blob: b81e4a7cd7f25c0167b826dd65a0788085eef98f [file] [log] [blame]
Jens Axboe75bb4622014-05-28 10:15:41 -06001/*
2 * Block multiqueue core code
3 *
4 * Copyright (C) 2013-2014 Jens Axboe
5 * Copyright (C) 2013-2014 Christoph Hellwig
6 */
Jens Axboe320ae512013-10-24 09:20:05 +01007#include <linux/kernel.h>
8#include <linux/module.h>
9#include <linux/backing-dev.h>
10#include <linux/bio.h>
11#include <linux/blkdev.h>
Catalin Marinasf75782e2015-09-14 18:16:02 +010012#include <linux/kmemleak.h>
Jens Axboe320ae512013-10-24 09:20:05 +010013#include <linux/mm.h>
14#include <linux/init.h>
15#include <linux/slab.h>
16#include <linux/workqueue.h>
17#include <linux/smp.h>
18#include <linux/llist.h>
19#include <linux/list_sort.h>
20#include <linux/cpu.h>
21#include <linux/cache.h>
22#include <linux/sched/sysctl.h>
Ingo Molnar105ab3d2017-02-01 16:36:40 +010023#include <linux/sched/topology.h>
Ingo Molnar174cd4b2017-02-02 19:15:33 +010024#include <linux/sched/signal.h>
Jens Axboe320ae512013-10-24 09:20:05 +010025#include <linux/delay.h>
Jens Axboeaedcd722014-09-17 08:27:03 -060026#include <linux/crash_dump.h>
Jens Axboe88c7b2b2016-08-25 08:07:30 -060027#include <linux/prefetch.h>
Jens Axboe320ae512013-10-24 09:20:05 +010028
29#include <trace/events/block.h>
30
31#include <linux/blk-mq.h>
32#include "blk.h"
33#include "blk-mq.h"
34#include "blk-mq-tag.h"
Jens Axboecf43e6b2016-11-07 21:32:37 -070035#include "blk-stat.h"
Jens Axboe87760e52016-11-09 12:38:14 -070036#include "blk-wbt.h"
Jens Axboebd166ef2017-01-17 06:03:22 -070037#include "blk-mq-sched.h"
Jens Axboe320ae512013-10-24 09:20:05 +010038
39static DEFINE_MUTEX(all_q_mutex);
40static LIST_HEAD(all_q_list);
41
Omar Sandoval34dbad52017-03-21 08:56:08 -070042static void blk_mq_poll_stats_start(struct request_queue *q);
43static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb);
44
Stephen Bates720b8cc2017-04-07 06:24:03 -060045static int blk_mq_poll_stats_bkt(const struct request *rq)
46{
47 int ddir, bytes, bucket;
48
Jens Axboe99c749a2017-04-21 07:55:42 -060049 ddir = rq_data_dir(rq);
Stephen Bates720b8cc2017-04-07 06:24:03 -060050 bytes = blk_rq_bytes(rq);
51
52 bucket = ddir + 2*(ilog2(bytes) - 9);
53
54 if (bucket < 0)
55 return -1;
56 else if (bucket >= BLK_MQ_POLL_STATS_BKTS)
57 return ddir + BLK_MQ_POLL_STATS_BKTS - 2;
58
59 return bucket;
60}
61
Jens Axboe320ae512013-10-24 09:20:05 +010062/*
63 * Check if any of the ctx's have pending work in this hardware queue
64 */
Jens Axboe50e1dab2017-01-26 14:42:34 -070065bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
Jens Axboe320ae512013-10-24 09:20:05 +010066{
Jens Axboebd166ef2017-01-17 06:03:22 -070067 return sbitmap_any_bit_set(&hctx->ctx_map) ||
68 !list_empty_careful(&hctx->dispatch) ||
69 blk_mq_sched_has_work(hctx);
Jens Axboe320ae512013-10-24 09:20:05 +010070}
71
72/*
73 * Mark this ctx as having pending work in this hardware queue
74 */
75static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
76 struct blk_mq_ctx *ctx)
77{
Omar Sandoval88459642016-09-17 08:38:44 -060078 if (!sbitmap_test_bit(&hctx->ctx_map, ctx->index_hw))
79 sbitmap_set_bit(&hctx->ctx_map, ctx->index_hw);
Jens Axboe1429d7c2014-05-19 09:23:55 -060080}
81
82static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
83 struct blk_mq_ctx *ctx)
84{
Omar Sandoval88459642016-09-17 08:38:44 -060085 sbitmap_clear_bit(&hctx->ctx_map, ctx->index_hw);
Jens Axboe320ae512013-10-24 09:20:05 +010086}
87
Ming Lei1671d522017-03-27 20:06:57 +080088void blk_freeze_queue_start(struct request_queue *q)
Ming Lei43a5e4e2013-12-26 21:31:35 +080089{
Christoph Hellwig4ecd4fe2015-05-07 09:38:13 +020090 int freeze_depth;
Tejun Heocddd5d12014-08-16 08:02:24 -040091
Christoph Hellwig4ecd4fe2015-05-07 09:38:13 +020092 freeze_depth = atomic_inc_return(&q->mq_freeze_depth);
93 if (freeze_depth == 1) {
Dan Williams3ef28e82015-10-21 13:20:12 -040094 percpu_ref_kill(&q->q_usage_counter);
Mike Snitzerb94ec292015-03-11 23:56:38 -040095 blk_mq_run_hw_queues(q, false);
Tejun Heocddd5d12014-08-16 08:02:24 -040096 }
Tejun Heof3af0202014-11-04 13:52:27 -050097}
Ming Lei1671d522017-03-27 20:06:57 +080098EXPORT_SYMBOL_GPL(blk_freeze_queue_start);
Tejun Heof3af0202014-11-04 13:52:27 -050099
Keith Busch6bae3632017-03-01 14:22:10 -0500100void blk_mq_freeze_queue_wait(struct request_queue *q)
Tejun Heof3af0202014-11-04 13:52:27 -0500101{
Dan Williams3ef28e82015-10-21 13:20:12 -0400102 wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
Ming Lei43a5e4e2013-12-26 21:31:35 +0800103}
Keith Busch6bae3632017-03-01 14:22:10 -0500104EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait);
Ming Lei43a5e4e2013-12-26 21:31:35 +0800105
Keith Buschf91328c2017-03-01 14:22:11 -0500106int blk_mq_freeze_queue_wait_timeout(struct request_queue *q,
107 unsigned long timeout)
108{
109 return wait_event_timeout(q->mq_freeze_wq,
110 percpu_ref_is_zero(&q->q_usage_counter),
111 timeout);
112}
113EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout);
Jens Axboe320ae512013-10-24 09:20:05 +0100114
Tejun Heof3af0202014-11-04 13:52:27 -0500115/*
116 * Guarantee no request is in use, so we can change any data structure of
117 * the queue afterward.
118 */
Dan Williams3ef28e82015-10-21 13:20:12 -0400119void blk_freeze_queue(struct request_queue *q)
Tejun Heof3af0202014-11-04 13:52:27 -0500120{
Dan Williams3ef28e82015-10-21 13:20:12 -0400121 /*
122 * In the !blk_mq case we are only calling this to kill the
123 * q_usage_counter, otherwise this increases the freeze depth
124 * and waits for it to return to zero. For this reason there is
125 * no blk_unfreeze_queue(), and blk_freeze_queue() is not
126 * exported to drivers as the only user for unfreeze is blk_mq.
127 */
Ming Lei1671d522017-03-27 20:06:57 +0800128 blk_freeze_queue_start(q);
Tejun Heof3af0202014-11-04 13:52:27 -0500129 blk_mq_freeze_queue_wait(q);
130}
Dan Williams3ef28e82015-10-21 13:20:12 -0400131
132void blk_mq_freeze_queue(struct request_queue *q)
133{
134 /*
135 * ...just an alias to keep freeze and unfreeze actions balanced
136 * in the blk_mq_* namespace
137 */
138 blk_freeze_queue(q);
139}
Jens Axboec761d962015-01-02 15:05:12 -0700140EXPORT_SYMBOL_GPL(blk_mq_freeze_queue);
Tejun Heof3af0202014-11-04 13:52:27 -0500141
Keith Buschb4c6a022014-12-19 17:54:14 -0700142void blk_mq_unfreeze_queue(struct request_queue *q)
Jens Axboe320ae512013-10-24 09:20:05 +0100143{
Christoph Hellwig4ecd4fe2015-05-07 09:38:13 +0200144 int freeze_depth;
Jens Axboe320ae512013-10-24 09:20:05 +0100145
Christoph Hellwig4ecd4fe2015-05-07 09:38:13 +0200146 freeze_depth = atomic_dec_return(&q->mq_freeze_depth);
147 WARN_ON_ONCE(freeze_depth < 0);
148 if (!freeze_depth) {
Dan Williams3ef28e82015-10-21 13:20:12 -0400149 percpu_ref_reinit(&q->q_usage_counter);
Jens Axboe320ae512013-10-24 09:20:05 +0100150 wake_up_all(&q->mq_freeze_wq);
Tejun Heoadd703f2014-07-01 10:34:38 -0600151 }
Jens Axboe320ae512013-10-24 09:20:05 +0100152}
Keith Buschb4c6a022014-12-19 17:54:14 -0700153EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue);
Jens Axboe320ae512013-10-24 09:20:05 +0100154
Bart Van Assche6a83e742016-11-02 10:09:51 -0600155/**
156 * blk_mq_quiesce_queue() - wait until all ongoing queue_rq calls have finished
157 * @q: request queue.
158 *
159 * Note: this function does not prevent that the struct request end_io()
160 * callback function is invoked. Additionally, it is not prevented that
161 * new queue_rq() calls occur unless the queue has been stopped first.
162 */
163void blk_mq_quiesce_queue(struct request_queue *q)
164{
165 struct blk_mq_hw_ctx *hctx;
166 unsigned int i;
167 bool rcu = false;
168
169 blk_mq_stop_hw_queues(q);
170
171 queue_for_each_hw_ctx(q, hctx, i) {
172 if (hctx->flags & BLK_MQ_F_BLOCKING)
173 synchronize_srcu(&hctx->queue_rq_srcu);
174 else
175 rcu = true;
176 }
177 if (rcu)
178 synchronize_rcu();
179}
180EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue);
181
Jens Axboeaed3ea92014-12-22 14:04:42 -0700182void blk_mq_wake_waiters(struct request_queue *q)
183{
184 struct blk_mq_hw_ctx *hctx;
185 unsigned int i;
186
187 queue_for_each_hw_ctx(q, hctx, i)
188 if (blk_mq_hw_queue_mapped(hctx))
189 blk_mq_tag_wakeup_all(hctx->tags, true);
Keith Busch3fd59402015-01-08 08:53:56 -0700190
191 /*
192 * If we are called because the queue has now been marked as
193 * dying, we need to ensure that processes currently waiting on
194 * the queue are notified as well.
195 */
196 wake_up_all(&q->mq_freeze_wq);
Jens Axboeaed3ea92014-12-22 14:04:42 -0700197}
198
Jens Axboe320ae512013-10-24 09:20:05 +0100199bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx)
200{
201 return blk_mq_has_free_tags(hctx->tags);
202}
203EXPORT_SYMBOL(blk_mq_can_queue);
204
Jens Axboe2c3ad662016-12-14 14:34:47 -0700205void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx,
206 struct request *rq, unsigned int op)
Jens Axboe320ae512013-10-24 09:20:05 +0100207{
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200208 INIT_LIST_HEAD(&rq->queuelist);
209 /* csd/requeue_work/fifo_time is initialized before use */
210 rq->q = q;
Jens Axboe320ae512013-10-24 09:20:05 +0100211 rq->mq_ctx = ctx;
Christoph Hellwigef295ec2016-10-28 08:48:16 -0600212 rq->cmd_flags = op;
Christoph Hellwige8064022016-10-20 15:12:13 +0200213 if (blk_queue_io_stat(q))
214 rq->rq_flags |= RQF_IO_STAT;
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200215 /* do not touch atomic flags, it needs atomic ops against the timer */
216 rq->cpu = -1;
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200217 INIT_HLIST_NODE(&rq->hash);
218 RB_CLEAR_NODE(&rq->rb_node);
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200219 rq->rq_disk = NULL;
220 rq->part = NULL;
Jens Axboe3ee32372014-06-09 09:36:53 -0600221 rq->start_time = jiffies;
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200222#ifdef CONFIG_BLK_CGROUP
223 rq->rl = NULL;
Ming Lei0fec08b2014-01-03 10:00:08 -0700224 set_start_time_ns(rq);
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200225 rq->io_start_time_ns = 0;
226#endif
227 rq->nr_phys_segments = 0;
228#if defined(CONFIG_BLK_DEV_INTEGRITY)
229 rq->nr_integrity_segments = 0;
230#endif
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200231 rq->special = NULL;
232 /* tag was already set */
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200233 rq->extra_len = 0;
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200234
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200235 INIT_LIST_HEAD(&rq->timeout_list);
Jens Axboef6be4fb2014-06-06 11:03:48 -0600236 rq->timeout = 0;
237
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200238 rq->end_io = NULL;
239 rq->end_io_data = NULL;
240 rq->next_rq = NULL;
241
Christoph Hellwigef295ec2016-10-28 08:48:16 -0600242 ctx->rq_dispatched[op_is_sync(op)]++;
Jens Axboe320ae512013-10-24 09:20:05 +0100243}
Jens Axboe2c3ad662016-12-14 14:34:47 -0700244EXPORT_SYMBOL_GPL(blk_mq_rq_ctx_init);
Jens Axboe320ae512013-10-24 09:20:05 +0100245
Jens Axboe2c3ad662016-12-14 14:34:47 -0700246struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data,
247 unsigned int op)
Christoph Hellwig5dee8572014-05-27 20:59:47 +0200248{
249 struct request *rq;
250 unsigned int tag;
251
Ming Leicb96a42c2014-06-01 00:43:37 +0800252 tag = blk_mq_get_tag(data);
Christoph Hellwig5dee8572014-05-27 20:59:47 +0200253 if (tag != BLK_MQ_TAG_FAIL) {
Jens Axboebd166ef2017-01-17 06:03:22 -0700254 struct blk_mq_tags *tags = blk_mq_tags_from_data(data);
255
256 rq = tags->static_rqs[tag];
Christoph Hellwig5dee8572014-05-27 20:59:47 +0200257
Jens Axboebd166ef2017-01-17 06:03:22 -0700258 if (data->flags & BLK_MQ_REQ_INTERNAL) {
259 rq->tag = -1;
260 rq->internal_tag = tag;
261 } else {
Jens Axboe200e86b2017-01-25 08:11:38 -0700262 if (blk_mq_tag_busy(data->hctx)) {
263 rq->rq_flags = RQF_MQ_INFLIGHT;
264 atomic_inc(&data->hctx->nr_active);
265 }
Jens Axboebd166ef2017-01-17 06:03:22 -0700266 rq->tag = tag;
267 rq->internal_tag = -1;
Omar Sandoval562bef42017-02-27 09:47:55 -0800268 data->hctx->tags->rqs[rq->tag] = rq;
Jens Axboebd166ef2017-01-17 06:03:22 -0700269 }
270
Christoph Hellwigef295ec2016-10-28 08:48:16 -0600271 blk_mq_rq_ctx_init(data->q, data->ctx, rq, op);
Christoph Hellwig5dee8572014-05-27 20:59:47 +0200272 return rq;
273 }
274
275 return NULL;
276}
Jens Axboe2c3ad662016-12-14 14:34:47 -0700277EXPORT_SYMBOL_GPL(__blk_mq_alloc_request);
Christoph Hellwig5dee8572014-05-27 20:59:47 +0200278
Christoph Hellwig6f3b0e82015-11-26 09:13:05 +0100279struct request *blk_mq_alloc_request(struct request_queue *q, int rw,
280 unsigned int flags)
Jens Axboe320ae512013-10-24 09:20:05 +0100281{
Jens Axboe5a797e02017-01-26 12:22:11 -0700282 struct blk_mq_alloc_data alloc_data = { .flags = flags };
Jens Axboebd166ef2017-01-17 06:03:22 -0700283 struct request *rq;
Joe Lawrencea492f072014-08-28 08:15:21 -0600284 int ret;
Jens Axboe320ae512013-10-24 09:20:05 +0100285
Christoph Hellwig6f3b0e82015-11-26 09:13:05 +0100286 ret = blk_queue_enter(q, flags & BLK_MQ_REQ_NOWAIT);
Joe Lawrencea492f072014-08-28 08:15:21 -0600287 if (ret)
288 return ERR_PTR(ret);
Jens Axboe320ae512013-10-24 09:20:05 +0100289
Jens Axboebd166ef2017-01-17 06:03:22 -0700290 rq = blk_mq_sched_get_request(q, NULL, rw, &alloc_data);
Jens Axboe841bac22016-09-21 10:08:43 -0600291
Jens Axboebd166ef2017-01-17 06:03:22 -0700292 blk_mq_put_ctx(alloc_data.ctx);
293 blk_queue_exit(q);
294
295 if (!rq)
Joe Lawrencea492f072014-08-28 08:15:21 -0600296 return ERR_PTR(-EWOULDBLOCK);
Christoph Hellwig0c4de0f2016-07-19 11:31:50 +0200297
298 rq->__data_len = 0;
299 rq->__sector = (sector_t) -1;
300 rq->bio = rq->biotail = NULL;
Jens Axboe320ae512013-10-24 09:20:05 +0100301 return rq;
302}
Jens Axboe4bb659b2014-05-09 09:36:49 -0600303EXPORT_SYMBOL(blk_mq_alloc_request);
Jens Axboe320ae512013-10-24 09:20:05 +0100304
Ming Lin1f5bd332016-06-13 16:45:21 +0200305struct request *blk_mq_alloc_request_hctx(struct request_queue *q, int rw,
306 unsigned int flags, unsigned int hctx_idx)
307{
Omar Sandoval6d2809d2017-02-27 10:28:27 -0800308 struct blk_mq_alloc_data alloc_data = { .flags = flags };
Ming Lin1f5bd332016-06-13 16:45:21 +0200309 struct request *rq;
Omar Sandoval6d2809d2017-02-27 10:28:27 -0800310 unsigned int cpu;
Ming Lin1f5bd332016-06-13 16:45:21 +0200311 int ret;
312
313 /*
314 * If the tag allocator sleeps we could get an allocation for a
315 * different hardware context. No need to complicate the low level
316 * allocator for this for the rare use case of a command tied to
317 * a specific queue.
318 */
319 if (WARN_ON_ONCE(!(flags & BLK_MQ_REQ_NOWAIT)))
320 return ERR_PTR(-EINVAL);
321
322 if (hctx_idx >= q->nr_hw_queues)
323 return ERR_PTR(-EIO);
324
325 ret = blk_queue_enter(q, true);
326 if (ret)
327 return ERR_PTR(ret);
328
Christoph Hellwigc8712c62016-09-23 10:25:48 -0600329 /*
330 * Check if the hardware context is actually mapped to anything.
331 * If not tell the caller that it should skip this queue.
332 */
Omar Sandoval6d2809d2017-02-27 10:28:27 -0800333 alloc_data.hctx = q->queue_hw_ctx[hctx_idx];
334 if (!blk_mq_hw_queue_mapped(alloc_data.hctx)) {
335 blk_queue_exit(q);
336 return ERR_PTR(-EXDEV);
Christoph Hellwigc8712c62016-09-23 10:25:48 -0600337 }
Omar Sandoval6d2809d2017-02-27 10:28:27 -0800338 cpu = cpumask_first(alloc_data.hctx->cpumask);
339 alloc_data.ctx = __blk_mq_get_ctx(q, cpu);
Ming Lin1f5bd332016-06-13 16:45:21 +0200340
Omar Sandoval6d2809d2017-02-27 10:28:27 -0800341 rq = blk_mq_sched_get_request(q, NULL, rw, &alloc_data);
342
Omar Sandoval6d2809d2017-02-27 10:28:27 -0800343 blk_queue_exit(q);
344
345 if (!rq)
346 return ERR_PTR(-EWOULDBLOCK);
Ming Lin1f5bd332016-06-13 16:45:21 +0200347
348 return rq;
349}
350EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);
351
Jens Axboebd166ef2017-01-17 06:03:22 -0700352void __blk_mq_finish_request(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
353 struct request *rq)
Jens Axboe320ae512013-10-24 09:20:05 +0100354{
Jens Axboebd166ef2017-01-17 06:03:22 -0700355 const int sched_tag = rq->internal_tag;
Jens Axboe320ae512013-10-24 09:20:05 +0100356 struct request_queue *q = rq->q;
357
Christoph Hellwige8064022016-10-20 15:12:13 +0200358 if (rq->rq_flags & RQF_MQ_INFLIGHT)
Jens Axboe0d2602c2014-05-13 15:10:52 -0600359 atomic_dec(&hctx->nr_active);
Jens Axboe87760e52016-11-09 12:38:14 -0700360
361 wbt_done(q->rq_wb, &rq->issue_stat);
Christoph Hellwige8064022016-10-20 15:12:13 +0200362 rq->rq_flags = 0;
Jens Axboe0d2602c2014-05-13 15:10:52 -0600363
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200364 clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
Jens Axboe06426ad2016-11-14 13:01:59 -0700365 clear_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);
Jens Axboebd166ef2017-01-17 06:03:22 -0700366 if (rq->tag != -1)
367 blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag);
368 if (sched_tag != -1)
Omar Sandovalc05f8522017-04-14 01:00:01 -0700369 blk_mq_put_tag(hctx, hctx->sched_tags, ctx, sched_tag);
Bart Van Assche6d8c6c02017-04-07 12:40:09 -0600370 blk_mq_sched_restart(hctx);
Dan Williams3ef28e82015-10-21 13:20:12 -0400371 blk_queue_exit(q);
Jens Axboe320ae512013-10-24 09:20:05 +0100372}
373
Jens Axboebd166ef2017-01-17 06:03:22 -0700374static void blk_mq_finish_hctx_request(struct blk_mq_hw_ctx *hctx,
Jens Axboe16a3c2a2016-12-15 14:27:46 -0700375 struct request *rq)
Jens Axboe320ae512013-10-24 09:20:05 +0100376{
377 struct blk_mq_ctx *ctx = rq->mq_ctx;
Jens Axboe7c7f2f22014-11-17 10:41:57 -0700378
379 ctx->rq_completed[rq_is_sync(rq)]++;
Jens Axboebd166ef2017-01-17 06:03:22 -0700380 __blk_mq_finish_request(hctx, ctx, rq);
381}
382
383void blk_mq_finish_request(struct request *rq)
384{
385 blk_mq_finish_hctx_request(blk_mq_map_queue(rq->q, rq->mq_ctx->cpu), rq);
Jens Axboe7c7f2f22014-11-17 10:41:57 -0700386}
Omar Sandoval5b727272017-04-14 01:00:00 -0700387EXPORT_SYMBOL_GPL(blk_mq_finish_request);
Jens Axboe7c7f2f22014-11-17 10:41:57 -0700388
389void blk_mq_free_request(struct request *rq)
390{
Jens Axboebd166ef2017-01-17 06:03:22 -0700391 blk_mq_sched_put_request(rq);
Jens Axboe320ae512013-10-24 09:20:05 +0100392}
Jens Axboe1a3b5952014-11-17 10:40:48 -0700393EXPORT_SYMBOL_GPL(blk_mq_free_request);
Jens Axboe320ae512013-10-24 09:20:05 +0100394
Christoph Hellwigc8a446a2014-09-13 16:40:10 -0700395inline void __blk_mq_end_request(struct request *rq, int error)
Jens Axboe320ae512013-10-24 09:20:05 +0100396{
Ming Lei0d11e6a2013-12-05 10:50:39 -0700397 blk_account_io_done(rq);
398
Christoph Hellwig91b63632014-04-16 09:44:53 +0200399 if (rq->end_io) {
Jens Axboe87760e52016-11-09 12:38:14 -0700400 wbt_done(rq->q->rq_wb, &rq->issue_stat);
Jens Axboe320ae512013-10-24 09:20:05 +0100401 rq->end_io(rq, error);
Christoph Hellwig91b63632014-04-16 09:44:53 +0200402 } else {
403 if (unlikely(blk_bidi_rq(rq)))
404 blk_mq_free_request(rq->next_rq);
Jens Axboe320ae512013-10-24 09:20:05 +0100405 blk_mq_free_request(rq);
Christoph Hellwig91b63632014-04-16 09:44:53 +0200406 }
Jens Axboe320ae512013-10-24 09:20:05 +0100407}
Christoph Hellwigc8a446a2014-09-13 16:40:10 -0700408EXPORT_SYMBOL(__blk_mq_end_request);
Christoph Hellwig63151a42014-04-16 09:44:52 +0200409
Christoph Hellwigc8a446a2014-09-13 16:40:10 -0700410void blk_mq_end_request(struct request *rq, int error)
Christoph Hellwig63151a42014-04-16 09:44:52 +0200411{
412 if (blk_update_request(rq, error, blk_rq_bytes(rq)))
413 BUG();
Christoph Hellwigc8a446a2014-09-13 16:40:10 -0700414 __blk_mq_end_request(rq, error);
Christoph Hellwig63151a42014-04-16 09:44:52 +0200415}
Christoph Hellwigc8a446a2014-09-13 16:40:10 -0700416EXPORT_SYMBOL(blk_mq_end_request);
Jens Axboe320ae512013-10-24 09:20:05 +0100417
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800418static void __blk_mq_complete_request_remote(void *data)
Jens Axboe320ae512013-10-24 09:20:05 +0100419{
Christoph Hellwig3d6efbf2014-01-08 09:33:37 -0800420 struct request *rq = data;
Jens Axboe320ae512013-10-24 09:20:05 +0100421
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800422 rq->q->softirq_done_fn(rq);
Jens Axboe320ae512013-10-24 09:20:05 +0100423}
424
Christoph Hellwig453f8342017-04-20 16:03:10 +0200425static void __blk_mq_complete_request(struct request *rq)
Jens Axboe320ae512013-10-24 09:20:05 +0100426{
427 struct blk_mq_ctx *ctx = rq->mq_ctx;
Christoph Hellwig38535202014-04-25 02:32:53 -0700428 bool shared = false;
Jens Axboe320ae512013-10-24 09:20:05 +0100429 int cpu;
430
Christoph Hellwig453f8342017-04-20 16:03:10 +0200431 if (rq->internal_tag != -1)
432 blk_mq_sched_completed_request(rq);
433 if (rq->rq_flags & RQF_STATS) {
434 blk_mq_poll_stats_start(rq->q);
435 blk_stat_add(rq);
436 }
437
Christoph Hellwig38535202014-04-25 02:32:53 -0700438 if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) {
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800439 rq->q->softirq_done_fn(rq);
440 return;
441 }
Jens Axboe320ae512013-10-24 09:20:05 +0100442
443 cpu = get_cpu();
Christoph Hellwig38535202014-04-25 02:32:53 -0700444 if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags))
445 shared = cpus_share_cache(cpu, ctx->cpu);
446
447 if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) {
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800448 rq->csd.func = __blk_mq_complete_request_remote;
Christoph Hellwig3d6efbf2014-01-08 09:33:37 -0800449 rq->csd.info = rq;
450 rq->csd.flags = 0;
Frederic Weisbeckerc46fff22014-02-24 16:40:02 +0100451 smp_call_function_single_async(ctx->cpu, &rq->csd);
Christoph Hellwig3d6efbf2014-01-08 09:33:37 -0800452 } else {
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800453 rq->q->softirq_done_fn(rq);
Christoph Hellwig3d6efbf2014-01-08 09:33:37 -0800454 }
Jens Axboe320ae512013-10-24 09:20:05 +0100455 put_cpu();
456}
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800457
458/**
459 * blk_mq_complete_request - end I/O on a request
460 * @rq: the request being processed
461 *
462 * Description:
463 * Ends all I/O on a request. It does not handle partial completions.
464 * The actual completion happens out-of-order, through a IPI handler.
465 **/
Christoph Hellwig08e00292017-04-20 16:03:09 +0200466void blk_mq_complete_request(struct request *rq)
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800467{
Jens Axboe95f09682014-05-27 17:46:48 -0600468 struct request_queue *q = rq->q;
469
470 if (unlikely(blk_should_fake_timeout(q)))
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800471 return;
Christoph Hellwig08e00292017-04-20 16:03:09 +0200472 if (!blk_mark_rq_complete(rq))
Jens Axboeed851862014-05-30 21:20:50 -0600473 __blk_mq_complete_request(rq);
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800474}
475EXPORT_SYMBOL(blk_mq_complete_request);
Jens Axboe320ae512013-10-24 09:20:05 +0100476
Keith Busch973c0192015-01-07 18:55:43 -0700477int blk_mq_request_started(struct request *rq)
478{
479 return test_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
480}
481EXPORT_SYMBOL_GPL(blk_mq_request_started);
482
Christoph Hellwige2490072014-09-13 16:40:09 -0700483void blk_mq_start_request(struct request *rq)
Jens Axboe320ae512013-10-24 09:20:05 +0100484{
485 struct request_queue *q = rq->q;
486
Jens Axboebd166ef2017-01-17 06:03:22 -0700487 blk_mq_sched_started_request(rq);
488
Jens Axboe320ae512013-10-24 09:20:05 +0100489 trace_block_rq_issue(q, rq);
490
Jens Axboecf43e6b2016-11-07 21:32:37 -0700491 if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {
Shaohua Li88eeca42017-03-27 15:19:41 -0700492 blk_stat_set_issue(&rq->issue_stat, blk_rq_sectors(rq));
Jens Axboecf43e6b2016-11-07 21:32:37 -0700493 rq->rq_flags |= RQF_STATS;
Jens Axboe87760e52016-11-09 12:38:14 -0700494 wbt_issue(q->rq_wb, &rq->issue_stat);
Jens Axboecf43e6b2016-11-07 21:32:37 -0700495 }
496
Ming Lei2b8393b2014-06-10 00:16:41 +0800497 blk_add_timer(rq);
Jens Axboe87ee7b12014-04-24 08:51:47 -0600498
499 /*
Jens Axboe538b7532014-09-16 10:37:37 -0600500 * Ensure that ->deadline is visible before set the started
501 * flag and clear the completed flag.
502 */
503 smp_mb__before_atomic();
504
505 /*
Jens Axboe87ee7b12014-04-24 08:51:47 -0600506 * Mark us as started and clear complete. Complete might have been
507 * set if requeue raced with timeout, which then marked it as
508 * complete. So be sure to clear complete again when we start
509 * the request, otherwise we'll ignore the completion event.
510 */
Jens Axboe4b570522014-05-29 11:00:11 -0600511 if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
512 set_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
513 if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
514 clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
Christoph Hellwig49f5baa2014-02-11 08:27:14 -0800515
516 if (q->dma_drain_size && blk_rq_bytes(rq)) {
517 /*
518 * Make sure space for the drain appears. We know we can do
519 * this because max_hw_segments has been adjusted to be one
520 * fewer than the device can handle.
521 */
522 rq->nr_phys_segments++;
523 }
Jens Axboe320ae512013-10-24 09:20:05 +0100524}
Christoph Hellwige2490072014-09-13 16:40:09 -0700525EXPORT_SYMBOL(blk_mq_start_request);
Jens Axboe320ae512013-10-24 09:20:05 +0100526
Ming Leid9d149a2017-03-27 20:06:55 +0800527/*
528 * When we reach here because queue is busy, REQ_ATOM_COMPLETE
Jens Axboe48b99c92017-03-29 11:10:34 -0600529 * flag isn't set yet, so there may be race with timeout handler,
Ming Leid9d149a2017-03-27 20:06:55 +0800530 * but given rq->deadline is just set in .queue_rq() under
531 * this situation, the race won't be possible in reality because
532 * rq->timeout should be set as big enough to cover the window
533 * between blk_mq_start_request() called from .queue_rq() and
534 * clearing REQ_ATOM_STARTED here.
535 */
Christoph Hellwiged0791b2014-04-16 09:44:57 +0200536static void __blk_mq_requeue_request(struct request *rq)
Jens Axboe320ae512013-10-24 09:20:05 +0100537{
538 struct request_queue *q = rq->q;
539
540 trace_block_rq_requeue(q, rq);
Jens Axboe87760e52016-11-09 12:38:14 -0700541 wbt_requeue(q->rq_wb, &rq->issue_stat);
Jens Axboebd166ef2017-01-17 06:03:22 -0700542 blk_mq_sched_requeue_request(rq);
Christoph Hellwig49f5baa2014-02-11 08:27:14 -0800543
Christoph Hellwige2490072014-09-13 16:40:09 -0700544 if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
545 if (q->dma_drain_size && blk_rq_bytes(rq))
546 rq->nr_phys_segments--;
547 }
Jens Axboe320ae512013-10-24 09:20:05 +0100548}
549
Bart Van Assche2b053ac2016-10-28 17:21:41 -0700550void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list)
Christoph Hellwiged0791b2014-04-16 09:44:57 +0200551{
Christoph Hellwiged0791b2014-04-16 09:44:57 +0200552 __blk_mq_requeue_request(rq);
Christoph Hellwiged0791b2014-04-16 09:44:57 +0200553
Christoph Hellwiged0791b2014-04-16 09:44:57 +0200554 BUG_ON(blk_queued_rq(rq));
Bart Van Assche2b053ac2016-10-28 17:21:41 -0700555 blk_mq_add_to_requeue_list(rq, true, kick_requeue_list);
Christoph Hellwiged0791b2014-04-16 09:44:57 +0200556}
557EXPORT_SYMBOL(blk_mq_requeue_request);
558
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600559static void blk_mq_requeue_work(struct work_struct *work)
560{
561 struct request_queue *q =
Mike Snitzer28494502016-09-14 13:28:30 -0400562 container_of(work, struct request_queue, requeue_work.work);
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600563 LIST_HEAD(rq_list);
564 struct request *rq, *next;
565 unsigned long flags;
566
567 spin_lock_irqsave(&q->requeue_lock, flags);
568 list_splice_init(&q->requeue_list, &rq_list);
569 spin_unlock_irqrestore(&q->requeue_lock, flags);
570
571 list_for_each_entry_safe(rq, next, &rq_list, queuelist) {
Christoph Hellwige8064022016-10-20 15:12:13 +0200572 if (!(rq->rq_flags & RQF_SOFTBARRIER))
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600573 continue;
574
Christoph Hellwige8064022016-10-20 15:12:13 +0200575 rq->rq_flags &= ~RQF_SOFTBARRIER;
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600576 list_del_init(&rq->queuelist);
Jens Axboebd6737f2017-01-27 01:00:47 -0700577 blk_mq_sched_insert_request(rq, true, false, false, true);
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600578 }
579
580 while (!list_empty(&rq_list)) {
581 rq = list_entry(rq_list.next, struct request, queuelist);
582 list_del_init(&rq->queuelist);
Jens Axboebd6737f2017-01-27 01:00:47 -0700583 blk_mq_sched_insert_request(rq, false, false, false, true);
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600584 }
585
Bart Van Assche52d7f1b2016-10-28 17:20:32 -0700586 blk_mq_run_hw_queues(q, false);
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600587}
588
Bart Van Assche2b053ac2016-10-28 17:21:41 -0700589void blk_mq_add_to_requeue_list(struct request *rq, bool at_head,
590 bool kick_requeue_list)
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600591{
592 struct request_queue *q = rq->q;
593 unsigned long flags;
594
595 /*
596 * We abuse this flag that is otherwise used by the I/O scheduler to
597 * request head insertation from the workqueue.
598 */
Christoph Hellwige8064022016-10-20 15:12:13 +0200599 BUG_ON(rq->rq_flags & RQF_SOFTBARRIER);
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600600
601 spin_lock_irqsave(&q->requeue_lock, flags);
602 if (at_head) {
Christoph Hellwige8064022016-10-20 15:12:13 +0200603 rq->rq_flags |= RQF_SOFTBARRIER;
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600604 list_add(&rq->queuelist, &q->requeue_list);
605 } else {
606 list_add_tail(&rq->queuelist, &q->requeue_list);
607 }
608 spin_unlock_irqrestore(&q->requeue_lock, flags);
Bart Van Assche2b053ac2016-10-28 17:21:41 -0700609
610 if (kick_requeue_list)
611 blk_mq_kick_requeue_list(q);
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600612}
613EXPORT_SYMBOL(blk_mq_add_to_requeue_list);
614
615void blk_mq_kick_requeue_list(struct request_queue *q)
616{
Mike Snitzer28494502016-09-14 13:28:30 -0400617 kblockd_schedule_delayed_work(&q->requeue_work, 0);
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600618}
619EXPORT_SYMBOL(blk_mq_kick_requeue_list);
620
Mike Snitzer28494502016-09-14 13:28:30 -0400621void blk_mq_delay_kick_requeue_list(struct request_queue *q,
622 unsigned long msecs)
623{
624 kblockd_schedule_delayed_work(&q->requeue_work,
625 msecs_to_jiffies(msecs));
626}
627EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list);
628
Jens Axboe1885b242015-01-07 18:55:45 -0700629void blk_mq_abort_requeue_list(struct request_queue *q)
630{
631 unsigned long flags;
632 LIST_HEAD(rq_list);
633
634 spin_lock_irqsave(&q->requeue_lock, flags);
635 list_splice_init(&q->requeue_list, &rq_list);
636 spin_unlock_irqrestore(&q->requeue_lock, flags);
637
638 while (!list_empty(&rq_list)) {
639 struct request *rq;
640
641 rq = list_first_entry(&rq_list, struct request, queuelist);
642 list_del_init(&rq->queuelist);
Christoph Hellwigcaf7df12017-04-20 16:03:16 +0200643 blk_mq_end_request(rq, -EIO);
Jens Axboe1885b242015-01-07 18:55:45 -0700644 }
645}
646EXPORT_SYMBOL(blk_mq_abort_requeue_list);
647
Jens Axboe0e62f512014-06-04 10:23:49 -0600648struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)
649{
Jens Axboe88c7b2b2016-08-25 08:07:30 -0600650 if (tag < tags->nr_tags) {
651 prefetch(tags->rqs[tag]);
Hannes Reinecke4ee86ba2016-03-15 12:03:28 -0700652 return tags->rqs[tag];
Jens Axboe88c7b2b2016-08-25 08:07:30 -0600653 }
Hannes Reinecke4ee86ba2016-03-15 12:03:28 -0700654
655 return NULL;
Christoph Hellwig24d2f902014-04-15 14:14:00 -0600656}
657EXPORT_SYMBOL(blk_mq_tag_to_rq);
658
Jens Axboe320ae512013-10-24 09:20:05 +0100659struct blk_mq_timeout_data {
Christoph Hellwig46f92d42014-09-13 16:40:12 -0700660 unsigned long next;
661 unsigned int next_set;
Jens Axboe320ae512013-10-24 09:20:05 +0100662};
663
Christoph Hellwig90415832014-09-22 10:21:48 -0600664void blk_mq_rq_timed_out(struct request *req, bool reserved)
Jens Axboe320ae512013-10-24 09:20:05 +0100665{
Jens Axboef8a5b122016-12-13 09:24:51 -0700666 const struct blk_mq_ops *ops = req->q->mq_ops;
Christoph Hellwig46f92d42014-09-13 16:40:12 -0700667 enum blk_eh_timer_return ret = BLK_EH_RESET_TIMER;
Jens Axboe87ee7b12014-04-24 08:51:47 -0600668
669 /*
670 * We know that complete is set at this point. If STARTED isn't set
671 * anymore, then the request isn't active and the "timeout" should
672 * just be ignored. This can happen due to the bitflag ordering.
673 * Timeout first checks if STARTED is set, and if it is, assumes
674 * the request is active. But if we race with completion, then
Jens Axboe48b99c92017-03-29 11:10:34 -0600675 * both flags will get cleared. So check here again, and ignore
Jens Axboe87ee7b12014-04-24 08:51:47 -0600676 * a timeout event with a request that isn't active.
677 */
Christoph Hellwig46f92d42014-09-13 16:40:12 -0700678 if (!test_bit(REQ_ATOM_STARTED, &req->atomic_flags))
679 return;
Jens Axboe87ee7b12014-04-24 08:51:47 -0600680
Christoph Hellwig46f92d42014-09-13 16:40:12 -0700681 if (ops->timeout)
Christoph Hellwig0152fb62014-09-13 16:40:13 -0700682 ret = ops->timeout(req, reserved);
Jens Axboe87ee7b12014-04-24 08:51:47 -0600683
Christoph Hellwig46f92d42014-09-13 16:40:12 -0700684 switch (ret) {
685 case BLK_EH_HANDLED:
686 __blk_mq_complete_request(req);
687 break;
688 case BLK_EH_RESET_TIMER:
689 blk_add_timer(req);
690 blk_clear_rq_complete(req);
691 break;
692 case BLK_EH_NOT_HANDLED:
693 break;
694 default:
695 printk(KERN_ERR "block: bad eh return: %d\n", ret);
696 break;
697 }
Jens Axboe87ee7b12014-04-24 08:51:47 -0600698}
Keith Busch5b3f25f2015-01-07 18:55:46 -0700699
Christoph Hellwig81481eb2014-09-13 16:40:11 -0700700static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,
701 struct request *rq, void *priv, bool reserved)
Jens Axboe320ae512013-10-24 09:20:05 +0100702{
Christoph Hellwig81481eb2014-09-13 16:40:11 -0700703 struct blk_mq_timeout_data *data = priv;
704
Ming Lei95a49602017-03-22 10:14:43 +0800705 if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
Christoph Hellwig46f92d42014-09-13 16:40:12 -0700706 return;
Jens Axboe320ae512013-10-24 09:20:05 +0100707
Ming Leid9d149a2017-03-27 20:06:55 +0800708 /*
709 * The rq being checked may have been freed and reallocated
710 * out already here, we avoid this race by checking rq->deadline
711 * and REQ_ATOM_COMPLETE flag together:
712 *
713 * - if rq->deadline is observed as new value because of
714 * reusing, the rq won't be timed out because of timing.
715 * - if rq->deadline is observed as previous value,
716 * REQ_ATOM_COMPLETE flag won't be cleared in reuse path
717 * because we put a barrier between setting rq->deadline
718 * and clearing the flag in blk_mq_start_request(), so
719 * this rq won't be timed out too.
720 */
Christoph Hellwig46f92d42014-09-13 16:40:12 -0700721 if (time_after_eq(jiffies, rq->deadline)) {
722 if (!blk_mark_rq_complete(rq))
Christoph Hellwig0152fb62014-09-13 16:40:13 -0700723 blk_mq_rq_timed_out(rq, reserved);
Christoph Hellwig46f92d42014-09-13 16:40:12 -0700724 } else if (!data->next_set || time_after(data->next, rq->deadline)) {
725 data->next = rq->deadline;
726 data->next_set = 1;
727 }
Jens Axboe320ae512013-10-24 09:20:05 +0100728}
729
Christoph Hellwig287922e2015-10-30 20:57:30 +0800730static void blk_mq_timeout_work(struct work_struct *work)
Jens Axboe320ae512013-10-24 09:20:05 +0100731{
Christoph Hellwig287922e2015-10-30 20:57:30 +0800732 struct request_queue *q =
733 container_of(work, struct request_queue, timeout_work);
Christoph Hellwig81481eb2014-09-13 16:40:11 -0700734 struct blk_mq_timeout_data data = {
735 .next = 0,
736 .next_set = 0,
737 };
Christoph Hellwig81481eb2014-09-13 16:40:11 -0700738 int i;
Jens Axboe320ae512013-10-24 09:20:05 +0100739
Gabriel Krisman Bertazi71f79fb2016-08-01 08:23:39 -0600740 /* A deadlock might occur if a request is stuck requiring a
741 * timeout at the same time a queue freeze is waiting
742 * completion, since the timeout code would not be able to
743 * acquire the queue reference here.
744 *
745 * That's why we don't use blk_queue_enter here; instead, we use
746 * percpu_ref_tryget directly, because we need to be able to
747 * obtain a reference even in the short window between the queue
748 * starting to freeze, by dropping the first reference in
Ming Lei1671d522017-03-27 20:06:57 +0800749 * blk_freeze_queue_start, and the moment the last request is
Gabriel Krisman Bertazi71f79fb2016-08-01 08:23:39 -0600750 * consumed, marked by the instant q_usage_counter reaches
751 * zero.
752 */
753 if (!percpu_ref_tryget(&q->q_usage_counter))
Christoph Hellwig287922e2015-10-30 20:57:30 +0800754 return;
755
Christoph Hellwig0bf6cd52015-09-27 21:01:51 +0200756 blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &data);
Jens Axboe320ae512013-10-24 09:20:05 +0100757
Christoph Hellwig81481eb2014-09-13 16:40:11 -0700758 if (data.next_set) {
759 data.next = blk_rq_timeout(round_jiffies_up(data.next));
760 mod_timer(&q->timeout, data.next);
Jens Axboe0d2602c2014-05-13 15:10:52 -0600761 } else {
Christoph Hellwig0bf6cd52015-09-27 21:01:51 +0200762 struct blk_mq_hw_ctx *hctx;
763
Ming Leif054b562015-04-21 10:00:19 +0800764 queue_for_each_hw_ctx(q, hctx, i) {
765 /* the hctx may be unmapped, so check it here */
766 if (blk_mq_hw_queue_mapped(hctx))
767 blk_mq_tag_idle(hctx);
768 }
Jens Axboe0d2602c2014-05-13 15:10:52 -0600769 }
Christoph Hellwig287922e2015-10-30 20:57:30 +0800770 blk_queue_exit(q);
Jens Axboe320ae512013-10-24 09:20:05 +0100771}
772
773/*
774 * Reverse check our software queue for entries that we could potentially
775 * merge with. Currently includes a hand-wavy stop count of 8, to not spend
776 * too much time checking for merges.
777 */
778static bool blk_mq_attempt_merge(struct request_queue *q,
779 struct blk_mq_ctx *ctx, struct bio *bio)
780{
781 struct request *rq;
782 int checked = 8;
783
784 list_for_each_entry_reverse(rq, &ctx->rq_list, queuelist) {
Christoph Hellwig34fe7c02017-02-08 14:46:48 +0100785 bool merged = false;
Jens Axboe320ae512013-10-24 09:20:05 +0100786
787 if (!checked--)
788 break;
789
790 if (!blk_rq_merge_ok(rq, bio))
791 continue;
792
Christoph Hellwig34fe7c02017-02-08 14:46:48 +0100793 switch (blk_try_merge(rq, bio)) {
794 case ELEVATOR_BACK_MERGE:
795 if (blk_mq_sched_allow_merge(q, rq, bio))
796 merged = bio_attempt_back_merge(q, rq, bio);
797 break;
798 case ELEVATOR_FRONT_MERGE:
799 if (blk_mq_sched_allow_merge(q, rq, bio))
800 merged = bio_attempt_front_merge(q, rq, bio);
801 break;
Christoph Hellwig1e739732017-02-08 14:46:49 +0100802 case ELEVATOR_DISCARD_MERGE:
803 merged = bio_attempt_discard_merge(q, rq, bio);
804 break;
Christoph Hellwig34fe7c02017-02-08 14:46:48 +0100805 default:
Jens Axboebd166ef2017-01-17 06:03:22 -0700806 continue;
Jens Axboe320ae512013-10-24 09:20:05 +0100807 }
Christoph Hellwig34fe7c02017-02-08 14:46:48 +0100808
809 if (merged)
810 ctx->rq_merged++;
811 return merged;
Jens Axboe320ae512013-10-24 09:20:05 +0100812 }
813
814 return false;
815}
816
Omar Sandoval88459642016-09-17 08:38:44 -0600817struct flush_busy_ctx_data {
818 struct blk_mq_hw_ctx *hctx;
819 struct list_head *list;
820};
821
822static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data)
823{
824 struct flush_busy_ctx_data *flush_data = data;
825 struct blk_mq_hw_ctx *hctx = flush_data->hctx;
826 struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
827
828 sbitmap_clear_bit(sb, bitnr);
829 spin_lock(&ctx->lock);
830 list_splice_tail_init(&ctx->rq_list, flush_data->list);
831 spin_unlock(&ctx->lock);
832 return true;
833}
834
Jens Axboe320ae512013-10-24 09:20:05 +0100835/*
Jens Axboe1429d7c2014-05-19 09:23:55 -0600836 * Process software queues that have been marked busy, splicing them
837 * to the for-dispatch
838 */
Jens Axboe2c3ad662016-12-14 14:34:47 -0700839void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
Jens Axboe1429d7c2014-05-19 09:23:55 -0600840{
Omar Sandoval88459642016-09-17 08:38:44 -0600841 struct flush_busy_ctx_data data = {
842 .hctx = hctx,
843 .list = list,
844 };
Jens Axboe1429d7c2014-05-19 09:23:55 -0600845
Omar Sandoval88459642016-09-17 08:38:44 -0600846 sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data);
Jens Axboe1429d7c2014-05-19 09:23:55 -0600847}
Jens Axboe2c3ad662016-12-14 14:34:47 -0700848EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs);
Jens Axboe1429d7c2014-05-19 09:23:55 -0600849
Jens Axboe703fd1c2016-09-16 13:59:14 -0600850static inline unsigned int queued_to_index(unsigned int queued)
851{
852 if (!queued)
853 return 0;
Jens Axboe1429d7c2014-05-19 09:23:55 -0600854
Jens Axboe703fd1c2016-09-16 13:59:14 -0600855 return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1);
Jens Axboe1429d7c2014-05-19 09:23:55 -0600856}
857
Jens Axboebd6737f2017-01-27 01:00:47 -0700858bool blk_mq_get_driver_tag(struct request *rq, struct blk_mq_hw_ctx **hctx,
859 bool wait)
Jens Axboebd166ef2017-01-17 06:03:22 -0700860{
861 struct blk_mq_alloc_data data = {
862 .q = rq->q,
Jens Axboebd166ef2017-01-17 06:03:22 -0700863 .hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu),
864 .flags = wait ? 0 : BLK_MQ_REQ_NOWAIT,
865 };
866
Jens Axboe5feeacd2017-04-20 17:23:13 -0600867 might_sleep_if(wait);
868
Omar Sandoval81380ca2017-04-07 08:56:26 -0600869 if (rq->tag != -1)
870 goto done;
Jens Axboebd166ef2017-01-17 06:03:22 -0700871
Sagi Grimberg415b8062017-02-27 10:04:39 -0700872 if (blk_mq_tag_is_reserved(data.hctx->sched_tags, rq->internal_tag))
873 data.flags |= BLK_MQ_REQ_RESERVED;
874
Jens Axboebd166ef2017-01-17 06:03:22 -0700875 rq->tag = blk_mq_get_tag(&data);
876 if (rq->tag >= 0) {
Jens Axboe200e86b2017-01-25 08:11:38 -0700877 if (blk_mq_tag_busy(data.hctx)) {
878 rq->rq_flags |= RQF_MQ_INFLIGHT;
879 atomic_inc(&data.hctx->nr_active);
880 }
Jens Axboebd166ef2017-01-17 06:03:22 -0700881 data.hctx->tags->rqs[rq->tag] = rq;
Jens Axboebd166ef2017-01-17 06:03:22 -0700882 }
883
Omar Sandoval81380ca2017-04-07 08:56:26 -0600884done:
885 if (hctx)
886 *hctx = data.hctx;
887 return rq->tag != -1;
Jens Axboebd166ef2017-01-17 06:03:22 -0700888}
889
Jens Axboe113285b2017-03-02 13:26:04 -0700890static void __blk_mq_put_driver_tag(struct blk_mq_hw_ctx *hctx,
891 struct request *rq)
Jens Axboe99cf1dc2017-01-26 12:32:32 -0700892{
Jens Axboe99cf1dc2017-01-26 12:32:32 -0700893 blk_mq_put_tag(hctx, hctx->tags, rq->mq_ctx, rq->tag);
894 rq->tag = -1;
895
896 if (rq->rq_flags & RQF_MQ_INFLIGHT) {
897 rq->rq_flags &= ~RQF_MQ_INFLIGHT;
898 atomic_dec(&hctx->nr_active);
899 }
900}
901
Jens Axboe113285b2017-03-02 13:26:04 -0700902static void blk_mq_put_driver_tag_hctx(struct blk_mq_hw_ctx *hctx,
903 struct request *rq)
904{
905 if (rq->tag == -1 || rq->internal_tag == -1)
906 return;
907
908 __blk_mq_put_driver_tag(hctx, rq);
909}
910
911static void blk_mq_put_driver_tag(struct request *rq)
912{
913 struct blk_mq_hw_ctx *hctx;
914
915 if (rq->tag == -1 || rq->internal_tag == -1)
916 return;
917
918 hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu);
919 __blk_mq_put_driver_tag(hctx, rq);
920}
921
Jens Axboebd166ef2017-01-17 06:03:22 -0700922/*
923 * If we fail getting a driver tag because all the driver tags are already
924 * assigned and on the dispatch list, BUT the first entry does not have a
925 * tag, then we could deadlock. For that case, move entries with assigned
926 * driver tags to the front, leaving the set of tagged requests in the
927 * same order, and the untagged set in the same order.
928 */
929static bool reorder_tags_to_front(struct list_head *list)
930{
931 struct request *rq, *tmp, *first = NULL;
932
933 list_for_each_entry_safe_reverse(rq, tmp, list, queuelist) {
934 if (rq == first)
935 break;
936 if (rq->tag != -1) {
937 list_move(&rq->queuelist, list);
938 if (!first)
939 first = rq;
940 }
941 }
942
943 return first != NULL;
944}
945
Omar Sandovalda55f2c2017-02-22 10:58:29 -0800946static int blk_mq_dispatch_wake(wait_queue_t *wait, unsigned mode, int flags,
947 void *key)
948{
949 struct blk_mq_hw_ctx *hctx;
950
951 hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait);
952
953 list_del(&wait->task_list);
954 clear_bit_unlock(BLK_MQ_S_TAG_WAITING, &hctx->state);
955 blk_mq_run_hw_queue(hctx, true);
956 return 1;
957}
958
959static bool blk_mq_dispatch_wait_add(struct blk_mq_hw_ctx *hctx)
960{
961 struct sbq_wait_state *ws;
962
963 /*
964 * The TAG_WAITING bit serves as a lock protecting hctx->dispatch_wait.
965 * The thread which wins the race to grab this bit adds the hardware
966 * queue to the wait queue.
967 */
968 if (test_bit(BLK_MQ_S_TAG_WAITING, &hctx->state) ||
969 test_and_set_bit_lock(BLK_MQ_S_TAG_WAITING, &hctx->state))
970 return false;
971
972 init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake);
973 ws = bt_wait_ptr(&hctx->tags->bitmap_tags, hctx);
974
975 /*
976 * As soon as this returns, it's no longer safe to fiddle with
977 * hctx->dispatch_wait, since a completion can wake up the wait queue
978 * and unlock the bit.
979 */
980 add_wait_queue(&ws->wait, &hctx->dispatch_wait);
981 return true;
982}
983
Omar Sandoval81380ca2017-04-07 08:56:26 -0600984bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list)
Jens Axboef04c3df2016-12-07 08:41:17 -0700985{
Omar Sandoval81380ca2017-04-07 08:56:26 -0600986 struct blk_mq_hw_ctx *hctx;
Jens Axboef04c3df2016-12-07 08:41:17 -0700987 struct request *rq;
Jens Axboe93efe982017-03-24 12:04:19 -0600988 int errors, queued, ret = BLK_MQ_RQ_QUEUE_OK;
Jens Axboef04c3df2016-12-07 08:41:17 -0700989
Omar Sandoval81380ca2017-04-07 08:56:26 -0600990 if (list_empty(list))
991 return false;
992
Jens Axboef04c3df2016-12-07 08:41:17 -0700993 /*
Jens Axboef04c3df2016-12-07 08:41:17 -0700994 * Now process all the entries, sending them to the driver.
995 */
Jens Axboe93efe982017-03-24 12:04:19 -0600996 errors = queued = 0;
Omar Sandoval81380ca2017-04-07 08:56:26 -0600997 do {
Jens Axboef04c3df2016-12-07 08:41:17 -0700998 struct blk_mq_queue_data bd;
999
1000 rq = list_first_entry(list, struct request, queuelist);
Jens Axboebd166ef2017-01-17 06:03:22 -07001001 if (!blk_mq_get_driver_tag(rq, &hctx, false)) {
1002 if (!queued && reorder_tags_to_front(list))
1003 continue;
Jens Axboe3c782d62017-01-26 12:50:36 -07001004
1005 /*
Omar Sandovalda55f2c2017-02-22 10:58:29 -08001006 * The initial allocation attempt failed, so we need to
1007 * rerun the hardware queue when a tag is freed.
Jens Axboe3c782d62017-01-26 12:50:36 -07001008 */
Omar Sandoval807b1042017-04-05 12:01:35 -07001009 if (!blk_mq_dispatch_wait_add(hctx))
Jens Axboe3c782d62017-01-26 12:50:36 -07001010 break;
Omar Sandoval807b1042017-04-05 12:01:35 -07001011
1012 /*
1013 * It's possible that a tag was freed in the window
1014 * between the allocation failure and adding the
1015 * hardware queue to the wait queue.
1016 */
1017 if (!blk_mq_get_driver_tag(rq, &hctx, false))
1018 break;
Jens Axboebd166ef2017-01-17 06:03:22 -07001019 }
Omar Sandovalda55f2c2017-02-22 10:58:29 -08001020
Jens Axboef04c3df2016-12-07 08:41:17 -07001021 list_del_init(&rq->queuelist);
1022
1023 bd.rq = rq;
Jens Axboe113285b2017-03-02 13:26:04 -07001024
1025 /*
1026 * Flag last if we have no more requests, or if we have more
1027 * but can't assign a driver tag to it.
1028 */
1029 if (list_empty(list))
1030 bd.last = true;
1031 else {
1032 struct request *nxt;
1033
1034 nxt = list_first_entry(list, struct request, queuelist);
1035 bd.last = !blk_mq_get_driver_tag(nxt, NULL, false);
1036 }
Jens Axboef04c3df2016-12-07 08:41:17 -07001037
1038 ret = q->mq_ops->queue_rq(hctx, &bd);
1039 switch (ret) {
1040 case BLK_MQ_RQ_QUEUE_OK:
1041 queued++;
1042 break;
1043 case BLK_MQ_RQ_QUEUE_BUSY:
Jens Axboe113285b2017-03-02 13:26:04 -07001044 blk_mq_put_driver_tag_hctx(hctx, rq);
Jens Axboef04c3df2016-12-07 08:41:17 -07001045 list_add(&rq->queuelist, list);
1046 __blk_mq_requeue_request(rq);
1047 break;
1048 default:
1049 pr_err("blk-mq: bad return on queue: %d\n", ret);
1050 case BLK_MQ_RQ_QUEUE_ERROR:
Jens Axboe93efe982017-03-24 12:04:19 -06001051 errors++;
Christoph Hellwigcaf7df12017-04-20 16:03:16 +02001052 blk_mq_end_request(rq, -EIO);
Jens Axboef04c3df2016-12-07 08:41:17 -07001053 break;
1054 }
1055
1056 if (ret == BLK_MQ_RQ_QUEUE_BUSY)
1057 break;
Omar Sandoval81380ca2017-04-07 08:56:26 -06001058 } while (!list_empty(list));
Jens Axboef04c3df2016-12-07 08:41:17 -07001059
1060 hctx->dispatched[queued_to_index(queued)]++;
1061
1062 /*
1063 * Any items that need requeuing? Stuff them into hctx->dispatch,
1064 * that is where we will continue on next queue run.
1065 */
1066 if (!list_empty(list)) {
Jens Axboe113285b2017-03-02 13:26:04 -07001067 /*
Bart Van Assche710c7852017-04-07 11:16:51 -07001068 * If an I/O scheduler has been configured and we got a driver
1069 * tag for the next request already, free it again.
Jens Axboe113285b2017-03-02 13:26:04 -07001070 */
1071 rq = list_first_entry(list, struct request, queuelist);
1072 blk_mq_put_driver_tag(rq);
1073
Jens Axboef04c3df2016-12-07 08:41:17 -07001074 spin_lock(&hctx->lock);
Jens Axboec13660a2017-01-26 12:40:07 -07001075 list_splice_init(list, &hctx->dispatch);
Jens Axboef04c3df2016-12-07 08:41:17 -07001076 spin_unlock(&hctx->lock);
1077
1078 /*
Bart Van Assche710c7852017-04-07 11:16:51 -07001079 * If SCHED_RESTART was set by the caller of this function and
1080 * it is no longer set that means that it was cleared by another
1081 * thread and hence that a queue rerun is needed.
Jens Axboef04c3df2016-12-07 08:41:17 -07001082 *
Bart Van Assche710c7852017-04-07 11:16:51 -07001083 * If TAG_WAITING is set that means that an I/O scheduler has
1084 * been configured and another thread is waiting for a driver
1085 * tag. To guarantee fairness, do not rerun this hardware queue
1086 * but let the other thread grab the driver tag.
Jens Axboebd166ef2017-01-17 06:03:22 -07001087 *
Bart Van Assche710c7852017-04-07 11:16:51 -07001088 * If no I/O scheduler has been configured it is possible that
1089 * the hardware queue got stopped and restarted before requests
1090 * were pushed back onto the dispatch list. Rerun the queue to
1091 * avoid starvation. Notes:
1092 * - blk_mq_run_hw_queue() checks whether or not a queue has
1093 * been stopped before rerunning a queue.
1094 * - Some but not all block drivers stop a queue before
1095 * returning BLK_MQ_RQ_QUEUE_BUSY. Two exceptions are scsi-mq
1096 * and dm-rq.
Jens Axboebd166ef2017-01-17 06:03:22 -07001097 */
Omar Sandovalda55f2c2017-02-22 10:58:29 -08001098 if (!blk_mq_sched_needs_restart(hctx) &&
1099 !test_bit(BLK_MQ_S_TAG_WAITING, &hctx->state))
Jens Axboebd166ef2017-01-17 06:03:22 -07001100 blk_mq_run_hw_queue(hctx, true);
Jens Axboef04c3df2016-12-07 08:41:17 -07001101 }
1102
Jens Axboe93efe982017-03-24 12:04:19 -06001103 return (queued + errors) != 0;
Jens Axboef04c3df2016-12-07 08:41:17 -07001104}
1105
Bart Van Assche6a83e742016-11-02 10:09:51 -06001106static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
1107{
1108 int srcu_idx;
1109
1110 WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask) &&
1111 cpu_online(hctx->next_cpu));
1112
1113 if (!(hctx->flags & BLK_MQ_F_BLOCKING)) {
1114 rcu_read_lock();
Jens Axboebd166ef2017-01-17 06:03:22 -07001115 blk_mq_sched_dispatch_requests(hctx);
Bart Van Assche6a83e742016-11-02 10:09:51 -06001116 rcu_read_unlock();
1117 } else {
Jens Axboebf4907c2017-03-30 12:30:39 -06001118 might_sleep();
1119
Bart Van Assche6a83e742016-11-02 10:09:51 -06001120 srcu_idx = srcu_read_lock(&hctx->queue_rq_srcu);
Jens Axboebd166ef2017-01-17 06:03:22 -07001121 blk_mq_sched_dispatch_requests(hctx);
Bart Van Assche6a83e742016-11-02 10:09:51 -06001122 srcu_read_unlock(&hctx->queue_rq_srcu, srcu_idx);
1123 }
1124}
1125
Jens Axboe506e9312014-05-07 10:26:44 -06001126/*
1127 * It'd be great if the workqueue API had a way to pass
1128 * in a mask and had some smarts for more clever placement.
1129 * For now we just round-robin here, switching for every
1130 * BLK_MQ_CPU_WORK_BATCH queued items.
1131 */
1132static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
1133{
Christoph Hellwigb657d7e2014-11-24 09:27:23 +01001134 if (hctx->queue->nr_hw_queues == 1)
1135 return WORK_CPU_UNBOUND;
Jens Axboe506e9312014-05-07 10:26:44 -06001136
1137 if (--hctx->next_cpu_batch <= 0) {
Gabriel Krisman Bertazic02ebfd2016-09-28 00:24:24 -03001138 int next_cpu;
Jens Axboe506e9312014-05-07 10:26:44 -06001139
1140 next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask);
1141 if (next_cpu >= nr_cpu_ids)
1142 next_cpu = cpumask_first(hctx->cpumask);
1143
1144 hctx->next_cpu = next_cpu;
1145 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
1146 }
1147
Christoph Hellwigb657d7e2014-11-24 09:27:23 +01001148 return hctx->next_cpu;
Jens Axboe506e9312014-05-07 10:26:44 -06001149}
1150
Bart Van Assche7587a5a2017-04-07 11:16:52 -07001151static void __blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async,
1152 unsigned long msecs)
Jens Axboe320ae512013-10-24 09:20:05 +01001153{
Bart Van Assche5d1b25c2016-10-28 17:19:15 -07001154 if (unlikely(blk_mq_hctx_stopped(hctx) ||
1155 !blk_mq_hw_queue_mapped(hctx)))
Jens Axboe320ae512013-10-24 09:20:05 +01001156 return;
1157
Jens Axboe1b792f22016-09-21 10:12:13 -06001158 if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) {
Paolo Bonzini2a90d4a2014-11-07 23:04:00 +01001159 int cpu = get_cpu();
1160 if (cpumask_test_cpu(cpu, hctx->cpumask)) {
Paolo Bonzini398205b2014-11-07 23:03:59 +01001161 __blk_mq_run_hw_queue(hctx);
Paolo Bonzini2a90d4a2014-11-07 23:04:00 +01001162 put_cpu();
Paolo Bonzini398205b2014-11-07 23:03:59 +01001163 return;
1164 }
Jens Axboee4043dc2014-04-09 10:18:23 -06001165
Paolo Bonzini2a90d4a2014-11-07 23:04:00 +01001166 put_cpu();
Jens Axboee4043dc2014-04-09 10:18:23 -06001167 }
Paolo Bonzini398205b2014-11-07 23:03:59 +01001168
Jens Axboe9f993732017-04-10 09:54:54 -06001169 kblockd_schedule_delayed_work_on(blk_mq_hctx_next_cpu(hctx),
1170 &hctx->run_work,
1171 msecs_to_jiffies(msecs));
Bart Van Assche7587a5a2017-04-07 11:16:52 -07001172}
1173
1174void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
1175{
1176 __blk_mq_delay_run_hw_queue(hctx, true, msecs);
1177}
1178EXPORT_SYMBOL(blk_mq_delay_run_hw_queue);
1179
1180void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
1181{
1182 __blk_mq_delay_run_hw_queue(hctx, async, 0);
Jens Axboe320ae512013-10-24 09:20:05 +01001183}
Omar Sandoval5b727272017-04-14 01:00:00 -07001184EXPORT_SYMBOL(blk_mq_run_hw_queue);
Jens Axboe320ae512013-10-24 09:20:05 +01001185
Mike Snitzerb94ec292015-03-11 23:56:38 -04001186void blk_mq_run_hw_queues(struct request_queue *q, bool async)
Jens Axboe320ae512013-10-24 09:20:05 +01001187{
1188 struct blk_mq_hw_ctx *hctx;
1189 int i;
1190
1191 queue_for_each_hw_ctx(q, hctx, i) {
Jens Axboebd166ef2017-01-17 06:03:22 -07001192 if (!blk_mq_hctx_has_pending(hctx) ||
Bart Van Assche5d1b25c2016-10-28 17:19:15 -07001193 blk_mq_hctx_stopped(hctx))
Jens Axboe320ae512013-10-24 09:20:05 +01001194 continue;
1195
Mike Snitzerb94ec292015-03-11 23:56:38 -04001196 blk_mq_run_hw_queue(hctx, async);
Jens Axboe320ae512013-10-24 09:20:05 +01001197 }
1198}
Mike Snitzerb94ec292015-03-11 23:56:38 -04001199EXPORT_SYMBOL(blk_mq_run_hw_queues);
Jens Axboe320ae512013-10-24 09:20:05 +01001200
Bart Van Asschefd001442016-10-28 17:19:37 -07001201/**
1202 * blk_mq_queue_stopped() - check whether one or more hctxs have been stopped
1203 * @q: request queue.
1204 *
1205 * The caller is responsible for serializing this function against
1206 * blk_mq_{start,stop}_hw_queue().
1207 */
1208bool blk_mq_queue_stopped(struct request_queue *q)
1209{
1210 struct blk_mq_hw_ctx *hctx;
1211 int i;
1212
1213 queue_for_each_hw_ctx(q, hctx, i)
1214 if (blk_mq_hctx_stopped(hctx))
1215 return true;
1216
1217 return false;
1218}
1219EXPORT_SYMBOL(blk_mq_queue_stopped);
1220
Jens Axboe320ae512013-10-24 09:20:05 +01001221void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
1222{
Jens Axboe9f993732017-04-10 09:54:54 -06001223 cancel_delayed_work_sync(&hctx->run_work);
Jens Axboe320ae512013-10-24 09:20:05 +01001224 set_bit(BLK_MQ_S_STOPPED, &hctx->state);
1225}
1226EXPORT_SYMBOL(blk_mq_stop_hw_queue);
1227
Christoph Hellwig280d45f2013-10-25 14:45:58 +01001228void blk_mq_stop_hw_queues(struct request_queue *q)
1229{
1230 struct blk_mq_hw_ctx *hctx;
1231 int i;
1232
1233 queue_for_each_hw_ctx(q, hctx, i)
1234 blk_mq_stop_hw_queue(hctx);
1235}
1236EXPORT_SYMBOL(blk_mq_stop_hw_queues);
1237
Jens Axboe320ae512013-10-24 09:20:05 +01001238void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
1239{
1240 clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
Jens Axboee4043dc2014-04-09 10:18:23 -06001241
Jens Axboe0ffbce82014-06-25 08:22:34 -06001242 blk_mq_run_hw_queue(hctx, false);
Jens Axboe320ae512013-10-24 09:20:05 +01001243}
1244EXPORT_SYMBOL(blk_mq_start_hw_queue);
1245
Christoph Hellwig2f268552014-04-16 09:44:56 +02001246void blk_mq_start_hw_queues(struct request_queue *q)
1247{
1248 struct blk_mq_hw_ctx *hctx;
1249 int i;
1250
1251 queue_for_each_hw_ctx(q, hctx, i)
1252 blk_mq_start_hw_queue(hctx);
1253}
1254EXPORT_SYMBOL(blk_mq_start_hw_queues);
1255
Jens Axboeae911c52016-12-08 13:19:30 -07001256void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
1257{
1258 if (!blk_mq_hctx_stopped(hctx))
1259 return;
1260
1261 clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
1262 blk_mq_run_hw_queue(hctx, async);
1263}
1264EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue);
1265
Christoph Hellwig1b4a3252014-04-16 09:44:54 +02001266void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
Jens Axboe320ae512013-10-24 09:20:05 +01001267{
1268 struct blk_mq_hw_ctx *hctx;
1269 int i;
1270
Jens Axboeae911c52016-12-08 13:19:30 -07001271 queue_for_each_hw_ctx(q, hctx, i)
1272 blk_mq_start_stopped_hw_queue(hctx, async);
Jens Axboe320ae512013-10-24 09:20:05 +01001273}
1274EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);
1275
Christoph Hellwig70f4db62014-04-16 10:48:08 -06001276static void blk_mq_run_work_fn(struct work_struct *work)
Jens Axboe320ae512013-10-24 09:20:05 +01001277{
1278 struct blk_mq_hw_ctx *hctx;
1279
Jens Axboe9f993732017-04-10 09:54:54 -06001280 hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work);
Jens Axboe21c6e932017-04-10 09:54:56 -06001281
1282 /*
1283 * If we are stopped, don't run the queue. The exception is if
1284 * BLK_MQ_S_START_ON_RUN is set. For that case, we auto-clear
1285 * the STOPPED bit and run it.
1286 */
1287 if (test_bit(BLK_MQ_S_STOPPED, &hctx->state)) {
1288 if (!test_bit(BLK_MQ_S_START_ON_RUN, &hctx->state))
1289 return;
1290
1291 clear_bit(BLK_MQ_S_START_ON_RUN, &hctx->state);
1292 clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
1293 }
Jens Axboee4043dc2014-04-09 10:18:23 -06001294
Jens Axboe320ae512013-10-24 09:20:05 +01001295 __blk_mq_run_hw_queue(hctx);
1296}
1297
Christoph Hellwig70f4db62014-04-16 10:48:08 -06001298
1299void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
1300{
Ming Lei19c66e52014-12-03 19:38:04 +08001301 if (unlikely(!blk_mq_hw_queue_mapped(hctx)))
1302 return;
Christoph Hellwig70f4db62014-04-16 10:48:08 -06001303
Jens Axboe21c6e932017-04-10 09:54:56 -06001304 /*
1305 * Stop the hw queue, then modify currently delayed work.
1306 * This should prevent us from running the queue prematurely.
1307 * Mark the queue as auto-clearing STOPPED when it runs.
1308 */
Jens Axboe7e79dad2017-01-19 07:58:59 -07001309 blk_mq_stop_hw_queue(hctx);
Jens Axboe21c6e932017-04-10 09:54:56 -06001310 set_bit(BLK_MQ_S_START_ON_RUN, &hctx->state);
1311 kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx),
1312 &hctx->run_work,
1313 msecs_to_jiffies(msecs));
Christoph Hellwig70f4db62014-04-16 10:48:08 -06001314}
1315EXPORT_SYMBOL(blk_mq_delay_queue);
1316
Ming Leicfd0c552015-10-20 23:13:57 +08001317static inline void __blk_mq_insert_req_list(struct blk_mq_hw_ctx *hctx,
Ming Leicfd0c552015-10-20 23:13:57 +08001318 struct request *rq,
1319 bool at_head)
Jens Axboe320ae512013-10-24 09:20:05 +01001320{
Jens Axboee57690f2016-08-24 15:34:35 -06001321 struct blk_mq_ctx *ctx = rq->mq_ctx;
1322
Jens Axboe01b983c2013-11-19 18:59:10 -07001323 trace_block_rq_insert(hctx->queue, rq);
1324
Christoph Hellwig72a0a362014-02-07 10:22:36 -08001325 if (at_head)
1326 list_add(&rq->queuelist, &ctx->rq_list);
1327 else
1328 list_add_tail(&rq->queuelist, &ctx->rq_list);
Ming Leicfd0c552015-10-20 23:13:57 +08001329}
Jens Axboe4bb659b2014-05-09 09:36:49 -06001330
Jens Axboe2c3ad662016-12-14 14:34:47 -07001331void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
1332 bool at_head)
Ming Leicfd0c552015-10-20 23:13:57 +08001333{
1334 struct blk_mq_ctx *ctx = rq->mq_ctx;
1335
Jens Axboee57690f2016-08-24 15:34:35 -06001336 __blk_mq_insert_req_list(hctx, rq, at_head);
Jens Axboe320ae512013-10-24 09:20:05 +01001337 blk_mq_hctx_mark_pending(hctx, ctx);
Jens Axboe320ae512013-10-24 09:20:05 +01001338}
1339
Jens Axboebd166ef2017-01-17 06:03:22 -07001340void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
1341 struct list_head *list)
Jens Axboe320ae512013-10-24 09:20:05 +01001342
1343{
Jens Axboe320ae512013-10-24 09:20:05 +01001344 /*
1345 * preemption doesn't flush plug list, so it's possible ctx->cpu is
1346 * offline now
1347 */
1348 spin_lock(&ctx->lock);
1349 while (!list_empty(list)) {
1350 struct request *rq;
1351
1352 rq = list_first_entry(list, struct request, queuelist);
Jens Axboee57690f2016-08-24 15:34:35 -06001353 BUG_ON(rq->mq_ctx != ctx);
Jens Axboe320ae512013-10-24 09:20:05 +01001354 list_del_init(&rq->queuelist);
Jens Axboee57690f2016-08-24 15:34:35 -06001355 __blk_mq_insert_req_list(hctx, rq, false);
Jens Axboe320ae512013-10-24 09:20:05 +01001356 }
Ming Leicfd0c552015-10-20 23:13:57 +08001357 blk_mq_hctx_mark_pending(hctx, ctx);
Jens Axboe320ae512013-10-24 09:20:05 +01001358 spin_unlock(&ctx->lock);
Jens Axboe320ae512013-10-24 09:20:05 +01001359}
1360
1361static int plug_ctx_cmp(void *priv, struct list_head *a, struct list_head *b)
1362{
1363 struct request *rqa = container_of(a, struct request, queuelist);
1364 struct request *rqb = container_of(b, struct request, queuelist);
1365
1366 return !(rqa->mq_ctx < rqb->mq_ctx ||
1367 (rqa->mq_ctx == rqb->mq_ctx &&
1368 blk_rq_pos(rqa) < blk_rq_pos(rqb)));
1369}
1370
1371void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
1372{
1373 struct blk_mq_ctx *this_ctx;
1374 struct request_queue *this_q;
1375 struct request *rq;
1376 LIST_HEAD(list);
1377 LIST_HEAD(ctx_list);
1378 unsigned int depth;
1379
1380 list_splice_init(&plug->mq_list, &list);
1381
1382 list_sort(NULL, &list, plug_ctx_cmp);
1383
1384 this_q = NULL;
1385 this_ctx = NULL;
1386 depth = 0;
1387
1388 while (!list_empty(&list)) {
1389 rq = list_entry_rq(list.next);
1390 list_del_init(&rq->queuelist);
1391 BUG_ON(!rq->q);
1392 if (rq->mq_ctx != this_ctx) {
1393 if (this_ctx) {
Jens Axboebd166ef2017-01-17 06:03:22 -07001394 trace_block_unplug(this_q, depth, from_schedule);
1395 blk_mq_sched_insert_requests(this_q, this_ctx,
1396 &ctx_list,
1397 from_schedule);
Jens Axboe320ae512013-10-24 09:20:05 +01001398 }
1399
1400 this_ctx = rq->mq_ctx;
1401 this_q = rq->q;
1402 depth = 0;
1403 }
1404
1405 depth++;
1406 list_add_tail(&rq->queuelist, &ctx_list);
1407 }
1408
1409 /*
1410 * If 'this_ctx' is set, we know we have entries to complete
1411 * on 'ctx_list'. Do those.
1412 */
1413 if (this_ctx) {
Jens Axboebd166ef2017-01-17 06:03:22 -07001414 trace_block_unplug(this_q, depth, from_schedule);
1415 blk_mq_sched_insert_requests(this_q, this_ctx, &ctx_list,
1416 from_schedule);
Jens Axboe320ae512013-10-24 09:20:05 +01001417 }
1418}
1419
1420static void blk_mq_bio_to_request(struct request *rq, struct bio *bio)
1421{
Bart Van Asscheda8d7f02017-04-19 14:01:24 -07001422 blk_init_request_from_bio(rq, bio);
Jens Axboe4b570522014-05-29 11:00:11 -06001423
Jens Axboe6e85eaf2016-12-02 20:00:14 -07001424 blk_account_io_start(rq, true);
Jens Axboe320ae512013-10-24 09:20:05 +01001425}
1426
Jens Axboe274a5842014-08-15 12:44:08 -06001427static inline bool hctx_allow_merges(struct blk_mq_hw_ctx *hctx)
1428{
1429 return (hctx->flags & BLK_MQ_F_SHOULD_MERGE) &&
1430 !blk_queue_nomerges(hctx->queue);
1431}
1432
Jens Axboe07068d52014-05-22 10:40:51 -06001433static inline bool blk_mq_merge_queue_io(struct blk_mq_hw_ctx *hctx,
1434 struct blk_mq_ctx *ctx,
1435 struct request *rq, struct bio *bio)
1436{
Ming Leie18378a2015-10-20 23:13:54 +08001437 if (!hctx_allow_merges(hctx) || !bio_mergeable(bio)) {
Jens Axboe07068d52014-05-22 10:40:51 -06001438 blk_mq_bio_to_request(rq, bio);
1439 spin_lock(&ctx->lock);
1440insert_rq:
1441 __blk_mq_insert_request(hctx, rq, false);
1442 spin_unlock(&ctx->lock);
1443 return false;
1444 } else {
Jens Axboe274a5842014-08-15 12:44:08 -06001445 struct request_queue *q = hctx->queue;
1446
Jens Axboe07068d52014-05-22 10:40:51 -06001447 spin_lock(&ctx->lock);
1448 if (!blk_mq_attempt_merge(q, ctx, bio)) {
1449 blk_mq_bio_to_request(rq, bio);
1450 goto insert_rq;
1451 }
1452
1453 spin_unlock(&ctx->lock);
Jens Axboebd166ef2017-01-17 06:03:22 -07001454 __blk_mq_finish_request(hctx, ctx, rq);
Jens Axboe07068d52014-05-22 10:40:51 -06001455 return true;
1456 }
1457}
1458
Jens Axboefd2d3322017-01-12 10:04:45 -07001459static blk_qc_t request_to_qc_t(struct blk_mq_hw_ctx *hctx, struct request *rq)
1460{
Jens Axboebd166ef2017-01-17 06:03:22 -07001461 if (rq->tag != -1)
1462 return blk_tag_to_qc_t(rq->tag, hctx->queue_num, false);
1463
1464 return blk_tag_to_qc_t(rq->internal_tag, hctx->queue_num, true);
Jens Axboefd2d3322017-01-12 10:04:45 -07001465}
1466
Christoph Hellwig5eb61262017-03-22 15:01:51 -04001467static void __blk_mq_try_issue_directly(struct request *rq, blk_qc_t *cookie,
Jens Axboe9c621102017-03-14 11:51:59 -06001468 bool may_sleep)
Shaohua Lif984df12015-05-08 10:51:32 -07001469{
Shaohua Lif984df12015-05-08 10:51:32 -07001470 struct request_queue *q = rq->q;
Shaohua Lif984df12015-05-08 10:51:32 -07001471 struct blk_mq_queue_data bd = {
1472 .rq = rq,
Omar Sandovald945a362017-04-05 12:01:36 -07001473 .last = true,
Shaohua Lif984df12015-05-08 10:51:32 -07001474 };
Jens Axboebd166ef2017-01-17 06:03:22 -07001475 struct blk_mq_hw_ctx *hctx;
1476 blk_qc_t new_cookie;
1477 int ret;
Shaohua Lif984df12015-05-08 10:51:32 -07001478
Jens Axboebd166ef2017-01-17 06:03:22 -07001479 if (q->elevator)
Bart Van Assche2253efc2016-10-28 17:20:02 -07001480 goto insert;
1481
Jens Axboebd166ef2017-01-17 06:03:22 -07001482 if (!blk_mq_get_driver_tag(rq, &hctx, false))
1483 goto insert;
1484
1485 new_cookie = request_to_qc_t(hctx, rq);
1486
Shaohua Lif984df12015-05-08 10:51:32 -07001487 /*
1488 * For OK queue, we are done. For error, kill it. Any other
1489 * error (busy), just add it to our list as we previously
1490 * would have done
1491 */
1492 ret = q->mq_ops->queue_rq(hctx, &bd);
Jens Axboe7b371632015-11-05 10:41:40 -07001493 if (ret == BLK_MQ_RQ_QUEUE_OK) {
1494 *cookie = new_cookie;
Bart Van Assche2253efc2016-10-28 17:20:02 -07001495 return;
Shaohua Lif984df12015-05-08 10:51:32 -07001496 }
Jens Axboe7b371632015-11-05 10:41:40 -07001497
Jens Axboe7b371632015-11-05 10:41:40 -07001498 if (ret == BLK_MQ_RQ_QUEUE_ERROR) {
1499 *cookie = BLK_QC_T_NONE;
Christoph Hellwigcaf7df12017-04-20 16:03:16 +02001500 blk_mq_end_request(rq, -EIO);
Bart Van Assche2253efc2016-10-28 17:20:02 -07001501 return;
Jens Axboe7b371632015-11-05 10:41:40 -07001502 }
1503
Josef Bacikb58e1762017-03-28 16:37:52 -04001504 __blk_mq_requeue_request(rq);
Bart Van Assche2253efc2016-10-28 17:20:02 -07001505insert:
Jens Axboe9c621102017-03-14 11:51:59 -06001506 blk_mq_sched_insert_request(rq, false, true, false, may_sleep);
Shaohua Lif984df12015-05-08 10:51:32 -07001507}
1508
Christoph Hellwig5eb61262017-03-22 15:01:51 -04001509static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
1510 struct request *rq, blk_qc_t *cookie)
1511{
1512 if (!(hctx->flags & BLK_MQ_F_BLOCKING)) {
1513 rcu_read_lock();
1514 __blk_mq_try_issue_directly(rq, cookie, false);
1515 rcu_read_unlock();
1516 } else {
Jens Axboebf4907c2017-03-30 12:30:39 -06001517 unsigned int srcu_idx;
1518
1519 might_sleep();
1520
1521 srcu_idx = srcu_read_lock(&hctx->queue_rq_srcu);
Christoph Hellwig5eb61262017-03-22 15:01:51 -04001522 __blk_mq_try_issue_directly(rq, cookie, true);
1523 srcu_read_unlock(&hctx->queue_rq_srcu, srcu_idx);
1524 }
1525}
1526
Jens Axboedece1632015-11-05 10:41:16 -07001527static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
Jens Axboe07068d52014-05-22 10:40:51 -06001528{
Christoph Hellwigef295ec2016-10-28 08:48:16 -06001529 const int is_sync = op_is_sync(bio->bi_opf);
Christoph Hellwigf73f44e2017-01-27 08:30:47 -07001530 const int is_flush_fua = op_is_flush(bio->bi_opf);
Jens Axboe5a797e02017-01-26 12:22:11 -07001531 struct blk_mq_alloc_data data = { .flags = 0 };
Jens Axboe07068d52014-05-22 10:40:51 -06001532 struct request *rq;
Christoph Hellwig5eb61262017-03-22 15:01:51 -04001533 unsigned int request_count = 0;
Shaohua Lif984df12015-05-08 10:51:32 -07001534 struct blk_plug *plug;
Shaohua Li5b3f3412015-05-08 10:51:33 -07001535 struct request *same_queue_rq = NULL;
Jens Axboe7b371632015-11-05 10:41:40 -07001536 blk_qc_t cookie;
Jens Axboe87760e52016-11-09 12:38:14 -07001537 unsigned int wb_acct;
Jens Axboe07068d52014-05-22 10:40:51 -06001538
1539 blk_queue_bounce(q, &bio);
1540
1541 if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02001542 bio_io_error(bio);
Jens Axboedece1632015-11-05 10:41:16 -07001543 return BLK_QC_T_NONE;
Jens Axboe07068d52014-05-22 10:40:51 -06001544 }
1545
Kent Overstreet54efd502015-04-23 22:37:18 -07001546 blk_queue_split(q, &bio, q->bio_split);
1547
Omar Sandoval87c279e2016-06-01 22:18:48 -07001548 if (!is_flush_fua && !blk_queue_nomerges(q) &&
1549 blk_attempt_plug_merge(q, bio, &request_count, &same_queue_rq))
1550 return BLK_QC_T_NONE;
Shaohua Lif984df12015-05-08 10:51:32 -07001551
Jens Axboebd166ef2017-01-17 06:03:22 -07001552 if (blk_mq_sched_bio_merge(q, bio))
1553 return BLK_QC_T_NONE;
1554
Jens Axboe87760e52016-11-09 12:38:14 -07001555 wb_acct = wbt_wait(q->rq_wb, bio, NULL);
1556
Jens Axboebd166ef2017-01-17 06:03:22 -07001557 trace_block_getrq(q, bio, bio->bi_opf);
1558
1559 rq = blk_mq_sched_get_request(q, bio, bio->bi_opf, &data);
Jens Axboe87760e52016-11-09 12:38:14 -07001560 if (unlikely(!rq)) {
1561 __wbt_done(q->rq_wb, wb_acct);
Jens Axboedece1632015-11-05 10:41:16 -07001562 return BLK_QC_T_NONE;
Jens Axboe87760e52016-11-09 12:38:14 -07001563 }
1564
1565 wbt_track(&rq->issue_stat, wb_acct);
Jens Axboe07068d52014-05-22 10:40:51 -06001566
Jens Axboefd2d3322017-01-12 10:04:45 -07001567 cookie = request_to_qc_t(data.hctx, rq);
Jens Axboe07068d52014-05-22 10:40:51 -06001568
Shaohua Lif984df12015-05-08 10:51:32 -07001569 plug = current->plug;
Christoph Hellwiga4d907b2017-03-22 15:01:53 -04001570 if (unlikely(is_flush_fua)) {
Shaohua Lif984df12015-05-08 10:51:32 -07001571 blk_mq_put_ctx(data.ctx);
Christoph Hellwiga4d907b2017-03-22 15:01:53 -04001572 blk_mq_bio_to_request(rq, bio);
1573 if (q->elevator) {
1574 blk_mq_sched_insert_request(rq, false, true, true,
1575 true);
Bart Van Assche6a83e742016-11-02 10:09:51 -06001576 } else {
Christoph Hellwiga4d907b2017-03-22 15:01:53 -04001577 blk_insert_flush(rq);
1578 blk_mq_run_hw_queue(data.hctx, true);
Bart Van Assche6a83e742016-11-02 10:09:51 -06001579 }
Christoph Hellwiga4d907b2017-03-22 15:01:53 -04001580 } else if (plug && q->nr_hw_queues == 1) {
Shaohua Li600271d2016-11-03 17:03:54 -07001581 struct request *last = NULL;
1582
Jens Axboeb00c53e2017-04-20 16:40:36 -06001583 blk_mq_put_ctx(data.ctx);
Jeff Moyere6c44382015-05-08 10:51:30 -07001584 blk_mq_bio_to_request(rq, bio);
Ming Lei0a6219a2016-11-16 18:07:05 +08001585
1586 /*
1587 * @request_count may become stale because of schedule
1588 * out, so check the list again.
1589 */
1590 if (list_empty(&plug->mq_list))
1591 request_count = 0;
Christoph Hellwig254d2592017-03-22 15:01:50 -04001592 else if (blk_queue_nomerges(q))
1593 request_count = blk_plug_queued_count(q);
1594
Ming Lei676d0602015-10-20 23:13:56 +08001595 if (!request_count)
Jeff Moyere6c44382015-05-08 10:51:30 -07001596 trace_block_plug(q);
Shaohua Li600271d2016-11-03 17:03:54 -07001597 else
1598 last = list_entry_rq(plug->mq_list.prev);
Jens Axboeb094f892015-11-20 20:29:45 -07001599
Shaohua Li600271d2016-11-03 17:03:54 -07001600 if (request_count >= BLK_MAX_REQUEST_COUNT || (last &&
1601 blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) {
Jeff Moyere6c44382015-05-08 10:51:30 -07001602 blk_flush_plug_list(plug, false);
1603 trace_block_plug(q);
Jens Axboe320ae512013-10-24 09:20:05 +01001604 }
Jens Axboeb094f892015-11-20 20:29:45 -07001605
Jeff Moyere6c44382015-05-08 10:51:30 -07001606 list_add_tail(&rq->queuelist, &plug->mq_list);
Christoph Hellwig22997222017-03-22 15:01:52 -04001607 } else if (plug && !blk_queue_nomerges(q)) {
Jens Axboe320ae512013-10-24 09:20:05 +01001608 blk_mq_bio_to_request(rq, bio);
Jens Axboe320ae512013-10-24 09:20:05 +01001609
Jens Axboe320ae512013-10-24 09:20:05 +01001610 /*
1611 * We do limited plugging. If the bio can be merged, do that.
1612 * Otherwise the existing request in the plug list will be
1613 * issued. So the plug list will have one request at most
Christoph Hellwig22997222017-03-22 15:01:52 -04001614 * The plug list might get flushed before this. If that happens,
1615 * the plug list is empty, and same_queue_rq is invalid.
Jens Axboe320ae512013-10-24 09:20:05 +01001616 */
Christoph Hellwig22997222017-03-22 15:01:52 -04001617 if (list_empty(&plug->mq_list))
1618 same_queue_rq = NULL;
1619 if (same_queue_rq)
1620 list_del_init(&same_queue_rq->queuelist);
1621 list_add_tail(&rq->queuelist, &plug->mq_list);
1622
Jens Axboebf4907c2017-03-30 12:30:39 -06001623 blk_mq_put_ctx(data.ctx);
1624
Christoph Hellwig22997222017-03-22 15:01:52 -04001625 if (same_queue_rq)
1626 blk_mq_try_issue_directly(data.hctx, same_queue_rq,
1627 &cookie);
Christoph Hellwiga4d907b2017-03-22 15:01:53 -04001628 } else if (q->nr_hw_queues > 1 && is_sync) {
Jens Axboebd166ef2017-01-17 06:03:22 -07001629 blk_mq_put_ctx(data.ctx);
1630 blk_mq_bio_to_request(rq, bio);
Christoph Hellwig22997222017-03-22 15:01:52 -04001631 blk_mq_try_issue_directly(data.hctx, rq, &cookie);
Christoph Hellwiga4d907b2017-03-22 15:01:53 -04001632 } else if (q->elevator) {
Jens Axboeb00c53e2017-04-20 16:40:36 -06001633 blk_mq_put_ctx(data.ctx);
Christoph Hellwiga4d907b2017-03-22 15:01:53 -04001634 blk_mq_bio_to_request(rq, bio);
1635 blk_mq_sched_insert_request(rq, false, true, true, true);
Jens Axboeb00c53e2017-04-20 16:40:36 -06001636 } else if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
1637 blk_mq_put_ctx(data.ctx);
Christoph Hellwiga4d907b2017-03-22 15:01:53 -04001638 blk_mq_run_hw_queue(data.hctx, true);
Bart Van Asscheabc25a62017-04-21 12:00:40 -06001639 } else
1640 blk_mq_put_ctx(data.ctx);
Jens Axboe320ae512013-10-24 09:20:05 +01001641
Jens Axboe7b371632015-11-05 10:41:40 -07001642 return cookie;
Jens Axboe320ae512013-10-24 09:20:05 +01001643}
1644
Jens Axboecc71a6f2017-01-11 14:29:56 -07001645void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
1646 unsigned int hctx_idx)
Jens Axboe320ae512013-10-24 09:20:05 +01001647{
1648 struct page *page;
1649
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001650 if (tags->rqs && set->ops->exit_request) {
Christoph Hellwige9b267d2014-04-15 13:59:10 -06001651 int i;
1652
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001653 for (i = 0; i < tags->nr_tags; i++) {
Jens Axboe2af8cbe2017-01-13 14:39:30 -07001654 struct request *rq = tags->static_rqs[i];
1655
1656 if (!rq)
Christoph Hellwige9b267d2014-04-15 13:59:10 -06001657 continue;
Christoph Hellwigd6296d392017-05-01 10:19:08 -06001658 set->ops->exit_request(set, rq, hctx_idx);
Jens Axboe2af8cbe2017-01-13 14:39:30 -07001659 tags->static_rqs[i] = NULL;
Christoph Hellwige9b267d2014-04-15 13:59:10 -06001660 }
1661 }
1662
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001663 while (!list_empty(&tags->page_list)) {
1664 page = list_first_entry(&tags->page_list, struct page, lru);
Dave Hansen67534712014-01-08 20:17:46 -07001665 list_del_init(&page->lru);
Catalin Marinasf75782e2015-09-14 18:16:02 +01001666 /*
1667 * Remove kmemleak object previously allocated in
1668 * blk_mq_init_rq_map().
1669 */
1670 kmemleak_free(page_address(page));
Jens Axboe320ae512013-10-24 09:20:05 +01001671 __free_pages(page, page->private);
1672 }
Jens Axboecc71a6f2017-01-11 14:29:56 -07001673}
Jens Axboe320ae512013-10-24 09:20:05 +01001674
Jens Axboecc71a6f2017-01-11 14:29:56 -07001675void blk_mq_free_rq_map(struct blk_mq_tags *tags)
1676{
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001677 kfree(tags->rqs);
Jens Axboecc71a6f2017-01-11 14:29:56 -07001678 tags->rqs = NULL;
Jens Axboe2af8cbe2017-01-13 14:39:30 -07001679 kfree(tags->static_rqs);
1680 tags->static_rqs = NULL;
Jens Axboe320ae512013-10-24 09:20:05 +01001681
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001682 blk_mq_free_tags(tags);
Jens Axboe320ae512013-10-24 09:20:05 +01001683}
1684
Jens Axboecc71a6f2017-01-11 14:29:56 -07001685struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
1686 unsigned int hctx_idx,
1687 unsigned int nr_tags,
1688 unsigned int reserved_tags)
Jens Axboe320ae512013-10-24 09:20:05 +01001689{
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001690 struct blk_mq_tags *tags;
Shaohua Li59f082e2017-02-01 09:53:14 -08001691 int node;
Jens Axboe320ae512013-10-24 09:20:05 +01001692
Shaohua Li59f082e2017-02-01 09:53:14 -08001693 node = blk_mq_hw_queue_to_node(set->mq_map, hctx_idx);
1694 if (node == NUMA_NO_NODE)
1695 node = set->numa_node;
1696
1697 tags = blk_mq_init_tags(nr_tags, reserved_tags, node,
Shaohua Li24391c02015-01-23 14:18:00 -07001698 BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags));
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001699 if (!tags)
1700 return NULL;
Jens Axboe320ae512013-10-24 09:20:05 +01001701
Jens Axboecc71a6f2017-01-11 14:29:56 -07001702 tags->rqs = kzalloc_node(nr_tags * sizeof(struct request *),
Gabriel Krisman Bertazi36e1f3d12016-12-06 13:31:44 -02001703 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
Shaohua Li59f082e2017-02-01 09:53:14 -08001704 node);
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001705 if (!tags->rqs) {
1706 blk_mq_free_tags(tags);
1707 return NULL;
1708 }
Jens Axboe320ae512013-10-24 09:20:05 +01001709
Jens Axboe2af8cbe2017-01-13 14:39:30 -07001710 tags->static_rqs = kzalloc_node(nr_tags * sizeof(struct request *),
1711 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
Shaohua Li59f082e2017-02-01 09:53:14 -08001712 node);
Jens Axboe2af8cbe2017-01-13 14:39:30 -07001713 if (!tags->static_rqs) {
1714 kfree(tags->rqs);
1715 blk_mq_free_tags(tags);
1716 return NULL;
1717 }
1718
Jens Axboecc71a6f2017-01-11 14:29:56 -07001719 return tags;
1720}
1721
1722static size_t order_to_size(unsigned int order)
1723{
1724 return (size_t)PAGE_SIZE << order;
1725}
1726
1727int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
1728 unsigned int hctx_idx, unsigned int depth)
1729{
1730 unsigned int i, j, entries_per_page, max_order = 4;
1731 size_t rq_size, left;
Shaohua Li59f082e2017-02-01 09:53:14 -08001732 int node;
1733
1734 node = blk_mq_hw_queue_to_node(set->mq_map, hctx_idx);
1735 if (node == NUMA_NO_NODE)
1736 node = set->numa_node;
Jens Axboecc71a6f2017-01-11 14:29:56 -07001737
1738 INIT_LIST_HEAD(&tags->page_list);
1739
Jens Axboe320ae512013-10-24 09:20:05 +01001740 /*
1741 * rq_size is the size of the request plus driver payload, rounded
1742 * to the cacheline size
1743 */
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001744 rq_size = round_up(sizeof(struct request) + set->cmd_size,
Jens Axboe320ae512013-10-24 09:20:05 +01001745 cache_line_size());
Jens Axboecc71a6f2017-01-11 14:29:56 -07001746 left = rq_size * depth;
Jens Axboe320ae512013-10-24 09:20:05 +01001747
Jens Axboecc71a6f2017-01-11 14:29:56 -07001748 for (i = 0; i < depth; ) {
Jens Axboe320ae512013-10-24 09:20:05 +01001749 int this_order = max_order;
1750 struct page *page;
1751 int to_do;
1752 void *p;
1753
Bartlomiej Zolnierkiewiczb3a834b2016-05-16 09:54:47 -06001754 while (this_order && left < order_to_size(this_order - 1))
Jens Axboe320ae512013-10-24 09:20:05 +01001755 this_order--;
1756
1757 do {
Shaohua Li59f082e2017-02-01 09:53:14 -08001758 page = alloc_pages_node(node,
Gabriel Krisman Bertazi36e1f3d12016-12-06 13:31:44 -02001759 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO,
Jens Axboea5164402014-09-10 09:02:03 -06001760 this_order);
Jens Axboe320ae512013-10-24 09:20:05 +01001761 if (page)
1762 break;
1763 if (!this_order--)
1764 break;
1765 if (order_to_size(this_order) < rq_size)
1766 break;
1767 } while (1);
1768
1769 if (!page)
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001770 goto fail;
Jens Axboe320ae512013-10-24 09:20:05 +01001771
1772 page->private = this_order;
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001773 list_add_tail(&page->lru, &tags->page_list);
Jens Axboe320ae512013-10-24 09:20:05 +01001774
1775 p = page_address(page);
Catalin Marinasf75782e2015-09-14 18:16:02 +01001776 /*
1777 * Allow kmemleak to scan these pages as they contain pointers
1778 * to additional allocations like via ops->init_request().
1779 */
Gabriel Krisman Bertazi36e1f3d12016-12-06 13:31:44 -02001780 kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO);
Jens Axboe320ae512013-10-24 09:20:05 +01001781 entries_per_page = order_to_size(this_order) / rq_size;
Jens Axboecc71a6f2017-01-11 14:29:56 -07001782 to_do = min(entries_per_page, depth - i);
Jens Axboe320ae512013-10-24 09:20:05 +01001783 left -= to_do * rq_size;
1784 for (j = 0; j < to_do; j++) {
Jens Axboe2af8cbe2017-01-13 14:39:30 -07001785 struct request *rq = p;
1786
1787 tags->static_rqs[i] = rq;
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001788 if (set->ops->init_request) {
Christoph Hellwigd6296d392017-05-01 10:19:08 -06001789 if (set->ops->init_request(set, rq, hctx_idx,
Shaohua Li59f082e2017-02-01 09:53:14 -08001790 node)) {
Jens Axboe2af8cbe2017-01-13 14:39:30 -07001791 tags->static_rqs[i] = NULL;
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001792 goto fail;
Jens Axboea5164402014-09-10 09:02:03 -06001793 }
Christoph Hellwige9b267d2014-04-15 13:59:10 -06001794 }
1795
Jens Axboe320ae512013-10-24 09:20:05 +01001796 p += rq_size;
1797 i++;
1798 }
1799 }
Jens Axboecc71a6f2017-01-11 14:29:56 -07001800 return 0;
Jens Axboe320ae512013-10-24 09:20:05 +01001801
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001802fail:
Jens Axboecc71a6f2017-01-11 14:29:56 -07001803 blk_mq_free_rqs(set, tags, hctx_idx);
1804 return -ENOMEM;
Jens Axboe320ae512013-10-24 09:20:05 +01001805}
1806
Jens Axboee57690f2016-08-24 15:34:35 -06001807/*
1808 * 'cpu' is going away. splice any existing rq_list entries from this
1809 * software queue to the hw queue dispatch list, and ensure that it
1810 * gets run.
1811 */
Thomas Gleixner9467f852016-09-22 08:05:17 -06001812static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node)
Jens Axboe484b4062014-05-21 14:01:15 -06001813{
Thomas Gleixner9467f852016-09-22 08:05:17 -06001814 struct blk_mq_hw_ctx *hctx;
Jens Axboe484b4062014-05-21 14:01:15 -06001815 struct blk_mq_ctx *ctx;
1816 LIST_HEAD(tmp);
1817
Thomas Gleixner9467f852016-09-22 08:05:17 -06001818 hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead);
Jens Axboee57690f2016-08-24 15:34:35 -06001819 ctx = __blk_mq_get_ctx(hctx->queue, cpu);
Jens Axboe484b4062014-05-21 14:01:15 -06001820
1821 spin_lock(&ctx->lock);
1822 if (!list_empty(&ctx->rq_list)) {
1823 list_splice_init(&ctx->rq_list, &tmp);
1824 blk_mq_hctx_clear_pending(hctx, ctx);
1825 }
1826 spin_unlock(&ctx->lock);
1827
1828 if (list_empty(&tmp))
Thomas Gleixner9467f852016-09-22 08:05:17 -06001829 return 0;
Jens Axboe484b4062014-05-21 14:01:15 -06001830
Jens Axboee57690f2016-08-24 15:34:35 -06001831 spin_lock(&hctx->lock);
1832 list_splice_tail_init(&tmp, &hctx->dispatch);
1833 spin_unlock(&hctx->lock);
Jens Axboe484b4062014-05-21 14:01:15 -06001834
1835 blk_mq_run_hw_queue(hctx, true);
Thomas Gleixner9467f852016-09-22 08:05:17 -06001836 return 0;
Jens Axboe484b4062014-05-21 14:01:15 -06001837}
1838
Thomas Gleixner9467f852016-09-22 08:05:17 -06001839static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)
Jens Axboe484b4062014-05-21 14:01:15 -06001840{
Thomas Gleixner9467f852016-09-22 08:05:17 -06001841 cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD,
1842 &hctx->cpuhp_dead);
Jens Axboe484b4062014-05-21 14:01:15 -06001843}
1844
Ming Leic3b4afc2015-06-04 22:25:04 +08001845/* hctx->ctxs will be freed in queue's release handler */
Ming Lei08e98fc2014-09-25 23:23:38 +08001846static void blk_mq_exit_hctx(struct request_queue *q,
1847 struct blk_mq_tag_set *set,
1848 struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
1849{
1850 blk_mq_tag_idle(hctx);
1851
Ming Leif70ced02014-09-25 23:23:47 +08001852 if (set->ops->exit_request)
Christoph Hellwigd6296d392017-05-01 10:19:08 -06001853 set->ops->exit_request(set, hctx->fq->flush_rq, hctx_idx);
Ming Leif70ced02014-09-25 23:23:47 +08001854
Omar Sandoval93252632017-04-05 12:01:31 -07001855 blk_mq_sched_exit_hctx(q, hctx, hctx_idx);
1856
Ming Lei08e98fc2014-09-25 23:23:38 +08001857 if (set->ops->exit_hctx)
1858 set->ops->exit_hctx(hctx, hctx_idx);
1859
Bart Van Assche6a83e742016-11-02 10:09:51 -06001860 if (hctx->flags & BLK_MQ_F_BLOCKING)
1861 cleanup_srcu_struct(&hctx->queue_rq_srcu);
1862
Thomas Gleixner9467f852016-09-22 08:05:17 -06001863 blk_mq_remove_cpuhp(hctx);
Ming Leif70ced02014-09-25 23:23:47 +08001864 blk_free_flush_queue(hctx->fq);
Omar Sandoval88459642016-09-17 08:38:44 -06001865 sbitmap_free(&hctx->ctx_map);
Ming Lei08e98fc2014-09-25 23:23:38 +08001866}
1867
Ming Lei624dbe42014-05-27 23:35:13 +08001868static void blk_mq_exit_hw_queues(struct request_queue *q,
1869 struct blk_mq_tag_set *set, int nr_queue)
1870{
1871 struct blk_mq_hw_ctx *hctx;
1872 unsigned int i;
1873
1874 queue_for_each_hw_ctx(q, hctx, i) {
1875 if (i == nr_queue)
1876 break;
Ming Lei08e98fc2014-09-25 23:23:38 +08001877 blk_mq_exit_hctx(q, set, hctx, i);
Ming Lei624dbe42014-05-27 23:35:13 +08001878 }
Ming Lei624dbe42014-05-27 23:35:13 +08001879}
1880
Ming Lei08e98fc2014-09-25 23:23:38 +08001881static int blk_mq_init_hctx(struct request_queue *q,
1882 struct blk_mq_tag_set *set,
1883 struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)
1884{
1885 int node;
1886
1887 node = hctx->numa_node;
1888 if (node == NUMA_NO_NODE)
1889 node = hctx->numa_node = set->numa_node;
1890
Jens Axboe9f993732017-04-10 09:54:54 -06001891 INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
Ming Lei08e98fc2014-09-25 23:23:38 +08001892 spin_lock_init(&hctx->lock);
1893 INIT_LIST_HEAD(&hctx->dispatch);
1894 hctx->queue = q;
1895 hctx->queue_num = hctx_idx;
Jeff Moyer2404e602015-11-03 10:40:06 -05001896 hctx->flags = set->flags & ~BLK_MQ_F_TAG_SHARED;
Ming Lei08e98fc2014-09-25 23:23:38 +08001897
Thomas Gleixner9467f852016-09-22 08:05:17 -06001898 cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead);
Ming Lei08e98fc2014-09-25 23:23:38 +08001899
1900 hctx->tags = set->tags[hctx_idx];
1901
1902 /*
1903 * Allocate space for all possible cpus to avoid allocation at
1904 * runtime
1905 */
1906 hctx->ctxs = kmalloc_node(nr_cpu_ids * sizeof(void *),
1907 GFP_KERNEL, node);
1908 if (!hctx->ctxs)
1909 goto unregister_cpu_notifier;
1910
Omar Sandoval88459642016-09-17 08:38:44 -06001911 if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8), GFP_KERNEL,
1912 node))
Ming Lei08e98fc2014-09-25 23:23:38 +08001913 goto free_ctxs;
1914
1915 hctx->nr_ctx = 0;
1916
1917 if (set->ops->init_hctx &&
1918 set->ops->init_hctx(hctx, set->driver_data, hctx_idx))
1919 goto free_bitmap;
1920
Omar Sandoval93252632017-04-05 12:01:31 -07001921 if (blk_mq_sched_init_hctx(q, hctx, hctx_idx))
1922 goto exit_hctx;
1923
Ming Leif70ced02014-09-25 23:23:47 +08001924 hctx->fq = blk_alloc_flush_queue(q, hctx->numa_node, set->cmd_size);
1925 if (!hctx->fq)
Omar Sandoval93252632017-04-05 12:01:31 -07001926 goto sched_exit_hctx;
Ming Leif70ced02014-09-25 23:23:47 +08001927
1928 if (set->ops->init_request &&
Christoph Hellwigd6296d392017-05-01 10:19:08 -06001929 set->ops->init_request(set, hctx->fq->flush_rq, hctx_idx,
1930 node))
Ming Leif70ced02014-09-25 23:23:47 +08001931 goto free_fq;
1932
Bart Van Assche6a83e742016-11-02 10:09:51 -06001933 if (hctx->flags & BLK_MQ_F_BLOCKING)
1934 init_srcu_struct(&hctx->queue_rq_srcu);
1935
Ming Lei08e98fc2014-09-25 23:23:38 +08001936 return 0;
1937
Ming Leif70ced02014-09-25 23:23:47 +08001938 free_fq:
1939 kfree(hctx->fq);
Omar Sandoval93252632017-04-05 12:01:31 -07001940 sched_exit_hctx:
1941 blk_mq_sched_exit_hctx(q, hctx, hctx_idx);
Ming Leif70ced02014-09-25 23:23:47 +08001942 exit_hctx:
1943 if (set->ops->exit_hctx)
1944 set->ops->exit_hctx(hctx, hctx_idx);
Ming Lei08e98fc2014-09-25 23:23:38 +08001945 free_bitmap:
Omar Sandoval88459642016-09-17 08:38:44 -06001946 sbitmap_free(&hctx->ctx_map);
Ming Lei08e98fc2014-09-25 23:23:38 +08001947 free_ctxs:
1948 kfree(hctx->ctxs);
1949 unregister_cpu_notifier:
Thomas Gleixner9467f852016-09-22 08:05:17 -06001950 blk_mq_remove_cpuhp(hctx);
Ming Lei08e98fc2014-09-25 23:23:38 +08001951 return -1;
1952}
1953
Jens Axboe320ae512013-10-24 09:20:05 +01001954static void blk_mq_init_cpu_queues(struct request_queue *q,
1955 unsigned int nr_hw_queues)
1956{
1957 unsigned int i;
1958
1959 for_each_possible_cpu(i) {
1960 struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
1961 struct blk_mq_hw_ctx *hctx;
1962
Jens Axboe320ae512013-10-24 09:20:05 +01001963 __ctx->cpu = i;
1964 spin_lock_init(&__ctx->lock);
1965 INIT_LIST_HEAD(&__ctx->rq_list);
1966 __ctx->queue = q;
1967
1968 /* If the cpu isn't online, the cpu is mapped to first hctx */
Jens Axboe320ae512013-10-24 09:20:05 +01001969 if (!cpu_online(i))
1970 continue;
1971
Christoph Hellwig7d7e0f92016-09-14 16:18:54 +02001972 hctx = blk_mq_map_queue(q, i);
Jens Axboee4043dc2014-04-09 10:18:23 -06001973
Jens Axboe320ae512013-10-24 09:20:05 +01001974 /*
1975 * Set local node, IFF we have more than one hw queue. If
1976 * not, we remain on the home node of the device
1977 */
1978 if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
Raghavendra K Tbffed452015-12-02 16:59:05 +05301979 hctx->numa_node = local_memory_node(cpu_to_node(i));
Jens Axboe320ae512013-10-24 09:20:05 +01001980 }
1981}
1982
Jens Axboecc71a6f2017-01-11 14:29:56 -07001983static bool __blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, int hctx_idx)
1984{
1985 int ret = 0;
1986
1987 set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx,
1988 set->queue_depth, set->reserved_tags);
1989 if (!set->tags[hctx_idx])
1990 return false;
1991
1992 ret = blk_mq_alloc_rqs(set, set->tags[hctx_idx], hctx_idx,
1993 set->queue_depth);
1994 if (!ret)
1995 return true;
1996
1997 blk_mq_free_rq_map(set->tags[hctx_idx]);
1998 set->tags[hctx_idx] = NULL;
1999 return false;
2000}
2001
2002static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set,
2003 unsigned int hctx_idx)
2004{
Jens Axboebd166ef2017-01-17 06:03:22 -07002005 if (set->tags[hctx_idx]) {
2006 blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx);
2007 blk_mq_free_rq_map(set->tags[hctx_idx]);
2008 set->tags[hctx_idx] = NULL;
2009 }
Jens Axboecc71a6f2017-01-11 14:29:56 -07002010}
2011
Akinobu Mita57783222015-09-27 02:09:23 +09002012static void blk_mq_map_swqueue(struct request_queue *q,
2013 const struct cpumask *online_mask)
Jens Axboe320ae512013-10-24 09:20:05 +01002014{
Gabriel Krisman Bertazid1b1cea2016-12-14 18:48:36 -02002015 unsigned int i, hctx_idx;
Jens Axboe320ae512013-10-24 09:20:05 +01002016 struct blk_mq_hw_ctx *hctx;
2017 struct blk_mq_ctx *ctx;
Ming Lei2a34c082015-04-21 10:00:20 +08002018 struct blk_mq_tag_set *set = q->tag_set;
Jens Axboe320ae512013-10-24 09:20:05 +01002019
Akinobu Mita60de0742015-09-27 02:09:25 +09002020 /*
2021 * Avoid others reading imcomplete hctx->cpumask through sysfs
2022 */
2023 mutex_lock(&q->sysfs_lock);
2024
Jens Axboe320ae512013-10-24 09:20:05 +01002025 queue_for_each_hw_ctx(q, hctx, i) {
Jens Axboee4043dc2014-04-09 10:18:23 -06002026 cpumask_clear(hctx->cpumask);
Jens Axboe320ae512013-10-24 09:20:05 +01002027 hctx->nr_ctx = 0;
2028 }
2029
2030 /*
2031 * Map software to hardware queues
2032 */
Thomas Gleixner897bb0c2016-03-19 11:30:33 +01002033 for_each_possible_cpu(i) {
Jens Axboe320ae512013-10-24 09:20:05 +01002034 /* If the cpu isn't online, the cpu is mapped to first hctx */
Akinobu Mita57783222015-09-27 02:09:23 +09002035 if (!cpumask_test_cpu(i, online_mask))
Jens Axboee4043dc2014-04-09 10:18:23 -06002036 continue;
2037
Gabriel Krisman Bertazid1b1cea2016-12-14 18:48:36 -02002038 hctx_idx = q->mq_map[i];
2039 /* unmapped hw queue can be remapped after CPU topo changed */
Jens Axboecc71a6f2017-01-11 14:29:56 -07002040 if (!set->tags[hctx_idx] &&
2041 !__blk_mq_alloc_rq_map(set, hctx_idx)) {
Gabriel Krisman Bertazid1b1cea2016-12-14 18:48:36 -02002042 /*
2043 * If tags initialization fail for some hctx,
2044 * that hctx won't be brought online. In this
2045 * case, remap the current ctx to hctx[0] which
2046 * is guaranteed to always have tags allocated
2047 */
Jens Axboecc71a6f2017-01-11 14:29:56 -07002048 q->mq_map[i] = 0;
Gabriel Krisman Bertazid1b1cea2016-12-14 18:48:36 -02002049 }
2050
Thomas Gleixner897bb0c2016-03-19 11:30:33 +01002051 ctx = per_cpu_ptr(q->queue_ctx, i);
Christoph Hellwig7d7e0f92016-09-14 16:18:54 +02002052 hctx = blk_mq_map_queue(q, i);
Keith Busch868f2f02015-12-17 17:08:14 -07002053
Jens Axboee4043dc2014-04-09 10:18:23 -06002054 cpumask_set_cpu(i, hctx->cpumask);
Jens Axboe320ae512013-10-24 09:20:05 +01002055 ctx->index_hw = hctx->nr_ctx;
2056 hctx->ctxs[hctx->nr_ctx++] = ctx;
2057 }
Jens Axboe506e9312014-05-07 10:26:44 -06002058
Akinobu Mita60de0742015-09-27 02:09:25 +09002059 mutex_unlock(&q->sysfs_lock);
2060
Jens Axboe506e9312014-05-07 10:26:44 -06002061 queue_for_each_hw_ctx(q, hctx, i) {
Jens Axboe484b4062014-05-21 14:01:15 -06002062 /*
Jens Axboea68aafa2014-08-15 13:19:15 -06002063 * If no software queues are mapped to this hardware queue,
2064 * disable it and free the request entries.
Jens Axboe484b4062014-05-21 14:01:15 -06002065 */
2066 if (!hctx->nr_ctx) {
Gabriel Krisman Bertazid1b1cea2016-12-14 18:48:36 -02002067 /* Never unmap queue 0. We need it as a
2068 * fallback in case of a new remap fails
2069 * allocation
2070 */
Jens Axboecc71a6f2017-01-11 14:29:56 -07002071 if (i && set->tags[i])
2072 blk_mq_free_map_and_requests(set, i);
2073
Ming Lei2a34c082015-04-21 10:00:20 +08002074 hctx->tags = NULL;
Jens Axboe484b4062014-05-21 14:01:15 -06002075 continue;
2076 }
2077
Ming Lei2a34c082015-04-21 10:00:20 +08002078 hctx->tags = set->tags[i];
2079 WARN_ON(!hctx->tags);
2080
Jens Axboe484b4062014-05-21 14:01:15 -06002081 /*
Chong Yuan889fa312015-04-15 11:39:29 -06002082 * Set the map size to the number of mapped software queues.
2083 * This is more accurate and more efficient than looping
2084 * over all possibly mapped software queues.
2085 */
Omar Sandoval88459642016-09-17 08:38:44 -06002086 sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx);
Chong Yuan889fa312015-04-15 11:39:29 -06002087
2088 /*
Jens Axboe484b4062014-05-21 14:01:15 -06002089 * Initialize batch roundrobin counts
2090 */
Jens Axboe506e9312014-05-07 10:26:44 -06002091 hctx->next_cpu = cpumask_first(hctx->cpumask);
2092 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
2093 }
Jens Axboe320ae512013-10-24 09:20:05 +01002094}
2095
Jeff Moyer2404e602015-11-03 10:40:06 -05002096static void queue_set_hctx_shared(struct request_queue *q, bool shared)
Jens Axboe0d2602c2014-05-13 15:10:52 -06002097{
2098 struct blk_mq_hw_ctx *hctx;
Jens Axboe0d2602c2014-05-13 15:10:52 -06002099 int i;
2100
Jeff Moyer2404e602015-11-03 10:40:06 -05002101 queue_for_each_hw_ctx(q, hctx, i) {
2102 if (shared)
2103 hctx->flags |= BLK_MQ_F_TAG_SHARED;
2104 else
2105 hctx->flags &= ~BLK_MQ_F_TAG_SHARED;
2106 }
2107}
2108
2109static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set, bool shared)
2110{
2111 struct request_queue *q;
Jens Axboe0d2602c2014-05-13 15:10:52 -06002112
Bart Van Assche705cda92017-04-07 11:16:49 -07002113 lockdep_assert_held(&set->tag_list_lock);
2114
Jens Axboe0d2602c2014-05-13 15:10:52 -06002115 list_for_each_entry(q, &set->tag_list, tag_set_list) {
2116 blk_mq_freeze_queue(q);
Jeff Moyer2404e602015-11-03 10:40:06 -05002117 queue_set_hctx_shared(q, shared);
Jens Axboe0d2602c2014-05-13 15:10:52 -06002118 blk_mq_unfreeze_queue(q);
2119 }
2120}
2121
2122static void blk_mq_del_queue_tag_set(struct request_queue *q)
2123{
2124 struct blk_mq_tag_set *set = q->tag_set;
2125
Jens Axboe0d2602c2014-05-13 15:10:52 -06002126 mutex_lock(&set->tag_list_lock);
Bart Van Assche705cda92017-04-07 11:16:49 -07002127 list_del_rcu(&q->tag_set_list);
2128 INIT_LIST_HEAD(&q->tag_set_list);
Jeff Moyer2404e602015-11-03 10:40:06 -05002129 if (list_is_singular(&set->tag_list)) {
2130 /* just transitioned to unshared */
2131 set->flags &= ~BLK_MQ_F_TAG_SHARED;
2132 /* update existing queue */
2133 blk_mq_update_tag_set_depth(set, false);
2134 }
Jens Axboe0d2602c2014-05-13 15:10:52 -06002135 mutex_unlock(&set->tag_list_lock);
Bart Van Assche705cda92017-04-07 11:16:49 -07002136
2137 synchronize_rcu();
Jens Axboe0d2602c2014-05-13 15:10:52 -06002138}
2139
2140static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
2141 struct request_queue *q)
2142{
2143 q->tag_set = set;
2144
2145 mutex_lock(&set->tag_list_lock);
Jeff Moyer2404e602015-11-03 10:40:06 -05002146
2147 /* Check to see if we're transitioning to shared (from 1 to 2 queues). */
2148 if (!list_empty(&set->tag_list) && !(set->flags & BLK_MQ_F_TAG_SHARED)) {
2149 set->flags |= BLK_MQ_F_TAG_SHARED;
2150 /* update existing queue */
2151 blk_mq_update_tag_set_depth(set, true);
2152 }
2153 if (set->flags & BLK_MQ_F_TAG_SHARED)
2154 queue_set_hctx_shared(q, true);
Bart Van Assche705cda92017-04-07 11:16:49 -07002155 list_add_tail_rcu(&q->tag_set_list, &set->tag_list);
Jeff Moyer2404e602015-11-03 10:40:06 -05002156
Jens Axboe0d2602c2014-05-13 15:10:52 -06002157 mutex_unlock(&set->tag_list_lock);
2158}
2159
Ming Leie09aae72015-01-29 20:17:27 +08002160/*
2161 * It is the actual release handler for mq, but we do it from
2162 * request queue's release handler for avoiding use-after-free
2163 * and headache because q->mq_kobj shouldn't have been introduced,
2164 * but we can't group ctx/kctx kobj without it.
2165 */
2166void blk_mq_release(struct request_queue *q)
2167{
2168 struct blk_mq_hw_ctx *hctx;
2169 unsigned int i;
2170
2171 /* hctx kobj stays in hctx */
Ming Leic3b4afc2015-06-04 22:25:04 +08002172 queue_for_each_hw_ctx(q, hctx, i) {
2173 if (!hctx)
2174 continue;
Ming Lei6c8b2322017-02-22 18:14:01 +08002175 kobject_put(&hctx->kobj);
Ming Leic3b4afc2015-06-04 22:25:04 +08002176 }
Ming Leie09aae72015-01-29 20:17:27 +08002177
Akinobu Mitaa723bab2015-09-27 02:09:21 +09002178 q->mq_map = NULL;
2179
Ming Leie09aae72015-01-29 20:17:27 +08002180 kfree(q->queue_hw_ctx);
2181
Ming Lei7ea5fe32017-02-22 18:14:00 +08002182 /*
2183 * release .mq_kobj and sw queue's kobject now because
2184 * both share lifetime with request queue.
2185 */
2186 blk_mq_sysfs_deinit(q);
2187
Ming Leie09aae72015-01-29 20:17:27 +08002188 free_percpu(q->queue_ctx);
2189}
2190
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002191struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
Jens Axboe320ae512013-10-24 09:20:05 +01002192{
Mike Snitzerb62c21b2015-03-12 23:56:02 -04002193 struct request_queue *uninit_q, *q;
2194
2195 uninit_q = blk_alloc_queue_node(GFP_KERNEL, set->numa_node);
2196 if (!uninit_q)
2197 return ERR_PTR(-ENOMEM);
2198
2199 q = blk_mq_init_allocated_queue(set, uninit_q);
2200 if (IS_ERR(q))
2201 blk_cleanup_queue(uninit_q);
2202
2203 return q;
2204}
2205EXPORT_SYMBOL(blk_mq_init_queue);
2206
Keith Busch868f2f02015-12-17 17:08:14 -07002207static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
2208 struct request_queue *q)
Mike Snitzerb62c21b2015-03-12 23:56:02 -04002209{
Keith Busch868f2f02015-12-17 17:08:14 -07002210 int i, j;
2211 struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx;
Jens Axboe320ae512013-10-24 09:20:05 +01002212
Keith Busch868f2f02015-12-17 17:08:14 -07002213 blk_mq_sysfs_unregister(q);
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002214 for (i = 0; i < set->nr_hw_queues; i++) {
Keith Busch868f2f02015-12-17 17:08:14 -07002215 int node;
Jens Axboef14bbe72014-05-27 12:06:53 -06002216
Keith Busch868f2f02015-12-17 17:08:14 -07002217 if (hctxs[i])
2218 continue;
2219
2220 node = blk_mq_hw_queue_to_node(q->mq_map, i);
Christoph Hellwigcdef54d2014-05-28 18:11:06 +02002221 hctxs[i] = kzalloc_node(sizeof(struct blk_mq_hw_ctx),
2222 GFP_KERNEL, node);
Jens Axboe320ae512013-10-24 09:20:05 +01002223 if (!hctxs[i])
Keith Busch868f2f02015-12-17 17:08:14 -07002224 break;
Jens Axboe320ae512013-10-24 09:20:05 +01002225
Jens Axboea86073e2014-10-13 15:41:54 -06002226 if (!zalloc_cpumask_var_node(&hctxs[i]->cpumask, GFP_KERNEL,
Keith Busch868f2f02015-12-17 17:08:14 -07002227 node)) {
2228 kfree(hctxs[i]);
2229 hctxs[i] = NULL;
2230 break;
2231 }
Jens Axboee4043dc2014-04-09 10:18:23 -06002232
Jens Axboe0d2602c2014-05-13 15:10:52 -06002233 atomic_set(&hctxs[i]->nr_active, 0);
Jens Axboef14bbe72014-05-27 12:06:53 -06002234 hctxs[i]->numa_node = node;
Jens Axboe320ae512013-10-24 09:20:05 +01002235 hctxs[i]->queue_num = i;
Keith Busch868f2f02015-12-17 17:08:14 -07002236
2237 if (blk_mq_init_hctx(q, set, hctxs[i], i)) {
2238 free_cpumask_var(hctxs[i]->cpumask);
2239 kfree(hctxs[i]);
2240 hctxs[i] = NULL;
2241 break;
2242 }
2243 blk_mq_hctx_kobj_init(hctxs[i]);
Jens Axboe320ae512013-10-24 09:20:05 +01002244 }
Keith Busch868f2f02015-12-17 17:08:14 -07002245 for (j = i; j < q->nr_hw_queues; j++) {
2246 struct blk_mq_hw_ctx *hctx = hctxs[j];
2247
2248 if (hctx) {
Jens Axboecc71a6f2017-01-11 14:29:56 -07002249 if (hctx->tags)
2250 blk_mq_free_map_and_requests(set, j);
Keith Busch868f2f02015-12-17 17:08:14 -07002251 blk_mq_exit_hctx(q, set, hctx, j);
Keith Busch868f2f02015-12-17 17:08:14 -07002252 kobject_put(&hctx->kobj);
Keith Busch868f2f02015-12-17 17:08:14 -07002253 hctxs[j] = NULL;
2254
2255 }
2256 }
2257 q->nr_hw_queues = i;
2258 blk_mq_sysfs_register(q);
2259}
2260
2261struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
2262 struct request_queue *q)
2263{
Ming Lei66841672016-02-12 15:27:00 +08002264 /* mark the queue as mq asap */
2265 q->mq_ops = set->ops;
2266
Omar Sandoval34dbad52017-03-21 08:56:08 -07002267 q->poll_cb = blk_stat_alloc_callback(blk_mq_poll_stats_fn,
Stephen Bates720b8cc2017-04-07 06:24:03 -06002268 blk_mq_poll_stats_bkt,
2269 BLK_MQ_POLL_STATS_BKTS, q);
Omar Sandoval34dbad52017-03-21 08:56:08 -07002270 if (!q->poll_cb)
2271 goto err_exit;
2272
Keith Busch868f2f02015-12-17 17:08:14 -07002273 q->queue_ctx = alloc_percpu(struct blk_mq_ctx);
2274 if (!q->queue_ctx)
Ming Linc7de5722016-05-25 23:23:27 -07002275 goto err_exit;
Keith Busch868f2f02015-12-17 17:08:14 -07002276
Ming Lei737f98c2017-02-22 18:13:59 +08002277 /* init q->mq_kobj and sw queues' kobjects */
2278 blk_mq_sysfs_init(q);
2279
Keith Busch868f2f02015-12-17 17:08:14 -07002280 q->queue_hw_ctx = kzalloc_node(nr_cpu_ids * sizeof(*(q->queue_hw_ctx)),
2281 GFP_KERNEL, set->numa_node);
2282 if (!q->queue_hw_ctx)
2283 goto err_percpu;
2284
Christoph Hellwigbdd17e72016-09-14 16:18:53 +02002285 q->mq_map = set->mq_map;
Keith Busch868f2f02015-12-17 17:08:14 -07002286
2287 blk_mq_realloc_hw_ctxs(set, q);
2288 if (!q->nr_hw_queues)
2289 goto err_hctxs;
Jens Axboe320ae512013-10-24 09:20:05 +01002290
Christoph Hellwig287922e2015-10-30 20:57:30 +08002291 INIT_WORK(&q->timeout_work, blk_mq_timeout_work);
Ming Leie56f6982015-07-16 19:53:22 +08002292 blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ);
Jens Axboe320ae512013-10-24 09:20:05 +01002293
2294 q->nr_queues = nr_cpu_ids;
Jens Axboe320ae512013-10-24 09:20:05 +01002295
Jens Axboe94eddfb2013-11-19 09:25:07 -07002296 q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
Jens Axboe320ae512013-10-24 09:20:05 +01002297
Jens Axboe05f1dd52014-05-29 09:53:32 -06002298 if (!(set->flags & BLK_MQ_F_SG_MERGE))
2299 q->queue_flags |= 1 << QUEUE_FLAG_NO_SG_MERGE;
2300
Christoph Hellwig1be036e2014-02-07 10:22:39 -08002301 q->sg_reserved_size = INT_MAX;
2302
Mike Snitzer28494502016-09-14 13:28:30 -04002303 INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work);
Christoph Hellwig6fca6a62014-05-28 08:08:02 -06002304 INIT_LIST_HEAD(&q->requeue_list);
2305 spin_lock_init(&q->requeue_lock);
2306
Christoph Hellwig254d2592017-03-22 15:01:50 -04002307 blk_queue_make_request(q, blk_mq_make_request);
Jens Axboe07068d52014-05-22 10:40:51 -06002308
Jens Axboeeba71762014-05-20 15:17:27 -06002309 /*
2310 * Do this after blk_queue_make_request() overrides it...
2311 */
2312 q->nr_requests = set->queue_depth;
2313
Jens Axboe64f1c212016-11-14 13:03:03 -07002314 /*
2315 * Default to classic polling
2316 */
2317 q->poll_nsec = -1;
2318
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002319 if (set->ops->complete)
2320 blk_queue_softirq_done(q, set->ops->complete);
Christoph Hellwig30a91cb2014-02-10 03:24:38 -08002321
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002322 blk_mq_init_cpu_queues(q, set->nr_hw_queues);
Jens Axboe320ae512013-10-24 09:20:05 +01002323
Akinobu Mita57783222015-09-27 02:09:23 +09002324 get_online_cpus();
Jens Axboe320ae512013-10-24 09:20:05 +01002325 mutex_lock(&all_q_mutex);
Akinobu Mita4593fdb2015-09-27 02:09:20 +09002326
Jens Axboe320ae512013-10-24 09:20:05 +01002327 list_add_tail(&q->all_q_node, &all_q_list);
Jens Axboe0d2602c2014-05-13 15:10:52 -06002328 blk_mq_add_queue_tag_set(set, q);
Akinobu Mita57783222015-09-27 02:09:23 +09002329 blk_mq_map_swqueue(q, cpu_online_mask);
Jens Axboe484b4062014-05-21 14:01:15 -06002330
Akinobu Mita4593fdb2015-09-27 02:09:20 +09002331 mutex_unlock(&all_q_mutex);
Akinobu Mita57783222015-09-27 02:09:23 +09002332 put_online_cpus();
Akinobu Mita4593fdb2015-09-27 02:09:20 +09002333
Jens Axboed3484992017-01-13 14:43:58 -07002334 if (!(set->flags & BLK_MQ_F_NO_SCHED)) {
2335 int ret;
2336
2337 ret = blk_mq_sched_init(q);
2338 if (ret)
2339 return ERR_PTR(ret);
2340 }
2341
Jens Axboe320ae512013-10-24 09:20:05 +01002342 return q;
Christoph Hellwig18741982014-02-10 09:29:00 -07002343
Jens Axboe320ae512013-10-24 09:20:05 +01002344err_hctxs:
Keith Busch868f2f02015-12-17 17:08:14 -07002345 kfree(q->queue_hw_ctx);
Jens Axboe320ae512013-10-24 09:20:05 +01002346err_percpu:
Keith Busch868f2f02015-12-17 17:08:14 -07002347 free_percpu(q->queue_ctx);
Ming Linc7de5722016-05-25 23:23:27 -07002348err_exit:
2349 q->mq_ops = NULL;
Jens Axboe320ae512013-10-24 09:20:05 +01002350 return ERR_PTR(-ENOMEM);
2351}
Mike Snitzerb62c21b2015-03-12 23:56:02 -04002352EXPORT_SYMBOL(blk_mq_init_allocated_queue);
Jens Axboe320ae512013-10-24 09:20:05 +01002353
2354void blk_mq_free_queue(struct request_queue *q)
2355{
Ming Lei624dbe42014-05-27 23:35:13 +08002356 struct blk_mq_tag_set *set = q->tag_set;
Jens Axboe320ae512013-10-24 09:20:05 +01002357
Akinobu Mita0e626362015-09-27 02:09:22 +09002358 mutex_lock(&all_q_mutex);
2359 list_del_init(&q->all_q_node);
2360 mutex_unlock(&all_q_mutex);
2361
Jens Axboe0d2602c2014-05-13 15:10:52 -06002362 blk_mq_del_queue_tag_set(q);
2363
Ming Lei624dbe42014-05-27 23:35:13 +08002364 blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
Jens Axboe320ae512013-10-24 09:20:05 +01002365}
Jens Axboe320ae512013-10-24 09:20:05 +01002366
2367/* Basically redo blk_mq_init_queue with queue frozen */
Akinobu Mita57783222015-09-27 02:09:23 +09002368static void blk_mq_queue_reinit(struct request_queue *q,
2369 const struct cpumask *online_mask)
Jens Axboe320ae512013-10-24 09:20:05 +01002370{
Christoph Hellwig4ecd4fe2015-05-07 09:38:13 +02002371 WARN_ON_ONCE(!atomic_read(&q->mq_freeze_depth));
Jens Axboe320ae512013-10-24 09:20:05 +01002372
Jens Axboe67aec142014-05-30 08:25:36 -06002373 blk_mq_sysfs_unregister(q);
2374
Jens Axboe320ae512013-10-24 09:20:05 +01002375 /*
2376 * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe
2377 * we should change hctx numa_node according to new topology (this
2378 * involves free and re-allocate memory, worthy doing?)
2379 */
2380
Akinobu Mita57783222015-09-27 02:09:23 +09002381 blk_mq_map_swqueue(q, online_mask);
Jens Axboe320ae512013-10-24 09:20:05 +01002382
Jens Axboe67aec142014-05-30 08:25:36 -06002383 blk_mq_sysfs_register(q);
Jens Axboe320ae512013-10-24 09:20:05 +01002384}
2385
Sebastian Andrzej Siewior65d52912016-09-22 08:05:19 -06002386/*
2387 * New online cpumask which is going to be set in this hotplug event.
2388 * Declare this cpumasks as global as cpu-hotplug operation is invoked
2389 * one-by-one and dynamically allocating this could result in a failure.
2390 */
2391static struct cpumask cpuhp_online_new;
2392
2393static void blk_mq_queue_reinit_work(void)
Jens Axboe320ae512013-10-24 09:20:05 +01002394{
2395 struct request_queue *q;
Jens Axboe320ae512013-10-24 09:20:05 +01002396
2397 mutex_lock(&all_q_mutex);
Tejun Heof3af0202014-11-04 13:52:27 -05002398 /*
2399 * We need to freeze and reinit all existing queues. Freezing
2400 * involves synchronous wait for an RCU grace period and doing it
2401 * one by one may take a long time. Start freezing all queues in
2402 * one swoop and then wait for the completions so that freezing can
2403 * take place in parallel.
2404 */
2405 list_for_each_entry(q, &all_q_list, all_q_node)
Ming Lei1671d522017-03-27 20:06:57 +08002406 blk_freeze_queue_start(q);
Gabriel Krisman Bertazi415d3da2016-11-28 15:01:48 -02002407 list_for_each_entry(q, &all_q_list, all_q_node)
Tejun Heof3af0202014-11-04 13:52:27 -05002408 blk_mq_freeze_queue_wait(q);
2409
Jens Axboe320ae512013-10-24 09:20:05 +01002410 list_for_each_entry(q, &all_q_list, all_q_node)
Sebastian Andrzej Siewior65d52912016-09-22 08:05:19 -06002411 blk_mq_queue_reinit(q, &cpuhp_online_new);
Tejun Heof3af0202014-11-04 13:52:27 -05002412
2413 list_for_each_entry(q, &all_q_list, all_q_node)
2414 blk_mq_unfreeze_queue(q);
2415
Jens Axboe320ae512013-10-24 09:20:05 +01002416 mutex_unlock(&all_q_mutex);
Sebastian Andrzej Siewior65d52912016-09-22 08:05:19 -06002417}
2418
2419static int blk_mq_queue_reinit_dead(unsigned int cpu)
2420{
Sebastian Andrzej Siewior97a32862016-09-23 15:02:38 +02002421 cpumask_copy(&cpuhp_online_new, cpu_online_mask);
Sebastian Andrzej Siewior65d52912016-09-22 08:05:19 -06002422 blk_mq_queue_reinit_work();
2423 return 0;
2424}
2425
2426/*
2427 * Before hotadded cpu starts handling requests, new mappings must be
2428 * established. Otherwise, these requests in hw queue might never be
2429 * dispatched.
2430 *
2431 * For example, there is a single hw queue (hctx) and two CPU queues (ctx0
2432 * for CPU0, and ctx1 for CPU1).
2433 *
2434 * Now CPU1 is just onlined and a request is inserted into ctx1->rq_list
2435 * and set bit0 in pending bitmap as ctx1->index_hw is still zero.
2436 *
Jens Axboe2c3ad662016-12-14 14:34:47 -07002437 * And then while running hw queue, blk_mq_flush_busy_ctxs() finds bit0 is set
2438 * in pending bitmap and tries to retrieve requests in hctx->ctxs[0]->rq_list.
2439 * But htx->ctxs[0] is a pointer to ctx0, so the request in ctx1->rq_list is
2440 * ignored.
Sebastian Andrzej Siewior65d52912016-09-22 08:05:19 -06002441 */
2442static int blk_mq_queue_reinit_prepare(unsigned int cpu)
2443{
2444 cpumask_copy(&cpuhp_online_new, cpu_online_mask);
2445 cpumask_set_cpu(cpu, &cpuhp_online_new);
2446 blk_mq_queue_reinit_work();
2447 return 0;
Jens Axboe320ae512013-10-24 09:20:05 +01002448}
2449
Jens Axboea5164402014-09-10 09:02:03 -06002450static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
2451{
2452 int i;
2453
Jens Axboecc71a6f2017-01-11 14:29:56 -07002454 for (i = 0; i < set->nr_hw_queues; i++)
2455 if (!__blk_mq_alloc_rq_map(set, i))
Jens Axboea5164402014-09-10 09:02:03 -06002456 goto out_unwind;
Jens Axboea5164402014-09-10 09:02:03 -06002457
2458 return 0;
2459
2460out_unwind:
2461 while (--i >= 0)
Jens Axboecc71a6f2017-01-11 14:29:56 -07002462 blk_mq_free_rq_map(set->tags[i]);
Jens Axboea5164402014-09-10 09:02:03 -06002463
Jens Axboea5164402014-09-10 09:02:03 -06002464 return -ENOMEM;
2465}
2466
2467/*
2468 * Allocate the request maps associated with this tag_set. Note that this
2469 * may reduce the depth asked for, if memory is tight. set->queue_depth
2470 * will be updated to reflect the allocated depth.
2471 */
2472static int blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
2473{
2474 unsigned int depth;
2475 int err;
2476
2477 depth = set->queue_depth;
2478 do {
2479 err = __blk_mq_alloc_rq_maps(set);
2480 if (!err)
2481 break;
2482
2483 set->queue_depth >>= 1;
2484 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) {
2485 err = -ENOMEM;
2486 break;
2487 }
2488 } while (set->queue_depth);
2489
2490 if (!set->queue_depth || err) {
2491 pr_err("blk-mq: failed to allocate request map\n");
2492 return -ENOMEM;
2493 }
2494
2495 if (depth != set->queue_depth)
2496 pr_info("blk-mq: reduced tag depth (%u -> %u)\n",
2497 depth, set->queue_depth);
2498
2499 return 0;
2500}
2501
Omar Sandovalebe8bdd2017-04-07 08:53:11 -06002502static int blk_mq_update_queue_map(struct blk_mq_tag_set *set)
2503{
2504 if (set->ops->map_queues)
2505 return set->ops->map_queues(set);
2506 else
2507 return blk_mq_map_queues(set);
2508}
2509
Jens Axboea4391c62014-06-05 15:21:56 -06002510/*
2511 * Alloc a tag set to be associated with one or more request queues.
2512 * May fail with EINVAL for various error conditions. May adjust the
2513 * requested depth down, if if it too large. In that case, the set
2514 * value will be stored in set->queue_depth.
2515 */
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002516int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
2517{
Christoph Hellwigda695ba2016-09-14 16:18:55 +02002518 int ret;
2519
Bart Van Assche205fb5f2014-10-30 14:45:11 +01002520 BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS);
2521
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002522 if (!set->nr_hw_queues)
2523 return -EINVAL;
Jens Axboea4391c62014-06-05 15:21:56 -06002524 if (!set->queue_depth)
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002525 return -EINVAL;
2526 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
2527 return -EINVAL;
2528
Christoph Hellwig7d7e0f92016-09-14 16:18:54 +02002529 if (!set->ops->queue_rq)
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002530 return -EINVAL;
2531
Jens Axboea4391c62014-06-05 15:21:56 -06002532 if (set->queue_depth > BLK_MQ_MAX_DEPTH) {
2533 pr_info("blk-mq: reduced tag depth to %u\n",
2534 BLK_MQ_MAX_DEPTH);
2535 set->queue_depth = BLK_MQ_MAX_DEPTH;
2536 }
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002537
Shaohua Li6637fad2014-11-30 16:00:58 -08002538 /*
2539 * If a crashdump is active, then we are potentially in a very
2540 * memory constrained environment. Limit us to 1 queue and
2541 * 64 tags to prevent using too much memory.
2542 */
2543 if (is_kdump_kernel()) {
2544 set->nr_hw_queues = 1;
2545 set->queue_depth = min(64U, set->queue_depth);
2546 }
Keith Busch868f2f02015-12-17 17:08:14 -07002547 /*
2548 * There is no use for more h/w queues than cpus.
2549 */
2550 if (set->nr_hw_queues > nr_cpu_ids)
2551 set->nr_hw_queues = nr_cpu_ids;
Shaohua Li6637fad2014-11-30 16:00:58 -08002552
Keith Busch868f2f02015-12-17 17:08:14 -07002553 set->tags = kzalloc_node(nr_cpu_ids * sizeof(struct blk_mq_tags *),
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002554 GFP_KERNEL, set->numa_node);
2555 if (!set->tags)
Jens Axboea5164402014-09-10 09:02:03 -06002556 return -ENOMEM;
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002557
Christoph Hellwigda695ba2016-09-14 16:18:55 +02002558 ret = -ENOMEM;
2559 set->mq_map = kzalloc_node(sizeof(*set->mq_map) * nr_cpu_ids,
2560 GFP_KERNEL, set->numa_node);
Christoph Hellwigbdd17e72016-09-14 16:18:53 +02002561 if (!set->mq_map)
2562 goto out_free_tags;
2563
Omar Sandovalebe8bdd2017-04-07 08:53:11 -06002564 ret = blk_mq_update_queue_map(set);
Christoph Hellwigda695ba2016-09-14 16:18:55 +02002565 if (ret)
2566 goto out_free_mq_map;
2567
2568 ret = blk_mq_alloc_rq_maps(set);
2569 if (ret)
Christoph Hellwigbdd17e72016-09-14 16:18:53 +02002570 goto out_free_mq_map;
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002571
Jens Axboe0d2602c2014-05-13 15:10:52 -06002572 mutex_init(&set->tag_list_lock);
2573 INIT_LIST_HEAD(&set->tag_list);
2574
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002575 return 0;
Christoph Hellwigbdd17e72016-09-14 16:18:53 +02002576
2577out_free_mq_map:
2578 kfree(set->mq_map);
2579 set->mq_map = NULL;
2580out_free_tags:
Robert Elliott5676e7b2014-09-02 11:38:44 -05002581 kfree(set->tags);
2582 set->tags = NULL;
Christoph Hellwigda695ba2016-09-14 16:18:55 +02002583 return ret;
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002584}
2585EXPORT_SYMBOL(blk_mq_alloc_tag_set);
2586
2587void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
2588{
2589 int i;
2590
Jens Axboecc71a6f2017-01-11 14:29:56 -07002591 for (i = 0; i < nr_cpu_ids; i++)
2592 blk_mq_free_map_and_requests(set, i);
Jens Axboe484b4062014-05-21 14:01:15 -06002593
Christoph Hellwigbdd17e72016-09-14 16:18:53 +02002594 kfree(set->mq_map);
2595 set->mq_map = NULL;
2596
Ming Lei981bd182014-04-24 00:07:34 +08002597 kfree(set->tags);
Robert Elliott5676e7b2014-09-02 11:38:44 -05002598 set->tags = NULL;
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002599}
2600EXPORT_SYMBOL(blk_mq_free_tag_set);
2601
Jens Axboee3a2b3f2014-05-20 11:49:02 -06002602int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
2603{
2604 struct blk_mq_tag_set *set = q->tag_set;
2605 struct blk_mq_hw_ctx *hctx;
2606 int i, ret;
2607
Jens Axboebd166ef2017-01-17 06:03:22 -07002608 if (!set)
Jens Axboee3a2b3f2014-05-20 11:49:02 -06002609 return -EINVAL;
2610
Jens Axboe70f36b62017-01-19 10:59:07 -07002611 blk_mq_freeze_queue(q);
2612 blk_mq_quiesce_queue(q);
2613
Jens Axboee3a2b3f2014-05-20 11:49:02 -06002614 ret = 0;
2615 queue_for_each_hw_ctx(q, hctx, i) {
Keith Busche9137d42016-02-18 14:56:35 -07002616 if (!hctx->tags)
2617 continue;
Jens Axboebd166ef2017-01-17 06:03:22 -07002618 /*
2619 * If we're using an MQ scheduler, just update the scheduler
2620 * queue depth. This is similar to what the old code would do.
2621 */
Jens Axboe70f36b62017-01-19 10:59:07 -07002622 if (!hctx->sched_tags) {
2623 ret = blk_mq_tag_update_depth(hctx, &hctx->tags,
2624 min(nr, set->queue_depth),
2625 false);
2626 } else {
2627 ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags,
2628 nr, true);
2629 }
Jens Axboee3a2b3f2014-05-20 11:49:02 -06002630 if (ret)
2631 break;
2632 }
2633
2634 if (!ret)
2635 q->nr_requests = nr;
2636
Jens Axboe70f36b62017-01-19 10:59:07 -07002637 blk_mq_unfreeze_queue(q);
2638 blk_mq_start_stopped_hw_queues(q, true);
2639
Jens Axboee3a2b3f2014-05-20 11:49:02 -06002640 return ret;
2641}
2642
Keith Busch868f2f02015-12-17 17:08:14 -07002643void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues)
2644{
2645 struct request_queue *q;
2646
Bart Van Assche705cda92017-04-07 11:16:49 -07002647 lockdep_assert_held(&set->tag_list_lock);
2648
Keith Busch868f2f02015-12-17 17:08:14 -07002649 if (nr_hw_queues > nr_cpu_ids)
2650 nr_hw_queues = nr_cpu_ids;
2651 if (nr_hw_queues < 1 || nr_hw_queues == set->nr_hw_queues)
2652 return;
2653
2654 list_for_each_entry(q, &set->tag_list, tag_set_list)
2655 blk_mq_freeze_queue(q);
2656
2657 set->nr_hw_queues = nr_hw_queues;
Omar Sandovalebe8bdd2017-04-07 08:53:11 -06002658 blk_mq_update_queue_map(set);
Keith Busch868f2f02015-12-17 17:08:14 -07002659 list_for_each_entry(q, &set->tag_list, tag_set_list) {
2660 blk_mq_realloc_hw_ctxs(set, q);
Keith Busch868f2f02015-12-17 17:08:14 -07002661 blk_mq_queue_reinit(q, cpu_online_mask);
2662 }
2663
2664 list_for_each_entry(q, &set->tag_list, tag_set_list)
2665 blk_mq_unfreeze_queue(q);
2666}
2667EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues);
2668
Omar Sandoval34dbad52017-03-21 08:56:08 -07002669/* Enable polling stats and return whether they were already enabled. */
2670static bool blk_poll_stats_enable(struct request_queue *q)
2671{
2672 if (test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) ||
2673 test_and_set_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags))
2674 return true;
2675 blk_stat_add_callback(q, q->poll_cb);
2676 return false;
2677}
2678
2679static void blk_mq_poll_stats_start(struct request_queue *q)
2680{
2681 /*
2682 * We don't arm the callback if polling stats are not enabled or the
2683 * callback is already active.
2684 */
2685 if (!test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) ||
2686 blk_stat_is_active(q->poll_cb))
2687 return;
2688
2689 blk_stat_activate_msecs(q->poll_cb, 100);
2690}
2691
2692static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb)
2693{
2694 struct request_queue *q = cb->data;
Stephen Bates720b8cc2017-04-07 06:24:03 -06002695 int bucket;
Omar Sandoval34dbad52017-03-21 08:56:08 -07002696
Stephen Bates720b8cc2017-04-07 06:24:03 -06002697 for (bucket = 0; bucket < BLK_MQ_POLL_STATS_BKTS; bucket++) {
2698 if (cb->stat[bucket].nr_samples)
2699 q->poll_stat[bucket] = cb->stat[bucket];
2700 }
Omar Sandoval34dbad52017-03-21 08:56:08 -07002701}
2702
Jens Axboe64f1c212016-11-14 13:03:03 -07002703static unsigned long blk_mq_poll_nsecs(struct request_queue *q,
2704 struct blk_mq_hw_ctx *hctx,
2705 struct request *rq)
2706{
Jens Axboe64f1c212016-11-14 13:03:03 -07002707 unsigned long ret = 0;
Stephen Bates720b8cc2017-04-07 06:24:03 -06002708 int bucket;
Jens Axboe64f1c212016-11-14 13:03:03 -07002709
2710 /*
2711 * If stats collection isn't on, don't sleep but turn it on for
2712 * future users
2713 */
Omar Sandoval34dbad52017-03-21 08:56:08 -07002714 if (!blk_poll_stats_enable(q))
Jens Axboe64f1c212016-11-14 13:03:03 -07002715 return 0;
2716
2717 /*
Jens Axboe64f1c212016-11-14 13:03:03 -07002718 * As an optimistic guess, use half of the mean service time
2719 * for this type of request. We can (and should) make this smarter.
2720 * For instance, if the completion latencies are tight, we can
2721 * get closer than just half the mean. This is especially
2722 * important on devices where the completion latencies are longer
Stephen Bates720b8cc2017-04-07 06:24:03 -06002723 * than ~10 usec. We do use the stats for the relevant IO size
2724 * if available which does lead to better estimates.
Jens Axboe64f1c212016-11-14 13:03:03 -07002725 */
Stephen Bates720b8cc2017-04-07 06:24:03 -06002726 bucket = blk_mq_poll_stats_bkt(rq);
2727 if (bucket < 0)
2728 return ret;
2729
2730 if (q->poll_stat[bucket].nr_samples)
2731 ret = (q->poll_stat[bucket].mean + 1) / 2;
Jens Axboe64f1c212016-11-14 13:03:03 -07002732
2733 return ret;
2734}
2735
Jens Axboe06426ad2016-11-14 13:01:59 -07002736static bool blk_mq_poll_hybrid_sleep(struct request_queue *q,
Jens Axboe64f1c212016-11-14 13:03:03 -07002737 struct blk_mq_hw_ctx *hctx,
Jens Axboe06426ad2016-11-14 13:01:59 -07002738 struct request *rq)
2739{
2740 struct hrtimer_sleeper hs;
2741 enum hrtimer_mode mode;
Jens Axboe64f1c212016-11-14 13:03:03 -07002742 unsigned int nsecs;
Jens Axboe06426ad2016-11-14 13:01:59 -07002743 ktime_t kt;
2744
Jens Axboe64f1c212016-11-14 13:03:03 -07002745 if (test_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags))
2746 return false;
2747
2748 /*
2749 * poll_nsec can be:
2750 *
2751 * -1: don't ever hybrid sleep
2752 * 0: use half of prev avg
2753 * >0: use this specific value
2754 */
2755 if (q->poll_nsec == -1)
2756 return false;
2757 else if (q->poll_nsec > 0)
2758 nsecs = q->poll_nsec;
2759 else
2760 nsecs = blk_mq_poll_nsecs(q, hctx, rq);
2761
2762 if (!nsecs)
Jens Axboe06426ad2016-11-14 13:01:59 -07002763 return false;
2764
2765 set_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);
2766
2767 /*
2768 * This will be replaced with the stats tracking code, using
2769 * 'avg_completion_time / 2' as the pre-sleep target.
2770 */
Thomas Gleixner8b0e1952016-12-25 12:30:41 +01002771 kt = nsecs;
Jens Axboe06426ad2016-11-14 13:01:59 -07002772
2773 mode = HRTIMER_MODE_REL;
2774 hrtimer_init_on_stack(&hs.timer, CLOCK_MONOTONIC, mode);
2775 hrtimer_set_expires(&hs.timer, kt);
2776
2777 hrtimer_init_sleeper(&hs, current);
2778 do {
2779 if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
2780 break;
2781 set_current_state(TASK_UNINTERRUPTIBLE);
2782 hrtimer_start_expires(&hs.timer, mode);
2783 if (hs.task)
2784 io_schedule();
2785 hrtimer_cancel(&hs.timer);
2786 mode = HRTIMER_MODE_ABS;
2787 } while (hs.task && !signal_pending(current));
2788
2789 __set_current_state(TASK_RUNNING);
2790 destroy_hrtimer_on_stack(&hs.timer);
2791 return true;
2792}
2793
Jens Axboebbd7bb72016-11-04 09:34:34 -06002794static bool __blk_mq_poll(struct blk_mq_hw_ctx *hctx, struct request *rq)
2795{
2796 struct request_queue *q = hctx->queue;
2797 long state;
2798
Jens Axboe06426ad2016-11-14 13:01:59 -07002799 /*
2800 * If we sleep, have the caller restart the poll loop to reset
2801 * the state. Like for the other success return cases, the
2802 * caller is responsible for checking if the IO completed. If
2803 * the IO isn't complete, we'll get called again and will go
2804 * straight to the busy poll loop.
2805 */
Jens Axboe64f1c212016-11-14 13:03:03 -07002806 if (blk_mq_poll_hybrid_sleep(q, hctx, rq))
Jens Axboe06426ad2016-11-14 13:01:59 -07002807 return true;
2808
Jens Axboebbd7bb72016-11-04 09:34:34 -06002809 hctx->poll_considered++;
2810
2811 state = current->state;
2812 while (!need_resched()) {
2813 int ret;
2814
2815 hctx->poll_invoked++;
2816
2817 ret = q->mq_ops->poll(hctx, rq->tag);
2818 if (ret > 0) {
2819 hctx->poll_success++;
2820 set_current_state(TASK_RUNNING);
2821 return true;
2822 }
2823
2824 if (signal_pending_state(state, current))
2825 set_current_state(TASK_RUNNING);
2826
2827 if (current->state == TASK_RUNNING)
2828 return true;
2829 if (ret < 0)
2830 break;
2831 cpu_relax();
2832 }
2833
2834 return false;
2835}
2836
2837bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie)
2838{
2839 struct blk_mq_hw_ctx *hctx;
2840 struct blk_plug *plug;
2841 struct request *rq;
2842
2843 if (!q->mq_ops || !q->mq_ops->poll || !blk_qc_t_valid(cookie) ||
2844 !test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
2845 return false;
2846
2847 plug = current->plug;
2848 if (plug)
2849 blk_flush_plug_list(plug, false);
2850
2851 hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)];
Jens Axboebd166ef2017-01-17 06:03:22 -07002852 if (!blk_qc_t_is_internal(cookie))
2853 rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie));
Jens Axboe3a07bb12017-04-20 14:53:28 -06002854 else {
Jens Axboebd166ef2017-01-17 06:03:22 -07002855 rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie));
Jens Axboe3a07bb12017-04-20 14:53:28 -06002856 /*
2857 * With scheduling, if the request has completed, we'll
2858 * get a NULL return here, as we clear the sched tag when
2859 * that happens. The request still remains valid, like always,
2860 * so we should be safe with just the NULL check.
2861 */
2862 if (!rq)
2863 return false;
2864 }
Jens Axboebbd7bb72016-11-04 09:34:34 -06002865
2866 return __blk_mq_poll(hctx, rq);
2867}
2868EXPORT_SYMBOL_GPL(blk_mq_poll);
2869
Jens Axboe676141e2014-03-20 13:29:18 -06002870void blk_mq_disable_hotplug(void)
2871{
2872 mutex_lock(&all_q_mutex);
2873}
2874
2875void blk_mq_enable_hotplug(void)
2876{
2877 mutex_unlock(&all_q_mutex);
2878}
2879
Jens Axboe320ae512013-10-24 09:20:05 +01002880static int __init blk_mq_init(void)
2881{
Thomas Gleixner9467f852016-09-22 08:05:17 -06002882 cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL,
2883 blk_mq_hctx_notify_dead);
Jens Axboe320ae512013-10-24 09:20:05 +01002884
Sebastian Andrzej Siewior65d52912016-09-22 08:05:19 -06002885 cpuhp_setup_state_nocalls(CPUHP_BLK_MQ_PREPARE, "block/mq:prepare",
2886 blk_mq_queue_reinit_prepare,
2887 blk_mq_queue_reinit_dead);
Jens Axboe320ae512013-10-24 09:20:05 +01002888 return 0;
2889}
2890subsys_initcall(blk_mq_init);