blob: 663aabfeba183ae87a346b75a5af24f4a233bb47 [file] [log] [blame]
Christoph Hellwig3dcf60b2019-04-30 14:42:43 -04001// SPDX-License-Identifier: GPL-2.0
Vivek Goyal31e4c282009-12-03 12:59:42 -05002/*
3 * Common Block IO controller cgroup interface
4 *
5 * Based on ideas and code from CFQ, CFS and BFQ:
6 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
7 *
8 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
9 * Paolo Valente <paolo.valente@unimore.it>
10 *
11 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
12 * Nauman Rafique <nauman@google.com>
Arianna Avanzinie48453c2015-06-05 23:38:42 +020013 *
14 * For policy-specific per-blkcg data:
15 * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it>
16 * Arianna Avanzini <avanzini.arianna@gmail.com>
Vivek Goyal31e4c282009-12-03 12:59:42 -050017 */
18#include <linux/ioprio.h>
Vivek Goyal22084192009-12-03 12:59:49 -050019#include <linux/kdev_t.h>
Vivek Goyal9d6a9862009-12-04 10:36:41 -050020#include <linux/module.h>
Ingo Molnar174cd4b2017-02-02 19:15:33 +010021#include <linux/sched/signal.h>
Stephen Rothwellaccee782009-12-07 19:29:39 +110022#include <linux/err.h>
Divyesh Shah91952912010-04-01 15:01:41 -070023#include <linux/blkdev.h>
Tejun Heo52ebea72015-05-22 17:13:37 -040024#include <linux/backing-dev.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090025#include <linux/slab.h>
Gui Jianfeng34d0f172010-04-13 16:05:49 +080026#include <linux/genhd.h>
Tejun Heo72e06c22012-03-05 13:15:00 -080027#include <linux/delay.h>
Tejun Heo9a9e8a22012-03-19 15:10:56 -070028#include <linux/atomic.h>
Tejun Heo36aa9e52015-08-18 14:55:31 -070029#include <linux/ctype.h>
Tejun Heoeea8f412015-05-22 17:13:17 -040030#include <linux/blk-cgroup.h>
Josef Bacikd09d8df2018-07-03 11:14:55 -040031#include <linux/tracehook.h>
Josef Bacikfd112c72019-07-09 14:41:29 -070032#include <linux/psi.h>
Tejun Heo5efd6112012-03-05 13:15:12 -080033#include "blk.h"
Bart Van Assche556910e2021-06-17 17:44:44 -070034#include "blk-ioprio.h"
Jens Axboea7b36ee2021-10-05 09:11:56 -060035#include "blk-throttle.h"
Vivek Goyal3e252062009-12-04 10:36:42 -050036
Tejun Heo838f13b2015-07-09 16:39:47 -040037/*
38 * blkcg_pol_mutex protects blkcg_policy[] and policy [de]activation.
39 * blkcg_pol_register_mutex nests outside of it and synchronizes entire
40 * policy [un]register operations including cgroup file additions /
41 * removals. Putting cgroup file registration outside blkcg_pol_mutex
42 * allows grabbing it from cgroup callbacks.
43 */
44static DEFINE_MUTEX(blkcg_pol_register_mutex);
Tejun Heobc0d6502012-04-13 13:11:26 -070045static DEFINE_MUTEX(blkcg_pol_mutex);
Tejun Heo923adde2012-03-05 13:15:13 -080046
Arianna Avanzinie48453c2015-06-05 23:38:42 +020047struct blkcg blkcg_root;
Tejun Heo3c798392012-04-16 13:57:25 -070048EXPORT_SYMBOL_GPL(blkcg_root);
Vivek Goyal9d6a9862009-12-04 10:36:41 -050049
Tejun Heo496d5e72015-05-22 17:13:21 -040050struct cgroup_subsys_state * const blkcg_root_css = &blkcg_root.css;
Tejun Heo9b0eb692019-06-27 13:39:48 -070051EXPORT_SYMBOL_GPL(blkcg_root_css);
Tejun Heo496d5e72015-05-22 17:13:21 -040052
Tejun Heo3c798392012-04-16 13:57:25 -070053static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];
Tejun Heo035d10b2012-03-05 13:15:04 -080054
Tejun Heo7876f932015-07-09 16:39:49 -040055static LIST_HEAD(all_blkcgs); /* protected by blkcg_pol_mutex */
56
Tejun Heo07b0fde2019-07-16 07:58:31 -070057bool blkcg_debug_stats = false;
Tejun Heod3f77df2019-06-27 13:39:52 -070058static struct workqueue_struct *blkcg_punt_bio_wq;
Josef Bacik903d23f2018-07-03 11:14:52 -040059
Yu Kuaia7317632021-07-07 09:56:49 +080060#define BLKG_DESTROY_BATCH_SIZE 64
61
Tejun Heoa2b16932012-04-13 13:11:33 -070062static bool blkcg_policy_enabled(struct request_queue *q,
Tejun Heo3c798392012-04-16 13:57:25 -070063 const struct blkcg_policy *pol)
Tejun Heoa2b16932012-04-13 13:11:33 -070064{
65 return pol && test_bit(pol->plid, q->blkcg_pols);
66}
67
Tejun Heo03814112012-03-05 13:15:14 -080068/**
69 * blkg_free - free a blkg
70 * @blkg: blkg to free
71 *
72 * Free @blkg which may be partially allocated.
73 */
Tejun Heo3c798392012-04-16 13:57:25 -070074static void blkg_free(struct blkcg_gq *blkg)
Tejun Heo03814112012-03-05 13:15:14 -080075{
Tejun Heoe8989fa2012-03-05 13:15:20 -080076 int i;
Tejun Heo549d3aa2012-03-05 13:15:16 -080077
78 if (!blkg)
79 return;
80
Tejun Heodb613672013-05-14 13:52:31 -070081 for (i = 0; i < BLKCG_MAX_POLS; i++)
Tejun Heo001bea72015-08-18 14:55:11 -070082 if (blkg->pd[i])
83 blkcg_policy[i]->pd_free_fn(blkg->pd[i]);
Tejun Heoe8989fa2012-03-05 13:15:20 -080084
Tejun Heof7331642019-11-07 11:18:03 -080085 free_percpu(blkg->iostat_cpu);
Tejun Heoef069b92019-06-13 15:30:39 -070086 percpu_ref_exit(&blkg->refcnt);
Tejun Heo549d3aa2012-03-05 13:15:16 -080087 kfree(blkg);
Tejun Heo03814112012-03-05 13:15:14 -080088}
89
Dennis Zhou7fcf2b02018-12-05 12:10:38 -050090static void __blkg_release(struct rcu_head *rcu)
91{
92 struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head);
93
Tejun Heod3f77df2019-06-27 13:39:52 -070094 WARN_ON(!bio_list_empty(&blkg->async_bios));
95
Dennis Zhou7fcf2b02018-12-05 12:10:38 -050096 /* release the blkcg and parent blkg refs this blkg has been holding */
97 css_put(&blkg->blkcg->css);
98 if (blkg->parent)
99 blkg_put(blkg->parent);
Dennis Zhou7fcf2b02018-12-05 12:10:38 -0500100 blkg_free(blkg);
101}
102
103/*
104 * A group is RCU protected, but having an rcu lock does not mean that one
105 * can access all the fields of blkg and assume these are valid. For
106 * example, don't try to follow throtl_data and request queue links.
107 *
108 * Having a reference to blkg under an rcu allows accesses to only values
109 * local to groups like group stats and group rate limits.
110 */
111static void blkg_release(struct percpu_ref *ref)
112{
113 struct blkcg_gq *blkg = container_of(ref, struct blkcg_gq, refcnt);
114
115 call_rcu(&blkg->rcu_head, __blkg_release);
116}
117
Tejun Heod3f77df2019-06-27 13:39:52 -0700118static void blkg_async_bio_workfn(struct work_struct *work)
119{
120 struct blkcg_gq *blkg = container_of(work, struct blkcg_gq,
121 async_bio_work);
122 struct bio_list bios = BIO_EMPTY_LIST;
123 struct bio *bio;
Xianting Tian192f1c62020-09-10 14:15:06 +0800124 struct blk_plug plug;
125 bool need_plug = false;
Tejun Heod3f77df2019-06-27 13:39:52 -0700126
127 /* as long as there are pending bios, @blkg can't go away */
128 spin_lock_bh(&blkg->async_bio_lock);
129 bio_list_merge(&bios, &blkg->async_bios);
130 bio_list_init(&blkg->async_bios);
131 spin_unlock_bh(&blkg->async_bio_lock);
132
Xianting Tian192f1c62020-09-10 14:15:06 +0800133 /* start plug only when bio_list contains at least 2 bios */
134 if (bios.head && bios.head->bi_next) {
135 need_plug = true;
136 blk_start_plug(&plug);
137 }
Tejun Heod3f77df2019-06-27 13:39:52 -0700138 while ((bio = bio_list_pop(&bios)))
139 submit_bio(bio);
Xianting Tian192f1c62020-09-10 14:15:06 +0800140 if (need_plug)
141 blk_finish_plug(&plug);
Tejun Heod3f77df2019-06-27 13:39:52 -0700142}
143
Tejun Heo03814112012-03-05 13:15:14 -0800144/**
145 * blkg_alloc - allocate a blkg
146 * @blkcg: block cgroup the new blkg is associated with
147 * @q: request_queue the new blkg is associated with
Tejun Heo15974992012-06-04 20:40:52 -0700148 * @gfp_mask: allocation mask to use
Tejun Heo03814112012-03-05 13:15:14 -0800149 *
Tejun Heoe8989fa2012-03-05 13:15:20 -0800150 * Allocate a new blkg assocating @blkcg and @q.
Tejun Heo03814112012-03-05 13:15:14 -0800151 */
Tejun Heo15974992012-06-04 20:40:52 -0700152static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q,
153 gfp_t gfp_mask)
Tejun Heo03814112012-03-05 13:15:14 -0800154{
Tejun Heo3c798392012-04-16 13:57:25 -0700155 struct blkcg_gq *blkg;
Tejun Heof7331642019-11-07 11:18:03 -0800156 int i, cpu;
Tejun Heo03814112012-03-05 13:15:14 -0800157
158 /* alloc and init base part */
Tejun Heo15974992012-06-04 20:40:52 -0700159 blkg = kzalloc_node(sizeof(*blkg), gfp_mask, q->node);
Tejun Heo03814112012-03-05 13:15:14 -0800160 if (!blkg)
161 return NULL;
162
Tejun Heoef069b92019-06-13 15:30:39 -0700163 if (percpu_ref_init(&blkg->refcnt, blkg_release, 0, gfp_mask))
164 goto err_free;
165
Tejun Heof7331642019-11-07 11:18:03 -0800166 blkg->iostat_cpu = alloc_percpu_gfp(struct blkg_iostat_set, gfp_mask);
167 if (!blkg->iostat_cpu)
Tejun Heo77ea7332015-08-18 14:55:24 -0700168 goto err_free;
169
Tejun Heoc875f4d2012-03-05 13:15:22 -0800170 blkg->q = q;
Tejun Heoe8989fa2012-03-05 13:15:20 -0800171 INIT_LIST_HEAD(&blkg->q_node);
Tejun Heod3f77df2019-06-27 13:39:52 -0700172 spin_lock_init(&blkg->async_bio_lock);
173 bio_list_init(&blkg->async_bios);
174 INIT_WORK(&blkg->async_bio_work, blkg_async_bio_workfn);
Tejun Heo03814112012-03-05 13:15:14 -0800175 blkg->blkcg = blkcg;
Tejun Heo03814112012-03-05 13:15:14 -0800176
Tejun Heof7331642019-11-07 11:18:03 -0800177 u64_stats_init(&blkg->iostat.sync);
178 for_each_possible_cpu(cpu)
179 u64_stats_init(&per_cpu_ptr(blkg->iostat_cpu, cpu)->sync);
180
Tejun Heo8bd435b2012-04-13 13:11:28 -0700181 for (i = 0; i < BLKCG_MAX_POLS; i++) {
Tejun Heo3c798392012-04-16 13:57:25 -0700182 struct blkcg_policy *pol = blkcg_policy[i];
Tejun Heoe8989fa2012-03-05 13:15:20 -0800183 struct blkg_policy_data *pd;
Tejun Heo03814112012-03-05 13:15:14 -0800184
Tejun Heoa2b16932012-04-13 13:11:33 -0700185 if (!blkcg_policy_enabled(q, pol))
Tejun Heoe8989fa2012-03-05 13:15:20 -0800186 continue;
Tejun Heo549d3aa2012-03-05 13:15:16 -0800187
Tejun Heoe8989fa2012-03-05 13:15:20 -0800188 /* alloc per-policy data and attach it to blkg */
Tejun Heocf09a8e2019-08-28 15:05:51 -0700189 pd = pol->pd_alloc_fn(gfp_mask, q, blkcg);
Tejun Heoa0516612012-06-26 15:05:44 -0700190 if (!pd)
191 goto err_free;
Tejun Heo549d3aa2012-03-05 13:15:16 -0800192
Tejun Heoe8989fa2012-03-05 13:15:20 -0800193 blkg->pd[i] = pd;
194 pd->blkg = blkg;
Tejun Heob276a872013-01-09 08:05:12 -0800195 pd->plid = i;
Tejun Heoe8989fa2012-03-05 13:15:20 -0800196 }
197
Tejun Heo03814112012-03-05 13:15:14 -0800198 return blkg;
Tejun Heoa0516612012-06-26 15:05:44 -0700199
200err_free:
201 blkg_free(blkg);
202 return NULL;
Tejun Heo03814112012-03-05 13:15:14 -0800203}
204
Tejun Heo24f29042015-08-18 14:55:17 -0700205struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg,
206 struct request_queue *q, bool update_hint)
Tejun Heo80fd9972012-04-13 14:50:53 -0700207{
Tejun Heo3c798392012-04-16 13:57:25 -0700208 struct blkcg_gq *blkg;
Tejun Heo80fd9972012-04-13 14:50:53 -0700209
Tejun Heoa6371202012-04-19 16:29:24 -0700210 /*
Tejun Heo86cde6b2013-01-09 08:05:10 -0800211 * Hint didn't match. Look up from the radix tree. Note that the
212 * hint can only be updated under queue_lock as otherwise @blkg
213 * could have already been removed from blkg_tree. The caller is
214 * responsible for grabbing queue_lock if @update_hint.
Tejun Heoa6371202012-04-19 16:29:24 -0700215 */
216 blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id);
Tejun Heo86cde6b2013-01-09 08:05:10 -0800217 if (blkg && blkg->q == q) {
218 if (update_hint) {
Christoph Hellwig0d945c12018-11-15 12:17:28 -0700219 lockdep_assert_held(&q->queue_lock);
Tejun Heo86cde6b2013-01-09 08:05:10 -0800220 rcu_assign_pointer(blkcg->blkg_hint, blkg);
221 }
Tejun Heoa6371202012-04-19 16:29:24 -0700222 return blkg;
Tejun Heo86cde6b2013-01-09 08:05:10 -0800223 }
Tejun Heoa6371202012-04-19 16:29:24 -0700224
Tejun Heo80fd9972012-04-13 14:50:53 -0700225 return NULL;
226}
Tejun Heoae118892015-08-18 14:55:20 -0700227EXPORT_SYMBOL_GPL(blkg_lookup_slowpath);
Tejun Heo80fd9972012-04-13 14:50:53 -0700228
Tejun Heo15974992012-06-04 20:40:52 -0700229/*
Jens Axboed708f0d2017-03-29 11:25:48 -0600230 * If @new_blkg is %NULL, this function tries to allocate a new one as
231 * necessary using %GFP_NOWAIT. @new_blkg is always consumed on return.
Tejun Heo15974992012-06-04 20:40:52 -0700232 */
Tejun Heo86cde6b2013-01-09 08:05:10 -0800233static struct blkcg_gq *blkg_create(struct blkcg *blkcg,
Jens Axboed708f0d2017-03-29 11:25:48 -0600234 struct request_queue *q,
235 struct blkcg_gq *new_blkg)
Vivek Goyal5624a4e2011-05-19 15:38:28 -0400236{
Jens Axboed708f0d2017-03-29 11:25:48 -0600237 struct blkcg_gq *blkg;
Tejun Heof427d902013-01-09 08:05:12 -0800238 int i, ret;
Vivek Goyal5624a4e2011-05-19 15:38:28 -0400239
Tejun Heocd1604f2012-03-05 13:15:06 -0800240 WARN_ON_ONCE(!rcu_read_lock_held());
Christoph Hellwig0d945c12018-11-15 12:17:28 -0700241 lockdep_assert_held(&q->queue_lock);
Vivek Goyal31e4c282009-12-03 12:59:42 -0500242
Dennis Zhou0273ac32018-12-11 18:03:08 -0500243 /* request_queue is dying, do not create/recreate a blkg */
244 if (blk_queue_dying(q)) {
245 ret = -ENODEV;
246 goto err_free_blkg;
247 }
248
Tejun Heo7ee9c562012-03-05 13:15:11 -0800249 /* blkg holds a reference to blkcg */
Tejun Heoec903c02014-05-13 12:11:01 -0400250 if (!css_tryget_online(&blkcg->css)) {
Tejun Heo20386ce2015-08-18 14:55:28 -0700251 ret = -ENODEV;
Tejun Heo93e6d5d2013-01-09 08:05:10 -0800252 goto err_free_blkg;
Tejun Heo15974992012-06-04 20:40:52 -0700253 }
Tejun Heocd1604f2012-03-05 13:15:06 -0800254
Jens Axboed708f0d2017-03-29 11:25:48 -0600255 /* allocate */
256 if (!new_blkg) {
257 new_blkg = blkg_alloc(blkcg, q, GFP_NOWAIT | __GFP_NOWARN);
258 if (unlikely(!new_blkg)) {
259 ret = -ENOMEM;
Christoph Hellwig8c911f32020-07-01 11:06:21 +0200260 goto err_put_css;
Jens Axboed708f0d2017-03-29 11:25:48 -0600261 }
262 }
263 blkg = new_blkg;
Tahsin Erdogan7fc6b872017-03-09 00:05:31 -0800264
Tejun Heodb613672013-05-14 13:52:31 -0700265 /* link parent */
Tejun Heo3c547862013-01-09 08:05:10 -0800266 if (blkcg_parent(blkcg)) {
267 blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false);
268 if (WARN_ON_ONCE(!blkg->parent)) {
Tejun Heo20386ce2015-08-18 14:55:28 -0700269 ret = -ENODEV;
Christoph Hellwig8c911f32020-07-01 11:06:21 +0200270 goto err_put_css;
Tejun Heo3c547862013-01-09 08:05:10 -0800271 }
272 blkg_get(blkg->parent);
273 }
274
Tejun Heodb613672013-05-14 13:52:31 -0700275 /* invoke per-policy init */
276 for (i = 0; i < BLKCG_MAX_POLS; i++) {
277 struct blkcg_policy *pol = blkcg_policy[i];
278
279 if (blkg->pd[i] && pol->pd_init_fn)
Tejun Heoa9520cd2015-08-18 14:55:14 -0700280 pol->pd_init_fn(blkg->pd[i]);
Tejun Heodb613672013-05-14 13:52:31 -0700281 }
282
283 /* insert */
Tejun Heoa6371202012-04-19 16:29:24 -0700284 spin_lock(&blkcg->lock);
285 ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg);
286 if (likely(!ret)) {
287 hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
288 list_add(&blkg->q_node, &q->blkg_list);
Tejun Heof427d902013-01-09 08:05:12 -0800289
290 for (i = 0; i < BLKCG_MAX_POLS; i++) {
291 struct blkcg_policy *pol = blkcg_policy[i];
292
293 if (blkg->pd[i] && pol->pd_online_fn)
Tejun Heoa9520cd2015-08-18 14:55:14 -0700294 pol->pd_online_fn(blkg->pd[i]);
Tejun Heof427d902013-01-09 08:05:12 -0800295 }
Tejun Heoa6371202012-04-19 16:29:24 -0700296 }
Tejun Heof427d902013-01-09 08:05:12 -0800297 blkg->online = true;
Tejun Heoa6371202012-04-19 16:29:24 -0700298 spin_unlock(&blkcg->lock);
299
Tejun Heoec13b1d2015-05-22 17:13:19 -0400300 if (!ret)
Tejun Heoa6371202012-04-19 16:29:24 -0700301 return blkg;
Tejun Heo15974992012-06-04 20:40:52 -0700302
Tejun Heo3c547862013-01-09 08:05:10 -0800303 /* @blkg failed fully initialized, use the usual release path */
304 blkg_put(blkg);
305 return ERR_PTR(ret);
306
Jens Axboed708f0d2017-03-29 11:25:48 -0600307err_put_css:
Tejun Heo496fb782012-04-19 16:29:23 -0700308 css_put(&blkcg->css);
Tejun Heo93e6d5d2013-01-09 08:05:10 -0800309err_free_blkg:
Jens Axboed708f0d2017-03-29 11:25:48 -0600310 blkg_free(new_blkg);
Tejun Heo93e6d5d2013-01-09 08:05:10 -0800311 return ERR_PTR(ret);
Vivek Goyal31e4c282009-12-03 12:59:42 -0500312}
Tejun Heo3c96cb32012-04-13 13:11:34 -0700313
Tejun Heo86cde6b2013-01-09 08:05:10 -0800314/**
Christoph Hellwig8c546282020-06-27 09:31:52 +0200315 * blkg_lookup_create - lookup blkg, try to create one if not there
Tejun Heo86cde6b2013-01-09 08:05:10 -0800316 * @blkcg: blkcg of interest
317 * @q: request_queue of interest
318 *
319 * Lookup blkg for the @blkcg - @q pair. If it doesn't exist, try to
Tejun Heo3c547862013-01-09 08:05:10 -0800320 * create one. blkg creation is performed recursively from blkcg_root such
321 * that all non-root blkg's have access to the parent blkg. This function
Christoph Hellwig8c546282020-06-27 09:31:52 +0200322 * should be called under RCU read lock and takes @q->queue_lock.
Tejun Heo86cde6b2013-01-09 08:05:10 -0800323 *
Dennis Zhoubeea9da2018-12-05 12:10:28 -0500324 * Returns the blkg or the closest blkg if blkg_create() fails as it walks
325 * down from root.
Tejun Heo86cde6b2013-01-09 08:05:10 -0800326 */
Christoph Hellwig8c546282020-06-27 09:31:52 +0200327static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
328 struct request_queue *q)
Tejun Heo3c96cb32012-04-13 13:11:34 -0700329{
Tejun Heo86cde6b2013-01-09 08:05:10 -0800330 struct blkcg_gq *blkg;
Christoph Hellwig8c546282020-06-27 09:31:52 +0200331 unsigned long flags;
Tejun Heo86cde6b2013-01-09 08:05:10 -0800332
333 WARN_ON_ONCE(!rcu_read_lock_held());
Tejun Heo86cde6b2013-01-09 08:05:10 -0800334
Christoph Hellwig8c546282020-06-27 09:31:52 +0200335 blkg = blkg_lookup(blkcg, q);
Tejun Heo86cde6b2013-01-09 08:05:10 -0800336 if (blkg)
337 return blkg;
338
Christoph Hellwig8c546282020-06-27 09:31:52 +0200339 spin_lock_irqsave(&q->queue_lock, flags);
340 blkg = __blkg_lookup(blkcg, q, true);
341 if (blkg)
342 goto found;
343
Tejun Heo3c547862013-01-09 08:05:10 -0800344 /*
345 * Create blkgs walking down from blkcg_root to @blkcg, so that all
Dennis Zhoubeea9da2018-12-05 12:10:28 -0500346 * non-root blkgs have access to their parents. Returns the closest
347 * blkg to the intended blkg should blkg_create() fail.
Tejun Heo3c547862013-01-09 08:05:10 -0800348 */
349 while (true) {
350 struct blkcg *pos = blkcg;
351 struct blkcg *parent = blkcg_parent(blkcg);
Dennis Zhoubeea9da2018-12-05 12:10:28 -0500352 struct blkcg_gq *ret_blkg = q->root_blkg;
Tejun Heo3c547862013-01-09 08:05:10 -0800353
Dennis Zhoubeea9da2018-12-05 12:10:28 -0500354 while (parent) {
355 blkg = __blkg_lookup(parent, q, false);
356 if (blkg) {
357 /* remember closest blkg */
358 ret_blkg = blkg;
359 break;
360 }
Tejun Heo3c547862013-01-09 08:05:10 -0800361 pos = parent;
362 parent = blkcg_parent(parent);
363 }
364
Jens Axboed708f0d2017-03-29 11:25:48 -0600365 blkg = blkg_create(pos, q, NULL);
Christoph Hellwig8c546282020-06-27 09:31:52 +0200366 if (IS_ERR(blkg)) {
367 blkg = ret_blkg;
368 break;
369 }
Dennis Zhoubeea9da2018-12-05 12:10:28 -0500370 if (pos == blkcg)
Christoph Hellwig8c546282020-06-27 09:31:52 +0200371 break;
Dennis Zhoub9789622018-12-05 12:10:27 -0500372 }
373
Christoph Hellwig8c546282020-06-27 09:31:52 +0200374found:
375 spin_unlock_irqrestore(&q->queue_lock, flags);
Dennis Zhoub9789622018-12-05 12:10:27 -0500376 return blkg;
377}
378
Tejun Heo3c798392012-04-16 13:57:25 -0700379static void blkg_destroy(struct blkcg_gq *blkg)
Tejun Heo72e06c22012-03-05 13:15:00 -0800380{
Tejun Heo3c798392012-04-16 13:57:25 -0700381 struct blkcg *blkcg = blkg->blkcg;
Dennis Zhou (Facebook)6b065462018-08-31 16:22:42 -0400382 int i;
Tejun Heo03aa2642012-03-05 13:15:19 -0800383
Christoph Hellwig0d945c12018-11-15 12:17:28 -0700384 lockdep_assert_held(&blkg->q->queue_lock);
Tejun Heo9f13ef62012-03-05 13:15:21 -0800385 lockdep_assert_held(&blkcg->lock);
Tejun Heo03aa2642012-03-05 13:15:19 -0800386
387 /* Something wrong if we are trying to remove same group twice */
Tejun Heoe8989fa2012-03-05 13:15:20 -0800388 WARN_ON_ONCE(list_empty(&blkg->q_node));
Tejun Heo9f13ef62012-03-05 13:15:21 -0800389 WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node));
Tejun Heoa6371202012-04-19 16:29:24 -0700390
Dennis Zhou (Facebook)6b065462018-08-31 16:22:42 -0400391 for (i = 0; i < BLKCG_MAX_POLS; i++) {
392 struct blkcg_policy *pol = blkcg_policy[i];
393
394 if (blkg->pd[i] && pol->pd_offline_fn)
395 pol->pd_offline_fn(blkg->pd[i]);
396 }
397
Tejun Heof427d902013-01-09 08:05:12 -0800398 blkg->online = false;
399
Tejun Heoa6371202012-04-19 16:29:24 -0700400 radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);
Tejun Heoe8989fa2012-03-05 13:15:20 -0800401 list_del_init(&blkg->q_node);
Tejun Heo9f13ef62012-03-05 13:15:21 -0800402 hlist_del_init_rcu(&blkg->blkcg_node);
Tejun Heo03aa2642012-03-05 13:15:19 -0800403
Tejun Heo03aa2642012-03-05 13:15:19 -0800404 /*
Tejun Heoa6371202012-04-19 16:29:24 -0700405 * Both setting lookup hint to and clearing it from @blkg are done
406 * under queue_lock. If it's not pointing to @blkg now, it never
407 * will. Hint assignment itself can race safely.
408 */
Paul E. McKenneyec6c676a2014-02-17 13:35:57 -0800409 if (rcu_access_pointer(blkcg->blkg_hint) == blkg)
Tejun Heoa6371202012-04-19 16:29:24 -0700410 rcu_assign_pointer(blkcg->blkg_hint, NULL);
411
412 /*
Tejun Heo03aa2642012-03-05 13:15:19 -0800413 * Put the reference taken at the time of creation so that when all
414 * queues are gone, group can be destroyed.
415 */
Dennis Zhou7fcf2b02018-12-05 12:10:38 -0500416 percpu_ref_kill(&blkg->refcnt);
Tejun Heo03aa2642012-03-05 13:15:19 -0800417}
418
Tejun Heo9f13ef62012-03-05 13:15:21 -0800419/**
420 * blkg_destroy_all - destroy all blkgs associated with a request_queue
421 * @q: request_queue of interest
Tejun Heo9f13ef62012-03-05 13:15:21 -0800422 *
Tejun Heo3c96cb32012-04-13 13:11:34 -0700423 * Destroy all blkgs associated with @q.
Tejun Heo9f13ef62012-03-05 13:15:21 -0800424 */
Tejun Heo3c96cb32012-04-13 13:11:34 -0700425static void blkg_destroy_all(struct request_queue *q)
Tejun Heo03aa2642012-03-05 13:15:19 -0800426{
Tejun Heo3c798392012-04-16 13:57:25 -0700427 struct blkcg_gq *blkg, *n;
Yu Kuaia7317632021-07-07 09:56:49 +0800428 int count = BLKG_DESTROY_BATCH_SIZE;
Tejun Heo72e06c22012-03-05 13:15:00 -0800429
Yu Kuaia7317632021-07-07 09:56:49 +0800430restart:
Christoph Hellwig0d945c12018-11-15 12:17:28 -0700431 spin_lock_irq(&q->queue_lock);
Tejun Heo9f13ef62012-03-05 13:15:21 -0800432 list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) {
Tejun Heo3c798392012-04-16 13:57:25 -0700433 struct blkcg *blkcg = blkg->blkcg;
Tejun Heo72e06c22012-03-05 13:15:00 -0800434
Tejun Heo9f13ef62012-03-05 13:15:21 -0800435 spin_lock(&blkcg->lock);
436 blkg_destroy(blkg);
437 spin_unlock(&blkcg->lock);
Yu Kuaia7317632021-07-07 09:56:49 +0800438
439 /*
440 * in order to avoid holding the spin lock for too long, release
441 * it when a batch of blkgs are destroyed.
442 */
443 if (!(--count)) {
444 count = BLKG_DESTROY_BATCH_SIZE;
445 spin_unlock_irq(&q->queue_lock);
446 cond_resched();
447 goto restart;
448 }
Tejun Heo72e06c22012-03-05 13:15:00 -0800449 }
Tejun Heo6fe810b2015-09-05 15:47:36 -0400450
451 q->root_blkg = NULL;
Christoph Hellwig0d945c12018-11-15 12:17:28 -0700452 spin_unlock_irq(&q->queue_lock);
Tejun Heo72e06c22012-03-05 13:15:00 -0800453}
454
Tejun Heo182446d2013-08-08 20:11:24 -0400455static int blkcg_reset_stats(struct cgroup_subsys_state *css,
456 struct cftype *cftype, u64 val)
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700457{
Tejun Heo182446d2013-08-08 20:11:24 -0400458 struct blkcg *blkcg = css_to_blkcg(css);
Tejun Heo3c798392012-04-16 13:57:25 -0700459 struct blkcg_gq *blkg;
Tejun Heof7331642019-11-07 11:18:03 -0800460 int i, cpu;
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700461
Tejun Heo838f13b2015-07-09 16:39:47 -0400462 mutex_lock(&blkcg_pol_mutex);
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700463 spin_lock_irq(&blkcg->lock);
Tejun Heo997a0262012-03-08 10:53:58 -0800464
465 /*
466 * Note that stat reset is racy - it doesn't synchronize against
467 * stat updates. This is a debug feature which shouldn't exist
468 * anyway. If you get hit by a race, retry.
469 */
Sasha Levinb67bfe02013-02-27 17:06:00 -0800470 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
Tejun Heof7331642019-11-07 11:18:03 -0800471 for_each_possible_cpu(cpu) {
472 struct blkg_iostat_set *bis =
473 per_cpu_ptr(blkg->iostat_cpu, cpu);
474 memset(bis, 0, sizeof(*bis));
475 }
476 memset(&blkg->iostat, 0, sizeof(blkg->iostat));
Tejun Heo77ea7332015-08-18 14:55:24 -0700477
Tejun Heo8bd435b2012-04-13 13:11:28 -0700478 for (i = 0; i < BLKCG_MAX_POLS; i++) {
Tejun Heo3c798392012-04-16 13:57:25 -0700479 struct blkcg_policy *pol = blkcg_policy[i];
Tejun Heo549d3aa2012-03-05 13:15:16 -0800480
Tejun Heoa9520cd2015-08-18 14:55:14 -0700481 if (blkg->pd[i] && pol->pd_reset_stats_fn)
482 pol->pd_reset_stats_fn(blkg->pd[i]);
Tejun Heobc0d6502012-04-13 13:11:26 -0700483 }
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700484 }
Vivek Goyalf0bdc8c2011-05-19 15:38:30 -0400485
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700486 spin_unlock_irq(&blkcg->lock);
Tejun Heobc0d6502012-04-13 13:11:26 -0700487 mutex_unlock(&blkcg_pol_mutex);
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700488 return 0;
489}
490
Tejun Heodd165eb2015-08-18 14:55:33 -0700491const char *blkg_dev_name(struct blkcg_gq *blkg)
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700492{
Christoph Hellwigd152c682021-08-16 15:46:24 +0200493 if (!blkg->q->disk || !blkg->q->disk->bdi->dev)
Christoph Hellwigedb08722021-08-09 16:17:43 +0200494 return NULL;
Christoph Hellwigd152c682021-08-16 15:46:24 +0200495 return bdi_dev_name(blkg->q->disk->bdi);
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700496}
497
Tejun Heod3d32e62012-04-01 14:38:42 -0700498/**
499 * blkcg_print_blkgs - helper for printing per-blkg data
500 * @sf: seq_file to print to
501 * @blkcg: blkcg of interest
502 * @prfill: fill function to print out a blkg
503 * @pol: policy in question
504 * @data: data to be passed to @prfill
505 * @show_total: to print out sum of prfill return values or not
506 *
507 * This function invokes @prfill on each blkg of @blkcg if pd for the
508 * policy specified by @pol exists. @prfill is invoked with @sf, the
Tejun Heo810ecfa2013-01-09 08:05:13 -0800509 * policy data and @data and the matching queue lock held. If @show_total
510 * is %true, the sum of the return values from @prfill is printed with
511 * "Total" label at the end.
Tejun Heod3d32e62012-04-01 14:38:42 -0700512 *
513 * This is to be used to construct print functions for
514 * cftype->read_seq_string method.
515 */
Tejun Heo3c798392012-04-16 13:57:25 -0700516void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
Tejun Heof95a04a2012-04-16 13:57:26 -0700517 u64 (*prfill)(struct seq_file *,
518 struct blkg_policy_data *, int),
Tejun Heo3c798392012-04-16 13:57:25 -0700519 const struct blkcg_policy *pol, int data,
Tejun Heoec399342012-04-13 13:11:27 -0700520 bool show_total)
Vivek Goyal5624a4e2011-05-19 15:38:28 -0400521{
Tejun Heo3c798392012-04-16 13:57:25 -0700522 struct blkcg_gq *blkg;
Tejun Heod3d32e62012-04-01 14:38:42 -0700523 u64 total = 0;
524
Tejun Heo810ecfa2013-01-09 08:05:13 -0800525 rcu_read_lock();
Linus Torvaldsee89f812013-02-28 12:52:24 -0800526 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
Christoph Hellwig0d945c12018-11-15 12:17:28 -0700527 spin_lock_irq(&blkg->q->queue_lock);
Tejun Heoa2b16932012-04-13 13:11:33 -0700528 if (blkcg_policy_enabled(blkg->q, pol))
Tejun Heof95a04a2012-04-16 13:57:26 -0700529 total += prfill(sf, blkg->pd[pol->plid], data);
Christoph Hellwig0d945c12018-11-15 12:17:28 -0700530 spin_unlock_irq(&blkg->q->queue_lock);
Tejun Heo810ecfa2013-01-09 08:05:13 -0800531 }
532 rcu_read_unlock();
Tejun Heod3d32e62012-04-01 14:38:42 -0700533
534 if (show_total)
535 seq_printf(sf, "Total %llu\n", (unsigned long long)total);
536}
Tejun Heo829fdb52012-04-01 14:38:43 -0700537EXPORT_SYMBOL_GPL(blkcg_print_blkgs);
Tejun Heod3d32e62012-04-01 14:38:42 -0700538
539/**
540 * __blkg_prfill_u64 - prfill helper for a single u64 value
541 * @sf: seq_file to print to
Tejun Heof95a04a2012-04-16 13:57:26 -0700542 * @pd: policy private data of interest
Tejun Heod3d32e62012-04-01 14:38:42 -0700543 * @v: value to print
544 *
Tejun Heof95a04a2012-04-16 13:57:26 -0700545 * Print @v to @sf for the device assocaited with @pd.
Tejun Heod3d32e62012-04-01 14:38:42 -0700546 */
Tejun Heof95a04a2012-04-16 13:57:26 -0700547u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v)
Tejun Heod3d32e62012-04-01 14:38:42 -0700548{
Tejun Heof95a04a2012-04-16 13:57:26 -0700549 const char *dname = blkg_dev_name(pd->blkg);
Tejun Heod3d32e62012-04-01 14:38:42 -0700550
551 if (!dname)
552 return 0;
553
554 seq_printf(sf, "%s %llu\n", dname, (unsigned long long)v);
555 return v;
556}
Tejun Heo829fdb52012-04-01 14:38:43 -0700557EXPORT_SYMBOL_GPL(__blkg_prfill_u64);
Tejun Heod3d32e62012-04-01 14:38:42 -0700558
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600559/* Performs queue bypass and policy enabled checks then looks up blkg. */
560static struct blkcg_gq *blkg_lookup_check(struct blkcg *blkcg,
561 const struct blkcg_policy *pol,
562 struct request_queue *q)
563{
564 WARN_ON_ONCE(!rcu_read_lock_held());
Christoph Hellwig0d945c12018-11-15 12:17:28 -0700565 lockdep_assert_held(&q->queue_lock);
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600566
567 if (!blkcg_policy_enabled(q, pol))
568 return ERR_PTR(-EOPNOTSUPP);
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600569 return __blkg_lookup(blkcg, q, true /* update_hint */);
570}
571
Tejun Heo16b3de62013-01-09 08:05:12 -0800572/**
Christoph Hellwig22ae8ce2020-11-26 09:23:26 +0100573 * blkcg_conf_open_bdev - parse and open bdev for per-blkg config update
Tejun Heo015d2542019-08-28 15:05:53 -0700574 * @inputp: input string pointer
575 *
576 * Parse the device node prefix part, MAJ:MIN, of per-blkg config update
Christoph Hellwig22ae8ce2020-11-26 09:23:26 +0100577 * from @input and get and return the matching bdev. *@inputp is
Tejun Heo015d2542019-08-28 15:05:53 -0700578 * updated to point past the device node prefix. Returns an ERR_PTR()
579 * value on error.
580 *
581 * Use this function iff blkg_conf_prep() can't be used for some reason.
582 */
Christoph Hellwig22ae8ce2020-11-26 09:23:26 +0100583struct block_device *blkcg_conf_open_bdev(char **inputp)
Tejun Heo015d2542019-08-28 15:05:53 -0700584{
585 char *input = *inputp;
586 unsigned int major, minor;
Christoph Hellwig22ae8ce2020-11-26 09:23:26 +0100587 struct block_device *bdev;
588 int key_len;
Tejun Heo015d2542019-08-28 15:05:53 -0700589
590 if (sscanf(input, "%u:%u%n", &major, &minor, &key_len) != 2)
591 return ERR_PTR(-EINVAL);
592
593 input += key_len;
594 if (!isspace(*input))
595 return ERR_PTR(-EINVAL);
596 input = skip_spaces(input);
597
Christoph Hellwig22ae8ce2020-11-26 09:23:26 +0100598 bdev = blkdev_get_no_open(MKDEV(major, minor));
599 if (!bdev)
Tejun Heo015d2542019-08-28 15:05:53 -0700600 return ERR_PTR(-ENODEV);
Christoph Hellwig22ae8ce2020-11-26 09:23:26 +0100601 if (bdev_is_partition(bdev)) {
602 blkdev_put_no_open(bdev);
Tejun Heo015d2542019-08-28 15:05:53 -0700603 return ERR_PTR(-ENODEV);
604 }
605
606 *inputp = input;
Christoph Hellwig22ae8ce2020-11-26 09:23:26 +0100607 return bdev;
Tejun Heo015d2542019-08-28 15:05:53 -0700608}
609
610/**
611 * blkg_conf_prep - parse and prepare for per-blkg config update
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700612 * @blkcg: target block cgroup
Tejun Heoda8b0662012-04-13 13:11:29 -0700613 * @pol: target policy
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700614 * @input: input string
615 * @ctx: blkg_conf_ctx to be filled
616 *
617 * Parse per-blkg config update from @input and initialize @ctx with the
Tejun Heo36aa9e52015-08-18 14:55:31 -0700618 * result. @ctx->blkg points to the blkg to be updated and @ctx->body the
619 * part of @input following MAJ:MIN. This function returns with RCU read
620 * lock and queue lock held and must be paired with blkg_conf_finish().
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700621 */
Tejun Heo3c798392012-04-16 13:57:25 -0700622int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
Tejun Heo36aa9e52015-08-18 14:55:31 -0700623 char *input, struct blkg_conf_ctx *ctx)
Pavel Begunkoved6cdde2021-10-14 15:03:30 +0100624 __acquires(rcu) __acquires(&bdev->bd_queue->queue_lock)
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800625{
Christoph Hellwig22ae8ce2020-11-26 09:23:26 +0100626 struct block_device *bdev;
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600627 struct request_queue *q;
Tejun Heo3c798392012-04-16 13:57:25 -0700628 struct blkcg_gq *blkg;
Tejun Heo015d2542019-08-28 15:05:53 -0700629 int ret;
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800630
Christoph Hellwig22ae8ce2020-11-26 09:23:26 +0100631 bdev = blkcg_conf_open_bdev(&input);
632 if (IS_ERR(bdev))
633 return PTR_ERR(bdev);
Tejun Heoe56da7e2012-03-05 13:15:07 -0800634
Pavel Begunkoved6cdde2021-10-14 15:03:30 +0100635 q = bdev_get_queue(bdev);
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600636
Yu Kuai0c9d3382021-10-20 09:40:36 +0800637 /*
638 * blkcg_deactivate_policy() requires queue to be frozen, we can grab
639 * q_usage_counter to prevent concurrent with blkcg_deactivate_policy().
640 */
641 ret = blk_queue_enter(q, 0);
642 if (ret)
Yu Kuai15c30102021-11-02 10:07:05 +0800643 goto fail;
Yu Kuai0c9d3382021-10-20 09:40:36 +0800644
Tejun Heoe56da7e2012-03-05 13:15:07 -0800645 rcu_read_lock();
Christoph Hellwig0d945c12018-11-15 12:17:28 -0700646 spin_lock_irq(&q->queue_lock);
Tejun Heoda8b0662012-04-13 13:11:29 -0700647
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600648 blkg = blkg_lookup_check(blkcg, pol, q);
Tejun Heo4bfd4822012-03-05 13:15:08 -0800649 if (IS_ERR(blkg)) {
650 ret = PTR_ERR(blkg);
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600651 goto fail_unlock;
Vivek Goyal062a6442010-09-15 17:06:33 -0400652 }
Tejun Heoe56da7e2012-03-05 13:15:07 -0800653
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600654 if (blkg)
655 goto success;
656
657 /*
658 * Create blkgs walking down from blkcg_root to @blkcg, so that all
659 * non-root blkgs have access to their parents.
660 */
661 while (true) {
662 struct blkcg *pos = blkcg;
663 struct blkcg *parent;
664 struct blkcg_gq *new_blkg;
665
666 parent = blkcg_parent(blkcg);
667 while (parent && !__blkg_lookup(parent, q, false)) {
668 pos = parent;
669 parent = blkcg_parent(parent);
670 }
671
672 /* Drop locks to do new blkg allocation with GFP_KERNEL. */
Christoph Hellwig0d945c12018-11-15 12:17:28 -0700673 spin_unlock_irq(&q->queue_lock);
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600674 rcu_read_unlock();
675
676 new_blkg = blkg_alloc(pos, q, GFP_KERNEL);
677 if (unlikely(!new_blkg)) {
678 ret = -ENOMEM;
Yu Kuai15c30102021-11-02 10:07:05 +0800679 goto fail_exit_queue;
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600680 }
681
Gabriel Krisman Bertazif255c192020-10-22 16:58:42 -0400682 if (radix_tree_preload(GFP_KERNEL)) {
683 blkg_free(new_blkg);
684 ret = -ENOMEM;
Yu Kuai15c30102021-11-02 10:07:05 +0800685 goto fail_exit_queue;
Gabriel Krisman Bertazif255c192020-10-22 16:58:42 -0400686 }
687
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600688 rcu_read_lock();
Christoph Hellwig0d945c12018-11-15 12:17:28 -0700689 spin_lock_irq(&q->queue_lock);
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600690
691 blkg = blkg_lookup_check(pos, pol, q);
692 if (IS_ERR(blkg)) {
693 ret = PTR_ERR(blkg);
Gabriel Krisman Bertazi52abfcb2020-10-22 16:58:41 -0400694 blkg_free(new_blkg);
Gabriel Krisman Bertazif255c192020-10-22 16:58:42 -0400695 goto fail_preloaded;
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600696 }
697
698 if (blkg) {
699 blkg_free(new_blkg);
700 } else {
701 blkg = blkg_create(pos, q, new_blkg);
Kefeng Wang98d669b2019-06-05 22:24:27 +0800702 if (IS_ERR(blkg)) {
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600703 ret = PTR_ERR(blkg);
Gabriel Krisman Bertazif255c192020-10-22 16:58:42 -0400704 goto fail_preloaded;
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600705 }
706 }
707
Gabriel Krisman Bertazif255c192020-10-22 16:58:42 -0400708 radix_tree_preload_end();
709
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600710 if (pos == blkcg)
711 goto success;
712 }
713success:
Yu Kuai0c9d3382021-10-20 09:40:36 +0800714 blk_queue_exit(q);
Christoph Hellwig22ae8ce2020-11-26 09:23:26 +0100715 ctx->bdev = bdev;
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700716 ctx->blkg = blkg;
Tejun Heo015d2542019-08-28 15:05:53 -0700717 ctx->body = input;
Tejun Heo726fa692012-04-01 14:38:43 -0700718 return 0;
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600719
Gabriel Krisman Bertazif255c192020-10-22 16:58:42 -0400720fail_preloaded:
721 radix_tree_preload_end();
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600722fail_unlock:
Christoph Hellwig0d945c12018-11-15 12:17:28 -0700723 spin_unlock_irq(&q->queue_lock);
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600724 rcu_read_unlock();
Yu Kuai15c30102021-11-02 10:07:05 +0800725fail_exit_queue:
726 blk_queue_exit(q);
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600727fail:
Christoph Hellwig22ae8ce2020-11-26 09:23:26 +0100728 blkdev_put_no_open(bdev);
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600729 /*
730 * If queue was bypassing, we should retry. Do so after a
731 * short msleep(). It isn't strictly necessary but queue
732 * can be bypassing for some time and it's always nice to
733 * avoid busy looping.
734 */
735 if (ret == -EBUSY) {
736 msleep(10);
737 ret = restart_syscall();
738 }
739 return ret;
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800740}
Pavel Begunkov89f3b6d2019-09-14 20:31:50 +0300741EXPORT_SYMBOL_GPL(blkg_conf_prep);
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800742
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700743/**
744 * blkg_conf_finish - finish up per-blkg config update
745 * @ctx: blkg_conf_ctx intiailized by blkg_conf_prep()
746 *
747 * Finish up after per-blkg config update. This function must be paired
748 * with blkg_conf_prep().
749 */
Tejun Heo829fdb52012-04-01 14:38:43 -0700750void blkg_conf_finish(struct blkg_conf_ctx *ctx)
Pavel Begunkoved6cdde2021-10-14 15:03:30 +0100751 __releases(&ctx->bdev->bd_queue->queue_lock) __releases(rcu)
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800752{
Pavel Begunkoved6cdde2021-10-14 15:03:30 +0100753 spin_unlock_irq(&bdev_get_queue(ctx->bdev)->queue_lock);
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700754 rcu_read_unlock();
Christoph Hellwig22ae8ce2020-11-26 09:23:26 +0100755 blkdev_put_no_open(ctx->bdev);
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800756}
Pavel Begunkov89f3b6d2019-09-14 20:31:50 +0300757EXPORT_SYMBOL_GPL(blkg_conf_finish);
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800758
Boris Burkovcd1fc4b2020-06-01 13:11:43 -0700759static void blkg_iostat_set(struct blkg_iostat *dst, struct blkg_iostat *src)
760{
761 int i;
762
763 for (i = 0; i < BLKG_IOSTAT_NR; i++) {
764 dst->bytes[i] = src->bytes[i];
765 dst->ios[i] = src->ios[i];
766 }
767}
768
769static void blkg_iostat_add(struct blkg_iostat *dst, struct blkg_iostat *src)
770{
771 int i;
772
773 for (i = 0; i < BLKG_IOSTAT_NR; i++) {
774 dst->bytes[i] += src->bytes[i];
775 dst->ios[i] += src->ios[i];
776 }
777}
778
779static void blkg_iostat_sub(struct blkg_iostat *dst, struct blkg_iostat *src)
780{
781 int i;
782
783 for (i = 0; i < BLKG_IOSTAT_NR; i++) {
784 dst->bytes[i] -= src->bytes[i];
785 dst->ios[i] -= src->ios[i];
786 }
787}
788
789static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)
790{
791 struct blkcg *blkcg = css_to_blkcg(css);
792 struct blkcg_gq *blkg;
793
Johannes Weinerdc265322021-04-29 22:56:23 -0700794 /* Root-level stats are sourced from system-wide IO stats */
795 if (!cgroup_parent(css->cgroup))
796 return;
797
Boris Burkovcd1fc4b2020-06-01 13:11:43 -0700798 rcu_read_lock();
799
800 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
801 struct blkcg_gq *parent = blkg->parent;
802 struct blkg_iostat_set *bisc = per_cpu_ptr(blkg->iostat_cpu, cpu);
803 struct blkg_iostat cur, delta;
Tejun Heoc3df5fb2021-07-27 13:12:20 -1000804 unsigned long flags;
Boris Burkovcd1fc4b2020-06-01 13:11:43 -0700805 unsigned int seq;
806
807 /* fetch the current per-cpu values */
808 do {
809 seq = u64_stats_fetch_begin(&bisc->sync);
810 blkg_iostat_set(&cur, &bisc->cur);
811 } while (u64_stats_fetch_retry(&bisc->sync, seq));
812
813 /* propagate percpu delta to global */
Tejun Heoc3df5fb2021-07-27 13:12:20 -1000814 flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
Boris Burkovcd1fc4b2020-06-01 13:11:43 -0700815 blkg_iostat_set(&delta, &cur);
816 blkg_iostat_sub(&delta, &bisc->last);
817 blkg_iostat_add(&blkg->iostat.cur, &delta);
818 blkg_iostat_add(&bisc->last, &delta);
Tejun Heoc3df5fb2021-07-27 13:12:20 -1000819 u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
Boris Burkovcd1fc4b2020-06-01 13:11:43 -0700820
Johannes Weinerdc265322021-04-29 22:56:23 -0700821 /* propagate global delta to parent (unless that's root) */
822 if (parent && parent->parent) {
Tejun Heoc3df5fb2021-07-27 13:12:20 -1000823 flags = u64_stats_update_begin_irqsave(&parent->iostat.sync);
Boris Burkovcd1fc4b2020-06-01 13:11:43 -0700824 blkg_iostat_set(&delta, &blkg->iostat.cur);
825 blkg_iostat_sub(&delta, &blkg->iostat.last);
826 blkg_iostat_add(&parent->iostat.cur, &delta);
827 blkg_iostat_add(&blkg->iostat.last, &delta);
Tejun Heoc3df5fb2021-07-27 13:12:20 -1000828 u64_stats_update_end_irqrestore(&parent->iostat.sync, flags);
Boris Burkovcd1fc4b2020-06-01 13:11:43 -0700829 }
830 }
831
832 rcu_read_unlock();
833}
834
Boris Burkovef45fe42020-06-01 13:12:05 -0700835/*
Johannes Weinerdc265322021-04-29 22:56:23 -0700836 * We source root cgroup stats from the system-wide stats to avoid
837 * tracking the same information twice and incurring overhead when no
838 * cgroups are defined. For that reason, cgroup_rstat_flush in
839 * blkcg_print_stat does not actually fill out the iostat in the root
840 * cgroup's blkcg_gq.
Boris Burkovef45fe42020-06-01 13:12:05 -0700841 *
842 * However, we would like to re-use the printing code between the root and
843 * non-root cgroups to the extent possible. For that reason, we simulate
844 * flushing the root cgroup's stats by explicitly filling in the iostat
845 * with disk level statistics.
846 */
847static void blkcg_fill_root_iostats(void)
848{
849 struct class_dev_iter iter;
850 struct device *dev;
851
852 class_dev_iter_init(&iter, &block_class, NULL, &disk_type);
853 while ((dev = class_dev_iter_next(&iter))) {
Christoph Hellwig0d021292020-11-27 16:43:51 +0100854 struct block_device *bdev = dev_to_bdev(dev);
855 struct blkcg_gq *blkg =
Pavel Begunkoved6cdde2021-10-14 15:03:30 +0100856 blk_queue_root_blkg(bdev_get_queue(bdev));
Boris Burkovef45fe42020-06-01 13:12:05 -0700857 struct blkg_iostat tmp;
858 int cpu;
859
860 memset(&tmp, 0, sizeof(tmp));
861 for_each_possible_cpu(cpu) {
862 struct disk_stats *cpu_dkstats;
Tejun Heoc3df5fb2021-07-27 13:12:20 -1000863 unsigned long flags;
Boris Burkovef45fe42020-06-01 13:12:05 -0700864
Christoph Hellwig0d021292020-11-27 16:43:51 +0100865 cpu_dkstats = per_cpu_ptr(bdev->bd_stats, cpu);
Boris Burkovef45fe42020-06-01 13:12:05 -0700866 tmp.ios[BLKG_IOSTAT_READ] +=
867 cpu_dkstats->ios[STAT_READ];
868 tmp.ios[BLKG_IOSTAT_WRITE] +=
869 cpu_dkstats->ios[STAT_WRITE];
870 tmp.ios[BLKG_IOSTAT_DISCARD] +=
871 cpu_dkstats->ios[STAT_DISCARD];
872 // convert sectors to bytes
873 tmp.bytes[BLKG_IOSTAT_READ] +=
874 cpu_dkstats->sectors[STAT_READ] << 9;
875 tmp.bytes[BLKG_IOSTAT_WRITE] +=
876 cpu_dkstats->sectors[STAT_WRITE] << 9;
877 tmp.bytes[BLKG_IOSTAT_DISCARD] +=
878 cpu_dkstats->sectors[STAT_DISCARD] << 9;
879
Tejun Heoc3df5fb2021-07-27 13:12:20 -1000880 flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
Boris Burkovef45fe42020-06-01 13:12:05 -0700881 blkg_iostat_set(&blkg->iostat.cur, &tmp);
Tejun Heoc3df5fb2021-07-27 13:12:20 -1000882 u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
Boris Burkovef45fe42020-06-01 13:12:05 -0700883 }
884 }
885}
886
Christoph Hellwig49cb5162021-08-10 17:26:22 +0200887static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s)
888{
889 struct blkg_iostat_set *bis = &blkg->iostat;
890 u64 rbytes, wbytes, rios, wios, dbytes, dios;
891 bool has_stats = false;
892 const char *dname;
893 unsigned seq;
Christoph Hellwig49cb5162021-08-10 17:26:22 +0200894 int i;
895
896 if (!blkg->online)
897 return;
898
899 dname = blkg_dev_name(blkg);
900 if (!dname)
901 return;
902
Christoph Hellwig252c6512021-08-10 17:26:23 +0200903 seq_printf(s, "%s ", dname);
Christoph Hellwig49cb5162021-08-10 17:26:22 +0200904
905 do {
906 seq = u64_stats_fetch_begin(&bis->sync);
907
908 rbytes = bis->cur.bytes[BLKG_IOSTAT_READ];
909 wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE];
910 dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD];
911 rios = bis->cur.ios[BLKG_IOSTAT_READ];
912 wios = bis->cur.ios[BLKG_IOSTAT_WRITE];
913 dios = bis->cur.ios[BLKG_IOSTAT_DISCARD];
914 } while (u64_stats_fetch_retry(&bis->sync, seq));
915
916 if (rbytes || wbytes || rios || wios) {
917 has_stats = true;
Christoph Hellwig252c6512021-08-10 17:26:23 +0200918 seq_printf(s, "rbytes=%llu wbytes=%llu rios=%llu wios=%llu dbytes=%llu dios=%llu",
Christoph Hellwig49cb5162021-08-10 17:26:22 +0200919 rbytes, wbytes, rios, wios,
920 dbytes, dios);
921 }
922
923 if (blkcg_debug_stats && atomic_read(&blkg->use_delay)) {
924 has_stats = true;
Christoph Hellwig252c6512021-08-10 17:26:23 +0200925 seq_printf(s, " use_delay=%d delay_nsec=%llu",
Christoph Hellwig49cb5162021-08-10 17:26:22 +0200926 atomic_read(&blkg->use_delay),
927 atomic64_read(&blkg->delay_nsec));
928 }
929
930 for (i = 0; i < BLKCG_MAX_POLS; i++) {
931 struct blkcg_policy *pol = blkcg_policy[i];
Christoph Hellwig49cb5162021-08-10 17:26:22 +0200932
933 if (!blkg->pd[i] || !pol->pd_stat_fn)
934 continue;
935
Christoph Hellwig252c6512021-08-10 17:26:23 +0200936 if (pol->pd_stat_fn(blkg->pd[i], s))
Christoph Hellwig49cb5162021-08-10 17:26:22 +0200937 has_stats = true;
Christoph Hellwig49cb5162021-08-10 17:26:22 +0200938 }
939
Christoph Hellwig252c6512021-08-10 17:26:23 +0200940 if (has_stats)
941 seq_printf(s, "\n");
Christoph Hellwig49cb5162021-08-10 17:26:22 +0200942}
943
Tejun Heo2ee867dc2015-08-18 14:55:34 -0700944static int blkcg_print_stat(struct seq_file *sf, void *v)
945{
946 struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
947 struct blkcg_gq *blkg;
948
Boris Burkovef45fe42020-06-01 13:12:05 -0700949 if (!seq_css(sf)->parent)
950 blkcg_fill_root_iostats();
951 else
952 cgroup_rstat_flush(blkcg->css.cgroup);
953
Tejun Heo2ee867dc2015-08-18 14:55:34 -0700954 rcu_read_lock();
Tejun Heo2ee867dc2015-08-18 14:55:34 -0700955 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
Tejun Heob0814362019-11-05 08:09:51 -0800956 spin_lock_irq(&blkg->q->queue_lock);
Christoph Hellwig49cb5162021-08-10 17:26:22 +0200957 blkcg_print_one_stat(blkg, sf);
Tejun Heob0814362019-11-05 08:09:51 -0800958 spin_unlock_irq(&blkg->q->queue_lock);
Tejun Heo2ee867dc2015-08-18 14:55:34 -0700959 }
Tejun Heo2ee867dc2015-08-18 14:55:34 -0700960 rcu_read_unlock();
961 return 0;
962}
963
Bart Van Asschee1f3b942016-06-14 17:04:32 +0200964static struct cftype blkcg_files[] = {
Tejun Heo2ee867dc2015-08-18 14:55:34 -0700965 {
966 .name = "stat",
967 .seq_show = blkcg_print_stat,
968 },
969 { } /* terminate */
970};
971
Bart Van Asschee1f3b942016-06-14 17:04:32 +0200972static struct cftype blkcg_legacy_files[] = {
Vivek Goyal31e4c282009-12-03 12:59:42 -0500973 {
Divyesh Shah84c124d2010-04-09 08:31:19 +0200974 .name = "reset_stats",
Tejun Heo3c798392012-04-16 13:57:25 -0700975 .write_u64 = blkcg_reset_stats,
Vivek Goyal22084192009-12-03 12:59:49 -0500976 },
Tejun Heo4baf6e32012-04-01 12:09:55 -0700977 { } /* terminate */
Vivek Goyal31e4c282009-12-03 12:59:42 -0500978};
979
Dennis Zhou (Facebook)59b57712018-08-31 16:22:43 -0400980/*
981 * blkcg destruction is a three-stage process.
982 *
983 * 1. Destruction starts. The blkcg_css_offline() callback is invoked
984 * which offlines writeback. Here we tie the next stage of blkg destruction
985 * to the completion of writeback associated with the blkcg. This lets us
986 * avoid punting potentially large amounts of outstanding writeback to root
987 * while maintaining any ongoing policies. The next stage is triggered when
988 * the nr_cgwbs count goes to zero.
989 *
990 * 2. When the nr_cgwbs count goes to zero, blkcg_destroy_blkgs() is called
991 * and handles the destruction of blkgs. Here the css reference held by
992 * the blkg is put back eventually allowing blkcg_css_free() to be called.
993 * This work may occur in cgwb_release_workfn() on the cgwb_release
994 * workqueue. Any submitted ios that fail to get the blkg ref will be
995 * punted to the root_blkg.
996 *
997 * 3. Once the blkcg ref count goes to zero, blkcg_css_free() is called.
998 * This finally frees the blkcg.
999 */
1000
Tejun Heo9f13ef62012-03-05 13:15:21 -08001001/**
Tejun Heo92fb9742012-11-19 08:13:38 -08001002 * blkcg_css_offline - cgroup css_offline callback
Tejun Heoeb954192013-08-08 20:11:23 -04001003 * @css: css of interest
Tejun Heo9f13ef62012-03-05 13:15:21 -08001004 *
Dennis Zhou (Facebook)59b57712018-08-31 16:22:43 -04001005 * This function is called when @css is about to go away. Here the cgwbs are
1006 * offlined first and only once writeback associated with the blkcg has
1007 * finished do we start step 2 (see above).
Tejun Heo9f13ef62012-03-05 13:15:21 -08001008 */
Tejun Heoeb954192013-08-08 20:11:23 -04001009static void blkcg_css_offline(struct cgroup_subsys_state *css)
Vivek Goyal31e4c282009-12-03 12:59:42 -05001010{
Tejun Heoeb954192013-08-08 20:11:23 -04001011 struct blkcg *blkcg = css_to_blkcg(css);
Vivek Goyal31e4c282009-12-03 12:59:42 -05001012
Dennis Zhou (Facebook)59b57712018-08-31 16:22:43 -04001013 /* this prevents anyone from attaching or migrating to this blkcg */
1014 wb_blkcg_offline(blkcg);
1015
Tejun Heod866dbf2019-07-24 10:37:22 -07001016 /* put the base online pin allowing step 2 to be triggered */
1017 blkcg_unpin_online(blkcg);
Dennis Zhou (Facebook)59b57712018-08-31 16:22:43 -04001018}
1019
1020/**
1021 * blkcg_destroy_blkgs - responsible for shooting down blkgs
1022 * @blkcg: blkcg of interest
1023 *
1024 * blkgs should be removed while holding both q and blkcg locks. As blkcg lock
1025 * is nested inside q lock, this function performs reverse double lock dancing.
1026 * Destroying the blkgs releases the reference held on the blkcg's css allowing
1027 * blkcg_css_free to eventually be called.
1028 *
1029 * This is the blkcg counterpart of ioc_release_fn().
1030 */
1031void blkcg_destroy_blkgs(struct blkcg *blkcg)
1032{
Baolin Wang6c635ca2021-01-28 13:58:15 +08001033 might_sleep();
1034
Tejun Heo9f13ef62012-03-05 13:15:21 -08001035 spin_lock_irq(&blkcg->lock);
Tejun Heo7ee9c562012-03-05 13:15:11 -08001036
Dennis Zhou (Facebook)6b065462018-08-31 16:22:42 -04001037 while (!hlist_empty(&blkcg->blkg_list)) {
1038 struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first,
1039 struct blkcg_gq, blkcg_node);
Tejun Heoc875f4d2012-03-05 13:15:22 -08001040 struct request_queue *q = blkg->q;
Vivek Goyalb1c35762009-12-03 12:59:47 -05001041
Baolin Wang6c635ca2021-01-28 13:58:15 +08001042 if (need_resched() || !spin_trylock(&q->queue_lock)) {
1043 /*
1044 * Given that the system can accumulate a huge number
1045 * of blkgs in pathological cases, check to see if we
1046 * need to rescheduling to avoid softlockup.
1047 */
Tejun Heo9f13ef62012-03-05 13:15:21 -08001048 spin_unlock_irq(&blkcg->lock);
Baolin Wang6c635ca2021-01-28 13:58:15 +08001049 cond_resched();
Dan Carpentera5567932012-03-29 20:57:08 +02001050 spin_lock_irq(&blkcg->lock);
Baolin Wang6c635ca2021-01-28 13:58:15 +08001051 continue;
Jens Axboe0f3942a2010-05-03 14:28:55 +02001052 }
Baolin Wang6c635ca2021-01-28 13:58:15 +08001053
1054 blkg_destroy(blkg);
1055 spin_unlock(&q->queue_lock);
Tejun Heo9f13ef62012-03-05 13:15:21 -08001056 }
Jens Axboe0f3942a2010-05-03 14:28:55 +02001057
Tejun Heo9f13ef62012-03-05 13:15:21 -08001058 spin_unlock_irq(&blkcg->lock);
Tejun Heo7ee9c562012-03-05 13:15:11 -08001059}
1060
Tejun Heoeb954192013-08-08 20:11:23 -04001061static void blkcg_css_free(struct cgroup_subsys_state *css)
Tejun Heo7ee9c562012-03-05 13:15:11 -08001062{
Tejun Heoeb954192013-08-08 20:11:23 -04001063 struct blkcg *blkcg = css_to_blkcg(css);
Tejun Heobc915e62015-08-18 14:55:08 -07001064 int i;
Tejun Heo7ee9c562012-03-05 13:15:11 -08001065
Tejun Heo7876f932015-07-09 16:39:49 -04001066 mutex_lock(&blkcg_pol_mutex);
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001067
Tejun Heo7876f932015-07-09 16:39:49 -04001068 list_del(&blkcg->all_blkcgs_node);
Tejun Heo7876f932015-07-09 16:39:49 -04001069
Tejun Heobc915e62015-08-18 14:55:08 -07001070 for (i = 0; i < BLKCG_MAX_POLS; i++)
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001071 if (blkcg->cpd[i])
1072 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
1073
1074 mutex_unlock(&blkcg_pol_mutex);
1075
Tejun Heobc915e62015-08-18 14:55:08 -07001076 kfree(blkcg);
Vivek Goyal31e4c282009-12-03 12:59:42 -05001077}
1078
Tejun Heoeb954192013-08-08 20:11:23 -04001079static struct cgroup_subsys_state *
1080blkcg_css_alloc(struct cgroup_subsys_state *parent_css)
Vivek Goyal31e4c282009-12-03 12:59:42 -05001081{
Tejun Heo3c798392012-04-16 13:57:25 -07001082 struct blkcg *blkcg;
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001083 struct cgroup_subsys_state *ret;
1084 int i;
Vivek Goyal31e4c282009-12-03 12:59:42 -05001085
Tejun Heo7876f932015-07-09 16:39:49 -04001086 mutex_lock(&blkcg_pol_mutex);
1087
Tejun Heoeb954192013-08-08 20:11:23 -04001088 if (!parent_css) {
Tejun Heo3c798392012-04-16 13:57:25 -07001089 blkcg = &blkcg_root;
Tejun Heobc915e62015-08-18 14:55:08 -07001090 } else {
1091 blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);
1092 if (!blkcg) {
1093 ret = ERR_PTR(-ENOMEM);
weiping zhang4c18c9e2017-08-25 23:49:32 +08001094 goto unlock;
Tejun Heobc915e62015-08-18 14:55:08 -07001095 }
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001096 }
Vivek Goyal31e4c282009-12-03 12:59:42 -05001097
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001098 for (i = 0; i < BLKCG_MAX_POLS ; i++) {
1099 struct blkcg_policy *pol = blkcg_policy[i];
1100 struct blkcg_policy_data *cpd;
1101
1102 /*
1103 * If the policy hasn't been attached yet, wait for it
1104 * to be attached before doing anything else. Otherwise,
1105 * check if the policy requires any specific per-cgroup
1106 * data: if it does, allocate and initialize it.
1107 */
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001108 if (!pol || !pol->cpd_alloc_fn)
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001109 continue;
1110
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001111 cpd = pol->cpd_alloc_fn(GFP_KERNEL);
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001112 if (!cpd) {
1113 ret = ERR_PTR(-ENOMEM);
1114 goto free_pd_blkcg;
1115 }
Tejun Heo81437642015-08-18 14:55:15 -07001116 blkcg->cpd[i] = cpd;
1117 cpd->blkcg = blkcg;
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001118 cpd->plid = i;
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001119 if (pol->cpd_init_fn)
1120 pol->cpd_init_fn(cpd);
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001121 }
1122
Vivek Goyal31e4c282009-12-03 12:59:42 -05001123 spin_lock_init(&blkcg->lock);
Tejun Heod866dbf2019-07-24 10:37:22 -07001124 refcount_set(&blkcg->online_pin, 1);
Tejun Heoe00f4f42016-11-21 18:03:32 -05001125 INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT | __GFP_NOWARN);
Vivek Goyal31e4c282009-12-03 12:59:42 -05001126 INIT_HLIST_HEAD(&blkcg->blkg_list);
Tejun Heo52ebea72015-05-22 17:13:37 -04001127#ifdef CONFIG_CGROUP_WRITEBACK
1128 INIT_LIST_HEAD(&blkcg->cgwb_list);
1129#endif
Tejun Heo7876f932015-07-09 16:39:49 -04001130 list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs);
1131
1132 mutex_unlock(&blkcg_pol_mutex);
Vivek Goyal31e4c282009-12-03 12:59:42 -05001133 return &blkcg->css;
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001134
1135free_pd_blkcg:
1136 for (i--; i >= 0; i--)
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001137 if (blkcg->cpd[i])
1138 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
weiping zhang4c18c9e2017-08-25 23:49:32 +08001139
1140 if (blkcg != &blkcg_root)
1141 kfree(blkcg);
1142unlock:
Tejun Heo7876f932015-07-09 16:39:49 -04001143 mutex_unlock(&blkcg_pol_mutex);
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001144 return ret;
Vivek Goyal31e4c282009-12-03 12:59:42 -05001145}
1146
Tejun Heo4308a432019-07-24 10:37:55 -07001147static int blkcg_css_online(struct cgroup_subsys_state *css)
1148{
1149 struct blkcg *blkcg = css_to_blkcg(css);
1150 struct blkcg *parent = blkcg_parent(blkcg);
1151
1152 /*
1153 * blkcg_pin_online() is used to delay blkcg offline so that blkgs
1154 * don't go offline while cgwbs are still active on them. Pin the
1155 * parent so that offline always happens towards the root.
1156 */
1157 if (parent)
1158 blkcg_pin_online(parent);
1159 return 0;
1160}
1161
Tejun Heo5efd6112012-03-05 13:15:12 -08001162/**
1163 * blkcg_init_queue - initialize blkcg part of request queue
1164 * @q: request_queue to initialize
1165 *
Christoph Hellwigc62b37d2020-07-01 10:59:43 +02001166 * Called from blk_alloc_queue(). Responsible for initializing blkcg
Tejun Heo5efd6112012-03-05 13:15:12 -08001167 * part of new request_queue @q.
1168 *
1169 * RETURNS:
1170 * 0 on success, -errno on failure.
1171 */
1172int blkcg_init_queue(struct request_queue *q)
1173{
Jens Axboed708f0d2017-03-29 11:25:48 -06001174 struct blkcg_gq *new_blkg, *blkg;
1175 bool preloaded;
Tejun Heoec13b1d2015-05-22 17:13:19 -04001176 int ret;
Tejun Heo5efd6112012-03-05 13:15:12 -08001177
Jens Axboed708f0d2017-03-29 11:25:48 -06001178 new_blkg = blkg_alloc(&blkcg_root, q, GFP_KERNEL);
1179 if (!new_blkg)
1180 return -ENOMEM;
1181
1182 preloaded = !radix_tree_preload(GFP_KERNEL);
1183
Jiang Biaobea54882018-04-19 12:04:26 +08001184 /* Make sure the root blkg exists. */
Tejun Heoec13b1d2015-05-22 17:13:19 -04001185 rcu_read_lock();
Christoph Hellwig0d945c12018-11-15 12:17:28 -07001186 spin_lock_irq(&q->queue_lock);
Jens Axboed708f0d2017-03-29 11:25:48 -06001187 blkg = blkg_create(&blkcg_root, q, new_blkg);
Jiang Biao901932a2018-04-19 12:06:09 +08001188 if (IS_ERR(blkg))
1189 goto err_unlock;
1190 q->root_blkg = blkg;
Christoph Hellwig0d945c12018-11-15 12:17:28 -07001191 spin_unlock_irq(&q->queue_lock);
Tejun Heoec13b1d2015-05-22 17:13:19 -04001192 rcu_read_unlock();
1193
Jens Axboed708f0d2017-03-29 11:25:48 -06001194 if (preloaded)
1195 radix_tree_preload_end();
1196
Bart Van Assche556910e2021-06-17 17:44:44 -07001197 ret = blk_ioprio_init(q);
1198 if (ret)
1199 goto err_destroy_all;
1200
Tejun Heoec13b1d2015-05-22 17:13:19 -04001201 ret = blk_throtl_init(q);
Christoph Hellwig04be60b2018-11-14 17:02:12 +01001202 if (ret)
1203 goto err_destroy_all;
Yufen Yu27029b42020-08-10 22:21:16 -04001204
Yanfei Xu6f5ddde2021-09-15 15:24:26 +08001205 ret = blk_iolatency_init(q);
1206 if (ret) {
1207 blk_throtl_exit(q);
1208 goto err_destroy_all;
1209 }
1210
Christoph Hellwig04be60b2018-11-14 17:02:12 +01001211 return 0;
Jiang Biao901932a2018-04-19 12:06:09 +08001212
Christoph Hellwig04be60b2018-11-14 17:02:12 +01001213err_destroy_all:
Christoph Hellwig04be60b2018-11-14 17:02:12 +01001214 blkg_destroy_all(q);
Christoph Hellwig04be60b2018-11-14 17:02:12 +01001215 return ret;
Jiang Biao901932a2018-04-19 12:06:09 +08001216err_unlock:
Christoph Hellwig0d945c12018-11-15 12:17:28 -07001217 spin_unlock_irq(&q->queue_lock);
Jiang Biao901932a2018-04-19 12:06:09 +08001218 rcu_read_unlock();
1219 if (preloaded)
1220 radix_tree_preload_end();
1221 return PTR_ERR(blkg);
Tejun Heo5efd6112012-03-05 13:15:12 -08001222}
1223
1224/**
Tejun Heo5efd6112012-03-05 13:15:12 -08001225 * blkcg_exit_queue - exit and release blkcg part of request_queue
1226 * @q: request_queue being released
1227 *
Marcos Paulo de Souza7585d502019-01-25 00:01:42 -02001228 * Called from blk_exit_queue(). Responsible for exiting blkcg part.
Tejun Heo5efd6112012-03-05 13:15:12 -08001229 */
1230void blkcg_exit_queue(struct request_queue *q)
1231{
Tejun Heo3c96cb32012-04-13 13:11:34 -07001232 blkg_destroy_all(q);
Tejun Heo5efd6112012-03-05 13:15:12 -08001233 blk_throtl_exit(q);
1234}
1235
Tejun Heo69d7fde2015-08-18 14:55:36 -07001236static void blkcg_bind(struct cgroup_subsys_state *root_css)
1237{
1238 int i;
1239
1240 mutex_lock(&blkcg_pol_mutex);
1241
1242 for (i = 0; i < BLKCG_MAX_POLS; i++) {
1243 struct blkcg_policy *pol = blkcg_policy[i];
1244 struct blkcg *blkcg;
1245
1246 if (!pol || !pol->cpd_bind_fn)
1247 continue;
1248
1249 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node)
1250 if (blkcg->cpd[pol->plid])
1251 pol->cpd_bind_fn(blkcg->cpd[pol->plid]);
1252 }
1253 mutex_unlock(&blkcg_pol_mutex);
1254}
1255
Josef Bacikd09d8df2018-07-03 11:14:55 -04001256static void blkcg_exit(struct task_struct *tsk)
1257{
1258 if (tsk->throttle_queue)
1259 blk_put_queue(tsk->throttle_queue);
1260 tsk->throttle_queue = NULL;
1261}
1262
Tejun Heoc165b3e2015-08-18 14:55:29 -07001263struct cgroup_subsys io_cgrp_subsys = {
Tejun Heo92fb9742012-11-19 08:13:38 -08001264 .css_alloc = blkcg_css_alloc,
Tejun Heo4308a432019-07-24 10:37:55 -07001265 .css_online = blkcg_css_online,
Tejun Heo92fb9742012-11-19 08:13:38 -08001266 .css_offline = blkcg_css_offline,
1267 .css_free = blkcg_css_free,
Tejun Heof7331642019-11-07 11:18:03 -08001268 .css_rstat_flush = blkcg_rstat_flush,
Tejun Heo69d7fde2015-08-18 14:55:36 -07001269 .bind = blkcg_bind,
Tejun Heo2ee867dc2015-08-18 14:55:34 -07001270 .dfl_cftypes = blkcg_files,
Tejun Heo880f50e2015-08-18 14:55:30 -07001271 .legacy_cftypes = blkcg_legacy_files,
Tejun Heoc165b3e2015-08-18 14:55:29 -07001272 .legacy_name = "blkio",
Josef Bacikd09d8df2018-07-03 11:14:55 -04001273 .exit = blkcg_exit,
Tejun Heo1ced9532014-07-08 18:02:57 -04001274#ifdef CONFIG_MEMCG
1275 /*
1276 * This ensures that, if available, memcg is automatically enabled
1277 * together on the default hierarchy so that the owner cgroup can
1278 * be retrieved from writeback pages.
1279 */
1280 .depends_on = 1 << memory_cgrp_id,
1281#endif
Tejun Heo676f7c82012-04-01 12:09:55 -07001282};
Tejun Heoc165b3e2015-08-18 14:55:29 -07001283EXPORT_SYMBOL_GPL(io_cgrp_subsys);
Tejun Heo676f7c82012-04-01 12:09:55 -07001284
Tejun Heo8bd435b2012-04-13 13:11:28 -07001285/**
Tejun Heoa2b16932012-04-13 13:11:33 -07001286 * blkcg_activate_policy - activate a blkcg policy on a request_queue
1287 * @q: request_queue of interest
1288 * @pol: blkcg policy to activate
1289 *
1290 * Activate @pol on @q. Requires %GFP_KERNEL context. @q goes through
1291 * bypass mode to populate its blkgs with policy_data for @pol.
1292 *
1293 * Activation happens with @q bypassed, so nobody would be accessing blkgs
1294 * from IO path. Update of each blkg is protected by both queue and blkcg
1295 * locks so that holding either lock and testing blkcg_policy_enabled() is
1296 * always enough for dereferencing policy data.
1297 *
1298 * The caller is responsible for synchronizing [de]activations and policy
1299 * [un]registerations. Returns 0 on success, -errno on failure.
1300 */
1301int blkcg_activate_policy(struct request_queue *q,
Tejun Heo3c798392012-04-16 13:57:25 -07001302 const struct blkcg_policy *pol)
Tejun Heoa2b16932012-04-13 13:11:33 -07001303{
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001304 struct blkg_policy_data *pd_prealloc = NULL;
Tejun Heo9d179b82019-10-15 09:03:47 -07001305 struct blkcg_gq *blkg, *pinned_blkg = NULL;
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001306 int ret;
Tejun Heoa2b16932012-04-13 13:11:33 -07001307
1308 if (blkcg_policy_enabled(q, pol))
1309 return 0;
1310
Jens Axboe344e9ff2018-11-15 12:22:51 -07001311 if (queue_is_mq(q))
Jens Axboebd166ef2017-01-17 06:03:22 -07001312 blk_mq_freeze_queue(q);
Tejun Heo9d179b82019-10-15 09:03:47 -07001313retry:
Christoph Hellwig0d945c12018-11-15 12:17:28 -07001314 spin_lock_irq(&q->queue_lock);
Tejun Heoa2b16932012-04-13 13:11:33 -07001315
Tejun Heo9d179b82019-10-15 09:03:47 -07001316 /* blkg_list is pushed at the head, reverse walk to allocate parents first */
Tejun Heo71c81402019-06-13 15:30:40 -07001317 list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) {
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001318 struct blkg_policy_data *pd;
Tejun Heoa2b16932012-04-13 13:11:33 -07001319
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001320 if (blkg->pd[pol->plid])
1321 continue;
1322
Tejun Heo9d179b82019-10-15 09:03:47 -07001323 /* If prealloc matches, use it; otherwise try GFP_NOWAIT */
1324 if (blkg == pinned_blkg) {
1325 pd = pd_prealloc;
1326 pd_prealloc = NULL;
1327 } else {
1328 pd = pol->pd_alloc_fn(GFP_NOWAIT | __GFP_NOWARN, q,
1329 blkg->blkcg);
1330 }
1331
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001332 if (!pd) {
Tejun Heo9d179b82019-10-15 09:03:47 -07001333 /*
1334 * GFP_NOWAIT failed. Free the existing one and
1335 * prealloc for @blkg w/ GFP_KERNEL.
1336 */
1337 if (pinned_blkg)
1338 blkg_put(pinned_blkg);
1339 blkg_get(blkg);
1340 pinned_blkg = blkg;
1341
Christoph Hellwig0d945c12018-11-15 12:17:28 -07001342 spin_unlock_irq(&q->queue_lock);
Tejun Heo9d179b82019-10-15 09:03:47 -07001343
1344 if (pd_prealloc)
1345 pol->pd_free_fn(pd_prealloc);
1346 pd_prealloc = pol->pd_alloc_fn(GFP_KERNEL, q,
1347 blkg->blkcg);
1348 if (pd_prealloc)
1349 goto retry;
1350 else
1351 goto enomem;
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001352 }
Tejun Heoa2b16932012-04-13 13:11:33 -07001353
1354 blkg->pd[pol->plid] = pd;
1355 pd->blkg = blkg;
Tejun Heob276a872013-01-09 08:05:12 -08001356 pd->plid = pol->plid;
Tejun Heoa2b16932012-04-13 13:11:33 -07001357 }
1358
Tejun Heo9d179b82019-10-15 09:03:47 -07001359 /* all allocated, init in the same order */
1360 if (pol->pd_init_fn)
1361 list_for_each_entry_reverse(blkg, &q->blkg_list, q_node)
1362 pol->pd_init_fn(blkg->pd[pol->plid]);
1363
Tejun Heoa2b16932012-04-13 13:11:33 -07001364 __set_bit(pol->plid, q->blkcg_pols);
1365 ret = 0;
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001366
Christoph Hellwig0d945c12018-11-15 12:17:28 -07001367 spin_unlock_irq(&q->queue_lock);
Tejun Heo9d179b82019-10-15 09:03:47 -07001368out:
Jens Axboe344e9ff2018-11-15 12:22:51 -07001369 if (queue_is_mq(q))
Jens Axboebd166ef2017-01-17 06:03:22 -07001370 blk_mq_unfreeze_queue(q);
Tejun Heo9d179b82019-10-15 09:03:47 -07001371 if (pinned_blkg)
1372 blkg_put(pinned_blkg);
Tejun Heo001bea72015-08-18 14:55:11 -07001373 if (pd_prealloc)
1374 pol->pd_free_fn(pd_prealloc);
Tejun Heoa2b16932012-04-13 13:11:33 -07001375 return ret;
Tejun Heo9d179b82019-10-15 09:03:47 -07001376
1377enomem:
1378 /* alloc failed, nothing's initialized yet, free everything */
1379 spin_lock_irq(&q->queue_lock);
1380 list_for_each_entry(blkg, &q->blkg_list, q_node) {
Li Jinlin858560b2021-09-14 12:26:05 +08001381 struct blkcg *blkcg = blkg->blkcg;
1382
1383 spin_lock(&blkcg->lock);
Tejun Heo9d179b82019-10-15 09:03:47 -07001384 if (blkg->pd[pol->plid]) {
1385 pol->pd_free_fn(blkg->pd[pol->plid]);
1386 blkg->pd[pol->plid] = NULL;
1387 }
Li Jinlin858560b2021-09-14 12:26:05 +08001388 spin_unlock(&blkcg->lock);
Tejun Heo9d179b82019-10-15 09:03:47 -07001389 }
1390 spin_unlock_irq(&q->queue_lock);
1391 ret = -ENOMEM;
1392 goto out;
Tejun Heoa2b16932012-04-13 13:11:33 -07001393}
1394EXPORT_SYMBOL_GPL(blkcg_activate_policy);
1395
1396/**
1397 * blkcg_deactivate_policy - deactivate a blkcg policy on a request_queue
1398 * @q: request_queue of interest
1399 * @pol: blkcg policy to deactivate
1400 *
1401 * Deactivate @pol on @q. Follows the same synchronization rules as
1402 * blkcg_activate_policy().
1403 */
1404void blkcg_deactivate_policy(struct request_queue *q,
Tejun Heo3c798392012-04-16 13:57:25 -07001405 const struct blkcg_policy *pol)
Tejun Heoa2b16932012-04-13 13:11:33 -07001406{
Tejun Heo3c798392012-04-16 13:57:25 -07001407 struct blkcg_gq *blkg;
Tejun Heoa2b16932012-04-13 13:11:33 -07001408
1409 if (!blkcg_policy_enabled(q, pol))
1410 return;
1411
Jens Axboe344e9ff2018-11-15 12:22:51 -07001412 if (queue_is_mq(q))
Jens Axboebd166ef2017-01-17 06:03:22 -07001413 blk_mq_freeze_queue(q);
Jens Axboebd166ef2017-01-17 06:03:22 -07001414
Christoph Hellwig0d945c12018-11-15 12:17:28 -07001415 spin_lock_irq(&q->queue_lock);
Tejun Heoa2b16932012-04-13 13:11:33 -07001416
1417 __clear_bit(pol->plid, q->blkcg_pols);
1418
1419 list_for_each_entry(blkg, &q->blkg_list, q_node) {
Li Jinlin858560b2021-09-14 12:26:05 +08001420 struct blkcg *blkcg = blkg->blkcg;
1421
1422 spin_lock(&blkcg->lock);
Tejun Heo001bea72015-08-18 14:55:11 -07001423 if (blkg->pd[pol->plid]) {
Dennis Zhou (Facebook)6b065462018-08-31 16:22:42 -04001424 if (pol->pd_offline_fn)
Tejun Heoa9520cd2015-08-18 14:55:14 -07001425 pol->pd_offline_fn(blkg->pd[pol->plid]);
Tejun Heo001bea72015-08-18 14:55:11 -07001426 pol->pd_free_fn(blkg->pd[pol->plid]);
1427 blkg->pd[pol->plid] = NULL;
1428 }
Li Jinlin858560b2021-09-14 12:26:05 +08001429 spin_unlock(&blkcg->lock);
Tejun Heoa2b16932012-04-13 13:11:33 -07001430 }
1431
Christoph Hellwig0d945c12018-11-15 12:17:28 -07001432 spin_unlock_irq(&q->queue_lock);
Jens Axboebd166ef2017-01-17 06:03:22 -07001433
Jens Axboe344e9ff2018-11-15 12:22:51 -07001434 if (queue_is_mq(q))
Jens Axboebd166ef2017-01-17 06:03:22 -07001435 blk_mq_unfreeze_queue(q);
Tejun Heoa2b16932012-04-13 13:11:33 -07001436}
1437EXPORT_SYMBOL_GPL(blkcg_deactivate_policy);
1438
1439/**
Tejun Heo3c798392012-04-16 13:57:25 -07001440 * blkcg_policy_register - register a blkcg policy
1441 * @pol: blkcg policy to register
Tejun Heo8bd435b2012-04-13 13:11:28 -07001442 *
Tejun Heo3c798392012-04-16 13:57:25 -07001443 * Register @pol with blkcg core. Might sleep and @pol may be modified on
1444 * successful registration. Returns 0 on success and -errno on failure.
Tejun Heo8bd435b2012-04-13 13:11:28 -07001445 */
Jens Axboed5bf0292014-06-22 16:31:56 -06001446int blkcg_policy_register(struct blkcg_policy *pol)
Vivek Goyal3e252062009-12-04 10:36:42 -05001447{
Tejun Heo06b285b2015-07-09 16:39:50 -04001448 struct blkcg *blkcg;
Tejun Heo8bd435b2012-04-13 13:11:28 -07001449 int i, ret;
Tejun Heoe8989fa2012-03-05 13:15:20 -08001450
Tejun Heo838f13b2015-07-09 16:39:47 -04001451 mutex_lock(&blkcg_pol_register_mutex);
Tejun Heobc0d6502012-04-13 13:11:26 -07001452 mutex_lock(&blkcg_pol_mutex);
1453
Tejun Heo8bd435b2012-04-13 13:11:28 -07001454 /* find an empty slot */
1455 ret = -ENOSPC;
1456 for (i = 0; i < BLKCG_MAX_POLS; i++)
Tejun Heo3c798392012-04-16 13:57:25 -07001457 if (!blkcg_policy[i])
Tejun Heo8bd435b2012-04-13 13:11:28 -07001458 break;
Jens Axboe01c5f852018-09-11 10:59:53 -06001459 if (i >= BLKCG_MAX_POLS) {
1460 pr_warn("blkcg_policy_register: BLKCG_MAX_POLS too small\n");
Tejun Heo838f13b2015-07-09 16:39:47 -04001461 goto err_unlock;
Jens Axboe01c5f852018-09-11 10:59:53 -06001462 }
Tejun Heo035d10b2012-03-05 13:15:04 -08001463
weiping zhange8401072017-10-17 23:56:21 +08001464 /* Make sure cpd/pd_alloc_fn and cpd/pd_free_fn in pairs */
1465 if ((!pol->cpd_alloc_fn ^ !pol->cpd_free_fn) ||
1466 (!pol->pd_alloc_fn ^ !pol->pd_free_fn))
1467 goto err_unlock;
1468
Tejun Heo06b285b2015-07-09 16:39:50 -04001469 /* register @pol */
Tejun Heo3c798392012-04-16 13:57:25 -07001470 pol->plid = i;
Tejun Heo06b285b2015-07-09 16:39:50 -04001471 blkcg_policy[pol->plid] = pol;
1472
1473 /* allocate and install cpd's */
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001474 if (pol->cpd_alloc_fn) {
Tejun Heo06b285b2015-07-09 16:39:50 -04001475 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1476 struct blkcg_policy_data *cpd;
1477
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001478 cpd = pol->cpd_alloc_fn(GFP_KERNEL);
Bart Van Asschebbb427e2016-09-29 08:33:30 -07001479 if (!cpd)
Tejun Heo06b285b2015-07-09 16:39:50 -04001480 goto err_free_cpds;
Tejun Heo06b285b2015-07-09 16:39:50 -04001481
Tejun Heo81437642015-08-18 14:55:15 -07001482 blkcg->cpd[pol->plid] = cpd;
1483 cpd->blkcg = blkcg;
Tejun Heo06b285b2015-07-09 16:39:50 -04001484 cpd->plid = pol->plid;
Tejun Heo86a5bba2019-08-28 15:05:52 -07001485 if (pol->cpd_init_fn)
1486 pol->cpd_init_fn(cpd);
Tejun Heo06b285b2015-07-09 16:39:50 -04001487 }
1488 }
1489
Tejun Heo838f13b2015-07-09 16:39:47 -04001490 mutex_unlock(&blkcg_pol_mutex);
Tejun Heo8bd435b2012-04-13 13:11:28 -07001491
Tejun Heo8bd435b2012-04-13 13:11:28 -07001492 /* everything is in place, add intf files for the new policy */
Tejun Heo2ee867dc2015-08-18 14:55:34 -07001493 if (pol->dfl_cftypes)
1494 WARN_ON(cgroup_add_dfl_cftypes(&io_cgrp_subsys,
1495 pol->dfl_cftypes));
Tejun Heo880f50e2015-08-18 14:55:30 -07001496 if (pol->legacy_cftypes)
Tejun Heoc165b3e2015-08-18 14:55:29 -07001497 WARN_ON(cgroup_add_legacy_cftypes(&io_cgrp_subsys,
Tejun Heo880f50e2015-08-18 14:55:30 -07001498 pol->legacy_cftypes));
Tejun Heo838f13b2015-07-09 16:39:47 -04001499 mutex_unlock(&blkcg_pol_register_mutex);
1500 return 0;
1501
Tejun Heo06b285b2015-07-09 16:39:50 -04001502err_free_cpds:
weiping zhang58a9edc2017-10-10 22:53:46 +08001503 if (pol->cpd_free_fn) {
Tejun Heo06b285b2015-07-09 16:39:50 -04001504 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001505 if (blkcg->cpd[pol->plid]) {
1506 pol->cpd_free_fn(blkcg->cpd[pol->plid]);
1507 blkcg->cpd[pol->plid] = NULL;
1508 }
Tejun Heo06b285b2015-07-09 16:39:50 -04001509 }
1510 }
1511 blkcg_policy[pol->plid] = NULL;
Tejun Heo838f13b2015-07-09 16:39:47 -04001512err_unlock:
Tejun Heobc0d6502012-04-13 13:11:26 -07001513 mutex_unlock(&blkcg_pol_mutex);
Tejun Heo838f13b2015-07-09 16:39:47 -04001514 mutex_unlock(&blkcg_pol_register_mutex);
Tejun Heo8bd435b2012-04-13 13:11:28 -07001515 return ret;
Vivek Goyal3e252062009-12-04 10:36:42 -05001516}
Tejun Heo3c798392012-04-16 13:57:25 -07001517EXPORT_SYMBOL_GPL(blkcg_policy_register);
Vivek Goyal3e252062009-12-04 10:36:42 -05001518
Tejun Heo8bd435b2012-04-13 13:11:28 -07001519/**
Tejun Heo3c798392012-04-16 13:57:25 -07001520 * blkcg_policy_unregister - unregister a blkcg policy
1521 * @pol: blkcg policy to unregister
Tejun Heo8bd435b2012-04-13 13:11:28 -07001522 *
Tejun Heo3c798392012-04-16 13:57:25 -07001523 * Undo blkcg_policy_register(@pol). Might sleep.
Tejun Heo8bd435b2012-04-13 13:11:28 -07001524 */
Tejun Heo3c798392012-04-16 13:57:25 -07001525void blkcg_policy_unregister(struct blkcg_policy *pol)
Vivek Goyal3e252062009-12-04 10:36:42 -05001526{
Tejun Heo06b285b2015-07-09 16:39:50 -04001527 struct blkcg *blkcg;
1528
Tejun Heo838f13b2015-07-09 16:39:47 -04001529 mutex_lock(&blkcg_pol_register_mutex);
Tejun Heobc0d6502012-04-13 13:11:26 -07001530
Tejun Heo3c798392012-04-16 13:57:25 -07001531 if (WARN_ON(blkcg_policy[pol->plid] != pol))
Tejun Heo8bd435b2012-04-13 13:11:28 -07001532 goto out_unlock;
1533
1534 /* kill the intf files first */
Tejun Heo2ee867dc2015-08-18 14:55:34 -07001535 if (pol->dfl_cftypes)
1536 cgroup_rm_cftypes(pol->dfl_cftypes);
Tejun Heo880f50e2015-08-18 14:55:30 -07001537 if (pol->legacy_cftypes)
1538 cgroup_rm_cftypes(pol->legacy_cftypes);
Tejun Heo44ea53d2012-04-01 14:38:43 -07001539
Tejun Heo06b285b2015-07-09 16:39:50 -04001540 /* remove cpds and unregister */
Tejun Heo838f13b2015-07-09 16:39:47 -04001541 mutex_lock(&blkcg_pol_mutex);
Tejun Heo06b285b2015-07-09 16:39:50 -04001542
weiping zhang58a9edc2017-10-10 22:53:46 +08001543 if (pol->cpd_free_fn) {
Tejun Heo06b285b2015-07-09 16:39:50 -04001544 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001545 if (blkcg->cpd[pol->plid]) {
1546 pol->cpd_free_fn(blkcg->cpd[pol->plid]);
1547 blkcg->cpd[pol->plid] = NULL;
1548 }
Tejun Heo06b285b2015-07-09 16:39:50 -04001549 }
1550 }
Tejun Heo3c798392012-04-16 13:57:25 -07001551 blkcg_policy[pol->plid] = NULL;
Tejun Heo06b285b2015-07-09 16:39:50 -04001552
Tejun Heobc0d6502012-04-13 13:11:26 -07001553 mutex_unlock(&blkcg_pol_mutex);
Tejun Heo838f13b2015-07-09 16:39:47 -04001554out_unlock:
1555 mutex_unlock(&blkcg_pol_register_mutex);
Vivek Goyal3e252062009-12-04 10:36:42 -05001556}
Tejun Heo3c798392012-04-16 13:57:25 -07001557EXPORT_SYMBOL_GPL(blkcg_policy_unregister);
Josef Bacik903d23f2018-07-03 11:14:52 -04001558
Tejun Heod3f77df2019-06-27 13:39:52 -07001559bool __blkcg_punt_bio_submit(struct bio *bio)
1560{
1561 struct blkcg_gq *blkg = bio->bi_blkg;
1562
1563 /* consume the flag first */
1564 bio->bi_opf &= ~REQ_CGROUP_PUNT;
1565
1566 /* never bounce for the root cgroup */
1567 if (!blkg->parent)
1568 return false;
1569
1570 spin_lock_bh(&blkg->async_bio_lock);
1571 bio_list_add(&blkg->async_bios, bio);
1572 spin_unlock_bh(&blkg->async_bio_lock);
1573
1574 queue_work(blkcg_punt_bio_wq, &blkg->async_bio_work);
1575 return true;
1576}
1577
Josef Bacikd09d8df2018-07-03 11:14:55 -04001578/*
1579 * Scale the accumulated delay based on how long it has been since we updated
1580 * the delay. We only call this when we are adding delay, in case it's been a
1581 * while since we added delay, and when we are checking to see if we need to
1582 * delay a task, to account for any delays that may have occurred.
1583 */
1584static void blkcg_scale_delay(struct blkcg_gq *blkg, u64 now)
1585{
1586 u64 old = atomic64_read(&blkg->delay_start);
1587
Tejun Heo54c52e12020-04-13 12:27:55 -04001588 /* negative use_delay means no scaling, see blkcg_set_delay() */
1589 if (atomic_read(&blkg->use_delay) < 0)
1590 return;
1591
Josef Bacikd09d8df2018-07-03 11:14:55 -04001592 /*
1593 * We only want to scale down every second. The idea here is that we
1594 * want to delay people for min(delay_nsec, NSEC_PER_SEC) in a certain
1595 * time window. We only want to throttle tasks for recent delay that
1596 * has occurred, in 1 second time windows since that's the maximum
1597 * things can be throttled. We save the current delay window in
1598 * blkg->last_delay so we know what amount is still left to be charged
1599 * to the blkg from this point onward. blkg->last_use keeps track of
1600 * the use_delay counter. The idea is if we're unthrottling the blkg we
1601 * are ok with whatever is happening now, and we can take away more of
1602 * the accumulated delay as we've already throttled enough that
1603 * everybody is happy with their IO latencies.
1604 */
1605 if (time_before64(old + NSEC_PER_SEC, now) &&
1606 atomic64_cmpxchg(&blkg->delay_start, old, now) == old) {
1607 u64 cur = atomic64_read(&blkg->delay_nsec);
1608 u64 sub = min_t(u64, blkg->last_delay, now - old);
1609 int cur_use = atomic_read(&blkg->use_delay);
1610
1611 /*
1612 * We've been unthrottled, subtract a larger chunk of our
1613 * accumulated delay.
1614 */
1615 if (cur_use < blkg->last_use)
1616 sub = max_t(u64, sub, blkg->last_delay >> 1);
1617
1618 /*
1619 * This shouldn't happen, but handle it anyway. Our delay_nsec
1620 * should only ever be growing except here where we subtract out
1621 * min(last_delay, 1 second), but lord knows bugs happen and I'd
1622 * rather not end up with negative numbers.
1623 */
1624 if (unlikely(cur < sub)) {
1625 atomic64_set(&blkg->delay_nsec, 0);
1626 blkg->last_delay = 0;
1627 } else {
1628 atomic64_sub(sub, &blkg->delay_nsec);
1629 blkg->last_delay = cur - sub;
1630 }
1631 blkg->last_use = cur_use;
1632 }
1633}
1634
1635/*
1636 * This is called when we want to actually walk up the hierarchy and check to
1637 * see if we need to throttle, and then actually throttle if there is some
1638 * accumulated delay. This should only be called upon return to user space so
1639 * we're not holding some lock that would induce a priority inversion.
1640 */
1641static void blkcg_maybe_throttle_blkg(struct blkcg_gq *blkg, bool use_memdelay)
1642{
Josef Bacikfd112c72019-07-09 14:41:29 -07001643 unsigned long pflags;
Tejun Heo5160a5a2020-09-01 14:52:52 -04001644 bool clamp;
Josef Bacikd09d8df2018-07-03 11:14:55 -04001645 u64 now = ktime_to_ns(ktime_get());
1646 u64 exp;
1647 u64 delay_nsec = 0;
1648 int tok;
1649
1650 while (blkg->parent) {
Tejun Heo5160a5a2020-09-01 14:52:52 -04001651 int use_delay = atomic_read(&blkg->use_delay);
1652
1653 if (use_delay) {
1654 u64 this_delay;
1655
Josef Bacikd09d8df2018-07-03 11:14:55 -04001656 blkcg_scale_delay(blkg, now);
Tejun Heo5160a5a2020-09-01 14:52:52 -04001657 this_delay = atomic64_read(&blkg->delay_nsec);
1658 if (this_delay > delay_nsec) {
1659 delay_nsec = this_delay;
1660 clamp = use_delay > 0;
1661 }
Josef Bacikd09d8df2018-07-03 11:14:55 -04001662 }
1663 blkg = blkg->parent;
1664 }
1665
1666 if (!delay_nsec)
1667 return;
1668
1669 /*
1670 * Let's not sleep for all eternity if we've amassed a huge delay.
1671 * Swapping or metadata IO can accumulate 10's of seconds worth of
1672 * delay, and we want userspace to be able to do _something_ so cap the
Tejun Heo5160a5a2020-09-01 14:52:52 -04001673 * delays at 0.25s. If there's 10's of seconds worth of delay then the
1674 * tasks will be delayed for 0.25 second for every syscall. If
1675 * blkcg_set_delay() was used as indicated by negative use_delay, the
1676 * caller is responsible for regulating the range.
Josef Bacikd09d8df2018-07-03 11:14:55 -04001677 */
Tejun Heo5160a5a2020-09-01 14:52:52 -04001678 if (clamp)
1679 delay_nsec = min_t(u64, delay_nsec, 250 * NSEC_PER_MSEC);
Josef Bacikd09d8df2018-07-03 11:14:55 -04001680
Josef Bacikfd112c72019-07-09 14:41:29 -07001681 if (use_memdelay)
1682 psi_memstall_enter(&pflags);
Josef Bacikd09d8df2018-07-03 11:14:55 -04001683
1684 exp = ktime_add_ns(now, delay_nsec);
1685 tok = io_schedule_prepare();
1686 do {
1687 __set_current_state(TASK_KILLABLE);
1688 if (!schedule_hrtimeout(&exp, HRTIMER_MODE_ABS))
1689 break;
1690 } while (!fatal_signal_pending(current));
1691 io_schedule_finish(tok);
Josef Bacikfd112c72019-07-09 14:41:29 -07001692
1693 if (use_memdelay)
1694 psi_memstall_leave(&pflags);
Josef Bacikd09d8df2018-07-03 11:14:55 -04001695}
1696
1697/**
1698 * blkcg_maybe_throttle_current - throttle the current task if it has been marked
1699 *
1700 * This is only called if we've been marked with set_notify_resume(). Obviously
1701 * we can be set_notify_resume() for reasons other than blkcg throttling, so we
1702 * check to see if current->throttle_queue is set and if not this doesn't do
1703 * anything. This should only ever be called by the resume code, it's not meant
1704 * to be called by people willy-nilly as it will actually do the work to
1705 * throttle the task if it is setup for throttling.
1706 */
1707void blkcg_maybe_throttle_current(void)
1708{
1709 struct request_queue *q = current->throttle_queue;
1710 struct cgroup_subsys_state *css;
1711 struct blkcg *blkcg;
1712 struct blkcg_gq *blkg;
1713 bool use_memdelay = current->use_memdelay;
1714
1715 if (!q)
1716 return;
1717
1718 current->throttle_queue = NULL;
1719 current->use_memdelay = false;
1720
1721 rcu_read_lock();
1722 css = kthread_blkcg();
1723 if (css)
1724 blkcg = css_to_blkcg(css);
1725 else
1726 blkcg = css_to_blkcg(task_css(current, io_cgrp_id));
1727
1728 if (!blkcg)
1729 goto out;
1730 blkg = blkg_lookup(blkcg, q);
1731 if (!blkg)
1732 goto out;
Dennis Zhou7754f662018-12-05 12:10:39 -05001733 if (!blkg_tryget(blkg))
Josef Bacikd09d8df2018-07-03 11:14:55 -04001734 goto out;
1735 rcu_read_unlock();
Josef Bacikd09d8df2018-07-03 11:14:55 -04001736
1737 blkcg_maybe_throttle_blkg(blkg, use_memdelay);
1738 blkg_put(blkg);
Josef Bacikcc7ecc252018-07-31 12:39:03 -04001739 blk_put_queue(q);
Josef Bacikd09d8df2018-07-03 11:14:55 -04001740 return;
1741out:
1742 rcu_read_unlock();
1743 blk_put_queue(q);
1744}
Josef Bacikd09d8df2018-07-03 11:14:55 -04001745
1746/**
1747 * blkcg_schedule_throttle - this task needs to check for throttling
Bart Van Assche537d71b2019-03-20 13:18:45 -07001748 * @q: the request queue IO was submitted on
1749 * @use_memdelay: do we charge this to memory delay for PSI
Josef Bacikd09d8df2018-07-03 11:14:55 -04001750 *
1751 * This is called by the IO controller when we know there's delay accumulated
1752 * for the blkg for this task. We do not pass the blkg because there are places
1753 * we call this that may not have that information, the swapping code for
1754 * instance will only have a request_queue at that point. This set's the
1755 * notify_resume for the task to check and see if it requires throttling before
1756 * returning to user space.
1757 *
1758 * We will only schedule once per syscall. You can call this over and over
1759 * again and it will only do the check once upon return to user space, and only
1760 * throttle once. If the task needs to be throttled again it'll need to be
1761 * re-set at the next time we see the task.
1762 */
1763void blkcg_schedule_throttle(struct request_queue *q, bool use_memdelay)
1764{
1765 if (unlikely(current->flags & PF_KTHREAD))
1766 return;
1767
Chunguang Xu49d18222021-01-25 13:05:28 +08001768 if (current->throttle_queue != q) {
1769 if (!blk_get_queue(q))
1770 return;
Josef Bacikd09d8df2018-07-03 11:14:55 -04001771
Chunguang Xu49d18222021-01-25 13:05:28 +08001772 if (current->throttle_queue)
1773 blk_put_queue(current->throttle_queue);
1774 current->throttle_queue = q;
1775 }
1776
Josef Bacikd09d8df2018-07-03 11:14:55 -04001777 if (use_memdelay)
1778 current->use_memdelay = use_memdelay;
1779 set_notify_resume(current);
1780}
Josef Bacikd09d8df2018-07-03 11:14:55 -04001781
1782/**
1783 * blkcg_add_delay - add delay to this blkg
Bart Van Assche537d71b2019-03-20 13:18:45 -07001784 * @blkg: blkg of interest
1785 * @now: the current time in nanoseconds
1786 * @delta: how many nanoseconds of delay to add
Josef Bacikd09d8df2018-07-03 11:14:55 -04001787 *
1788 * Charge @delta to the blkg's current delay accumulation. This is used to
1789 * throttle tasks if an IO controller thinks we need more throttling.
1790 */
1791void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta)
1792{
Tejun Heo54c52e12020-04-13 12:27:55 -04001793 if (WARN_ON_ONCE(atomic_read(&blkg->use_delay) < 0))
1794 return;
Josef Bacikd09d8df2018-07-03 11:14:55 -04001795 blkcg_scale_delay(blkg, now);
1796 atomic64_add(delta, &blkg->delay_nsec);
1797}
Josef Bacikd09d8df2018-07-03 11:14:55 -04001798
Christoph Hellwig28fc5912020-06-27 09:31:51 +02001799/**
1800 * blkg_tryget_closest - try and get a blkg ref on the closet blkg
Christoph Hellwig13c78632020-06-27 09:31:54 +02001801 * @bio: target bio
1802 * @css: target css
Christoph Hellwig28fc5912020-06-27 09:31:51 +02001803 *
Christoph Hellwig13c78632020-06-27 09:31:54 +02001804 * As the failure mode here is to walk up the blkg tree, this ensure that the
1805 * blkg->parent pointers are always valid. This returns the blkg that it ended
1806 * up taking a reference on or %NULL if no reference was taken.
Christoph Hellwig28fc5912020-06-27 09:31:51 +02001807 */
Christoph Hellwig13c78632020-06-27 09:31:54 +02001808static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio,
1809 struct cgroup_subsys_state *css)
Christoph Hellwig28fc5912020-06-27 09:31:51 +02001810{
Christoph Hellwig13c78632020-06-27 09:31:54 +02001811 struct blkcg_gq *blkg, *ret_blkg = NULL;
Christoph Hellwig28fc5912020-06-27 09:31:51 +02001812
Christoph Hellwig13c78632020-06-27 09:31:54 +02001813 rcu_read_lock();
Christoph Hellwig309dca302021-01-24 11:02:34 +01001814 blkg = blkg_lookup_create(css_to_blkcg(css),
Pavel Begunkoved6cdde2021-10-14 15:03:30 +01001815 bdev_get_queue(bio->bi_bdev));
Christoph Hellwig28fc5912020-06-27 09:31:51 +02001816 while (blkg) {
1817 if (blkg_tryget(blkg)) {
1818 ret_blkg = blkg;
1819 break;
1820 }
1821 blkg = blkg->parent;
1822 }
Christoph Hellwig13c78632020-06-27 09:31:54 +02001823 rcu_read_unlock();
Christoph Hellwig28fc5912020-06-27 09:31:51 +02001824
1825 return ret_blkg;
1826}
1827
1828/**
1829 * bio_associate_blkg_from_css - associate a bio with a specified css
1830 * @bio: target bio
1831 * @css: target css
1832 *
1833 * Associate @bio with the blkg found by combining the css's blkg and the
1834 * request_queue of the @bio. An association failure is handled by walking up
1835 * the blkg tree. Therefore, the blkg associated can be anything between @blkg
1836 * and q->root_blkg. This situation only happens when a cgroup is dying and
1837 * then the remaining bios will spill to the closest alive blkg.
1838 *
1839 * A reference will be taken on the blkg and will be released when @bio is
1840 * freed.
1841 */
1842void bio_associate_blkg_from_css(struct bio *bio,
1843 struct cgroup_subsys_state *css)
1844{
Christoph Hellwig28fc5912020-06-27 09:31:51 +02001845 if (bio->bi_blkg)
1846 blkg_put(bio->bi_blkg);
1847
Christoph Hellwiga5b97522020-06-27 09:31:53 +02001848 if (css && css->parent) {
Christoph Hellwig13c78632020-06-27 09:31:54 +02001849 bio->bi_blkg = blkg_tryget_closest(bio, css);
Christoph Hellwiga5b97522020-06-27 09:31:53 +02001850 } else {
Pavel Begunkoved6cdde2021-10-14 15:03:30 +01001851 blkg_get(bdev_get_queue(bio->bi_bdev)->root_blkg);
1852 bio->bi_blkg = bdev_get_queue(bio->bi_bdev)->root_blkg;
Christoph Hellwiga5b97522020-06-27 09:31:53 +02001853 }
Christoph Hellwig28fc5912020-06-27 09:31:51 +02001854}
1855EXPORT_SYMBOL_GPL(bio_associate_blkg_from_css);
1856
1857/**
1858 * bio_associate_blkg - associate a bio with a blkg
1859 * @bio: target bio
1860 *
1861 * Associate @bio with the blkg found from the bio's css and request_queue.
1862 * If one is not found, bio_lookup_blkg() creates the blkg. If a blkg is
1863 * already associated, the css is reused and association redone as the
1864 * request_queue may have changed.
1865 */
1866void bio_associate_blkg(struct bio *bio)
1867{
1868 struct cgroup_subsys_state *css;
1869
1870 rcu_read_lock();
1871
1872 if (bio->bi_blkg)
1873 css = &bio_blkcg(bio)->css;
1874 else
1875 css = blkcg_css();
1876
1877 bio_associate_blkg_from_css(bio, css);
1878
1879 rcu_read_unlock();
1880}
1881EXPORT_SYMBOL_GPL(bio_associate_blkg);
1882
1883/**
1884 * bio_clone_blkg_association - clone blkg association from src to dst bio
1885 * @dst: destination bio
1886 * @src: source bio
1887 */
1888void bio_clone_blkg_association(struct bio *dst, struct bio *src)
1889{
1890 if (src->bi_blkg) {
1891 if (dst->bi_blkg)
1892 blkg_put(dst->bi_blkg);
1893 blkg_get(src->bi_blkg);
1894 dst->bi_blkg = src->bi_blkg;
1895 }
1896}
1897EXPORT_SYMBOL_GPL(bio_clone_blkg_association);
1898
Christoph Hellwigdb18a532020-06-27 09:31:58 +02001899static int blk_cgroup_io_type(struct bio *bio)
1900{
1901 if (op_is_discard(bio->bi_opf))
1902 return BLKG_IOSTAT_DISCARD;
1903 if (op_is_write(bio->bi_opf))
1904 return BLKG_IOSTAT_WRITE;
1905 return BLKG_IOSTAT_READ;
1906}
1907
1908void blk_cgroup_bio_start(struct bio *bio)
1909{
1910 int rwd = blk_cgroup_io_type(bio), cpu;
1911 struct blkg_iostat_set *bis;
Tejun Heo3c08b092021-10-14 13:20:22 -10001912 unsigned long flags;
Christoph Hellwigdb18a532020-06-27 09:31:58 +02001913
1914 cpu = get_cpu();
1915 bis = per_cpu_ptr(bio->bi_blkg->iostat_cpu, cpu);
Tejun Heo3c08b092021-10-14 13:20:22 -10001916 flags = u64_stats_update_begin_irqsave(&bis->sync);
Christoph Hellwigdb18a532020-06-27 09:31:58 +02001917
1918 /*
1919 * If the bio is flagged with BIO_CGROUP_ACCT it means this is a split
1920 * bio and we would have already accounted for the size of the bio.
1921 */
1922 if (!bio_flagged(bio, BIO_CGROUP_ACCT)) {
1923 bio_set_flag(bio, BIO_CGROUP_ACCT);
Colin Ian King0b8cc252020-06-30 16:54:41 +01001924 bis->cur.bytes[rwd] += bio->bi_iter.bi_size;
Christoph Hellwigdb18a532020-06-27 09:31:58 +02001925 }
1926 bis->cur.ios[rwd]++;
1927
Tejun Heo3c08b092021-10-14 13:20:22 -10001928 u64_stats_update_end_irqrestore(&bis->sync, flags);
Christoph Hellwigdb18a532020-06-27 09:31:58 +02001929 if (cgroup_subsys_on_dfl(io_cgrp_subsys))
1930 cgroup_rstat_updated(bio->bi_blkg->blkcg->css.cgroup, cpu);
1931 put_cpu();
1932}
1933
Tejun Heod3f77df2019-06-27 13:39:52 -07001934static int __init blkcg_init(void)
1935{
1936 blkcg_punt_bio_wq = alloc_workqueue("blkcg_punt_bio",
1937 WQ_MEM_RECLAIM | WQ_FREEZABLE |
1938 WQ_UNBOUND | WQ_SYSFS, 0);
1939 if (!blkcg_punt_bio_wq)
1940 return -ENOMEM;
1941 return 0;
1942}
1943subsys_initcall(blkcg_init);
1944
Josef Bacik903d23f2018-07-03 11:14:52 -04001945module_param(blkcg_debug_stats, bool, 0644);
1946MODULE_PARM_DESC(blkcg_debug_stats, "True if you want debug stats, false if not");