blob: 491df0fa0835ed3cb6c8519234826284bc129f96 [file] [log] [blame]
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001/*
2 * Copyright (C) 2015 IT University of Copenhagen (rrpc.c)
3 * Copyright (C) 2016 CNEX Labs
4 * Initial release: Javier Gonzalez <javier@cnexlabs.com>
5 * Matias Bjorling <matias@cnexlabs.com>
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License version
9 * 2 as published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * General Public License for more details.
15 *
16 * Implementation of a physical block-device target for Open-channel SSDs.
17 *
18 * pblk-init.c - pblk's initialization.
19 */
20
21#include "pblk.h"
22
Wei Yongjun21ff1392018-06-05 09:15:12 +000023static unsigned int write_buffer_size;
Marcin Dziegielewski4a828882018-06-01 16:41:10 +020024
25module_param(write_buffer_size, uint, 0644);
26MODULE_PARM_DESC(write_buffer_size, "number of entries in a write buffer");
27
Javier Gonzálezb84ae4a82017-10-13 14:46:07 +020028static struct kmem_cache *pblk_ws_cache, *pblk_rec_cache, *pblk_g_rq_cache,
Javier Gonzáleze72ec1d2017-10-13 14:46:09 +020029 *pblk_w_rq_cache;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +020030static DECLARE_RWSEM(pblk_lock);
Kent Overstreetb906bbb2018-05-20 18:25:50 -040031struct bio_set pblk_bio_set;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +020032
33static int pblk_rw_io(struct request_queue *q, struct pblk *pblk,
34 struct bio *bio)
35{
36 int ret;
37
38 /* Read requests must be <= 256kb due to NVMe's 64 bit completion bitmap
39 * constraint. Writes can be of arbitrary size.
40 */
41 if (bio_data_dir(bio) == READ) {
NeilBrownaf67c312017-06-18 14:38:57 +100042 blk_queue_split(q, &bio);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +020043 ret = pblk_submit_read(pblk, bio);
44 if (ret == NVM_IO_DONE && bio_flagged(bio, BIO_CLONED))
45 bio_put(bio);
46
47 return ret;
48 }
49
50 /* Prevent deadlock in the case of a modest LUN configuration and large
51 * user I/Os. Unless stalled, the rate limiter leaves at least 256KB
52 * available for user I/O.
53 */
Javier Gonzálezda67e68f2017-10-13 14:46:05 +020054 if (pblk_get_secs(bio) > pblk_rl_max_io(&pblk->rl))
NeilBrownaf67c312017-06-18 14:38:57 +100055 blk_queue_split(q, &bio);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +020056
57 return pblk_write_to_cache(pblk, bio, PBLK_IOTYPE_USER);
58}
59
60static blk_qc_t pblk_make_rq(struct request_queue *q, struct bio *bio)
61{
62 struct pblk *pblk = q->queuedata;
63
64 if (bio_op(bio) == REQ_OP_DISCARD) {
65 pblk_discard(pblk, bio);
66 if (!(bio->bi_opf & REQ_PREFLUSH)) {
67 bio_endio(bio);
68 return BLK_QC_T_NONE;
69 }
70 }
71
72 switch (pblk_rw_io(q, pblk, bio)) {
73 case NVM_IO_ERR:
74 bio_io_error(bio);
75 break;
76 case NVM_IO_DONE:
77 bio_endio(bio);
78 break;
79 }
80
81 return BLK_QC_T_NONE;
82}
83
Hans Holmbergc5586192017-10-13 14:46:40 +020084static size_t pblk_trans_map_size(struct pblk *pblk)
85{
86 int entry_size = 8;
87
Javier Gonzálezbb845ae42018-03-30 00:05:19 +020088 if (pblk->addrf_len < 32)
Hans Holmbergc5586192017-10-13 14:46:40 +020089 entry_size = 4;
90
91 return entry_size * pblk->rl.nr_secs;
92}
93
94#ifdef CONFIG_NVM_DEBUG
95static u32 pblk_l2p_crc(struct pblk *pblk)
96{
97 size_t map_size;
98 u32 crc = ~(u32)0;
99
100 map_size = pblk_trans_map_size(pblk);
101 crc = crc32_le(crc, pblk->trans_map, map_size);
102 return crc;
103}
104#endif
105
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200106static void pblk_l2p_free(struct pblk *pblk)
107{
108 vfree(pblk->trans_map);
109}
110
Javier González43d47122018-03-30 00:05:09 +0200111static int pblk_l2p_recover(struct pblk *pblk, bool factory_init)
112{
113 struct pblk_line *line = NULL;
114
115 if (factory_init) {
116 pblk_setup_uuid(pblk);
117 } else {
118 line = pblk_recov_l2p(pblk);
119 if (IS_ERR(line)) {
120 pr_err("pblk: could not recover l2p table\n");
121 return -EFAULT;
122 }
123 }
124
125#ifdef CONFIG_NVM_DEBUG
126 pr_info("pblk init: L2P CRC: %x\n", pblk_l2p_crc(pblk));
127#endif
128
129 /* Free full lines directly as GC has not been started yet */
130 pblk_gc_free_full_lines(pblk);
131
132 if (!line) {
133 /* Configure next line for user data */
134 line = pblk_line_get_first_data(pblk);
Javier González1d8b33e2018-06-01 15:04:16 +0200135 if (!line)
Javier González43d47122018-03-30 00:05:09 +0200136 return -EFAULT;
Javier González43d47122018-03-30 00:05:09 +0200137 }
138
139 return 0;
140}
141
142static int pblk_l2p_init(struct pblk *pblk, bool factory_init)
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200143{
144 sector_t i;
145 struct ppa_addr ppa;
Hans Holmbergc5586192017-10-13 14:46:40 +0200146 size_t map_size;
Javier González1d8b33e2018-06-01 15:04:16 +0200147 int ret = 0;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200148
Hans Holmbergc5586192017-10-13 14:46:40 +0200149 map_size = pblk_trans_map_size(pblk);
150 pblk->trans_map = vmalloc(map_size);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200151 if (!pblk->trans_map)
152 return -ENOMEM;
153
154 pblk_ppa_set_empty(&ppa);
155
156 for (i = 0; i < pblk->rl.nr_secs; i++)
157 pblk_trans_map_set(pblk, i, ppa);
158
Javier González1d8b33e2018-06-01 15:04:16 +0200159 ret = pblk_l2p_recover(pblk, factory_init);
160 if (ret)
161 vfree(pblk->trans_map);
162
163 return ret;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200164}
165
166static void pblk_rwb_free(struct pblk *pblk)
167{
168 if (pblk_rb_tear_down_check(&pblk->rwb))
169 pr_err("pblk: write buffer error on tear down\n");
170
171 pblk_rb_data_free(&pblk->rwb);
172 vfree(pblk_rb_entries_ref(&pblk->rwb));
173}
174
175static int pblk_rwb_init(struct pblk *pblk)
176{
177 struct nvm_tgt_dev *dev = pblk->dev;
178 struct nvm_geo *geo = &dev->geo;
179 struct pblk_rb_entry *entries;
Marcin Dziegielewski4a828882018-06-01 16:41:10 +0200180 unsigned long nr_entries, buffer_size;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200181 unsigned int power_size, power_seg_sz;
182
Marcin Dziegielewski4a828882018-06-01 16:41:10 +0200183 if (write_buffer_size && (write_buffer_size > pblk->pgs_in_buffer))
184 buffer_size = write_buffer_size;
185 else
186 buffer_size = pblk->pgs_in_buffer;
187
188 nr_entries = pblk_rb_calculate_size(buffer_size);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200189
190 entries = vzalloc(nr_entries * sizeof(struct pblk_rb_entry));
191 if (!entries)
192 return -ENOMEM;
193
194 power_size = get_count_order(nr_entries);
Javier Gonzáleze46f4e42018-03-30 00:05:10 +0200195 power_seg_sz = get_count_order(geo->csecs);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200196
197 return pblk_rb_init(&pblk->rwb, entries, power_size, power_seg_sz);
198}
199
200/* Minimum pages needed within a lun */
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200201#define ADDR_POOL_SIZE 64
202
Javier Gonzáleze46f4e42018-03-30 00:05:10 +0200203static int pblk_set_addrf_12(struct nvm_geo *geo, struct nvm_addrf_12 *dst)
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200204{
Javier Gonzáleze46f4e42018-03-30 00:05:10 +0200205 struct nvm_addrf_12 *src = (struct nvm_addrf_12 *)&geo->addrf;
206 int power_len;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200207
208 /* Re-calculate channel and lun format to adapt to configuration */
Javier Gonzáleza40afad2018-03-30 00:05:14 +0200209 power_len = get_count_order(geo->num_ch);
210 if (1 << power_len != geo->num_ch) {
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200211 pr_err("pblk: supports only power-of-two channel config.\n");
212 return -EINVAL;
213 }
Javier Gonzáleze46f4e42018-03-30 00:05:10 +0200214 dst->ch_len = power_len;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200215
Javier Gonzáleza40afad2018-03-30 00:05:14 +0200216 power_len = get_count_order(geo->num_lun);
217 if (1 << power_len != geo->num_lun) {
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200218 pr_err("pblk: supports only power-of-two LUN config.\n");
219 return -EINVAL;
220 }
Javier Gonzáleze46f4e42018-03-30 00:05:10 +0200221 dst->lun_len = power_len;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200222
Javier Gonzáleze46f4e42018-03-30 00:05:10 +0200223 dst->blk_len = src->blk_len;
224 dst->pg_len = src->pg_len;
225 dst->pln_len = src->pln_len;
Javier Gonzáleza40afad2018-03-30 00:05:14 +0200226 dst->sec_len = src->sec_len;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200227
Javier Gonzáleza40afad2018-03-30 00:05:14 +0200228 dst->sec_offset = 0;
229 dst->pln_offset = dst->sec_len;
Javier Gonzáleze46f4e42018-03-30 00:05:10 +0200230 dst->ch_offset = dst->pln_offset + dst->pln_len;
231 dst->lun_offset = dst->ch_offset + dst->ch_len;
232 dst->pg_offset = dst->lun_offset + dst->lun_len;
233 dst->blk_offset = dst->pg_offset + dst->pg_len;
234
Javier Gonzáleza40afad2018-03-30 00:05:14 +0200235 dst->sec_mask = ((1ULL << dst->sec_len) - 1) << dst->sec_offset;
Javier Gonzáleze46f4e42018-03-30 00:05:10 +0200236 dst->pln_mask = ((1ULL << dst->pln_len) - 1) << dst->pln_offset;
237 dst->ch_mask = ((1ULL << dst->ch_len) - 1) << dst->ch_offset;
238 dst->lun_mask = ((1ULL << dst->lun_len) - 1) << dst->lun_offset;
239 dst->pg_mask = ((1ULL << dst->pg_len) - 1) << dst->pg_offset;
240 dst->blk_mask = ((1ULL << dst->blk_len) - 1) << dst->blk_offset;
241
242 return dst->blk_offset + src->blk_len;
243}
244
Javier González3b2a3ad2018-03-30 00:05:21 +0200245static int pblk_set_addrf_20(struct nvm_geo *geo, struct nvm_addrf *adst,
246 struct pblk_addrf *udst)
247{
248 struct nvm_addrf *src = &geo->addrf;
249
250 adst->ch_len = get_count_order(geo->num_ch);
251 adst->lun_len = get_count_order(geo->num_lun);
252 adst->chk_len = src->chk_len;
253 adst->sec_len = src->sec_len;
254
255 adst->sec_offset = 0;
256 adst->ch_offset = adst->sec_len;
257 adst->lun_offset = adst->ch_offset + adst->ch_len;
258 adst->chk_offset = adst->lun_offset + adst->lun_len;
259
260 adst->sec_mask = ((1ULL << adst->sec_len) - 1) << adst->sec_offset;
261 adst->chk_mask = ((1ULL << adst->chk_len) - 1) << adst->chk_offset;
262 adst->lun_mask = ((1ULL << adst->lun_len) - 1) << adst->lun_offset;
263 adst->ch_mask = ((1ULL << adst->ch_len) - 1) << adst->ch_offset;
264
265 udst->sec_stripe = geo->ws_opt;
266 udst->ch_stripe = geo->num_ch;
267 udst->lun_stripe = geo->num_lun;
268
269 udst->sec_lun_stripe = udst->sec_stripe * udst->ch_stripe;
270 udst->sec_ws_stripe = udst->sec_lun_stripe * udst->lun_stripe;
271
272 return adst->chk_offset + adst->chk_len;
273}
274
Javier Gonzálezbb845ae42018-03-30 00:05:19 +0200275static int pblk_set_addrf(struct pblk *pblk)
Javier Gonzáleze46f4e42018-03-30 00:05:10 +0200276{
277 struct nvm_tgt_dev *dev = pblk->dev;
278 struct nvm_geo *geo = &dev->geo;
279 int mod;
280
Javier González3b2a3ad2018-03-30 00:05:21 +0200281 switch (geo->version) {
282 case NVM_OCSSD_SPEC_12:
283 div_u64_rem(geo->clba, pblk->min_write_pgs, &mod);
284 if (mod) {
285 pr_err("pblk: bad configuration of sectors/pages\n");
286 return -EINVAL;
287 }
288
289 pblk->addrf_len = pblk_set_addrf_12(geo, (void *)&pblk->addrf);
290 break;
291 case NVM_OCSSD_SPEC_20:
292 pblk->addrf_len = pblk_set_addrf_20(geo, (void *)&pblk->addrf,
293 &pblk->uaddrf);
294 break;
295 default:
296 pr_err("pblk: OCSSD revision not supported (%d)\n",
297 geo->version);
Javier Gonzáleze46f4e42018-03-30 00:05:10 +0200298 return -EINVAL;
299 }
300
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200301 return 0;
302}
303
304static int pblk_init_global_caches(struct pblk *pblk)
305{
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200306 down_write(&pblk_lock);
Javier Gonzálezb84ae4a82017-10-13 14:46:07 +0200307 pblk_ws_cache = kmem_cache_create("pblk_blk_ws",
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200308 sizeof(struct pblk_line_ws), 0, 0, NULL);
Javier Gonzálezb84ae4a82017-10-13 14:46:07 +0200309 if (!pblk_ws_cache) {
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200310 up_write(&pblk_lock);
311 return -ENOMEM;
312 }
313
314 pblk_rec_cache = kmem_cache_create("pblk_rec",
315 sizeof(struct pblk_rec_ctx), 0, 0, NULL);
316 if (!pblk_rec_cache) {
Javier Gonzálezb84ae4a82017-10-13 14:46:07 +0200317 kmem_cache_destroy(pblk_ws_cache);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200318 up_write(&pblk_lock);
319 return -ENOMEM;
320 }
321
Javier González084ec9b2017-06-26 16:27:13 -0600322 pblk_g_rq_cache = kmem_cache_create("pblk_g_rq", pblk_g_rq_size,
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200323 0, 0, NULL);
Javier González084ec9b2017-06-26 16:27:13 -0600324 if (!pblk_g_rq_cache) {
Javier Gonzálezb84ae4a82017-10-13 14:46:07 +0200325 kmem_cache_destroy(pblk_ws_cache);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200326 kmem_cache_destroy(pblk_rec_cache);
327 up_write(&pblk_lock);
328 return -ENOMEM;
329 }
330
331 pblk_w_rq_cache = kmem_cache_create("pblk_w_rq", pblk_w_rq_size,
332 0, 0, NULL);
333 if (!pblk_w_rq_cache) {
Javier Gonzálezb84ae4a82017-10-13 14:46:07 +0200334 kmem_cache_destroy(pblk_ws_cache);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200335 kmem_cache_destroy(pblk_rec_cache);
Javier González084ec9b2017-06-26 16:27:13 -0600336 kmem_cache_destroy(pblk_g_rq_cache);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200337 up_write(&pblk_lock);
338 return -ENOMEM;
339 }
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200340 up_write(&pblk_lock);
341
342 return 0;
343}
344
Rakesh Pandit22a4e062017-10-13 14:46:33 +0200345static void pblk_free_global_caches(struct pblk *pblk)
346{
347 kmem_cache_destroy(pblk_ws_cache);
348 kmem_cache_destroy(pblk_rec_cache);
349 kmem_cache_destroy(pblk_g_rq_cache);
350 kmem_cache_destroy(pblk_w_rq_cache);
351}
352
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200353static int pblk_core_init(struct pblk *pblk)
354{
355 struct nvm_tgt_dev *dev = pblk->dev;
356 struct nvm_geo *geo = &dev->geo;
Kent Overstreetb906bbb2018-05-20 18:25:50 -0400357 int ret, max_write_ppas;
Javier González43d47122018-03-30 00:05:09 +0200358
359 atomic64_set(&pblk->user_wa, 0);
360 atomic64_set(&pblk->pad_wa, 0);
361 atomic64_set(&pblk->gc_wa, 0);
362 pblk->user_rst_wa = 0;
363 pblk->pad_rst_wa = 0;
364 pblk->gc_rst_wa = 0;
365
366 atomic64_set(&pblk->nr_flush, 0);
367 pblk->nr_flush_rst = 0;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200368
Javier Gonzáleze46f4e42018-03-30 00:05:10 +0200369 pblk->pgs_in_buffer = geo->mw_cunits * geo->all_luns;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200370
Javier Gonzáleze46f4e42018-03-30 00:05:10 +0200371 pblk->min_write_pgs = geo->ws_opt * (geo->csecs / PAGE_SIZE);
Javier González43d47122018-03-30 00:05:09 +0200372 max_write_ppas = pblk->min_write_pgs * geo->all_luns;
373 pblk->max_write_pgs = min_t(int, max_write_ppas, NVM_MAX_VLBA);
374 pblk_set_sec_per_write(pblk, pblk->min_write_pgs);
375
376 if (pblk->max_write_pgs > PBLK_MAX_REQ_ADDRS) {
377 pr_err("pblk: vector list too big(%u > %u)\n",
378 pblk->max_write_pgs, PBLK_MAX_REQ_ADDRS);
379 return -EINVAL;
380 }
381
382 pblk->pad_dist = kzalloc((pblk->min_write_pgs - 1) * sizeof(atomic64_t),
383 GFP_KERNEL);
384 if (!pblk->pad_dist)
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200385 return -ENOMEM;
386
Javier González43d47122018-03-30 00:05:09 +0200387 if (pblk_init_global_caches(pblk))
388 goto fail_free_pad_dist;
389
Javier Gonzálezb84ae4a82017-10-13 14:46:07 +0200390 /* Internal bios can be at most the sectors signaled by the device. */
Kent Overstreetb906bbb2018-05-20 18:25:50 -0400391 ret = mempool_init_page_pool(&pblk->page_bio_pool, NVM_MAX_VLBA, 0);
392 if (ret)
Rakesh Pandit22a4e062017-10-13 14:46:33 +0200393 goto free_global_caches;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200394
Kent Overstreetb906bbb2018-05-20 18:25:50 -0400395 ret = mempool_init_slab_pool(&pblk->gen_ws_pool, PBLK_GEN_WS_POOL_SIZE,
396 pblk_ws_cache);
397 if (ret)
Javier Gonzálezbd432412017-10-13 14:46:06 +0200398 goto free_page_bio_pool;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200399
Kent Overstreetb906bbb2018-05-20 18:25:50 -0400400 ret = mempool_init_slab_pool(&pblk->rec_pool, geo->all_luns,
401 pblk_rec_cache);
402 if (ret)
Javier Gonzálezb84ae4a82017-10-13 14:46:07 +0200403 goto free_gen_ws_pool;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200404
Kent Overstreetb906bbb2018-05-20 18:25:50 -0400405 ret = mempool_init_slab_pool(&pblk->r_rq_pool, geo->all_luns,
406 pblk_g_rq_cache);
407 if (ret)
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200408 goto free_rec_pool;
409
Kent Overstreetb906bbb2018-05-20 18:25:50 -0400410 ret = mempool_init_slab_pool(&pblk->e_rq_pool, geo->all_luns,
411 pblk_g_rq_cache);
412 if (ret)
Javier González0d880392017-10-13 14:46:08 +0200413 goto free_r_rq_pool;
414
Kent Overstreetb906bbb2018-05-20 18:25:50 -0400415 ret = mempool_init_slab_pool(&pblk->w_rq_pool, geo->all_luns,
416 pblk_w_rq_cache);
417 if (ret)
Javier González0d880392017-10-13 14:46:08 +0200418 goto free_e_rq_pool;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200419
Javier Gonzálezef576492017-06-26 11:57:28 +0200420 pblk->close_wq = alloc_workqueue("pblk-close-wq",
421 WQ_MEM_RECLAIM | WQ_UNBOUND, PBLK_NR_CLOSE_JOBS);
422 if (!pblk->close_wq)
Javier Gonzáleze72ec1d2017-10-13 14:46:09 +0200423 goto free_w_rq_pool;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200424
Javier Gonzálezef576492017-06-26 11:57:28 +0200425 pblk->bb_wq = alloc_workqueue("pblk-bb-wq",
426 WQ_MEM_RECLAIM | WQ_UNBOUND, 0);
427 if (!pblk->bb_wq)
428 goto free_close_wq;
429
Javier González7bd4d372017-10-13 14:46:23 +0200430 pblk->r_end_wq = alloc_workqueue("pblk-read-end-wq",
431 WQ_MEM_RECLAIM | WQ_UNBOUND, 0);
432 if (!pblk->r_end_wq)
Javier Gonzálezef576492017-06-26 11:57:28 +0200433 goto free_bb_wq;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200434
Javier Gonzálezbb845ae42018-03-30 00:05:19 +0200435 if (pblk_set_addrf(pblk))
Javier González7bd4d372017-10-13 14:46:23 +0200436 goto free_r_end_wq;
437
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200438 INIT_LIST_HEAD(&pblk->compl_list);
Hans Holmberg6a3abf52018-06-01 16:41:05 +0200439 INIT_LIST_HEAD(&pblk->resubmit_list);
Javier González43d47122018-03-30 00:05:09 +0200440
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200441 return 0;
442
Javier González7bd4d372017-10-13 14:46:23 +0200443free_r_end_wq:
444 destroy_workqueue(pblk->r_end_wq);
Javier Gonzálezef576492017-06-26 11:57:28 +0200445free_bb_wq:
446 destroy_workqueue(pblk->bb_wq);
447free_close_wq:
448 destroy_workqueue(pblk->close_wq);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200449free_w_rq_pool:
Kent Overstreetb906bbb2018-05-20 18:25:50 -0400450 mempool_exit(&pblk->w_rq_pool);
Javier González0d880392017-10-13 14:46:08 +0200451free_e_rq_pool:
Kent Overstreetb906bbb2018-05-20 18:25:50 -0400452 mempool_exit(&pblk->e_rq_pool);
Javier González0d880392017-10-13 14:46:08 +0200453free_r_rq_pool:
Kent Overstreetb906bbb2018-05-20 18:25:50 -0400454 mempool_exit(&pblk->r_rq_pool);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200455free_rec_pool:
Kent Overstreetb906bbb2018-05-20 18:25:50 -0400456 mempool_exit(&pblk->rec_pool);
Javier Gonzálezb84ae4a82017-10-13 14:46:07 +0200457free_gen_ws_pool:
Kent Overstreetb906bbb2018-05-20 18:25:50 -0400458 mempool_exit(&pblk->gen_ws_pool);
Javier Gonzálezbd432412017-10-13 14:46:06 +0200459free_page_bio_pool:
Kent Overstreetb906bbb2018-05-20 18:25:50 -0400460 mempool_exit(&pblk->page_bio_pool);
Rakesh Pandit22a4e062017-10-13 14:46:33 +0200461free_global_caches:
462 pblk_free_global_caches(pblk);
Javier González43d47122018-03-30 00:05:09 +0200463fail_free_pad_dist:
464 kfree(pblk->pad_dist);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200465 return -ENOMEM;
466}
467
468static void pblk_core_free(struct pblk *pblk)
469{
Javier Gonzálezef576492017-06-26 11:57:28 +0200470 if (pblk->close_wq)
471 destroy_workqueue(pblk->close_wq);
472
Javier González7bd4d372017-10-13 14:46:23 +0200473 if (pblk->r_end_wq)
474 destroy_workqueue(pblk->r_end_wq);
475
Javier Gonzálezef576492017-06-26 11:57:28 +0200476 if (pblk->bb_wq)
477 destroy_workqueue(pblk->bb_wq);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200478
Kent Overstreetb906bbb2018-05-20 18:25:50 -0400479 mempool_exit(&pblk->page_bio_pool);
480 mempool_exit(&pblk->gen_ws_pool);
481 mempool_exit(&pblk->rec_pool);
482 mempool_exit(&pblk->r_rq_pool);
483 mempool_exit(&pblk->e_rq_pool);
484 mempool_exit(&pblk->w_rq_pool);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200485
Rakesh Pandit22a4e062017-10-13 14:46:33 +0200486 pblk_free_global_caches(pblk);
Javier González43d47122018-03-30 00:05:09 +0200487 kfree(pblk->pad_dist);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200488}
489
Javier Gonzáleze411b332018-03-30 00:04:59 +0200490static void pblk_line_mg_free(struct pblk *pblk)
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200491{
492 struct pblk_line_mgmt *l_mg = &pblk->l_mg;
493 int i;
494
495 kfree(l_mg->bb_template);
496 kfree(l_mg->bb_aux);
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200497 kfree(l_mg->vsc_list);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200498
499 for (i = 0; i < PBLK_DATA_LINES; i++) {
Javier Gonzálezf680f192017-06-26 11:57:21 +0200500 kfree(l_mg->sline_meta[i]);
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200501 pblk_mfree(l_mg->eline_meta[i]->buf, l_mg->emeta_alloc_type);
Javier Gonzálezf680f192017-06-26 11:57:21 +0200502 kfree(l_mg->eline_meta[i]);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200503 }
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200504}
505
Hans Holmberg48b8d202018-06-01 16:41:06 +0200506static void pblk_line_meta_free(struct pblk_line_mgmt *l_mg,
507 struct pblk_line *line)
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200508{
Hans Holmberg48b8d202018-06-01 16:41:06 +0200509 struct pblk_w_err_gc *w_err_gc = line->w_err_gc;
510
Javier Gonzáleze411b332018-03-30 00:04:59 +0200511 kfree(line->blk_bitmap);
512 kfree(line->erase_bitmap);
Javier González32ef9412018-03-30 00:05:20 +0200513 kfree(line->chks);
Hans Holmberg48b8d202018-06-01 16:41:06 +0200514
515 pblk_mfree(w_err_gc->lba_list, l_mg->emeta_alloc_type);
516 kfree(w_err_gc);
Javier Gonzáleze411b332018-03-30 00:04:59 +0200517}
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200518
Javier Gonzáleze411b332018-03-30 00:04:59 +0200519static void pblk_lines_free(struct pblk *pblk)
520{
521 struct pblk_line_mgmt *l_mg = &pblk->l_mg;
522 struct pblk_line *line;
523 int i;
524
525 spin_lock(&l_mg->free_lock);
526 for (i = 0; i < l_mg->nr_lines; i++) {
527 line = &pblk->lines[i];
528
Javier González8e55c072018-06-01 15:04:22 +0200529 pblk_line_free(line);
Hans Holmberg48b8d202018-06-01 16:41:06 +0200530 pblk_line_meta_free(l_mg, line);
Javier Gonzáleze411b332018-03-30 00:04:59 +0200531 }
532 spin_unlock(&l_mg->free_lock);
Javier González43d47122018-03-30 00:05:09 +0200533
534 pblk_line_mg_free(pblk);
535
536 kfree(pblk->luns);
537 kfree(pblk->lines);
Javier Gonzáleze411b332018-03-30 00:04:59 +0200538}
539
540static int pblk_bb_get_tbl(struct nvm_tgt_dev *dev, struct pblk_lun *rlun,
541 u8 *blks, int nr_blks)
542{
543 struct ppa_addr ppa;
544 int ret;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200545
546 ppa.ppa = 0;
547 ppa.g.ch = rlun->bppa.g.ch;
548 ppa.g.lun = rlun->bppa.g.lun;
549
550 ret = nvm_get_tgt_bb_tbl(dev, ppa, blks);
551 if (ret)
Javier Gonzáleze411b332018-03-30 00:04:59 +0200552 return ret;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200553
554 nr_blks = nvm_bb_tbl_fold(dev->parent, blks, nr_blks);
Javier Gonzáleze411b332018-03-30 00:04:59 +0200555 if (nr_blks < 0)
556 return -EIO;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200557
Wei Yongjun5136a4f2017-04-25 16:15:30 +0000558 return 0;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200559}
560
Javier González32ef9412018-03-30 00:05:20 +0200561static void *pblk_bb_get_meta(struct pblk *pblk)
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200562{
Javier Gonzálezdffdd962017-06-26 11:57:22 +0200563 struct nvm_tgt_dev *dev = pblk->dev;
564 struct nvm_geo *geo = &dev->geo;
Javier González32ef9412018-03-30 00:05:20 +0200565 u8 *meta;
Javier Gonzáleze411b332018-03-30 00:04:59 +0200566 int i, nr_blks, blk_per_lun;
567 int ret;
568
Javier Gonzáleza40afad2018-03-30 00:05:14 +0200569 blk_per_lun = geo->num_chk * geo->pln_mode;
Javier Gonzáleze411b332018-03-30 00:04:59 +0200570 nr_blks = blk_per_lun * geo->all_luns;
571
Javier González32ef9412018-03-30 00:05:20 +0200572 meta = kmalloc(nr_blks, GFP_KERNEL);
573 if (!meta)
Javier Gonzáleze411b332018-03-30 00:04:59 +0200574 return ERR_PTR(-ENOMEM);
575
576 for (i = 0; i < geo->all_luns; i++) {
577 struct pblk_lun *rlun = &pblk->luns[i];
Javier González32ef9412018-03-30 00:05:20 +0200578 u8 *meta_pos = meta + i * blk_per_lun;
Javier Gonzáleze411b332018-03-30 00:04:59 +0200579
Javier González32ef9412018-03-30 00:05:20 +0200580 ret = pblk_bb_get_tbl(dev, rlun, meta_pos, blk_per_lun);
Javier Gonzáleze411b332018-03-30 00:04:59 +0200581 if (ret) {
Javier González32ef9412018-03-30 00:05:20 +0200582 kfree(meta);
Javier Gonzáleze411b332018-03-30 00:04:59 +0200583 return ERR_PTR(-EIO);
584 }
585 }
586
Javier González32ef9412018-03-30 00:05:20 +0200587 return meta;
Javier Gonzáleze411b332018-03-30 00:04:59 +0200588}
589
Javier González32ef9412018-03-30 00:05:20 +0200590static void *pblk_chunk_get_meta(struct pblk *pblk)
Javier Gonzáleze411b332018-03-30 00:04:59 +0200591{
592 struct nvm_tgt_dev *dev = pblk->dev;
593 struct nvm_geo *geo = &dev->geo;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200594
Javier González32ef9412018-03-30 00:05:20 +0200595 if (geo->version == NVM_OCSSD_SPEC_12)
596 return pblk_bb_get_meta(pblk);
597 else
598 return pblk_chunk_get_info(pblk);
Javier Gonzálezdffdd962017-06-26 11:57:22 +0200599}
600
Javier González43d47122018-03-30 00:05:09 +0200601static int pblk_luns_init(struct pblk *pblk)
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200602{
603 struct nvm_tgt_dev *dev = pblk->dev;
604 struct nvm_geo *geo = &dev->geo;
605 struct pblk_lun *rlun;
Javier Gonzáleze411b332018-03-30 00:04:59 +0200606 int i;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200607
608 /* TODO: Implement unbalanced LUN support */
Javier Gonzáleza40afad2018-03-30 00:05:14 +0200609 if (geo->num_lun < 0) {
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200610 pr_err("pblk: unbalanced LUN config.\n");
611 return -EINVAL;
612 }
613
Matias Bjørlingfae7fae2018-01-05 14:16:03 +0100614 pblk->luns = kcalloc(geo->all_luns, sizeof(struct pblk_lun),
615 GFP_KERNEL);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200616 if (!pblk->luns)
617 return -ENOMEM;
618
Matias Bjørlingfae7fae2018-01-05 14:16:03 +0100619 for (i = 0; i < geo->all_luns; i++) {
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200620 /* Stripe across channels */
Javier Gonzáleza40afad2018-03-30 00:05:14 +0200621 int ch = i % geo->num_ch;
622 int lun_raw = i / geo->num_ch;
623 int lunid = lun_raw + ch * geo->num_lun;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200624
625 rlun = &pblk->luns[i];
Javier González43d47122018-03-30 00:05:09 +0200626 rlun->bppa = dev->luns[lunid];
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200627
628 sema_init(&rlun->wr_sem, 1);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200629 }
630
631 return 0;
632}
633
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200634/* See comment over struct line_emeta definition */
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200635static unsigned int calc_emeta_len(struct pblk *pblk)
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200636{
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200637 struct pblk_line_meta *lm = &pblk->lm;
638 struct pblk_line_mgmt *l_mg = &pblk->l_mg;
639 struct nvm_tgt_dev *dev = pblk->dev;
640 struct nvm_geo *geo = &dev->geo;
641
642 /* Round to sector size so that lba_list starts on its own sector */
643 lm->emeta_sec[1] = DIV_ROUND_UP(
Hans Holmberg76758392018-03-30 00:04:52 +0200644 sizeof(struct line_emeta) + lm->blk_bitmap_len +
Javier Gonzáleze46f4e42018-03-30 00:05:10 +0200645 sizeof(struct wa_counters), geo->csecs);
646 lm->emeta_len[1] = lm->emeta_sec[1] * geo->csecs;
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200647
648 /* Round to sector size so that vsc_list starts on its own sector */
649 lm->dsec_per_line = lm->sec_per_line - lm->emeta_sec[0];
650 lm->emeta_sec[2] = DIV_ROUND_UP(lm->dsec_per_line * sizeof(u64),
Javier Gonzáleze46f4e42018-03-30 00:05:10 +0200651 geo->csecs);
652 lm->emeta_len[2] = lm->emeta_sec[2] * geo->csecs;
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200653
654 lm->emeta_sec[3] = DIV_ROUND_UP(l_mg->nr_lines * sizeof(u32),
Javier Gonzáleze46f4e42018-03-30 00:05:10 +0200655 geo->csecs);
656 lm->emeta_len[3] = lm->emeta_sec[3] * geo->csecs;
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200657
658 lm->vsc_list_len = l_mg->nr_lines * sizeof(u32);
659
660 return (lm->emeta_len[1] + lm->emeta_len[2] + lm->emeta_len[3]);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200661}
662
663static void pblk_set_provision(struct pblk *pblk, long nr_free_blks)
664{
665 struct nvm_tgt_dev *dev = pblk->dev;
Javier Gonzáleza7689932018-01-05 14:16:13 +0100666 struct pblk_line_mgmt *l_mg = &pblk->l_mg;
667 struct pblk_line_meta *lm = &pblk->lm;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200668 struct nvm_geo *geo = &dev->geo;
669 sector_t provisioned;
Javier Gonzáleza7689932018-01-05 14:16:13 +0100670 int sec_meta, blk_meta;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200671
Javier Gonzáleze5392732018-01-05 14:16:14 +0100672 if (geo->op == NVM_TARGET_DEFAULT_OP)
673 pblk->op = PBLK_DEFAULT_OP;
674 else
675 pblk->op = geo->op;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200676
677 provisioned = nr_free_blks;
Javier Gonzáleza7689932018-01-05 14:16:13 +0100678 provisioned *= (100 - pblk->op);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200679 sector_div(provisioned, 100);
680
Javier Gonzáleza7689932018-01-05 14:16:13 +0100681 pblk->op_blks = nr_free_blks - provisioned;
682
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200683 /* Internally pblk manages all free blocks, but all calculations based
684 * on user capacity consider only provisioned blocks
685 */
686 pblk->rl.total_blocks = nr_free_blks;
Javier Gonzáleze46f4e42018-03-30 00:05:10 +0200687 pblk->rl.nr_secs = nr_free_blks * geo->clba;
Javier Gonzáleza7689932018-01-05 14:16:13 +0100688
689 /* Consider sectors used for metadata */
690 sec_meta = (lm->smeta_sec + lm->emeta_sec[0]) * l_mg->nr_free_lines;
Javier Gonzáleze46f4e42018-03-30 00:05:10 +0200691 blk_meta = DIV_ROUND_UP(sec_meta, geo->clba);
Javier Gonzáleza7689932018-01-05 14:16:13 +0100692
Javier Gonzáleze46f4e42018-03-30 00:05:10 +0200693 pblk->capacity = (provisioned - blk_meta) * geo->clba;
Javier Gonzáleza7689932018-01-05 14:16:13 +0100694
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200695 atomic_set(&pblk->rl.free_blocks, nr_free_blks);
Javier Gonzáleza7689932018-01-05 14:16:13 +0100696 atomic_set(&pblk->rl.free_user_blocks, nr_free_blks);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200697}
698
Javier González32ef9412018-03-30 00:05:20 +0200699static int pblk_setup_line_meta_12(struct pblk *pblk, struct pblk_line *line,
700 void *chunk_meta)
701{
702 struct nvm_tgt_dev *dev = pblk->dev;
703 struct nvm_geo *geo = &dev->geo;
704 struct pblk_line_meta *lm = &pblk->lm;
705 int i, chk_per_lun, nr_bad_chks = 0;
706
707 chk_per_lun = geo->num_chk * geo->pln_mode;
708
709 for (i = 0; i < lm->blk_per_line; i++) {
710 struct pblk_lun *rlun = &pblk->luns[i];
711 struct nvm_chk_meta *chunk;
712 int pos = pblk_ppa_to_pos(geo, rlun->bppa);
713 u8 *lun_bb_meta = chunk_meta + pos * chk_per_lun;
714
715 chunk = &line->chks[pos];
716
717 /*
718 * In 1.2 spec. chunk state is not persisted by the device. Thus
719 * some of the values are reset each time pblk is instantiated.
720 */
721 if (lun_bb_meta[line->id] == NVM_BLK_T_FREE)
722 chunk->state = NVM_CHK_ST_FREE;
723 else
724 chunk->state = NVM_CHK_ST_OFFLINE;
725
726 chunk->type = NVM_CHK_TP_W_SEQ;
727 chunk->wi = 0;
728 chunk->slba = -1;
729 chunk->cnlb = geo->clba;
730 chunk->wp = 0;
731
732 if (!(chunk->state & NVM_CHK_ST_OFFLINE))
733 continue;
734
735 set_bit(pos, line->blk_bitmap);
736 nr_bad_chks++;
737 }
738
739 return nr_bad_chks;
740}
741
742static int pblk_setup_line_meta_20(struct pblk *pblk, struct pblk_line *line,
743 struct nvm_chk_meta *meta)
744{
745 struct nvm_tgt_dev *dev = pblk->dev;
746 struct nvm_geo *geo = &dev->geo;
747 struct pblk_line_meta *lm = &pblk->lm;
748 int i, nr_bad_chks = 0;
749
750 for (i = 0; i < lm->blk_per_line; i++) {
751 struct pblk_lun *rlun = &pblk->luns[i];
752 struct nvm_chk_meta *chunk;
753 struct nvm_chk_meta *chunk_meta;
754 struct ppa_addr ppa;
755 int pos;
756
757 ppa = rlun->bppa;
758 pos = pblk_ppa_to_pos(geo, ppa);
759 chunk = &line->chks[pos];
760
761 ppa.m.chk = line->id;
762 chunk_meta = pblk_chunk_get_off(pblk, meta, ppa);
763
764 chunk->state = chunk_meta->state;
765 chunk->type = chunk_meta->type;
766 chunk->wi = chunk_meta->wi;
767 chunk->slba = chunk_meta->slba;
768 chunk->cnlb = chunk_meta->cnlb;
769 chunk->wp = chunk_meta->wp;
770
Javier González32ef9412018-03-30 00:05:20 +0200771 if (chunk->type & NVM_CHK_TP_SZ_SPEC) {
772 WARN_ONCE(1, "pblk: custom-sized chunks unsupported\n");
773 continue;
774 }
775
Javier González6f9c9602018-06-01 15:04:23 +0200776 if (!(chunk->state & NVM_CHK_ST_OFFLINE))
777 continue;
778
Javier González32ef9412018-03-30 00:05:20 +0200779 set_bit(pos, line->blk_bitmap);
780 nr_bad_chks++;
781 }
782
783 return nr_bad_chks;
784}
785
786static long pblk_setup_line_meta(struct pblk *pblk, struct pblk_line *line,
787 void *chunk_meta, int line_id)
788{
789 struct nvm_tgt_dev *dev = pblk->dev;
790 struct nvm_geo *geo = &dev->geo;
791 struct pblk_line_mgmt *l_mg = &pblk->l_mg;
792 struct pblk_line_meta *lm = &pblk->lm;
793 long nr_bad_chks, chk_in_line;
794
795 line->pblk = pblk;
796 line->id = line_id;
797 line->type = PBLK_LINETYPE_FREE;
798 line->state = PBLK_LINESTATE_NEW;
799 line->gc_group = PBLK_LINEGC_NONE;
800 line->vsc = &l_mg->vsc_list[line_id];
801 spin_lock_init(&line->lock);
802
803 if (geo->version == NVM_OCSSD_SPEC_12)
804 nr_bad_chks = pblk_setup_line_meta_12(pblk, line, chunk_meta);
805 else
806 nr_bad_chks = pblk_setup_line_meta_20(pblk, line, chunk_meta);
807
808 chk_in_line = lm->blk_per_line - nr_bad_chks;
809 if (nr_bad_chks < 0 || nr_bad_chks > lm->blk_per_line ||
810 chk_in_line < lm->min_blk_line) {
811 line->state = PBLK_LINESTATE_BAD;
812 list_add_tail(&line->list, &l_mg->bad_list);
813 return 0;
814 }
815
816 atomic_set(&line->blk_in_line, chk_in_line);
817 list_add_tail(&line->list, &l_mg->free_list);
818 l_mg->nr_free_lines++;
819
820 return chk_in_line;
821}
822
823static int pblk_alloc_line_meta(struct pblk *pblk, struct pblk_line *line)
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200824{
Javier González43d47122018-03-30 00:05:09 +0200825 struct pblk_line_meta *lm = &pblk->lm;
826
827 line->blk_bitmap = kzalloc(lm->blk_bitmap_len, GFP_KERNEL);
828 if (!line->blk_bitmap)
829 return -ENOMEM;
830
831 line->erase_bitmap = kzalloc(lm->blk_bitmap_len, GFP_KERNEL);
Hans Holmberg48b8d202018-06-01 16:41:06 +0200832 if (!line->erase_bitmap)
833 goto free_blk_bitmap;
834
Javier González43d47122018-03-30 00:05:09 +0200835
Javier González32ef9412018-03-30 00:05:20 +0200836 line->chks = kmalloc(lm->blk_per_line * sizeof(struct nvm_chk_meta),
837 GFP_KERNEL);
Hans Holmberg48b8d202018-06-01 16:41:06 +0200838 if (!line->chks)
839 goto free_erase_bitmap;
840
841 line->w_err_gc = kzalloc(sizeof(struct pblk_w_err_gc), GFP_KERNEL);
842 if (!line->w_err_gc)
843 goto free_chks;
Javier González43d47122018-03-30 00:05:09 +0200844
845 return 0;
Hans Holmberg48b8d202018-06-01 16:41:06 +0200846
847free_chks:
848 kfree(line->chks);
849free_erase_bitmap:
850 kfree(line->erase_bitmap);
851free_blk_bitmap:
852 kfree(line->blk_bitmap);
853 return -ENOMEM;
Javier González43d47122018-03-30 00:05:09 +0200854}
855
856static int pblk_line_mg_init(struct pblk *pblk)
857{
858 struct nvm_tgt_dev *dev = pblk->dev;
859 struct nvm_geo *geo = &dev->geo;
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200860 struct pblk_line_mgmt *l_mg = &pblk->l_mg;
861 struct pblk_line_meta *lm = &pblk->lm;
Javier González43d47122018-03-30 00:05:09 +0200862 int i, bb_distance;
863
Javier Gonzáleza40afad2018-03-30 00:05:14 +0200864 l_mg->nr_lines = geo->num_chk;
Javier González43d47122018-03-30 00:05:09 +0200865 l_mg->log_line = l_mg->data_line = NULL;
866 l_mg->l_seq_nr = l_mg->d_seq_nr = 0;
867 l_mg->nr_free_lines = 0;
868 bitmap_zero(&l_mg->meta_bitmap, PBLK_DATA_LINES);
869
870 INIT_LIST_HEAD(&l_mg->free_list);
871 INIT_LIST_HEAD(&l_mg->corrupt_list);
872 INIT_LIST_HEAD(&l_mg->bad_list);
873 INIT_LIST_HEAD(&l_mg->gc_full_list);
874 INIT_LIST_HEAD(&l_mg->gc_high_list);
875 INIT_LIST_HEAD(&l_mg->gc_mid_list);
876 INIT_LIST_HEAD(&l_mg->gc_low_list);
877 INIT_LIST_HEAD(&l_mg->gc_empty_list);
Hans Holmberg48b8d202018-06-01 16:41:06 +0200878 INIT_LIST_HEAD(&l_mg->gc_werr_list);
Javier González43d47122018-03-30 00:05:09 +0200879
880 INIT_LIST_HEAD(&l_mg->emeta_list);
881
Hans Holmberg48b8d202018-06-01 16:41:06 +0200882 l_mg->gc_lists[0] = &l_mg->gc_werr_list;
883 l_mg->gc_lists[1] = &l_mg->gc_high_list;
884 l_mg->gc_lists[2] = &l_mg->gc_mid_list;
885 l_mg->gc_lists[3] = &l_mg->gc_low_list;
Javier González43d47122018-03-30 00:05:09 +0200886
887 spin_lock_init(&l_mg->free_lock);
888 spin_lock_init(&l_mg->close_lock);
889 spin_lock_init(&l_mg->gc_lock);
890
891 l_mg->vsc_list = kcalloc(l_mg->nr_lines, sizeof(__le32), GFP_KERNEL);
892 if (!l_mg->vsc_list)
893 goto fail;
894
895 l_mg->bb_template = kzalloc(lm->sec_bitmap_len, GFP_KERNEL);
896 if (!l_mg->bb_template)
897 goto fail_free_vsc_list;
898
899 l_mg->bb_aux = kzalloc(lm->sec_bitmap_len, GFP_KERNEL);
900 if (!l_mg->bb_aux)
901 goto fail_free_bb_template;
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200902
903 /* smeta is always small enough to fit on a kmalloc memory allocation,
904 * emeta depends on the number of LUNs allocated to the pblk instance
905 */
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200906 for (i = 0; i < PBLK_DATA_LINES; i++) {
907 l_mg->sline_meta[i] = kmalloc(lm->smeta_len, GFP_KERNEL);
908 if (!l_mg->sline_meta[i])
909 goto fail_free_smeta;
910 }
911
912 /* emeta allocates three different buffers for managing metadata with
913 * in-memory and in-media layouts
914 */
915 for (i = 0; i < PBLK_DATA_LINES; i++) {
916 struct pblk_emeta *emeta;
917
918 emeta = kmalloc(sizeof(struct pblk_emeta), GFP_KERNEL);
919 if (!emeta)
920 goto fail_free_emeta;
921
922 if (lm->emeta_len[0] > KMALLOC_MAX_CACHE_SIZE) {
923 l_mg->emeta_alloc_type = PBLK_VMALLOC_META;
924
925 emeta->buf = vmalloc(lm->emeta_len[0]);
926 if (!emeta->buf) {
927 kfree(emeta);
928 goto fail_free_emeta;
929 }
930
931 emeta->nr_entries = lm->emeta_sec[0];
932 l_mg->eline_meta[i] = emeta;
933 } else {
934 l_mg->emeta_alloc_type = PBLK_KMALLOC_META;
935
936 emeta->buf = kmalloc(lm->emeta_len[0], GFP_KERNEL);
937 if (!emeta->buf) {
938 kfree(emeta);
939 goto fail_free_emeta;
940 }
941
942 emeta->nr_entries = lm->emeta_sec[0];
943 l_mg->eline_meta[i] = emeta;
944 }
945 }
946
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200947 for (i = 0; i < l_mg->nr_lines; i++)
948 l_mg->vsc_list[i] = cpu_to_le32(EMPTY_ENTRY);
949
Javier González43d47122018-03-30 00:05:09 +0200950 bb_distance = (geo->all_luns) * geo->ws_opt;
951 for (i = 0; i < lm->sec_per_line; i += bb_distance)
952 bitmap_set(l_mg->bb_template, i, geo->ws_opt);
953
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200954 return 0;
955
956fail_free_emeta:
957 while (--i >= 0) {
Rakesh Panditc9d84b32017-10-13 14:45:54 +0200958 if (l_mg->emeta_alloc_type == PBLK_VMALLOC_META)
959 vfree(l_mg->eline_meta[i]->buf);
960 else
961 kfree(l_mg->eline_meta[i]->buf);
Javier Gonzálezf680f192017-06-26 11:57:21 +0200962 kfree(l_mg->eline_meta[i]);
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200963 }
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200964fail_free_smeta:
965 for (i = 0; i < PBLK_DATA_LINES; i++)
Javier Gonzálezf680f192017-06-26 11:57:21 +0200966 kfree(l_mg->sline_meta[i]);
Javier González43d47122018-03-30 00:05:09 +0200967 kfree(l_mg->bb_aux);
968fail_free_bb_template:
969 kfree(l_mg->bb_template);
970fail_free_vsc_list:
971 kfree(l_mg->vsc_list);
972fail:
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200973 return -ENOMEM;
974}
975
Javier González43d47122018-03-30 00:05:09 +0200976static int pblk_line_meta_init(struct pblk *pblk)
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200977{
978 struct nvm_tgt_dev *dev = pblk->dev;
979 struct nvm_geo *geo = &dev->geo;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200980 struct pblk_line_meta *lm = &pblk->lm;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200981 unsigned int smeta_len, emeta_len;
Javier González43d47122018-03-30 00:05:09 +0200982 int i;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200983
Javier Gonzáleze46f4e42018-03-30 00:05:10 +0200984 lm->sec_per_line = geo->clba * geo->all_luns;
Matias Bjørlingfae7fae2018-01-05 14:16:03 +0100985 lm->blk_per_line = geo->all_luns;
986 lm->blk_bitmap_len = BITS_TO_LONGS(geo->all_luns) * sizeof(long);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200987 lm->sec_bitmap_len = BITS_TO_LONGS(lm->sec_per_line) * sizeof(long);
Matias Bjørlingfae7fae2018-01-05 14:16:03 +0100988 lm->lun_bitmap_len = BITS_TO_LONGS(geo->all_luns) * sizeof(long);
Rakesh Pandit27b97872017-10-13 14:46:28 +0200989 lm->mid_thrs = lm->sec_per_line / 2;
990 lm->high_thrs = lm->sec_per_line / 4;
Matias Bjørlingfae7fae2018-01-05 14:16:03 +0100991 lm->meta_distance = (geo->all_luns / 2) * pblk->min_write_pgs;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200992
993 /* Calculate necessary pages for smeta. See comment over struct
994 * line_smeta definition
995 */
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200996 i = 1;
997add_smeta_page:
Javier Gonzáleze46f4e42018-03-30 00:05:10 +0200998 lm->smeta_sec = i * geo->ws_opt;
999 lm->smeta_len = lm->smeta_sec * geo->csecs;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001000
Javier Gonzálezdd2a4342017-06-26 11:57:17 +02001001 smeta_len = sizeof(struct line_smeta) + lm->lun_bitmap_len;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001002 if (smeta_len > lm->smeta_len) {
1003 i++;
1004 goto add_smeta_page;
1005 }
1006
1007 /* Calculate necessary pages for emeta. See comment over struct
1008 * line_emeta definition
1009 */
1010 i = 1;
1011add_emeta_page:
Javier Gonzáleze46f4e42018-03-30 00:05:10 +02001012 lm->emeta_sec[0] = i * geo->ws_opt;
1013 lm->emeta_len[0] = lm->emeta_sec[0] * geo->csecs;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001014
Javier Gonzálezdd2a4342017-06-26 11:57:17 +02001015 emeta_len = calc_emeta_len(pblk);
1016 if (emeta_len > lm->emeta_len[0]) {
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001017 i++;
1018 goto add_emeta_page;
1019 }
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001020
Matias Bjørlingfae7fae2018-01-05 14:16:03 +01001021 lm->emeta_bb = geo->all_luns > i ? geo->all_luns - i : 0;
Javier González21d22872017-10-13 14:46:26 +02001022
1023 lm->min_blk_line = 1;
Matias Bjørlingfae7fae2018-01-05 14:16:03 +01001024 if (geo->all_luns > 1)
Javier González21d22872017-10-13 14:46:26 +02001025 lm->min_blk_line += DIV_ROUND_UP(lm->smeta_sec +
Javier Gonzáleze46f4e42018-03-30 00:05:10 +02001026 lm->emeta_sec[0], geo->clba);
Javier González21d22872017-10-13 14:46:26 +02001027
Javier Gonzálezb5e063a2017-06-30 17:56:41 +02001028 if (lm->min_blk_line > lm->blk_per_line) {
1029 pr_err("pblk: config. not supported. Min. LUN in line:%d\n",
1030 lm->blk_per_line);
Javier Gonzáleze411b332018-03-30 00:04:59 +02001031 return -EINVAL;
Javier Gonzálezb5e063a2017-06-30 17:56:41 +02001032 }
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001033
Javier González43d47122018-03-30 00:05:09 +02001034 return 0;
1035}
1036
1037static int pblk_lines_init(struct pblk *pblk)
1038{
1039 struct pblk_line_mgmt *l_mg = &pblk->l_mg;
Javier González43d47122018-03-30 00:05:09 +02001040 struct pblk_line *line;
Javier González32ef9412018-03-30 00:05:20 +02001041 void *chunk_meta;
1042 long nr_free_chks = 0;
Javier González43d47122018-03-30 00:05:09 +02001043 int i, ret;
1044
1045 ret = pblk_line_meta_init(pblk);
Javier Gonzálezdd2a4342017-06-26 11:57:17 +02001046 if (ret)
Javier Gonzáleze411b332018-03-30 00:04:59 +02001047 return ret;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001048
Javier González43d47122018-03-30 00:05:09 +02001049 ret = pblk_line_mg_init(pblk);
1050 if (ret)
1051 return ret;
1052
1053 ret = pblk_luns_init(pblk);
1054 if (ret)
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001055 goto fail_free_meta;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001056
Javier González32ef9412018-03-30 00:05:20 +02001057 chunk_meta = pblk_chunk_get_meta(pblk);
1058 if (IS_ERR(chunk_meta)) {
1059 ret = PTR_ERR(chunk_meta);
Javier González43d47122018-03-30 00:05:09 +02001060 goto fail_free_luns;
1061 }
1062
1063 pblk->lines = kcalloc(l_mg->nr_lines, sizeof(struct pblk_line),
1064 GFP_KERNEL);
1065 if (!pblk->lines) {
1066 ret = -ENOMEM;
Javier González32ef9412018-03-30 00:05:20 +02001067 goto fail_free_chunk_meta;
Javier Gonzáleze411b332018-03-30 00:04:59 +02001068 }
1069
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001070 for (i = 0; i < l_mg->nr_lines; i++) {
1071 line = &pblk->lines[i];
1072
Javier González32ef9412018-03-30 00:05:20 +02001073 ret = pblk_alloc_line_meta(pblk, line);
Javier Gonzálezdffdd962017-06-26 11:57:22 +02001074 if (ret)
Javier González43d47122018-03-30 00:05:09 +02001075 goto fail_free_lines;
Javier Gonzálezdffdd962017-06-26 11:57:22 +02001076
Javier González32ef9412018-03-30 00:05:20 +02001077 nr_free_chks += pblk_setup_line_meta(pblk, line, chunk_meta, i);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001078 }
1079
Javier González2deeefc2018-06-01 15:04:15 +02001080 if (!nr_free_chks) {
1081 pr_err("pblk: too many bad blocks prevent for sane instance\n");
1082 return -EINTR;
1083 }
1084
Javier González32ef9412018-03-30 00:05:20 +02001085 pblk_set_provision(pblk, nr_free_chks);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001086
Javier González32ef9412018-03-30 00:05:20 +02001087 kfree(chunk_meta);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001088 return 0;
Javier Gonzáleze411b332018-03-30 00:04:59 +02001089
Javier González43d47122018-03-30 00:05:09 +02001090fail_free_lines:
Javier Gonzálezdffdd962017-06-26 11:57:22 +02001091 while (--i >= 0)
Hans Holmberg48b8d202018-06-01 16:41:06 +02001092 pblk_line_meta_free(l_mg, &pblk->lines[i]);
Javier González43d47122018-03-30 00:05:09 +02001093 kfree(pblk->lines);
Javier González32ef9412018-03-30 00:05:20 +02001094fail_free_chunk_meta:
1095 kfree(chunk_meta);
Javier González43d47122018-03-30 00:05:09 +02001096fail_free_luns:
1097 kfree(pblk->luns);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001098fail_free_meta:
Javier Gonzáleze411b332018-03-30 00:04:59 +02001099 pblk_line_mg_free(pblk);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001100
1101 return ret;
1102}
1103
1104static int pblk_writer_init(struct pblk *pblk)
1105{
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001106 pblk->writer_ts = kthread_create(pblk_write_ts, pblk, "pblk-writer-t");
1107 if (IS_ERR(pblk->writer_ts)) {
Javier Gonzálezcc4f5ba2018-01-05 14:16:17 +01001108 int err = PTR_ERR(pblk->writer_ts);
1109
1110 if (err != -EINTR)
1111 pr_err("pblk: could not allocate writer kthread (%d)\n",
1112 err);
1113 return err;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001114 }
1115
Javier Gonzálezcc4f5ba2018-01-05 14:16:17 +01001116 timer_setup(&pblk->wtimer, pblk_write_timer_fn, 0);
1117 mod_timer(&pblk->wtimer, jiffies + msecs_to_jiffies(100));
1118
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001119 return 0;
1120}
1121
1122static void pblk_writer_stop(struct pblk *pblk)
1123{
Javier Gonzálezee8d5c12017-06-30 17:56:40 +02001124 /* The pipeline must be stopped and the write buffer emptied before the
1125 * write thread is stopped
1126 */
1127 WARN(pblk_rb_read_count(&pblk->rwb),
1128 "Stopping not fully persisted write buffer\n");
1129
1130 WARN(pblk_rb_sync_count(&pblk->rwb),
1131 "Stopping not fully synced write buffer\n");
1132
Hans Holmberg7be970b2018-03-30 00:04:56 +02001133 del_timer_sync(&pblk->wtimer);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001134 if (pblk->writer_ts)
1135 kthread_stop(pblk->writer_ts);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001136}
1137
1138static void pblk_free(struct pblk *pblk)
1139{
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001140 pblk_lines_free(pblk);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001141 pblk_l2p_free(pblk);
Javier González43d47122018-03-30 00:05:09 +02001142 pblk_rwb_free(pblk);
1143 pblk_core_free(pblk);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001144
1145 kfree(pblk);
1146}
1147
Javier Gonzáleza7c9e912018-06-01 15:04:24 +02001148static void pblk_tear_down(struct pblk *pblk, bool graceful)
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001149{
Javier Gonzáleza7c9e912018-06-01 15:04:24 +02001150 if (graceful)
1151 __pblk_pipeline_flush(pblk);
1152 __pblk_pipeline_stop(pblk);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001153 pblk_writer_stop(pblk);
1154 pblk_rb_sync_l2p(&pblk->rwb);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001155 pblk_rl_free(&pblk->rl);
1156
Javier Gonzáleza7c9e912018-06-01 15:04:24 +02001157 pr_debug("pblk: consistent tear down (graceful:%d)\n", graceful);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001158}
1159
Javier Gonzáleza7c9e912018-06-01 15:04:24 +02001160static void pblk_exit(void *private, bool graceful)
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001161{
1162 struct pblk *pblk = private;
1163
1164 down_write(&pblk_lock);
Javier Gonzáleza7c9e912018-06-01 15:04:24 +02001165 pblk_gc_exit(pblk, graceful);
1166 pblk_tear_down(pblk, graceful);
Hans Holmbergc5586192017-10-13 14:46:40 +02001167
1168#ifdef CONFIG_NVM_DEBUG
1169 pr_info("pblk exit: L2P CRC: %x\n", pblk_l2p_crc(pblk));
1170#endif
1171
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001172 pblk_free(pblk);
1173 up_write(&pblk_lock);
1174}
1175
1176static sector_t pblk_capacity(void *private)
1177{
1178 struct pblk *pblk = private;
1179
1180 return pblk->capacity * NR_PHY_IN_LOG;
1181}
1182
1183static void *pblk_init(struct nvm_tgt_dev *dev, struct gendisk *tdisk,
1184 int flags)
1185{
1186 struct nvm_geo *geo = &dev->geo;
1187 struct request_queue *bqueue = dev->q;
1188 struct request_queue *tqueue = tdisk->queue;
1189 struct pblk *pblk;
1190 int ret;
1191
Javier González3b2a3ad2018-03-30 00:05:21 +02001192 /* pblk supports 1.2 and 2.0 versions */
1193 if (!(geo->version == NVM_OCSSD_SPEC_12 ||
1194 geo->version == NVM_OCSSD_SPEC_20)) {
Javier González7ad50392018-03-30 00:05:18 +02001195 pr_err("pblk: OCSSD version not supported (%u)\n",
1196 geo->version);
1197 return ERR_PTR(-EINVAL);
1198 }
1199
1200 if (geo->version == NVM_OCSSD_SPEC_12 && geo->dom & NVM_RSP_L2P) {
Rakesh Pandit4e76af52017-10-13 14:45:57 +02001201 pr_err("pblk: host-side L2P table not supported. (%x)\n",
Javier González7ad50392018-03-30 00:05:18 +02001202 geo->dom);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001203 return ERR_PTR(-EINVAL);
1204 }
1205
1206 pblk = kzalloc(sizeof(struct pblk), GFP_KERNEL);
1207 if (!pblk)
1208 return ERR_PTR(-ENOMEM);
1209
1210 pblk->dev = dev;
1211 pblk->disk = tdisk;
Javier González588726d32017-06-26 11:57:29 +02001212 pblk->state = PBLK_STATE_RUNNING;
Hans Holmberg3e3a5b82017-10-13 14:46:34 +02001213 pblk->gc.gc_enabled = 0;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001214
Hans Holmberg6a3abf52018-06-01 16:41:05 +02001215 spin_lock_init(&pblk->resubmit_lock);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001216 spin_lock_init(&pblk->trans_lock);
1217 spin_lock_init(&pblk->lock);
1218
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001219#ifdef CONFIG_NVM_DEBUG
1220 atomic_long_set(&pblk->inflight_writes, 0);
1221 atomic_long_set(&pblk->padded_writes, 0);
1222 atomic_long_set(&pblk->padded_wb, 0);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001223 atomic_long_set(&pblk->req_writes, 0);
1224 atomic_long_set(&pblk->sub_writes, 0);
1225 atomic_long_set(&pblk->sync_writes, 0);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001226 atomic_long_set(&pblk->inflight_reads, 0);
Javier Gonzálezdb7ada32017-06-26 11:57:13 +02001227 atomic_long_set(&pblk->cache_reads, 0);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001228 atomic_long_set(&pblk->sync_reads, 0);
1229 atomic_long_set(&pblk->recov_writes, 0);
1230 atomic_long_set(&pblk->recov_writes, 0);
1231 atomic_long_set(&pblk->recov_gc_writes, 0);
Javier Gonzáleza1121172017-10-13 14:46:01 +02001232 atomic_long_set(&pblk->recov_gc_reads, 0);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001233#endif
1234
1235 atomic_long_set(&pblk->read_failed, 0);
1236 atomic_long_set(&pblk->read_empty, 0);
1237 atomic_long_set(&pblk->read_high_ecc, 0);
1238 atomic_long_set(&pblk->read_failed_gc, 0);
1239 atomic_long_set(&pblk->write_failed, 0);
1240 atomic_long_set(&pblk->erase_failed, 0);
1241
Javier González43d47122018-03-30 00:05:09 +02001242 ret = pblk_core_init(pblk);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001243 if (ret) {
Javier González43d47122018-03-30 00:05:09 +02001244 pr_err("pblk: could not initialize core\n");
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001245 goto fail;
1246 }
1247
1248 ret = pblk_lines_init(pblk);
1249 if (ret) {
1250 pr_err("pblk: could not initialize lines\n");
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001251 goto fail_free_core;
1252 }
1253
Javier González43d47122018-03-30 00:05:09 +02001254 ret = pblk_rwb_init(pblk);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001255 if (ret) {
Javier González43d47122018-03-30 00:05:09 +02001256 pr_err("pblk: could not initialize write buffer\n");
1257 goto fail_free_lines;
1258 }
1259
1260 ret = pblk_l2p_init(pblk, flags & NVM_TARGET_FACTORY);
1261 if (ret) {
1262 pr_err("pblk: could not initialize maps\n");
1263 goto fail_free_rwb;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001264 }
1265
1266 ret = pblk_writer_init(pblk);
1267 if (ret) {
Javier Gonzálezcc4f5ba2018-01-05 14:16:17 +01001268 if (ret != -EINTR)
1269 pr_err("pblk: could not initialize write thread\n");
Javier González43d47122018-03-30 00:05:09 +02001270 goto fail_free_l2p;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001271 }
1272
1273 ret = pblk_gc_init(pblk);
1274 if (ret) {
1275 pr_err("pblk: could not initialize gc\n");
1276 goto fail_stop_writer;
1277 }
1278
1279 /* inherit the size from the underlying device */
1280 blk_queue_logical_block_size(tqueue, queue_physical_block_size(bqueue));
1281 blk_queue_max_hw_sectors(tqueue, queue_max_hw_sectors(bqueue));
1282
1283 blk_queue_write_cache(tqueue, true, false);
1284
Javier Gonzáleze46f4e42018-03-30 00:05:10 +02001285 tqueue->limits.discard_granularity = geo->clba * geo->csecs;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001286 tqueue->limits.discard_alignment = 0;
1287 blk_queue_max_discard_sectors(tqueue, UINT_MAX >> 9);
Bart Van Assche8b904b52018-03-07 17:10:10 -08001288 blk_queue_flag_set(QUEUE_FLAG_DISCARD, tqueue);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001289
Javier González30d82a862018-01-05 14:16:19 +01001290 pr_info("pblk(%s): luns:%u, lines:%d, secs:%llu, buf entries:%u\n",
1291 tdisk->disk_name,
Matias Bjørlingfae7fae2018-01-05 14:16:03 +01001292 geo->all_luns, pblk->l_mg.nr_lines,
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001293 (unsigned long long)pblk->rl.nr_secs,
1294 pblk->rwb.nr_entries);
1295
1296 wake_up_process(pblk->writer_ts);
Hans Holmberg03661b52017-10-13 14:46:37 +02001297
1298 /* Check if we need to start GC */
1299 pblk_gc_should_kick(pblk);
1300
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001301 return pblk;
1302
1303fail_stop_writer:
1304 pblk_writer_stop(pblk);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001305fail_free_l2p:
1306 pblk_l2p_free(pblk);
Javier González43d47122018-03-30 00:05:09 +02001307fail_free_rwb:
1308 pblk_rwb_free(pblk);
1309fail_free_lines:
1310 pblk_lines_free(pblk);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001311fail_free_core:
1312 pblk_core_free(pblk);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001313fail:
1314 kfree(pblk);
1315 return ERR_PTR(ret);
1316}
1317
1318/* physical block device target */
1319static struct nvm_tgt_type tt_pblk = {
1320 .name = "pblk",
1321 .version = {1, 0, 0},
1322
1323 .make_rq = pblk_make_rq,
1324 .capacity = pblk_capacity,
1325
1326 .init = pblk_init,
1327 .exit = pblk_exit,
1328
1329 .sysfs_init = pblk_sysfs_init,
1330 .sysfs_exit = pblk_sysfs_exit,
Rakesh Pandit90014822017-10-13 14:45:50 +02001331 .owner = THIS_MODULE,
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001332};
1333
1334static int __init pblk_module_init(void)
1335{
NeilBrownb25d5232017-06-18 14:38:58 +10001336 int ret;
1337
Kent Overstreetb906bbb2018-05-20 18:25:50 -04001338 ret = bioset_init(&pblk_bio_set, BIO_POOL_SIZE, 0, 0);
1339 if (ret)
1340 return ret;
NeilBrownb25d5232017-06-18 14:38:58 +10001341 ret = nvm_register_tgt_type(&tt_pblk);
1342 if (ret)
Kent Overstreetb906bbb2018-05-20 18:25:50 -04001343 bioset_exit(&pblk_bio_set);
NeilBrownb25d5232017-06-18 14:38:58 +10001344 return ret;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001345}
1346
1347static void pblk_module_exit(void)
1348{
Kent Overstreetb906bbb2018-05-20 18:25:50 -04001349 bioset_exit(&pblk_bio_set);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001350 nvm_unregister_tgt_type(&tt_pblk);
1351}
1352
1353module_init(pblk_module_init);
1354module_exit(pblk_module_exit);
1355MODULE_AUTHOR("Javier Gonzalez <javier@cnexlabs.com>");
1356MODULE_AUTHOR("Matias Bjorling <matias@cnexlabs.com>");
1357MODULE_LICENSE("GPL v2");
1358MODULE_DESCRIPTION("Physical Block-Device for Open-Channel SSDs");