Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2016 CNEX Labs |
| 3 | * Initial release: Javier Gonzalez <javier@cnexlabs.com> |
| 4 | * Matias Bjorling <matias@cnexlabs.com> |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or |
| 7 | * modify it under the terms of the GNU General Public License version |
| 8 | * 2 as published by the Free Software Foundation. |
| 9 | * |
| 10 | * This program is distributed in the hope that it will be useful, but |
| 11 | * WITHOUT ANY WARRANTY; without even the implied warranty of |
| 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 13 | * General Public License for more details. |
| 14 | * |
| 15 | * pblk-read.c - pblk's read path |
| 16 | */ |
| 17 | |
| 18 | #include "pblk.h" |
| 19 | |
| 20 | /* |
| 21 | * There is no guarantee that the value read from cache has not been updated and |
| 22 | * resides at another location in the cache. We guarantee though that if the |
| 23 | * value is read from the cache, it belongs to the mapped lba. In order to |
| 24 | * guarantee and order between writes and reads are ordered, a flush must be |
| 25 | * issued. |
| 26 | */ |
| 27 | static int pblk_read_from_cache(struct pblk *pblk, struct bio *bio, |
| 28 | sector_t lba, struct ppa_addr ppa, |
Javier González | 75cb8e9 | 2017-07-28 15:13:16 +0200 | [diff] [blame] | 29 | int bio_iter, bool advanced_bio) |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 30 | { |
Matias Bjørling | 880eda5 | 2018-07-13 10:48:37 +0200 | [diff] [blame] | 31 | #ifdef CONFIG_NVM_PBLK_DEBUG |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 32 | /* Callers must ensure that the ppa points to a cache address */ |
| 33 | BUG_ON(pblk_ppa_empty(ppa)); |
| 34 | BUG_ON(!pblk_addr_in_cache(ppa)); |
| 35 | #endif |
| 36 | |
Javier González | 75cb8e9 | 2017-07-28 15:13:16 +0200 | [diff] [blame] | 37 | return pblk_rb_copy_to_bio(&pblk->rwb, bio, lba, ppa, |
| 38 | bio_iter, advanced_bio); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 39 | } |
| 40 | |
| 41 | static void pblk_read_ppalist_rq(struct pblk *pblk, struct nvm_rq *rqd, |
Javier González | 87cc40b | 2018-06-01 16:41:11 +0200 | [diff] [blame] | 42 | struct bio *bio, sector_t blba, |
| 43 | unsigned long *read_bitmap) |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 44 | { |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 45 | struct pblk_sec_meta *meta_list = rqd->meta_list; |
Matias Bjørling | afdc23c | 2018-10-09 13:11:37 +0200 | [diff] [blame] | 46 | struct ppa_addr ppas[NVM_MAX_VLBA]; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 47 | int nr_secs = rqd->nr_ppas; |
Javier González | 75cb8e9 | 2017-07-28 15:13:16 +0200 | [diff] [blame] | 48 | bool advanced_bio = false; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 49 | int i, j = 0; |
| 50 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 51 | pblk_lookup_l2p_seq(pblk, ppas, blba, nr_secs); |
| 52 | |
| 53 | for (i = 0; i < nr_secs; i++) { |
| 54 | struct ppa_addr p = ppas[i]; |
| 55 | sector_t lba = blba + i; |
| 56 | |
| 57 | retry: |
| 58 | if (pblk_ppa_empty(p)) { |
| 59 | WARN_ON(test_and_set_bit(i, read_bitmap)); |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 60 | meta_list[i].lba = cpu_to_le64(ADDR_EMPTY); |
Javier González | 75cb8e9 | 2017-07-28 15:13:16 +0200 | [diff] [blame] | 61 | |
| 62 | if (unlikely(!advanced_bio)) { |
| 63 | bio_advance(bio, (i) * PBLK_EXPOSED_PAGE_SIZE); |
| 64 | advanced_bio = true; |
| 65 | } |
| 66 | |
| 67 | goto next; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 68 | } |
| 69 | |
| 70 | /* Try to read from write buffer. The address is later checked |
| 71 | * on the write buffer to prevent retrieving overwritten data. |
| 72 | */ |
| 73 | if (pblk_addr_in_cache(p)) { |
Javier González | 75cb8e9 | 2017-07-28 15:13:16 +0200 | [diff] [blame] | 74 | if (!pblk_read_from_cache(pblk, bio, lba, p, i, |
| 75 | advanced_bio)) { |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 76 | pblk_lookup_l2p_seq(pblk, &p, lba, 1); |
| 77 | goto retry; |
| 78 | } |
| 79 | WARN_ON(test_and_set_bit(i, read_bitmap)); |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 80 | meta_list[i].lba = cpu_to_le64(lba); |
Javier González | 75cb8e9 | 2017-07-28 15:13:16 +0200 | [diff] [blame] | 81 | advanced_bio = true; |
Matias Bjørling | 880eda5 | 2018-07-13 10:48:37 +0200 | [diff] [blame] | 82 | #ifdef CONFIG_NVM_PBLK_DEBUG |
Javier González | db7ada3 | 2017-06-26 11:57:13 +0200 | [diff] [blame] | 83 | atomic_long_inc(&pblk->cache_reads); |
| 84 | #endif |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 85 | } else { |
| 86 | /* Read from media non-cached sectors */ |
| 87 | rqd->ppa_list[j++] = p; |
| 88 | } |
| 89 | |
Javier González | 75cb8e9 | 2017-07-28 15:13:16 +0200 | [diff] [blame] | 90 | next: |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 91 | if (advanced_bio) |
| 92 | bio_advance(bio, PBLK_EXPOSED_PAGE_SIZE); |
| 93 | } |
| 94 | |
Javier González | f9c1015 | 2017-06-26 11:57:20 +0200 | [diff] [blame] | 95 | if (pblk_io_aligned(pblk, nr_secs)) |
Matias Bjørling | d7b6801 | 2018-10-09 13:11:32 +0200 | [diff] [blame] | 96 | rqd->is_seq = 1; |
Javier González | f9c1015 | 2017-06-26 11:57:20 +0200 | [diff] [blame] | 97 | |
Matias Bjørling | 880eda5 | 2018-07-13 10:48:37 +0200 | [diff] [blame] | 98 | #ifdef CONFIG_NVM_PBLK_DEBUG |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 99 | atomic_long_add(nr_secs, &pblk->inflight_reads); |
| 100 | #endif |
| 101 | } |
| 102 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 103 | |
Javier González | 03a34b2 | 2018-06-01 15:04:18 +0200 | [diff] [blame] | 104 | static void pblk_read_check_seq(struct pblk *pblk, struct nvm_rq *rqd, |
| 105 | sector_t blba) |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 106 | { |
Javier González | 03a34b2 | 2018-06-01 15:04:18 +0200 | [diff] [blame] | 107 | struct pblk_sec_meta *meta_lba_list = rqd->meta_list; |
| 108 | int nr_lbas = rqd->nr_ppas; |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 109 | int i; |
| 110 | |
| 111 | for (i = 0; i < nr_lbas; i++) { |
Javier González | 310df58 | 2018-06-01 15:04:17 +0200 | [diff] [blame] | 112 | u64 lba = le64_to_cpu(meta_lba_list[i].lba); |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 113 | |
| 114 | if (lba == ADDR_EMPTY) |
| 115 | continue; |
| 116 | |
Javier González | 03a34b2 | 2018-06-01 15:04:18 +0200 | [diff] [blame] | 117 | if (lba != blba + i) { |
Matias Bjørling | 880eda5 | 2018-07-13 10:48:37 +0200 | [diff] [blame] | 118 | #ifdef CONFIG_NVM_PBLK_DEBUG |
Hans Holmberg | d68a934 | 2018-10-09 13:11:46 +0200 | [diff] [blame] | 119 | struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd); |
Javier González | 03a34b2 | 2018-06-01 15:04:18 +0200 | [diff] [blame] | 120 | |
Hans Holmberg | d68a934 | 2018-10-09 13:11:46 +0200 | [diff] [blame] | 121 | print_ppa(pblk, &ppa_list[i], "seq", i); |
Javier González | 03a34b2 | 2018-06-01 15:04:18 +0200 | [diff] [blame] | 122 | #endif |
Matias Bjørling | 4e495a4 | 2018-07-13 10:48:42 +0200 | [diff] [blame] | 123 | pblk_err(pblk, "corrupted read LBA (%llu/%llu)\n", |
Javier González | 03a34b2 | 2018-06-01 15:04:18 +0200 | [diff] [blame] | 124 | lba, (u64)blba + i); |
| 125 | WARN_ON(1); |
| 126 | } |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 127 | } |
| 128 | } |
| 129 | |
Javier González | 310df58 | 2018-06-01 15:04:17 +0200 | [diff] [blame] | 130 | /* |
| 131 | * There can be holes in the lba list. |
| 132 | */ |
Javier González | 03a34b2 | 2018-06-01 15:04:18 +0200 | [diff] [blame] | 133 | static void pblk_read_check_rand(struct pblk *pblk, struct nvm_rq *rqd, |
| 134 | u64 *lba_list, int nr_lbas) |
Javier González | 310df58 | 2018-06-01 15:04:17 +0200 | [diff] [blame] | 135 | { |
Javier González | 03a34b2 | 2018-06-01 15:04:18 +0200 | [diff] [blame] | 136 | struct pblk_sec_meta *meta_lba_list = rqd->meta_list; |
Javier González | 310df58 | 2018-06-01 15:04:17 +0200 | [diff] [blame] | 137 | int i, j; |
| 138 | |
| 139 | for (i = 0, j = 0; i < nr_lbas; i++) { |
| 140 | u64 lba = lba_list[i]; |
| 141 | u64 meta_lba; |
| 142 | |
| 143 | if (lba == ADDR_EMPTY) |
| 144 | continue; |
| 145 | |
Javier González | 03a34b2 | 2018-06-01 15:04:18 +0200 | [diff] [blame] | 146 | meta_lba = le64_to_cpu(meta_lba_list[j].lba); |
Javier González | 310df58 | 2018-06-01 15:04:17 +0200 | [diff] [blame] | 147 | |
| 148 | if (lba != meta_lba) { |
Matias Bjørling | 880eda5 | 2018-07-13 10:48:37 +0200 | [diff] [blame] | 149 | #ifdef CONFIG_NVM_PBLK_DEBUG |
Hans Holmberg | d68a934 | 2018-10-09 13:11:46 +0200 | [diff] [blame] | 150 | struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd); |
Javier González | 03a34b2 | 2018-06-01 15:04:18 +0200 | [diff] [blame] | 151 | |
Hans Holmberg | ea1d24b | 2018-10-09 13:11:49 +0200 | [diff] [blame^] | 152 | print_ppa(pblk, &ppa_list[j], "rnd", j); |
Javier González | 03a34b2 | 2018-06-01 15:04:18 +0200 | [diff] [blame] | 153 | #endif |
Matias Bjørling | 4e495a4 | 2018-07-13 10:48:42 +0200 | [diff] [blame] | 154 | pblk_err(pblk, "corrupted read LBA (%llu/%llu)\n", |
Hans Holmberg | ea1d24b | 2018-10-09 13:11:49 +0200 | [diff] [blame^] | 155 | meta_lba, lba); |
Javier González | 310df58 | 2018-06-01 15:04:17 +0200 | [diff] [blame] | 156 | WARN_ON(1); |
| 157 | } |
Javier González | 03a34b2 | 2018-06-01 15:04:18 +0200 | [diff] [blame] | 158 | |
| 159 | j++; |
Javier González | 310df58 | 2018-06-01 15:04:17 +0200 | [diff] [blame] | 160 | } |
Javier González | 03a34b2 | 2018-06-01 15:04:18 +0200 | [diff] [blame] | 161 | |
| 162 | WARN_ONCE(j != rqd->nr_ppas, "pblk: corrupted random request\n"); |
Javier González | 310df58 | 2018-06-01 15:04:17 +0200 | [diff] [blame] | 163 | } |
| 164 | |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 165 | static void pblk_end_user_read(struct bio *bio) |
| 166 | { |
Matias Bjørling | 880eda5 | 2018-07-13 10:48:37 +0200 | [diff] [blame] | 167 | #ifdef CONFIG_NVM_PBLK_DEBUG |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 168 | WARN_ONCE(bio->bi_status, "pblk: corrupted read bio\n"); |
| 169 | #endif |
| 170 | bio_endio(bio); |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 171 | } |
| 172 | |
| 173 | static void __pblk_end_io_read(struct pblk *pblk, struct nvm_rq *rqd, |
| 174 | bool put_line) |
| 175 | { |
Javier González | 998ba62 | 2018-01-05 14:16:20 +0100 | [diff] [blame] | 176 | struct nvm_tgt_dev *dev = pblk->dev; |
Javier González | 084ec9b | 2017-06-26 16:27:13 -0600 | [diff] [blame] | 177 | struct pblk_g_ctx *r_ctx = nvm_rq_to_pdu(rqd); |
Javier González | 87cc40b | 2018-06-01 16:41:11 +0200 | [diff] [blame] | 178 | struct bio *int_bio = rqd->bio; |
Javier González | 998ba62 | 2018-01-05 14:16:20 +0100 | [diff] [blame] | 179 | unsigned long start_time = r_ctx->start_time; |
| 180 | |
Michael Callahan | ddcf35d | 2018-07-18 04:47:39 -0700 | [diff] [blame] | 181 | generic_end_io_acct(dev->q, REQ_OP_READ, &pblk->disk->part0, start_time); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 182 | |
| 183 | if (rqd->error) |
| 184 | pblk_log_read_err(pblk, rqd); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 185 | |
Javier González | 03a34b2 | 2018-06-01 15:04:18 +0200 | [diff] [blame] | 186 | pblk_read_check_seq(pblk, rqd, r_ctx->lba); |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 187 | |
Javier González | 87cc40b | 2018-06-01 16:41:11 +0200 | [diff] [blame] | 188 | if (int_bio) |
| 189 | bio_put(int_bio); |
Javier González | 084ec9b | 2017-06-26 16:27:13 -0600 | [diff] [blame] | 190 | |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 191 | if (put_line) |
Matias Bjørling | ae14cc0 | 2018-10-09 13:11:40 +0200 | [diff] [blame] | 192 | pblk_rq_to_line_put(pblk, rqd); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 193 | |
Matias Bjørling | 880eda5 | 2018-07-13 10:48:37 +0200 | [diff] [blame] | 194 | #ifdef CONFIG_NVM_PBLK_DEBUG |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 195 | atomic_long_add(rqd->nr_ppas, &pblk->sync_reads); |
| 196 | atomic_long_sub(rqd->nr_ppas, &pblk->inflight_reads); |
| 197 | #endif |
| 198 | |
Javier González | e2cddf2 | 2017-10-13 14:46:19 +0200 | [diff] [blame] | 199 | pblk_free_rqd(pblk, rqd, PBLK_READ); |
Javier González | 588726d3 | 2017-06-26 11:57:29 +0200 | [diff] [blame] | 200 | atomic_dec(&pblk->inflight_io); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 201 | } |
| 202 | |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 203 | static void pblk_end_io_read(struct nvm_rq *rqd) |
| 204 | { |
| 205 | struct pblk *pblk = rqd->private; |
Javier González | 87cc40b | 2018-06-01 16:41:11 +0200 | [diff] [blame] | 206 | struct pblk_g_ctx *r_ctx = nvm_rq_to_pdu(rqd); |
| 207 | struct bio *bio = (struct bio *)r_ctx->private; |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 208 | |
Javier González | 87cc40b | 2018-06-01 16:41:11 +0200 | [diff] [blame] | 209 | pblk_end_user_read(bio); |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 210 | __pblk_end_io_read(pblk, rqd, true); |
| 211 | } |
| 212 | |
Heiner Litz | 11f6ad6 | 2018-07-13 10:48:44 +0200 | [diff] [blame] | 213 | static void pblk_end_partial_read(struct nvm_rq *rqd) |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 214 | { |
Heiner Litz | 11f6ad6 | 2018-07-13 10:48:44 +0200 | [diff] [blame] | 215 | struct pblk *pblk = rqd->private; |
| 216 | struct pblk_g_ctx *r_ctx = nvm_rq_to_pdu(rqd); |
| 217 | struct pblk_pr_ctx *pr_ctx = r_ctx->private; |
| 218 | struct bio *new_bio = rqd->bio; |
| 219 | struct bio *bio = pr_ctx->orig_bio; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 220 | struct bio_vec src_bv, dst_bv; |
Heiner Litz | 11f6ad6 | 2018-07-13 10:48:44 +0200 | [diff] [blame] | 221 | struct pblk_sec_meta *meta_list = rqd->meta_list; |
| 222 | int bio_init_idx = pr_ctx->bio_init_idx; |
| 223 | unsigned long *read_bitmap = pr_ctx->bitmap; |
| 224 | int nr_secs = pr_ctx->orig_nr_secs; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 225 | int nr_holes = nr_secs - bitmap_weight(read_bitmap, nr_secs); |
Heiner Litz | 11f6ad6 | 2018-07-13 10:48:44 +0200 | [diff] [blame] | 226 | __le64 *lba_list_mem, *lba_list_media; |
| 227 | void *src_p, *dst_p; |
| 228 | int hole, i; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 229 | |
Javier González | 0f9248c | 2017-10-13 14:46:24 +0200 | [diff] [blame] | 230 | if (unlikely(nr_holes == 1)) { |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 231 | struct ppa_addr ppa; |
| 232 | |
| 233 | ppa = rqd->ppa_addr; |
Heiner Litz | 11f6ad6 | 2018-07-13 10:48:44 +0200 | [diff] [blame] | 234 | rqd->ppa_list = pr_ctx->ppa_ptr; |
| 235 | rqd->dma_ppa_list = pr_ctx->dma_ppa_list; |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 236 | rqd->ppa_list[0] = ppa; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 237 | } |
| 238 | |
Heiner Litz | 11f6ad6 | 2018-07-13 10:48:44 +0200 | [diff] [blame] | 239 | /* Re-use allocated memory for intermediate lbas */ |
| 240 | lba_list_mem = (((void *)rqd->ppa_list) + pblk_dma_ppa_size); |
| 241 | lba_list_media = (((void *)rqd->ppa_list) + 2 * pblk_dma_ppa_size); |
| 242 | |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 243 | for (i = 0; i < nr_secs; i++) { |
| 244 | lba_list_media[i] = meta_list[i].lba; |
| 245 | meta_list[i].lba = lba_list_mem[i]; |
| 246 | } |
| 247 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 248 | /* Fill the holes in the original bio */ |
| 249 | i = 0; |
| 250 | hole = find_first_zero_bit(read_bitmap, nr_secs); |
| 251 | do { |
Javier González | cb21665 | 2018-10-09 13:11:42 +0200 | [diff] [blame] | 252 | struct pblk_line *line; |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 253 | |
Javier González | cb21665 | 2018-10-09 13:11:42 +0200 | [diff] [blame] | 254 | line = pblk_ppa_to_line(pblk, rqd->ppa_list[i]); |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 255 | kref_put(&line->ref, pblk_line_put); |
| 256 | |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 257 | meta_list[hole].lba = lba_list_media[i]; |
| 258 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 259 | src_bv = new_bio->bi_io_vec[i++]; |
Heiner Litz | 11f6ad6 | 2018-07-13 10:48:44 +0200 | [diff] [blame] | 260 | dst_bv = bio->bi_io_vec[bio_init_idx + hole]; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 261 | |
| 262 | src_p = kmap_atomic(src_bv.bv_page); |
| 263 | dst_p = kmap_atomic(dst_bv.bv_page); |
| 264 | |
| 265 | memcpy(dst_p + dst_bv.bv_offset, |
| 266 | src_p + src_bv.bv_offset, |
| 267 | PBLK_EXPOSED_PAGE_SIZE); |
| 268 | |
| 269 | kunmap_atomic(src_p); |
| 270 | kunmap_atomic(dst_p); |
| 271 | |
Kent Overstreet | b906bbb | 2018-05-20 18:25:50 -0400 | [diff] [blame] | 272 | mempool_free(src_bv.bv_page, &pblk->page_bio_pool); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 273 | |
| 274 | hole = find_next_zero_bit(read_bitmap, nr_secs, hole + 1); |
| 275 | } while (hole < nr_secs); |
| 276 | |
| 277 | bio_put(new_bio); |
Heiner Litz | 11f6ad6 | 2018-07-13 10:48:44 +0200 | [diff] [blame] | 278 | kfree(pr_ctx); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 279 | |
Javier González | 87cc40b | 2018-06-01 16:41:11 +0200 | [diff] [blame] | 280 | /* restore original request */ |
| 281 | rqd->bio = NULL; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 282 | rqd->nr_ppas = nr_secs; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 283 | |
Heiner Litz | 11f6ad6 | 2018-07-13 10:48:44 +0200 | [diff] [blame] | 284 | bio_endio(bio); |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 285 | __pblk_end_io_read(pblk, rqd, false); |
Heiner Litz | 11f6ad6 | 2018-07-13 10:48:44 +0200 | [diff] [blame] | 286 | } |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 287 | |
Heiner Litz | 11f6ad6 | 2018-07-13 10:48:44 +0200 | [diff] [blame] | 288 | static int pblk_setup_partial_read(struct pblk *pblk, struct nvm_rq *rqd, |
| 289 | unsigned int bio_init_idx, |
| 290 | unsigned long *read_bitmap, |
| 291 | int nr_holes) |
| 292 | { |
| 293 | struct pblk_sec_meta *meta_list = rqd->meta_list; |
| 294 | struct pblk_g_ctx *r_ctx = nvm_rq_to_pdu(rqd); |
| 295 | struct pblk_pr_ctx *pr_ctx; |
| 296 | struct bio *new_bio, *bio = r_ctx->private; |
| 297 | __le64 *lba_list_mem; |
| 298 | int nr_secs = rqd->nr_ppas; |
| 299 | int i; |
| 300 | |
| 301 | /* Re-use allocated memory for intermediate lbas */ |
| 302 | lba_list_mem = (((void *)rqd->ppa_list) + pblk_dma_ppa_size); |
| 303 | |
| 304 | new_bio = bio_alloc(GFP_KERNEL, nr_holes); |
| 305 | |
| 306 | if (pblk_bio_add_pages(pblk, new_bio, GFP_KERNEL, nr_holes)) |
| 307 | goto fail_bio_put; |
| 308 | |
| 309 | if (nr_holes != new_bio->bi_vcnt) { |
| 310 | WARN_ONCE(1, "pblk: malformed bio\n"); |
| 311 | goto fail_free_pages; |
| 312 | } |
| 313 | |
| 314 | pr_ctx = kmalloc(sizeof(struct pblk_pr_ctx), GFP_KERNEL); |
| 315 | if (!pr_ctx) |
| 316 | goto fail_free_pages; |
| 317 | |
| 318 | for (i = 0; i < nr_secs; i++) |
| 319 | lba_list_mem[i] = meta_list[i].lba; |
| 320 | |
| 321 | new_bio->bi_iter.bi_sector = 0; /* internal bio */ |
| 322 | bio_set_op_attrs(new_bio, REQ_OP_READ, 0); |
| 323 | |
| 324 | rqd->bio = new_bio; |
| 325 | rqd->nr_ppas = nr_holes; |
Heiner Litz | 11f6ad6 | 2018-07-13 10:48:44 +0200 | [diff] [blame] | 326 | |
| 327 | pr_ctx->ppa_ptr = NULL; |
| 328 | pr_ctx->orig_bio = bio; |
| 329 | bitmap_copy(pr_ctx->bitmap, read_bitmap, NVM_MAX_VLBA); |
| 330 | pr_ctx->bio_init_idx = bio_init_idx; |
| 331 | pr_ctx->orig_nr_secs = nr_secs; |
| 332 | r_ctx->private = pr_ctx; |
| 333 | |
| 334 | if (unlikely(nr_holes == 1)) { |
| 335 | pr_ctx->ppa_ptr = rqd->ppa_list; |
| 336 | pr_ctx->dma_ppa_list = rqd->dma_ppa_list; |
| 337 | rqd->ppa_addr = rqd->ppa_list[0]; |
| 338 | } |
| 339 | return 0; |
| 340 | |
| 341 | fail_free_pages: |
Igor Konopko | fbadca7 | 2018-06-01 16:41:09 +0200 | [diff] [blame] | 342 | pblk_bio_free_pages(pblk, new_bio, 0, new_bio->bi_vcnt); |
Heiner Litz | 11f6ad6 | 2018-07-13 10:48:44 +0200 | [diff] [blame] | 343 | fail_bio_put: |
| 344 | bio_put(new_bio); |
| 345 | |
| 346 | return -ENOMEM; |
| 347 | } |
| 348 | |
| 349 | static int pblk_partial_read_bio(struct pblk *pblk, struct nvm_rq *rqd, |
| 350 | unsigned int bio_init_idx, |
| 351 | unsigned long *read_bitmap, int nr_secs) |
| 352 | { |
| 353 | int nr_holes; |
| 354 | int ret; |
| 355 | |
| 356 | nr_holes = nr_secs - bitmap_weight(read_bitmap, nr_secs); |
| 357 | |
| 358 | if (pblk_setup_partial_read(pblk, rqd, bio_init_idx, read_bitmap, |
| 359 | nr_holes)) |
| 360 | return NVM_IO_ERR; |
| 361 | |
| 362 | rqd->end_io = pblk_end_partial_read; |
| 363 | |
| 364 | ret = pblk_submit_io(pblk, rqd); |
| 365 | if (ret) { |
| 366 | bio_put(rqd->bio); |
| 367 | pblk_err(pblk, "partial read IO submission failed\n"); |
| 368 | goto err; |
| 369 | } |
| 370 | |
| 371 | return NVM_IO_OK; |
| 372 | |
| 373 | err: |
Matias Bjørling | 4e495a4 | 2018-07-13 10:48:42 +0200 | [diff] [blame] | 374 | pblk_err(pblk, "failed to perform partial read\n"); |
Heiner Litz | 11f6ad6 | 2018-07-13 10:48:44 +0200 | [diff] [blame] | 375 | |
| 376 | /* Free allocated pages in new bio */ |
| 377 | pblk_bio_free_pages(pblk, rqd->bio, 0, rqd->bio->bi_vcnt); |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 378 | __pblk_end_io_read(pblk, rqd, false); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 379 | return NVM_IO_ERR; |
| 380 | } |
| 381 | |
Javier González | 87cc40b | 2018-06-01 16:41:11 +0200 | [diff] [blame] | 382 | static void pblk_read_rq(struct pblk *pblk, struct nvm_rq *rqd, struct bio *bio, |
Javier González | 84454e6 | 2017-10-13 14:46:13 +0200 | [diff] [blame] | 383 | sector_t lba, unsigned long *read_bitmap) |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 384 | { |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 385 | struct pblk_sec_meta *meta_list = rqd->meta_list; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 386 | struct ppa_addr ppa; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 387 | |
| 388 | pblk_lookup_l2p_seq(pblk, &ppa, lba, 1); |
| 389 | |
Matias Bjørling | 880eda5 | 2018-07-13 10:48:37 +0200 | [diff] [blame] | 390 | #ifdef CONFIG_NVM_PBLK_DEBUG |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 391 | atomic_long_inc(&pblk->inflight_reads); |
| 392 | #endif |
| 393 | |
| 394 | retry: |
| 395 | if (pblk_ppa_empty(ppa)) { |
| 396 | WARN_ON(test_and_set_bit(0, read_bitmap)); |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 397 | meta_list[0].lba = cpu_to_le64(ADDR_EMPTY); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 398 | return; |
| 399 | } |
| 400 | |
| 401 | /* Try to read from write buffer. The address is later checked on the |
| 402 | * write buffer to prevent retrieving overwritten data. |
| 403 | */ |
| 404 | if (pblk_addr_in_cache(ppa)) { |
Javier González | 75cb8e9 | 2017-07-28 15:13:16 +0200 | [diff] [blame] | 405 | if (!pblk_read_from_cache(pblk, bio, lba, ppa, 0, 1)) { |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 406 | pblk_lookup_l2p_seq(pblk, &ppa, lba, 1); |
| 407 | goto retry; |
| 408 | } |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 409 | |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 410 | WARN_ON(test_and_set_bit(0, read_bitmap)); |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 411 | meta_list[0].lba = cpu_to_le64(lba); |
| 412 | |
Matias Bjørling | 880eda5 | 2018-07-13 10:48:37 +0200 | [diff] [blame] | 413 | #ifdef CONFIG_NVM_PBLK_DEBUG |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 414 | atomic_long_inc(&pblk->cache_reads); |
Javier González | db7ada3 | 2017-06-26 11:57:13 +0200 | [diff] [blame] | 415 | #endif |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 416 | } else { |
| 417 | rqd->ppa_addr = ppa; |
| 418 | } |
| 419 | } |
| 420 | |
| 421 | int pblk_submit_read(struct pblk *pblk, struct bio *bio) |
| 422 | { |
| 423 | struct nvm_tgt_dev *dev = pblk->dev; |
Javier González | 998ba62 | 2018-01-05 14:16:20 +0100 | [diff] [blame] | 424 | struct request_queue *q = dev->q; |
Javier González | 84454e6 | 2017-10-13 14:46:13 +0200 | [diff] [blame] | 425 | sector_t blba = pblk_get_lba(bio); |
Dan Carpenter | 5bf1e1ee | 2017-04-21 16:48:40 -0600 | [diff] [blame] | 426 | unsigned int nr_secs = pblk_get_secs(bio); |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 427 | struct pblk_g_ctx *r_ctx; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 428 | struct nvm_rq *rqd; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 429 | unsigned int bio_init_idx; |
Matias Bjørling | 921aebf | 2018-07-13 10:48:40 +0200 | [diff] [blame] | 430 | DECLARE_BITMAP(read_bitmap, NVM_MAX_VLBA); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 431 | int ret = NVM_IO_ERR; |
| 432 | |
Michael Callahan | ddcf35d | 2018-07-18 04:47:39 -0700 | [diff] [blame] | 433 | generic_start_io_acct(q, REQ_OP_READ, bio_sectors(bio), |
| 434 | &pblk->disk->part0); |
Javier González | 998ba62 | 2018-01-05 14:16:20 +0100 | [diff] [blame] | 435 | |
Matias Bjørling | 921aebf | 2018-07-13 10:48:40 +0200 | [diff] [blame] | 436 | bitmap_zero(read_bitmap, nr_secs); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 437 | |
Javier González | e2cddf2 | 2017-10-13 14:46:19 +0200 | [diff] [blame] | 438 | rqd = pblk_alloc_rqd(pblk, PBLK_READ); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 439 | |
| 440 | rqd->opcode = NVM_OP_PREAD; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 441 | rqd->nr_ppas = nr_secs; |
Javier González | 87cc40b | 2018-06-01 16:41:11 +0200 | [diff] [blame] | 442 | rqd->bio = NULL; /* cloned bio if needed */ |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 443 | rqd->private = pblk; |
| 444 | rqd->end_io = pblk_end_io_read; |
| 445 | |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 446 | r_ctx = nvm_rq_to_pdu(rqd); |
Javier González | 998ba62 | 2018-01-05 14:16:20 +0100 | [diff] [blame] | 447 | r_ctx->start_time = jiffies; |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 448 | r_ctx->lba = blba; |
Javier González | 87cc40b | 2018-06-01 16:41:11 +0200 | [diff] [blame] | 449 | r_ctx->private = bio; /* original bio */ |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 450 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 451 | /* Save the index for this bio's start. This is needed in case |
| 452 | * we need to fill a partial read. |
| 453 | */ |
| 454 | bio_init_idx = pblk_get_bi_idx(bio); |
| 455 | |
Javier González | 63e3809 | 2017-06-26 11:57:24 +0200 | [diff] [blame] | 456 | rqd->meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL, |
| 457 | &rqd->dma_meta_list); |
| 458 | if (!rqd->meta_list) { |
Matias Bjørling | 4e495a4 | 2018-07-13 10:48:42 +0200 | [diff] [blame] | 459 | pblk_err(pblk, "not able to allocate ppa list\n"); |
Javier González | 63e3809 | 2017-06-26 11:57:24 +0200 | [diff] [blame] | 460 | goto fail_rqd_free; |
| 461 | } |
| 462 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 463 | if (nr_secs > 1) { |
Javier González | 63e3809 | 2017-06-26 11:57:24 +0200 | [diff] [blame] | 464 | rqd->ppa_list = rqd->meta_list + pblk_dma_meta_size; |
| 465 | rqd->dma_ppa_list = rqd->dma_meta_list + pblk_dma_meta_size; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 466 | |
Matias Bjørling | 921aebf | 2018-07-13 10:48:40 +0200 | [diff] [blame] | 467 | pblk_read_ppalist_rq(pblk, rqd, bio, blba, read_bitmap); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 468 | } else { |
Matias Bjørling | 921aebf | 2018-07-13 10:48:40 +0200 | [diff] [blame] | 469 | pblk_read_rq(pblk, rqd, bio, blba, read_bitmap); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 470 | } |
| 471 | |
Matias Bjørling | 921aebf | 2018-07-13 10:48:40 +0200 | [diff] [blame] | 472 | if (bitmap_full(read_bitmap, nr_secs)) { |
Javier González | 588726d3 | 2017-06-26 11:57:29 +0200 | [diff] [blame] | 473 | atomic_inc(&pblk->inflight_io); |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 474 | __pblk_end_io_read(pblk, rqd, false); |
Javier González | 87cc40b | 2018-06-01 16:41:11 +0200 | [diff] [blame] | 475 | return NVM_IO_DONE; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 476 | } |
| 477 | |
| 478 | /* All sectors are to be read from the device */ |
Matias Bjørling | 921aebf | 2018-07-13 10:48:40 +0200 | [diff] [blame] | 479 | if (bitmap_empty(read_bitmap, rqd->nr_ppas)) { |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 480 | struct bio *int_bio = NULL; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 481 | |
| 482 | /* Clone read bio to deal with read errors internally */ |
Kent Overstreet | b906bbb | 2018-05-20 18:25:50 -0400 | [diff] [blame] | 483 | int_bio = bio_clone_fast(bio, GFP_KERNEL, &pblk_bio_set); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 484 | if (!int_bio) { |
Matias Bjørling | 4e495a4 | 2018-07-13 10:48:42 +0200 | [diff] [blame] | 485 | pblk_err(pblk, "could not clone read bio\n"); |
Javier González | 998ba62 | 2018-01-05 14:16:20 +0100 | [diff] [blame] | 486 | goto fail_end_io; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 487 | } |
| 488 | |
| 489 | rqd->bio = int_bio; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 490 | |
Javier González | e13f421 | 2018-06-01 15:04:21 +0200 | [diff] [blame] | 491 | if (pblk_submit_io(pblk, rqd)) { |
Matias Bjørling | 4e495a4 | 2018-07-13 10:48:42 +0200 | [diff] [blame] | 492 | pblk_err(pblk, "read IO submission failed\n"); |
Javier González | e13f421 | 2018-06-01 15:04:21 +0200 | [diff] [blame] | 493 | ret = NVM_IO_ERR; |
Javier González | 998ba62 | 2018-01-05 14:16:20 +0100 | [diff] [blame] | 494 | goto fail_end_io; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 495 | } |
| 496 | |
| 497 | return NVM_IO_OK; |
| 498 | } |
| 499 | |
| 500 | /* The read bio request could be partially filled by the write buffer, |
| 501 | * but there are some holes that need to be read from the drive. |
| 502 | */ |
Heiner Litz | 11f6ad6 | 2018-07-13 10:48:44 +0200 | [diff] [blame] | 503 | ret = pblk_partial_read_bio(pblk, rqd, bio_init_idx, read_bitmap, |
| 504 | nr_secs); |
| 505 | if (ret) |
| 506 | goto fail_meta_free; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 507 | |
Heiner Litz | 11f6ad6 | 2018-07-13 10:48:44 +0200 | [diff] [blame] | 508 | return NVM_IO_OK; |
| 509 | |
| 510 | fail_meta_free: |
| 511 | nvm_dev_dma_free(dev->parent, rqd->meta_list, rqd->dma_meta_list); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 512 | fail_rqd_free: |
Javier González | e2cddf2 | 2017-10-13 14:46:19 +0200 | [diff] [blame] | 513 | pblk_free_rqd(pblk, rqd, PBLK_READ); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 514 | return ret; |
Javier González | 998ba62 | 2018-01-05 14:16:20 +0100 | [diff] [blame] | 515 | fail_end_io: |
| 516 | __pblk_end_io_read(pblk, rqd, false); |
| 517 | return ret; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 518 | } |
| 519 | |
| 520 | static int read_ppalist_rq_gc(struct pblk *pblk, struct nvm_rq *rqd, |
| 521 | struct pblk_line *line, u64 *lba_list, |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 522 | u64 *paddr_list_gc, unsigned int nr_secs) |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 523 | { |
Matias Bjørling | afdc23c | 2018-10-09 13:11:37 +0200 | [diff] [blame] | 524 | struct ppa_addr ppa_list_l2p[NVM_MAX_VLBA]; |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 525 | struct ppa_addr ppa_gc; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 526 | int valid_secs = 0; |
| 527 | int i; |
| 528 | |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 529 | pblk_lookup_l2p_rand(pblk, ppa_list_l2p, lba_list, nr_secs); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 530 | |
| 531 | for (i = 0; i < nr_secs; i++) { |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 532 | if (lba_list[i] == ADDR_EMPTY) |
| 533 | continue; |
| 534 | |
| 535 | ppa_gc = addr_to_gen_ppa(pblk, paddr_list_gc[i], line->id); |
| 536 | if (!pblk_ppa_comp(ppa_list_l2p[i], ppa_gc)) { |
| 537 | paddr_list_gc[i] = lba_list[i] = ADDR_EMPTY; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 538 | continue; |
| 539 | } |
| 540 | |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 541 | rqd->ppa_list[valid_secs++] = ppa_list_l2p[i]; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 542 | } |
| 543 | |
Matias Bjørling | 880eda5 | 2018-07-13 10:48:37 +0200 | [diff] [blame] | 544 | #ifdef CONFIG_NVM_PBLK_DEBUG |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 545 | atomic_long_add(valid_secs, &pblk->inflight_reads); |
| 546 | #endif |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 547 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 548 | return valid_secs; |
| 549 | } |
| 550 | |
| 551 | static int read_rq_gc(struct pblk *pblk, struct nvm_rq *rqd, |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 552 | struct pblk_line *line, sector_t lba, |
| 553 | u64 paddr_gc) |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 554 | { |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 555 | struct ppa_addr ppa_l2p, ppa_gc; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 556 | int valid_secs = 0; |
| 557 | |
Dan Carpenter | 659226e | 2017-04-21 16:49:56 -0600 | [diff] [blame] | 558 | if (lba == ADDR_EMPTY) |
| 559 | goto out; |
| 560 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 561 | /* logic error: lba out-of-bounds */ |
Dan Carpenter | 2a79efd | 2017-04-15 20:55:52 +0200 | [diff] [blame] | 562 | if (lba >= pblk->rl.nr_secs) { |
| 563 | WARN(1, "pblk: read lba out of bounds\n"); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 564 | goto out; |
| 565 | } |
| 566 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 567 | spin_lock(&pblk->trans_lock); |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 568 | ppa_l2p = pblk_trans_map_get(pblk, lba); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 569 | spin_unlock(&pblk->trans_lock); |
| 570 | |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 571 | ppa_gc = addr_to_gen_ppa(pblk, paddr_gc, line->id); |
| 572 | if (!pblk_ppa_comp(ppa_l2p, ppa_gc)) |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 573 | goto out; |
| 574 | |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 575 | rqd->ppa_addr = ppa_l2p; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 576 | valid_secs = 1; |
| 577 | |
Matias Bjørling | 880eda5 | 2018-07-13 10:48:37 +0200 | [diff] [blame] | 578 | #ifdef CONFIG_NVM_PBLK_DEBUG |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 579 | atomic_long_inc(&pblk->inflight_reads); |
| 580 | #endif |
| 581 | |
| 582 | out: |
| 583 | return valid_secs; |
| 584 | } |
| 585 | |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 586 | int pblk_submit_read_gc(struct pblk *pblk, struct pblk_gc_rq *gc_rq) |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 587 | { |
| 588 | struct nvm_tgt_dev *dev = pblk->dev; |
| 589 | struct nvm_geo *geo = &dev->geo; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 590 | struct bio *bio; |
| 591 | struct nvm_rq rqd; |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 592 | int data_len; |
| 593 | int ret = NVM_IO_OK; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 594 | |
| 595 | memset(&rqd, 0, sizeof(struct nvm_rq)); |
| 596 | |
Javier González | 63e3809 | 2017-06-26 11:57:24 +0200 | [diff] [blame] | 597 | rqd.meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL, |
| 598 | &rqd.dma_meta_list); |
| 599 | if (!rqd.meta_list) |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 600 | return -ENOMEM; |
Javier González | 63e3809 | 2017-06-26 11:57:24 +0200 | [diff] [blame] | 601 | |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 602 | if (gc_rq->nr_secs > 1) { |
Javier González | 63e3809 | 2017-06-26 11:57:24 +0200 | [diff] [blame] | 603 | rqd.ppa_list = rqd.meta_list + pblk_dma_meta_size; |
| 604 | rqd.dma_ppa_list = rqd.dma_meta_list + pblk_dma_meta_size; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 605 | |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 606 | gc_rq->secs_to_gc = read_ppalist_rq_gc(pblk, &rqd, gc_rq->line, |
| 607 | gc_rq->lba_list, |
| 608 | gc_rq->paddr_list, |
| 609 | gc_rq->nr_secs); |
| 610 | if (gc_rq->secs_to_gc == 1) |
Javier González | 63e3809 | 2017-06-26 11:57:24 +0200 | [diff] [blame] | 611 | rqd.ppa_addr = rqd.ppa_list[0]; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 612 | } else { |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 613 | gc_rq->secs_to_gc = read_rq_gc(pblk, &rqd, gc_rq->line, |
| 614 | gc_rq->lba_list[0], |
| 615 | gc_rq->paddr_list[0]); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 616 | } |
| 617 | |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 618 | if (!(gc_rq->secs_to_gc)) |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 619 | goto out; |
| 620 | |
Javier González | e46f4e4 | 2018-03-30 00:05:10 +0200 | [diff] [blame] | 621 | data_len = (gc_rq->secs_to_gc) * geo->csecs; |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 622 | bio = pblk_bio_map_addr(pblk, gc_rq->data, gc_rq->secs_to_gc, data_len, |
Javier González | 7d327a9 | 2017-10-13 14:46:02 +0200 | [diff] [blame] | 623 | PBLK_VMALLOC_META, GFP_KERNEL); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 624 | if (IS_ERR(bio)) { |
Matias Bjørling | 4e495a4 | 2018-07-13 10:48:42 +0200 | [diff] [blame] | 625 | pblk_err(pblk, "could not allocate GC bio (%lu)\n", |
| 626 | PTR_ERR(bio)); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 627 | goto err_free_dma; |
| 628 | } |
| 629 | |
| 630 | bio->bi_iter.bi_sector = 0; /* internal bio */ |
| 631 | bio_set_op_attrs(bio, REQ_OP_READ, 0); |
| 632 | |
| 633 | rqd.opcode = NVM_OP_PREAD; |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 634 | rqd.nr_ppas = gc_rq->secs_to_gc; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 635 | rqd.bio = bio; |
| 636 | |
Javier González | 1a94b2d | 2017-10-13 14:46:47 +0200 | [diff] [blame] | 637 | if (pblk_submit_io_sync(pblk, &rqd)) { |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 638 | ret = -EIO; |
Matias Bjørling | 4e495a4 | 2018-07-13 10:48:42 +0200 | [diff] [blame] | 639 | pblk_err(pblk, "GC read request failed\n"); |
Javier González | 7d327a9 | 2017-10-13 14:46:02 +0200 | [diff] [blame] | 640 | goto err_free_bio; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 641 | } |
| 642 | |
Javier González | 03a34b2 | 2018-06-01 15:04:18 +0200 | [diff] [blame] | 643 | pblk_read_check_rand(pblk, &rqd, gc_rq->lba_list, gc_rq->nr_secs); |
Javier González | 310df58 | 2018-06-01 15:04:17 +0200 | [diff] [blame] | 644 | |
Javier González | 588726d3 | 2017-06-26 11:57:29 +0200 | [diff] [blame] | 645 | atomic_dec(&pblk->inflight_io); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 646 | |
| 647 | if (rqd.error) { |
| 648 | atomic_long_inc(&pblk->read_failed_gc); |
Matias Bjørling | 880eda5 | 2018-07-13 10:48:37 +0200 | [diff] [blame] | 649 | #ifdef CONFIG_NVM_PBLK_DEBUG |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 650 | pblk_print_failed_rqd(pblk, &rqd, rqd.error); |
| 651 | #endif |
| 652 | } |
| 653 | |
Matias Bjørling | 880eda5 | 2018-07-13 10:48:37 +0200 | [diff] [blame] | 654 | #ifdef CONFIG_NVM_PBLK_DEBUG |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 655 | atomic_long_add(gc_rq->secs_to_gc, &pblk->sync_reads); |
| 656 | atomic_long_add(gc_rq->secs_to_gc, &pblk->recov_gc_reads); |
| 657 | atomic_long_sub(gc_rq->secs_to_gc, &pblk->inflight_reads); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 658 | #endif |
| 659 | |
| 660 | out: |
Javier González | 63e3809 | 2017-06-26 11:57:24 +0200 | [diff] [blame] | 661 | nvm_dev_dma_free(dev->parent, rqd.meta_list, rqd.dma_meta_list); |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 662 | return ret; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 663 | |
Javier González | 7d327a9 | 2017-10-13 14:46:02 +0200 | [diff] [blame] | 664 | err_free_bio: |
| 665 | bio_put(bio); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 666 | err_free_dma: |
Javier González | 63e3809 | 2017-06-26 11:57:24 +0200 | [diff] [blame] | 667 | nvm_dev_dma_free(dev->parent, rqd.meta_list, rqd.dma_meta_list); |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 668 | return ret; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 669 | } |