Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 1 | //#define DEBUG |
| 2 | #include <linux/spinlock.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 3 | #include <linux/slab.h> |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 4 | #include <linux/blkdev.h> |
| 5 | #include <linux/hdreg.h> |
Paul Gortmaker | 0c8d44f | 2011-07-01 15:56:05 -0400 | [diff] [blame] | 6 | #include <linux/module.h> |
Michael S. Tsirkin | 4678d6f9 | 2012-01-12 15:44:44 +1030 | [diff] [blame] | 7 | #include <linux/mutex.h> |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 8 | #include <linux/virtio.h> |
| 9 | #include <linux/virtio_blk.h> |
Jens Axboe | 3d1266c | 2007-10-24 13:21:21 +0200 | [diff] [blame] | 10 | #include <linux/scatterlist.h> |
Christoph Hellwig | 7a7c924 | 2011-02-01 21:43:48 +0100 | [diff] [blame] | 11 | #include <linux/string_helpers.h> |
Liu Yuan | 6917f83 | 2011-04-24 02:49:26 +0800 | [diff] [blame] | 12 | #include <scsi/scsi_cmnd.h> |
Michael S. Tsirkin | 5087a50 | 2011-10-30 21:29:59 +0200 | [diff] [blame] | 13 | #include <linux/idr.h> |
Jens Axboe | 3d1266c | 2007-10-24 13:21:21 +0200 | [diff] [blame] | 14 | |
Christian Borntraeger | 4f3bf19 | 2008-01-31 15:53:53 +0100 | [diff] [blame] | 15 | #define PART_BITS 4 |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 16 | |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 17 | static bool use_bio; |
| 18 | module_param(use_bio, bool, S_IRUGO); |
| 19 | |
Michael S. Tsirkin | 5087a50 | 2011-10-30 21:29:59 +0200 | [diff] [blame] | 20 | static int major; |
| 21 | static DEFINE_IDA(vd_index_ida); |
| 22 | |
Christoph Hellwig | 7a7c924 | 2011-02-01 21:43:48 +0100 | [diff] [blame] | 23 | struct workqueue_struct *virtblk_wq; |
Christian Borntraeger | 4f3bf19 | 2008-01-31 15:53:53 +0100 | [diff] [blame] | 24 | |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 25 | struct virtio_blk |
| 26 | { |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 27 | struct virtio_device *vdev; |
| 28 | struct virtqueue *vq; |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 29 | wait_queue_head_t queue_wait; |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 30 | |
| 31 | /* The disk structure for the kernel. */ |
| 32 | struct gendisk *disk; |
| 33 | |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 34 | mempool_t *pool; |
| 35 | |
Christoph Hellwig | 7a7c924 | 2011-02-01 21:43:48 +0100 | [diff] [blame] | 36 | /* Process context for config space updates */ |
| 37 | struct work_struct config_work; |
| 38 | |
Michael S. Tsirkin | 4678d6f9 | 2012-01-12 15:44:44 +1030 | [diff] [blame] | 39 | /* Lock for config space updates */ |
| 40 | struct mutex config_lock; |
| 41 | |
| 42 | /* enable config space updates */ |
| 43 | bool config_enable; |
| 44 | |
Rusty Russell | 0864b79 | 2008-12-30 09:26:05 -0600 | [diff] [blame] | 45 | /* What host tells us, plus 2 for header & tailer. */ |
| 46 | unsigned int sg_elems; |
| 47 | |
Michael S. Tsirkin | 5087a50 | 2011-10-30 21:29:59 +0200 | [diff] [blame] | 48 | /* Ida index - used to track minor number allocations. */ |
| 49 | int index; |
| 50 | |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 51 | /* Scatterlist: can be too big for stack. */ |
Rusty Russell | 0864b79 | 2008-12-30 09:26:05 -0600 | [diff] [blame] | 52 | struct scatterlist sg[/*sg_elems*/]; |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 53 | }; |
| 54 | |
| 55 | struct virtblk_req |
| 56 | { |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 57 | struct request *req; |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 58 | struct bio *bio; |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 59 | struct virtio_blk_outhdr out_hdr; |
Hannes Reinecke | 1cde26f | 2009-05-18 14:41:30 +0200 | [diff] [blame] | 60 | struct virtio_scsi_inhdr in_hdr; |
Asias He | c85a1f9 | 2012-08-08 16:07:05 +0800 | [diff] [blame^] | 61 | struct work_struct work; |
| 62 | struct virtio_blk *vblk; |
| 63 | int flags; |
Rusty Russell | cb38fa2 | 2008-05-02 21:50:45 -0500 | [diff] [blame] | 64 | u8 status; |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 65 | struct scatterlist sg[]; |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 66 | }; |
| 67 | |
Asias He | c85a1f9 | 2012-08-08 16:07:05 +0800 | [diff] [blame^] | 68 | enum { |
| 69 | VBLK_IS_FLUSH = 1, |
| 70 | VBLK_REQ_FLUSH = 2, |
| 71 | VBLK_REQ_DATA = 4, |
| 72 | VBLK_REQ_FUA = 8, |
| 73 | }; |
| 74 | |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 75 | static inline int virtblk_result(struct virtblk_req *vbr) |
| 76 | { |
| 77 | switch (vbr->status) { |
| 78 | case VIRTIO_BLK_S_OK: |
| 79 | return 0; |
| 80 | case VIRTIO_BLK_S_UNSUPP: |
| 81 | return -ENOTTY; |
| 82 | default: |
| 83 | return -EIO; |
| 84 | } |
| 85 | } |
| 86 | |
Asias He | c85a1f9 | 2012-08-08 16:07:05 +0800 | [diff] [blame^] | 87 | static inline struct virtblk_req *virtblk_alloc_req(struct virtio_blk *vblk, |
| 88 | gfp_t gfp_mask) |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 89 | { |
Asias He | c85a1f9 | 2012-08-08 16:07:05 +0800 | [diff] [blame^] | 90 | struct virtblk_req *vbr; |
| 91 | |
| 92 | vbr = mempool_alloc(vblk->pool, gfp_mask); |
| 93 | if (vbr && use_bio) |
| 94 | sg_init_table(vbr->sg, vblk->sg_elems); |
| 95 | |
| 96 | vbr->vblk = vblk; |
| 97 | |
| 98 | return vbr; |
| 99 | } |
| 100 | |
| 101 | static void virtblk_add_buf_wait(struct virtio_blk *vblk, |
| 102 | struct virtblk_req *vbr, |
| 103 | unsigned long out, |
| 104 | unsigned long in) |
| 105 | { |
| 106 | DEFINE_WAIT(wait); |
| 107 | |
| 108 | for (;;) { |
| 109 | prepare_to_wait_exclusive(&vblk->queue_wait, &wait, |
| 110 | TASK_UNINTERRUPTIBLE); |
| 111 | |
| 112 | spin_lock_irq(vblk->disk->queue->queue_lock); |
| 113 | if (virtqueue_add_buf(vblk->vq, vbr->sg, out, in, vbr, |
| 114 | GFP_ATOMIC) < 0) { |
| 115 | spin_unlock_irq(vblk->disk->queue->queue_lock); |
| 116 | io_schedule(); |
| 117 | } else { |
| 118 | virtqueue_kick(vblk->vq); |
| 119 | spin_unlock_irq(vblk->disk->queue->queue_lock); |
| 120 | break; |
| 121 | } |
| 122 | |
| 123 | } |
| 124 | |
| 125 | finish_wait(&vblk->queue_wait, &wait); |
| 126 | } |
| 127 | |
| 128 | static inline void virtblk_add_req(struct virtblk_req *vbr, |
| 129 | unsigned int out, unsigned int in) |
| 130 | { |
| 131 | struct virtio_blk *vblk = vbr->vblk; |
| 132 | |
| 133 | spin_lock_irq(vblk->disk->queue->queue_lock); |
| 134 | if (unlikely(virtqueue_add_buf(vblk->vq, vbr->sg, out, in, vbr, |
| 135 | GFP_ATOMIC) < 0)) { |
| 136 | spin_unlock_irq(vblk->disk->queue->queue_lock); |
| 137 | virtblk_add_buf_wait(vblk, vbr, out, in); |
| 138 | return; |
| 139 | } |
| 140 | virtqueue_kick(vblk->vq); |
| 141 | spin_unlock_irq(vblk->disk->queue->queue_lock); |
| 142 | } |
| 143 | |
| 144 | static int virtblk_bio_send_flush(struct virtblk_req *vbr) |
| 145 | { |
| 146 | unsigned int out = 0, in = 0; |
| 147 | |
| 148 | vbr->flags |= VBLK_IS_FLUSH; |
| 149 | vbr->out_hdr.type = VIRTIO_BLK_T_FLUSH; |
| 150 | vbr->out_hdr.sector = 0; |
| 151 | vbr->out_hdr.ioprio = 0; |
| 152 | sg_set_buf(&vbr->sg[out++], &vbr->out_hdr, sizeof(vbr->out_hdr)); |
| 153 | sg_set_buf(&vbr->sg[out + in++], &vbr->status, sizeof(vbr->status)); |
| 154 | |
| 155 | virtblk_add_req(vbr, out, in); |
| 156 | |
| 157 | return 0; |
| 158 | } |
| 159 | |
| 160 | static int virtblk_bio_send_data(struct virtblk_req *vbr) |
| 161 | { |
| 162 | struct virtio_blk *vblk = vbr->vblk; |
| 163 | unsigned int num, out = 0, in = 0; |
| 164 | struct bio *bio = vbr->bio; |
| 165 | |
| 166 | vbr->flags &= ~VBLK_IS_FLUSH; |
| 167 | vbr->out_hdr.type = 0; |
| 168 | vbr->out_hdr.sector = bio->bi_sector; |
| 169 | vbr->out_hdr.ioprio = bio_prio(bio); |
| 170 | |
| 171 | sg_set_buf(&vbr->sg[out++], &vbr->out_hdr, sizeof(vbr->out_hdr)); |
| 172 | |
| 173 | num = blk_bio_map_sg(vblk->disk->queue, bio, vbr->sg + out); |
| 174 | |
| 175 | sg_set_buf(&vbr->sg[num + out + in++], &vbr->status, |
| 176 | sizeof(vbr->status)); |
| 177 | |
| 178 | if (num) { |
| 179 | if (bio->bi_rw & REQ_WRITE) { |
| 180 | vbr->out_hdr.type |= VIRTIO_BLK_T_OUT; |
| 181 | out += num; |
| 182 | } else { |
| 183 | vbr->out_hdr.type |= VIRTIO_BLK_T_IN; |
| 184 | in += num; |
| 185 | } |
| 186 | } |
| 187 | |
| 188 | virtblk_add_req(vbr, out, in); |
| 189 | |
| 190 | return 0; |
| 191 | } |
| 192 | |
| 193 | static void virtblk_bio_send_data_work(struct work_struct *work) |
| 194 | { |
| 195 | struct virtblk_req *vbr; |
| 196 | |
| 197 | vbr = container_of(work, struct virtblk_req, work); |
| 198 | |
| 199 | virtblk_bio_send_data(vbr); |
| 200 | } |
| 201 | |
| 202 | static void virtblk_bio_send_flush_work(struct work_struct *work) |
| 203 | { |
| 204 | struct virtblk_req *vbr; |
| 205 | |
| 206 | vbr = container_of(work, struct virtblk_req, work); |
| 207 | |
| 208 | virtblk_bio_send_flush(vbr); |
| 209 | } |
| 210 | |
| 211 | static inline void virtblk_request_done(struct virtblk_req *vbr) |
| 212 | { |
| 213 | struct virtio_blk *vblk = vbr->vblk; |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 214 | struct request *req = vbr->req; |
| 215 | int error = virtblk_result(vbr); |
| 216 | |
| 217 | if (req->cmd_type == REQ_TYPE_BLOCK_PC) { |
| 218 | req->resid_len = vbr->in_hdr.residual; |
| 219 | req->sense_len = vbr->in_hdr.sense_len; |
| 220 | req->errors = vbr->in_hdr.errors; |
| 221 | } else if (req->cmd_type == REQ_TYPE_SPECIAL) { |
| 222 | req->errors = (error != 0); |
| 223 | } |
| 224 | |
| 225 | __blk_end_request_all(req, error); |
| 226 | mempool_free(vbr, vblk->pool); |
| 227 | } |
| 228 | |
Asias He | c85a1f9 | 2012-08-08 16:07:05 +0800 | [diff] [blame^] | 229 | static inline void virtblk_bio_flush_done(struct virtblk_req *vbr) |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 230 | { |
Asias He | c85a1f9 | 2012-08-08 16:07:05 +0800 | [diff] [blame^] | 231 | struct virtio_blk *vblk = vbr->vblk; |
| 232 | |
| 233 | if (vbr->flags & VBLK_REQ_DATA) { |
| 234 | /* Send out the actual write data */ |
| 235 | INIT_WORK(&vbr->work, virtblk_bio_send_data_work); |
| 236 | queue_work(virtblk_wq, &vbr->work); |
| 237 | } else { |
| 238 | bio_endio(vbr->bio, virtblk_result(vbr)); |
| 239 | mempool_free(vbr, vblk->pool); |
| 240 | } |
| 241 | } |
| 242 | |
| 243 | static inline void virtblk_bio_data_done(struct virtblk_req *vbr) |
| 244 | { |
| 245 | struct virtio_blk *vblk = vbr->vblk; |
| 246 | |
| 247 | if (unlikely(vbr->flags & VBLK_REQ_FUA)) { |
| 248 | /* Send out a flush before end the bio */ |
| 249 | vbr->flags &= ~VBLK_REQ_DATA; |
| 250 | INIT_WORK(&vbr->work, virtblk_bio_send_flush_work); |
| 251 | queue_work(virtblk_wq, &vbr->work); |
| 252 | } else { |
| 253 | bio_endio(vbr->bio, virtblk_result(vbr)); |
| 254 | mempool_free(vbr, vblk->pool); |
| 255 | } |
| 256 | } |
| 257 | |
| 258 | static inline void virtblk_bio_done(struct virtblk_req *vbr) |
| 259 | { |
| 260 | if (unlikely(vbr->flags & VBLK_IS_FLUSH)) |
| 261 | virtblk_bio_flush_done(vbr); |
| 262 | else |
| 263 | virtblk_bio_data_done(vbr); |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 264 | } |
| 265 | |
| 266 | static void virtblk_done(struct virtqueue *vq) |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 267 | { |
| 268 | struct virtio_blk *vblk = vq->vdev->priv; |
Asias He | c85a1f9 | 2012-08-08 16:07:05 +0800 | [diff] [blame^] | 269 | bool bio_done = false, req_done = false; |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 270 | struct virtblk_req *vbr; |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 271 | unsigned long flags; |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 272 | unsigned int len; |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 273 | |
Asias He | 2c95a32 | 2012-05-25 16:03:27 +0800 | [diff] [blame] | 274 | spin_lock_irqsave(vblk->disk->queue->queue_lock, flags); |
Michael S. Tsirkin | 09ec6b6 | 2010-04-12 16:18:36 +0300 | [diff] [blame] | 275 | while ((vbr = virtqueue_get_buf(vblk->vq, &len)) != NULL) { |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 276 | if (vbr->bio) { |
Asias He | c85a1f9 | 2012-08-08 16:07:05 +0800 | [diff] [blame^] | 277 | virtblk_bio_done(vbr); |
| 278 | bio_done = true; |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 279 | } else { |
Asias He | c85a1f9 | 2012-08-08 16:07:05 +0800 | [diff] [blame^] | 280 | virtblk_request_done(vbr); |
| 281 | req_done = true; |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 282 | } |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 283 | } |
| 284 | /* In case queue is stopped waiting for more buffers. */ |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 285 | if (req_done) |
| 286 | blk_start_queue(vblk->disk->queue); |
Asias He | 2c95a32 | 2012-05-25 16:03:27 +0800 | [diff] [blame] | 287 | spin_unlock_irqrestore(vblk->disk->queue->queue_lock, flags); |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 288 | |
| 289 | if (bio_done) |
| 290 | wake_up(&vblk->queue_wait); |
| 291 | } |
| 292 | |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 293 | static bool do_req(struct request_queue *q, struct virtio_blk *vblk, |
| 294 | struct request *req) |
| 295 | { |
Hannes Reinecke | 1cde26f | 2009-05-18 14:41:30 +0200 | [diff] [blame] | 296 | unsigned long num, out = 0, in = 0; |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 297 | struct virtblk_req *vbr; |
| 298 | |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 299 | vbr = virtblk_alloc_req(vblk, GFP_ATOMIC); |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 300 | if (!vbr) |
| 301 | /* When another request finishes we'll try again. */ |
| 302 | return false; |
| 303 | |
| 304 | vbr->req = req; |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 305 | vbr->bio = NULL; |
FUJITA Tomonori | dd40e45 | 2010-07-03 17:45:38 +0900 | [diff] [blame] | 306 | if (req->cmd_flags & REQ_FLUSH) { |
| 307 | vbr->out_hdr.type = VIRTIO_BLK_T_FLUSH; |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 308 | vbr->out_hdr.sector = 0; |
Fernando Luis Vázquez Cao | 766ca44 | 2008-08-14 09:59:13 +0200 | [diff] [blame] | 309 | vbr->out_hdr.ioprio = req_get_ioprio(vbr->req); |
FUJITA Tomonori | dd40e45 | 2010-07-03 17:45:38 +0900 | [diff] [blame] | 310 | } else { |
| 311 | switch (req->cmd_type) { |
| 312 | case REQ_TYPE_FS: |
| 313 | vbr->out_hdr.type = 0; |
| 314 | vbr->out_hdr.sector = blk_rq_pos(vbr->req); |
| 315 | vbr->out_hdr.ioprio = req_get_ioprio(vbr->req); |
| 316 | break; |
| 317 | case REQ_TYPE_BLOCK_PC: |
| 318 | vbr->out_hdr.type = VIRTIO_BLK_T_SCSI_CMD; |
Christoph Hellwig | f1b0ef06 | 2009-09-17 19:57:42 +0200 | [diff] [blame] | 319 | vbr->out_hdr.sector = 0; |
| 320 | vbr->out_hdr.ioprio = req_get_ioprio(vbr->req); |
| 321 | break; |
FUJITA Tomonori | dd40e45 | 2010-07-03 17:45:38 +0900 | [diff] [blame] | 322 | case REQ_TYPE_SPECIAL: |
| 323 | vbr->out_hdr.type = VIRTIO_BLK_T_GET_ID; |
| 324 | vbr->out_hdr.sector = 0; |
| 325 | vbr->out_hdr.ioprio = req_get_ioprio(vbr->req); |
| 326 | break; |
| 327 | default: |
| 328 | /* We don't put anything else in the queue. */ |
| 329 | BUG(); |
Christoph Hellwig | f1b0ef06 | 2009-09-17 19:57:42 +0200 | [diff] [blame] | 330 | } |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 331 | } |
| 332 | |
Hannes Reinecke | 1cde26f | 2009-05-18 14:41:30 +0200 | [diff] [blame] | 333 | sg_set_buf(&vblk->sg[out++], &vbr->out_hdr, sizeof(vbr->out_hdr)); |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 334 | |
Hannes Reinecke | 1cde26f | 2009-05-18 14:41:30 +0200 | [diff] [blame] | 335 | /* |
| 336 | * If this is a packet command we need a couple of additional headers. |
| 337 | * Behind the normal outhdr we put a segment with the scsi command |
| 338 | * block, and before the normal inhdr we put the sense data and the |
| 339 | * inhdr with additional status information before the normal inhdr. |
| 340 | */ |
Christoph Hellwig | 33659eb | 2010-08-07 18:17:56 +0200 | [diff] [blame] | 341 | if (vbr->req->cmd_type == REQ_TYPE_BLOCK_PC) |
Hannes Reinecke | 1cde26f | 2009-05-18 14:41:30 +0200 | [diff] [blame] | 342 | sg_set_buf(&vblk->sg[out++], vbr->req->cmd, vbr->req->cmd_len); |
| 343 | |
| 344 | num = blk_rq_map_sg(q, vbr->req, vblk->sg + out); |
| 345 | |
Christoph Hellwig | 33659eb | 2010-08-07 18:17:56 +0200 | [diff] [blame] | 346 | if (vbr->req->cmd_type == REQ_TYPE_BLOCK_PC) { |
Liu Yuan | 6917f83 | 2011-04-24 02:49:26 +0800 | [diff] [blame] | 347 | sg_set_buf(&vblk->sg[num + out + in++], vbr->req->sense, SCSI_SENSE_BUFFERSIZE); |
Hannes Reinecke | 1cde26f | 2009-05-18 14:41:30 +0200 | [diff] [blame] | 348 | sg_set_buf(&vblk->sg[num + out + in++], &vbr->in_hdr, |
| 349 | sizeof(vbr->in_hdr)); |
| 350 | } |
| 351 | |
| 352 | sg_set_buf(&vblk->sg[num + out + in++], &vbr->status, |
| 353 | sizeof(vbr->status)); |
| 354 | |
| 355 | if (num) { |
| 356 | if (rq_data_dir(vbr->req) == WRITE) { |
| 357 | vbr->out_hdr.type |= VIRTIO_BLK_T_OUT; |
| 358 | out += num; |
| 359 | } else { |
| 360 | vbr->out_hdr.type |= VIRTIO_BLK_T_IN; |
| 361 | in += num; |
| 362 | } |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 363 | } |
| 364 | |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 365 | if (virtqueue_add_buf(vblk->vq, vblk->sg, out, in, vbr, |
| 366 | GFP_ATOMIC) < 0) { |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 367 | mempool_free(vbr, vblk->pool); |
| 368 | return false; |
| 369 | } |
| 370 | |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 371 | return true; |
| 372 | } |
| 373 | |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 374 | static void virtblk_request(struct request_queue *q) |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 375 | { |
Christoph Hellwig | 6c3b46f | 2009-05-18 14:38:28 +0200 | [diff] [blame] | 376 | struct virtio_blk *vblk = q->queuedata; |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 377 | struct request *req; |
| 378 | unsigned int issued = 0; |
| 379 | |
Tejun Heo | 9934c8c | 2009-05-08 11:54:16 +0900 | [diff] [blame] | 380 | while ((req = blk_peek_request(q)) != NULL) { |
Rusty Russell | 0864b79 | 2008-12-30 09:26:05 -0600 | [diff] [blame] | 381 | BUG_ON(req->nr_phys_segments + 2 > vblk->sg_elems); |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 382 | |
| 383 | /* If this request fails, stop queue and wait for something to |
| 384 | finish to restart it. */ |
| 385 | if (!do_req(q, vblk, req)) { |
| 386 | blk_stop_queue(q); |
| 387 | break; |
| 388 | } |
Tejun Heo | 9934c8c | 2009-05-08 11:54:16 +0900 | [diff] [blame] | 389 | blk_start_request(req); |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 390 | issued++; |
| 391 | } |
| 392 | |
| 393 | if (issued) |
Michael S. Tsirkin | 09ec6b6 | 2010-04-12 16:18:36 +0300 | [diff] [blame] | 394 | virtqueue_kick(vblk->vq); |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 395 | } |
| 396 | |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 397 | static void virtblk_make_request(struct request_queue *q, struct bio *bio) |
| 398 | { |
| 399 | struct virtio_blk *vblk = q->queuedata; |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 400 | struct virtblk_req *vbr; |
| 401 | |
| 402 | BUG_ON(bio->bi_phys_segments + 2 > vblk->sg_elems); |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 403 | |
| 404 | vbr = virtblk_alloc_req(vblk, GFP_NOIO); |
| 405 | if (!vbr) { |
| 406 | bio_endio(bio, -ENOMEM); |
| 407 | return; |
| 408 | } |
| 409 | |
| 410 | vbr->bio = bio; |
Asias He | c85a1f9 | 2012-08-08 16:07:05 +0800 | [diff] [blame^] | 411 | vbr->flags = 0; |
| 412 | if (bio->bi_rw & REQ_FLUSH) |
| 413 | vbr->flags |= VBLK_REQ_FLUSH; |
| 414 | if (bio->bi_rw & REQ_FUA) |
| 415 | vbr->flags |= VBLK_REQ_FUA; |
| 416 | if (bio->bi_size) |
| 417 | vbr->flags |= VBLK_REQ_DATA; |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 418 | |
Asias He | c85a1f9 | 2012-08-08 16:07:05 +0800 | [diff] [blame^] | 419 | if (unlikely(vbr->flags & VBLK_REQ_FLUSH)) |
| 420 | virtblk_bio_send_flush(vbr); |
| 421 | else |
| 422 | virtblk_bio_send_data(vbr); |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 423 | } |
| 424 | |
john cooper | 4cb2ea2 | 2010-03-25 01:33:33 -0400 | [diff] [blame] | 425 | /* return id (s/n) string for *disk to *id_str |
| 426 | */ |
| 427 | static int virtblk_get_id(struct gendisk *disk, char *id_str) |
| 428 | { |
| 429 | struct virtio_blk *vblk = disk->private_data; |
| 430 | struct request *req; |
| 431 | struct bio *bio; |
Mike Snitzer | e4c4776 | 2010-10-09 12:12:13 +1030 | [diff] [blame] | 432 | int err; |
john cooper | 4cb2ea2 | 2010-03-25 01:33:33 -0400 | [diff] [blame] | 433 | |
| 434 | bio = bio_map_kern(vblk->disk->queue, id_str, VIRTIO_BLK_ID_BYTES, |
| 435 | GFP_KERNEL); |
| 436 | if (IS_ERR(bio)) |
| 437 | return PTR_ERR(bio); |
| 438 | |
| 439 | req = blk_make_request(vblk->disk->queue, bio, GFP_KERNEL); |
| 440 | if (IS_ERR(req)) { |
| 441 | bio_put(bio); |
| 442 | return PTR_ERR(req); |
| 443 | } |
| 444 | |
| 445 | req->cmd_type = REQ_TYPE_SPECIAL; |
Mike Snitzer | e4c4776 | 2010-10-09 12:12:13 +1030 | [diff] [blame] | 446 | err = blk_execute_rq(vblk->disk->queue, vblk->disk, req, false); |
| 447 | blk_put_request(req); |
| 448 | |
| 449 | return err; |
john cooper | 4cb2ea2 | 2010-03-25 01:33:33 -0400 | [diff] [blame] | 450 | } |
| 451 | |
Christoph Hellwig | fe5a50a | 2010-09-15 01:27:23 +0200 | [diff] [blame] | 452 | static int virtblk_ioctl(struct block_device *bdev, fmode_t mode, |
| 453 | unsigned int cmd, unsigned long data) |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 454 | { |
Hannes Reinecke | 1cde26f | 2009-05-18 14:41:30 +0200 | [diff] [blame] | 455 | struct gendisk *disk = bdev->bd_disk; |
| 456 | struct virtio_blk *vblk = disk->private_data; |
| 457 | |
| 458 | /* |
| 459 | * Only allow the generic SCSI ioctls if the host can support it. |
| 460 | */ |
| 461 | if (!virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_SCSI)) |
Christoph Hellwig | d9ecdea | 2009-06-20 21:29:41 +0200 | [diff] [blame] | 462 | return -ENOTTY; |
Hannes Reinecke | 1cde26f | 2009-05-18 14:41:30 +0200 | [diff] [blame] | 463 | |
Paolo Bonzini | 577ebb3 | 2012-01-12 16:01:27 +0100 | [diff] [blame] | 464 | return scsi_cmd_blk_ioctl(bdev, mode, cmd, |
| 465 | (void __user *)data); |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 466 | } |
| 467 | |
Christian Borntraeger | 135da0b | 2008-01-23 17:56:50 +0100 | [diff] [blame] | 468 | /* We provide getgeo only to please some old bootloader/partitioning tools */ |
| 469 | static int virtblk_getgeo(struct block_device *bd, struct hd_geometry *geo) |
| 470 | { |
Ryan Harper | 48e4043 | 2008-04-16 13:56:37 -0500 | [diff] [blame] | 471 | struct virtio_blk *vblk = bd->bd_disk->private_data; |
| 472 | struct virtio_blk_geometry vgeo; |
| 473 | int err; |
| 474 | |
| 475 | /* see if the host passed in geometry config */ |
| 476 | err = virtio_config_val(vblk->vdev, VIRTIO_BLK_F_GEOMETRY, |
| 477 | offsetof(struct virtio_blk_config, geometry), |
| 478 | &vgeo); |
| 479 | |
| 480 | if (!err) { |
| 481 | geo->heads = vgeo.heads; |
| 482 | geo->sectors = vgeo.sectors; |
| 483 | geo->cylinders = vgeo.cylinders; |
| 484 | } else { |
| 485 | /* some standard values, similar to sd */ |
| 486 | geo->heads = 1 << 6; |
| 487 | geo->sectors = 1 << 5; |
| 488 | geo->cylinders = get_capacity(bd->bd_disk) >> 11; |
| 489 | } |
Christian Borntraeger | 135da0b | 2008-01-23 17:56:50 +0100 | [diff] [blame] | 490 | return 0; |
| 491 | } |
| 492 | |
Alexey Dobriyan | 83d5cde | 2009-09-21 17:01:13 -0700 | [diff] [blame] | 493 | static const struct block_device_operations virtblk_fops = { |
Arnd Bergmann | 8a6cfeb | 2010-07-08 10:18:46 +0200 | [diff] [blame] | 494 | .ioctl = virtblk_ioctl, |
Christian Borntraeger | 135da0b | 2008-01-23 17:56:50 +0100 | [diff] [blame] | 495 | .owner = THIS_MODULE, |
| 496 | .getgeo = virtblk_getgeo, |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 497 | }; |
| 498 | |
Christian Borntraeger | d50ed90 | 2008-02-01 09:05:00 +0100 | [diff] [blame] | 499 | static int index_to_minor(int index) |
| 500 | { |
| 501 | return index << PART_BITS; |
| 502 | } |
| 503 | |
Michael S. Tsirkin | 5087a50 | 2011-10-30 21:29:59 +0200 | [diff] [blame] | 504 | static int minor_to_index(int minor) |
| 505 | { |
| 506 | return minor >> PART_BITS; |
| 507 | } |
| 508 | |
Ryan Harper | a5eb9e4 | 2010-06-23 22:19:57 -0500 | [diff] [blame] | 509 | static ssize_t virtblk_serial_show(struct device *dev, |
| 510 | struct device_attribute *attr, char *buf) |
| 511 | { |
| 512 | struct gendisk *disk = dev_to_disk(dev); |
| 513 | int err; |
| 514 | |
| 515 | /* sysfs gives us a PAGE_SIZE buffer */ |
| 516 | BUILD_BUG_ON(PAGE_SIZE < VIRTIO_BLK_ID_BYTES); |
| 517 | |
| 518 | buf[VIRTIO_BLK_ID_BYTES] = '\0'; |
| 519 | err = virtblk_get_id(disk, buf); |
| 520 | if (!err) |
| 521 | return strlen(buf); |
| 522 | |
| 523 | if (err == -EIO) /* Unsupported? Make it empty. */ |
| 524 | return 0; |
| 525 | |
| 526 | return err; |
| 527 | } |
| 528 | DEVICE_ATTR(serial, S_IRUGO, virtblk_serial_show, NULL); |
| 529 | |
Christoph Hellwig | 7a7c924 | 2011-02-01 21:43:48 +0100 | [diff] [blame] | 530 | static void virtblk_config_changed_work(struct work_struct *work) |
| 531 | { |
| 532 | struct virtio_blk *vblk = |
| 533 | container_of(work, struct virtio_blk, config_work); |
| 534 | struct virtio_device *vdev = vblk->vdev; |
| 535 | struct request_queue *q = vblk->disk->queue; |
| 536 | char cap_str_2[10], cap_str_10[10]; |
| 537 | u64 capacity, size; |
| 538 | |
Michael S. Tsirkin | 4678d6f9 | 2012-01-12 15:44:44 +1030 | [diff] [blame] | 539 | mutex_lock(&vblk->config_lock); |
| 540 | if (!vblk->config_enable) |
| 541 | goto done; |
| 542 | |
Christoph Hellwig | 7a7c924 | 2011-02-01 21:43:48 +0100 | [diff] [blame] | 543 | /* Host must always specify the capacity. */ |
| 544 | vdev->config->get(vdev, offsetof(struct virtio_blk_config, capacity), |
| 545 | &capacity, sizeof(capacity)); |
| 546 | |
| 547 | /* If capacity is too big, truncate with warning. */ |
| 548 | if ((sector_t)capacity != capacity) { |
| 549 | dev_warn(&vdev->dev, "Capacity %llu too large: truncating\n", |
| 550 | (unsigned long long)capacity); |
| 551 | capacity = (sector_t)-1; |
| 552 | } |
| 553 | |
| 554 | size = capacity * queue_logical_block_size(q); |
| 555 | string_get_size(size, STRING_UNITS_2, cap_str_2, sizeof(cap_str_2)); |
| 556 | string_get_size(size, STRING_UNITS_10, cap_str_10, sizeof(cap_str_10)); |
| 557 | |
| 558 | dev_notice(&vdev->dev, |
| 559 | "new size: %llu %d-byte logical blocks (%s/%s)\n", |
| 560 | (unsigned long long)capacity, |
| 561 | queue_logical_block_size(q), |
| 562 | cap_str_10, cap_str_2); |
| 563 | |
| 564 | set_capacity(vblk->disk, capacity); |
Vivek Goyal | e9986f3 | 2012-03-29 10:09:44 +0200 | [diff] [blame] | 565 | revalidate_disk(vblk->disk); |
Michael S. Tsirkin | 4678d6f9 | 2012-01-12 15:44:44 +1030 | [diff] [blame] | 566 | done: |
| 567 | mutex_unlock(&vblk->config_lock); |
Christoph Hellwig | 7a7c924 | 2011-02-01 21:43:48 +0100 | [diff] [blame] | 568 | } |
| 569 | |
| 570 | static void virtblk_config_changed(struct virtio_device *vdev) |
| 571 | { |
| 572 | struct virtio_blk *vblk = vdev->priv; |
| 573 | |
| 574 | queue_work(virtblk_wq, &vblk->config_work); |
| 575 | } |
| 576 | |
Amit Shah | 6abd6e5 | 2011-12-22 16:58:29 +0530 | [diff] [blame] | 577 | static int init_vq(struct virtio_blk *vblk) |
| 578 | { |
| 579 | int err = 0; |
| 580 | |
| 581 | /* We expect one virtqueue, for output. */ |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 582 | vblk->vq = virtio_find_single_vq(vblk->vdev, virtblk_done, "requests"); |
Amit Shah | 6abd6e5 | 2011-12-22 16:58:29 +0530 | [diff] [blame] | 583 | if (IS_ERR(vblk->vq)) |
| 584 | err = PTR_ERR(vblk->vq); |
| 585 | |
| 586 | return err; |
| 587 | } |
| 588 | |
Ren Mingxin | c0aa3e0 | 2012-04-10 15:28:05 +0800 | [diff] [blame] | 589 | /* |
| 590 | * Legacy naming scheme used for virtio devices. We are stuck with it for |
| 591 | * virtio blk but don't ever use it for any new driver. |
| 592 | */ |
| 593 | static int virtblk_name_format(char *prefix, int index, char *buf, int buflen) |
| 594 | { |
| 595 | const int base = 'z' - 'a' + 1; |
| 596 | char *begin = buf + strlen(prefix); |
| 597 | char *end = buf + buflen; |
| 598 | char *p; |
| 599 | int unit; |
| 600 | |
| 601 | p = end - 1; |
| 602 | *p = '\0'; |
| 603 | unit = base; |
| 604 | do { |
| 605 | if (p == begin) |
| 606 | return -EINVAL; |
| 607 | *--p = 'a' + (index % unit); |
| 608 | index = (index / unit) - 1; |
| 609 | } while (index >= 0); |
| 610 | |
| 611 | memmove(begin, p, end - p); |
| 612 | memcpy(buf, prefix, strlen(prefix)); |
| 613 | |
| 614 | return 0; |
| 615 | } |
| 616 | |
Paolo Bonzini | cd5d503 | 2012-07-03 15:19:37 +0200 | [diff] [blame] | 617 | static int virtblk_get_cache_mode(struct virtio_device *vdev) |
| 618 | { |
| 619 | u8 writeback; |
| 620 | int err; |
| 621 | |
| 622 | err = virtio_config_val(vdev, VIRTIO_BLK_F_CONFIG_WCE, |
| 623 | offsetof(struct virtio_blk_config, wce), |
| 624 | &writeback); |
| 625 | if (err) |
| 626 | writeback = virtio_has_feature(vdev, VIRTIO_BLK_F_WCE); |
| 627 | |
| 628 | return writeback; |
| 629 | } |
| 630 | |
| 631 | static void virtblk_update_cache_mode(struct virtio_device *vdev) |
| 632 | { |
| 633 | u8 writeback = virtblk_get_cache_mode(vdev); |
| 634 | struct virtio_blk *vblk = vdev->priv; |
| 635 | |
Asias He | c85a1f9 | 2012-08-08 16:07:05 +0800 | [diff] [blame^] | 636 | if (writeback) |
Paolo Bonzini | cd5d503 | 2012-07-03 15:19:37 +0200 | [diff] [blame] | 637 | blk_queue_flush(vblk->disk->queue, REQ_FLUSH); |
| 638 | else |
| 639 | blk_queue_flush(vblk->disk->queue, 0); |
| 640 | |
| 641 | revalidate_disk(vblk->disk); |
| 642 | } |
| 643 | |
| 644 | static const char *const virtblk_cache_types[] = { |
| 645 | "write through", "write back" |
| 646 | }; |
| 647 | |
| 648 | static ssize_t |
| 649 | virtblk_cache_type_store(struct device *dev, struct device_attribute *attr, |
| 650 | const char *buf, size_t count) |
| 651 | { |
| 652 | struct gendisk *disk = dev_to_disk(dev); |
| 653 | struct virtio_blk *vblk = disk->private_data; |
| 654 | struct virtio_device *vdev = vblk->vdev; |
| 655 | int i; |
| 656 | u8 writeback; |
| 657 | |
| 658 | BUG_ON(!virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_CONFIG_WCE)); |
| 659 | for (i = ARRAY_SIZE(virtblk_cache_types); --i >= 0; ) |
| 660 | if (sysfs_streq(buf, virtblk_cache_types[i])) |
| 661 | break; |
| 662 | |
| 663 | if (i < 0) |
| 664 | return -EINVAL; |
| 665 | |
| 666 | writeback = i; |
| 667 | vdev->config->set(vdev, |
| 668 | offsetof(struct virtio_blk_config, wce), |
| 669 | &writeback, sizeof(writeback)); |
| 670 | |
| 671 | virtblk_update_cache_mode(vdev); |
| 672 | return count; |
| 673 | } |
| 674 | |
| 675 | static ssize_t |
| 676 | virtblk_cache_type_show(struct device *dev, struct device_attribute *attr, |
| 677 | char *buf) |
| 678 | { |
| 679 | struct gendisk *disk = dev_to_disk(dev); |
| 680 | struct virtio_blk *vblk = disk->private_data; |
| 681 | u8 writeback = virtblk_get_cache_mode(vblk->vdev); |
| 682 | |
| 683 | BUG_ON(writeback >= ARRAY_SIZE(virtblk_cache_types)); |
| 684 | return snprintf(buf, 40, "%s\n", virtblk_cache_types[writeback]); |
| 685 | } |
| 686 | |
| 687 | static const struct device_attribute dev_attr_cache_type_ro = |
| 688 | __ATTR(cache_type, S_IRUGO, |
| 689 | virtblk_cache_type_show, NULL); |
| 690 | static const struct device_attribute dev_attr_cache_type_rw = |
| 691 | __ATTR(cache_type, S_IRUGO|S_IWUSR, |
| 692 | virtblk_cache_type_show, virtblk_cache_type_store); |
| 693 | |
Mike Frysinger | 98e9444 | 2009-05-18 03:39:09 -0400 | [diff] [blame] | 694 | static int __devinit virtblk_probe(struct virtio_device *vdev) |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 695 | { |
| 696 | struct virtio_blk *vblk; |
Christoph Hellwig | 69740c8 | 2010-02-24 14:22:25 -0600 | [diff] [blame] | 697 | struct request_queue *q; |
Michael S. Tsirkin | 5087a50 | 2011-10-30 21:29:59 +0200 | [diff] [blame] | 698 | int err, index; |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 699 | int pool_size; |
| 700 | |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 701 | u64 cap; |
Christoph Hellwig | 69740c8 | 2010-02-24 14:22:25 -0600 | [diff] [blame] | 702 | u32 v, blk_size, sg_elems, opt_io_size; |
| 703 | u16 min_io_size; |
| 704 | u8 physical_block_exp, alignment_offset; |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 705 | |
Michael S. Tsirkin | 5087a50 | 2011-10-30 21:29:59 +0200 | [diff] [blame] | 706 | err = ida_simple_get(&vd_index_ida, 0, minor_to_index(1 << MINORBITS), |
| 707 | GFP_KERNEL); |
| 708 | if (err < 0) |
| 709 | goto out; |
| 710 | index = err; |
Christian Borntraeger | 4f3bf19 | 2008-01-31 15:53:53 +0100 | [diff] [blame] | 711 | |
Rusty Russell | 0864b79 | 2008-12-30 09:26:05 -0600 | [diff] [blame] | 712 | /* We need to know how many segments before we allocate. */ |
| 713 | err = virtio_config_val(vdev, VIRTIO_BLK_F_SEG_MAX, |
| 714 | offsetof(struct virtio_blk_config, seg_max), |
| 715 | &sg_elems); |
Christoph Hellwig | a5b365a | 2010-05-25 14:17:54 +0200 | [diff] [blame] | 716 | |
| 717 | /* We need at least one SG element, whatever they say. */ |
| 718 | if (err || !sg_elems) |
Rusty Russell | 0864b79 | 2008-12-30 09:26:05 -0600 | [diff] [blame] | 719 | sg_elems = 1; |
| 720 | |
| 721 | /* We need an extra sg elements at head and tail. */ |
| 722 | sg_elems += 2; |
| 723 | vdev->priv = vblk = kmalloc(sizeof(*vblk) + |
| 724 | sizeof(vblk->sg[0]) * sg_elems, GFP_KERNEL); |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 725 | if (!vblk) { |
| 726 | err = -ENOMEM; |
Michael S. Tsirkin | 5087a50 | 2011-10-30 21:29:59 +0200 | [diff] [blame] | 727 | goto out_free_index; |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 728 | } |
| 729 | |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 730 | init_waitqueue_head(&vblk->queue_wait); |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 731 | vblk->vdev = vdev; |
Rusty Russell | 0864b79 | 2008-12-30 09:26:05 -0600 | [diff] [blame] | 732 | vblk->sg_elems = sg_elems; |
| 733 | sg_init_table(vblk->sg, vblk->sg_elems); |
Michael S. Tsirkin | 4678d6f9 | 2012-01-12 15:44:44 +1030 | [diff] [blame] | 734 | mutex_init(&vblk->config_lock); |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 735 | |
Christoph Hellwig | 7a7c924 | 2011-02-01 21:43:48 +0100 | [diff] [blame] | 736 | INIT_WORK(&vblk->config_work, virtblk_config_changed_work); |
Michael S. Tsirkin | 4678d6f9 | 2012-01-12 15:44:44 +1030 | [diff] [blame] | 737 | vblk->config_enable = true; |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 738 | |
Amit Shah | 6abd6e5 | 2011-12-22 16:58:29 +0530 | [diff] [blame] | 739 | err = init_vq(vblk); |
| 740 | if (err) |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 741 | goto out_free_vblk; |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 742 | |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 743 | pool_size = sizeof(struct virtblk_req); |
| 744 | if (use_bio) |
| 745 | pool_size += sizeof(struct scatterlist) * sg_elems; |
| 746 | vblk->pool = mempool_create_kmalloc_pool(1, pool_size); |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 747 | if (!vblk->pool) { |
| 748 | err = -ENOMEM; |
| 749 | goto out_free_vq; |
| 750 | } |
| 751 | |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 752 | /* FIXME: How many partitions? How long is a piece of string? */ |
Christian Borntraeger | 4f3bf19 | 2008-01-31 15:53:53 +0100 | [diff] [blame] | 753 | vblk->disk = alloc_disk(1 << PART_BITS); |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 754 | if (!vblk->disk) { |
| 755 | err = -ENOMEM; |
Christian Borntraeger | 4f3bf19 | 2008-01-31 15:53:53 +0100 | [diff] [blame] | 756 | goto out_mempool; |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 757 | } |
| 758 | |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 759 | q = vblk->disk->queue = blk_init_queue(virtblk_request, NULL); |
Christoph Hellwig | 69740c8 | 2010-02-24 14:22:25 -0600 | [diff] [blame] | 760 | if (!q) { |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 761 | err = -ENOMEM; |
| 762 | goto out_put_disk; |
| 763 | } |
| 764 | |
Asias He | a98755c | 2012-08-08 16:07:04 +0800 | [diff] [blame] | 765 | if (use_bio) |
| 766 | blk_queue_make_request(q, virtblk_make_request); |
Christoph Hellwig | 69740c8 | 2010-02-24 14:22:25 -0600 | [diff] [blame] | 767 | q->queuedata = vblk; |
Fernando Luis Vázquez Cao | 7d116b6 | 2008-10-27 18:45:15 +0900 | [diff] [blame] | 768 | |
Ren Mingxin | c0aa3e0 | 2012-04-10 15:28:05 +0800 | [diff] [blame] | 769 | virtblk_name_format("vd", index, vblk->disk->disk_name, DISK_NAME_LEN); |
Christian Borntraeger | d50ed90 | 2008-02-01 09:05:00 +0100 | [diff] [blame] | 770 | |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 771 | vblk->disk->major = major; |
Christian Borntraeger | d50ed90 | 2008-02-01 09:05:00 +0100 | [diff] [blame] | 772 | vblk->disk->first_minor = index_to_minor(index); |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 773 | vblk->disk->private_data = vblk; |
| 774 | vblk->disk->fops = &virtblk_fops; |
Jeremy Katz | c483934 | 2008-03-02 17:00:15 -0500 | [diff] [blame] | 775 | vblk->disk->driverfs_dev = &vdev->dev; |
Michael S. Tsirkin | 5087a50 | 2011-10-30 21:29:59 +0200 | [diff] [blame] | 776 | vblk->index = index; |
Christian Borntraeger | 4f3bf19 | 2008-01-31 15:53:53 +0100 | [diff] [blame] | 777 | |
Tejun Heo | 02c42b7 | 2010-09-03 11:56:18 +0200 | [diff] [blame] | 778 | /* configure queue flush support */ |
Paolo Bonzini | cd5d503 | 2012-07-03 15:19:37 +0200 | [diff] [blame] | 779 | virtblk_update_cache_mode(vdev); |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 780 | |
Christian Borntraeger | 3ef5360 | 2008-05-16 11:17:03 +0200 | [diff] [blame] | 781 | /* If disk is read-only in the host, the guest should obey */ |
| 782 | if (virtio_has_feature(vdev, VIRTIO_BLK_F_RO)) |
| 783 | set_disk_ro(vblk->disk, 1); |
| 784 | |
Rusty Russell | a586d4f | 2008-02-04 23:49:56 -0500 | [diff] [blame] | 785 | /* Host must always specify the capacity. */ |
Rusty Russell | 72e61eb | 2008-05-02 21:50:49 -0500 | [diff] [blame] | 786 | vdev->config->get(vdev, offsetof(struct virtio_blk_config, capacity), |
| 787 | &cap, sizeof(cap)); |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 788 | |
| 789 | /* If capacity is too big, truncate with warning. */ |
| 790 | if ((sector_t)cap != cap) { |
| 791 | dev_warn(&vdev->dev, "Capacity %llu too large: truncating\n", |
| 792 | (unsigned long long)cap); |
| 793 | cap = (sector_t)-1; |
| 794 | } |
| 795 | set_capacity(vblk->disk, cap); |
| 796 | |
Rusty Russell | 0864b79 | 2008-12-30 09:26:05 -0600 | [diff] [blame] | 797 | /* We can handle whatever the host told us to handle. */ |
Martin K. Petersen | ee714f2 | 2010-03-10 00:48:32 -0500 | [diff] [blame] | 798 | blk_queue_max_segments(q, vblk->sg_elems-2); |
Rusty Russell | 0864b79 | 2008-12-30 09:26:05 -0600 | [diff] [blame] | 799 | |
Christoph Hellwig | 4eff3ca | 2009-07-17 21:47:45 -0600 | [diff] [blame] | 800 | /* No need to bounce any requests */ |
Christoph Hellwig | 69740c8 | 2010-02-24 14:22:25 -0600 | [diff] [blame] | 801 | blk_queue_bounce_limit(q, BLK_BOUNCE_ANY); |
Christoph Hellwig | 4eff3ca | 2009-07-17 21:47:45 -0600 | [diff] [blame] | 802 | |
Rusty Russell | 4b7f7e2 | 2008-12-30 09:26:04 -0600 | [diff] [blame] | 803 | /* No real sector limit. */ |
Martin K. Petersen | ee714f2 | 2010-03-10 00:48:32 -0500 | [diff] [blame] | 804 | blk_queue_max_hw_sectors(q, -1U); |
Rusty Russell | 4b7f7e2 | 2008-12-30 09:26:04 -0600 | [diff] [blame] | 805 | |
Rusty Russell | a586d4f | 2008-02-04 23:49:56 -0500 | [diff] [blame] | 806 | /* Host can optionally specify maximum segment size and number of |
| 807 | * segments. */ |
| 808 | err = virtio_config_val(vdev, VIRTIO_BLK_F_SIZE_MAX, |
| 809 | offsetof(struct virtio_blk_config, size_max), |
| 810 | &v); |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 811 | if (!err) |
Christoph Hellwig | 69740c8 | 2010-02-24 14:22:25 -0600 | [diff] [blame] | 812 | blk_queue_max_segment_size(q, v); |
Rusty Russell | 4b7f7e2 | 2008-12-30 09:26:04 -0600 | [diff] [blame] | 813 | else |
Christoph Hellwig | 69740c8 | 2010-02-24 14:22:25 -0600 | [diff] [blame] | 814 | blk_queue_max_segment_size(q, -1U); |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 815 | |
Christian Borntraeger | 066f4d8 | 2008-05-29 11:08:26 +0200 | [diff] [blame] | 816 | /* Host can optionally specify the block size of the device */ |
| 817 | err = virtio_config_val(vdev, VIRTIO_BLK_F_BLK_SIZE, |
| 818 | offsetof(struct virtio_blk_config, blk_size), |
| 819 | &blk_size); |
| 820 | if (!err) |
Christoph Hellwig | 69740c8 | 2010-02-24 14:22:25 -0600 | [diff] [blame] | 821 | blk_queue_logical_block_size(q, blk_size); |
| 822 | else |
| 823 | blk_size = queue_logical_block_size(q); |
| 824 | |
| 825 | /* Use topology information if available */ |
| 826 | err = virtio_config_val(vdev, VIRTIO_BLK_F_TOPOLOGY, |
| 827 | offsetof(struct virtio_blk_config, physical_block_exp), |
| 828 | &physical_block_exp); |
| 829 | if (!err && physical_block_exp) |
| 830 | blk_queue_physical_block_size(q, |
| 831 | blk_size * (1 << physical_block_exp)); |
| 832 | |
| 833 | err = virtio_config_val(vdev, VIRTIO_BLK_F_TOPOLOGY, |
| 834 | offsetof(struct virtio_blk_config, alignment_offset), |
| 835 | &alignment_offset); |
| 836 | if (!err && alignment_offset) |
| 837 | blk_queue_alignment_offset(q, blk_size * alignment_offset); |
| 838 | |
| 839 | err = virtio_config_val(vdev, VIRTIO_BLK_F_TOPOLOGY, |
| 840 | offsetof(struct virtio_blk_config, min_io_size), |
| 841 | &min_io_size); |
| 842 | if (!err && min_io_size) |
| 843 | blk_queue_io_min(q, blk_size * min_io_size); |
| 844 | |
| 845 | err = virtio_config_val(vdev, VIRTIO_BLK_F_TOPOLOGY, |
| 846 | offsetof(struct virtio_blk_config, opt_io_size), |
| 847 | &opt_io_size); |
| 848 | if (!err && opt_io_size) |
| 849 | blk_queue_io_opt(q, blk_size * opt_io_size); |
| 850 | |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 851 | add_disk(vblk->disk); |
Ryan Harper | a5eb9e4 | 2010-06-23 22:19:57 -0500 | [diff] [blame] | 852 | err = device_create_file(disk_to_dev(vblk->disk), &dev_attr_serial); |
| 853 | if (err) |
| 854 | goto out_del_disk; |
| 855 | |
Paolo Bonzini | cd5d503 | 2012-07-03 15:19:37 +0200 | [diff] [blame] | 856 | if (virtio_has_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE)) |
| 857 | err = device_create_file(disk_to_dev(vblk->disk), |
| 858 | &dev_attr_cache_type_rw); |
| 859 | else |
| 860 | err = device_create_file(disk_to_dev(vblk->disk), |
| 861 | &dev_attr_cache_type_ro); |
| 862 | if (err) |
| 863 | goto out_del_disk; |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 864 | return 0; |
| 865 | |
Ryan Harper | a5eb9e4 | 2010-06-23 22:19:57 -0500 | [diff] [blame] | 866 | out_del_disk: |
| 867 | del_gendisk(vblk->disk); |
| 868 | blk_cleanup_queue(vblk->disk->queue); |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 869 | out_put_disk: |
| 870 | put_disk(vblk->disk); |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 871 | out_mempool: |
| 872 | mempool_destroy(vblk->pool); |
| 873 | out_free_vq: |
Michael S. Tsirkin | d2a7ddd | 2009-06-12 22:16:36 -0600 | [diff] [blame] | 874 | vdev->config->del_vqs(vdev); |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 875 | out_free_vblk: |
| 876 | kfree(vblk); |
Michael S. Tsirkin | 5087a50 | 2011-10-30 21:29:59 +0200 | [diff] [blame] | 877 | out_free_index: |
| 878 | ida_simple_remove(&vd_index_ida, index); |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 879 | out: |
| 880 | return err; |
| 881 | } |
| 882 | |
Mike Frysinger | 98e9444 | 2009-05-18 03:39:09 -0400 | [diff] [blame] | 883 | static void __devexit virtblk_remove(struct virtio_device *vdev) |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 884 | { |
| 885 | struct virtio_blk *vblk = vdev->priv; |
Michael S. Tsirkin | 5087a50 | 2011-10-30 21:29:59 +0200 | [diff] [blame] | 886 | int index = vblk->index; |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 887 | |
Michael S. Tsirkin | 4678d6f9 | 2012-01-12 15:44:44 +1030 | [diff] [blame] | 888 | /* Prevent config work handler from accessing the device. */ |
| 889 | mutex_lock(&vblk->config_lock); |
| 890 | vblk->config_enable = false; |
| 891 | mutex_unlock(&vblk->config_lock); |
Christoph Hellwig | 7a7c924 | 2011-02-01 21:43:48 +0100 | [diff] [blame] | 892 | |
Asias He | 02e2b12 | 2012-05-25 10:34:47 +0800 | [diff] [blame] | 893 | del_gendisk(vblk->disk); |
Asias He | 483001c | 2012-05-25 10:34:48 +0800 | [diff] [blame] | 894 | blk_cleanup_queue(vblk->disk->queue); |
Asias He | 02e2b12 | 2012-05-25 10:34:47 +0800 | [diff] [blame] | 895 | |
Rusty Russell | 6e5aa7e | 2008-02-04 23:50:03 -0500 | [diff] [blame] | 896 | /* Stop all the virtqueues. */ |
| 897 | vdev->config->reset(vdev); |
| 898 | |
Michael S. Tsirkin | 4678d6f9 | 2012-01-12 15:44:44 +1030 | [diff] [blame] | 899 | flush_work(&vblk->config_work); |
| 900 | |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 901 | put_disk(vblk->disk); |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 902 | mempool_destroy(vblk->pool); |
Michael S. Tsirkin | d2a7ddd | 2009-06-12 22:16:36 -0600 | [diff] [blame] | 903 | vdev->config->del_vqs(vdev); |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 904 | kfree(vblk); |
Michael S. Tsirkin | 5087a50 | 2011-10-30 21:29:59 +0200 | [diff] [blame] | 905 | ida_simple_remove(&vd_index_ida, index); |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 906 | } |
| 907 | |
Amit Shah | f8fb5bc | 2011-12-22 16:58:30 +0530 | [diff] [blame] | 908 | #ifdef CONFIG_PM |
| 909 | static int virtblk_freeze(struct virtio_device *vdev) |
| 910 | { |
| 911 | struct virtio_blk *vblk = vdev->priv; |
| 912 | |
| 913 | /* Ensure we don't receive any more interrupts */ |
| 914 | vdev->config->reset(vdev); |
| 915 | |
| 916 | /* Prevent config work handler from accessing the device. */ |
| 917 | mutex_lock(&vblk->config_lock); |
| 918 | vblk->config_enable = false; |
| 919 | mutex_unlock(&vblk->config_lock); |
| 920 | |
| 921 | flush_work(&vblk->config_work); |
| 922 | |
| 923 | spin_lock_irq(vblk->disk->queue->queue_lock); |
| 924 | blk_stop_queue(vblk->disk->queue); |
| 925 | spin_unlock_irq(vblk->disk->queue->queue_lock); |
| 926 | blk_sync_queue(vblk->disk->queue); |
| 927 | |
| 928 | vdev->config->del_vqs(vdev); |
| 929 | return 0; |
| 930 | } |
| 931 | |
| 932 | static int virtblk_restore(struct virtio_device *vdev) |
| 933 | { |
| 934 | struct virtio_blk *vblk = vdev->priv; |
| 935 | int ret; |
| 936 | |
| 937 | vblk->config_enable = true; |
| 938 | ret = init_vq(vdev->priv); |
| 939 | if (!ret) { |
| 940 | spin_lock_irq(vblk->disk->queue->queue_lock); |
| 941 | blk_start_queue(vblk->disk->queue); |
| 942 | spin_unlock_irq(vblk->disk->queue->queue_lock); |
| 943 | } |
| 944 | return ret; |
| 945 | } |
| 946 | #endif |
| 947 | |
Márton Németh | 47483e2 | 2010-01-10 13:40:02 +0100 | [diff] [blame] | 948 | static const struct virtio_device_id id_table[] = { |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 949 | { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID }, |
| 950 | { 0 }, |
| 951 | }; |
| 952 | |
Rusty Russell | c45a681 | 2008-05-02 21:50:50 -0500 | [diff] [blame] | 953 | static unsigned int features[] = { |
Tejun Heo | 02c42b7 | 2010-09-03 11:56:18 +0200 | [diff] [blame] | 954 | VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY, |
| 955 | VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE, VIRTIO_BLK_F_SCSI, |
Paolo Bonzini | cd5d503 | 2012-07-03 15:19:37 +0200 | [diff] [blame] | 956 | VIRTIO_BLK_F_WCE, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE |
Rusty Russell | c45a681 | 2008-05-02 21:50:50 -0500 | [diff] [blame] | 957 | }; |
| 958 | |
Rakib Mullick | 4fbfff76 | 2009-07-17 20:13:22 +0600 | [diff] [blame] | 959 | /* |
| 960 | * virtio_blk causes spurious section mismatch warning by |
| 961 | * simultaneously referring to a __devinit and a __devexit function. |
| 962 | * Use __refdata to avoid this warning. |
| 963 | */ |
| 964 | static struct virtio_driver __refdata virtio_blk = { |
Christoph Hellwig | 7a7c924 | 2011-02-01 21:43:48 +0100 | [diff] [blame] | 965 | .feature_table = features, |
| 966 | .feature_table_size = ARRAY_SIZE(features), |
| 967 | .driver.name = KBUILD_MODNAME, |
| 968 | .driver.owner = THIS_MODULE, |
| 969 | .id_table = id_table, |
| 970 | .probe = virtblk_probe, |
| 971 | .remove = __devexit_p(virtblk_remove), |
| 972 | .config_changed = virtblk_config_changed, |
Amit Shah | f8fb5bc | 2011-12-22 16:58:30 +0530 | [diff] [blame] | 973 | #ifdef CONFIG_PM |
| 974 | .freeze = virtblk_freeze, |
| 975 | .restore = virtblk_restore, |
| 976 | #endif |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 977 | }; |
| 978 | |
| 979 | static int __init init(void) |
| 980 | { |
Christoph Hellwig | 7a7c924 | 2011-02-01 21:43:48 +0100 | [diff] [blame] | 981 | int error; |
| 982 | |
| 983 | virtblk_wq = alloc_workqueue("virtio-blk", 0, 0); |
| 984 | if (!virtblk_wq) |
| 985 | return -ENOMEM; |
| 986 | |
Christian Borntraeger | 4f3bf19 | 2008-01-31 15:53:53 +0100 | [diff] [blame] | 987 | major = register_blkdev(0, "virtblk"); |
Christoph Hellwig | 7a7c924 | 2011-02-01 21:43:48 +0100 | [diff] [blame] | 988 | if (major < 0) { |
| 989 | error = major; |
| 990 | goto out_destroy_workqueue; |
| 991 | } |
| 992 | |
| 993 | error = register_virtio_driver(&virtio_blk); |
| 994 | if (error) |
| 995 | goto out_unregister_blkdev; |
| 996 | return 0; |
| 997 | |
| 998 | out_unregister_blkdev: |
| 999 | unregister_blkdev(major, "virtblk"); |
| 1000 | out_destroy_workqueue: |
| 1001 | destroy_workqueue(virtblk_wq); |
| 1002 | return error; |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 1003 | } |
| 1004 | |
| 1005 | static void __exit fini(void) |
| 1006 | { |
Christian Borntraeger | 4f3bf19 | 2008-01-31 15:53:53 +0100 | [diff] [blame] | 1007 | unregister_blkdev(major, "virtblk"); |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 1008 | unregister_virtio_driver(&virtio_blk); |
Christoph Hellwig | 7a7c924 | 2011-02-01 21:43:48 +0100 | [diff] [blame] | 1009 | destroy_workqueue(virtblk_wq); |
Rusty Russell | e467cde | 2007-10-22 11:03:38 +1000 | [diff] [blame] | 1010 | } |
| 1011 | module_init(init); |
| 1012 | module_exit(fini); |
| 1013 | |
| 1014 | MODULE_DEVICE_TABLE(virtio, id_table); |
| 1015 | MODULE_DESCRIPTION("Virtio block driver"); |
| 1016 | MODULE_LICENSE("GPL"); |