blob: 23374603e4d900850afd3b667eb25cdd082743b5 [file] [log] [blame]
Rusty Russell48925e32009-09-24 09:59:20 -06001/* A network driver using virtio.
Rusty Russell296f96f2007-10-22 11:03:37 +10002 *
3 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
Jeff Kirsheradf8d3f2013-12-06 06:28:47 -080016 * along with this program; if not, see <http://www.gnu.org/licenses/>.
Rusty Russell296f96f2007-10-22 11:03:37 +100017 */
18//#define DEBUG
19#include <linux/netdevice.h>
20#include <linux/etherdevice.h>
Herbert Xua9ea3fc2008-04-18 11:21:42 +080021#include <linux/ethtool.h>
Rusty Russell296f96f2007-10-22 11:03:37 +100022#include <linux/module.h>
23#include <linux/virtio.h>
24#include <linux/virtio_net.h>
John Fastabendf600b692016-12-15 12:13:24 -080025#include <linux/bpf.h>
Daniel Borkmanna67edbf2017-01-25 02:28:18 +010026#include <linux/bpf_trace.h>
Rusty Russell296f96f2007-10-22 11:03:37 +100027#include <linux/scatterlist.h>
Alex Williamsone918085a2009-01-25 18:06:26 -080028#include <linux/if_vlan.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090029#include <linux/slab.h>
Wanlong Gao8de4b2f2013-01-24 23:51:31 +000030#include <linux/cpu.h>
Michael Daltonab7db912014-01-16 22:23:27 -080031#include <linux/average.h>
Jason Wang186b3c92017-09-19 17:42:43 +080032#include <linux/filter.h>
Michael S. Tsirkind85b758f72017-03-09 02:21:21 +020033#include <net/route.h>
Jesper Dangaard Brouer754b8a22018-01-03 11:26:04 +010034#include <net/xdp.h>
Rusty Russell296f96f2007-10-22 11:03:37 +100035
Amerigo Wangd34710e2013-05-09 19:50:51 +000036static int napi_weight = NAPI_POLL_WEIGHT;
Dor Laor6c0cd7c2007-12-16 15:19:43 +020037module_param(napi_weight, int, 0444);
38
Willem de Bruijnb92f1e62017-04-24 13:49:27 -040039static bool csum = true, gso = true, napi_tx;
Rusty Russell34a48572008-02-04 23:50:02 -050040module_param(csum, bool, 0444);
41module_param(gso, bool, 0444);
Willem de Bruijnb92f1e62017-04-24 13:49:27 -040042module_param(napi_tx, bool, 0644);
Rusty Russell34a48572008-02-04 23:50:02 -050043
Rusty Russell296f96f2007-10-22 11:03:37 +100044/* FIXME: MTU in config. */
Michael Dalton5061de32013-11-14 10:41:04 -080045#define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -080046#define GOOD_COPY_LEN 128
Rusty Russell296f96f2007-10-22 11:03:37 +100047
Jason Wangf6b10202017-02-21 16:46:28 +080048#define VIRTNET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)
49
John Fastabend2de2f7f2017-02-02 19:16:29 -080050/* Amount of XDP headroom to prepend to packets for use by xdp_adjust_head */
51#define VIRTIO_XDP_HEADROOM 256
52
Johannes Berg5377d7582015-08-19 09:48:40 +020053/* RX packet size EWMA. The average packet size is used to determine the packet
54 * buffer size when refilling RX rings. As the entire RX ring may be refilled
55 * at once, the weight is chosen so that the EWMA will be insensitive to short-
56 * term, transient changes in packet size.
Michael Daltonab7db912014-01-16 22:23:27 -080057 */
Johannes Bergeb1e0112017-02-15 09:49:26 +010058DECLARE_EWMA(pkt_len, 0, 64)
Michael Daltonab7db912014-01-16 22:23:27 -080059
Rick Jones66846042011-11-14 14:17:08 +000060#define VIRTNET_DRIVER_VERSION "1.0.0"
Alex Williamson2a41f712009-02-04 09:02:34 +000061
Colin Ian King7acd4322017-08-12 22:45:53 +010062static const unsigned long guest_offloads[] = {
63 VIRTIO_NET_F_GUEST_TSO4,
64 VIRTIO_NET_F_GUEST_TSO6,
65 VIRTIO_NET_F_GUEST_ECN,
66 VIRTIO_NET_F_GUEST_UFO
67};
Jason Wang3f935222017-07-19 16:54:49 +080068
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +090069struct virtnet_stat_desc {
70 char desc[ETH_GSTRING_LEN];
71 size_t offset;
stephen hemminger3fa2a1d2011-06-15 06:36:29 +000072};
73
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +090074struct virtnet_sq_stats {
75 struct u64_stats_sync syncp;
76 u64 packets;
77 u64 bytes;
78};
79
80struct virtnet_rq_stats {
81 struct u64_stats_sync syncp;
82 u64 packets;
83 u64 bytes;
84};
85
86#define VIRTNET_SQ_STAT(m) offsetof(struct virtnet_sq_stats, m)
87#define VIRTNET_RQ_STAT(m) offsetof(struct virtnet_rq_stats, m)
88
89static const struct virtnet_stat_desc virtnet_sq_stats_desc[] = {
90 { "packets", VIRTNET_SQ_STAT(packets) },
91 { "bytes", VIRTNET_SQ_STAT(bytes) },
92};
93
94static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = {
95 { "packets", VIRTNET_RQ_STAT(packets) },
96 { "bytes", VIRTNET_RQ_STAT(bytes) },
97};
98
99#define VIRTNET_SQ_STATS_LEN ARRAY_SIZE(virtnet_sq_stats_desc)
100#define VIRTNET_RQ_STATS_LEN ARRAY_SIZE(virtnet_rq_stats_desc)
101
Jason Wange9d74172012-12-07 07:04:55 +0000102/* Internal representation of a send virtqueue */
103struct send_queue {
104 /* Virtqueue associated with this send _queue */
105 struct virtqueue *vq;
106
107 /* TX: fragments + linear part + virtio header */
108 struct scatterlist sg[MAX_SKB_FRAGS + 2];
Jason Wang986a4f42012-12-07 07:04:56 +0000109
110 /* Name of the send queue: output.$index */
111 char name[40];
Willem de Bruijnb92f1e62017-04-24 13:49:27 -0400112
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +0900113 struct virtnet_sq_stats stats;
114
Willem de Bruijnb92f1e62017-04-24 13:49:27 -0400115 struct napi_struct napi;
Jason Wange9d74172012-12-07 07:04:55 +0000116};
117
118/* Internal representation of a receive virtqueue */
119struct receive_queue {
120 /* Virtqueue associated with this receive_queue */
121 struct virtqueue *vq;
122
Rusty Russell296f96f2007-10-22 11:03:37 +1000123 struct napi_struct napi;
124
John Fastabendf600b692016-12-15 12:13:24 -0800125 struct bpf_prog __rcu *xdp_prog;
126
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +0900127 struct virtnet_rq_stats stats;
128
Jason Wange9d74172012-12-07 07:04:55 +0000129 /* Chain pages by the private ptr. */
130 struct page *pages;
131
Michael Daltonab7db912014-01-16 22:23:27 -0800132 /* Average packet length for mergeable receive buffers. */
Johannes Berg5377d7582015-08-19 09:48:40 +0200133 struct ewma_pkt_len mrg_avg_pkt_len;
Michael Daltonab7db912014-01-16 22:23:27 -0800134
Michael Daltonfb518792014-01-16 22:23:26 -0800135 /* Page frag for packet buffer allocation. */
136 struct page_frag alloc_frag;
137
Jason Wange9d74172012-12-07 07:04:55 +0000138 /* RX: fragments + linear part + virtio header */
139 struct scatterlist sg[MAX_SKB_FRAGS + 2];
Jason Wang986a4f42012-12-07 07:04:56 +0000140
Michael S. Tsirkind85b758f72017-03-09 02:21:21 +0200141 /* Min single buffer size for mergeable buffers case. */
142 unsigned int min_buf_len;
143
Jason Wang986a4f42012-12-07 07:04:56 +0000144 /* Name of this receive queue: input.$index */
145 char name[40];
Jesper Dangaard Brouer754b8a22018-01-03 11:26:04 +0100146
147 struct xdp_rxq_info xdp_rxq;
Jason Wange9d74172012-12-07 07:04:55 +0000148};
149
150struct virtnet_info {
151 struct virtio_device *vdev;
152 struct virtqueue *cvq;
153 struct net_device *dev;
Jason Wang986a4f42012-12-07 07:04:56 +0000154 struct send_queue *sq;
155 struct receive_queue *rq;
Jason Wange9d74172012-12-07 07:04:55 +0000156 unsigned int status;
157
Jason Wang986a4f42012-12-07 07:04:56 +0000158 /* Max # of queue pairs supported by the device */
159 u16 max_queue_pairs;
160
161 /* # of queue pairs currently used by the driver */
162 u16 curr_queue_pairs;
163
John Fastabend672aafd2016-12-15 12:13:49 -0800164 /* # of XDP queue pairs currently used by the driver */
165 u16 xdp_queue_pairs;
166
Herbert Xu97402b92008-04-18 11:24:27 +0800167 /* I like... big packets and I cannot lie! */
168 bool big_packets;
169
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -0800170 /* Host will merge rx buffers for big packets (shake it! shake it!) */
171 bool mergeable_rx_bufs;
172
Jason Wang986a4f42012-12-07 07:04:56 +0000173 /* Has control virtqueue */
174 bool has_cvq;
175
Michael S. Tsirkine7428e92013-07-25 10:20:23 +0930176 /* Host can handle any s/g split between our header and packet data */
177 bool any_header_sg;
178
Michael S. Tsirkin012873d2014-10-24 16:55:57 +0300179 /* Packet virtio header size */
180 u8 hdr_len;
181
Rusty Russell3161e452009-08-26 12:22:32 -0700182 /* Work struct for refilling if we run low on memory. */
183 struct delayed_work refill;
184
Jason Wang586d17c2012-04-11 20:43:52 +0000185 /* Work struct for config space updates */
186 struct work_struct config_work;
187
Jason Wang986a4f42012-12-07 07:04:56 +0000188 /* Does the affinity hint is set for virtqueues? */
189 bool affinity_hint_set;
Wanlong Gao47be2472013-01-24 23:51:29 +0000190
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +0200191 /* CPU hotplug instances for online & dead */
192 struct hlist_node node;
193 struct hlist_node node_dead;
Michael S. Tsirkin2ac46032015-11-15 15:11:00 +0200194
195 /* Control VQ buffers: protected by the rtnl lock */
196 struct virtio_net_ctrl_hdr ctrl_hdr;
197 virtio_net_ctrl_ack ctrl_status;
Andy Lutomirskia725ee32016-07-18 15:34:49 -0700198 struct virtio_net_ctrl_mq ctrl_mq;
Michael S. Tsirkin2ac46032015-11-15 15:11:00 +0200199 u8 ctrl_promisc;
200 u8 ctrl_allmulti;
Andy Lutomirskia725ee32016-07-18 15:34:49 -0700201 u16 ctrl_vid;
Jason Wang3f935222017-07-19 16:54:49 +0800202 u64 ctrl_offloads;
Nikolay Aleksandrov16032be2016-02-03 04:04:37 +0100203
204 /* Ethtool settings */
205 u8 duplex;
206 u32 speed;
Jason Wang3f935222017-07-19 16:54:49 +0800207
208 unsigned long guest_offloads;
Rusty Russell296f96f2007-10-22 11:03:37 +1000209};
210
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000211struct padded_vnet_hdr {
Michael S. Tsirkin012873d2014-10-24 16:55:57 +0300212 struct virtio_net_hdr_mrg_rxbuf hdr;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000213 /*
Michael S. Tsirkin012873d2014-10-24 16:55:57 +0300214 * hdr is in a separate sg buffer, and data sg buffer shares same page
215 * with this header sg. This padding makes next sg 16 byte aligned
216 * after the header.
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000217 */
Michael S. Tsirkin012873d2014-10-24 16:55:57 +0300218 char padding[4];
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000219};
220
Jason Wang986a4f42012-12-07 07:04:56 +0000221/* Converting between virtqueue no. and kernel tx/rx queue no.
222 * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq
223 */
224static int vq2txq(struct virtqueue *vq)
225{
Rusty Russell9d0ca6e2013-03-21 14:17:34 +0000226 return (vq->index - 1) / 2;
Jason Wang986a4f42012-12-07 07:04:56 +0000227}
228
229static int txq2vq(int txq)
230{
231 return txq * 2 + 1;
232}
233
234static int vq2rxq(struct virtqueue *vq)
235{
Rusty Russell9d0ca6e2013-03-21 14:17:34 +0000236 return vq->index / 2;
Jason Wang986a4f42012-12-07 07:04:56 +0000237}
238
239static int rxq2vq(int rxq)
240{
241 return rxq * 2;
242}
243
Michael S. Tsirkin012873d2014-10-24 16:55:57 +0300244static inline struct virtio_net_hdr_mrg_rxbuf *skb_vnet_hdr(struct sk_buff *skb)
Rusty Russell296f96f2007-10-22 11:03:37 +1000245{
Michael S. Tsirkin012873d2014-10-24 16:55:57 +0300246 return (struct virtio_net_hdr_mrg_rxbuf *)skb->cb;
Rusty Russell296f96f2007-10-22 11:03:37 +1000247}
248
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000249/*
250 * private is used to chain pages for big packets, put the whole
251 * most recent used list in the beginning for reuse
252 */
Jason Wange9d74172012-12-07 07:04:55 +0000253static void give_pages(struct receive_queue *rq, struct page *page)
Rusty Russellfb6813f2008-07-25 12:06:01 -0500254{
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000255 struct page *end;
256
Jason Wange9d74172012-12-07 07:04:55 +0000257 /* Find end of list, sew whole thing into vi->rq.pages. */
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000258 for (end = page; end->private; end = (struct page *)end->private);
Jason Wange9d74172012-12-07 07:04:55 +0000259 end->private = (unsigned long)rq->pages;
260 rq->pages = page;
Rusty Russellfb6813f2008-07-25 12:06:01 -0500261}
262
Jason Wange9d74172012-12-07 07:04:55 +0000263static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask)
Rusty Russellfb6813f2008-07-25 12:06:01 -0500264{
Jason Wange9d74172012-12-07 07:04:55 +0000265 struct page *p = rq->pages;
Rusty Russellfb6813f2008-07-25 12:06:01 -0500266
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000267 if (p) {
Jason Wange9d74172012-12-07 07:04:55 +0000268 rq->pages = (struct page *)p->private;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000269 /* clear private here, it is used to chain pages */
270 p->private = 0;
271 } else
Rusty Russellfb6813f2008-07-25 12:06:01 -0500272 p = alloc_page(gfp_mask);
273 return p;
274}
275
Willem de Bruijne4e84522017-04-24 13:49:26 -0400276static void virtqueue_napi_schedule(struct napi_struct *napi,
277 struct virtqueue *vq)
278{
279 if (napi_schedule_prep(napi)) {
280 virtqueue_disable_cb(vq);
281 __napi_schedule(napi);
282 }
283}
284
285static void virtqueue_napi_complete(struct napi_struct *napi,
286 struct virtqueue *vq, int processed)
287{
288 int opaque;
289
290 opaque = virtqueue_enable_cb_prepare(vq);
Toshiaki Makitafdaa7672017-12-07 13:15:15 +0900291 if (napi_complete_done(napi, processed)) {
292 if (unlikely(virtqueue_poll(vq, opaque)))
293 virtqueue_napi_schedule(napi, vq);
294 } else {
295 virtqueue_disable_cb(vq);
296 }
Willem de Bruijne4e84522017-04-24 13:49:26 -0400297}
298
Jason Wange9d74172012-12-07 07:04:55 +0000299static void skb_xmit_done(struct virtqueue *vq)
Rusty Russell296f96f2007-10-22 11:03:37 +1000300{
Jason Wange9d74172012-12-07 07:04:55 +0000301 struct virtnet_info *vi = vq->vdev->priv;
Willem de Bruijnb92f1e62017-04-24 13:49:27 -0400302 struct napi_struct *napi = &vi->sq[vq2txq(vq)].napi;
Rusty Russell296f96f2007-10-22 11:03:37 +1000303
Rusty Russell2cb9c6b2008-02-04 23:50:07 -0500304 /* Suppress further interrupts. */
Jason Wange9d74172012-12-07 07:04:55 +0000305 virtqueue_disable_cb(vq);
Rusty Russell11a3a152008-05-26 17:48:13 +1000306
Willem de Bruijnb92f1e62017-04-24 13:49:27 -0400307 if (napi->weight)
308 virtqueue_napi_schedule(napi, vq);
309 else
310 /* We were probably waiting for more output buffers. */
311 netif_wake_subqueue(vi->dev, vq2txq(vq));
Rusty Russell296f96f2007-10-22 11:03:37 +1000312}
313
Jason Wang28b39bc2017-07-19 16:54:46 +0800314#define MRG_CTX_HEADER_SHIFT 22
315static void *mergeable_len_to_ctx(unsigned int truesize,
316 unsigned int headroom)
317{
318 return (void *)(unsigned long)((headroom << MRG_CTX_HEADER_SHIFT) | truesize);
319}
320
321static unsigned int mergeable_ctx_to_headroom(void *mrg_ctx)
322{
323 return (unsigned long)mrg_ctx >> MRG_CTX_HEADER_SHIFT;
324}
325
326static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx)
327{
328 return (unsigned long)mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1);
329}
330
Mike Waychison34646452012-01-04 12:52:32 +0000331/* Called from bottom half context */
Michael S. Tsirkin946fa562014-10-24 00:12:10 +0300332static struct sk_buff *page_to_skb(struct virtnet_info *vi,
333 struct receive_queue *rq,
Michael Dalton2613af02013-10-28 15:44:18 -0700334 struct page *page, unsigned int offset,
335 unsigned int len, unsigned int truesize)
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000336{
337 struct sk_buff *skb;
Michael S. Tsirkin012873d2014-10-24 16:55:57 +0300338 struct virtio_net_hdr_mrg_rxbuf *hdr;
Michael Dalton2613af02013-10-28 15:44:18 -0700339 unsigned int copy, hdr_len, hdr_padded_len;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000340 char *p;
341
Michael Dalton2613af02013-10-28 15:44:18 -0700342 p = page_address(page) + offset;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000343
344 /* copy small packet so we can reuse these pages for small data */
Paolo Abenic67f5db2016-03-17 15:44:00 +0100345 skb = napi_alloc_skb(&rq->napi, GOOD_COPY_LEN);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000346 if (unlikely(!skb))
347 return NULL;
348
349 hdr = skb_vnet_hdr(skb);
350
Michael S. Tsirkin012873d2014-10-24 16:55:57 +0300351 hdr_len = vi->hdr_len;
352 if (vi->mergeable_rx_bufs)
stephen hemmingera4a76502017-08-15 10:29:17 -0700353 hdr_padded_len = sizeof(*hdr);
Michael S. Tsirkin012873d2014-10-24 16:55:57 +0300354 else
Michael Dalton2613af02013-10-28 15:44:18 -0700355 hdr_padded_len = sizeof(struct padded_vnet_hdr);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000356
357 memcpy(hdr, p, hdr_len);
358
359 len -= hdr_len;
Michael Dalton2613af02013-10-28 15:44:18 -0700360 offset += hdr_padded_len;
361 p += hdr_padded_len;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000362
363 copy = len;
364 if (copy > skb_tailroom(skb))
365 copy = skb_tailroom(skb);
Johannes Berg59ae1d12017-06-16 14:29:20 +0200366 skb_put_data(skb, p, copy);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000367
368 len -= copy;
369 offset += copy;
370
Michael Dalton2613af02013-10-28 15:44:18 -0700371 if (vi->mergeable_rx_bufs) {
372 if (len)
373 skb_add_rx_frag(skb, 0, page, offset, len, truesize);
374 else
375 put_page(page);
376 return skb;
377 }
378
Sasha Levine878d782011-09-28 04:40:54 +0000379 /*
380 * Verify that we can indeed put this data into a skb.
381 * This is here to handle cases when the device erroneously
382 * tries to receive more than is possible. This is usually
383 * the case of a broken device.
384 */
385 if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) {
Amerigo Wangbe443892012-11-08 17:47:28 +0000386 net_dbg_ratelimited("%s: too much data\n", skb->dev->name);
Sasha Levine878d782011-09-28 04:40:54 +0000387 dev_kfree_skb(skb);
388 return NULL;
389 }
Michael Dalton2613af02013-10-28 15:44:18 -0700390 BUG_ON(offset >= PAGE_SIZE);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000391 while (len) {
Michael Dalton2613af02013-10-28 15:44:18 -0700392 unsigned int frag_size = min((unsigned)PAGE_SIZE - offset, len);
393 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset,
394 frag_size, truesize);
395 len -= frag_size;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000396 page = (struct page *)page->private;
397 offset = 0;
398 }
399
400 if (page)
Jason Wange9d74172012-12-07 07:04:55 +0000401 give_pages(rq, page);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000402
403 return skb;
404}
405
Jason Wang186b3c92017-09-19 17:42:43 +0800406static void virtnet_xdp_flush(struct net_device *dev)
407{
408 struct virtnet_info *vi = netdev_priv(dev);
409 struct send_queue *sq;
410 unsigned int qp;
411
412 qp = vi->curr_queue_pairs - vi->xdp_queue_pairs + smp_processor_id();
413 sq = &vi->sq[qp];
414
415 virtqueue_kick(sq->vq);
416}
417
418static bool __virtnet_xdp_xmit(struct virtnet_info *vi,
419 struct xdp_buff *xdp)
John Fastabend56434a02016-12-15 12:14:13 -0800420{
John Fastabend56434a02016-12-15 12:14:13 -0800421 struct virtio_net_hdr_mrg_rxbuf *hdr;
Jason Wangf6b10202017-02-21 16:46:28 +0800422 unsigned int len;
John Fastabend722d8282017-02-02 19:15:32 -0800423 struct send_queue *sq;
424 unsigned int qp;
John Fastabend56434a02016-12-15 12:14:13 -0800425 void *xdp_sent;
426 int err;
427
John Fastabend722d8282017-02-02 19:15:32 -0800428 qp = vi->curr_queue_pairs - vi->xdp_queue_pairs + smp_processor_id();
429 sq = &vi->sq[qp];
430
John Fastabend56434a02016-12-15 12:14:13 -0800431 /* Free up any pending old buffers before queueing new ones. */
432 while ((xdp_sent = virtqueue_get_buf(sq->vq, &len)) != NULL) {
Jason Wangf6b10202017-02-21 16:46:28 +0800433 struct page *sent_page = virt_to_head_page(xdp_sent);
Jason Wangbb91acc2016-12-23 22:37:32 +0800434
Jason Wangf6b10202017-02-21 16:46:28 +0800435 put_page(sent_page);
John Fastabend56434a02016-12-15 12:14:13 -0800436 }
437
Jason Wangf6b10202017-02-21 16:46:28 +0800438 xdp->data -= vi->hdr_len;
439 /* Zero header and leave csum up to XDP layers */
440 hdr = xdp->data;
441 memset(hdr, 0, vi->hdr_len);
John Fastabend56434a02016-12-15 12:14:13 -0800442
Jason Wangf6b10202017-02-21 16:46:28 +0800443 sg_init_one(sq->sg, xdp->data, xdp->data_end - xdp->data);
Jason Wangbb91acc2016-12-23 22:37:32 +0800444
Jason Wangf6b10202017-02-21 16:46:28 +0800445 err = virtqueue_add_outbuf(sq->vq, sq->sg, 1, xdp->data, GFP_ATOMIC);
Jesper Dangaard Brouer11b7d8972018-02-20 14:32:15 +0100446 if (unlikely(err))
447 return false; /* Caller handle free/refcnt */
John Fastabend56434a02016-12-15 12:14:13 -0800448
Daniel Borkmanna67edbf2017-01-25 02:28:18 +0100449 return true;
John Fastabend56434a02016-12-15 12:14:13 -0800450}
451
Jason Wang186b3c92017-09-19 17:42:43 +0800452static int virtnet_xdp_xmit(struct net_device *dev, struct xdp_buff *xdp)
453{
454 struct virtnet_info *vi = netdev_priv(dev);
Jesper Dangaard Brouer8dcc5b02018-02-20 14:32:20 +0100455 struct receive_queue *rq = vi->rq;
456 struct bpf_prog *xdp_prog;
457 bool sent;
Jason Wang186b3c92017-09-19 17:42:43 +0800458
Jesper Dangaard Brouer8dcc5b02018-02-20 14:32:20 +0100459 /* Only allow ndo_xdp_xmit if XDP is loaded on dev, as this
460 * indicate XDP resources have been successfully allocated.
461 */
462 xdp_prog = rcu_dereference(rq->xdp_prog);
463 if (!xdp_prog)
464 return -ENXIO;
465
466 sent = __virtnet_xdp_xmit(vi, xdp);
Jason Wang186b3c92017-09-19 17:42:43 +0800467 if (!sent)
468 return -ENOSPC;
469 return 0;
470}
471
Jason Wangf6b10202017-02-21 16:46:28 +0800472static unsigned int virtnet_get_headroom(struct virtnet_info *vi)
473{
474 return vi->xdp_queue_pairs ? VIRTIO_XDP_HEADROOM : 0;
475}
476
Jason Wang4941d472017-07-19 16:54:48 +0800477/* We copy the packet for XDP in the following cases:
478 *
479 * 1) Packet is scattered across multiple rx buffers.
480 * 2) Headroom space is insufficient.
481 *
482 * This is inefficient but it's a temporary condition that
483 * we hit right after XDP is enabled and until queue is refilled
484 * with large buffers with sufficient headroom - so it should affect
485 * at most queue size packets.
486 * Afterwards, the conditions to enable
487 * XDP should preclude the underlying device from sending packets
488 * across multiple buffers (num_buf > 1), and we make sure buffers
489 * have enough headroom.
John Fastabend72979a62016-12-15 12:14:36 -0800490 */
491static struct page *xdp_linearize_page(struct receive_queue *rq,
Jason Wang56a86f82016-12-23 22:37:26 +0800492 u16 *num_buf,
John Fastabend72979a62016-12-15 12:14:36 -0800493 struct page *p,
494 int offset,
Jason Wang4941d472017-07-19 16:54:48 +0800495 int page_off,
John Fastabend72979a62016-12-15 12:14:36 -0800496 unsigned int *len)
497{
498 struct page *page = alloc_page(GFP_ATOMIC);
John Fastabend72979a62016-12-15 12:14:36 -0800499
500 if (!page)
501 return NULL;
502
503 memcpy(page_address(page) + page_off, page_address(p) + offset, *len);
504 page_off += *len;
505
Jason Wang56a86f82016-12-23 22:37:26 +0800506 while (--*num_buf) {
Jason Wang3cc81a92018-03-02 17:29:14 +0800507 int tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
John Fastabend72979a62016-12-15 12:14:36 -0800508 unsigned int buflen;
John Fastabend72979a62016-12-15 12:14:36 -0800509 void *buf;
510 int off;
511
Michael S. Tsirkin680557c2017-03-06 21:29:47 +0200512 buf = virtqueue_get_buf(rq->vq, &buflen);
513 if (unlikely(!buf))
John Fastabend72979a62016-12-15 12:14:36 -0800514 goto err_buf;
515
John Fastabend72979a62016-12-15 12:14:36 -0800516 p = virt_to_head_page(buf);
517 off = buf - page_address(p);
518
Jason Wang56a86f82016-12-23 22:37:26 +0800519 /* guard against a misconfigured or uncooperative backend that
520 * is sending packet larger than the MTU.
521 */
Jason Wang3cc81a92018-03-02 17:29:14 +0800522 if ((page_off + buflen + tailroom) > PAGE_SIZE) {
Jason Wang56a86f82016-12-23 22:37:26 +0800523 put_page(p);
524 goto err_buf;
525 }
526
John Fastabend72979a62016-12-15 12:14:36 -0800527 memcpy(page_address(page) + page_off,
528 page_address(p) + off, buflen);
529 page_off += buflen;
Jason Wang56a86f82016-12-23 22:37:26 +0800530 put_page(p);
John Fastabend72979a62016-12-15 12:14:36 -0800531 }
532
John Fastabend2de2f7f2017-02-02 19:16:29 -0800533 /* Headroom does not contribute to packet length */
534 *len = page_off - VIRTIO_XDP_HEADROOM;
John Fastabend72979a62016-12-15 12:14:36 -0800535 return page;
536err_buf:
537 __free_pages(page, 0);
538 return NULL;
539}
540
Jason Wang4941d472017-07-19 16:54:48 +0800541static struct sk_buff *receive_small(struct net_device *dev,
542 struct virtnet_info *vi,
543 struct receive_queue *rq,
544 void *buf, void *ctx,
Jason Wang186b3c92017-09-19 17:42:43 +0800545 unsigned int len,
546 bool *xdp_xmit)
Jason Wang4941d472017-07-19 16:54:48 +0800547{
548 struct sk_buff *skb;
549 struct bpf_prog *xdp_prog;
550 unsigned int xdp_headroom = (unsigned long)ctx;
551 unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom;
552 unsigned int headroom = vi->hdr_len + header_offset;
553 unsigned int buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
554 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
555 struct page *page = virt_to_head_page(buf);
Jesper Dangaard Brouer11b7d8972018-02-20 14:32:15 +0100556 unsigned int delta = 0;
Jason Wang4941d472017-07-19 16:54:48 +0800557 struct page *xdp_page;
Jesper Dangaard Brouer11b7d8972018-02-20 14:32:15 +0100558 bool sent;
559 int err;
560
Jason Wang4941d472017-07-19 16:54:48 +0800561 len -= vi->hdr_len;
562
563 rcu_read_lock();
564 xdp_prog = rcu_dereference(rq->xdp_prog);
565 if (xdp_prog) {
566 struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset;
567 struct xdp_buff xdp;
568 void *orig_data;
569 u32 act;
570
Jesper Dangaard Brouer95dbe9e2018-02-20 14:32:10 +0100571 if (unlikely(hdr->hdr.gso_type))
Jason Wang4941d472017-07-19 16:54:48 +0800572 goto err_xdp;
573
574 if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) {
575 int offset = buf - page_address(page) + header_offset;
576 unsigned int tlen = len + vi->hdr_len;
577 u16 num_buf = 1;
578
579 xdp_headroom = virtnet_get_headroom(vi);
580 header_offset = VIRTNET_RX_PAD + xdp_headroom;
581 headroom = vi->hdr_len + header_offset;
582 buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
583 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
584 xdp_page = xdp_linearize_page(rq, &num_buf, page,
585 offset, header_offset,
586 &tlen);
587 if (!xdp_page)
588 goto err_xdp;
589
590 buf = page_address(xdp_page);
591 put_page(page);
592 page = xdp_page;
593 }
594
595 xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len;
596 xdp.data = xdp.data_hard_start + xdp_headroom;
Daniel Borkmannde8f3a82017-09-25 02:25:51 +0200597 xdp_set_data_meta_invalid(&xdp);
Jason Wang4941d472017-07-19 16:54:48 +0800598 xdp.data_end = xdp.data + len;
Jesper Dangaard Brouer754b8a22018-01-03 11:26:04 +0100599 xdp.rxq = &rq->xdp_rxq;
Jason Wang4941d472017-07-19 16:54:48 +0800600 orig_data = xdp.data;
601 act = bpf_prog_run_xdp(xdp_prog, &xdp);
602
603 switch (act) {
604 case XDP_PASS:
605 /* Recalculate length in case bpf program changed it */
606 delta = orig_data - xdp.data;
607 break;
608 case XDP_TX:
Jesper Dangaard Brouer11b7d8972018-02-20 14:32:15 +0100609 sent = __virtnet_xdp_xmit(vi, &xdp);
610 if (unlikely(!sent)) {
Jason Wang4941d472017-07-19 16:54:48 +0800611 trace_xdp_exception(vi->dev, xdp_prog, act);
Jesper Dangaard Brouer11b7d8972018-02-20 14:32:15 +0100612 goto err_xdp;
613 }
614 *xdp_xmit = true;
Jason Wang186b3c92017-09-19 17:42:43 +0800615 rcu_read_unlock();
616 goto xdp_xmit;
617 case XDP_REDIRECT:
618 err = xdp_do_redirect(dev, &xdp, xdp_prog);
Jesper Dangaard Brouer11b7d8972018-02-20 14:32:15 +0100619 if (err)
620 goto err_xdp;
621 *xdp_xmit = true;
Jason Wang4941d472017-07-19 16:54:48 +0800622 rcu_read_unlock();
623 goto xdp_xmit;
624 default:
625 bpf_warn_invalid_xdp_action(act);
626 case XDP_ABORTED:
627 trace_xdp_exception(vi->dev, xdp_prog, act);
628 case XDP_DROP:
629 goto err_xdp;
630 }
631 }
632 rcu_read_unlock();
633
634 skb = build_skb(buf, buflen);
635 if (!skb) {
636 put_page(page);
637 goto err;
638 }
639 skb_reserve(skb, headroom - delta);
640 skb_put(skb, len + delta);
641 if (!delta) {
642 buf += header_offset;
643 memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len);
644 } /* keep zeroed vnet hdr since packet was changed by bpf */
645
646err:
647 return skb;
648
649err_xdp:
650 rcu_read_unlock();
651 dev->stats.rx_dropped++;
652 put_page(page);
653xdp_xmit:
654 return NULL;
655}
656
657static struct sk_buff *receive_big(struct net_device *dev,
658 struct virtnet_info *vi,
659 struct receive_queue *rq,
660 void *buf,
661 unsigned int len)
662{
663 struct page *page = buf;
664 struct sk_buff *skb = page_to_skb(vi, rq, page, 0, len, PAGE_SIZE);
665
666 if (unlikely(!skb))
667 goto err;
668
669 return skb;
670
671err:
672 dev->stats.rx_dropped++;
673 give_pages(rq, page);
674 return NULL;
675}
676
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200677static struct sk_buff *receive_mergeable(struct net_device *dev,
Michael S. Tsirkinfdd819b2014-10-07 16:39:48 +0200678 struct virtnet_info *vi,
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200679 struct receive_queue *rq,
Michael S. Tsirkin680557c2017-03-06 21:29:47 +0200680 void *buf,
681 void *ctx,
Jason Wang186b3c92017-09-19 17:42:43 +0800682 unsigned int len,
683 bool *xdp_xmit)
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000684{
Michael S. Tsirkin012873d2014-10-24 16:55:57 +0300685 struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
686 u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200687 struct page *page = virt_to_head_page(buf);
688 int offset = buf - page_address(page);
John Fastabendf600b692016-12-15 12:13:24 -0800689 struct sk_buff *head_skb, *curr_skb;
690 struct bpf_prog *xdp_prog;
691 unsigned int truesize;
Jason Wang4941d472017-07-19 16:54:48 +0800692 unsigned int headroom = mergeable_ctx_to_headroom(ctx);
Jesper Dangaard Brouer11b7d8972018-02-20 14:32:15 +0100693 bool sent;
Jason Wang3cc81a92018-03-02 17:29:14 +0800694 int err;
Michael Daltonab7db912014-01-16 22:23:27 -0800695
John Fastabend56434a02016-12-15 12:14:13 -0800696 head_skb = NULL;
697
John Fastabendf600b692016-12-15 12:13:24 -0800698 rcu_read_lock();
699 xdp_prog = rcu_dereference(rq->xdp_prog);
700 if (xdp_prog) {
John Fastabend72979a62016-12-15 12:14:36 -0800701 struct page *xdp_page;
John Fastabend0354e4d2017-02-02 19:15:01 -0800702 struct xdp_buff xdp;
John Fastabend0354e4d2017-02-02 19:15:01 -0800703 void *data;
John Fastabendf600b692016-12-15 12:13:24 -0800704 u32 act;
705
Jason Wang3cc81a92018-03-02 17:29:14 +0800706 /* This happens when rx buffer size is underestimated
707 * or headroom is not enough because of the buffer
708 * was refilled before XDP is set. This should only
709 * happen for the first several packets, so we don't
710 * care much about its performance.
711 */
Jason Wang4941d472017-07-19 16:54:48 +0800712 if (unlikely(num_buf > 1 ||
713 headroom < virtnet_get_headroom(vi))) {
John Fastabend72979a62016-12-15 12:14:36 -0800714 /* linearize data for XDP */
Jason Wang56a86f82016-12-23 22:37:26 +0800715 xdp_page = xdp_linearize_page(rq, &num_buf,
Jason Wang4941d472017-07-19 16:54:48 +0800716 page, offset,
717 VIRTIO_XDP_HEADROOM,
718 &len);
John Fastabend72979a62016-12-15 12:14:36 -0800719 if (!xdp_page)
720 goto err_xdp;
John Fastabend2de2f7f2017-02-02 19:16:29 -0800721 offset = VIRTIO_XDP_HEADROOM;
John Fastabend72979a62016-12-15 12:14:36 -0800722 } else {
723 xdp_page = page;
John Fastabendf600b692016-12-15 12:13:24 -0800724 }
725
726 /* Transient failure which in theory could occur if
727 * in-flight packets from before XDP was enabled reach
728 * the receive path after XDP is loaded. In practice I
729 * was not able to create this condition.
730 */
Jason Wangb00f70b2016-12-23 22:37:28 +0800731 if (unlikely(hdr->hdr.gso_type))
John Fastabendf600b692016-12-15 12:13:24 -0800732 goto err_xdp;
733
John Fastabend2de2f7f2017-02-02 19:16:29 -0800734 /* Allow consuming headroom but reserve enough space to push
735 * the descriptor on if we get an XDP_TX return code.
736 */
John Fastabend0354e4d2017-02-02 19:15:01 -0800737 data = page_address(xdp_page) + offset;
John Fastabend2de2f7f2017-02-02 19:16:29 -0800738 xdp.data_hard_start = data - VIRTIO_XDP_HEADROOM + vi->hdr_len;
John Fastabend0354e4d2017-02-02 19:15:01 -0800739 xdp.data = data + vi->hdr_len;
Daniel Borkmannde8f3a82017-09-25 02:25:51 +0200740 xdp_set_data_meta_invalid(&xdp);
John Fastabend0354e4d2017-02-02 19:15:01 -0800741 xdp.data_end = xdp.data + (len - vi->hdr_len);
Jesper Dangaard Brouer754b8a22018-01-03 11:26:04 +0100742 xdp.rxq = &rq->xdp_rxq;
743
John Fastabend0354e4d2017-02-02 19:15:01 -0800744 act = bpf_prog_run_xdp(xdp_prog, &xdp);
745
John Fastabend56434a02016-12-15 12:14:13 -0800746 switch (act) {
747 case XDP_PASS:
John Fastabend2de2f7f2017-02-02 19:16:29 -0800748 /* recalculate offset to account for any header
749 * adjustments. Note other cases do not build an
750 * skb and avoid using offset
751 */
752 offset = xdp.data -
753 page_address(xdp_page) - vi->hdr_len;
754
Jason Wang1830f892016-12-23 22:37:27 +0800755 /* We can only create skb based on xdp_page. */
756 if (unlikely(xdp_page != page)) {
757 rcu_read_unlock();
758 put_page(page);
759 head_skb = page_to_skb(vi, rq, xdp_page,
John Fastabend2de2f7f2017-02-02 19:16:29 -0800760 offset, len, PAGE_SIZE);
Jason Wang1830f892016-12-23 22:37:27 +0800761 return head_skb;
762 }
John Fastabend56434a02016-12-15 12:14:13 -0800763 break;
764 case XDP_TX:
Jesper Dangaard Brouer11b7d8972018-02-20 14:32:15 +0100765 sent = __virtnet_xdp_xmit(vi, &xdp);
766 if (unlikely(!sent)) {
John Fastabend0354e4d2017-02-02 19:15:01 -0800767 trace_xdp_exception(vi->dev, xdp_prog, act);
Jesper Dangaard Brouer11b7d8972018-02-20 14:32:15 +0100768 if (unlikely(xdp_page != page))
769 put_page(xdp_page);
770 goto err_xdp;
771 }
772 *xdp_xmit = true;
John Fastabend72979a62016-12-15 12:14:36 -0800773 if (unlikely(xdp_page != page))
774 goto err_xdp;
John Fastabend56434a02016-12-15 12:14:13 -0800775 rcu_read_unlock();
776 goto xdp_xmit;
Jason Wang3cc81a92018-03-02 17:29:14 +0800777 case XDP_REDIRECT:
778 err = xdp_do_redirect(dev, &xdp, xdp_prog);
779 if (err) {
780 if (unlikely(xdp_page != page))
781 put_page(xdp_page);
782 goto err_xdp;
783 }
784 *xdp_xmit = true;
785 if (unlikely(xdp_page != page))
786 goto err_xdp;
787 rcu_read_unlock();
788 goto xdp_xmit;
John Fastabend56434a02016-12-15 12:14:13 -0800789 default:
John Fastabend0354e4d2017-02-02 19:15:01 -0800790 bpf_warn_invalid_xdp_action(act);
791 case XDP_ABORTED:
792 trace_xdp_exception(vi->dev, xdp_prog, act);
793 case XDP_DROP:
John Fastabend72979a62016-12-15 12:14:36 -0800794 if (unlikely(xdp_page != page))
795 __free_pages(xdp_page, 0);
John Fastabendf600b692016-12-15 12:13:24 -0800796 goto err_xdp;
John Fastabend56434a02016-12-15 12:14:13 -0800797 }
John Fastabendf600b692016-12-15 12:13:24 -0800798 }
799 rcu_read_unlock();
800
Jason Wang28b39bc2017-07-19 16:54:46 +0800801 truesize = mergeable_ctx_to_truesize(ctx);
802 if (unlikely(len > truesize)) {
Dan Carpenter56da5fd2017-04-06 12:04:47 +0300803 pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
Michael S. Tsirkin680557c2017-03-06 21:29:47 +0200804 dev->name, len, (unsigned long)ctx);
805 dev->stats.rx_length_errors++;
806 goto err_skb;
807 }
Jason Wang28b39bc2017-07-19 16:54:46 +0800808
John Fastabendf600b692016-12-15 12:13:24 -0800809 head_skb = page_to_skb(vi, rq, page, offset, len, truesize);
810 curr_skb = head_skb;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000811
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200812 if (unlikely(!curr_skb))
813 goto err_skb;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000814 while (--num_buf) {
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200815 int num_skb_frags;
816
Michael S. Tsirkin680557c2017-03-06 21:29:47 +0200817 buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx);
Yunjian Wang03e9f8a2017-12-04 14:02:19 +0800818 if (unlikely(!buf)) {
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200819 pr_debug("%s: rx error: %d buffers out of %d missing\n",
Michael S. Tsirkinfdd819b2014-10-07 16:39:48 +0200820 dev->name, num_buf,
Michael S. Tsirkin012873d2014-10-24 16:55:57 +0300821 virtio16_to_cpu(vi->vdev,
822 hdr->num_buffers));
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200823 dev->stats.rx_length_errors++;
824 goto err_buf;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000825 }
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200826
827 page = virt_to_head_page(buf);
Jason Wang28b39bc2017-07-19 16:54:46 +0800828
829 truesize = mergeable_ctx_to_truesize(ctx);
830 if (unlikely(len > truesize)) {
Dan Carpenter56da5fd2017-04-06 12:04:47 +0300831 pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
Michael S. Tsirkin680557c2017-03-06 21:29:47 +0200832 dev->name, len, (unsigned long)ctx);
833 dev->stats.rx_length_errors++;
834 goto err_skb;
835 }
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200836
837 num_skb_frags = skb_shinfo(curr_skb)->nr_frags;
Michael Dalton2613af02013-10-28 15:44:18 -0700838 if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) {
839 struct sk_buff *nskb = alloc_skb(0, GFP_ATOMIC);
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200840
841 if (unlikely(!nskb))
842 goto err_skb;
Michael Dalton2613af02013-10-28 15:44:18 -0700843 if (curr_skb == head_skb)
844 skb_shinfo(curr_skb)->frag_list = nskb;
845 else
846 curr_skb->next = nskb;
847 curr_skb = nskb;
848 head_skb->truesize += nskb->truesize;
849 num_skb_frags = 0;
850 }
851 if (curr_skb != head_skb) {
852 head_skb->data_len += len;
853 head_skb->len += len;
Michael Daltonfb518792014-01-16 22:23:26 -0800854 head_skb->truesize += truesize;
Michael Dalton2613af02013-10-28 15:44:18 -0700855 }
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200856 offset = buf - page_address(page);
Jason Wangba275242013-11-01 14:07:48 +0800857 if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
858 put_page(page);
859 skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
Michael Daltonfb518792014-01-16 22:23:26 -0800860 len, truesize);
Jason Wangba275242013-11-01 14:07:48 +0800861 } else {
862 skb_add_rx_frag(curr_skb, num_skb_frags, page,
Michael Daltonfb518792014-01-16 22:23:26 -0800863 offset, len, truesize);
Jason Wangba275242013-11-01 14:07:48 +0800864 }
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200865 }
866
Johannes Berg5377d7582015-08-19 09:48:40 +0200867 ewma_pkt_len_add(&rq->mrg_avg_pkt_len, head_skb->len);
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200868 return head_skb;
869
John Fastabendf600b692016-12-15 12:13:24 -0800870err_xdp:
871 rcu_read_unlock();
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200872err_skb:
873 put_page(page);
874 while (--num_buf) {
Michael S. Tsirkin680557c2017-03-06 21:29:47 +0200875 buf = virtqueue_get_buf(rq->vq, &len);
876 if (unlikely(!buf)) {
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200877 pr_debug("%s: rx error: %d buffers missing\n",
878 dev->name, num_buf);
879 dev->stats.rx_length_errors++;
880 break;
881 }
Michael S. Tsirkin680557c2017-03-06 21:29:47 +0200882 page = virt_to_head_page(buf);
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200883 put_page(page);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000884 }
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200885err_buf:
886 dev->stats.rx_dropped++;
887 dev_kfree_skb(head_skb);
John Fastabend56434a02016-12-15 12:14:13 -0800888xdp_xmit:
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200889 return NULL;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000890}
891
Jason Wang61845d22017-02-17 11:33:09 +0800892static int receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
Jason Wang186b3c92017-09-19 17:42:43 +0800893 void *buf, unsigned int len, void **ctx, bool *xdp_xmit)
Rusty Russell296f96f2007-10-22 11:03:37 +1000894{
Jason Wange9d74172012-12-07 07:04:55 +0000895 struct net_device *dev = vi->dev;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000896 struct sk_buff *skb;
Michael S. Tsirkin012873d2014-10-24 16:55:57 +0300897 struct virtio_net_hdr_mrg_rxbuf *hdr;
Jason Wang61845d22017-02-17 11:33:09 +0800898 int ret;
Rusty Russell296f96f2007-10-22 11:03:37 +1000899
Michael S. Tsirkinbcff3162014-10-24 00:22:11 +0300900 if (unlikely(len < vi->hdr_len + ETH_HLEN)) {
Rusty Russell296f96f2007-10-22 11:03:37 +1000901 pr_debug("%s: short packet %i\n", dev->name, len);
902 dev->stats.rx_length_errors++;
Michael Daltonab7db912014-01-16 22:23:27 -0800903 if (vi->mergeable_rx_bufs) {
Michael S. Tsirkin680557c2017-03-06 21:29:47 +0200904 put_page(virt_to_head_page(buf));
Michael Daltonab7db912014-01-16 22:23:27 -0800905 } else if (vi->big_packets) {
Michael Dalton98bfd232013-12-05 13:14:05 -0800906 give_pages(rq, buf);
Michael Daltonab7db912014-01-16 22:23:27 -0800907 } else {
Jason Wangf6b10202017-02-21 16:46:28 +0800908 put_page(virt_to_head_page(buf));
Michael Daltonab7db912014-01-16 22:23:27 -0800909 }
Jason Wang61845d22017-02-17 11:33:09 +0800910 return 0;
Rusty Russell296f96f2007-10-22 11:03:37 +1000911 }
Rusty Russell296f96f2007-10-22 11:03:37 +1000912
Michael S. Tsirkinf1211592013-11-28 13:30:59 +0200913 if (vi->mergeable_rx_bufs)
Jason Wang186b3c92017-09-19 17:42:43 +0800914 skb = receive_mergeable(dev, vi, rq, buf, ctx, len, xdp_xmit);
Michael S. Tsirkinf1211592013-11-28 13:30:59 +0200915 else if (vi->big_packets)
Michael S. Tsirkin946fa562014-10-24 00:12:10 +0300916 skb = receive_big(dev, vi, rq, buf, len);
Michael S. Tsirkinf1211592013-11-28 13:30:59 +0200917 else
Jason Wang186b3c92017-09-19 17:42:43 +0800918 skb = receive_small(dev, vi, rq, buf, ctx, len, xdp_xmit);
Michael S. Tsirkinf1211592013-11-28 13:30:59 +0200919
920 if (unlikely(!skb))
Jason Wang61845d22017-02-17 11:33:09 +0800921 return 0;
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -0800922
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000923 hdr = skb_vnet_hdr(skb);
stephen hemminger3fa2a1d2011-06-15 06:36:29 +0000924
Jason Wang61845d22017-02-17 11:33:09 +0800925 ret = skb->len;
Rusty Russell296f96f2007-10-22 11:03:37 +1000926
Mike Rapoporte858fae2016-06-08 16:09:21 +0300927 if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID)
Jason Wang10a8d942011-06-10 00:56:17 +0000928 skb->ip_summed = CHECKSUM_UNNECESSARY;
Rusty Russell296f96f2007-10-22 11:03:37 +1000929
Mike Rapoporte858fae2016-06-08 16:09:21 +0300930 if (virtio_net_hdr_to_skb(skb, &hdr->hdr,
931 virtio_is_little_endian(vi->vdev))) {
932 net_warn_ratelimited("%s: bad gso: type: %u, size: %u\n",
933 dev->name, hdr->hdr.gso_type,
934 hdr->hdr.gso_size);
935 goto frame_err;
Rusty Russell296f96f2007-10-22 11:03:37 +1000936 }
937
Mike Rapoportd1dc06d2016-06-14 08:29:38 +0300938 skb->protocol = eth_type_trans(skb, dev);
939 pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
940 ntohs(skb->protocol), skb->len, skb->pkt_type);
941
Eric Dumazet0fbd0502015-07-31 18:25:17 +0200942 napi_gro_receive(&rq->napi, skb);
Jason Wang61845d22017-02-17 11:33:09 +0800943 return ret;
Rusty Russell296f96f2007-10-22 11:03:37 +1000944
945frame_err:
946 dev->stats.rx_frame_errors++;
Rusty Russell296f96f2007-10-22 11:03:37 +1000947 dev_kfree_skb(skb);
Jason Wang61845d22017-02-17 11:33:09 +0800948 return 0;
Rusty Russell296f96f2007-10-22 11:03:37 +1000949}
950
Jason Wang192f68c2017-07-19 16:54:47 +0800951/* Unlike mergeable buffers, all buffers are allocated to the
952 * same size, except for the headroom. For this reason we do
953 * not need to use mergeable_len_to_ctx here - it is enough
954 * to store the headroom as the context ignoring the truesize.
955 */
Michael S. Tsirkin946fa562014-10-24 00:12:10 +0300956static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,
957 gfp_t gfp)
Rusty Russell296f96f2007-10-22 11:03:37 +1000958{
Jason Wangf6b10202017-02-21 16:46:28 +0800959 struct page_frag *alloc_frag = &rq->alloc_frag;
960 char *buf;
John Fastabend2de2f7f2017-02-02 19:16:29 -0800961 unsigned int xdp_headroom = virtnet_get_headroom(vi);
Jason Wang192f68c2017-07-19 16:54:47 +0800962 void *ctx = (void *)(unsigned long)xdp_headroom;
Jason Wangf6b10202017-02-21 16:46:28 +0800963 int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000964 int err;
Rusty Russell296f96f2007-10-22 11:03:37 +1000965
Jason Wangf6b10202017-02-21 16:46:28 +0800966 len = SKB_DATA_ALIGN(len) +
967 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
968 if (unlikely(!skb_page_frag_refill(len, alloc_frag, gfp)))
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000969 return -ENOMEM;
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -0800970
Jason Wangf6b10202017-02-21 16:46:28 +0800971 buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
972 get_page(alloc_frag->page);
973 alloc_frag->offset += len;
974 sg_init_one(rq->sg, buf + VIRTNET_RX_PAD + xdp_headroom,
975 vi->hdr_len + GOOD_PACKET_LEN);
Jason Wang192f68c2017-07-19 16:54:47 +0800976 err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000977 if (err < 0)
Jason Wangf6b10202017-02-21 16:46:28 +0800978 put_page(virt_to_head_page(buf));
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000979 return err;
980}
981
Michael S. Tsirkin012873d2014-10-24 16:55:57 +0300982static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq,
983 gfp_t gfp)
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000984{
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000985 struct page *first, *list = NULL;
986 char *p;
987 int i, err, offset;
988
Rusty Russella5835442014-09-11 10:17:36 +0930989 sg_init_table(rq->sg, MAX_SKB_FRAGS + 2);
990
Jason Wange9d74172012-12-07 07:04:55 +0000991 /* page in rq->sg[MAX_SKB_FRAGS + 1] is list tail */
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000992 for (i = MAX_SKB_FRAGS + 1; i > 1; --i) {
Jason Wange9d74172012-12-07 07:04:55 +0000993 first = get_a_page(rq, gfp);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000994 if (!first) {
995 if (list)
Jason Wange9d74172012-12-07 07:04:55 +0000996 give_pages(rq, list);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000997 return -ENOMEM;
Rusty Russell3161e452009-08-26 12:22:32 -0700998 }
Jason Wange9d74172012-12-07 07:04:55 +0000999 sg_set_buf(&rq->sg[i], page_address(first), PAGE_SIZE);
Rusty Russell296f96f2007-10-22 11:03:37 +10001000
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001001 /* chain new page in list head to match sg */
1002 first->private = (unsigned long)list;
1003 list = first;
1004 }
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -08001005
Jason Wange9d74172012-12-07 07:04:55 +00001006 first = get_a_page(rq, gfp);
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001007 if (!first) {
Jason Wange9d74172012-12-07 07:04:55 +00001008 give_pages(rq, list);
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001009 return -ENOMEM;
1010 }
1011 p = page_address(first);
Herbert Xu97402b92008-04-18 11:24:27 +08001012
Jason Wange9d74172012-12-07 07:04:55 +00001013 /* rq->sg[0], rq->sg[1] share the same page */
Michael S. Tsirkin012873d2014-10-24 16:55:57 +03001014 /* a separated rq->sg[0] for header - required in case !any_header_sg */
1015 sg_set_buf(&rq->sg[0], p, vi->hdr_len);
Herbert Xu97402b92008-04-18 11:24:27 +08001016
Jason Wange9d74172012-12-07 07:04:55 +00001017 /* rq->sg[1] for data packet, from offset */
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001018 offset = sizeof(struct padded_vnet_hdr);
Jason Wange9d74172012-12-07 07:04:55 +00001019 sg_set_buf(&rq->sg[1], p + offset, PAGE_SIZE - offset);
Herbert Xu97402b92008-04-18 11:24:27 +08001020
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001021 /* chain first in list head */
1022 first->private = (unsigned long)list;
Rusty Russell9dc7b9e2013-03-20 15:44:28 +10301023 err = virtqueue_add_inbuf(rq->vq, rq->sg, MAX_SKB_FRAGS + 2,
1024 first, gfp);
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001025 if (err < 0)
Jason Wange9d74172012-12-07 07:04:55 +00001026 give_pages(rq, first);
Herbert Xu97402b92008-04-18 11:24:27 +08001027
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001028 return err;
1029}
Herbert Xu97402b92008-04-18 11:24:27 +08001030
Michael S. Tsirkind85b758f72017-03-09 02:21:21 +02001031static unsigned int get_mergeable_buf_len(struct receive_queue *rq,
Jason Wang3cc81a92018-03-02 17:29:14 +08001032 struct ewma_pkt_len *avg_pkt_len,
1033 unsigned int room)
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001034{
Michael Daltonab7db912014-01-16 22:23:27 -08001035 const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
Michael Daltonfbf28d72014-01-16 22:23:30 -08001036 unsigned int len;
1037
Jason Wang3cc81a92018-03-02 17:29:14 +08001038 if (room)
1039 return PAGE_SIZE - room;
1040
1041 len = hdr_len + clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len),
Michael S. Tsirkinf0c31922017-06-02 17:54:33 +03001042 rq->min_buf_len, PAGE_SIZE - hdr_len);
Jason Wang3cc81a92018-03-02 17:29:14 +08001043
Michael S. Tsirkine377fcc2017-03-06 22:21:35 +02001044 return ALIGN(len, L1_CACHE_BYTES);
Michael Daltonfbf28d72014-01-16 22:23:30 -08001045}
1046
John Fastabend2de2f7f2017-02-02 19:16:29 -08001047static int add_recvbuf_mergeable(struct virtnet_info *vi,
1048 struct receive_queue *rq, gfp_t gfp)
Michael Daltonfbf28d72014-01-16 22:23:30 -08001049{
Michael Daltonfb518792014-01-16 22:23:26 -08001050 struct page_frag *alloc_frag = &rq->alloc_frag;
John Fastabend2de2f7f2017-02-02 19:16:29 -08001051 unsigned int headroom = virtnet_get_headroom(vi);
Jason Wang3cc81a92018-03-02 17:29:14 +08001052 unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
1053 unsigned int room = SKB_DATA_ALIGN(headroom + tailroom);
Michael Daltonfb518792014-01-16 22:23:26 -08001054 char *buf;
Michael S. Tsirkin680557c2017-03-06 21:29:47 +02001055 void *ctx;
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001056 int err;
Michael Daltonfb518792014-01-16 22:23:26 -08001057 unsigned int len, hole;
Rusty Russell296f96f2007-10-22 11:03:37 +10001058
Jason Wang3cc81a92018-03-02 17:29:14 +08001059 /* Extra tailroom is needed to satisfy XDP's assumption. This
1060 * means rx frags coalescing won't work, but consider we've
1061 * disabled GSO for XDP, it won't be a big issue.
1062 */
1063 len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len, room);
1064 if (unlikely(!skb_page_frag_refill(len + room, alloc_frag, gfp)))
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001065 return -ENOMEM;
Michael Daltonab7db912014-01-16 22:23:27 -08001066
Michael Daltonfb518792014-01-16 22:23:26 -08001067 buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
John Fastabend2de2f7f2017-02-02 19:16:29 -08001068 buf += headroom; /* advance address leaving hole at front of pkt */
Michael Daltonfb518792014-01-16 22:23:26 -08001069 get_page(alloc_frag->page);
Jason Wang3cc81a92018-03-02 17:29:14 +08001070 alloc_frag->offset += len + room;
Michael Daltonfb518792014-01-16 22:23:26 -08001071 hole = alloc_frag->size - alloc_frag->offset;
Jason Wang3cc81a92018-03-02 17:29:14 +08001072 if (hole < len + room) {
Michael Daltonab7db912014-01-16 22:23:27 -08001073 /* To avoid internal fragmentation, if there is very likely not
1074 * enough space for another buffer, add the remaining space to
Michael S. Tsirkin1daa8792017-07-31 21:49:49 +03001075 * the current buffer.
Michael Daltonab7db912014-01-16 22:23:27 -08001076 */
Michael Daltonfb518792014-01-16 22:23:26 -08001077 len += hole;
1078 alloc_frag->offset += hole;
1079 }
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001080
Michael Daltonfb518792014-01-16 22:23:26 -08001081 sg_init_one(rq->sg, buf, len);
David S. Miller29fda252017-08-01 10:07:50 -07001082 ctx = mergeable_len_to_ctx(len, headroom);
Michael S. Tsirkin680557c2017-03-06 21:29:47 +02001083 err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001084 if (err < 0)
Michael Dalton2613af02013-10-28 15:44:18 -07001085 put_page(virt_to_head_page(buf));
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001086
1087 return err;
Rusty Russell296f96f2007-10-22 11:03:37 +10001088}
1089
Rusty Russellb2baed62011-12-29 00:42:38 +00001090/*
1091 * Returns false if we couldn't fill entirely (OOM).
1092 *
1093 * Normally run in the receive path, but can also be run from ndo_open
1094 * before we're receiving packets, or from refill_work which is
1095 * careful to disable receiving (using napi_disable).
1096 */
Michael S. Tsirkin946fa562014-10-24 00:12:10 +03001097static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq,
1098 gfp_t gfp)
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -08001099{
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -08001100 int err;
Michael S. Tsirkin1788f4952010-07-02 16:32:55 +00001101 bool oom;
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -08001102
Amit Shah0aea51c2009-08-26 14:58:28 +05301103 do {
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001104 if (vi->mergeable_rx_bufs)
John Fastabend2de2f7f2017-02-02 19:16:29 -08001105 err = add_recvbuf_mergeable(vi, rq, gfp);
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001106 else if (vi->big_packets)
Michael S. Tsirkin012873d2014-10-24 16:55:57 +03001107 err = add_recvbuf_big(vi, rq, gfp);
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001108 else
Michael S. Tsirkin946fa562014-10-24 00:12:10 +03001109 err = add_recvbuf_small(vi, rq, gfp);
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -08001110
Michael S. Tsirkin1788f4952010-07-02 16:32:55 +00001111 oom = err == -ENOMEM;
Rusty Russell9ed4cb02012-10-16 23:56:14 +10301112 if (err)
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -08001113 break;
Linus Torvaldsb7dfde92012-12-20 08:37:04 -08001114 } while (rq->vq->num_free);
Jason Wang681daee22014-03-26 13:03:00 +08001115 virtqueue_kick(rq->vq);
Rusty Russell3161e452009-08-26 12:22:32 -07001116 return !oom;
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -08001117}
1118
Rusty Russell18445c42008-02-04 23:49:57 -05001119static void skb_recv_done(struct virtqueue *rvq)
Rusty Russell296f96f2007-10-22 11:03:37 +10001120{
1121 struct virtnet_info *vi = rvq->vdev->priv;
Jason Wang986a4f42012-12-07 07:04:56 +00001122 struct receive_queue *rq = &vi->rq[vq2rxq(rvq)];
Jason Wange9d74172012-12-07 07:04:55 +00001123
Willem de Bruijne4e84522017-04-24 13:49:26 -04001124 virtqueue_napi_schedule(&rq->napi, rvq);
Rusty Russell296f96f2007-10-22 11:03:37 +10001125}
1126
Willem de Bruijne4e84522017-04-24 13:49:26 -04001127static void virtnet_napi_enable(struct virtqueue *vq, struct napi_struct *napi)
Bruce Rogers3e9d08e2011-02-10 11:03:31 -08001128{
Willem de Bruijne4e84522017-04-24 13:49:26 -04001129 napi_enable(napi);
Bruce Rogers3e9d08e2011-02-10 11:03:31 -08001130
1131 /* If all buffers were filled by other side before we napi_enabled, we
Willem de Bruijne4e84522017-04-24 13:49:26 -04001132 * won't get another interrupt, so process any outstanding packets now.
1133 * Call local_bh_enable after to trigger softIRQ processing.
1134 */
1135 local_bh_disable();
1136 virtqueue_napi_schedule(napi, vq);
1137 local_bh_enable();
Bruce Rogers3e9d08e2011-02-10 11:03:31 -08001138}
1139
Willem de Bruijnb92f1e62017-04-24 13:49:27 -04001140static void virtnet_napi_tx_enable(struct virtnet_info *vi,
1141 struct virtqueue *vq,
1142 struct napi_struct *napi)
1143{
1144 if (!napi->weight)
1145 return;
1146
1147 /* Tx napi touches cachelines on the cpu handling tx interrupts. Only
1148 * enable the feature if this is likely affine with the transmit path.
1149 */
1150 if (!vi->affinity_hint_set) {
1151 napi->weight = 0;
1152 return;
1153 }
1154
1155 return virtnet_napi_enable(vq, napi);
1156}
1157
Willem de Bruijn78a57b42017-04-25 15:59:17 -04001158static void virtnet_napi_tx_disable(struct napi_struct *napi)
1159{
1160 if (napi->weight)
1161 napi_disable(napi);
1162}
1163
Rusty Russell3161e452009-08-26 12:22:32 -07001164static void refill_work(struct work_struct *work)
1165{
Jason Wange9d74172012-12-07 07:04:55 +00001166 struct virtnet_info *vi =
1167 container_of(work, struct virtnet_info, refill.work);
Rusty Russell3161e452009-08-26 12:22:32 -07001168 bool still_empty;
Jason Wang986a4f42012-12-07 07:04:56 +00001169 int i;
Rusty Russell3161e452009-08-26 12:22:32 -07001170
Sasha Levin55257d72013-04-29 12:00:08 +09301171 for (i = 0; i < vi->curr_queue_pairs; i++) {
Jason Wang986a4f42012-12-07 07:04:56 +00001172 struct receive_queue *rq = &vi->rq[i];
Rusty Russell3161e452009-08-26 12:22:32 -07001173
Jason Wang986a4f42012-12-07 07:04:56 +00001174 napi_disable(&rq->napi);
Michael S. Tsirkin946fa562014-10-24 00:12:10 +03001175 still_empty = !try_fill_recv(vi, rq, GFP_KERNEL);
Willem de Bruijne4e84522017-04-24 13:49:26 -04001176 virtnet_napi_enable(rq->vq, &rq->napi);
Jason Wang986a4f42012-12-07 07:04:56 +00001177
1178 /* In theory, this can happen: if we don't get any buffers in
1179 * we will *never* try to fill again.
1180 */
1181 if (still_empty)
1182 schedule_delayed_work(&vi->refill, HZ/2);
1183 }
Rusty Russell3161e452009-08-26 12:22:32 -07001184}
1185
Jason Wang186b3c92017-09-19 17:42:43 +08001186static int virtnet_receive(struct receive_queue *rq, int budget, bool *xdp_xmit)
Rusty Russell296f96f2007-10-22 11:03:37 +10001187{
Jason Wange9d74172012-12-07 07:04:55 +00001188 struct virtnet_info *vi = rq->vq->vdev->priv;
Jason Wang61845d22017-02-17 11:33:09 +08001189 unsigned int len, received = 0, bytes = 0;
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001190 void *buf;
Rusty Russell296f96f2007-10-22 11:03:37 +10001191
Jason Wang192f68c2017-07-19 16:54:47 +08001192 if (!vi->big_packets || vi->mergeable_rx_bufs) {
Michael S. Tsirkin680557c2017-03-06 21:29:47 +02001193 void *ctx;
1194
1195 while (received < budget &&
1196 (buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx))) {
Jason Wang186b3c92017-09-19 17:42:43 +08001197 bytes += receive_buf(vi, rq, buf, len, ctx, xdp_xmit);
Michael S. Tsirkin680557c2017-03-06 21:29:47 +02001198 received++;
1199 }
1200 } else {
1201 while (received < budget &&
1202 (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) {
Jason Wang186b3c92017-09-19 17:42:43 +08001203 bytes += receive_buf(vi, rq, buf, len, NULL, xdp_xmit);
Michael S. Tsirkin680557c2017-03-06 21:29:47 +02001204 received++;
1205 }
Rusty Russell296f96f2007-10-22 11:03:37 +10001206 }
1207
Jason Wangbe121f42014-01-16 14:45:24 +08001208 if (rq->vq->num_free > virtqueue_get_vring_size(rq->vq) / 2) {
Michael S. Tsirkin946fa562014-10-24 00:12:10 +03001209 if (!try_fill_recv(vi, rq, GFP_ATOMIC))
Tejun Heo3b07e9c2012-08-20 14:51:24 -07001210 schedule_delayed_work(&vi->refill, 0);
Rusty Russell3161e452009-08-26 12:22:32 -07001211 }
Rusty Russell296f96f2007-10-22 11:03:37 +10001212
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09001213 u64_stats_update_begin(&rq->stats.syncp);
1214 rq->stats.bytes += bytes;
1215 rq->stats.packets += received;
1216 u64_stats_update_end(&rq->stats.syncp);
Jason Wang61845d22017-02-17 11:33:09 +08001217
Jason Wang2ffa7592014-07-23 16:33:54 +08001218 return received;
1219}
1220
Willem de Bruijnea7735d2017-04-24 13:49:28 -04001221static void free_old_xmit_skbs(struct send_queue *sq)
1222{
1223 struct sk_buff *skb;
1224 unsigned int len;
Willem de Bruijnea7735d2017-04-24 13:49:28 -04001225 unsigned int packets = 0;
1226 unsigned int bytes = 0;
1227
1228 while ((skb = virtqueue_get_buf(sq->vq, &len)) != NULL) {
1229 pr_debug("Sent skb %p\n", skb);
1230
1231 bytes += skb->len;
1232 packets++;
1233
Eric Dumazetdadc0732017-08-24 09:02:49 -07001234 dev_consume_skb_any(skb);
Willem de Bruijnea7735d2017-04-24 13:49:28 -04001235 }
1236
1237 /* Avoid overhead when no packets have been processed
1238 * happens when called speculatively from start_xmit.
1239 */
1240 if (!packets)
1241 return;
1242
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09001243 u64_stats_update_begin(&sq->stats.syncp);
1244 sq->stats.bytes += bytes;
1245 sq->stats.packets += packets;
1246 u64_stats_update_end(&sq->stats.syncp);
Willem de Bruijnea7735d2017-04-24 13:49:28 -04001247}
1248
Willem de Bruijn7b0411e2017-04-24 13:49:29 -04001249static void virtnet_poll_cleantx(struct receive_queue *rq)
1250{
1251 struct virtnet_info *vi = rq->vq->vdev->priv;
1252 unsigned int index = vq2rxq(rq->vq);
1253 struct send_queue *sq = &vi->sq[index];
1254 struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index);
1255
1256 if (!sq->napi.weight)
1257 return;
1258
1259 if (__netif_tx_trylock(txq)) {
1260 free_old_xmit_skbs(sq);
1261 __netif_tx_unlock(txq);
1262 }
1263
1264 if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
1265 netif_tx_wake_queue(txq);
1266}
1267
Jason Wang2ffa7592014-07-23 16:33:54 +08001268static int virtnet_poll(struct napi_struct *napi, int budget)
1269{
1270 struct receive_queue *rq =
1271 container_of(napi, struct receive_queue, napi);
Willem de Bruijne4e84522017-04-24 13:49:26 -04001272 unsigned int received;
Jason Wang186b3c92017-09-19 17:42:43 +08001273 bool xdp_xmit = false;
Jason Wang2ffa7592014-07-23 16:33:54 +08001274
Willem de Bruijn7b0411e2017-04-24 13:49:29 -04001275 virtnet_poll_cleantx(rq);
1276
Jason Wang186b3c92017-09-19 17:42:43 +08001277 received = virtnet_receive(rq, budget, &xdp_xmit);
Jason Wang2ffa7592014-07-23 16:33:54 +08001278
Rusty Russell8329d982007-11-19 11:20:43 -05001279 /* Out of packets? */
Willem de Bruijne4e84522017-04-24 13:49:26 -04001280 if (received < budget)
1281 virtqueue_napi_complete(napi, rq->vq, received);
Rusty Russell296f96f2007-10-22 11:03:37 +10001282
Jason Wang186b3c92017-09-19 17:42:43 +08001283 if (xdp_xmit)
1284 xdp_do_flush_map();
1285
Rusty Russell296f96f2007-10-22 11:03:37 +10001286 return received;
1287}
1288
Jason Wang986a4f42012-12-07 07:04:56 +00001289static int virtnet_open(struct net_device *dev)
1290{
1291 struct virtnet_info *vi = netdev_priv(dev);
Jesper Dangaard Brouer754b8a22018-01-03 11:26:04 +01001292 int i, err;
Jason Wang986a4f42012-12-07 07:04:56 +00001293
Jason Wange4166622013-05-21 20:03:58 +00001294 for (i = 0; i < vi->max_queue_pairs; i++) {
1295 if (i < vi->curr_queue_pairs)
1296 /* Make sure we have some buffers: if oom use wq. */
Michael S. Tsirkin946fa562014-10-24 00:12:10 +03001297 if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))
Jason Wange4166622013-05-21 20:03:58 +00001298 schedule_delayed_work(&vi->refill, 0);
Jesper Dangaard Brouer754b8a22018-01-03 11:26:04 +01001299
1300 err = xdp_rxq_info_reg(&vi->rq[i].xdp_rxq, dev, i);
1301 if (err < 0)
1302 return err;
1303
Willem de Bruijne4e84522017-04-24 13:49:26 -04001304 virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
Willem de Bruijnb92f1e62017-04-24 13:49:27 -04001305 virtnet_napi_tx_enable(vi, vi->sq[i].vq, &vi->sq[i].napi);
Jason Wang986a4f42012-12-07 07:04:56 +00001306 }
1307
1308 return 0;
1309}
1310
Willem de Bruijnb92f1e62017-04-24 13:49:27 -04001311static int virtnet_poll_tx(struct napi_struct *napi, int budget)
1312{
1313 struct send_queue *sq = container_of(napi, struct send_queue, napi);
1314 struct virtnet_info *vi = sq->vq->vdev->priv;
1315 struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, vq2txq(sq->vq));
1316
1317 __netif_tx_lock(txq, raw_smp_processor_id());
1318 free_old_xmit_skbs(sq);
1319 __netif_tx_unlock(txq);
1320
1321 virtqueue_napi_complete(napi, sq->vq, 0);
1322
1323 if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
1324 netif_tx_wake_queue(txq);
1325
1326 return 0;
1327}
1328
Jason Wange9d74172012-12-07 07:04:55 +00001329static int xmit_skb(struct send_queue *sq, struct sk_buff *skb)
Rusty Russell296f96f2007-10-22 11:03:37 +10001330{
Michael S. Tsirkin012873d2014-10-24 16:55:57 +03001331 struct virtio_net_hdr_mrg_rxbuf *hdr;
Rusty Russell296f96f2007-10-22 11:03:37 +10001332 const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;
Jason Wange9d74172012-12-07 07:04:55 +00001333 struct virtnet_info *vi = sq->vq->vdev->priv;
Jason A. Donenfelde2fcad52017-06-04 04:16:26 +02001334 int num_sg;
Michael S. Tsirkin012873d2014-10-24 16:55:57 +03001335 unsigned hdr_len = vi->hdr_len;
Michael S. Tsirkine7428e92013-07-25 10:20:23 +09301336 bool can_push;
Rusty Russell296f96f2007-10-22 11:03:37 +10001337
Johannes Berge1749612008-10-27 15:59:26 -07001338 pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
Michael S. Tsirkine7428e92013-07-25 10:20:23 +09301339
1340 can_push = vi->any_header_sg &&
1341 !((unsigned long)skb->data & (__alignof__(*hdr) - 1)) &&
1342 !skb_header_cloned(skb) && skb_headroom(skb) >= hdr_len;
1343 /* Even if we can, don't push here yet as this would skew
1344 * csum_start offset below. */
1345 if (can_push)
Michael S. Tsirkin012873d2014-10-24 16:55:57 +03001346 hdr = (struct virtio_net_hdr_mrg_rxbuf *)(skb->data - hdr_len);
Michael S. Tsirkine7428e92013-07-25 10:20:23 +09301347 else
1348 hdr = skb_vnet_hdr(skb);
Rusty Russell296f96f2007-10-22 11:03:37 +10001349
Mike Rapoporte858fae2016-06-08 16:09:21 +03001350 if (virtio_net_hdr_from_skb(skb, &hdr->hdr,
Jason Wang6391a442017-01-20 14:32:42 +08001351 virtio_is_little_endian(vi->vdev), false))
Mike Rapoporte858fae2016-06-08 16:09:21 +03001352 BUG();
Rusty Russell296f96f2007-10-22 11:03:37 +10001353
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -08001354 if (vi->mergeable_rx_bufs)
Michael S. Tsirkin012873d2014-10-24 16:55:57 +03001355 hdr->num_buffers = 0;
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -08001356
Jason Wang547c8902015-08-27 14:53:06 +08001357 sg_init_table(sq->sg, skb_shinfo(skb)->nr_frags + (can_push ? 1 : 2));
Michael S. Tsirkine7428e92013-07-25 10:20:23 +09301358 if (can_push) {
1359 __skb_push(skb, hdr_len);
1360 num_sg = skb_to_sgvec(skb, sq->sg, 0, skb->len);
Jason A. Donenfelde2fcad52017-06-04 04:16:26 +02001361 if (unlikely(num_sg < 0))
1362 return num_sg;
Michael S. Tsirkine7428e92013-07-25 10:20:23 +09301363 /* Pull header back to avoid skew in tx bytes calculations. */
1364 __skb_pull(skb, hdr_len);
1365 } else {
1366 sg_set_buf(sq->sg, hdr, hdr_len);
Jason A. Donenfelde2fcad52017-06-04 04:16:26 +02001367 num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len);
1368 if (unlikely(num_sg < 0))
1369 return num_sg;
1370 num_sg++;
Michael S. Tsirkine7428e92013-07-25 10:20:23 +09301371 }
Rusty Russell9dc7b9e2013-03-20 15:44:28 +10301372 return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC);
Rusty Russell11a3a152008-05-26 17:48:13 +10001373}
1374
Stephen Hemminger424efe92009-08-31 19:50:51 +00001375static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
Rusty Russell99ffc692008-05-02 21:50:46 -05001376{
1377 struct virtnet_info *vi = netdev_priv(dev);
Jason Wang986a4f42012-12-07 07:04:56 +00001378 int qnum = skb_get_queue_mapping(skb);
1379 struct send_queue *sq = &vi->sq[qnum];
Rusty Russell9ed4cb02012-10-16 23:56:14 +10301380 int err;
Michael S. Tsirkin4b7fd2e62014-10-15 16:23:28 +03001381 struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum);
1382 bool kick = !skb->xmit_more;
Willem de Bruijnb92f1e62017-04-24 13:49:27 -04001383 bool use_napi = sq->napi.weight;
Rusty Russell2cb9c6b2008-02-04 23:50:07 -05001384
Rusty Russell2cb9c6b2008-02-04 23:50:07 -05001385 /* Free up any pending old buffers before queueing new ones. */
Jason Wange9d74172012-12-07 07:04:55 +00001386 free_old_xmit_skbs(sq);
Rusty Russell2cb9c6b2008-02-04 23:50:07 -05001387
Willem de Bruijnbdb12e02017-04-24 13:49:30 -04001388 if (use_napi && kick)
1389 virtqueue_enable_cb_delayed(sq->vq);
1390
Jacob Keller074c3582014-06-25 02:37:13 +00001391 /* timestamp packet in software */
1392 skb_tx_timestamp(skb);
1393
Michael S. Tsirkin03f191b2009-10-28 04:03:38 -07001394 /* Try to transmit */
Linus Torvaldsb7dfde92012-12-20 08:37:04 -08001395 err = xmit_skb(sq, skb);
Rusty Russell48925e32009-09-24 09:59:20 -06001396
Rusty Russell9ed4cb02012-10-16 23:56:14 +10301397 /* This should not happen! */
Jason Wang681daee22014-03-26 13:03:00 +08001398 if (unlikely(err)) {
Rusty Russell9ed4cb02012-10-16 23:56:14 +10301399 dev->stats.tx_fifo_errors++;
1400 if (net_ratelimit())
1401 dev_warn(&dev->dev,
Linus Torvaldsb7dfde92012-12-20 08:37:04 -08001402 "Unexpected TXQ (%d) queue failure: %d\n", qnum, err);
Rusty Russell58eba97d2010-07-02 16:34:01 +00001403 dev->stats.tx_dropped++;
Eric W. Biederman85e94522014-03-15 18:43:33 -07001404 dev_kfree_skb_any(skb);
Rusty Russell58eba97d2010-07-02 16:34:01 +00001405 return NETDEV_TX_OK;
Rusty Russell99ffc692008-05-02 21:50:46 -05001406 }
Michael S. Tsirkin03f191b2009-10-28 04:03:38 -07001407
Rusty Russell48925e32009-09-24 09:59:20 -06001408 /* Don't wait up for transmitted skbs to be freed. */
Willem de Bruijnb92f1e62017-04-24 13:49:27 -04001409 if (!use_napi) {
1410 skb_orphan(skb);
1411 nf_reset(skb);
1412 }
Rusty Russell99ffc692008-05-02 21:50:46 -05001413
Michael S. Tsirkin60302ff2015-04-02 13:05:47 +02001414 /* If running out of space, stop queue to avoid getting packets that we
1415 * are then unable to transmit.
1416 * An alternative would be to force queuing layer to requeue the skb by
1417 * returning NETDEV_TX_BUSY. However, NETDEV_TX_BUSY should not be
1418 * returned in a normal path of operation: it means that driver is not
1419 * maintaining the TX queue stop/start state properly, and causes
1420 * the stack to do a non-trivial amount of useless work.
1421 * Since most packets only take 1 or 2 ring slots, stopping the queue
1422 * early means 16 slots are typically wasted.
stephen hemmingerd631b942015-03-24 16:22:07 -07001423 */
Linus Torvaldsb7dfde92012-12-20 08:37:04 -08001424 if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
Jason Wang986a4f42012-12-07 07:04:56 +00001425 netif_stop_subqueue(dev, qnum);
Willem de Bruijnb92f1e62017-04-24 13:49:27 -04001426 if (!use_napi &&
1427 unlikely(!virtqueue_enable_cb_delayed(sq->vq))) {
Rusty Russell48925e32009-09-24 09:59:20 -06001428 /* More just got used, free them then recheck. */
Linus Torvaldsb7dfde92012-12-20 08:37:04 -08001429 free_old_xmit_skbs(sq);
1430 if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
Jason Wang986a4f42012-12-07 07:04:56 +00001431 netif_start_subqueue(dev, qnum);
Jason Wange9d74172012-12-07 07:04:55 +00001432 virtqueue_disable_cb(sq->vq);
Rusty Russell48925e32009-09-24 09:59:20 -06001433 }
1434 }
Rusty Russell99ffc692008-05-02 21:50:46 -05001435 }
Rusty Russell48925e32009-09-24 09:59:20 -06001436
Michael S. Tsirkin4b7fd2e62014-10-15 16:23:28 +03001437 if (kick || netif_xmit_stopped(txq))
David S. Miller0b725a22014-08-25 15:51:53 -07001438 virtqueue_kick(sq->vq);
1439
Rusty Russell48925e32009-09-24 09:59:20 -06001440 return NETDEV_TX_OK;
Rusty Russell296f96f2007-10-22 11:03:37 +10001441}
1442
Amos Kong40cbfc32013-01-21 01:17:21 +00001443/*
1444 * Send command via the control virtqueue and check status. Commands
1445 * supported by the hypervisor, as indicated by feature bits, should
stephen hemminger788a8b62013-12-09 16:18:45 -08001446 * never fail unless improperly formatted.
Amos Kong40cbfc32013-01-21 01:17:21 +00001447 */
1448static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,
stephen hemmingerd24bae32013-12-09 16:17:40 -08001449 struct scatterlist *out)
Amos Kong40cbfc32013-01-21 01:17:21 +00001450{
Rusty Russellf7bc9592013-03-20 15:44:28 +10301451 struct scatterlist *sgs[4], hdr, stat;
stephen hemmingerd24bae32013-12-09 16:17:40 -08001452 unsigned out_num = 0, tmp;
Amos Kong40cbfc32013-01-21 01:17:21 +00001453
1454 /* Caller should know better */
Rusty Russellf7bc9592013-03-20 15:44:28 +10301455 BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ));
Amos Kong40cbfc32013-01-21 01:17:21 +00001456
Michael S. Tsirkin2ac46032015-11-15 15:11:00 +02001457 vi->ctrl_status = ~0;
1458 vi->ctrl_hdr.class = class;
1459 vi->ctrl_hdr.cmd = cmd;
Rusty Russellf7bc9592013-03-20 15:44:28 +10301460 /* Add header */
Michael S. Tsirkin2ac46032015-11-15 15:11:00 +02001461 sg_init_one(&hdr, &vi->ctrl_hdr, sizeof(vi->ctrl_hdr));
Rusty Russellf7bc9592013-03-20 15:44:28 +10301462 sgs[out_num++] = &hdr;
Amos Kong40cbfc32013-01-21 01:17:21 +00001463
Rusty Russellf7bc9592013-03-20 15:44:28 +10301464 if (out)
1465 sgs[out_num++] = out;
Amos Kong40cbfc32013-01-21 01:17:21 +00001466
Rusty Russellf7bc9592013-03-20 15:44:28 +10301467 /* Add return status. */
Michael S. Tsirkin2ac46032015-11-15 15:11:00 +02001468 sg_init_one(&stat, &vi->ctrl_status, sizeof(vi->ctrl_status));
stephen hemmingerd24bae32013-12-09 16:17:40 -08001469 sgs[out_num] = &stat;
Amos Kong40cbfc32013-01-21 01:17:21 +00001470
stephen hemmingerd24bae32013-12-09 16:17:40 -08001471 BUG_ON(out_num + 1 > ARRAY_SIZE(sgs));
Rusty Russella7c58142014-03-13 11:23:39 +10301472 virtqueue_add_sgs(vi->cvq, sgs, out_num, 1, vi, GFP_ATOMIC);
Amos Kong40cbfc32013-01-21 01:17:21 +00001473
Heinz Graalfs67975902013-10-29 09:40:02 +10301474 if (unlikely(!virtqueue_kick(vi->cvq)))
Michael S. Tsirkin2ac46032015-11-15 15:11:00 +02001475 return vi->ctrl_status == VIRTIO_NET_OK;
Amos Kong40cbfc32013-01-21 01:17:21 +00001476
1477 /* Spin for a response, the kick causes an ioport write, trapping
1478 * into the hypervisor, so the request should be handled immediately.
1479 */
Heinz Graalfs047b9b92013-10-29 09:40:47 +10301480 while (!virtqueue_get_buf(vi->cvq, &tmp) &&
1481 !virtqueue_is_broken(vi->cvq))
Amos Kong40cbfc32013-01-21 01:17:21 +00001482 cpu_relax();
1483
Michael S. Tsirkin2ac46032015-11-15 15:11:00 +02001484 return vi->ctrl_status == VIRTIO_NET_OK;
Amos Kong40cbfc32013-01-21 01:17:21 +00001485}
1486
Alex Williamson9c46f6d2009-02-04 16:36:34 -08001487static int virtnet_set_mac_address(struct net_device *dev, void *p)
1488{
1489 struct virtnet_info *vi = netdev_priv(dev);
1490 struct virtio_device *vdev = vi->vdev;
Jiri Pirkof2f2c8b2012-06-29 05:10:06 +00001491 int ret;
Andy Lutomirskie37e2ff2016-12-05 18:10:58 -08001492 struct sockaddr *addr;
Amos Kong7e58d5a2013-01-21 01:17:23 +00001493 struct scatterlist sg;
Alex Williamson9c46f6d2009-02-04 16:36:34 -08001494
Shyam Saini801822d2016-12-24 00:44:58 +05301495 addr = kmemdup(p, sizeof(*addr), GFP_KERNEL);
Andy Lutomirskie37e2ff2016-12-05 18:10:58 -08001496 if (!addr)
1497 return -ENOMEM;
Andy Lutomirskie37e2ff2016-12-05 18:10:58 -08001498
1499 ret = eth_prepare_mac_addr_change(dev, addr);
Jiri Pirkof2f2c8b2012-06-29 05:10:06 +00001500 if (ret)
Andy Lutomirskie37e2ff2016-12-05 18:10:58 -08001501 goto out;
Alex Williamson9c46f6d2009-02-04 16:36:34 -08001502
Amos Kong7e58d5a2013-01-21 01:17:23 +00001503 if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
1504 sg_init_one(&sg, addr->sa_data, dev->addr_len);
1505 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
stephen hemmingerd24bae32013-12-09 16:17:40 -08001506 VIRTIO_NET_CTRL_MAC_ADDR_SET, &sg)) {
Amos Kong7e58d5a2013-01-21 01:17:23 +00001507 dev_warn(&vdev->dev,
1508 "Failed to set mac address by vq command.\n");
Andy Lutomirskie37e2ff2016-12-05 18:10:58 -08001509 ret = -EINVAL;
1510 goto out;
Amos Kong7e58d5a2013-01-21 01:17:23 +00001511 }
Michael S. Tsirkin7e93a022014-11-26 15:58:28 +02001512 } else if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC) &&
1513 !virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) {
Rusty Russell855e0c52013-10-14 18:11:51 +10301514 unsigned int i;
1515
1516 /* Naturally, this has an atomicity problem. */
1517 for (i = 0; i < dev->addr_len; i++)
1518 virtio_cwrite8(vdev,
1519 offsetof(struct virtio_net_config, mac) +
1520 i, addr->sa_data[i]);
Amos Kong7e58d5a2013-01-21 01:17:23 +00001521 }
1522
1523 eth_commit_mac_addr_change(dev, p);
Andy Lutomirskie37e2ff2016-12-05 18:10:58 -08001524 ret = 0;
Alex Williamson9c46f6d2009-02-04 16:36:34 -08001525
Andy Lutomirskie37e2ff2016-12-05 18:10:58 -08001526out:
1527 kfree(addr);
1528 return ret;
Alex Williamson9c46f6d2009-02-04 16:36:34 -08001529}
1530
stephen hemmingerbc1f4472017-01-06 19:12:52 -08001531static void virtnet_stats(struct net_device *dev,
1532 struct rtnl_link_stats64 *tot)
stephen hemminger3fa2a1d2011-06-15 06:36:29 +00001533{
1534 struct virtnet_info *vi = netdev_priv(dev);
stephen hemminger3fa2a1d2011-06-15 06:36:29 +00001535 unsigned int start;
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09001536 int i;
stephen hemminger3fa2a1d2011-06-15 06:36:29 +00001537
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09001538 for (i = 0; i < vi->max_queue_pairs; i++) {
stephen hemminger3fa2a1d2011-06-15 06:36:29 +00001539 u64 tpackets, tbytes, rpackets, rbytes;
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09001540 struct receive_queue *rq = &vi->rq[i];
1541 struct send_queue *sq = &vi->sq[i];
stephen hemminger3fa2a1d2011-06-15 06:36:29 +00001542
1543 do {
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09001544 start = u64_stats_fetch_begin_irq(&sq->stats.syncp);
1545 tpackets = sq->stats.packets;
1546 tbytes = sq->stats.bytes;
1547 } while (u64_stats_fetch_retry_irq(&sq->stats.syncp, start));
Eric Dumazet83a27052012-06-05 22:35:24 +00001548
1549 do {
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09001550 start = u64_stats_fetch_begin_irq(&rq->stats.syncp);
1551 rpackets = rq->stats.packets;
1552 rbytes = rq->stats.bytes;
1553 } while (u64_stats_fetch_retry_irq(&rq->stats.syncp, start));
stephen hemminger3fa2a1d2011-06-15 06:36:29 +00001554
1555 tot->rx_packets += rpackets;
1556 tot->tx_packets += tpackets;
1557 tot->rx_bytes += rbytes;
1558 tot->tx_bytes += tbytes;
1559 }
1560
1561 tot->tx_dropped = dev->stats.tx_dropped;
Rick Jones021ac8d2011-11-21 09:28:17 +00001562 tot->tx_fifo_errors = dev->stats.tx_fifo_errors;
stephen hemminger3fa2a1d2011-06-15 06:36:29 +00001563 tot->rx_dropped = dev->stats.rx_dropped;
1564 tot->rx_length_errors = dev->stats.rx_length_errors;
1565 tot->rx_frame_errors = dev->stats.rx_frame_errors;
stephen hemminger3fa2a1d2011-06-15 06:36:29 +00001566}
1567
Amit Shahda74e892008-02-29 16:24:50 +05301568#ifdef CONFIG_NET_POLL_CONTROLLER
1569static void virtnet_netpoll(struct net_device *dev)
1570{
1571 struct virtnet_info *vi = netdev_priv(dev);
Jason Wang986a4f42012-12-07 07:04:56 +00001572 int i;
Amit Shahda74e892008-02-29 16:24:50 +05301573
Jason Wang986a4f42012-12-07 07:04:56 +00001574 for (i = 0; i < vi->curr_queue_pairs; i++)
1575 napi_schedule(&vi->rq[i].napi);
Amit Shahda74e892008-02-29 16:24:50 +05301576}
1577#endif
1578
Jason Wang586d17c2012-04-11 20:43:52 +00001579static void virtnet_ack_link_announce(struct virtnet_info *vi)
1580{
1581 rtnl_lock();
1582 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE,
stephen hemmingerd24bae32013-12-09 16:17:40 -08001583 VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL))
Jason Wang586d17c2012-04-11 20:43:52 +00001584 dev_warn(&vi->dev->dev, "Failed to ack link announce.\n");
1585 rtnl_unlock();
1586}
1587
John Fastabend473153292017-02-02 19:14:32 -08001588static int _virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
Jason Wang986a4f42012-12-07 07:04:56 +00001589{
1590 struct scatterlist sg;
Jason Wang986a4f42012-12-07 07:04:56 +00001591 struct net_device *dev = vi->dev;
1592
1593 if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ))
1594 return 0;
1595
Andy Lutomirskia725ee32016-07-18 15:34:49 -07001596 vi->ctrl_mq.virtqueue_pairs = cpu_to_virtio16(vi->vdev, queue_pairs);
1597 sg_init_one(&sg, &vi->ctrl_mq, sizeof(vi->ctrl_mq));
Jason Wang986a4f42012-12-07 07:04:56 +00001598
1599 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ,
stephen hemmingerd24bae32013-12-09 16:17:40 -08001600 VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &sg)) {
Jason Wang986a4f42012-12-07 07:04:56 +00001601 dev_warn(&dev->dev, "Fail to set num of queue pairs to %d\n",
1602 queue_pairs);
1603 return -EINVAL;
Sasha Levin55257d72013-04-29 12:00:08 +09301604 } else {
Jason Wang986a4f42012-12-07 07:04:56 +00001605 vi->curr_queue_pairs = queue_pairs;
Jason Wang35ed1592013-10-15 11:18:59 +08001606 /* virtnet_open() will refill when device is going to up. */
1607 if (dev->flags & IFF_UP)
1608 schedule_delayed_work(&vi->refill, 0);
Sasha Levin55257d72013-04-29 12:00:08 +09301609 }
Jason Wang986a4f42012-12-07 07:04:56 +00001610
1611 return 0;
1612}
1613
John Fastabend473153292017-02-02 19:14:32 -08001614static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
1615{
1616 int err;
1617
1618 rtnl_lock();
1619 err = _virtnet_set_queues(vi, queue_pairs);
1620 rtnl_unlock();
1621 return err;
1622}
1623
Rusty Russell296f96f2007-10-22 11:03:37 +10001624static int virtnet_close(struct net_device *dev)
1625{
1626 struct virtnet_info *vi = netdev_priv(dev);
Jason Wang986a4f42012-12-07 07:04:56 +00001627 int i;
Rusty Russell296f96f2007-10-22 11:03:37 +10001628
Rusty Russellb2baed62011-12-29 00:42:38 +00001629 /* Make sure refill_work doesn't re-enable napi! */
1630 cancel_delayed_work_sync(&vi->refill);
Jason Wang986a4f42012-12-07 07:04:56 +00001631
Willem de Bruijnb92f1e62017-04-24 13:49:27 -04001632 for (i = 0; i < vi->max_queue_pairs; i++) {
Jesper Dangaard Brouer754b8a22018-01-03 11:26:04 +01001633 xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq);
Jason Wang986a4f42012-12-07 07:04:56 +00001634 napi_disable(&vi->rq[i].napi);
Willem de Bruijn78a57b42017-04-25 15:59:17 -04001635 virtnet_napi_tx_disable(&vi->sq[i].napi);
Willem de Bruijnb92f1e62017-04-24 13:49:27 -04001636 }
Rusty Russell296f96f2007-10-22 11:03:37 +10001637
Rusty Russell296f96f2007-10-22 11:03:37 +10001638 return 0;
1639}
1640
Alex Williamson2af76982009-02-04 09:02:40 +00001641static void virtnet_set_rx_mode(struct net_device *dev)
1642{
1643 struct virtnet_info *vi = netdev_priv(dev);
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001644 struct scatterlist sg[2];
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001645 struct virtio_net_ctrl_mac *mac_data;
Jiri Pirkoccffad252009-05-22 23:22:17 +00001646 struct netdev_hw_addr *ha;
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08001647 int uc_count;
Jiri Pirko4cd24ea2010-02-08 04:30:35 +00001648 int mc_count;
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001649 void *buf;
1650 int i;
Alex Williamson2af76982009-02-04 09:02:40 +00001651
stephen hemminger788a8b62013-12-09 16:18:45 -08001652 /* We can't dynamically set ndo_set_rx_mode, so return gracefully */
Alex Williamson2af76982009-02-04 09:02:40 +00001653 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
1654 return;
1655
Michael S. Tsirkin2ac46032015-11-15 15:11:00 +02001656 vi->ctrl_promisc = ((dev->flags & IFF_PROMISC) != 0);
1657 vi->ctrl_allmulti = ((dev->flags & IFF_ALLMULTI) != 0);
Alex Williamson2af76982009-02-04 09:02:40 +00001658
Michael S. Tsirkin2ac46032015-11-15 15:11:00 +02001659 sg_init_one(sg, &vi->ctrl_promisc, sizeof(vi->ctrl_promisc));
Alex Williamson2af76982009-02-04 09:02:40 +00001660
1661 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
stephen hemmingerd24bae32013-12-09 16:17:40 -08001662 VIRTIO_NET_CTRL_RX_PROMISC, sg))
Alex Williamson2af76982009-02-04 09:02:40 +00001663 dev_warn(&dev->dev, "Failed to %sable promisc mode.\n",
Michael S. Tsirkin2ac46032015-11-15 15:11:00 +02001664 vi->ctrl_promisc ? "en" : "dis");
Alex Williamson2af76982009-02-04 09:02:40 +00001665
Michael S. Tsirkin2ac46032015-11-15 15:11:00 +02001666 sg_init_one(sg, &vi->ctrl_allmulti, sizeof(vi->ctrl_allmulti));
Alex Williamson2af76982009-02-04 09:02:40 +00001667
1668 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
stephen hemmingerd24bae32013-12-09 16:17:40 -08001669 VIRTIO_NET_CTRL_RX_ALLMULTI, sg))
Alex Williamson2af76982009-02-04 09:02:40 +00001670 dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n",
Michael S. Tsirkin2ac46032015-11-15 15:11:00 +02001671 vi->ctrl_allmulti ? "en" : "dis");
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001672
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08001673 uc_count = netdev_uc_count(dev);
Jiri Pirko4cd24ea2010-02-08 04:30:35 +00001674 mc_count = netdev_mc_count(dev);
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001675 /* MAC filter - use one buffer for both lists */
Jiri Pirko4cd24ea2010-02-08 04:30:35 +00001676 buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
1677 (2 * sizeof(mac_data->entries)), GFP_ATOMIC);
1678 mac_data = buf;
Joe Perchese68ed8f2013-02-03 17:28:15 +00001679 if (!buf)
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001680 return;
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001681
Alex Williamson23e258e2009-05-01 17:27:56 +00001682 sg_init_table(sg, 2);
1683
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001684 /* Store the unicast list and count in the front of the buffer */
Michael S. Tsirkinfdd819b2014-10-07 16:39:48 +02001685 mac_data->entries = cpu_to_virtio32(vi->vdev, uc_count);
Jiri Pirkoccffad252009-05-22 23:22:17 +00001686 i = 0;
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08001687 netdev_for_each_uc_addr(ha, dev)
Jiri Pirkoccffad252009-05-22 23:22:17 +00001688 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001689
1690 sg_set_buf(&sg[0], mac_data,
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08001691 sizeof(mac_data->entries) + (uc_count * ETH_ALEN));
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001692
1693 /* multicast list and count fill the end */
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08001694 mac_data = (void *)&mac_data->macs[uc_count][0];
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001695
Michael S. Tsirkinfdd819b2014-10-07 16:39:48 +02001696 mac_data->entries = cpu_to_virtio32(vi->vdev, mc_count);
Jiri Pirko567ec872010-02-23 23:17:07 +00001697 i = 0;
Jiri Pirko22bedad32010-04-01 21:22:57 +00001698 netdev_for_each_mc_addr(ha, dev)
1699 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001700
1701 sg_set_buf(&sg[1], mac_data,
Jiri Pirko4cd24ea2010-02-08 04:30:35 +00001702 sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001703
1704 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
stephen hemmingerd24bae32013-12-09 16:17:40 -08001705 VIRTIO_NET_CTRL_MAC_TABLE_SET, sg))
Thomas Huth99e872a2013-11-29 10:02:19 +01001706 dev_warn(&dev->dev, "Failed to set MAC filter table.\n");
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001707
1708 kfree(buf);
Alex Williamson2af76982009-02-04 09:02:40 +00001709}
1710
Patrick McHardy80d5c362013-04-19 02:04:28 +00001711static int virtnet_vlan_rx_add_vid(struct net_device *dev,
1712 __be16 proto, u16 vid)
Alex Williamson0bde95692009-02-04 09:02:50 +00001713{
1714 struct virtnet_info *vi = netdev_priv(dev);
1715 struct scatterlist sg;
1716
Andy Lutomirskia725ee32016-07-18 15:34:49 -07001717 vi->ctrl_vid = vid;
1718 sg_init_one(&sg, &vi->ctrl_vid, sizeof(vi->ctrl_vid));
Alex Williamson0bde95692009-02-04 09:02:50 +00001719
1720 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
stephen hemmingerd24bae32013-12-09 16:17:40 -08001721 VIRTIO_NET_CTRL_VLAN_ADD, &sg))
Alex Williamson0bde95692009-02-04 09:02:50 +00001722 dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid);
Jiri Pirko8e586132011-12-08 19:52:37 -05001723 return 0;
Alex Williamson0bde95692009-02-04 09:02:50 +00001724}
1725
Patrick McHardy80d5c362013-04-19 02:04:28 +00001726static int virtnet_vlan_rx_kill_vid(struct net_device *dev,
1727 __be16 proto, u16 vid)
Alex Williamson0bde95692009-02-04 09:02:50 +00001728{
1729 struct virtnet_info *vi = netdev_priv(dev);
1730 struct scatterlist sg;
1731
Andy Lutomirskia725ee32016-07-18 15:34:49 -07001732 vi->ctrl_vid = vid;
1733 sg_init_one(&sg, &vi->ctrl_vid, sizeof(vi->ctrl_vid));
Alex Williamson0bde95692009-02-04 09:02:50 +00001734
1735 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
stephen hemmingerd24bae32013-12-09 16:17:40 -08001736 VIRTIO_NET_CTRL_VLAN_DEL, &sg))
Alex Williamson0bde95692009-02-04 09:02:50 +00001737 dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid);
Jiri Pirko8e586132011-12-08 19:52:37 -05001738 return 0;
Alex Williamson0bde95692009-02-04 09:02:50 +00001739}
1740
Wanlong Gao8898c212013-01-24 23:51:30 +00001741static void virtnet_clean_affinity(struct virtnet_info *vi, long hcpu)
Jason Wang986a4f42012-12-07 07:04:56 +00001742{
1743 int i;
Wanlong Gao8898c212013-01-24 23:51:30 +00001744
1745 if (vi->affinity_hint_set) {
1746 for (i = 0; i < vi->max_queue_pairs; i++) {
1747 virtqueue_set_affinity(vi->rq[i].vq, -1);
1748 virtqueue_set_affinity(vi->sq[i].vq, -1);
1749 }
1750
1751 vi->affinity_hint_set = false;
1752 }
Wanlong Gao8898c212013-01-24 23:51:30 +00001753}
1754
1755static void virtnet_set_affinity(struct virtnet_info *vi)
Jason Wang986a4f42012-12-07 07:04:56 +00001756{
1757 int i;
Wanlong Gao47be2472013-01-24 23:51:29 +00001758 int cpu;
Jason Wang986a4f42012-12-07 07:04:56 +00001759
1760 /* In multiqueue mode, when the number of cpu is equal to the number of
1761 * queue pairs, we let the queue pairs to be private to one cpu by
1762 * setting the affinity hint to eliminate the contention.
1763 */
Wanlong Gao8898c212013-01-24 23:51:30 +00001764 if (vi->curr_queue_pairs == 1 ||
1765 vi->max_queue_pairs != num_online_cpus()) {
1766 virtnet_clean_affinity(vi, -1);
1767 return;
Jason Wang986a4f42012-12-07 07:04:56 +00001768 }
1769
Wanlong Gao8898c212013-01-24 23:51:30 +00001770 i = 0;
1771 for_each_online_cpu(cpu) {
Jason Wang986a4f42012-12-07 07:04:56 +00001772 virtqueue_set_affinity(vi->rq[i].vq, cpu);
1773 virtqueue_set_affinity(vi->sq[i].vq, cpu);
Jason Wang9bb8ca82013-11-05 18:19:45 +08001774 netif_set_xps_queue(vi->dev, cpumask_of(cpu), i);
Wanlong Gao8898c212013-01-24 23:51:30 +00001775 i++;
Jason Wang986a4f42012-12-07 07:04:56 +00001776 }
1777
Wanlong Gao8898c212013-01-24 23:51:30 +00001778 vi->affinity_hint_set = true;
Jason Wang986a4f42012-12-07 07:04:56 +00001779}
1780
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +02001781static int virtnet_cpu_online(unsigned int cpu, struct hlist_node *node)
Wanlong Gao8de4b2f2013-01-24 23:51:31 +00001782{
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +02001783 struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
1784 node);
1785 virtnet_set_affinity(vi);
1786 return 0;
1787}
Wanlong Gao8de4b2f2013-01-24 23:51:31 +00001788
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +02001789static int virtnet_cpu_dead(unsigned int cpu, struct hlist_node *node)
1790{
1791 struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
1792 node_dead);
1793 virtnet_set_affinity(vi);
1794 return 0;
1795}
Jason Wang3ab098d2013-10-15 11:18:58 +08001796
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +02001797static int virtnet_cpu_down_prep(unsigned int cpu, struct hlist_node *node)
1798{
1799 struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
1800 node);
1801
1802 virtnet_clean_affinity(vi, cpu);
1803 return 0;
1804}
1805
1806static enum cpuhp_state virtionet_online;
1807
1808static int virtnet_cpu_notif_add(struct virtnet_info *vi)
1809{
1810 int ret;
1811
1812 ret = cpuhp_state_add_instance_nocalls(virtionet_online, &vi->node);
1813 if (ret)
1814 return ret;
1815 ret = cpuhp_state_add_instance_nocalls(CPUHP_VIRT_NET_DEAD,
1816 &vi->node_dead);
1817 if (!ret)
1818 return ret;
1819 cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);
1820 return ret;
1821}
1822
1823static void virtnet_cpu_notif_remove(struct virtnet_info *vi)
1824{
1825 cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);
1826 cpuhp_state_remove_instance_nocalls(CPUHP_VIRT_NET_DEAD,
1827 &vi->node_dead);
Herbert Xua9ea3fc2008-04-18 11:21:42 +08001828}
1829
Rick Jones8f9f4662011-10-19 08:10:59 +00001830static void virtnet_get_ringparam(struct net_device *dev,
1831 struct ethtool_ringparam *ring)
1832{
1833 struct virtnet_info *vi = netdev_priv(dev);
1834
Jason Wang986a4f42012-12-07 07:04:56 +00001835 ring->rx_max_pending = virtqueue_get_vring_size(vi->rq[0].vq);
1836 ring->tx_max_pending = virtqueue_get_vring_size(vi->sq[0].vq);
Rick Jones8f9f4662011-10-19 08:10:59 +00001837 ring->rx_pending = ring->rx_max_pending;
1838 ring->tx_pending = ring->tx_max_pending;
Rick Jones8f9f4662011-10-19 08:10:59 +00001839}
1840
Rick Jones66846042011-11-14 14:17:08 +00001841
1842static void virtnet_get_drvinfo(struct net_device *dev,
1843 struct ethtool_drvinfo *info)
1844{
1845 struct virtnet_info *vi = netdev_priv(dev);
1846 struct virtio_device *vdev = vi->vdev;
1847
1848 strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
1849 strlcpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version));
1850 strlcpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info));
1851
1852}
1853
Jason Wangd73bcd22012-12-07 07:04:57 +00001854/* TODO: Eliminate OOO packets during switching */
1855static int virtnet_set_channels(struct net_device *dev,
1856 struct ethtool_channels *channels)
1857{
1858 struct virtnet_info *vi = netdev_priv(dev);
1859 u16 queue_pairs = channels->combined_count;
1860 int err;
1861
1862 /* We don't support separate rx/tx channels.
1863 * We don't allow setting 'other' channels.
1864 */
1865 if (channels->rx_count || channels->tx_count || channels->other_count)
1866 return -EINVAL;
1867
Amos Kongc18e9cd2014-04-18 13:45:41 +08001868 if (queue_pairs > vi->max_queue_pairs || queue_pairs == 0)
Jason Wangd73bcd22012-12-07 07:04:57 +00001869 return -EINVAL;
1870
John Fastabendf600b692016-12-15 12:13:24 -08001871 /* For now we don't support modifying channels while XDP is loaded
1872 * also when XDP is loaded all RX queues have XDP programs so we only
1873 * need to check a single RX queue.
1874 */
1875 if (vi->rq[0].xdp_prog)
1876 return -EINVAL;
1877
Wanlong Gao47be2472013-01-24 23:51:29 +00001878 get_online_cpus();
John Fastabend473153292017-02-02 19:14:32 -08001879 err = _virtnet_set_queues(vi, queue_pairs);
Jason Wangd73bcd22012-12-07 07:04:57 +00001880 if (!err) {
1881 netif_set_real_num_tx_queues(dev, queue_pairs);
1882 netif_set_real_num_rx_queues(dev, queue_pairs);
1883
Wanlong Gao8898c212013-01-24 23:51:30 +00001884 virtnet_set_affinity(vi);
Jason Wangd73bcd22012-12-07 07:04:57 +00001885 }
Wanlong Gao47be2472013-01-24 23:51:29 +00001886 put_online_cpus();
Jason Wangd73bcd22012-12-07 07:04:57 +00001887
1888 return err;
1889}
1890
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09001891static void virtnet_get_strings(struct net_device *dev, u32 stringset, u8 *data)
1892{
1893 struct virtnet_info *vi = netdev_priv(dev);
1894 char *p = (char *)data;
1895 unsigned int i, j;
1896
1897 switch (stringset) {
1898 case ETH_SS_STATS:
1899 for (i = 0; i < vi->curr_queue_pairs; i++) {
1900 for (j = 0; j < VIRTNET_RQ_STATS_LEN; j++) {
1901 snprintf(p, ETH_GSTRING_LEN, "rx_queue_%u_%s",
1902 i, virtnet_rq_stats_desc[j].desc);
1903 p += ETH_GSTRING_LEN;
1904 }
1905 }
1906
1907 for (i = 0; i < vi->curr_queue_pairs; i++) {
1908 for (j = 0; j < VIRTNET_SQ_STATS_LEN; j++) {
1909 snprintf(p, ETH_GSTRING_LEN, "tx_queue_%u_%s",
1910 i, virtnet_sq_stats_desc[j].desc);
1911 p += ETH_GSTRING_LEN;
1912 }
1913 }
1914 break;
1915 }
1916}
1917
1918static int virtnet_get_sset_count(struct net_device *dev, int sset)
1919{
1920 struct virtnet_info *vi = netdev_priv(dev);
1921
1922 switch (sset) {
1923 case ETH_SS_STATS:
1924 return vi->curr_queue_pairs * (VIRTNET_RQ_STATS_LEN +
1925 VIRTNET_SQ_STATS_LEN);
1926 default:
1927 return -EOPNOTSUPP;
1928 }
1929}
1930
1931static void virtnet_get_ethtool_stats(struct net_device *dev,
1932 struct ethtool_stats *stats, u64 *data)
1933{
1934 struct virtnet_info *vi = netdev_priv(dev);
1935 unsigned int idx = 0, start, i, j;
1936 const u8 *stats_base;
1937 size_t offset;
1938
1939 for (i = 0; i < vi->curr_queue_pairs; i++) {
1940 struct receive_queue *rq = &vi->rq[i];
1941
1942 stats_base = (u8 *)&rq->stats;
1943 do {
1944 start = u64_stats_fetch_begin_irq(&rq->stats.syncp);
1945 for (j = 0; j < VIRTNET_RQ_STATS_LEN; j++) {
1946 offset = virtnet_rq_stats_desc[j].offset;
1947 data[idx + j] = *(u64 *)(stats_base + offset);
1948 }
1949 } while (u64_stats_fetch_retry_irq(&rq->stats.syncp, start));
1950 idx += VIRTNET_RQ_STATS_LEN;
1951 }
1952
1953 for (i = 0; i < vi->curr_queue_pairs; i++) {
1954 struct send_queue *sq = &vi->sq[i];
1955
1956 stats_base = (u8 *)&sq->stats;
1957 do {
1958 start = u64_stats_fetch_begin_irq(&sq->stats.syncp);
1959 for (j = 0; j < VIRTNET_SQ_STATS_LEN; j++) {
1960 offset = virtnet_sq_stats_desc[j].offset;
1961 data[idx + j] = *(u64 *)(stats_base + offset);
1962 }
1963 } while (u64_stats_fetch_retry_irq(&sq->stats.syncp, start));
1964 idx += VIRTNET_SQ_STATS_LEN;
1965 }
1966}
1967
Jason Wangd73bcd22012-12-07 07:04:57 +00001968static void virtnet_get_channels(struct net_device *dev,
1969 struct ethtool_channels *channels)
1970{
1971 struct virtnet_info *vi = netdev_priv(dev);
1972
1973 channels->combined_count = vi->curr_queue_pairs;
1974 channels->max_combined = vi->max_queue_pairs;
1975 channels->max_other = 0;
1976 channels->rx_count = 0;
1977 channels->tx_count = 0;
1978 channels->other_count = 0;
1979}
1980
Nikolay Aleksandrov16032be2016-02-03 04:04:37 +01001981/* Check if the user is trying to change anything besides speed/duplex */
Philippe Reynesebb6b4b2017-03-21 23:24:24 +01001982static bool
1983virtnet_validate_ethtool_cmd(const struct ethtool_link_ksettings *cmd)
Nikolay Aleksandrov16032be2016-02-03 04:04:37 +01001984{
Philippe Reynesebb6b4b2017-03-21 23:24:24 +01001985 struct ethtool_link_ksettings diff1 = *cmd;
1986 struct ethtool_link_ksettings diff2 = {};
Nikolay Aleksandrov16032be2016-02-03 04:04:37 +01001987
Nikolay Aleksandrov0cf3ace2016-02-07 21:52:24 +01001988 /* cmd is always set so we need to clear it, validate the port type
1989 * and also without autonegotiation we can ignore advertising
1990 */
Philippe Reynesebb6b4b2017-03-21 23:24:24 +01001991 diff1.base.speed = 0;
1992 diff2.base.port = PORT_OTHER;
1993 ethtool_link_ksettings_zero_link_mode(&diff1, advertising);
1994 diff1.base.duplex = 0;
1995 diff1.base.cmd = 0;
1996 diff1.base.link_mode_masks_nwords = 0;
Nikolay Aleksandrov16032be2016-02-03 04:04:37 +01001997
Philippe Reynesebb6b4b2017-03-21 23:24:24 +01001998 return !memcmp(&diff1.base, &diff2.base, sizeof(diff1.base)) &&
1999 bitmap_empty(diff1.link_modes.supported,
2000 __ETHTOOL_LINK_MODE_MASK_NBITS) &&
2001 bitmap_empty(diff1.link_modes.advertising,
2002 __ETHTOOL_LINK_MODE_MASK_NBITS) &&
2003 bitmap_empty(diff1.link_modes.lp_advertising,
2004 __ETHTOOL_LINK_MODE_MASK_NBITS);
Nikolay Aleksandrov16032be2016-02-03 04:04:37 +01002005}
2006
Philippe Reynesebb6b4b2017-03-21 23:24:24 +01002007static int virtnet_set_link_ksettings(struct net_device *dev,
2008 const struct ethtool_link_ksettings *cmd)
Nikolay Aleksandrov16032be2016-02-03 04:04:37 +01002009{
2010 struct virtnet_info *vi = netdev_priv(dev);
2011 u32 speed;
2012
Philippe Reynesebb6b4b2017-03-21 23:24:24 +01002013 speed = cmd->base.speed;
Nikolay Aleksandrov16032be2016-02-03 04:04:37 +01002014 /* don't allow custom speed and duplex */
2015 if (!ethtool_validate_speed(speed) ||
Philippe Reynesebb6b4b2017-03-21 23:24:24 +01002016 !ethtool_validate_duplex(cmd->base.duplex) ||
Nikolay Aleksandrov16032be2016-02-03 04:04:37 +01002017 !virtnet_validate_ethtool_cmd(cmd))
2018 return -EINVAL;
2019 vi->speed = speed;
Philippe Reynesebb6b4b2017-03-21 23:24:24 +01002020 vi->duplex = cmd->base.duplex;
Nikolay Aleksandrov16032be2016-02-03 04:04:37 +01002021
2022 return 0;
2023}
2024
Philippe Reynesebb6b4b2017-03-21 23:24:24 +01002025static int virtnet_get_link_ksettings(struct net_device *dev,
2026 struct ethtool_link_ksettings *cmd)
Nikolay Aleksandrov16032be2016-02-03 04:04:37 +01002027{
2028 struct virtnet_info *vi = netdev_priv(dev);
2029
Philippe Reynesebb6b4b2017-03-21 23:24:24 +01002030 cmd->base.speed = vi->speed;
2031 cmd->base.duplex = vi->duplex;
2032 cmd->base.port = PORT_OTHER;
Nikolay Aleksandrov16032be2016-02-03 04:04:37 +01002033
2034 return 0;
2035}
2036
2037static void virtnet_init_settings(struct net_device *dev)
2038{
2039 struct virtnet_info *vi = netdev_priv(dev);
2040
2041 vi->speed = SPEED_UNKNOWN;
2042 vi->duplex = DUPLEX_UNKNOWN;
2043}
2044
Jason Baronfaa9b392018-01-05 17:44:54 -05002045static void virtnet_update_settings(struct virtnet_info *vi)
2046{
2047 u32 speed;
2048 u8 duplex;
2049
2050 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_SPEED_DUPLEX))
2051 return;
2052
2053 speed = virtio_cread32(vi->vdev, offsetof(struct virtio_net_config,
2054 speed));
2055 if (ethtool_validate_speed(speed))
2056 vi->speed = speed;
2057 duplex = virtio_cread8(vi->vdev, offsetof(struct virtio_net_config,
2058 duplex));
2059 if (ethtool_validate_duplex(duplex))
2060 vi->duplex = duplex;
2061}
2062
Stephen Hemminger0fc0b732009-09-02 01:03:33 -07002063static const struct ethtool_ops virtnet_ethtool_ops = {
Rick Jones66846042011-11-14 14:17:08 +00002064 .get_drvinfo = virtnet_get_drvinfo,
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08002065 .get_link = ethtool_op_get_link,
Rick Jones8f9f4662011-10-19 08:10:59 +00002066 .get_ringparam = virtnet_get_ringparam,
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09002067 .get_strings = virtnet_get_strings,
2068 .get_sset_count = virtnet_get_sset_count,
2069 .get_ethtool_stats = virtnet_get_ethtool_stats,
Jason Wangd73bcd22012-12-07 07:04:57 +00002070 .set_channels = virtnet_set_channels,
2071 .get_channels = virtnet_get_channels,
Jacob Keller074c3582014-06-25 02:37:13 +00002072 .get_ts_info = ethtool_op_get_ts_info,
Philippe Reynesebb6b4b2017-03-21 23:24:24 +01002073 .get_link_ksettings = virtnet_get_link_ksettings,
2074 .set_link_ksettings = virtnet_set_link_ksettings,
Herbert Xua9ea3fc2008-04-18 11:21:42 +08002075};
2076
John Fastabend9fe7bfc2017-02-02 19:16:01 -08002077static void virtnet_freeze_down(struct virtio_device *vdev)
2078{
2079 struct virtnet_info *vi = vdev->priv;
2080 int i;
2081
2082 /* Make sure no work handler is accessing the device */
2083 flush_work(&vi->config_work);
2084
2085 netif_device_detach(vi->dev);
Jason Wang713a98d2017-06-28 09:51:03 +08002086 netif_tx_disable(vi->dev);
John Fastabend9fe7bfc2017-02-02 19:16:01 -08002087 cancel_delayed_work_sync(&vi->refill);
2088
2089 if (netif_running(vi->dev)) {
Willem de Bruijnb92f1e62017-04-24 13:49:27 -04002090 for (i = 0; i < vi->max_queue_pairs; i++) {
John Fastabend9fe7bfc2017-02-02 19:16:01 -08002091 napi_disable(&vi->rq[i].napi);
Willem de Bruijn78a57b42017-04-25 15:59:17 -04002092 virtnet_napi_tx_disable(&vi->sq[i].napi);
Willem de Bruijnb92f1e62017-04-24 13:49:27 -04002093 }
John Fastabend9fe7bfc2017-02-02 19:16:01 -08002094 }
2095}
2096
2097static int init_vqs(struct virtnet_info *vi);
2098
2099static int virtnet_restore_up(struct virtio_device *vdev)
2100{
2101 struct virtnet_info *vi = vdev->priv;
2102 int err, i;
2103
2104 err = init_vqs(vi);
2105 if (err)
2106 return err;
2107
2108 virtio_device_ready(vdev);
2109
2110 if (netif_running(vi->dev)) {
2111 for (i = 0; i < vi->curr_queue_pairs; i++)
2112 if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))
2113 schedule_delayed_work(&vi->refill, 0);
2114
Willem de Bruijnb92f1e62017-04-24 13:49:27 -04002115 for (i = 0; i < vi->max_queue_pairs; i++) {
Willem de Bruijne4e84522017-04-24 13:49:26 -04002116 virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
Willem de Bruijnb92f1e62017-04-24 13:49:27 -04002117 virtnet_napi_tx_enable(vi, vi->sq[i].vq,
2118 &vi->sq[i].napi);
2119 }
John Fastabend9fe7bfc2017-02-02 19:16:01 -08002120 }
2121
2122 netif_device_attach(vi->dev);
2123 return err;
2124}
2125
Jason Wang3f935222017-07-19 16:54:49 +08002126static int virtnet_set_guest_offloads(struct virtnet_info *vi, u64 offloads)
2127{
2128 struct scatterlist sg;
2129 vi->ctrl_offloads = cpu_to_virtio64(vi->vdev, offloads);
2130
2131 sg_init_one(&sg, &vi->ctrl_offloads, sizeof(vi->ctrl_offloads));
2132
2133 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_GUEST_OFFLOADS,
2134 VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET, &sg)) {
2135 dev_warn(&vi->dev->dev, "Fail to set guest offload. \n");
2136 return -EINVAL;
2137 }
2138
2139 return 0;
2140}
2141
2142static int virtnet_clear_guest_offloads(struct virtnet_info *vi)
2143{
2144 u64 offloads = 0;
2145
2146 if (!vi->guest_offloads)
2147 return 0;
2148
2149 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM))
2150 offloads = 1ULL << VIRTIO_NET_F_GUEST_CSUM;
2151
2152 return virtnet_set_guest_offloads(vi, offloads);
2153}
2154
2155static int virtnet_restore_guest_offloads(struct virtnet_info *vi)
2156{
2157 u64 offloads = vi->guest_offloads;
2158
2159 if (!vi->guest_offloads)
2160 return 0;
2161 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM))
2162 offloads |= 1ULL << VIRTIO_NET_F_GUEST_CSUM;
2163
2164 return virtnet_set_guest_offloads(vi, offloads);
2165}
2166
Jakub Kicinski9861ce02017-04-30 21:46:48 -07002167static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog,
2168 struct netlink_ext_ack *extack)
John Fastabendf600b692016-12-15 12:13:24 -08002169{
2170 unsigned long int max_sz = PAGE_SIZE - sizeof(struct padded_vnet_hdr);
2171 struct virtnet_info *vi = netdev_priv(dev);
2172 struct bpf_prog *old_prog;
Jason Wang017b29c2017-02-20 11:50:20 +08002173 u16 xdp_qp = 0, curr_qp;
John Fastabend672aafd2016-12-15 12:13:49 -08002174 int i, err;
John Fastabendf600b692016-12-15 12:13:24 -08002175
Jason Wang3f935222017-07-19 16:54:49 +08002176 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)
2177 && (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) ||
2178 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) ||
2179 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) ||
2180 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO))) {
Daniel Borkmann4d463c42017-05-03 00:39:17 +02002181 NL_SET_ERR_MSG_MOD(extack, "Can't set XDP while host is implementing LRO, disable LRO first");
John Fastabendf600b692016-12-15 12:13:24 -08002182 return -EOPNOTSUPP;
2183 }
2184
2185 if (vi->mergeable_rx_bufs && !vi->any_header_sg) {
Daniel Borkmann4d463c42017-05-03 00:39:17 +02002186 NL_SET_ERR_MSG_MOD(extack, "XDP expects header/data in single page, any_header_sg required");
John Fastabendf600b692016-12-15 12:13:24 -08002187 return -EINVAL;
2188 }
2189
2190 if (dev->mtu > max_sz) {
Daniel Borkmann4d463c42017-05-03 00:39:17 +02002191 NL_SET_ERR_MSG_MOD(extack, "MTU too large to enable XDP");
John Fastabendf600b692016-12-15 12:13:24 -08002192 netdev_warn(dev, "XDP requires MTU less than %lu\n", max_sz);
2193 return -EINVAL;
2194 }
2195
John Fastabend672aafd2016-12-15 12:13:49 -08002196 curr_qp = vi->curr_queue_pairs - vi->xdp_queue_pairs;
2197 if (prog)
2198 xdp_qp = nr_cpu_ids;
2199
2200 /* XDP requires extra queues for XDP_TX */
2201 if (curr_qp + xdp_qp > vi->max_queue_pairs) {
Daniel Borkmann4d463c42017-05-03 00:39:17 +02002202 NL_SET_ERR_MSG_MOD(extack, "Too few free TX rings available");
John Fastabend672aafd2016-12-15 12:13:49 -08002203 netdev_warn(dev, "request %i queues but max is %i\n",
2204 curr_qp + xdp_qp, vi->max_queue_pairs);
2205 return -ENOMEM;
2206 }
2207
John Fastabend2de2f7f2017-02-02 19:16:29 -08002208 if (prog) {
2209 prog = bpf_prog_add(prog, vi->max_queue_pairs - 1);
2210 if (IS_ERR(prog))
2211 return PTR_ERR(prog);
2212 }
2213
Jason Wang4941d472017-07-19 16:54:48 +08002214 /* Make sure NAPI is not using any XDP TX queues for RX. */
Jason Wang4e09ff52018-02-28 18:20:04 +08002215 if (netif_running(dev))
2216 for (i = 0; i < vi->max_queue_pairs; i++)
2217 napi_disable(&vi->rq[i].napi);
John Fastabendf600b692016-12-15 12:13:24 -08002218
John Fastabend672aafd2016-12-15 12:13:49 -08002219 netif_set_real_num_rx_queues(dev, curr_qp + xdp_qp);
Jason Wang4941d472017-07-19 16:54:48 +08002220 err = _virtnet_set_queues(vi, curr_qp + xdp_qp);
2221 if (err)
2222 goto err;
2223 vi->xdp_queue_pairs = xdp_qp;
John Fastabend672aafd2016-12-15 12:13:49 -08002224
John Fastabendf600b692016-12-15 12:13:24 -08002225 for (i = 0; i < vi->max_queue_pairs; i++) {
2226 old_prog = rtnl_dereference(vi->rq[i].xdp_prog);
2227 rcu_assign_pointer(vi->rq[i].xdp_prog, prog);
Jason Wang3f935222017-07-19 16:54:49 +08002228 if (i == 0) {
2229 if (!old_prog)
2230 virtnet_clear_guest_offloads(vi);
2231 if (!prog)
2232 virtnet_restore_guest_offloads(vi);
2233 }
John Fastabendf600b692016-12-15 12:13:24 -08002234 if (old_prog)
2235 bpf_prog_put(old_prog);
Jason Wang4e09ff52018-02-28 18:20:04 +08002236 if (netif_running(dev))
2237 virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
John Fastabendf600b692016-12-15 12:13:24 -08002238 }
2239
2240 return 0;
John Fastabend2de2f7f2017-02-02 19:16:29 -08002241
Jason Wang4941d472017-07-19 16:54:48 +08002242err:
2243 for (i = 0; i < vi->max_queue_pairs; i++)
2244 virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
John Fastabend2de2f7f2017-02-02 19:16:29 -08002245 if (prog)
2246 bpf_prog_sub(prog, vi->max_queue_pairs - 1);
2247 return err;
John Fastabendf600b692016-12-15 12:13:24 -08002248}
2249
Martin KaFai Lau5b0e6622017-06-15 17:29:12 -07002250static u32 virtnet_xdp_query(struct net_device *dev)
John Fastabendf600b692016-12-15 12:13:24 -08002251{
2252 struct virtnet_info *vi = netdev_priv(dev);
Martin KaFai Lau5b0e6622017-06-15 17:29:12 -07002253 const struct bpf_prog *xdp_prog;
John Fastabendf600b692016-12-15 12:13:24 -08002254 int i;
2255
2256 for (i = 0; i < vi->max_queue_pairs; i++) {
Martin KaFai Lau5b0e6622017-06-15 17:29:12 -07002257 xdp_prog = rtnl_dereference(vi->rq[i].xdp_prog);
2258 if (xdp_prog)
2259 return xdp_prog->aux->id;
John Fastabendf600b692016-12-15 12:13:24 -08002260 }
Martin KaFai Lau5b0e6622017-06-15 17:29:12 -07002261 return 0;
John Fastabendf600b692016-12-15 12:13:24 -08002262}
2263
Jakub Kicinskif4e63522017-11-03 13:56:16 -07002264static int virtnet_xdp(struct net_device *dev, struct netdev_bpf *xdp)
John Fastabendf600b692016-12-15 12:13:24 -08002265{
2266 switch (xdp->command) {
2267 case XDP_SETUP_PROG:
Jakub Kicinski9861ce02017-04-30 21:46:48 -07002268 return virtnet_xdp_set(dev, xdp->prog, xdp->extack);
John Fastabendf600b692016-12-15 12:13:24 -08002269 case XDP_QUERY_PROG:
Martin KaFai Lau5b0e6622017-06-15 17:29:12 -07002270 xdp->prog_id = virtnet_xdp_query(dev);
2271 xdp->prog_attached = !!xdp->prog_id;
John Fastabendf600b692016-12-15 12:13:24 -08002272 return 0;
2273 default:
2274 return -EINVAL;
2275 }
2276}
2277
Stephen Hemminger76288b42009-01-06 10:44:22 -08002278static const struct net_device_ops virtnet_netdev = {
2279 .ndo_open = virtnet_open,
2280 .ndo_stop = virtnet_close,
2281 .ndo_start_xmit = start_xmit,
2282 .ndo_validate_addr = eth_validate_addr,
Alex Williamson9c46f6d2009-02-04 16:36:34 -08002283 .ndo_set_mac_address = virtnet_set_mac_address,
Alex Williamson2af76982009-02-04 09:02:40 +00002284 .ndo_set_rx_mode = virtnet_set_rx_mode,
stephen hemminger3fa2a1d2011-06-15 06:36:29 +00002285 .ndo_get_stats64 = virtnet_stats,
Alex Williamson1824a982009-05-01 17:31:10 +00002286 .ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid,
2287 .ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid,
Stephen Hemminger76288b42009-01-06 10:44:22 -08002288#ifdef CONFIG_NET_POLL_CONTROLLER
2289 .ndo_poll_controller = virtnet_netpoll,
2290#endif
Jakub Kicinskif4e63522017-11-03 13:56:16 -07002291 .ndo_bpf = virtnet_xdp,
Jason Wang186b3c92017-09-19 17:42:43 +08002292 .ndo_xdp_xmit = virtnet_xdp_xmit,
2293 .ndo_xdp_flush = virtnet_xdp_flush,
Vlad Yasevich2836b4f2017-05-23 13:38:43 -04002294 .ndo_features_check = passthru_features_check,
Stephen Hemminger76288b42009-01-06 10:44:22 -08002295};
2296
Jason Wang586d17c2012-04-11 20:43:52 +00002297static void virtnet_config_changed_work(struct work_struct *work)
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08002298{
Jason Wang586d17c2012-04-11 20:43:52 +00002299 struct virtnet_info *vi =
2300 container_of(work, struct virtnet_info, config_work);
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08002301 u16 v;
2302
Rusty Russell855e0c52013-10-14 18:11:51 +10302303 if (virtio_cread_feature(vi->vdev, VIRTIO_NET_F_STATUS,
2304 struct virtio_net_config, status, &v) < 0)
Michael S. Tsirkin507613b2014-10-15 10:22:30 +10302305 return;
Jason Wang586d17c2012-04-11 20:43:52 +00002306
2307 if (v & VIRTIO_NET_S_ANNOUNCE) {
Amerigo Wangee89bab2012-08-09 22:14:56 +00002308 netdev_notify_peers(vi->dev);
Jason Wang586d17c2012-04-11 20:43:52 +00002309 virtnet_ack_link_announce(vi);
2310 }
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08002311
2312 /* Ignore unknown (future) status bits */
2313 v &= VIRTIO_NET_S_LINK_UP;
2314
2315 if (vi->status == v)
Michael S. Tsirkin507613b2014-10-15 10:22:30 +10302316 return;
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08002317
2318 vi->status = v;
2319
2320 if (vi->status & VIRTIO_NET_S_LINK_UP) {
Jason Baronfaa9b392018-01-05 17:44:54 -05002321 virtnet_update_settings(vi);
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08002322 netif_carrier_on(vi->dev);
Jason Wang986a4f42012-12-07 07:04:56 +00002323 netif_tx_wake_all_queues(vi->dev);
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08002324 } else {
2325 netif_carrier_off(vi->dev);
Jason Wang986a4f42012-12-07 07:04:56 +00002326 netif_tx_stop_all_queues(vi->dev);
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08002327 }
2328}
2329
2330static void virtnet_config_changed(struct virtio_device *vdev)
2331{
2332 struct virtnet_info *vi = vdev->priv;
2333
Tejun Heo3b07e9c2012-08-20 14:51:24 -07002334 schedule_work(&vi->config_work);
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08002335}
2336
Jason Wang986a4f42012-12-07 07:04:56 +00002337static void virtnet_free_queues(struct virtnet_info *vi)
2338{
Andrey Vagind4fb84e2013-12-05 18:36:21 +04002339 int i;
2340
Jason Wangab3971b2015-03-12 13:57:44 +08002341 for (i = 0; i < vi->max_queue_pairs; i++) {
2342 napi_hash_del(&vi->rq[i].napi);
Andrey Vagind4fb84e2013-12-05 18:36:21 +04002343 netif_napi_del(&vi->rq[i].napi);
Willem de Bruijnb92f1e62017-04-24 13:49:27 -04002344 netif_napi_del(&vi->sq[i].napi);
Jason Wangab3971b2015-03-12 13:57:44 +08002345 }
Andrey Vagind4fb84e2013-12-05 18:36:21 +04002346
Eric Dumazet963abe52016-11-15 22:24:12 -08002347 /* We called napi_hash_del() before netif_napi_del(),
2348 * we need to respect an RCU grace period before freeing vi->rq
2349 */
2350 synchronize_net();
2351
Jason Wang986a4f42012-12-07 07:04:56 +00002352 kfree(vi->rq);
2353 kfree(vi->sq);
2354}
2355
John Fastabend473153292017-02-02 19:14:32 -08002356static void _free_receive_bufs(struct virtnet_info *vi)
Jason Wang986a4f42012-12-07 07:04:56 +00002357{
John Fastabendf600b692016-12-15 12:13:24 -08002358 struct bpf_prog *old_prog;
Jason Wang986a4f42012-12-07 07:04:56 +00002359 int i;
2360
2361 for (i = 0; i < vi->max_queue_pairs; i++) {
2362 while (vi->rq[i].pages)
2363 __free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0);
John Fastabendf600b692016-12-15 12:13:24 -08002364
2365 old_prog = rtnl_dereference(vi->rq[i].xdp_prog);
2366 RCU_INIT_POINTER(vi->rq[i].xdp_prog, NULL);
2367 if (old_prog)
2368 bpf_prog_put(old_prog);
Jason Wang986a4f42012-12-07 07:04:56 +00002369 }
John Fastabend473153292017-02-02 19:14:32 -08002370}
2371
2372static void free_receive_bufs(struct virtnet_info *vi)
2373{
2374 rtnl_lock();
2375 _free_receive_bufs(vi);
John Fastabendf600b692016-12-15 12:13:24 -08002376 rtnl_unlock();
Jason Wang986a4f42012-12-07 07:04:56 +00002377}
2378
Michael Daltonfb518792014-01-16 22:23:26 -08002379static void free_receive_page_frags(struct virtnet_info *vi)
2380{
2381 int i;
2382 for (i = 0; i < vi->max_queue_pairs; i++)
2383 if (vi->rq[i].alloc_frag.page)
2384 put_page(vi->rq[i].alloc_frag.page);
2385}
2386
John Fastabendb68df012017-01-25 18:22:48 -08002387static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)
John Fastabend56434a02016-12-15 12:14:13 -08002388{
2389 if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs))
2390 return false;
2391 else if (q < vi->curr_queue_pairs)
2392 return true;
2393 else
2394 return false;
2395}
2396
Jason Wang986a4f42012-12-07 07:04:56 +00002397static void free_unused_bufs(struct virtnet_info *vi)
2398{
2399 void *buf;
2400 int i;
2401
2402 for (i = 0; i < vi->max_queue_pairs; i++) {
2403 struct virtqueue *vq = vi->sq[i].vq;
John Fastabend56434a02016-12-15 12:14:13 -08002404 while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
John Fastabendb68df012017-01-25 18:22:48 -08002405 if (!is_xdp_raw_buffer_queue(vi, i))
John Fastabend56434a02016-12-15 12:14:13 -08002406 dev_kfree_skb(buf);
2407 else
2408 put_page(virt_to_head_page(buf));
2409 }
Jason Wang986a4f42012-12-07 07:04:56 +00002410 }
2411
2412 for (i = 0; i < vi->max_queue_pairs; i++) {
2413 struct virtqueue *vq = vi->rq[i].vq;
2414
2415 while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
Michael Daltonab7db912014-01-16 22:23:27 -08002416 if (vi->mergeable_rx_bufs) {
Michael S. Tsirkin680557c2017-03-06 21:29:47 +02002417 put_page(virt_to_head_page(buf));
Michael Daltonab7db912014-01-16 22:23:27 -08002418 } else if (vi->big_packets) {
Andrey Vaginfa9fac12013-12-05 18:36:20 +04002419 give_pages(&vi->rq[i], buf);
Michael Daltonab7db912014-01-16 22:23:27 -08002420 } else {
Jason Wangf6b10202017-02-21 16:46:28 +08002421 put_page(virt_to_head_page(buf));
Michael Daltonab7db912014-01-16 22:23:27 -08002422 }
Jason Wang986a4f42012-12-07 07:04:56 +00002423 }
Jason Wang986a4f42012-12-07 07:04:56 +00002424 }
2425}
2426
Jason Wange9d74172012-12-07 07:04:55 +00002427static void virtnet_del_vqs(struct virtnet_info *vi)
2428{
2429 struct virtio_device *vdev = vi->vdev;
2430
Wanlong Gao8898c212013-01-24 23:51:30 +00002431 virtnet_clean_affinity(vi, -1);
Jason Wang986a4f42012-12-07 07:04:56 +00002432
Jason Wange9d74172012-12-07 07:04:55 +00002433 vdev->config->del_vqs(vdev);
Jason Wang986a4f42012-12-07 07:04:56 +00002434
2435 virtnet_free_queues(vi);
2436}
2437
Michael S. Tsirkind85b758f72017-03-09 02:21:21 +02002438/* How large should a single buffer be so a queue full of these can fit at
2439 * least one full packet?
2440 * Logic below assumes the mergeable buffer header is used.
2441 */
2442static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqueue *vq)
2443{
2444 const unsigned int hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
2445 unsigned int rq_size = virtqueue_get_vring_size(vq);
2446 unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu;
2447 unsigned int buf_len = hdr_len + ETH_HLEN + VLAN_HLEN + packet_len;
2448 unsigned int min_buf_len = DIV_ROUND_UP(buf_len, rq_size);
2449
Michael S. Tsirkinf0c31922017-06-02 17:54:33 +03002450 return max(max(min_buf_len, hdr_len) - hdr_len,
2451 (unsigned int)GOOD_PACKET_LEN);
Michael S. Tsirkind85b758f72017-03-09 02:21:21 +02002452}
2453
Jason Wang986a4f42012-12-07 07:04:56 +00002454static int virtnet_find_vqs(struct virtnet_info *vi)
2455{
2456 vq_callback_t **callbacks;
2457 struct virtqueue **vqs;
2458 int ret = -ENOMEM;
2459 int i, total_vqs;
2460 const char **names;
Michael S. Tsirkind45b8972017-03-06 20:31:21 +02002461 bool *ctx;
Jason Wang986a4f42012-12-07 07:04:56 +00002462
2463 /* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by
2464 * possible N-1 RX/TX queue pairs used in multiqueue mode, followed by
2465 * possible control vq.
2466 */
2467 total_vqs = vi->max_queue_pairs * 2 +
2468 virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ);
2469
2470 /* Allocate space for find_vqs parameters */
2471 vqs = kzalloc(total_vqs * sizeof(*vqs), GFP_KERNEL);
2472 if (!vqs)
2473 goto err_vq;
2474 callbacks = kmalloc(total_vqs * sizeof(*callbacks), GFP_KERNEL);
2475 if (!callbacks)
2476 goto err_callback;
2477 names = kmalloc(total_vqs * sizeof(*names), GFP_KERNEL);
2478 if (!names)
2479 goto err_names;
Jason Wang192f68c2017-07-19 16:54:47 +08002480 if (!vi->big_packets || vi->mergeable_rx_bufs) {
Michael S. Tsirkind45b8972017-03-06 20:31:21 +02002481 ctx = kzalloc(total_vqs * sizeof(*ctx), GFP_KERNEL);
2482 if (!ctx)
2483 goto err_ctx;
2484 } else {
2485 ctx = NULL;
2486 }
Jason Wang986a4f42012-12-07 07:04:56 +00002487
2488 /* Parameters for control virtqueue, if any */
2489 if (vi->has_cvq) {
2490 callbacks[total_vqs - 1] = NULL;
2491 names[total_vqs - 1] = "control";
2492 }
2493
2494 /* Allocate/initialize parameters for send/receive virtqueues */
2495 for (i = 0; i < vi->max_queue_pairs; i++) {
2496 callbacks[rxq2vq(i)] = skb_recv_done;
2497 callbacks[txq2vq(i)] = skb_xmit_done;
2498 sprintf(vi->rq[i].name, "input.%d", i);
2499 sprintf(vi->sq[i].name, "output.%d", i);
2500 names[rxq2vq(i)] = vi->rq[i].name;
2501 names[txq2vq(i)] = vi->sq[i].name;
Michael S. Tsirkind45b8972017-03-06 20:31:21 +02002502 if (ctx)
2503 ctx[rxq2vq(i)] = true;
Jason Wang986a4f42012-12-07 07:04:56 +00002504 }
2505
2506 ret = vi->vdev->config->find_vqs(vi->vdev, total_vqs, vqs, callbacks,
Michael S. Tsirkind45b8972017-03-06 20:31:21 +02002507 names, ctx, NULL);
Jason Wang986a4f42012-12-07 07:04:56 +00002508 if (ret)
2509 goto err_find;
2510
2511 if (vi->has_cvq) {
2512 vi->cvq = vqs[total_vqs - 1];
2513 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
Patrick McHardyf6469682013-04-19 02:04:27 +00002514 vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
Jason Wang986a4f42012-12-07 07:04:56 +00002515 }
2516
2517 for (i = 0; i < vi->max_queue_pairs; i++) {
2518 vi->rq[i].vq = vqs[rxq2vq(i)];
Michael S. Tsirkind85b758f72017-03-09 02:21:21 +02002519 vi->rq[i].min_buf_len = mergeable_min_buf_len(vi, vi->rq[i].vq);
Jason Wang986a4f42012-12-07 07:04:56 +00002520 vi->sq[i].vq = vqs[txq2vq(i)];
2521 }
2522
2523 kfree(names);
2524 kfree(callbacks);
2525 kfree(vqs);
Jason Wang55281622017-07-07 19:56:09 +08002526 kfree(ctx);
Jason Wang986a4f42012-12-07 07:04:56 +00002527
2528 return 0;
2529
2530err_find:
Michael S. Tsirkind45b8972017-03-06 20:31:21 +02002531 kfree(ctx);
2532err_ctx:
Jason Wang986a4f42012-12-07 07:04:56 +00002533 kfree(names);
2534err_names:
2535 kfree(callbacks);
2536err_callback:
2537 kfree(vqs);
2538err_vq:
2539 return ret;
2540}
2541
2542static int virtnet_alloc_queues(struct virtnet_info *vi)
2543{
2544 int i;
2545
2546 vi->sq = kzalloc(sizeof(*vi->sq) * vi->max_queue_pairs, GFP_KERNEL);
2547 if (!vi->sq)
2548 goto err_sq;
2549 vi->rq = kzalloc(sizeof(*vi->rq) * vi->max_queue_pairs, GFP_KERNEL);
Amerigo Wang008d4272012-12-10 02:24:08 +00002550 if (!vi->rq)
Jason Wang986a4f42012-12-07 07:04:56 +00002551 goto err_rq;
2552
2553 INIT_DELAYED_WORK(&vi->refill, refill_work);
2554 for (i = 0; i < vi->max_queue_pairs; i++) {
2555 vi->rq[i].pages = NULL;
2556 netif_napi_add(vi->dev, &vi->rq[i].napi, virtnet_poll,
2557 napi_weight);
Willem de Bruijn1d11e732017-04-27 20:37:58 -04002558 netif_tx_napi_add(vi->dev, &vi->sq[i].napi, virtnet_poll_tx,
2559 napi_tx ? napi_weight : 0);
Jason Wang986a4f42012-12-07 07:04:56 +00002560
2561 sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));
Johannes Berg5377d7582015-08-19 09:48:40 +02002562 ewma_pkt_len_init(&vi->rq[i].mrg_avg_pkt_len);
Jason Wang986a4f42012-12-07 07:04:56 +00002563 sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09002564
2565 u64_stats_init(&vi->rq[i].stats.syncp);
2566 u64_stats_init(&vi->sq[i].stats.syncp);
Jason Wang986a4f42012-12-07 07:04:56 +00002567 }
2568
2569 return 0;
2570
2571err_rq:
2572 kfree(vi->sq);
2573err_sq:
2574 return -ENOMEM;
Jason Wange9d74172012-12-07 07:04:55 +00002575}
2576
Amit Shah3f9c10b2011-12-22 16:58:31 +05302577static int init_vqs(struct virtnet_info *vi)
2578{
Jason Wang986a4f42012-12-07 07:04:56 +00002579 int ret;
Amit Shah3f9c10b2011-12-22 16:58:31 +05302580
Jason Wang986a4f42012-12-07 07:04:56 +00002581 /* Allocate send & receive queues */
2582 ret = virtnet_alloc_queues(vi);
2583 if (ret)
2584 goto err;
Amit Shah3f9c10b2011-12-22 16:58:31 +05302585
Jason Wang986a4f42012-12-07 07:04:56 +00002586 ret = virtnet_find_vqs(vi);
2587 if (ret)
2588 goto err_free;
Amit Shah3f9c10b2011-12-22 16:58:31 +05302589
Wanlong Gao47be2472013-01-24 23:51:29 +00002590 get_online_cpus();
Wanlong Gao8898c212013-01-24 23:51:30 +00002591 virtnet_set_affinity(vi);
Wanlong Gao47be2472013-01-24 23:51:29 +00002592 put_online_cpus();
2593
Amit Shah3f9c10b2011-12-22 16:58:31 +05302594 return 0;
Jason Wang986a4f42012-12-07 07:04:56 +00002595
2596err_free:
2597 virtnet_free_queues(vi);
2598err:
2599 return ret;
Amit Shah3f9c10b2011-12-22 16:58:31 +05302600}
2601
Michael Daltonfbf28d72014-01-16 22:23:30 -08002602#ifdef CONFIG_SYSFS
2603static ssize_t mergeable_rx_buffer_size_show(struct netdev_rx_queue *queue,
stephen hemminger718ad682017-08-18 13:46:24 -07002604 char *buf)
Michael Daltonfbf28d72014-01-16 22:23:30 -08002605{
2606 struct virtnet_info *vi = netdev_priv(queue->dev);
2607 unsigned int queue_index = get_netdev_rx_queue_index(queue);
Jason Wang3cc81a92018-03-02 17:29:14 +08002608 unsigned int headroom = virtnet_get_headroom(vi);
2609 unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
Johannes Berg5377d7582015-08-19 09:48:40 +02002610 struct ewma_pkt_len *avg;
Michael Daltonfbf28d72014-01-16 22:23:30 -08002611
2612 BUG_ON(queue_index >= vi->max_queue_pairs);
2613 avg = &vi->rq[queue_index].mrg_avg_pkt_len;
Michael S. Tsirkind85b758f72017-03-09 02:21:21 +02002614 return sprintf(buf, "%u\n",
Jason Wang3cc81a92018-03-02 17:29:14 +08002615 get_mergeable_buf_len(&vi->rq[queue_index], avg,
2616 SKB_DATA_ALIGN(headroom + tailroom)));
Michael Daltonfbf28d72014-01-16 22:23:30 -08002617}
2618
2619static struct rx_queue_attribute mergeable_rx_buffer_size_attribute =
2620 __ATTR_RO(mergeable_rx_buffer_size);
2621
2622static struct attribute *virtio_net_mrg_rx_attrs[] = {
2623 &mergeable_rx_buffer_size_attribute.attr,
2624 NULL
2625};
2626
2627static const struct attribute_group virtio_net_mrg_rx_group = {
2628 .name = "virtio_net",
2629 .attrs = virtio_net_mrg_rx_attrs
2630};
2631#endif
2632
Jason Wang892d6eb2014-11-20 17:03:05 +08002633static bool virtnet_fail_on_feature(struct virtio_device *vdev,
2634 unsigned int fbit,
2635 const char *fname, const char *dname)
2636{
2637 if (!virtio_has_feature(vdev, fbit))
2638 return false;
2639
2640 dev_err(&vdev->dev, "device advertises feature %s but not %s",
2641 fname, dname);
2642
2643 return true;
2644}
2645
2646#define VIRTNET_FAIL_ON(vdev, fbit, dbit) \
2647 virtnet_fail_on_feature(vdev, fbit, #fbit, dbit)
2648
2649static bool virtnet_validate_features(struct virtio_device *vdev)
2650{
2651 if (!virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) &&
2652 (VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_RX,
2653 "VIRTIO_NET_F_CTRL_VQ") ||
2654 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_VLAN,
2655 "VIRTIO_NET_F_CTRL_VQ") ||
2656 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE,
2657 "VIRTIO_NET_F_CTRL_VQ") ||
2658 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_MQ, "VIRTIO_NET_F_CTRL_VQ") ||
2659 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR,
2660 "VIRTIO_NET_F_CTRL_VQ"))) {
2661 return false;
2662 }
2663
2664 return true;
2665}
2666
Jarod Wilsond0c2c992016-10-20 13:55:21 -04002667#define MIN_MTU ETH_MIN_MTU
2668#define MAX_MTU ETH_MAX_MTU
2669
Michael S. Tsirkinfe36cbe2017-03-29 19:09:14 +03002670static int virtnet_validate(struct virtio_device *vdev)
Rusty Russell296f96f2007-10-22 11:03:37 +10002671{
Michael S. Tsirkin6ba42242015-01-12 16:23:37 +02002672 if (!vdev->config->get) {
2673 dev_err(&vdev->dev, "%s failure: config access disabled\n",
2674 __func__);
2675 return -EINVAL;
2676 }
2677
Jason Wang892d6eb2014-11-20 17:03:05 +08002678 if (!virtnet_validate_features(vdev))
2679 return -EINVAL;
2680
Michael S. Tsirkinfe36cbe2017-03-29 19:09:14 +03002681 if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) {
2682 int mtu = virtio_cread16(vdev,
2683 offsetof(struct virtio_net_config,
2684 mtu));
2685 if (mtu < MIN_MTU)
2686 __virtio_clear_bit(vdev, VIRTIO_NET_F_MTU);
2687 }
2688
2689 return 0;
2690}
2691
2692static int virtnet_probe(struct virtio_device *vdev)
2693{
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09002694 int i, err = -ENOMEM;
Michael S. Tsirkinfe36cbe2017-03-29 19:09:14 +03002695 struct net_device *dev;
2696 struct virtnet_info *vi;
2697 u16 max_queue_pairs;
2698 int mtu;
2699
Jason Wang986a4f42012-12-07 07:04:56 +00002700 /* Find if host supports multiqueue virtio_net device */
Rusty Russell855e0c52013-10-14 18:11:51 +10302701 err = virtio_cread_feature(vdev, VIRTIO_NET_F_MQ,
2702 struct virtio_net_config,
2703 max_virtqueue_pairs, &max_queue_pairs);
Jason Wang986a4f42012-12-07 07:04:56 +00002704
2705 /* We need at least 2 queue's */
2706 if (err || max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
2707 max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX ||
2708 !virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
2709 max_queue_pairs = 1;
Rusty Russell296f96f2007-10-22 11:03:37 +10002710
2711 /* Allocate ourselves a network device with room for our info */
Jason Wang986a4f42012-12-07 07:04:56 +00002712 dev = alloc_etherdev_mq(sizeof(struct virtnet_info), max_queue_pairs);
Rusty Russell296f96f2007-10-22 11:03:37 +10002713 if (!dev)
2714 return -ENOMEM;
2715
2716 /* Set up network device as normal. */
Jiri Pirkof2f2c8b2012-06-29 05:10:06 +00002717 dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE;
Stephen Hemminger76288b42009-01-06 10:44:22 -08002718 dev->netdev_ops = &virtnet_netdev;
Rusty Russell296f96f2007-10-22 11:03:37 +10002719 dev->features = NETIF_F_HIGHDMA;
stephen hemminger3fa2a1d2011-06-15 06:36:29 +00002720
Wilfried Klaebe7ad24ea2014-05-11 00:12:32 +00002721 dev->ethtool_ops = &virtnet_ethtool_ops;
Rusty Russell296f96f2007-10-22 11:03:37 +10002722 SET_NETDEV_DEV(dev, &vdev->dev);
2723
2724 /* Do we support "hardware" checksums? */
Michał Mirosław98e778c2011-03-31 01:01:35 +00002725 if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) {
Rusty Russell296f96f2007-10-22 11:03:37 +10002726 /* This opens up the world of extra features. */
Jason Wang48900cb2015-08-05 10:34:04 +08002727 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_SG;
Michał Mirosław98e778c2011-03-31 01:01:35 +00002728 if (csum)
Jason Wang48900cb2015-08-05 10:34:04 +08002729 dev->features |= NETIF_F_HW_CSUM | NETIF_F_SG;
Michał Mirosław98e778c2011-03-31 01:01:35 +00002730
2731 if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
David S. Millere078de02017-07-03 06:37:32 -07002732 dev->hw_features |= NETIF_F_TSO
Rusty Russell34a48572008-02-04 23:50:02 -05002733 | NETIF_F_TSO_ECN | NETIF_F_TSO6;
2734 }
Rusty Russell5539ae962008-05-02 21:50:46 -05002735 /* Individual feature bits: what can host handle? */
Michał Mirosław98e778c2011-03-31 01:01:35 +00002736 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4))
2737 dev->hw_features |= NETIF_F_TSO;
2738 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6))
2739 dev->hw_features |= NETIF_F_TSO6;
2740 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
2741 dev->hw_features |= NETIF_F_TSO_ECN;
Michał Mirosław98e778c2011-03-31 01:01:35 +00002742
Jason Wang41f2f122014-12-24 11:03:52 +08002743 dev->features |= NETIF_F_GSO_ROBUST;
2744
Michał Mirosław98e778c2011-03-31 01:01:35 +00002745 if (gso)
David S. Millere078de02017-07-03 06:37:32 -07002746 dev->features |= dev->hw_features & NETIF_F_ALL_TSO;
Michał Mirosław98e778c2011-03-31 01:01:35 +00002747 /* (!csum && gso) case will be fixed by register_netdev() */
Rusty Russell296f96f2007-10-22 11:03:37 +10002748 }
Thomas Huth4f491292013-08-27 17:09:02 +02002749 if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_CSUM))
2750 dev->features |= NETIF_F_RXCSUM;
Rusty Russell296f96f2007-10-22 11:03:37 +10002751
Jason Wang4fda8302013-04-10 23:32:21 +00002752 dev->vlan_features = dev->features;
2753
Jarod Wilsond0c2c992016-10-20 13:55:21 -04002754 /* MTU range: 68 - 65535 */
2755 dev->min_mtu = MIN_MTU;
2756 dev->max_mtu = MAX_MTU;
2757
Rusty Russell296f96f2007-10-22 11:03:37 +10002758 /* Configuration may specify what MAC to use. Otherwise random. */
Rusty Russell855e0c52013-10-14 18:11:51 +10302759 if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC))
2760 virtio_cread_bytes(vdev,
2761 offsetof(struct virtio_net_config, mac),
2762 dev->dev_addr, dev->addr_len);
2763 else
Danny Kukawkaf2cedb62012-02-15 06:45:39 +00002764 eth_hw_addr_random(dev);
Rusty Russell296f96f2007-10-22 11:03:37 +10002765
2766 /* Set up our device-specific information */
2767 vi = netdev_priv(dev);
Rusty Russell296f96f2007-10-22 11:03:37 +10002768 vi->dev = dev;
2769 vi->vdev = vdev;
Christian Borntraegerd9d5dcc2008-02-18 10:02:51 +01002770 vdev->priv = vi;
John Stultz827da442013-10-07 15:51:58 -07002771
Jason Wang586d17c2012-04-11 20:43:52 +00002772 INIT_WORK(&vi->config_work, virtnet_config_changed_work);
Rusty Russell296f96f2007-10-22 11:03:37 +10002773
Herbert Xu97402b92008-04-18 11:24:27 +08002774 /* If we can receive ANY GSO packets, we must allocate large ones. */
Joe Perches8e95a202009-12-03 07:58:21 +00002775 if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
2776 virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) ||
Vlad Yaseviche3e3c422015-02-03 16:36:17 -05002777 virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN) ||
2778 virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_UFO))
Herbert Xu97402b92008-04-18 11:24:27 +08002779 vi->big_packets = true;
2780
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -08002781 if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
2782 vi->mergeable_rx_bufs = true;
2783
Michael S. Tsirkind04302b2014-10-24 00:24:03 +03002784 if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) ||
2785 virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
Michael S. Tsirkin012873d2014-10-24 16:55:57 +03002786 vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
2787 else
2788 vi->hdr_len = sizeof(struct virtio_net_hdr);
2789
Michael S. Tsirkin75993302015-07-15 15:26:19 +03002790 if (virtio_has_feature(vdev, VIRTIO_F_ANY_LAYOUT) ||
2791 virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
Michael S. Tsirkine7428e92013-07-25 10:20:23 +09302792 vi->any_header_sg = true;
2793
Jason Wang986a4f42012-12-07 07:04:56 +00002794 if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
2795 vi->has_cvq = true;
2796
Aaron Conole14de9d12016-06-03 16:57:12 -04002797 if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) {
2798 mtu = virtio_cread16(vdev,
2799 offsetof(struct virtio_net_config,
2800 mtu));
Aaron Conole93a205e2016-10-25 16:12:12 -04002801 if (mtu < dev->min_mtu) {
Michael S. Tsirkinfe36cbe2017-03-29 19:09:14 +03002802 /* Should never trigger: MTU was previously validated
2803 * in virtnet_validate.
2804 */
2805 dev_err(&vdev->dev, "device MTU appears to have changed "
2806 "it is now %d < %d", mtu, dev->min_mtu);
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09002807 goto free;
Aaron Conole93a205e2016-10-25 16:12:12 -04002808 }
Michael S. Tsirkin2e123b42017-03-08 02:14:25 +02002809
Michael S. Tsirkinfe36cbe2017-03-29 19:09:14 +03002810 dev->mtu = mtu;
2811 dev->max_mtu = mtu;
2812
Michael S. Tsirkin2e123b42017-03-08 02:14:25 +02002813 /* TODO: size buffers correctly in this case. */
2814 if (dev->mtu > ETH_DATA_LEN)
2815 vi->big_packets = true;
Aaron Conole14de9d12016-06-03 16:57:12 -04002816 }
2817
Michael S. Tsirkin012873d2014-10-24 16:55:57 +03002818 if (vi->any_header_sg)
2819 dev->needed_headroom = vi->hdr_len;
Zhangjie \(HZ\)6ebbc1a2014-04-29 18:43:22 +08002820
Jason Wang44900012016-11-25 12:37:26 +08002821 /* Enable multiqueue by default */
2822 if (num_online_cpus() >= max_queue_pairs)
2823 vi->curr_queue_pairs = max_queue_pairs;
2824 else
2825 vi->curr_queue_pairs = num_online_cpus();
Jason Wang986a4f42012-12-07 07:04:56 +00002826 vi->max_queue_pairs = max_queue_pairs;
2827
2828 /* Allocate/initialize the rx/tx queues, and invoke find_vqs */
Amit Shah3f9c10b2011-12-22 16:58:31 +05302829 err = init_vqs(vi);
Michael S. Tsirkind2a7ddd2009-06-12 22:16:36 -06002830 if (err)
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09002831 goto free;
Michael S. Tsirkind2a7ddd2009-06-12 22:16:36 -06002832
Michael Daltonfbf28d72014-01-16 22:23:30 -08002833#ifdef CONFIG_SYSFS
2834 if (vi->mergeable_rx_bufs)
2835 dev->sysfs_rx_queue_group = &virtio_net_mrg_rx_group;
2836#endif
Zhi Yong Wu0f13b66b2013-11-18 21:19:27 +08002837 netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs);
2838 netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs);
Jason Wang986a4f42012-12-07 07:04:56 +00002839
Nikolay Aleksandrov16032be2016-02-03 04:04:37 +01002840 virtnet_init_settings(dev);
2841
Rusty Russell296f96f2007-10-22 11:03:37 +10002842 err = register_netdev(dev);
2843 if (err) {
2844 pr_debug("virtio_net: registering device failed\n");
Michael S. Tsirkind2a7ddd2009-06-12 22:16:36 -06002845 goto free_vqs;
Rusty Russell296f96f2007-10-22 11:03:37 +10002846 }
Rusty Russellb3369c12008-02-04 23:50:02 -05002847
Michael S. Tsirkin4baf1e32014-10-15 10:22:30 +10302848 virtio_device_ready(vdev);
2849
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +02002850 err = virtnet_cpu_notif_add(vi);
Wanlong Gao8de4b2f2013-01-24 23:51:31 +00002851 if (err) {
2852 pr_debug("virtio_net: registering cpu notifier failed\n");
wangyunjianf00e35e2016-05-31 11:52:43 +08002853 goto free_unregister_netdev;
Wanlong Gao8de4b2f2013-01-24 23:51:31 +00002854 }
2855
Jason Wanga2208712016-12-13 14:23:05 +08002856 virtnet_set_queues(vi, vi->curr_queue_pairs);
Jason Wang44900012016-11-25 12:37:26 +08002857
Jason Wang167c25e2010-11-10 14:45:41 +00002858 /* Assume link up if device can't report link status,
2859 otherwise get link status from config. */
2860 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {
2861 netif_carrier_off(dev);
Tejun Heo3b07e9c2012-08-20 14:51:24 -07002862 schedule_work(&vi->config_work);
Jason Wang167c25e2010-11-10 14:45:41 +00002863 } else {
2864 vi->status = VIRTIO_NET_S_LINK_UP;
Jason Baronfaa9b392018-01-05 17:44:54 -05002865 virtnet_update_settings(vi);
Jason Wang167c25e2010-11-10 14:45:41 +00002866 netif_carrier_on(dev);
2867 }
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08002868
Jason Wang3f935222017-07-19 16:54:49 +08002869 for (i = 0; i < ARRAY_SIZE(guest_offloads); i++)
2870 if (virtio_has_feature(vi->vdev, guest_offloads[i]))
2871 set_bit(guest_offloads[i], &vi->guest_offloads);
2872
Jason Wang986a4f42012-12-07 07:04:56 +00002873 pr_debug("virtnet: registered device %s with %d RX and TX vq's\n",
2874 dev->name, max_queue_pairs);
2875
Rusty Russell296f96f2007-10-22 11:03:37 +10002876 return 0;
2877
wangyunjianf00e35e2016-05-31 11:52:43 +08002878free_unregister_netdev:
Michael S. Tsirkin02465552014-10-15 10:22:31 +10302879 vi->vdev->config->reset(vdev);
2880
Rusty Russellb3369c12008-02-04 23:50:02 -05002881 unregister_netdev(dev);
Michael S. Tsirkind2a7ddd2009-06-12 22:16:36 -06002882free_vqs:
Jason Wang986a4f42012-12-07 07:04:56 +00002883 cancel_delayed_work_sync(&vi->refill);
Michael Daltonfb518792014-01-16 22:23:26 -08002884 free_receive_page_frags(vi);
Jason Wange9d74172012-12-07 07:04:55 +00002885 virtnet_del_vqs(vi);
Rusty Russell296f96f2007-10-22 11:03:37 +10002886free:
2887 free_netdev(dev);
2888 return err;
2889}
2890
Amit Shah04486ed2011-12-22 16:58:32 +05302891static void remove_vq_common(struct virtnet_info *vi)
Rusty Russell296f96f2007-10-22 11:03:37 +10002892{
Amit Shah04486ed2011-12-22 16:58:32 +05302893 vi->vdev->config->reset(vi->vdev);
Shirley Ma830a8a92010-02-08 14:14:42 +00002894
2895 /* Free unused buffers in both send and recv, if any. */
Shirley Ma9ab86bb2010-01-29 03:20:04 +00002896 free_unused_bufs(vi);
Rusty Russellfb6813f2008-07-25 12:06:01 -05002897
Jason Wang986a4f42012-12-07 07:04:56 +00002898 free_receive_bufs(vi);
Michael S. Tsirkind2a7ddd2009-06-12 22:16:36 -06002899
Michael Daltonfb518792014-01-16 22:23:26 -08002900 free_receive_page_frags(vi);
2901
Jason Wang986a4f42012-12-07 07:04:56 +00002902 virtnet_del_vqs(vi);
Amit Shah04486ed2011-12-22 16:58:32 +05302903}
2904
Bill Pemberton8cc085d2012-12-03 09:24:15 -05002905static void virtnet_remove(struct virtio_device *vdev)
Amit Shah04486ed2011-12-22 16:58:32 +05302906{
2907 struct virtnet_info *vi = vdev->priv;
2908
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +02002909 virtnet_cpu_notif_remove(vi);
Wanlong Gao8de4b2f2013-01-24 23:51:31 +00002910
Michael S. Tsirkin102a2782014-10-15 10:22:29 +10302911 /* Make sure no work handler is accessing the device. */
2912 flush_work(&vi->config_work);
Jason Wang586d17c2012-04-11 20:43:52 +00002913
Amit Shah04486ed2011-12-22 16:58:32 +05302914 unregister_netdev(vi->dev);
2915
2916 remove_vq_common(vi);
Rusty Russellfb6813f2008-07-25 12:06:01 -05002917
Rusty Russell74b25532007-11-19 11:20:42 -05002918 free_netdev(vi->dev);
Rusty Russell296f96f2007-10-22 11:03:37 +10002919}
2920
Arnd Bergmann67a75192017-07-25 17:35:50 +02002921static __maybe_unused int virtnet_freeze(struct virtio_device *vdev)
Amit Shah0741bcb2011-12-22 16:58:33 +05302922{
2923 struct virtnet_info *vi = vdev->priv;
2924
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +02002925 virtnet_cpu_notif_remove(vi);
John Fastabend9fe7bfc2017-02-02 19:16:01 -08002926 virtnet_freeze_down(vdev);
Amit Shah0741bcb2011-12-22 16:58:33 +05302927 remove_vq_common(vi);
2928
2929 return 0;
2930}
2931
Arnd Bergmann67a75192017-07-25 17:35:50 +02002932static __maybe_unused int virtnet_restore(struct virtio_device *vdev)
Amit Shah0741bcb2011-12-22 16:58:33 +05302933{
2934 struct virtnet_info *vi = vdev->priv;
John Fastabend9fe7bfc2017-02-02 19:16:01 -08002935 int err;
Amit Shah0741bcb2011-12-22 16:58:33 +05302936
John Fastabend9fe7bfc2017-02-02 19:16:01 -08002937 err = virtnet_restore_up(vdev);
Amit Shah0741bcb2011-12-22 16:58:33 +05302938 if (err)
2939 return err;
Jason Wang986a4f42012-12-07 07:04:56 +00002940 virtnet_set_queues(vi, vi->curr_queue_pairs);
2941
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +02002942 err = virtnet_cpu_notif_add(vi);
Jason Wangec9debb2013-10-29 15:11:07 +08002943 if (err)
2944 return err;
2945
Amit Shah0741bcb2011-12-22 16:58:33 +05302946 return 0;
2947}
Amit Shah0741bcb2011-12-22 16:58:33 +05302948
Rusty Russell296f96f2007-10-22 11:03:37 +10002949static struct virtio_device_id id_table[] = {
2950 { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
2951 { 0 },
2952};
2953
Michael S. Tsirkinf3358502016-11-04 12:55:36 +02002954#define VIRTNET_FEATURES \
2955 VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM, \
2956 VIRTIO_NET_F_MAC, \
2957 VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6, \
2958 VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6, \
2959 VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO, \
2960 VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ, \
2961 VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN, \
2962 VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \
2963 VIRTIO_NET_F_CTRL_MAC_ADDR, \
Jason Baronfaa9b392018-01-05 17:44:54 -05002964 VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \
2965 VIRTIO_NET_F_SPEED_DUPLEX
Michael S. Tsirkinf3358502016-11-04 12:55:36 +02002966
Rusty Russellc45a6812008-05-02 21:50:50 -05002967static unsigned int features[] = {
Michael S. Tsirkinf3358502016-11-04 12:55:36 +02002968 VIRTNET_FEATURES,
2969};
2970
2971static unsigned int features_legacy[] = {
2972 VIRTNET_FEATURES,
2973 VIRTIO_NET_F_GSO,
Michael S. Tsirkine7428e92013-07-25 10:20:23 +09302974 VIRTIO_F_ANY_LAYOUT,
Rusty Russellc45a6812008-05-02 21:50:50 -05002975};
2976
Uwe Kleine-König22402522009-11-05 01:32:44 -08002977static struct virtio_driver virtio_net_driver = {
Rusty Russellc45a6812008-05-02 21:50:50 -05002978 .feature_table = features,
2979 .feature_table_size = ARRAY_SIZE(features),
Michael S. Tsirkinf3358502016-11-04 12:55:36 +02002980 .feature_table_legacy = features_legacy,
2981 .feature_table_size_legacy = ARRAY_SIZE(features_legacy),
Rusty Russell296f96f2007-10-22 11:03:37 +10002982 .driver.name = KBUILD_MODNAME,
2983 .driver.owner = THIS_MODULE,
2984 .id_table = id_table,
Michael S. Tsirkinfe36cbe2017-03-29 19:09:14 +03002985 .validate = virtnet_validate,
Rusty Russell296f96f2007-10-22 11:03:37 +10002986 .probe = virtnet_probe,
Bill Pemberton8cc085d2012-12-03 09:24:15 -05002987 .remove = virtnet_remove,
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08002988 .config_changed = virtnet_config_changed,
Aaron Lu89107002013-09-17 09:25:23 +09302989#ifdef CONFIG_PM_SLEEP
Amit Shah0741bcb2011-12-22 16:58:33 +05302990 .freeze = virtnet_freeze,
2991 .restore = virtnet_restore,
2992#endif
Rusty Russell296f96f2007-10-22 11:03:37 +10002993};
2994
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +02002995static __init int virtio_net_driver_init(void)
2996{
2997 int ret;
2998
Thomas Gleixner73c1b412016-12-21 20:19:54 +01002999 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "virtio/net:online",
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +02003000 virtnet_cpu_online,
3001 virtnet_cpu_down_prep);
3002 if (ret < 0)
3003 goto out;
3004 virtionet_online = ret;
Thomas Gleixner73c1b412016-12-21 20:19:54 +01003005 ret = cpuhp_setup_state_multi(CPUHP_VIRT_NET_DEAD, "virtio/net:dead",
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +02003006 NULL, virtnet_cpu_dead);
3007 if (ret)
3008 goto err_dead;
3009
3010 ret = register_virtio_driver(&virtio_net_driver);
3011 if (ret)
3012 goto err_virtio;
3013 return 0;
3014err_virtio:
3015 cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD);
3016err_dead:
3017 cpuhp_remove_multi_state(virtionet_online);
3018out:
3019 return ret;
3020}
3021module_init(virtio_net_driver_init);
3022
3023static __exit void virtio_net_driver_exit(void)
3024{
Andrew Jonescfa0ebc2017-07-24 15:38:32 +02003025 unregister_virtio_driver(&virtio_net_driver);
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +02003026 cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD);
3027 cpuhp_remove_multi_state(virtionet_online);
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +02003028}
3029module_exit(virtio_net_driver_exit);
Rusty Russell296f96f2007-10-22 11:03:37 +10003030
3031MODULE_DEVICE_TABLE(virtio, id_table);
3032MODULE_DESCRIPTION("Virtio network driver");
3033MODULE_LICENSE("GPL");