blob: 07015b1c42c6300b9a47e415c5a595fdc693cc75 [file] [log] [blame]
Hank Janssenfceaf242009-07-13 15:34:54 -07001/*
Hank Janssenfceaf242009-07-13 15:34:54 -07002 * Copyright (c) 2009, Microsoft Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
Jeff Kirsheradf8d3f2013-12-06 06:28:47 -080014 * this program; if not, see <http://www.gnu.org/licenses/>.
Hank Janssenfceaf242009-07-13 15:34:54 -070015 *
16 * Authors:
Haiyang Zhangd0e94d12009-11-23 17:00:22 +000017 * Haiyang Zhang <haiyangz@microsoft.com>
Hank Janssenfceaf242009-07-13 15:34:54 -070018 * Hank Janssen <hjanssen@microsoft.com>
Hank Janssenfceaf242009-07-13 15:34:54 -070019 */
Hank Jansseneb335bc2011-03-29 13:58:48 -070020#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
21
Hank Janssenfceaf242009-07-13 15:34:54 -070022#include <linux/init.h>
K. Y. Srinivasan9079ce62011-06-16 13:16:37 -070023#include <linux/atomic.h>
Hank Janssenfceaf242009-07-13 15:34:54 -070024#include <linux/module.h>
25#include <linux/highmem.h>
26#include <linux/device.h>
Hank Janssenfceaf242009-07-13 15:34:54 -070027#include <linux/io.h>
Hank Janssenfceaf242009-07-13 15:34:54 -070028#include <linux/delay.h>
29#include <linux/netdevice.h>
30#include <linux/inetdevice.h>
31#include <linux/etherdevice.h>
32#include <linux/skbuff.h>
Haiyang Zhangc802db12013-05-28 06:15:56 +000033#include <linux/if_vlan.h>
Hank Janssenfceaf242009-07-13 15:34:54 -070034#include <linux/in.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090035#include <linux/slab.h>
stephen hemminger27f5aa92017-07-24 10:57:29 -070036#include <linux/rtnetlink.h>
stephen hemminger0c195562017-08-01 19:58:53 -070037#include <linux/netpoll.h>
stephen hemminger27f5aa92017-07-24 10:57:29 -070038
Hank Janssenfceaf242009-07-13 15:34:54 -070039#include <net/arp.h>
40#include <net/route.h>
41#include <net/sock.h>
42#include <net/pkt_sched.h>
Michael Kelley8eb1b3c2017-05-30 11:36:56 -070043#include <net/checksum.h>
44#include <net/ip6_checksum.h>
K. Y. Srinivasan3f335ea2011-05-12 19:34:15 -070045
K. Y. Srinivasan5ca72522011-05-12 19:34:37 -070046#include "hyperv_net.h"
Hank Janssenfceaf242009-07-13 15:34:54 -070047
Haiyang Zhangfa85a6c2012-07-25 08:08:41 +000048#define RING_SIZE_MIN 64
Vitaly Kuznetsov27a70af2015-11-27 11:39:55 +010049#define LINKCHANGE_INT (2 * HZ)
stephen hemminger6123c662017-08-09 17:46:03 -070050#define VF_TAKEOVER_INT (HZ / 10)
stephen hemmingera50af862016-12-06 13:43:54 -080051
Hank Janssen99c8da02010-10-12 10:45:23 -070052static int ring_size = 128;
Stephen Hemminger450d7a42010-05-04 09:58:53 -070053module_param(ring_size, int, S_IRUGO);
54MODULE_PARM_DESC(ring_size, "Ring buffer size (# of pages)");
Hank Janssenfceaf242009-07-13 15:34:54 -070055
Simon Xiao3f300ff2015-04-28 01:05:17 -070056static const u32 default_msg = NETIF_MSG_DRV | NETIF_MSG_PROBE |
57 NETIF_MSG_LINK | NETIF_MSG_IFUP |
58 NETIF_MSG_IFDOWN | NETIF_MSG_RX_ERR |
59 NETIF_MSG_TX_ERR;
60
61static int debug = -1;
62module_param(debug, int, S_IRUGO);
63MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
64
Greg Kroah-Hartman4e9bfef2009-07-15 12:45:20 -070065static void netvsc_set_multicast_list(struct net_device *net)
Hank Janssenfceaf242009-07-13 15:34:54 -070066{
Wenqi Ma792df872012-04-19 00:39:37 +000067 struct net_device_context *net_device_ctx = netdev_priv(net);
stephen hemminger4f19c0d2017-06-07 15:53:49 -070068 struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
Haiyang Zhangd426b2e2011-11-30 07:19:08 -080069
stephen hemminger4f19c0d2017-06-07 15:53:49 -070070 rndis_filter_update(nvdev);
Hank Janssenfceaf242009-07-13 15:34:54 -070071}
72
Hank Janssenfceaf242009-07-13 15:34:54 -070073static int netvsc_open(struct net_device *net)
74{
Haiyang Zhang53fa1a62017-06-21 16:40:47 -070075 struct net_device_context *ndev_ctx = netdev_priv(net);
stephen hemminger0c195562017-08-01 19:58:53 -070076 struct net_device *vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
stephen hemminger79e8cbe2017-07-19 11:53:13 -070077 struct netvsc_device *nvdev = rtnl_dereference(ndev_ctx->nvdev);
Haiyang Zhang891de742014-02-12 16:54:27 -080078 struct rndis_device *rdev;
Greg Kroah-Hartman02fafbc2009-08-31 21:09:45 -070079 int ret = 0;
Hank Janssenfceaf242009-07-13 15:34:54 -070080
Haiyang Zhang891de742014-02-12 16:54:27 -080081 netif_carrier_off(net);
82
Haiyang Zhangd515d0f2011-09-28 13:24:15 -070083 /* Open up the device */
Vitaly Kuznetsov2f5fa6c2016-06-03 17:51:00 +020084 ret = rndis_filter_open(nvdev);
Haiyang Zhangd515d0f2011-09-28 13:24:15 -070085 if (ret != 0) {
86 netdev_err(net, "unable to open device (ret %d).\n", ret);
87 return ret;
Hank Janssenfceaf242009-07-13 15:34:54 -070088 }
89
Haiyang Zhang2de85302015-07-13 13:09:16 -070090 netif_tx_wake_all_queues(net);
Haiyang Zhangd515d0f2011-09-28 13:24:15 -070091
Haiyang Zhang891de742014-02-12 16:54:27 -080092 rdev = nvdev->extension;
stephen hemminger0c195562017-08-01 19:58:53 -070093
94 if (!rdev->link_state)
Haiyang Zhang891de742014-02-12 16:54:27 -080095 netif_carrier_on(net);
96
stephen hemminger0c195562017-08-01 19:58:53 -070097 if (vf_netdev) {
98 /* Setting synthetic device up transparently sets
99 * slave as up. If open fails, then slave will be
100 * still be offline (and not used).
101 */
102 ret = dev_open(vf_netdev);
103 if (ret)
104 netdev_warn(net,
105 "unable to open slave: %s: %d\n",
106 vf_netdev->name, ret);
107 }
108 return 0;
Hank Janssenfceaf242009-07-13 15:34:54 -0700109}
110
Hank Janssenfceaf242009-07-13 15:34:54 -0700111static int netvsc_close(struct net_device *net)
112{
Hank Janssenfceaf242009-07-13 15:34:54 -0700113 struct net_device_context *net_device_ctx = netdev_priv(net);
stephen hemminger0c195562017-08-01 19:58:53 -0700114 struct net_device *vf_netdev
115 = rtnl_dereference(net_device_ctx->vf_netdev);
stephen hemminger545a8e72017-03-22 14:51:00 -0700116 struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
Greg Kroah-Hartman02fafbc2009-08-31 21:09:45 -0700117 int ret;
stephen hemminger40975962017-06-08 16:21:19 -0700118 u32 aread, i, msec = 10, retry = 0, retry_max = 20;
Haiyang Zhang2de85302015-07-13 13:09:16 -0700119 struct vmbus_channel *chn;
Hank Janssenfceaf242009-07-13 15:34:54 -0700120
Haiyang Zhang0a282532012-02-02 07:17:59 +0000121 netif_tx_disable(net);
Hank Janssenfceaf242009-07-13 15:34:54 -0700122
Vitaly Kuznetsov2f5fa6c2016-06-03 17:51:00 +0200123 ret = rndis_filter_close(nvdev);
Haiyang Zhang2de85302015-07-13 13:09:16 -0700124 if (ret != 0) {
Hank Jansseneb335bc2011-03-29 13:58:48 -0700125 netdev_err(net, "unable to close device (ret %d).\n", ret);
Haiyang Zhang2de85302015-07-13 13:09:16 -0700126 return ret;
127 }
128
129 /* Ensure pending bytes in ring are read */
130 while (true) {
131 aread = 0;
132 for (i = 0; i < nvdev->num_chn; i++) {
stephen hemmingerb8b835a2017-01-24 13:06:07 -0800133 chn = nvdev->chan_table[i].channel;
Haiyang Zhang2de85302015-07-13 13:09:16 -0700134 if (!chn)
135 continue;
136
stephen hemminger40975962017-06-08 16:21:19 -0700137 aread = hv_get_bytes_to_read(&chn->inbound);
Haiyang Zhang2de85302015-07-13 13:09:16 -0700138 if (aread)
139 break;
140
stephen hemminger40975962017-06-08 16:21:19 -0700141 aread = hv_get_bytes_to_read(&chn->outbound);
Haiyang Zhang2de85302015-07-13 13:09:16 -0700142 if (aread)
143 break;
144 }
145
146 retry++;
147 if (retry > retry_max || aread == 0)
148 break;
149
150 msleep(msec);
151
152 if (msec < 1000)
153 msec *= 2;
154 }
155
156 if (aread) {
157 netdev_err(net, "Ring buffer not empty after closing rndis\n");
158 ret = -ETIMEDOUT;
159 }
Hank Janssenfceaf242009-07-13 15:34:54 -0700160
stephen hemminger0c195562017-08-01 19:58:53 -0700161 if (vf_netdev)
162 dev_close(vf_netdev);
163
Hank Janssenfceaf242009-07-13 15:34:54 -0700164 return ret;
165}
166
KY Srinivasan8a002512014-03-08 19:23:14 -0800167static void *init_ppi_data(struct rndis_message *msg, u32 ppi_size,
168 int pkt_type)
169{
170 struct rndis_packet *rndis_pkt;
171 struct rndis_per_packet_info *ppi;
172
173 rndis_pkt = &msg->msg.pkt;
174 rndis_pkt->data_offset += ppi_size;
175
176 ppi = (struct rndis_per_packet_info *)((void *)rndis_pkt +
177 rndis_pkt->per_pkt_info_offset + rndis_pkt->per_pkt_info_len);
178
179 ppi->size = ppi_size;
180 ppi->type = pkt_type;
181 ppi->ppi_offset = sizeof(struct rndis_per_packet_info);
182
183 rndis_pkt->per_pkt_info_len += ppi_size;
184
185 return ppi;
186}
187
Haiyang Zhangf72860a2017-04-12 11:45:18 -0700188/* Azure hosts don't support non-TCP port numbers in hashing yet. We compute
189 * hash for non-TCP traffic with only IP numbers.
190 */
191static inline u32 netvsc_get_hash(struct sk_buff *skb, struct sock *sk)
192{
193 struct flow_keys flow;
194 u32 hash;
195 static u32 hashrnd __read_mostly;
196
197 net_get_random_once(&hashrnd, sizeof(hashrnd));
198
199 if (!skb_flow_dissect_flow_keys(skb, &flow, 0))
200 return 0;
201
202 if (flow.basic.ip_proto == IPPROTO_TCP) {
203 return skb_get_hash(skb);
204 } else {
205 if (flow.basic.n_proto == htons(ETH_P_IP))
206 hash = jhash2((u32 *)&flow.addrs.v4addrs, 2, hashrnd);
207 else if (flow.basic.n_proto == htons(ETH_P_IPV6))
208 hash = jhash2((u32 *)&flow.addrs.v6addrs, 8, hashrnd);
209 else
210 hash = 0;
211
212 skb_set_hash(skb, hash, PKT_HASH_TYPE_L3);
213 }
214
215 return hash;
216}
217
Haiyang Zhang8db91f62017-04-12 11:35:05 -0700218static inline int netvsc_get_tx_queue(struct net_device *ndev,
219 struct sk_buff *skb, int old_idx)
220{
221 const struct net_device_context *ndc = netdev_priv(ndev);
222 struct sock *sk = skb->sk;
223 int q_idx;
224
Haiyang Zhangf72860a2017-04-12 11:45:18 -0700225 q_idx = ndc->tx_send_table[netvsc_get_hash(skb, sk) &
Haiyang Zhang8db91f62017-04-12 11:35:05 -0700226 (VRSS_SEND_TAB_SIZE - 1)];
227
228 /* If queue index changed record the new value */
229 if (q_idx != old_idx &&
230 sk && sk_fullsock(sk) && rcu_access_pointer(sk->sk_dst_cache))
231 sk_tx_queue_set(sk, q_idx);
232
233 return q_idx;
234}
235
stephen hemmingerd8e18ee2017-01-24 13:06:05 -0800236/*
237 * Select queue for transmit.
238 *
239 * If a valid queue has already been assigned, then use that.
240 * Otherwise compute tx queue based on hash and the send table.
241 *
242 * This is basically similar to default (__netdev_pick_tx) with the added step
243 * of using the host send_table when no other queue has been assigned.
244 *
245 * TODO support XPS - but get_xps_queue not exported
246 */
stephen hemminger0c195562017-08-01 19:58:53 -0700247static u16 netvsc_pick_tx(struct net_device *ndev, struct sk_buff *skb)
Haiyang Zhang5b54dac2014-04-21 10:20:28 -0700248{
Haiyang Zhang8db91f62017-04-12 11:35:05 -0700249 int q_idx = sk_tx_queue_get(skb->sk);
Haiyang Zhang5b54dac2014-04-21 10:20:28 -0700250
stephen hemminger0c195562017-08-01 19:58:53 -0700251 if (q_idx < 0 || skb->ooo_okay || q_idx >= ndev->real_num_tx_queues) {
Haiyang Zhang8db91f62017-04-12 11:35:05 -0700252 /* If forwarding a packet, we use the recorded queue when
253 * available for better cache locality.
254 */
255 if (skb_rx_queue_recorded(skb))
256 q_idx = skb_get_rx_queue(skb);
257 else
258 q_idx = netvsc_get_tx_queue(ndev, skb, q_idx);
stephen hemmingerd8e18ee2017-01-24 13:06:05 -0800259 }
260
Haiyang Zhang5b54dac2014-04-21 10:20:28 -0700261 return q_idx;
262}
263
stephen hemminger0c195562017-08-01 19:58:53 -0700264static u16 netvsc_select_queue(struct net_device *ndev, struct sk_buff *skb,
265 void *accel_priv,
266 select_queue_fallback_t fallback)
267{
268 struct net_device_context *ndc = netdev_priv(ndev);
269 struct net_device *vf_netdev;
270 u16 txq;
271
272 rcu_read_lock();
273 vf_netdev = rcu_dereference(ndc->vf_netdev);
274 if (vf_netdev) {
275 txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : 0;
276 qdisc_skb_cb(skb)->slave_dev_queue_mapping = skb->queue_mapping;
277 } else {
278 txq = netvsc_pick_tx(ndev, skb);
279 }
280 rcu_read_unlock();
281
282 while (unlikely(txq >= ndev->real_num_tx_queues))
283 txq -= ndev->real_num_tx_queues;
284
285 return txq;
286}
287
KY Srinivasan54a73572014-03-08 19:23:13 -0800288static u32 fill_pg_buf(struct page *page, u32 offset, u32 len,
289 struct hv_page_buffer *pb)
290{
291 int j = 0;
292
293 /* Deal with compund pages by ignoring unused part
294 * of the page.
295 */
296 page += (offset >> PAGE_SHIFT);
297 offset &= ~PAGE_MASK;
298
299 while (len > 0) {
300 unsigned long bytes;
301
302 bytes = PAGE_SIZE - offset;
303 if (bytes > len)
304 bytes = len;
305 pb[j].pfn = page_to_pfn(page);
306 pb[j].offset = offset;
307 pb[j].len = bytes;
308
309 offset += bytes;
310 len -= bytes;
311
312 if (offset == PAGE_SIZE && len) {
313 page++;
314 offset = 0;
315 j++;
316 }
317 }
318
319 return j + 1;
320}
321
KY Srinivasan8a002512014-03-08 19:23:14 -0800322static u32 init_page_array(void *hdr, u32 len, struct sk_buff *skb,
KY Srinivasana9f2e2d2015-12-01 16:43:13 -0800323 struct hv_netvsc_packet *packet,
stephen hemminger02b6de02017-07-28 08:59:44 -0700324 struct hv_page_buffer *pb)
KY Srinivasan54a73572014-03-08 19:23:13 -0800325{
326 u32 slots_used = 0;
327 char *data = skb->data;
328 int frags = skb_shinfo(skb)->nr_frags;
329 int i;
330
331 /* The packet is laid out thus:
Haiyang Zhangaa0a34b2015-04-13 16:34:35 -0700332 * 1. hdr: RNDIS header and PPI
KY Srinivasan54a73572014-03-08 19:23:13 -0800333 * 2. skb linear data
334 * 3. skb fragment data
335 */
336 if (hdr != NULL)
337 slots_used += fill_pg_buf(virt_to_page(hdr),
338 offset_in_page(hdr),
339 len, &pb[slots_used]);
340
Haiyang Zhangaa0a34b2015-04-13 16:34:35 -0700341 packet->rmsg_size = len;
342 packet->rmsg_pgcnt = slots_used;
343
KY Srinivasan54a73572014-03-08 19:23:13 -0800344 slots_used += fill_pg_buf(virt_to_page(data),
345 offset_in_page(data),
346 skb_headlen(skb), &pb[slots_used]);
347
348 for (i = 0; i < frags; i++) {
349 skb_frag_t *frag = skb_shinfo(skb)->frags + i;
350
351 slots_used += fill_pg_buf(skb_frag_page(frag),
352 frag->page_offset,
353 skb_frag_size(frag), &pb[slots_used]);
354 }
KY Srinivasan8a002512014-03-08 19:23:14 -0800355 return slots_used;
KY Srinivasan54a73572014-03-08 19:23:13 -0800356}
357
stephen hemminger80d887d2017-07-24 21:03:19 -0700358static int count_skb_frag_slots(struct sk_buff *skb)
KY Srinivasan54a73572014-03-08 19:23:13 -0800359{
stephen hemminger80d887d2017-07-24 21:03:19 -0700360 int i, frags = skb_shinfo(skb)->nr_frags;
361 int pages = 0;
362
363 for (i = 0; i < frags; i++) {
364 skb_frag_t *frag = skb_shinfo(skb)->frags + i;
365 unsigned long size = skb_frag_size(frag);
366 unsigned long offset = frag->page_offset;
367
368 /* Skip unused frames from start of page */
369 offset &= ~PAGE_MASK;
370 pages += PFN_UP(offset + size);
371 }
372 return pages;
373}
374
375static int netvsc_get_slots(struct sk_buff *skb)
376{
377 char *data = skb->data;
378 unsigned int offset = offset_in_page(data);
379 unsigned int len = skb_headlen(skb);
380 int slots;
381 int frag_slots;
382
383 slots = DIV_ROUND_UP(offset + len, PAGE_SIZE);
384 frag_slots = count_skb_frag_slots(skb);
385 return slots + frag_slots;
KY Srinivasan54a73572014-03-08 19:23:13 -0800386}
387
stephen hemminger23312a32017-01-24 13:05:59 -0800388static u32 net_checksum_info(struct sk_buff *skb)
KY Srinivasan08cd04b2014-03-08 19:23:17 -0800389{
stephen hemminger23312a32017-01-24 13:05:59 -0800390 if (skb->protocol == htons(ETH_P_IP)) {
391 struct iphdr *ip = ip_hdr(skb);
KY Srinivasan08cd04b2014-03-08 19:23:17 -0800392
stephen hemminger23312a32017-01-24 13:05:59 -0800393 if (ip->protocol == IPPROTO_TCP)
394 return TRANSPORT_INFO_IPV4_TCP;
395 else if (ip->protocol == IPPROTO_UDP)
396 return TRANSPORT_INFO_IPV4_UDP;
KY Srinivasan08cd04b2014-03-08 19:23:17 -0800397 } else {
stephen hemminger23312a32017-01-24 13:05:59 -0800398 struct ipv6hdr *ip6 = ipv6_hdr(skb);
399
400 if (ip6->nexthdr == IPPROTO_TCP)
401 return TRANSPORT_INFO_IPV6_TCP;
Mohammed Gamal37b9dfa2017-07-24 10:57:26 -0700402 else if (ip6->nexthdr == IPPROTO_UDP)
stephen hemminger23312a32017-01-24 13:05:59 -0800403 return TRANSPORT_INFO_IPV6_UDP;
KY Srinivasan08cd04b2014-03-08 19:23:17 -0800404 }
405
stephen hemminger23312a32017-01-24 13:05:59 -0800406 return TRANSPORT_INFO_NOT_IP;
KY Srinivasan08cd04b2014-03-08 19:23:17 -0800407}
408
stephen hemminger0c195562017-08-01 19:58:53 -0700409/* Send skb on the slave VF device. */
410static int netvsc_vf_xmit(struct net_device *net, struct net_device *vf_netdev,
411 struct sk_buff *skb)
412{
413 struct net_device_context *ndev_ctx = netdev_priv(net);
414 unsigned int len = skb->len;
415 int rc;
416
417 skb->dev = vf_netdev;
418 skb->queue_mapping = qdisc_skb_cb(skb)->slave_dev_queue_mapping;
419
420 rc = dev_queue_xmit(skb);
421 if (likely(rc == NET_XMIT_SUCCESS || rc == NET_XMIT_CN)) {
422 struct netvsc_vf_pcpu_stats *pcpu_stats
423 = this_cpu_ptr(ndev_ctx->vf_stats);
424
425 u64_stats_update_begin(&pcpu_stats->syncp);
426 pcpu_stats->tx_packets++;
427 pcpu_stats->tx_bytes += len;
428 u64_stats_update_end(&pcpu_stats->syncp);
429 } else {
430 this_cpu_inc(ndev_ctx->vf_stats->tx_dropped);
431 }
432
433 return rc;
434}
435
Greg Kroah-Hartman02fafbc2009-08-31 21:09:45 -0700436static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net)
Hank Janssenfceaf242009-07-13 15:34:54 -0700437{
Hank Janssenfceaf242009-07-13 15:34:54 -0700438 struct net_device_context *net_device_ctx = netdev_priv(net);
Vitaly Kuznetsov981a1bd2015-04-08 17:54:05 +0200439 struct hv_netvsc_packet *packet = NULL;
Greg Kroah-Hartman02fafbc2009-08-31 21:09:45 -0700440 int ret;
KY Srinivasan8a002512014-03-08 19:23:14 -0800441 unsigned int num_data_pgs;
442 struct rndis_message *rndis_msg;
443 struct rndis_packet *rndis_pkt;
stephen hemminger0c195562017-08-01 19:58:53 -0700444 struct net_device *vf_netdev;
KY Srinivasan8a002512014-03-08 19:23:14 -0800445 u32 rndis_msg_size;
KY Srinivasan8a002512014-03-08 19:23:14 -0800446 struct rndis_per_packet_info *ppi;
Haiyang Zhang307f0992014-05-21 12:55:39 -0700447 u32 hash;
stephen hemminger02b6de02017-07-28 08:59:44 -0700448 struct hv_page_buffer pb[MAX_PAGE_BUFFER_COUNT];
Hank Janssenfceaf242009-07-13 15:34:54 -0700449
stephen hemminger0c195562017-08-01 19:58:53 -0700450 /* if VF is present and up then redirect packets
451 * already called with rcu_read_lock_bh
452 */
453 vf_netdev = rcu_dereference_bh(net_device_ctx->vf_netdev);
454 if (vf_netdev && netif_running(vf_netdev) &&
455 !netpoll_tx_running(net))
456 return netvsc_vf_xmit(net, vf_netdev, skb);
457
stephen hemminger80d887d2017-07-24 21:03:19 -0700458 /* We will atmost need two pages to describe the rndis
459 * header. We can only transmit MAX_PAGE_BUFFER_COUNT number
Vitaly Kuznetsove88f7e02015-04-08 17:54:06 +0200460 * of pages in a single packet. If skb is scattered around
461 * more pages we try linearizing it.
KY Srinivasan54a73572014-03-08 19:23:13 -0800462 */
stephen hemminger80d887d2017-07-24 21:03:19 -0700463
464 num_data_pgs = netvsc_get_slots(skb) + 2;
465
Stephen Hemminger0ab05142016-08-23 12:17:52 -0700466 if (unlikely(num_data_pgs > MAX_PAGE_BUFFER_COUNT)) {
Stephen Hemminger4323b472016-08-23 12:17:57 -0700467 ++net_device_ctx->eth_stats.tx_scattered;
468
469 if (skb_linearize(skb))
470 goto no_memory;
Stephen Hemminger0ab05142016-08-23 12:17:52 -0700471
stephen hemminger80d887d2017-07-24 21:03:19 -0700472 num_data_pgs = netvsc_get_slots(skb) + 2;
Stephen Hemminger0ab05142016-08-23 12:17:52 -0700473 if (num_data_pgs > MAX_PAGE_BUFFER_COUNT) {
Stephen Hemminger4323b472016-08-23 12:17:57 -0700474 ++net_device_ctx->eth_stats.tx_too_big;
Stephen Hemminger0ab05142016-08-23 12:17:52 -0700475 goto drop;
476 }
KY Srinivasan54a73572014-03-08 19:23:13 -0800477 }
Hank Janssenfceaf242009-07-13 15:34:54 -0700478
KY Srinivasanc0eb4542015-12-01 16:43:10 -0800479 /*
480 * Place the rndis header in the skb head room and
481 * the skb->cb will be used for hv_netvsc_packet
482 * structure.
483 */
484 ret = skb_cow_head(skb, RNDIS_AND_PPI_SIZE);
Stephen Hemminger4323b472016-08-23 12:17:57 -0700485 if (ret)
486 goto no_memory;
487
KY Srinivasanc0eb4542015-12-01 16:43:10 -0800488 /* Use the skb control buffer for building up the packet */
489 BUILD_BUG_ON(sizeof(struct hv_netvsc_packet) >
490 FIELD_SIZEOF(struct sk_buff, cb));
491 packet = (struct hv_netvsc_packet *)skb->cb;
Hank Janssenfceaf242009-07-13 15:34:54 -0700492
Haiyang Zhang5b54dac2014-04-21 10:20:28 -0700493 packet->q_idx = skb_get_queue_mapping(skb);
494
Haiyang Zhang4d447c92011-12-15 13:45:17 -0800495 packet->total_data_buflen = skb->len;
stephen hemminger793e3952017-01-24 13:06:12 -0800496 packet->total_bytes = skb->len;
497 packet->total_packets = 1;
Hank Janssenfceaf242009-07-13 15:34:54 -0700498
KY Srinivasanc0eb4542015-12-01 16:43:10 -0800499 rndis_msg = (struct rndis_message *)skb->head;
KY Srinivasanb08cc792015-03-29 21:08:42 -0700500
KY Srinivasan24476762015-12-01 16:43:06 -0800501 memset(rndis_msg, 0, RNDIS_AND_PPI_SIZE);
Hank Janssenfceaf242009-07-13 15:34:54 -0700502
KY Srinivasan8a002512014-03-08 19:23:14 -0800503 /* Add the rndis header */
KY Srinivasan8a002512014-03-08 19:23:14 -0800504 rndis_msg->ndis_msg_type = RNDIS_MSG_PACKET;
505 rndis_msg->msg_len = packet->total_data_buflen;
506 rndis_pkt = &rndis_msg->msg.pkt;
507 rndis_pkt->data_offset = sizeof(struct rndis_packet);
508 rndis_pkt->data_len = packet->total_data_buflen;
509 rndis_pkt->per_pkt_info_offset = sizeof(struct rndis_packet);
510
511 rndis_msg_size = RNDIS_MESSAGE_SIZE(struct rndis_packet);
512
Haiyang Zhang307f0992014-05-21 12:55:39 -0700513 hash = skb_get_hash_raw(skb);
514 if (hash != 0 && net->real_num_tx_queues > 1) {
515 rndis_msg_size += NDIS_HASH_PPI_SIZE;
516 ppi = init_ppi_data(rndis_msg, NDIS_HASH_PPI_SIZE,
517 NBL_HASH_VALUE);
518 *(u32 *)((void *)ppi + ppi->ppi_offset) = hash;
519 }
520
Stephen Hemminger0ab05142016-08-23 12:17:52 -0700521 if (skb_vlan_tag_present(skb)) {
KY Srinivasan8a002512014-03-08 19:23:14 -0800522 struct ndis_pkt_8021q_info *vlan;
523
524 rndis_msg_size += NDIS_VLAN_PPI_SIZE;
525 ppi = init_ppi_data(rndis_msg, NDIS_VLAN_PPI_SIZE,
526 IEEE_8021Q_INFO);
527 vlan = (struct ndis_pkt_8021q_info *)((void *)ppi +
528 ppi->ppi_offset);
KY Srinivasan760d1e32015-12-01 16:43:19 -0800529 vlan->vlanid = skb->vlan_tci & VLAN_VID_MASK;
530 vlan->pri = (skb->vlan_tci & VLAN_PRIO_MASK) >>
KY Srinivasan8a002512014-03-08 19:23:14 -0800531 VLAN_PRIO_SHIFT;
532 }
533
stephen hemminger23312a32017-01-24 13:05:59 -0800534 if (skb_is_gso(skb)) {
Stephen Hemminger0ab05142016-08-23 12:17:52 -0700535 struct ndis_tcp_lso_info *lso_info;
536
537 rndis_msg_size += NDIS_LSO_PPI_SIZE;
538 ppi = init_ppi_data(rndis_msg, NDIS_LSO_PPI_SIZE,
539 TCP_LARGESEND_PKTINFO);
540
541 lso_info = (struct ndis_tcp_lso_info *)((void *)ppi +
542 ppi->ppi_offset);
543
544 lso_info->lso_v2_transmit.type = NDIS_TCP_LARGE_SEND_OFFLOAD_V2_TYPE;
stephen hemminger23312a32017-01-24 13:05:59 -0800545 if (skb->protocol == htons(ETH_P_IP)) {
Stephen Hemminger0ab05142016-08-23 12:17:52 -0700546 lso_info->lso_v2_transmit.ip_version =
547 NDIS_TCP_LARGE_SEND_OFFLOAD_IPV4;
548 ip_hdr(skb)->tot_len = 0;
549 ip_hdr(skb)->check = 0;
550 tcp_hdr(skb)->check =
551 ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
552 ip_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
553 } else {
554 lso_info->lso_v2_transmit.ip_version =
555 NDIS_TCP_LARGE_SEND_OFFLOAD_IPV6;
556 ipv6_hdr(skb)->payload_len = 0;
557 tcp_hdr(skb)->check =
558 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
559 &ipv6_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
560 }
stephen hemminger23312a32017-01-24 13:05:59 -0800561 lso_info->lso_v2_transmit.tcp_header_offset = skb_transport_offset(skb);
Stephen Hemminger0ab05142016-08-23 12:17:52 -0700562 lso_info->lso_v2_transmit.mss = skb_shinfo(skb)->gso_size;
stephen hemmingerad19bc82016-10-11 14:03:07 -0700563 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
stephen hemminger23312a32017-01-24 13:05:59 -0800564 if (net_checksum_info(skb) & net_device_ctx->tx_checksum_mask) {
565 struct ndis_tcp_ip_checksum_info *csum_info;
566
stephen hemmingerad19bc82016-10-11 14:03:07 -0700567 rndis_msg_size += NDIS_CSUM_PPI_SIZE;
568 ppi = init_ppi_data(rndis_msg, NDIS_CSUM_PPI_SIZE,
569 TCPIP_CHKSUM_PKTINFO);
570
571 csum_info = (struct ndis_tcp_ip_checksum_info *)((void *)ppi +
572 ppi->ppi_offset);
573
stephen hemminger23312a32017-01-24 13:05:59 -0800574 csum_info->transmit.tcp_header_offset = skb_transport_offset(skb);
575
576 if (skb->protocol == htons(ETH_P_IP)) {
stephen hemmingerad19bc82016-10-11 14:03:07 -0700577 csum_info->transmit.is_ipv4 = 1;
stephen hemminger23312a32017-01-24 13:05:59 -0800578
579 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
580 csum_info->transmit.tcp_checksum = 1;
581 else
582 csum_info->transmit.udp_checksum = 1;
583 } else {
stephen hemmingerad19bc82016-10-11 14:03:07 -0700584 csum_info->transmit.is_ipv6 = 1;
585
stephen hemminger23312a32017-01-24 13:05:59 -0800586 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
587 csum_info->transmit.tcp_checksum = 1;
588 else
589 csum_info->transmit.udp_checksum = 1;
590 }
stephen hemmingerad19bc82016-10-11 14:03:07 -0700591 } else {
stephen hemminger23312a32017-01-24 13:05:59 -0800592 /* Can't do offload of this type of checksum */
stephen hemmingerad19bc82016-10-11 14:03:07 -0700593 if (skb_checksum_help(skb))
594 goto drop;
595 }
Stephen Hemminger0ab05142016-08-23 12:17:52 -0700596 }
KY Srinivasan08cd04b2014-03-08 19:23:17 -0800597
KY Srinivasan8a002512014-03-08 19:23:14 -0800598 /* Start filling in the page buffers with the rndis hdr */
599 rndis_msg->msg_len += rndis_msg_size;
Haiyang Zhang942396b2014-10-22 13:47:18 -0700600 packet->total_data_buflen = rndis_msg->msg_len;
KY Srinivasan8a002512014-03-08 19:23:14 -0800601 packet->page_buf_cnt = init_page_array(rndis_msg, rndis_msg_size,
stephen hemminger02b6de02017-07-28 08:59:44 -0700602 skb, packet, pb);
KY Srinivasan8a002512014-03-08 19:23:14 -0800603
sixiao@microsoft.com76d13b52016-02-17 16:43:59 -0800604 /* timestamp packet in software */
605 skb_tx_timestamp(skb);
stephen hemminger2a926f72017-07-19 11:53:17 -0700606
stephen hemminger02b6de02017-07-28 08:59:44 -0700607 ret = netvsc_send(net_device_ctx, packet, rndis_msg, pb, skb);
stephen hemminger793e3952017-01-24 13:06:12 -0800608 if (likely(ret == 0))
Stephen Hemminger0ab05142016-08-23 12:17:52 -0700609 return NETDEV_TX_OK;
Stephen Hemminger4323b472016-08-23 12:17:57 -0700610
611 if (ret == -EAGAIN) {
612 ++net_device_ctx->eth_stats.tx_busy;
Stephen Hemminger0ab05142016-08-23 12:17:52 -0700613 return NETDEV_TX_BUSY;
Stephen Hemminger4323b472016-08-23 12:17:57 -0700614 }
615
616 if (ret == -ENOSPC)
617 ++net_device_ctx->eth_stats.tx_no_space;
Hank Janssenfceaf242009-07-13 15:34:54 -0700618
Stephen Hemminger0ab05142016-08-23 12:17:52 -0700619drop:
620 dev_kfree_skb_any(skb);
621 net->stats.tx_dropped++;
622
623 return NETDEV_TX_OK;
Stephen Hemminger4323b472016-08-23 12:17:57 -0700624
625no_memory:
626 ++net_device_ctx->eth_stats.tx_no_memory;
627 goto drop;
Hank Janssenfceaf242009-07-13 15:34:54 -0700628}
Hank Janssen3e189512010-03-04 22:11:00 +0000629/*
Greg Kroah-Hartman02fafbc2009-08-31 21:09:45 -0700630 * netvsc_linkstatus_callback - Link up/down notification
631 */
K. Y. Srinivasan90ef1172011-05-12 19:34:50 -0700632void netvsc_linkstatus_callback(struct hv_device *device_obj,
Haiyang Zhang3a494e72014-06-19 18:34:36 -0700633 struct rndis_message *resp)
Hank Janssenfceaf242009-07-13 15:34:54 -0700634{
Haiyang Zhang3a494e72014-06-19 18:34:36 -0700635 struct rndis_indicate_status *indicate = &resp->msg.indicate_status;
K. Y. Srinivasan2ddd5e5f2011-09-13 10:59:49 -0700636 struct net_device *net;
Haiyang Zhangc996edc2011-04-06 15:18:00 -0700637 struct net_device_context *ndev_ctx;
Vitaly Kuznetsov27a70af2015-11-27 11:39:55 +0100638 struct netvsc_reconfig *event;
639 unsigned long flags;
640
Haiyang Zhang7f5d5af2016-08-04 10:42:15 -0700641 net = hv_get_drvdata(device_obj);
642
643 if (!net)
644 return;
645
646 ndev_ctx = netdev_priv(net);
647
648 /* Update the physical link speed when changing to another vSwitch */
649 if (indicate->status == RNDIS_STATUS_LINK_SPEED_CHANGE) {
650 u32 speed;
651
652 speed = *(u32 *)((void *)indicate + indicate->
653 status_buf_offset) / 10000;
654 ndev_ctx->speed = speed;
655 return;
656 }
657
658 /* Handle these link change statuses below */
Vitaly Kuznetsov27a70af2015-11-27 11:39:55 +0100659 if (indicate->status != RNDIS_STATUS_NETWORK_CHANGE &&
660 indicate->status != RNDIS_STATUS_MEDIA_CONNECT &&
661 indicate->status != RNDIS_STATUS_MEDIA_DISCONNECT)
662 return;
K. Y. Srinivasan2ddd5e5f2011-09-13 10:59:49 -0700663
Haiyang Zhang7f5d5af2016-08-04 10:42:15 -0700664 if (net->reg_state != NETREG_REGISTERED)
Hank Janssenfceaf242009-07-13 15:34:54 -0700665 return;
Hank Janssenfceaf242009-07-13 15:34:54 -0700666
Vitaly Kuznetsov27a70af2015-11-27 11:39:55 +0100667 event = kzalloc(sizeof(*event), GFP_ATOMIC);
668 if (!event)
669 return;
670 event->event = indicate->status;
671
672 spin_lock_irqsave(&ndev_ctx->lock, flags);
673 list_add_tail(&event->list, &ndev_ctx->reconfig_events);
674 spin_unlock_irqrestore(&ndev_ctx->lock, flags);
675
676 schedule_delayed_work(&ndev_ctx->dwork, 0);
Hank Janssenfceaf242009-07-13 15:34:54 -0700677}
678
KY Srinivasan84bf9ce2016-04-14 16:31:54 -0700679static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net,
stephen hemmingere91e7dd2017-02-27 10:26:51 -0800680 struct napi_struct *napi,
stephen hemmingerdc54a082017-01-24 13:06:08 -0800681 const struct ndis_tcp_ip_checksum_info *csum_info,
682 const struct ndis_pkt_8021q_info *vlan,
683 void *data, u32 buflen)
Hank Janssenfceaf242009-07-13 15:34:54 -0700684{
Hank Janssenfceaf242009-07-13 15:34:54 -0700685 struct sk_buff *skb;
K. Y. Srinivasan2ddd5e5f2011-09-13 10:59:49 -0700686
stephen hemmingere91e7dd2017-02-27 10:26:51 -0800687 skb = napi_alloc_skb(napi, buflen);
KY Srinivasan84bf9ce2016-04-14 16:31:54 -0700688 if (!skb)
689 return skb;
Hank Janssenfceaf242009-07-13 15:34:54 -0700690
Greg Kroah-Hartman02fafbc2009-08-31 21:09:45 -0700691 /*
692 * Copy to skb. This copy is needed here since the memory pointed by
693 * hv_netvsc_packet cannot be deallocated
694 */
Johannes Berg59ae1d12017-06-16 14:29:20 +0200695 skb_put_data(skb, data, buflen);
Hank Janssenfceaf242009-07-13 15:34:54 -0700696
697 skb->protocol = eth_type_trans(skb, net);
Stephen Hemmingere52fed72016-10-23 21:32:47 -0700698
699 /* skb is already created with CHECKSUM_NONE */
700 skb_checksum_none_assert(skb);
701
702 /*
703 * In Linux, the IP checksum is always checked.
704 * Do L4 checksum offload if enabled and present.
705 */
706 if (csum_info && (net->features & NETIF_F_RXCSUM)) {
707 if (csum_info->receive.tcp_checksum_succeeded ||
708 csum_info->receive.udp_checksum_succeeded)
KY Srinivasane3d605e2014-03-08 19:23:16 -0800709 skb->ip_summed = CHECKSUM_UNNECESSARY;
KY Srinivasane3d605e2014-03-08 19:23:16 -0800710 }
711
stephen hemmingerdc54a082017-01-24 13:06:08 -0800712 if (vlan) {
713 u16 vlan_tci = vlan->vlanid | (vlan->pri << VLAN_PRIO_SHIFT);
714
Haiyang Zhang93725cb2013-06-17 15:36:49 -0700715 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
KY Srinivasan760d1e32015-12-01 16:43:19 -0800716 vlan_tci);
stephen hemmingerdc54a082017-01-24 13:06:08 -0800717 }
Hank Janssenfceaf242009-07-13 15:34:54 -0700718
KY Srinivasan84bf9ce2016-04-14 16:31:54 -0700719 return skb;
720}
721
722/*
723 * netvsc_recv_callback - Callback when we receive a packet from the
724 * "wire" on the specified device.
725 */
stephen hemmingerdc54a082017-01-24 13:06:08 -0800726int netvsc_recv_callback(struct net_device *net,
727 struct vmbus_channel *channel,
728 void *data, u32 len,
729 const struct ndis_tcp_ip_checksum_info *csum_info,
730 const struct ndis_pkt_8021q_info *vlan)
KY Srinivasan84bf9ce2016-04-14 16:31:54 -0700731{
Vitaly Kuznetsov3d541ac2016-05-13 13:55:22 +0200732 struct net_device_context *net_device_ctx = netdev_priv(net);
stephen hemminger545a8e72017-03-22 14:51:00 -0700733 struct netvsc_device *net_device;
stephen hemminger742fe542017-02-27 10:26:50 -0800734 u16 q_idx = channel->offermsg.offer.sub_channel_index;
stephen hemminger545a8e72017-03-22 14:51:00 -0700735 struct netvsc_channel *nvchan;
KY Srinivasan84bf9ce2016-04-14 16:31:54 -0700736 struct sk_buff *skb;
KY Srinivasan84bf9ce2016-04-14 16:31:54 -0700737 struct netvsc_stats *rx_stats;
KY Srinivasan84bf9ce2016-04-14 16:31:54 -0700738
Stephen Hemminger9cbcc422016-09-22 16:56:34 -0700739 if (net->reg_state != NETREG_REGISTERED)
KY Srinivasan84bf9ce2016-04-14 16:31:54 -0700740 return NVSP_STAT_FAIL;
741
stephen hemminger0719e722017-01-11 09:16:32 -0800742 rcu_read_lock();
stephen hemminger545a8e72017-03-22 14:51:00 -0700743 net_device = rcu_dereference(net_device_ctx->nvdev);
744 if (unlikely(!net_device))
745 goto drop;
746
747 nvchan = &net_device->chan_table[q_idx];
KY Srinivasan84bf9ce2016-04-14 16:31:54 -0700748
749 /* Allocate a skb - TODO direct I/O to pages? */
stephen hemmingere91e7dd2017-02-27 10:26:51 -0800750 skb = netvsc_alloc_recv_skb(net, &nvchan->napi,
751 csum_info, vlan, data, len);
KY Srinivasan84bf9ce2016-04-14 16:31:54 -0700752 if (unlikely(!skb)) {
stephen hemminger545a8e72017-03-22 14:51:00 -0700753drop:
KY Srinivasan84bf9ce2016-04-14 16:31:54 -0700754 ++net->stats.rx_dropped;
stephen hemminger0719e722017-01-11 09:16:32 -0800755 rcu_read_unlock();
KY Srinivasan84bf9ce2016-04-14 16:31:54 -0700756 return NVSP_STAT_FAIL;
757 }
Haiyang Zhang5b54dac2014-04-21 10:20:28 -0700758
stephen hemminger0c195562017-08-01 19:58:53 -0700759 skb_record_rx_queue(skb, q_idx);
Stephen Hemminger9cbcc422016-09-22 16:56:34 -0700760
761 /*
762 * Even if injecting the packet, record the statistics
763 * on the synthetic device because modifying the VF device
764 * statistics will not work correctly.
765 */
stephen hemminger742fe542017-02-27 10:26:50 -0800766 rx_stats = &nvchan->rx_stats;
sixiao@microsoft.com4b02b582015-05-15 02:33:03 -0700767 u64_stats_update_begin(&rx_stats->syncp);
sixiao@microsoft.com7eafd9b2015-05-14 01:00:25 -0700768 rx_stats->packets++;
stephen hemmingerdc54a082017-01-24 13:06:08 -0800769 rx_stats->bytes += len;
Stephen Hemmingerf7ad75b2016-09-22 16:56:35 -0700770
771 if (skb->pkt_type == PACKET_BROADCAST)
772 ++rx_stats->broadcast;
773 else if (skb->pkt_type == PACKET_MULTICAST)
774 ++rx_stats->multicast;
sixiao@microsoft.com4b02b582015-05-15 02:33:03 -0700775 u64_stats_update_end(&rx_stats->syncp);
Stephen Hemminger9495c282010-03-09 17:42:17 -0800776
stephen hemminger742fe542017-02-27 10:26:50 -0800777 napi_gro_receive(&nvchan->napi, skb);
stephen hemminger0719e722017-01-11 09:16:32 -0800778 rcu_read_unlock();
Hank Janssenfceaf242009-07-13 15:34:54 -0700779
Hank Janssenfceaf242009-07-13 15:34:54 -0700780 return 0;
781}
782
Stephen Hemmingerf82f4ad2010-05-04 09:58:57 -0700783static void netvsc_get_drvinfo(struct net_device *net,
784 struct ethtool_drvinfo *info)
785{
Jiri Pirko7826d432013-01-06 00:44:26 +0000786 strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
Jiri Pirko7826d432013-01-06 00:44:26 +0000787 strlcpy(info->fw_version, "N/A", sizeof(info->fw_version));
Stephen Hemmingerf82f4ad2010-05-04 09:58:57 -0700788}
789
Andrew Schwartzmeyer59995372015-02-26 16:27:14 -0800790static void netvsc_get_channels(struct net_device *net,
791 struct ethtool_channels *channel)
792{
793 struct net_device_context *net_device_ctx = netdev_priv(net);
stephen hemminger545a8e72017-03-22 14:51:00 -0700794 struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
Andrew Schwartzmeyer59995372015-02-26 16:27:14 -0800795
796 if (nvdev) {
797 channel->max_combined = nvdev->max_chn;
798 channel->combined_count = nvdev->num_chn;
799 }
800}
801
Andrew Schwartzmeyerb5960e62015-08-11 17:14:32 -0700802static int netvsc_set_channels(struct net_device *net,
803 struct ethtool_channels *channels)
804{
805 struct net_device_context *net_device_ctx = netdev_priv(net);
806 struct hv_device *dev = net_device_ctx->device_ctx;
stephen hemminger545a8e72017-03-22 14:51:00 -0700807 struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
stephen hemminger7ca45932017-07-24 10:57:28 -0700808 unsigned int orig, count = channels->combined_count;
809 struct netvsc_device_info device_info;
stephen hemmingerea383bf2017-07-19 11:53:15 -0700810 bool was_opened;
stephen hemminger7ca45932017-07-24 10:57:28 -0700811 int ret = 0;
stephen hemminger2b018882017-01-24 13:06:03 -0800812
813 /* We do not support separate count for rx, tx, or other */
814 if (count == 0 ||
815 channels->rx_count || channels->tx_count || channels->other_count)
816 return -EINVAL;
817
Arnd Bergmannb92b7d32017-06-22 00:16:37 +0200818 if (count > net->num_tx_queues || count > VRSS_CHANNEL_MAX)
stephen hemminger2b018882017-01-24 13:06:03 -0800819 return -EINVAL;
Andrew Schwartzmeyerb5960e62015-08-11 17:14:32 -0700820
stephen hemmingera0be4502017-03-22 14:51:01 -0700821 if (!nvdev || nvdev->destroy)
Andrew Schwartzmeyerb5960e62015-08-11 17:14:32 -0700822 return -ENODEV;
823
stephen hemminger2b018882017-01-24 13:06:03 -0800824 if (nvdev->nvsp_version < NVSP_PROTOCOL_VERSION_5)
Andrew Schwartzmeyerb5960e62015-08-11 17:14:32 -0700825 return -EINVAL;
826
stephen hemminger2b018882017-01-24 13:06:03 -0800827 if (count > nvdev->max_chn)
828 return -EINVAL;
Andrew Schwartzmeyerb5960e62015-08-11 17:14:32 -0700829
stephen hemminger7ca45932017-07-24 10:57:28 -0700830 orig = nvdev->num_chn;
stephen hemmingerea383bf2017-07-19 11:53:15 -0700831 was_opened = rndis_filter_opened(nvdev);
832 if (was_opened)
833 rndis_filter_close(nvdev);
Andrew Schwartzmeyerb5960e62015-08-11 17:14:32 -0700834
stephen hemminger2289f0a2017-01-24 13:06:10 -0800835 rndis_filter_device_remove(dev, nvdev);
Andrew Schwartzmeyerb5960e62015-08-11 17:14:32 -0700836
stephen hemminger7ca45932017-07-24 10:57:28 -0700837 memset(&device_info, 0, sizeof(device_info));
838 device_info.num_chn = count;
839 device_info.ring_size = ring_size;
Andrew Schwartzmeyerb5960e62015-08-11 17:14:32 -0700840
stephen hemminger7ca45932017-07-24 10:57:28 -0700841 nvdev = rndis_filter_device_add(dev, &device_info);
842 if (!IS_ERR(nvdev)) {
843 netif_set_real_num_tx_queues(net, nvdev->num_chn);
844 netif_set_real_num_rx_queues(net, nvdev->num_chn);
stephen hemminger7ca45932017-07-24 10:57:28 -0700845 } else {
stephen hemmingerd6aac1f2017-07-28 08:59:41 -0700846 ret = PTR_ERR(nvdev);
stephen hemminger7ca45932017-07-24 10:57:28 -0700847 device_info.num_chn = orig;
stephen hemminger7ca45932017-07-24 10:57:28 -0700848 rndis_filter_device_add(dev, &device_info);
849 }
850
stephen hemmingerea383bf2017-07-19 11:53:15 -0700851 if (was_opened)
852 rndis_filter_open(nvdev);
stephen hemminger163891d2017-03-22 14:50:58 -0700853
Vitaly Kuznetsov1bdcec82016-05-13 13:55:21 +0200854 /* We may have missed link change notifications */
stephen hemminger1b019942017-07-19 11:53:12 -0700855 net_device_ctx->last_reconfig = 0;
Vitaly Kuznetsov1bdcec82016-05-13 13:55:21 +0200856 schedule_delayed_work(&net_device_ctx->dwork, 0);
Andrew Schwartzmeyerb5960e62015-08-11 17:14:32 -0700857
858 return ret;
Andrew Schwartzmeyerb5960e62015-08-11 17:14:32 -0700859}
860
Philippe Reynes5e8456f2017-03-08 23:41:04 +0100861static bool
862netvsc_validate_ethtool_ss_cmd(const struct ethtool_link_ksettings *cmd)
sixiao@microsoft.com49eb9382016-02-25 15:24:08 -0800863{
Philippe Reynes5e8456f2017-03-08 23:41:04 +0100864 struct ethtool_link_ksettings diff1 = *cmd;
865 struct ethtool_link_ksettings diff2 = {};
sixiao@microsoft.com49eb9382016-02-25 15:24:08 -0800866
Philippe Reynes5e8456f2017-03-08 23:41:04 +0100867 diff1.base.speed = 0;
868 diff1.base.duplex = 0;
sixiao@microsoft.com49eb9382016-02-25 15:24:08 -0800869 /* advertising and cmd are usually set */
Philippe Reynes5e8456f2017-03-08 23:41:04 +0100870 ethtool_link_ksettings_zero_link_mode(&diff1, advertising);
871 diff1.base.cmd = 0;
sixiao@microsoft.com49eb9382016-02-25 15:24:08 -0800872 /* We set port to PORT_OTHER */
Philippe Reynes5e8456f2017-03-08 23:41:04 +0100873 diff2.base.port = PORT_OTHER;
sixiao@microsoft.com49eb9382016-02-25 15:24:08 -0800874
875 return !memcmp(&diff1, &diff2, sizeof(diff1));
876}
877
878static void netvsc_init_settings(struct net_device *dev)
879{
880 struct net_device_context *ndc = netdev_priv(dev);
881
882 ndc->speed = SPEED_UNKNOWN;
Simon Xiaof3c9d40e2017-04-14 14:42:58 -0700883 ndc->duplex = DUPLEX_FULL;
sixiao@microsoft.com49eb9382016-02-25 15:24:08 -0800884}
885
Philippe Reynes5e8456f2017-03-08 23:41:04 +0100886static int netvsc_get_link_ksettings(struct net_device *dev,
887 struct ethtool_link_ksettings *cmd)
sixiao@microsoft.com49eb9382016-02-25 15:24:08 -0800888{
889 struct net_device_context *ndc = netdev_priv(dev);
890
Philippe Reynes5e8456f2017-03-08 23:41:04 +0100891 cmd->base.speed = ndc->speed;
892 cmd->base.duplex = ndc->duplex;
893 cmd->base.port = PORT_OTHER;
sixiao@microsoft.com49eb9382016-02-25 15:24:08 -0800894
895 return 0;
896}
897
Philippe Reynes5e8456f2017-03-08 23:41:04 +0100898static int netvsc_set_link_ksettings(struct net_device *dev,
899 const struct ethtool_link_ksettings *cmd)
sixiao@microsoft.com49eb9382016-02-25 15:24:08 -0800900{
901 struct net_device_context *ndc = netdev_priv(dev);
902 u32 speed;
903
Philippe Reynes5e8456f2017-03-08 23:41:04 +0100904 speed = cmd->base.speed;
sixiao@microsoft.com49eb9382016-02-25 15:24:08 -0800905 if (!ethtool_validate_speed(speed) ||
Philippe Reynes5e8456f2017-03-08 23:41:04 +0100906 !ethtool_validate_duplex(cmd->base.duplex) ||
sixiao@microsoft.com49eb9382016-02-25 15:24:08 -0800907 !netvsc_validate_ethtool_ss_cmd(cmd))
908 return -EINVAL;
909
910 ndc->speed = speed;
Philippe Reynes5e8456f2017-03-08 23:41:04 +0100911 ndc->duplex = cmd->base.duplex;
sixiao@microsoft.com49eb9382016-02-25 15:24:08 -0800912
913 return 0;
914}
915
Haiyang Zhang4d447c92011-12-15 13:45:17 -0800916static int netvsc_change_mtu(struct net_device *ndev, int mtu)
917{
918 struct net_device_context *ndevctx = netdev_priv(ndev);
stephen hemminger0c195562017-08-01 19:58:53 -0700919 struct net_device *vf_netdev = rtnl_dereference(ndevctx->vf_netdev);
stephen hemminger545a8e72017-03-22 14:51:00 -0700920 struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
Vitaly Kuznetsov3d541ac2016-05-13 13:55:22 +0200921 struct hv_device *hdev = ndevctx->device_ctx;
stephen hemminger9749fed2017-07-19 11:53:16 -0700922 int orig_mtu = ndev->mtu;
Haiyang Zhang4d447c92011-12-15 13:45:17 -0800923 struct netvsc_device_info device_info;
stephen hemmingerea383bf2017-07-19 11:53:15 -0700924 bool was_opened;
stephen hemminger9749fed2017-07-19 11:53:16 -0700925 int ret = 0;
Haiyang Zhang4d447c92011-12-15 13:45:17 -0800926
stephen hemmingera0be4502017-03-22 14:51:01 -0700927 if (!nvdev || nvdev->destroy)
Haiyang Zhang4d447c92011-12-15 13:45:17 -0800928 return -ENODEV;
929
stephen hemminger0c195562017-08-01 19:58:53 -0700930 /* Change MTU of underlying VF netdev first. */
931 if (vf_netdev) {
932 ret = dev_set_mtu(vf_netdev, mtu);
933 if (ret)
934 return ret;
935 }
936
stephen hemmingerea383bf2017-07-19 11:53:15 -0700937 netif_device_detach(ndev);
938 was_opened = rndis_filter_opened(nvdev);
939 if (was_opened)
940 rndis_filter_close(nvdev);
Haiyang Zhang2de85302015-07-13 13:09:16 -0700941
Andrew Schwartzmeyer8ebdcc52015-08-11 17:14:31 -0700942 memset(&device_info, 0, sizeof(device_info));
Haiyang Zhang4d447c92011-12-15 13:45:17 -0800943 device_info.ring_size = ring_size;
stephen hemminger2b018882017-01-24 13:06:03 -0800944 device_info.num_chn = nvdev->num_chn;
Dexuan Cui152669b2017-03-02 13:00:53 +0000945
Dexuan Cui152669b2017-03-02 13:00:53 +0000946 rndis_filter_device_remove(hdev, nvdev);
947
Dexuan Cui152669b2017-03-02 13:00:53 +0000948 ndev->mtu = mtu;
949
stephen hemminger9749fed2017-07-19 11:53:16 -0700950 nvdev = rndis_filter_device_add(hdev, &device_info);
951 if (IS_ERR(nvdev)) {
952 ret = PTR_ERR(nvdev);
953
954 /* Attempt rollback to original MTU */
955 ndev->mtu = orig_mtu;
956 rndis_filter_device_add(hdev, &device_info);
stephen hemminger0c195562017-08-01 19:58:53 -0700957
958 if (vf_netdev)
959 dev_set_mtu(vf_netdev, orig_mtu);
stephen hemminger9749fed2017-07-19 11:53:16 -0700960 }
Haiyang Zhang4d447c92011-12-15 13:45:17 -0800961
stephen hemmingerea383bf2017-07-19 11:53:15 -0700962 if (was_opened)
963 rndis_filter_open(nvdev);
964
965 netif_device_attach(ndev);
stephen hemminger163891d2017-03-22 14:50:58 -0700966
Vitaly Kuznetsov1bdcec82016-05-13 13:55:21 +0200967 /* We may have missed link change notifications */
968 schedule_delayed_work(&ndevctx->dwork, 0);
969
stephen hemminger9749fed2017-07-19 11:53:16 -0700970 return ret;
Haiyang Zhang4d447c92011-12-15 13:45:17 -0800971}
972
stephen hemminger0c195562017-08-01 19:58:53 -0700973static void netvsc_get_vf_stats(struct net_device *net,
974 struct netvsc_vf_pcpu_stats *tot)
975{
976 struct net_device_context *ndev_ctx = netdev_priv(net);
977 int i;
978
979 memset(tot, 0, sizeof(*tot));
980
981 for_each_possible_cpu(i) {
982 const struct netvsc_vf_pcpu_stats *stats
983 = per_cpu_ptr(ndev_ctx->vf_stats, i);
984 u64 rx_packets, rx_bytes, tx_packets, tx_bytes;
985 unsigned int start;
986
987 do {
988 start = u64_stats_fetch_begin_irq(&stats->syncp);
989 rx_packets = stats->rx_packets;
990 tx_packets = stats->tx_packets;
991 rx_bytes = stats->rx_bytes;
992 tx_bytes = stats->tx_bytes;
993 } while (u64_stats_fetch_retry_irq(&stats->syncp, start));
994
995 tot->rx_packets += rx_packets;
996 tot->tx_packets += tx_packets;
997 tot->rx_bytes += rx_bytes;
998 tot->tx_bytes += tx_bytes;
999 tot->tx_dropped += stats->tx_dropped;
1000 }
1001}
1002
stephen hemmingerbc1f4472017-01-06 19:12:52 -08001003static void netvsc_get_stats64(struct net_device *net,
1004 struct rtnl_link_stats64 *t)
sixiao@microsoft.com7eafd9b2015-05-14 01:00:25 -07001005{
1006 struct net_device_context *ndev_ctx = netdev_priv(net);
stephen hemminger776e7262017-04-14 14:42:57 -07001007 struct netvsc_device *nvdev = rcu_dereference_rtnl(ndev_ctx->nvdev);
stephen hemminger0c195562017-08-01 19:58:53 -07001008 struct netvsc_vf_pcpu_stats vf_tot;
1009 int i;
sixiao@microsoft.com7eafd9b2015-05-14 01:00:25 -07001010
Simon Xiao6c80f3f2017-01-24 13:06:13 -08001011 if (!nvdev)
1012 return;
1013
stephen hemminger0c195562017-08-01 19:58:53 -07001014 netdev_stats_to_stats64(t, &net->stats);
1015
1016 netvsc_get_vf_stats(net, &vf_tot);
1017 t->rx_packets += vf_tot.rx_packets;
1018 t->tx_packets += vf_tot.tx_packets;
1019 t->rx_bytes += vf_tot.rx_bytes;
1020 t->tx_bytes += vf_tot.tx_bytes;
1021 t->tx_dropped += vf_tot.tx_dropped;
1022
Simon Xiao6c80f3f2017-01-24 13:06:13 -08001023 for (i = 0; i < nvdev->num_chn; i++) {
1024 const struct netvsc_channel *nvchan = &nvdev->chan_table[i];
1025 const struct netvsc_stats *stats;
1026 u64 packets, bytes, multicast;
sixiao@microsoft.com7eafd9b2015-05-14 01:00:25 -07001027 unsigned int start;
1028
Simon Xiao6c80f3f2017-01-24 13:06:13 -08001029 stats = &nvchan->tx_stats;
sixiao@microsoft.com7eafd9b2015-05-14 01:00:25 -07001030 do {
Simon Xiao6c80f3f2017-01-24 13:06:13 -08001031 start = u64_stats_fetch_begin_irq(&stats->syncp);
1032 packets = stats->packets;
1033 bytes = stats->bytes;
1034 } while (u64_stats_fetch_retry_irq(&stats->syncp, start));
sixiao@microsoft.com7eafd9b2015-05-14 01:00:25 -07001035
Simon Xiao6c80f3f2017-01-24 13:06:13 -08001036 t->tx_bytes += bytes;
1037 t->tx_packets += packets;
1038
1039 stats = &nvchan->rx_stats;
sixiao@microsoft.com7eafd9b2015-05-14 01:00:25 -07001040 do {
Simon Xiao6c80f3f2017-01-24 13:06:13 -08001041 start = u64_stats_fetch_begin_irq(&stats->syncp);
1042 packets = stats->packets;
1043 bytes = stats->bytes;
1044 multicast = stats->multicast + stats->broadcast;
1045 } while (u64_stats_fetch_retry_irq(&stats->syncp, start));
sixiao@microsoft.com7eafd9b2015-05-14 01:00:25 -07001046
Simon Xiao6c80f3f2017-01-24 13:06:13 -08001047 t->rx_bytes += bytes;
1048 t->rx_packets += packets;
1049 t->multicast += multicast;
sixiao@microsoft.com7eafd9b2015-05-14 01:00:25 -07001050 }
sixiao@microsoft.com7eafd9b2015-05-14 01:00:25 -07001051}
Haiyang Zhang1ce09e82012-07-10 07:19:22 +00001052
1053static int netvsc_set_mac_addr(struct net_device *ndev, void *p)
1054{
stephen hemminger867047c2017-07-28 08:59:42 -07001055 struct net_device_context *ndc = netdev_priv(ndev);
stephen hemminger16ba3262017-08-09 17:46:05 -07001056 struct net_device *vf_netdev = rtnl_dereference(ndc->vf_netdev);
stephen hemminger867047c2017-07-28 08:59:42 -07001057 struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
Haiyang Zhang1ce09e82012-07-10 07:19:22 +00001058 struct sockaddr *addr = p;
Haiyang Zhang1ce09e82012-07-10 07:19:22 +00001059 int err;
1060
stephen hemminger16ba3262017-08-09 17:46:05 -07001061 err = eth_prepare_mac_addr_change(ndev, p);
1062 if (err)
Haiyang Zhang1ce09e82012-07-10 07:19:22 +00001063 return err;
1064
stephen hemminger867047c2017-07-28 08:59:42 -07001065 if (!nvdev)
1066 return -ENODEV;
1067
stephen hemminger16ba3262017-08-09 17:46:05 -07001068 if (vf_netdev) {
1069 err = dev_set_mac_address(vf_netdev, addr);
1070 if (err)
1071 return err;
1072 }
1073
stephen hemminger867047c2017-07-28 08:59:42 -07001074 err = rndis_filter_set_device_mac(nvdev, addr->sa_data);
stephen hemminger16ba3262017-08-09 17:46:05 -07001075 if (!err) {
1076 eth_commit_mac_addr_change(ndev, p);
1077 } else if (vf_netdev) {
1078 /* rollback change on VF */
1079 memcpy(addr->sa_data, ndev->dev_addr, ETH_ALEN);
1080 dev_set_mac_address(vf_netdev, addr);
Haiyang Zhang1ce09e82012-07-10 07:19:22 +00001081 }
1082
1083 return err;
1084}
1085
Stephen Hemminger4323b472016-08-23 12:17:57 -07001086static const struct {
1087 char name[ETH_GSTRING_LEN];
1088 u16 offset;
1089} netvsc_stats[] = {
1090 { "tx_scattered", offsetof(struct netvsc_ethtool_stats, tx_scattered) },
1091 { "tx_no_memory", offsetof(struct netvsc_ethtool_stats, tx_no_memory) },
1092 { "tx_no_space", offsetof(struct netvsc_ethtool_stats, tx_no_space) },
1093 { "tx_too_big", offsetof(struct netvsc_ethtool_stats, tx_too_big) },
1094 { "tx_busy", offsetof(struct netvsc_ethtool_stats, tx_busy) },
stephen hemminger0c195562017-08-01 19:58:53 -07001095}, vf_stats[] = {
1096 { "vf_rx_packets", offsetof(struct netvsc_vf_pcpu_stats, rx_packets) },
1097 { "vf_rx_bytes", offsetof(struct netvsc_vf_pcpu_stats, rx_bytes) },
1098 { "vf_tx_packets", offsetof(struct netvsc_vf_pcpu_stats, tx_packets) },
1099 { "vf_tx_bytes", offsetof(struct netvsc_vf_pcpu_stats, tx_bytes) },
1100 { "vf_tx_dropped", offsetof(struct netvsc_vf_pcpu_stats, tx_dropped) },
Stephen Hemminger4323b472016-08-23 12:17:57 -07001101};
1102
Simon Xiao6c80f3f2017-01-24 13:06:13 -08001103#define NETVSC_GLOBAL_STATS_LEN ARRAY_SIZE(netvsc_stats)
stephen hemminger0c195562017-08-01 19:58:53 -07001104#define NETVSC_VF_STATS_LEN ARRAY_SIZE(vf_stats)
Simon Xiao6c80f3f2017-01-24 13:06:13 -08001105
1106/* 4 statistics per queue (rx/tx packets/bytes) */
1107#define NETVSC_QUEUE_STATS_LEN(dev) ((dev)->num_chn * 4)
1108
Stephen Hemminger4323b472016-08-23 12:17:57 -07001109static int netvsc_get_sset_count(struct net_device *dev, int string_set)
1110{
Simon Xiao6c80f3f2017-01-24 13:06:13 -08001111 struct net_device_context *ndc = netdev_priv(dev);
stephen hemmingerfbd4c7e2017-06-07 15:53:47 -07001112 struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
stephen hemminger545a8e72017-03-22 14:51:00 -07001113
1114 if (!nvdev)
1115 return -ENODEV;
Simon Xiao6c80f3f2017-01-24 13:06:13 -08001116
Stephen Hemminger4323b472016-08-23 12:17:57 -07001117 switch (string_set) {
1118 case ETH_SS_STATS:
stephen hemminger0c195562017-08-01 19:58:53 -07001119 return NETVSC_GLOBAL_STATS_LEN
1120 + NETVSC_VF_STATS_LEN
1121 + NETVSC_QUEUE_STATS_LEN(nvdev);
Stephen Hemminger4323b472016-08-23 12:17:57 -07001122 default:
1123 return -EINVAL;
1124 }
1125}
1126
1127static void netvsc_get_ethtool_stats(struct net_device *dev,
1128 struct ethtool_stats *stats, u64 *data)
1129{
1130 struct net_device_context *ndc = netdev_priv(dev);
stephen hemminger867047c2017-07-28 08:59:42 -07001131 struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
Stephen Hemminger4323b472016-08-23 12:17:57 -07001132 const void *nds = &ndc->eth_stats;
Simon Xiao6c80f3f2017-01-24 13:06:13 -08001133 const struct netvsc_stats *qstats;
stephen hemminger0c195562017-08-01 19:58:53 -07001134 struct netvsc_vf_pcpu_stats sum;
Simon Xiao6c80f3f2017-01-24 13:06:13 -08001135 unsigned int start;
1136 u64 packets, bytes;
1137 int i, j;
Stephen Hemminger4323b472016-08-23 12:17:57 -07001138
stephen hemminger545a8e72017-03-22 14:51:00 -07001139 if (!nvdev)
1140 return;
1141
Simon Xiao6c80f3f2017-01-24 13:06:13 -08001142 for (i = 0; i < NETVSC_GLOBAL_STATS_LEN; i++)
Stephen Hemminger4323b472016-08-23 12:17:57 -07001143 data[i] = *(unsigned long *)(nds + netvsc_stats[i].offset);
Simon Xiao6c80f3f2017-01-24 13:06:13 -08001144
stephen hemminger0c195562017-08-01 19:58:53 -07001145 netvsc_get_vf_stats(dev, &sum);
1146 for (j = 0; j < NETVSC_VF_STATS_LEN; j++)
1147 data[i++] = *(u64 *)((void *)&sum + vf_stats[j].offset);
1148
Simon Xiao6c80f3f2017-01-24 13:06:13 -08001149 for (j = 0; j < nvdev->num_chn; j++) {
1150 qstats = &nvdev->chan_table[j].tx_stats;
1151
1152 do {
1153 start = u64_stats_fetch_begin_irq(&qstats->syncp);
1154 packets = qstats->packets;
1155 bytes = qstats->bytes;
1156 } while (u64_stats_fetch_retry_irq(&qstats->syncp, start));
1157 data[i++] = packets;
1158 data[i++] = bytes;
1159
1160 qstats = &nvdev->chan_table[j].rx_stats;
1161 do {
1162 start = u64_stats_fetch_begin_irq(&qstats->syncp);
1163 packets = qstats->packets;
1164 bytes = qstats->bytes;
1165 } while (u64_stats_fetch_retry_irq(&qstats->syncp, start));
1166 data[i++] = packets;
1167 data[i++] = bytes;
1168 }
Stephen Hemminger4323b472016-08-23 12:17:57 -07001169}
1170
1171static void netvsc_get_strings(struct net_device *dev, u32 stringset, u8 *data)
1172{
Simon Xiao6c80f3f2017-01-24 13:06:13 -08001173 struct net_device_context *ndc = netdev_priv(dev);
stephen hemminger867047c2017-07-28 08:59:42 -07001174 struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
Simon Xiao6c80f3f2017-01-24 13:06:13 -08001175 u8 *p = data;
Stephen Hemminger4323b472016-08-23 12:17:57 -07001176 int i;
1177
stephen hemminger545a8e72017-03-22 14:51:00 -07001178 if (!nvdev)
1179 return;
1180
Stephen Hemminger4323b472016-08-23 12:17:57 -07001181 switch (stringset) {
1182 case ETH_SS_STATS:
stephen hemminger0c195562017-08-01 19:58:53 -07001183 for (i = 0; i < ARRAY_SIZE(netvsc_stats); i++) {
1184 memcpy(p, netvsc_stats[i].name, ETH_GSTRING_LEN);
1185 p += ETH_GSTRING_LEN;
1186 }
Simon Xiao6c80f3f2017-01-24 13:06:13 -08001187
stephen hemminger0c195562017-08-01 19:58:53 -07001188 for (i = 0; i < ARRAY_SIZE(vf_stats); i++) {
1189 memcpy(p, vf_stats[i].name, ETH_GSTRING_LEN);
1190 p += ETH_GSTRING_LEN;
1191 }
1192
Simon Xiao6c80f3f2017-01-24 13:06:13 -08001193 for (i = 0; i < nvdev->num_chn; i++) {
1194 sprintf(p, "tx_queue_%u_packets", i);
1195 p += ETH_GSTRING_LEN;
1196 sprintf(p, "tx_queue_%u_bytes", i);
1197 p += ETH_GSTRING_LEN;
1198 sprintf(p, "rx_queue_%u_packets", i);
1199 p += ETH_GSTRING_LEN;
1200 sprintf(p, "rx_queue_%u_bytes", i);
1201 p += ETH_GSTRING_LEN;
1202 }
1203
Stephen Hemminger4323b472016-08-23 12:17:57 -07001204 break;
1205 }
1206}
1207
stephen hemmingerb448f4e2017-01-24 13:06:00 -08001208static int
stephen hemmingerb5a5dc82017-01-24 13:06:01 -08001209netvsc_get_rss_hash_opts(struct netvsc_device *nvdev,
1210 struct ethtool_rxnfc *info)
1211{
1212 info->data = RXH_IP_SRC | RXH_IP_DST;
1213
1214 switch (info->flow_type) {
1215 case TCP_V4_FLOW:
1216 case TCP_V6_FLOW:
1217 info->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3;
1218 /* fallthrough */
1219 case UDP_V4_FLOW:
1220 case UDP_V6_FLOW:
1221 case IPV4_FLOW:
1222 case IPV6_FLOW:
1223 break;
1224 default:
1225 info->data = 0;
1226 break;
1227 }
1228
1229 return 0;
1230}
1231
1232static int
stephen hemmingerb448f4e2017-01-24 13:06:00 -08001233netvsc_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
1234 u32 *rules)
1235{
1236 struct net_device_context *ndc = netdev_priv(dev);
stephen hemminger867047c2017-07-28 08:59:42 -07001237 struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
stephen hemminger545a8e72017-03-22 14:51:00 -07001238
1239 if (!nvdev)
1240 return -ENODEV;
stephen hemmingerb448f4e2017-01-24 13:06:00 -08001241
1242 switch (info->cmd) {
1243 case ETHTOOL_GRXRINGS:
1244 info->data = nvdev->num_chn;
1245 return 0;
stephen hemmingerb5a5dc82017-01-24 13:06:01 -08001246
1247 case ETHTOOL_GRXFH:
1248 return netvsc_get_rss_hash_opts(nvdev, info);
stephen hemmingerb448f4e2017-01-24 13:06:00 -08001249 }
1250 return -EOPNOTSUPP;
1251}
1252
Richard Weinberger316158f2014-07-09 16:23:59 +02001253#ifdef CONFIG_NET_POLL_CONTROLLER
stephen hemmingera5ecd432017-06-07 15:53:48 -07001254static void netvsc_poll_controller(struct net_device *dev)
Richard Weinberger316158f2014-07-09 16:23:59 +02001255{
stephen hemmingera5ecd432017-06-07 15:53:48 -07001256 struct net_device_context *ndc = netdev_priv(dev);
1257 struct netvsc_device *ndev;
1258 int i;
1259
1260 rcu_read_lock();
1261 ndev = rcu_dereference(ndc->nvdev);
1262 if (ndev) {
1263 for (i = 0; i < ndev->num_chn; i++) {
1264 struct netvsc_channel *nvchan = &ndev->chan_table[i];
1265
1266 napi_schedule(&nvchan->napi);
1267 }
1268 }
1269 rcu_read_unlock();
Richard Weinberger316158f2014-07-09 16:23:59 +02001270}
1271#endif
Haiyang Zhang1ce09e82012-07-10 07:19:22 +00001272
stephen hemminger962f3fe2017-01-24 13:06:02 -08001273static u32 netvsc_get_rxfh_key_size(struct net_device *dev)
1274{
1275 return NETVSC_HASH_KEYLEN;
1276}
1277
1278static u32 netvsc_rss_indir_size(struct net_device *dev)
1279{
stephen hemmingerff4a4412017-01-24 13:06:04 -08001280 return ITAB_NUM;
stephen hemminger962f3fe2017-01-24 13:06:02 -08001281}
1282
1283static int netvsc_get_rxfh(struct net_device *dev, u32 *indir, u8 *key,
1284 u8 *hfunc)
1285{
1286 struct net_device_context *ndc = netdev_priv(dev);
stephen hemminger867047c2017-07-28 08:59:42 -07001287 struct netvsc_device *ndev = rtnl_dereference(ndc->nvdev);
Colin Ian Kingeb996ed2017-03-25 14:26:39 +00001288 struct rndis_device *rndis_dev;
stephen hemmingerff4a4412017-01-24 13:06:04 -08001289 int i;
stephen hemminger962f3fe2017-01-24 13:06:02 -08001290
stephen hemminger545a8e72017-03-22 14:51:00 -07001291 if (!ndev)
1292 return -ENODEV;
1293
stephen hemminger962f3fe2017-01-24 13:06:02 -08001294 if (hfunc)
1295 *hfunc = ETH_RSS_HASH_TOP; /* Toeplitz */
1296
Colin Ian Kingeb996ed2017-03-25 14:26:39 +00001297 rndis_dev = ndev->extension;
stephen hemmingerff4a4412017-01-24 13:06:04 -08001298 if (indir) {
1299 for (i = 0; i < ITAB_NUM; i++)
1300 indir[i] = rndis_dev->ind_table[i];
1301 }
1302
stephen hemminger962f3fe2017-01-24 13:06:02 -08001303 if (key)
1304 memcpy(key, rndis_dev->rss_key, NETVSC_HASH_KEYLEN);
1305
1306 return 0;
1307}
1308
1309static int netvsc_set_rxfh(struct net_device *dev, const u32 *indir,
1310 const u8 *key, const u8 hfunc)
1311{
1312 struct net_device_context *ndc = netdev_priv(dev);
stephen hemminger545a8e72017-03-22 14:51:00 -07001313 struct netvsc_device *ndev = rtnl_dereference(ndc->nvdev);
Colin Ian Kingeb996ed2017-03-25 14:26:39 +00001314 struct rndis_device *rndis_dev;
stephen hemmingerff4a4412017-01-24 13:06:04 -08001315 int i;
stephen hemminger962f3fe2017-01-24 13:06:02 -08001316
stephen hemminger545a8e72017-03-22 14:51:00 -07001317 if (!ndev)
1318 return -ENODEV;
1319
stephen hemminger962f3fe2017-01-24 13:06:02 -08001320 if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)
1321 return -EOPNOTSUPP;
1322
Colin Ian Kingeb996ed2017-03-25 14:26:39 +00001323 rndis_dev = ndev->extension;
stephen hemmingerff4a4412017-01-24 13:06:04 -08001324 if (indir) {
1325 for (i = 0; i < ITAB_NUM; i++)
Arnd Bergmannb92b7d32017-06-22 00:16:37 +02001326 if (indir[i] >= VRSS_CHANNEL_MAX)
stephen hemmingerff4a4412017-01-24 13:06:04 -08001327 return -EINVAL;
1328
1329 for (i = 0; i < ITAB_NUM; i++)
1330 rndis_dev->ind_table[i] = indir[i];
1331 }
1332
1333 if (!key) {
1334 if (!indir)
1335 return 0;
1336
1337 key = rndis_dev->rss_key;
1338 }
stephen hemminger962f3fe2017-01-24 13:06:02 -08001339
1340 return rndis_filter_set_rss_param(rndis_dev, key, ndev->num_chn);
1341}
1342
Stephen Hemmingerf82f4ad2010-05-04 09:58:57 -07001343static const struct ethtool_ops ethtool_ops = {
1344 .get_drvinfo = netvsc_get_drvinfo,
Stephen Hemmingerf82f4ad2010-05-04 09:58:57 -07001345 .get_link = ethtool_op_get_link,
Stephen Hemminger4323b472016-08-23 12:17:57 -07001346 .get_ethtool_stats = netvsc_get_ethtool_stats,
1347 .get_sset_count = netvsc_get_sset_count,
1348 .get_strings = netvsc_get_strings,
Andrew Schwartzmeyer59995372015-02-26 16:27:14 -08001349 .get_channels = netvsc_get_channels,
Andrew Schwartzmeyerb5960e62015-08-11 17:14:32 -07001350 .set_channels = netvsc_set_channels,
sixiao@microsoft.com76d13b52016-02-17 16:43:59 -08001351 .get_ts_info = ethtool_op_get_ts_info,
stephen hemmingerb448f4e2017-01-24 13:06:00 -08001352 .get_rxnfc = netvsc_get_rxnfc,
stephen hemminger962f3fe2017-01-24 13:06:02 -08001353 .get_rxfh_key_size = netvsc_get_rxfh_key_size,
1354 .get_rxfh_indir_size = netvsc_rss_indir_size,
1355 .get_rxfh = netvsc_get_rxfh,
1356 .set_rxfh = netvsc_set_rxfh,
Philippe Reynes5e8456f2017-03-08 23:41:04 +01001357 .get_link_ksettings = netvsc_get_link_ksettings,
1358 .set_link_ksettings = netvsc_set_link_ksettings,
Stephen Hemmingerf82f4ad2010-05-04 09:58:57 -07001359};
1360
Greg Kroah-Hartmandf2fff22009-08-31 21:11:12 -07001361static const struct net_device_ops device_ops = {
1362 .ndo_open = netvsc_open,
1363 .ndo_stop = netvsc_close,
1364 .ndo_start_xmit = netvsc_start_xmit,
Jiri Pirkoafc4b132011-08-16 06:29:01 +00001365 .ndo_set_rx_mode = netvsc_set_multicast_list,
Haiyang Zhang4d447c92011-12-15 13:45:17 -08001366 .ndo_change_mtu = netvsc_change_mtu,
Haiyang Zhangb681b582010-08-03 19:15:31 +00001367 .ndo_validate_addr = eth_validate_addr,
Haiyang Zhang1ce09e82012-07-10 07:19:22 +00001368 .ndo_set_mac_address = netvsc_set_mac_addr,
Haiyang Zhang5b54dac2014-04-21 10:20:28 -07001369 .ndo_select_queue = netvsc_select_queue,
sixiao@microsoft.com7eafd9b2015-05-14 01:00:25 -07001370 .ndo_get_stats64 = netvsc_get_stats64,
Richard Weinberger316158f2014-07-09 16:23:59 +02001371#ifdef CONFIG_NET_POLL_CONTROLLER
1372 .ndo_poll_controller = netvsc_poll_controller,
1373#endif
Greg Kroah-Hartmandf2fff22009-08-31 21:11:12 -07001374};
1375
Haiyang Zhangc996edc2011-04-06 15:18:00 -07001376/*
Vitaly Kuznetsov27a70af2015-11-27 11:39:55 +01001377 * Handle link status changes. For RNDIS_STATUS_NETWORK_CHANGE emulate link
1378 * down/up sequence. In case of RNDIS_STATUS_MEDIA_CONNECT when carrier is
1379 * present send GARP packet to network peers with netif_notify_peers().
Haiyang Zhangc996edc2011-04-06 15:18:00 -07001380 */
Haiyang Zhang891de742014-02-12 16:54:27 -08001381static void netvsc_link_change(struct work_struct *w)
Haiyang Zhangc996edc2011-04-06 15:18:00 -07001382{
Vitaly Kuznetsov0a1275c2016-05-13 13:55:23 +02001383 struct net_device_context *ndev_ctx =
1384 container_of(w, struct net_device_context, dwork.work);
1385 struct hv_device *device_obj = ndev_ctx->device_ctx;
1386 struct net_device *net = hv_get_drvdata(device_obj);
K. Y. Srinivasan2ddd5e5f2011-09-13 10:59:49 -07001387 struct netvsc_device *net_device;
Haiyang Zhang891de742014-02-12 16:54:27 -08001388 struct rndis_device *rdev;
Vitaly Kuznetsov27a70af2015-11-27 11:39:55 +01001389 struct netvsc_reconfig *event = NULL;
1390 bool notify = false, reschedule = false;
1391 unsigned long flags, next_reconfig, delay;
Haiyang Zhangc996edc2011-04-06 15:18:00 -07001392
Vitaly Kuznetsov1bdcec82016-05-13 13:55:21 +02001393 rtnl_lock();
stephen hemmingera0be4502017-03-22 14:51:01 -07001394 net_device = rtnl_dereference(ndev_ctx->nvdev);
1395 if (!net_device)
Vitaly Kuznetsov1bdcec82016-05-13 13:55:21 +02001396 goto out_unlock;
1397
Haiyang Zhang891de742014-02-12 16:54:27 -08001398 rdev = net_device->extension;
Haiyang Zhang891de742014-02-12 16:54:27 -08001399
Vitaly Kuznetsov27a70af2015-11-27 11:39:55 +01001400 next_reconfig = ndev_ctx->last_reconfig + LINKCHANGE_INT;
1401 if (time_is_after_jiffies(next_reconfig)) {
1402 /* link_watch only sends one notification with current state
1403 * per second, avoid doing reconfig more frequently. Handle
1404 * wrap around.
1405 */
1406 delay = next_reconfig - jiffies;
1407 delay = delay < LINKCHANGE_INT ? delay : LINKCHANGE_INT;
1408 schedule_delayed_work(&ndev_ctx->dwork, delay);
Vitaly Kuznetsov1bdcec82016-05-13 13:55:21 +02001409 goto out_unlock;
Vitaly Kuznetsov27a70af2015-11-27 11:39:55 +01001410 }
1411 ndev_ctx->last_reconfig = jiffies;
1412
1413 spin_lock_irqsave(&ndev_ctx->lock, flags);
1414 if (!list_empty(&ndev_ctx->reconfig_events)) {
1415 event = list_first_entry(&ndev_ctx->reconfig_events,
1416 struct netvsc_reconfig, list);
1417 list_del(&event->list);
1418 reschedule = !list_empty(&ndev_ctx->reconfig_events);
1419 }
1420 spin_unlock_irqrestore(&ndev_ctx->lock, flags);
1421
1422 if (!event)
Vitaly Kuznetsov1bdcec82016-05-13 13:55:21 +02001423 goto out_unlock;
Vitaly Kuznetsov27a70af2015-11-27 11:39:55 +01001424
1425 switch (event->event) {
1426 /* Only the following events are possible due to the check in
1427 * netvsc_linkstatus_callback()
1428 */
1429 case RNDIS_STATUS_MEDIA_CONNECT:
1430 if (rdev->link_state) {
1431 rdev->link_state = false;
stephen hemminger0c195562017-08-01 19:58:53 -07001432 netif_carrier_on(net);
Vitaly Kuznetsov27a70af2015-11-27 11:39:55 +01001433 netif_tx_wake_all_queues(net);
1434 } else {
1435 notify = true;
Haiyang Zhang3a494e72014-06-19 18:34:36 -07001436 }
Vitaly Kuznetsov27a70af2015-11-27 11:39:55 +01001437 kfree(event);
1438 break;
1439 case RNDIS_STATUS_MEDIA_DISCONNECT:
1440 if (!rdev->link_state) {
1441 rdev->link_state = true;
1442 netif_carrier_off(net);
1443 netif_tx_stop_all_queues(net);
1444 }
1445 kfree(event);
1446 break;
1447 case RNDIS_STATUS_NETWORK_CHANGE:
1448 /* Only makes sense if carrier is present */
1449 if (!rdev->link_state) {
1450 rdev->link_state = true;
1451 netif_carrier_off(net);
1452 netif_tx_stop_all_queues(net);
1453 event->event = RNDIS_STATUS_MEDIA_CONNECT;
1454 spin_lock_irqsave(&ndev_ctx->lock, flags);
Haiyang Zhang15cfd402016-04-21 16:13:01 -07001455 list_add(&event->list, &ndev_ctx->reconfig_events);
Vitaly Kuznetsov27a70af2015-11-27 11:39:55 +01001456 spin_unlock_irqrestore(&ndev_ctx->lock, flags);
1457 reschedule = true;
1458 }
1459 break;
Haiyang Zhang891de742014-02-12 16:54:27 -08001460 }
1461
1462 rtnl_unlock();
1463
1464 if (notify)
1465 netdev_notify_peers(net);
Vitaly Kuznetsov27a70af2015-11-27 11:39:55 +01001466
1467 /* link_watch only sends one notification with current state per
1468 * second, handle next reconfig event in 2 seconds.
1469 */
1470 if (reschedule)
1471 schedule_delayed_work(&ndev_ctx->dwork, LINKCHANGE_INT);
Vitaly Kuznetsov1bdcec82016-05-13 13:55:21 +02001472
1473 return;
1474
1475out_unlock:
1476 rtnl_unlock();
Haiyang Zhangc996edc2011-04-06 15:18:00 -07001477}
1478
Stephen Hemmingere8ff40d2016-09-22 16:56:32 -07001479static struct net_device *get_netvsc_bymac(const u8 *mac)
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001480{
Stephen Hemmingere8ff40d2016-09-22 16:56:32 -07001481 struct net_device *dev;
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001482
Stephen Hemminger8737caa2016-08-23 12:17:44 -07001483 ASSERT_RTNL();
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001484
1485 for_each_netdev(&init_net, dev) {
Stephen Hemmingere8ff40d2016-09-22 16:56:32 -07001486 if (dev->netdev_ops != &device_ops)
1487 continue; /* not a netvsc device */
1488
1489 if (ether_addr_equal(mac, dev->perm_addr))
1490 return dev;
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001491 }
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001492
Stephen Hemmingere8ff40d2016-09-22 16:56:32 -07001493 return NULL;
1494}
1495
Stephen Hemmingerf207c102016-09-22 16:56:33 -07001496static struct net_device *get_netvsc_byref(struct net_device *vf_netdev)
Stephen Hemmingere8ff40d2016-09-22 16:56:32 -07001497{
1498 struct net_device *dev;
1499
1500 ASSERT_RTNL();
1501
1502 for_each_netdev(&init_net, dev) {
1503 struct net_device_context *net_device_ctx;
1504
1505 if (dev->netdev_ops != &device_ops)
1506 continue; /* not a netvsc device */
1507
1508 net_device_ctx = netdev_priv(dev);
stephen hemminger79e8cbe2017-07-19 11:53:13 -07001509 if (!rtnl_dereference(net_device_ctx->nvdev))
Stephen Hemmingere8ff40d2016-09-22 16:56:32 -07001510 continue; /* device is removed */
1511
Stephen Hemmingerf207c102016-09-22 16:56:33 -07001512 if (rtnl_dereference(net_device_ctx->vf_netdev) == vf_netdev)
Stephen Hemmingere8ff40d2016-09-22 16:56:32 -07001513 return dev; /* a match */
1514 }
1515
1516 return NULL;
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001517}
1518
stephen hemminger0c195562017-08-01 19:58:53 -07001519/* Called when VF is injecting data into network stack.
1520 * Change the associated network device from VF to netvsc.
1521 * note: already called with rcu_read_lock
1522 */
1523static rx_handler_result_t netvsc_vf_handle_frame(struct sk_buff **pskb)
1524{
1525 struct sk_buff *skb = *pskb;
1526 struct net_device *ndev = rcu_dereference(skb->dev->rx_handler_data);
1527 struct net_device_context *ndev_ctx = netdev_priv(ndev);
1528 struct netvsc_vf_pcpu_stats *pcpu_stats
1529 = this_cpu_ptr(ndev_ctx->vf_stats);
1530
1531 skb->dev = ndev;
1532
1533 u64_stats_update_begin(&pcpu_stats->syncp);
1534 pcpu_stats->rx_packets++;
1535 pcpu_stats->rx_bytes += skb->len;
1536 u64_stats_update_end(&pcpu_stats->syncp);
1537
1538 return RX_HANDLER_ANOTHER;
1539}
1540
1541static int netvsc_vf_join(struct net_device *vf_netdev,
1542 struct net_device *ndev)
1543{
1544 struct net_device_context *ndev_ctx = netdev_priv(ndev);
1545 int ret;
1546
1547 ret = netdev_rx_handler_register(vf_netdev,
1548 netvsc_vf_handle_frame, ndev);
1549 if (ret != 0) {
1550 netdev_err(vf_netdev,
1551 "can not register netvsc VF receive handler (err = %d)\n",
1552 ret);
1553 goto rx_handler_failed;
1554 }
1555
1556 ret = netdev_upper_dev_link(vf_netdev, ndev);
1557 if (ret != 0) {
1558 netdev_err(vf_netdev,
1559 "can not set master device %s (err = %d)\n",
1560 ndev->name, ret);
1561 goto upper_link_failed;
1562 }
1563
1564 /* set slave flag before open to prevent IPv6 addrconf */
1565 vf_netdev->flags |= IFF_SLAVE;
1566
stephen hemminger6123c662017-08-09 17:46:03 -07001567 schedule_delayed_work(&ndev_ctx->vf_takeover, VF_TAKEOVER_INT);
1568
1569 call_netdevice_notifiers(NETDEV_JOIN, vf_netdev);
stephen hemminger0c195562017-08-01 19:58:53 -07001570
1571 netdev_info(vf_netdev, "joined to %s\n", ndev->name);
1572 return 0;
1573
1574upper_link_failed:
1575 netdev_rx_handler_unregister(vf_netdev);
1576rx_handler_failed:
1577 return ret;
1578}
1579
1580static void __netvsc_vf_setup(struct net_device *ndev,
1581 struct net_device *vf_netdev)
1582{
1583 int ret;
1584
stephen hemminger0c195562017-08-01 19:58:53 -07001585 /* Align MTU of VF with master */
1586 ret = dev_set_mtu(vf_netdev, ndev->mtu);
1587 if (ret)
1588 netdev_warn(vf_netdev,
1589 "unable to change mtu to %u\n", ndev->mtu);
1590
1591 if (netif_running(ndev)) {
1592 ret = dev_open(vf_netdev);
1593 if (ret)
1594 netdev_warn(vf_netdev,
1595 "unable to open: %d\n", ret);
1596 }
1597}
1598
1599/* Setup VF as slave of the synthetic device.
1600 * Runs in workqueue to avoid recursion in netlink callbacks.
1601 */
1602static void netvsc_vf_setup(struct work_struct *w)
1603{
1604 struct net_device_context *ndev_ctx
stephen hemminger6123c662017-08-09 17:46:03 -07001605 = container_of(w, struct net_device_context, vf_takeover.work);
stephen hemminger0c195562017-08-01 19:58:53 -07001606 struct net_device *ndev = hv_get_drvdata(ndev_ctx->device_ctx);
1607 struct net_device *vf_netdev;
1608
stephen hemmingerfb84af82017-08-04 12:14:00 -07001609 if (!rtnl_trylock()) {
stephen hemminger6123c662017-08-09 17:46:03 -07001610 schedule_delayed_work(&ndev_ctx->vf_takeover, 0);
stephen hemmingerfb84af82017-08-04 12:14:00 -07001611 return;
1612 }
1613
stephen hemminger0c195562017-08-01 19:58:53 -07001614 vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
1615 if (vf_netdev)
1616 __netvsc_vf_setup(ndev, vf_netdev);
1617
1618 rtnl_unlock();
1619}
1620
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001621static int netvsc_register_vf(struct net_device *vf_netdev)
1622{
Vitaly Kuznetsov0a1275c2016-05-13 13:55:23 +02001623 struct net_device *ndev;
1624 struct net_device_context *net_device_ctx;
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001625 struct netvsc_device *netvsc_dev;
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001626
Stephen Hemmingere8ff40d2016-09-22 16:56:32 -07001627 if (vf_netdev->addr_len != ETH_ALEN)
1628 return NOTIFY_DONE;
1629
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001630 /*
1631 * We will use the MAC address to locate the synthetic interface to
1632 * associate with the VF interface. If we don't find a matching
1633 * synthetic interface, move on.
1634 */
Stephen Hemmingere8ff40d2016-09-22 16:56:32 -07001635 ndev = get_netvsc_bymac(vf_netdev->perm_addr);
Vitaly Kuznetsov0a1275c2016-05-13 13:55:23 +02001636 if (!ndev)
1637 return NOTIFY_DONE;
1638
1639 net_device_ctx = netdev_priv(ndev);
stephen hemminger545a8e72017-03-22 14:51:00 -07001640 netvsc_dev = rtnl_dereference(net_device_ctx->nvdev);
Stephen Hemmingerf207c102016-09-22 16:56:33 -07001641 if (!netvsc_dev || rtnl_dereference(net_device_ctx->vf_netdev))
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001642 return NOTIFY_DONE;
1643
stephen hemminger0c195562017-08-01 19:58:53 -07001644 if (netvsc_vf_join(vf_netdev, ndev) != 0)
1645 return NOTIFY_DONE;
1646
Vitaly Kuznetsov0a1275c2016-05-13 13:55:23 +02001647 netdev_info(ndev, "VF registering: %s\n", vf_netdev->name);
stephen hemminger0c195562017-08-01 19:58:53 -07001648
1649 /* Prevent this module from being unloaded while VF is registered */
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001650 try_module_get(THIS_MODULE);
Stephen Hemminger07d0f002016-09-22 16:56:30 -07001651
1652 dev_hold(vf_netdev);
Stephen Hemmingerf207c102016-09-22 16:56:33 -07001653 rcu_assign_pointer(net_device_ctx->vf_netdev, vf_netdev);
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001654 return NOTIFY_OK;
1655}
1656
stephen hemminger7b83f522017-08-07 11:30:00 -07001657static int netvsc_vf_up(struct net_device *vf_netdev)
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001658{
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001659 struct net_device_context *net_device_ctx;
stephen hemminger7b83f522017-08-07 11:30:00 -07001660 struct netvsc_device *netvsc_dev;
stephen hemminger0c195562017-08-01 19:58:53 -07001661 struct net_device *ndev;
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001662
Stephen Hemmingere8ff40d2016-09-22 16:56:32 -07001663 ndev = get_netvsc_byref(vf_netdev);
Vitaly Kuznetsov0a1275c2016-05-13 13:55:23 +02001664 if (!ndev)
1665 return NOTIFY_DONE;
1666
1667 net_device_ctx = netdev_priv(ndev);
stephen hemminger7b83f522017-08-07 11:30:00 -07001668 netvsc_dev = rtnl_dereference(net_device_ctx->nvdev);
1669 if (!netvsc_dev)
1670 return NOTIFY_DONE;
1671
1672 /* Bump refcount when datapath is acvive - Why? */
1673 rndis_filter_open(netvsc_dev);
1674
1675 /* notify the host to switch the data path. */
1676 netvsc_switch_datapath(ndev, true);
1677 netdev_info(ndev, "Data path switched to VF: %s\n", vf_netdev->name);
1678
1679 return NOTIFY_OK;
1680}
1681
1682static int netvsc_vf_down(struct net_device *vf_netdev)
1683{
1684 struct net_device_context *net_device_ctx;
1685 struct netvsc_device *netvsc_dev;
1686 struct net_device *ndev;
1687
1688 ndev = get_netvsc_byref(vf_netdev);
1689 if (!ndev)
1690 return NOTIFY_DONE;
1691
1692 net_device_ctx = netdev_priv(ndev);
1693 netvsc_dev = rtnl_dereference(net_device_ctx->nvdev);
1694 if (!netvsc_dev)
1695 return NOTIFY_DONE;
1696
1697 netvsc_switch_datapath(ndev, false);
1698 netdev_info(ndev, "Data path switched from VF: %s\n", vf_netdev->name);
1699 rndis_filter_close(netvsc_dev);
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001700
1701 return NOTIFY_OK;
1702}
1703
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001704static int netvsc_unregister_vf(struct net_device *vf_netdev)
1705{
Vitaly Kuznetsov0a1275c2016-05-13 13:55:23 +02001706 struct net_device *ndev;
Vitaly Kuznetsov0a1275c2016-05-13 13:55:23 +02001707 struct net_device_context *net_device_ctx;
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001708
Stephen Hemmingere8ff40d2016-09-22 16:56:32 -07001709 ndev = get_netvsc_byref(vf_netdev);
Vitaly Kuznetsov0a1275c2016-05-13 13:55:23 +02001710 if (!ndev)
1711 return NOTIFY_DONE;
1712
1713 net_device_ctx = netdev_priv(ndev);
stephen hemminger6123c662017-08-09 17:46:03 -07001714 cancel_delayed_work_sync(&net_device_ctx->vf_takeover);
Stephen Hemmingere8ff40d2016-09-22 16:56:32 -07001715
Vitaly Kuznetsov0a1275c2016-05-13 13:55:23 +02001716 netdev_info(ndev, "VF unregistering: %s\n", vf_netdev->name);
Stephen Hemmingerf207c102016-09-22 16:56:33 -07001717
stephen hemminger0c195562017-08-01 19:58:53 -07001718 netdev_upper_dev_unlink(vf_netdev, ndev);
Stephen Hemmingerf207c102016-09-22 16:56:33 -07001719 RCU_INIT_POINTER(net_device_ctx->vf_netdev, NULL);
Stephen Hemminger07d0f002016-09-22 16:56:30 -07001720 dev_put(vf_netdev);
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001721 module_put(THIS_MODULE);
1722 return NOTIFY_OK;
1723}
1724
K. Y. Srinivasan84946892011-09-13 10:59:38 -07001725static int netvsc_probe(struct hv_device *dev,
1726 const struct hv_vmbus_device_id *dev_id)
Greg Kroah-Hartmandf2fff22009-08-31 21:11:12 -07001727{
Greg Kroah-Hartmandf2fff22009-08-31 21:11:12 -07001728 struct net_device *net = NULL;
1729 struct net_device_context *net_device_ctx;
1730 struct netvsc_device_info device_info;
Haiyang Zhang5b54dac2014-04-21 10:20:28 -07001731 struct netvsc_device *nvdev;
stephen hemminger0c195562017-08-01 19:58:53 -07001732 int ret = -ENOMEM;
Greg Kroah-Hartmandf2fff22009-08-31 21:11:12 -07001733
Haiyang Zhang5b54dac2014-04-21 10:20:28 -07001734 net = alloc_etherdev_mq(sizeof(struct net_device_context),
stephen hemminger2b018882017-01-24 13:06:03 -08001735 VRSS_CHANNEL_MAX);
Greg Kroah-Hartmandf2fff22009-08-31 21:11:12 -07001736 if (!net)
stephen hemminger0c195562017-08-01 19:58:53 -07001737 goto no_net;
Greg Kroah-Hartmandf2fff22009-08-31 21:11:12 -07001738
Haiyang Zhang1b07da52014-03-04 14:11:06 -08001739 netif_carrier_off(net);
1740
Haiyang Zhangb37879e2016-08-04 10:42:14 -07001741 netvsc_init_settings(net);
1742
Greg Kroah-Hartmandf2fff22009-08-31 21:11:12 -07001743 net_device_ctx = netdev_priv(net);
K. Y. Srinivasan9efd21e2011-04-29 13:45:10 -07001744 net_device_ctx->device_ctx = dev;
Simon Xiao3f300ff2015-04-28 01:05:17 -07001745 net_device_ctx->msg_enable = netif_msg_init(debug, default_msg);
1746 if (netif_msg_probe(net_device_ctx))
1747 netdev_dbg(net, "netvsc msg_enable: %d\n",
1748 net_device_ctx->msg_enable);
1749
K. Y. Srinivasan2ddd5e5f2011-09-13 10:59:49 -07001750 hv_set_drvdata(dev, net);
Vitaly Kuznetsovf580aec2016-05-13 13:55:20 +02001751
Haiyang Zhang891de742014-02-12 16:54:27 -08001752 INIT_DELAYED_WORK(&net_device_ctx->dwork, netvsc_link_change);
Greg Kroah-Hartmandf2fff22009-08-31 21:11:12 -07001753
Vitaly Kuznetsov27a70af2015-11-27 11:39:55 +01001754 spin_lock_init(&net_device_ctx->lock);
1755 INIT_LIST_HEAD(&net_device_ctx->reconfig_events);
stephen hemminger6123c662017-08-09 17:46:03 -07001756 INIT_DELAYED_WORK(&net_device_ctx->vf_takeover, netvsc_vf_setup);
stephen hemminger0c195562017-08-01 19:58:53 -07001757
1758 net_device_ctx->vf_stats
1759 = netdev_alloc_pcpu_stats(struct netvsc_vf_pcpu_stats);
1760 if (!net_device_ctx->vf_stats)
1761 goto no_stats;
Vitaly Kuznetsov27a70af2015-11-27 11:39:55 +01001762
Greg Kroah-Hartmandf2fff22009-08-31 21:11:12 -07001763 net->netdev_ops = &device_ops;
Wilfried Klaebe7ad24ea2014-05-11 00:12:32 +00001764 net->ethtool_ops = &ethtool_ops;
K. Y. Srinivasan9efd21e2011-04-29 13:45:10 -07001765 SET_NETDEV_DEV(net, &dev->device);
Greg Kroah-Hartmandf2fff22009-08-31 21:11:12 -07001766
Vitaly Kuznetsov14a03cf2016-02-05 17:29:08 +01001767 /* We always need headroom for rndis header */
1768 net->needed_headroom = RNDIS_AND_PPI_SIZE;
1769
Haiyang Zhang692e0842011-09-01 12:19:43 -07001770 /* Notify the netvsc driver of the new device */
Andrew Schwartzmeyer8ebdcc52015-08-11 17:14:31 -07001771 memset(&device_info, 0, sizeof(device_info));
Haiyang Zhang692e0842011-09-01 12:19:43 -07001772 device_info.ring_size = ring_size;
stephen hemminger3071ada2017-03-22 14:50:59 -07001773 device_info.num_chn = VRSS_CHANNEL_DEFAULT;
stephen hemminger9749fed2017-07-19 11:53:16 -07001774
1775 nvdev = rndis_filter_device_add(dev, &device_info);
1776 if (IS_ERR(nvdev)) {
1777 ret = PTR_ERR(nvdev);
Haiyang Zhang692e0842011-09-01 12:19:43 -07001778 netdev_err(net, "unable to add netvsc device (ret %d)\n", ret);
stephen hemminger0c195562017-08-01 19:58:53 -07001779 goto rndis_failed;
Haiyang Zhang692e0842011-09-01 12:19:43 -07001780 }
stephen hemminger0c195562017-08-01 19:58:53 -07001781
Haiyang Zhang692e0842011-09-01 12:19:43 -07001782 memcpy(net->dev_addr, device_info.mac_adr, ETH_ALEN);
1783
stephen hemminger23312a32017-01-24 13:05:59 -08001784 /* hw_features computed in rndis_filter_device_add */
1785 net->features = net->hw_features |
1786 NETIF_F_HIGHDMA | NETIF_F_SG |
1787 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
1788 net->vlan_features = net->features;
1789
Haiyang Zhang5b54dac2014-04-21 10:20:28 -07001790 netif_set_real_num_tx_queues(net, nvdev->num_chn);
1791 netif_set_real_num_rx_queues(net, nvdev->num_chn);
Haiyang Zhang5b54dac2014-04-21 10:20:28 -07001792
stephen hemminger9749fed2017-07-19 11:53:16 -07001793 netdev_lockdep_set_classes(net);
1794
Jarod Wilsond0c2c992016-10-20 13:55:21 -04001795 /* MTU range: 68 - 1500 or 65521 */
1796 net->min_mtu = NETVSC_MTU_MIN;
1797 if (nvdev->nvsp_version >= NVSP_PROTOCOL_VERSION_2)
1798 net->max_mtu = NETVSC_MTU - ETH_HLEN;
1799 else
1800 net->max_mtu = ETH_DATA_LEN;
1801
Haiyang Zhanga68f9612013-12-20 16:52:31 -08001802 ret = register_netdev(net);
1803 if (ret != 0) {
1804 pr_err("Unable to register netdev.\n");
stephen hemminger0c195562017-08-01 19:58:53 -07001805 goto register_failed;
Haiyang Zhanga68f9612013-12-20 16:52:31 -08001806 }
1807
Greg Kroah-Hartmandf2fff22009-08-31 21:11:12 -07001808 return ret;
stephen hemminger0c195562017-08-01 19:58:53 -07001809
1810register_failed:
1811 rndis_filter_device_remove(dev, nvdev);
1812rndis_failed:
1813 free_percpu(net_device_ctx->vf_stats);
1814no_stats:
1815 hv_set_drvdata(dev, NULL);
1816 free_netdev(net);
1817no_net:
1818 return ret;
Greg Kroah-Hartmandf2fff22009-08-31 21:11:12 -07001819}
1820
K. Y. Srinivasan415b0232011-04-29 13:45:12 -07001821static int netvsc_remove(struct hv_device *dev)
Greg Kroah-Hartmandf2fff22009-08-31 21:11:12 -07001822{
K. Y. Srinivasan2ddd5e5f2011-09-13 10:59:49 -07001823 struct net_device *net;
Haiyang Zhang122a5f62011-05-27 06:21:55 -07001824 struct net_device_context *ndev_ctx;
K. Y. Srinivasan2ddd5e5f2011-09-13 10:59:49 -07001825
Vitaly Kuznetsov3d541ac2016-05-13 13:55:22 +02001826 net = hv_get_drvdata(dev);
Greg Kroah-Hartmandf2fff22009-08-31 21:11:12 -07001827
Greg Kroah-Hartmandf2fff22009-08-31 21:11:12 -07001828 if (net == NULL) {
K. Y. Srinivasan415b0232011-04-29 13:45:12 -07001829 dev_err(&dev->device, "No net device to remove\n");
Greg Kroah-Hartmandf2fff22009-08-31 21:11:12 -07001830 return 0;
1831 }
1832
Haiyang Zhang122a5f62011-05-27 06:21:55 -07001833 ndev_ctx = netdev_priv(net);
Vitaly Kuznetsov3d541ac2016-05-13 13:55:22 +02001834
stephen hemmingera0be4502017-03-22 14:51:01 -07001835 netif_device_detach(net);
Vitaly Kuznetsovf580aec2016-05-13 13:55:20 +02001836
Haiyang Zhang122a5f62011-05-27 06:21:55 -07001837 cancel_delayed_work_sync(&ndev_ctx->dwork);
1838
Greg Kroah-Hartmandf2fff22009-08-31 21:11:12 -07001839 /*
1840 * Call to the vsc driver to let it know that the device is being
stephen hemmingera0be4502017-03-22 14:51:01 -07001841 * removed. Also blocks mtu and channel changes.
Greg Kroah-Hartmandf2fff22009-08-31 21:11:12 -07001842 */
stephen hemmingera0be4502017-03-22 14:51:01 -07001843 rtnl_lock();
stephen hemminger79e8cbe2017-07-19 11:53:13 -07001844 rndis_filter_device_remove(dev,
1845 rtnl_dereference(ndev_ctx->nvdev));
stephen hemmingera0be4502017-03-22 14:51:01 -07001846 rtnl_unlock();
1847
1848 unregister_netdev(net);
Greg Kroah-Hartmandf2fff22009-08-31 21:11:12 -07001849
Vitaly Kuznetsov3d541ac2016-05-13 13:55:22 +02001850 hv_set_drvdata(dev, NULL);
1851
stephen hemminger0c195562017-08-01 19:58:53 -07001852 free_percpu(ndev_ctx->vf_stats);
Simon Xiao6c80f3f2017-01-24 13:06:13 -08001853 free_netdev(net);
Haiyang Zhangdf06bcf2011-05-23 09:03:47 -07001854 return 0;
Greg Kroah-Hartmandf2fff22009-08-31 21:11:12 -07001855}
1856
K. Y. Srinivasan345c4cc2011-08-25 09:48:34 -07001857static const struct hv_vmbus_device_id id_table[] = {
Greg Kroah-Hartmanc45cf2d2011-08-25 11:41:33 -07001858 /* Network guid */
K. Y. Srinivasan8f505942013-01-23 17:42:42 -08001859 { HV_NIC_GUID, },
Greg Kroah-Hartmanc45cf2d2011-08-25 11:41:33 -07001860 { },
K. Y. Srinivasan345c4cc2011-08-25 09:48:34 -07001861};
1862
1863MODULE_DEVICE_TABLE(vmbus, id_table);
1864
K. Y. Srinivasanf1542a62011-05-10 07:55:16 -07001865/* The one and only one */
K. Y. Srinivasanfde0ef92011-05-12 19:35:08 -07001866static struct hv_driver netvsc_drv = {
Haiyang Zhangd31b20f2012-03-07 10:02:00 +00001867 .name = KBUILD_MODNAME,
K. Y. Srinivasan345c4cc2011-08-25 09:48:34 -07001868 .id_table = id_table,
K. Y. Srinivasanfde0ef92011-05-12 19:35:08 -07001869 .probe = netvsc_probe,
1870 .remove = netvsc_remove,
K. Y. Srinivasand4890972011-05-10 07:55:17 -07001871};
K. Y. Srinivasanf1542a62011-05-10 07:55:16 -07001872
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001873/*
1874 * On Hyper-V, every VF interface is matched with a corresponding
1875 * synthetic interface. The synthetic interface is presented first
1876 * to the guest. When the corresponding VF instance is registered,
1877 * we will take care of switching the data path.
1878 */
1879static int netvsc_netdev_event(struct notifier_block *this,
1880 unsigned long event, void *ptr)
1881{
1882 struct net_device *event_dev = netdev_notifier_info_to_dev(ptr);
1883
Stephen Hemmingeree837a12016-09-22 16:56:31 -07001884 /* Skip our own events */
1885 if (event_dev->netdev_ops == &device_ops)
1886 return NOTIFY_DONE;
1887
1888 /* Avoid non-Ethernet type devices */
1889 if (event_dev->type != ARPHRD_ETHER)
1890 return NOTIFY_DONE;
1891
Vitaly Kuznetsov0dbff142016-08-15 17:48:43 +02001892 /* Avoid Vlan dev with same MAC registering as VF */
Parav Panditd0d7b102017-02-04 11:00:49 -06001893 if (is_vlan_dev(event_dev))
Vitaly Kuznetsov0dbff142016-08-15 17:48:43 +02001894 return NOTIFY_DONE;
1895
1896 /* Avoid Bonding master dev with same MAC registering as VF */
Stephen Hemmingeree837a12016-09-22 16:56:31 -07001897 if ((event_dev->priv_flags & IFF_BONDING) &&
1898 (event_dev->flags & IFF_MASTER))
Haiyang Zhangcb2911f2016-06-02 12:02:04 -07001899 return NOTIFY_DONE;
1900
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001901 switch (event) {
1902 case NETDEV_REGISTER:
1903 return netvsc_register_vf(event_dev);
1904 case NETDEV_UNREGISTER:
1905 return netvsc_unregister_vf(event_dev);
1906 case NETDEV_UP:
stephen hemminger7b83f522017-08-07 11:30:00 -07001907 return netvsc_vf_up(event_dev);
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001908 case NETDEV_DOWN:
stephen hemminger7b83f522017-08-07 11:30:00 -07001909 return netvsc_vf_down(event_dev);
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001910 default:
1911 return NOTIFY_DONE;
1912 }
1913}
1914
1915static struct notifier_block netvsc_netdev_notifier = {
1916 .notifier_call = netvsc_netdev_event,
1917};
1918
K. Y. Srinivasana9869c92011-05-12 19:35:17 -07001919static void __exit netvsc_drv_exit(void)
Hank Janssenfceaf242009-07-13 15:34:54 -07001920{
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001921 unregister_netdevice_notifier(&netvsc_netdev_notifier);
Greg Kroah-Hartman768fa212011-08-25 15:07:32 -07001922 vmbus_driver_unregister(&netvsc_drv);
Hank Janssenfceaf242009-07-13 15:34:54 -07001923}
1924
K. Y. Srinivasan1fde28c2011-05-12 19:35:16 -07001925static int __init netvsc_drv_init(void)
Greg Kroah-Hartmandf2fff22009-08-31 21:11:12 -07001926{
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001927 int ret;
1928
Haiyang Zhangfa85a6c2012-07-25 08:08:41 +00001929 if (ring_size < RING_SIZE_MIN) {
1930 ring_size = RING_SIZE_MIN;
1931 pr_info("Increased ring_size to %d (min allowed)\n",
1932 ring_size);
1933 }
KY Srinivasan84bf9ce2016-04-14 16:31:54 -07001934 ret = vmbus_driver_register(&netvsc_drv);
1935
1936 if (ret)
1937 return ret;
1938
1939 register_netdevice_notifier(&netvsc_netdev_notifier);
1940 return 0;
Greg Kroah-Hartmandf2fff22009-08-31 21:11:12 -07001941}
1942
Hank Janssen26c14cc2010-02-11 23:02:42 +00001943MODULE_LICENSE("GPL");
Stephen Hemminger7880fc52010-05-04 09:58:52 -07001944MODULE_DESCRIPTION("Microsoft Hyper-V network driver");
Hank Janssenfceaf242009-07-13 15:34:54 -07001945
K. Y. Srinivasan1fde28c2011-05-12 19:35:16 -07001946module_init(netvsc_drv_init);
K. Y. Srinivasana9869c92011-05-12 19:35:17 -07001947module_exit(netvsc_drv_exit);