blob: 19565f7497ee4516fbdf58b534d6488ba1e72688 [file] [log] [blame]
Thomas Gleixner2874c5f2019-05-27 08:55:01 +02001// SPDX-License-Identifier: GPL-2.0-or-later
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
tcharding722c9a02017-02-09 17:56:04 +11003 * NET3 Protocol independent device support routines.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07005 * Derived from the non IP parts of dev.c 1.0.19
tcharding722c9a02017-02-09 17:56:04 +11006 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
8 * Mark Evans, <evansmp@uhura.aston.ac.uk>
9 *
10 * Additional Authors:
11 * Florian la Roche <rzsfl@rz.uni-sb.de>
12 * Alan Cox <gw4pts@gw4pts.ampr.org>
13 * David Hinds <dahinds@users.sourceforge.net>
14 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
15 * Adam Sulmicki <adam@cfar.umd.edu>
16 * Pekka Riikonen <priikone@poesidon.pspt.fi>
17 *
18 * Changes:
19 * D.J. Barrow : Fixed bug where dev->refcnt gets set
tcharding722c9a02017-02-09 17:56:04 +110020 * to 2 if register_netdev gets called
21 * before net_dev_init & also removed a
22 * few lines of code in the process.
Linus Torvalds1da177e2005-04-16 15:20:36 -070023 * Alan Cox : device private ioctl copies fields back.
24 * Alan Cox : Transmit queue code does relevant
25 * stunts to keep the queue safe.
26 * Alan Cox : Fixed double lock.
27 * Alan Cox : Fixed promisc NULL pointer trap
28 * ???????? : Support the full private ioctl range
29 * Alan Cox : Moved ioctl permission check into
30 * drivers
31 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
32 * Alan Cox : 100 backlog just doesn't cut it when
33 * you start doing multicast video 8)
34 * Alan Cox : Rewrote net_bh and list manager.
tcharding722c9a02017-02-09 17:56:04 +110035 * Alan Cox : Fix ETH_P_ALL echoback lengths.
Linus Torvalds1da177e2005-04-16 15:20:36 -070036 * Alan Cox : Took out transmit every packet pass
37 * Saved a few bytes in the ioctl handler
38 * Alan Cox : Network driver sets packet type before
39 * calling netif_rx. Saves a function
40 * call a packet.
41 * Alan Cox : Hashed net_bh()
42 * Richard Kooijman: Timestamp fixes.
43 * Alan Cox : Wrong field in SIOCGIFDSTADDR
44 * Alan Cox : Device lock protection.
tcharding722c9a02017-02-09 17:56:04 +110045 * Alan Cox : Fixed nasty side effect of device close
Linus Torvalds1da177e2005-04-16 15:20:36 -070046 * changes.
47 * Rudi Cilibrasi : Pass the right thing to
48 * set_mac_address()
49 * Dave Miller : 32bit quantity for the device lock to
50 * make it work out on a Sparc.
51 * Bjorn Ekwall : Added KERNELD hack.
52 * Alan Cox : Cleaned up the backlog initialise.
53 * Craig Metz : SIOCGIFCONF fix if space for under
54 * 1 device.
55 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
56 * is no device open function.
57 * Andi Kleen : Fix error reporting for SIOCGIFCONF
58 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
59 * Cyrus Durgin : Cleaned for KMOD
60 * Adam Sulmicki : Bug Fix : Network Device Unload
61 * A network device unload needs to purge
62 * the backlog queue.
63 * Paul Rusty Russell : SIOCSIFNAME
64 * Pekka Riikonen : Netdev boot-time settings code
65 * Andrew Morton : Make unregister_netdevice wait
tcharding722c9a02017-02-09 17:56:04 +110066 * indefinitely on dev->refcnt
67 * J Hadi Salim : - Backlog queue sampling
Linus Torvalds1da177e2005-04-16 15:20:36 -070068 * - netif_rx() feedback
69 */
70
Linus Torvalds7c0f6ba2016-12-24 11:46:01 -080071#include <linux/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070072#include <linux/bitops.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080073#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070074#include <linux/cpu.h>
75#include <linux/types.h>
76#include <linux/kernel.h>
stephen hemminger08e98972009-11-10 07:20:34 +000077#include <linux/hash.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090078#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070079#include <linux/sched.h>
Vlastimil Babkaf1083042017-05-08 15:59:53 -070080#include <linux/sched/mm.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080081#include <linux/mutex.h>
Ahmed S. Darwish11d60112020-06-03 16:49:44 +020082#include <linux/rwsem.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070083#include <linux/string.h>
84#include <linux/mm.h>
85#include <linux/socket.h>
86#include <linux/sockios.h>
87#include <linux/errno.h>
88#include <linux/interrupt.h>
89#include <linux/if_ether.h>
90#include <linux/netdevice.h>
91#include <linux/etherdevice.h>
Ben Hutchings0187bdf2008-06-19 16:15:47 -070092#include <linux/ethtool.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070093#include <linux/skbuff.h>
Wei Wang29863d42021-02-08 11:34:09 -080094#include <linux/kthread.h>
Brenden Blancoa7862b42016-07-19 12:16:48 -070095#include <linux/bpf.h>
David S. Millerb5cdae32017-04-18 15:36:58 -040096#include <linux/bpf_trace.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020097#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070098#include <net/sock.h>
Eric Dumazet02d62e82015-11-18 06:30:52 -080099#include <net/busy_poll.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100#include <linux/rtnetlink.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101#include <linux/stat.h>
Vladimir Olteanb14a9fc2020-09-12 02:26:07 +0300102#include <net/dsa.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103#include <net/dst.h>
Pravin B Shelarfc4099f2015-10-22 18:17:16 -0700104#include <net/dst_metadata.h>
Leon Romanovsky04f00ab2021-02-03 15:51:11 +0200105#include <net/gro.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106#include <net/pkt_sched.h>
Jiri Pirko87d83092017-05-17 11:07:54 +0200107#include <net/pkt_cls.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108#include <net/checksum.h>
Arnd Bergmann44540962009-11-26 06:07:08 +0000109#include <net/xfrm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110#include <linux/highmem.h>
111#include <linux/init.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113#include <linux/netpoll.h>
114#include <linux/rcupdate.h>
115#include <linux/delay.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116#include <net/iw_handler.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117#include <asm/current.h>
Steve Grubb5bdb9882005-12-03 08:39:35 -0500118#include <linux/audit.h>
Chris Leechdb217332006-06-17 21:24:58 -0700119#include <linux/dmaengine.h>
Herbert Xuf6a78bf2006-06-22 02:57:17 -0700120#include <linux/err.h>
David S. Millerc7fa9d12006-08-15 16:34:13 -0700121#include <linux/ctype.h>
Jarek Poplawski723e98b72007-05-15 22:46:18 -0700122#include <linux/if_arp.h>
Ben Hutchings6de329e2008-06-16 17:02:28 -0700123#include <linux/if_vlan.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700124#include <linux/ip.h>
Alexander Duyckad55dca2008-09-20 22:05:50 -0700125#include <net/ip.h>
Simon Horman25cd9ba2014-10-06 05:05:13 -0700126#include <net/mpls.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700127#include <linux/ipv6.h>
128#include <linux/in.h>
David S. Millerb6b2fed2008-07-21 09:48:06 -0700129#include <linux/jhash.h>
130#include <linux/random.h>
David S. Miller9cbc1cb2009-06-15 03:02:23 -0700131#include <trace/events/napi.h>
Koki Sanagicf66ba52010-08-23 18:45:02 +0900132#include <trace/events/net.h>
Koki Sanagi07dc22e2010-08-23 18:46:12 +0900133#include <trace/events/skb.h>
Qitao Xu70713dd2021-07-14 23:03:24 -0700134#include <trace/events/qdisc.h>
Stephen Rothwellcaeda9b2010-09-16 21:39:16 -0700135#include <linux/inetdevice.h>
Ben Hutchingsc4454772011-01-19 11:03:53 +0000136#include <linux/cpu_rmap.h>
Ingo Molnarc5905af2012-02-24 08:31:31 +0100137#include <linux/static_key.h>
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300138#include <linux/hashtable.h>
Eric Dumazet60877a32013-06-20 01:15:51 -0700139#include <linux/vmalloc.h>
Michal Kubeček529d0482013-11-15 06:18:50 +0100140#include <linux/if_macvlan.h>
Willem de Bruijne7fd2882014-08-04 22:11:48 -0400141#include <linux/errqueue.h>
Eric Dumazet3b47d302014-11-06 21:09:44 -0800142#include <linux/hrtimer.h>
Daniel Borkmann357b6cc2020-03-18 10:33:22 +0100143#include <linux/netfilter_ingress.h>
Hariprasad Shenai40e4e712016-06-08 18:09:08 +0530144#include <linux/crash_dump.h>
Davide Carattib72b5bf2017-05-18 15:44:38 +0200145#include <linux/sctp.h>
Sabrina Dubrocaae847f42017-07-21 12:49:31 +0200146#include <net/udp_tunnel.h>
Nicolas Dichtel6621dd22017-10-03 13:53:23 +0200147#include <linux/net_namespace.h>
Paolo Abeniaaa5d902018-12-14 11:51:58 +0100148#include <linux/indirect_call_wrapper.h>
Jiri Pirkoaf3836d2019-03-28 13:56:37 +0100149#include <net/devlink.h>
Heiner Kallweitbd869242020-06-20 22:35:42 +0200150#include <linux/pm_runtime.h>
Willy Tarreau37447412020-08-10 10:27:42 +0200151#include <linux/prandom.h>
Tanner Love127d7352021-06-28 09:50:07 -0400152#include <linux/once_lite.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153
Pavel Emelyanov342709e2007-10-23 21:14:45 -0700154#include "net-sysfs.h"
155
Herbert Xud565b0a2008-12-15 23:38:52 -0800156#define MAX_GRO_SKBS 8
157
Herbert Xu5d38a072009-01-04 16:13:40 -0800158/* This should be increased if a protocol with a bigger head is added. */
159#define GRO_MAX_HEAD (MAX_HEADER + 128)
160
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161static DEFINE_SPINLOCK(ptype_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000162static DEFINE_SPINLOCK(offload_lock);
Cong Wang900ff8c2013-02-18 19:20:33 +0000163struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
164struct list_head ptype_all __read_mostly; /* Taps */
Vlad Yasevich62532da2012-11-15 08:49:10 +0000165static struct list_head offload_base __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166
Ben Hutchingsae78dbf2014-01-10 22:17:24 +0000167static int netif_rx_internal(struct sk_buff *skb);
Loic Prylli54951192014-07-01 21:39:43 -0700168static int call_netdevice_notifiers_info(unsigned long val,
Loic Prylli54951192014-07-01 21:39:43 -0700169 struct netdev_notifier_info *info);
Petr Machata26372602018-12-06 17:05:45 +0000170static int call_netdevice_notifiers_extack(unsigned long val,
171 struct net_device *dev,
172 struct netlink_ext_ack *extack);
Miroslav Lichvar90b602f2017-05-19 17:52:37 +0200173static struct napi_struct *napi_by_id(unsigned int napi_id);
Ben Hutchingsae78dbf2014-01-10 22:17:24 +0000174
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175/*
Pavel Emelianov7562f872007-05-03 15:13:45 -0700176 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177 * semaphore.
178 *
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800179 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700180 *
181 * Writers must hold the rtnl semaphore while they loop through the
Pavel Emelianov7562f872007-05-03 15:13:45 -0700182 * dev_base_head list, and hold dev_base_lock for writing when they do the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183 * actual updates. This allows pure readers to access the list even
184 * while a writer is preparing to update it.
185 *
186 * To put it another way, dev_base_lock is held for writing only to
187 * protect against pure readers; the rtnl semaphore provides the
188 * protection against other writers.
189 *
190 * See, for example usages, register_netdevice() and
191 * unregister_netdevice(), which must be called with the rtnl
192 * semaphore held.
193 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700194DEFINE_RWLOCK(dev_base_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195EXPORT_SYMBOL(dev_base_lock);
196
Florian Westphal6c557002017-10-02 23:50:05 +0200197static DEFINE_MUTEX(ifalias_mutex);
198
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300199/* protects napi_hash addition/deletion and napi_gen_id */
200static DEFINE_SPINLOCK(napi_hash_lock);
201
Eric Dumazet52bd2d62015-11-18 06:30:50 -0800202static unsigned int napi_gen_id = NR_CPUS;
Eric Dumazet6180d9d2015-11-18 06:31:01 -0800203static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300204
Ahmed S. Darwish11d60112020-06-03 16:49:44 +0200205static DECLARE_RWSEM(devnet_rename_sem);
Brian Haleyc91f6df2012-11-26 05:21:08 +0000206
Thomas Graf4e985ad2011-06-21 03:11:20 +0000207static inline void dev_base_seq_inc(struct net *net)
208{
tcharding643aa9c2017-02-09 17:56:05 +1100209 while (++net->dev_base_seq == 0)
210 ;
Thomas Graf4e985ad2011-06-21 03:11:20 +0000211}
212
Eric W. Biederman881d9662007-09-17 11:56:21 -0700213static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214{
Linus Torvalds8387ff22016-06-10 07:51:30 -0700215 unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
Eric Dumazet95c96172012-04-15 05:58:06 +0000216
stephen hemminger08e98972009-11-10 07:20:34 +0000217 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700218}
219
Eric W. Biederman881d9662007-09-17 11:56:21 -0700220static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221{
Eric Dumazet7c28bd02009-10-24 06:13:17 -0700222 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223}
224
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000225static inline void rps_lock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000226{
227#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000228 spin_lock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000229#endif
230}
231
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000232static inline void rps_unlock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000233{
234#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000235 spin_unlock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000236#endif
237}
238
Jiri Pirkoff927412019-09-30 11:48:15 +0200239static struct netdev_name_node *netdev_name_node_alloc(struct net_device *dev,
240 const char *name)
241{
242 struct netdev_name_node *name_node;
243
244 name_node = kmalloc(sizeof(*name_node), GFP_KERNEL);
245 if (!name_node)
246 return NULL;
247 INIT_HLIST_NODE(&name_node->hlist);
248 name_node->dev = dev;
249 name_node->name = name;
250 return name_node;
251}
252
253static struct netdev_name_node *
254netdev_name_node_head_alloc(struct net_device *dev)
255{
Jiri Pirko36fbf1e2019-09-30 11:48:16 +0200256 struct netdev_name_node *name_node;
257
258 name_node = netdev_name_node_alloc(dev, dev->name);
259 if (!name_node)
260 return NULL;
261 INIT_LIST_HEAD(&name_node->list);
262 return name_node;
Jiri Pirkoff927412019-09-30 11:48:15 +0200263}
264
265static void netdev_name_node_free(struct netdev_name_node *name_node)
266{
267 kfree(name_node);
268}
269
270static void netdev_name_node_add(struct net *net,
271 struct netdev_name_node *name_node)
272{
273 hlist_add_head_rcu(&name_node->hlist,
274 dev_name_hash(net, name_node->name));
275}
276
277static void netdev_name_node_del(struct netdev_name_node *name_node)
278{
279 hlist_del_rcu(&name_node->hlist);
280}
281
282static struct netdev_name_node *netdev_name_node_lookup(struct net *net,
283 const char *name)
284{
285 struct hlist_head *head = dev_name_hash(net, name);
286 struct netdev_name_node *name_node;
287
288 hlist_for_each_entry(name_node, head, hlist)
289 if (!strcmp(name_node->name, name))
290 return name_node;
291 return NULL;
292}
293
294static struct netdev_name_node *netdev_name_node_lookup_rcu(struct net *net,
295 const char *name)
296{
297 struct hlist_head *head = dev_name_hash(net, name);
298 struct netdev_name_node *name_node;
299
300 hlist_for_each_entry_rcu(name_node, head, hlist)
301 if (!strcmp(name_node->name, name))
302 return name_node;
303 return NULL;
304}
305
Jiri Pirko36fbf1e2019-09-30 11:48:16 +0200306int netdev_name_node_alt_create(struct net_device *dev, const char *name)
307{
308 struct netdev_name_node *name_node;
309 struct net *net = dev_net(dev);
310
311 name_node = netdev_name_node_lookup(net, name);
312 if (name_node)
313 return -EEXIST;
314 name_node = netdev_name_node_alloc(dev, name);
315 if (!name_node)
316 return -ENOMEM;
317 netdev_name_node_add(net, name_node);
318 /* The node that holds dev->name acts as a head of per-device list. */
319 list_add_tail(&name_node->list, &dev->name_node->list);
320
321 return 0;
322}
323EXPORT_SYMBOL(netdev_name_node_alt_create);
324
325static void __netdev_name_node_alt_destroy(struct netdev_name_node *name_node)
326{
327 list_del(&name_node->list);
328 netdev_name_node_del(name_node);
329 kfree(name_node->name);
330 netdev_name_node_free(name_node);
331}
332
333int netdev_name_node_alt_destroy(struct net_device *dev, const char *name)
334{
335 struct netdev_name_node *name_node;
336 struct net *net = dev_net(dev);
337
338 name_node = netdev_name_node_lookup(net, name);
339 if (!name_node)
340 return -ENOENT;
Eric Dumazete08ad802020-02-14 07:53:53 -0800341 /* lookup might have found our primary name or a name belonging
342 * to another device.
343 */
344 if (name_node == dev->name_node || name_node->dev != dev)
345 return -EINVAL;
346
Jiri Pirko36fbf1e2019-09-30 11:48:16 +0200347 __netdev_name_node_alt_destroy(name_node);
348
349 return 0;
350}
351EXPORT_SYMBOL(netdev_name_node_alt_destroy);
352
353static void netdev_name_node_alt_flush(struct net_device *dev)
354{
355 struct netdev_name_node *name_node, *tmp;
356
357 list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list)
358 __netdev_name_node_alt_destroy(name_node);
359}
360
Eric W. Biedermance286d32007-09-12 13:53:49 +0200361/* Device list insertion */
dingtianhong53759be2013-04-17 22:17:50 +0000362static void list_netdevice(struct net_device *dev)
Eric W. Biedermance286d32007-09-12 13:53:49 +0200363{
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900364 struct net *net = dev_net(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200365
366 ASSERT_RTNL();
367
368 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800369 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
Jiri Pirkoff927412019-09-30 11:48:15 +0200370 netdev_name_node_add(net, dev->name_node);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000371 hlist_add_head_rcu(&dev->index_hlist,
372 dev_index_hash(net, dev->ifindex));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200373 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000374
375 dev_base_seq_inc(net);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200376}
377
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000378/* Device list removal
379 * caller must respect a RCU grace period before freeing/reusing dev
380 */
Eric W. Biedermance286d32007-09-12 13:53:49 +0200381static void unlist_netdevice(struct net_device *dev)
382{
383 ASSERT_RTNL();
384
385 /* Unlink dev from the device chain */
386 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800387 list_del_rcu(&dev->dev_list);
Jiri Pirkoff927412019-09-30 11:48:15 +0200388 netdev_name_node_del(dev->name_node);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000389 hlist_del_rcu(&dev->index_hlist);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200390 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000391
392 dev_base_seq_inc(dev_net(dev));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200393}
394
Linus Torvalds1da177e2005-04-16 15:20:36 -0700395/*
396 * Our notifier list
397 */
398
Alan Sternf07d5b92006-05-09 15:23:03 -0700399static RAW_NOTIFIER_HEAD(netdev_chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700400
401/*
402 * Device drivers call our routines to queue packets here. We empty the
403 * queue in the local softnet handler.
404 */
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700405
Eric Dumazet9958da02010-04-17 04:17:02 +0000406DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700407EXPORT_PER_CPU_SYMBOL(softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408
Cong Wang1a33e102020-05-02 22:22:19 -0700409#ifdef CONFIG_LOCKDEP
410/*
411 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
412 * according to dev->type
413 */
414static const unsigned short netdev_lock_type[] = {
415 ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
416 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
417 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
418 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
419 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
420 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
421 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
422 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
423 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
424 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
425 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
426 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
427 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
428 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
429 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
430
431static const char *const netdev_lock_name[] = {
432 "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
433 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
434 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
435 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
436 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
437 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
438 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
439 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
440 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
441 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
442 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
443 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
444 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
445 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
446 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
447
448static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
Cong Wang845e0eb2020-06-08 14:53:01 -0700449static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
Cong Wang1a33e102020-05-02 22:22:19 -0700450
451static inline unsigned short netdev_lock_pos(unsigned short dev_type)
452{
453 int i;
454
455 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
456 if (netdev_lock_type[i] == dev_type)
457 return i;
458 /* the last key is used by default */
459 return ARRAY_SIZE(netdev_lock_type) - 1;
460}
461
462static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
463 unsigned short dev_type)
464{
465 int i;
466
467 i = netdev_lock_pos(dev_type);
468 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
469 netdev_lock_name[i]);
470}
Cong Wang845e0eb2020-06-08 14:53:01 -0700471
472static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
473{
474 int i;
475
476 i = netdev_lock_pos(dev->type);
477 lockdep_set_class_and_name(&dev->addr_list_lock,
478 &netdev_addr_lock_key[i],
479 netdev_lock_name[i]);
480}
Cong Wang1a33e102020-05-02 22:22:19 -0700481#else
482static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
483 unsigned short dev_type)
484{
485}
Cong Wang845e0eb2020-06-08 14:53:01 -0700486
487static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
488{
489}
Cong Wang1a33e102020-05-02 22:22:19 -0700490#endif
491
Linus Torvalds1da177e2005-04-16 15:20:36 -0700492/*******************************************************************************
tchardingeb13da12017-02-09 17:56:06 +1100493 *
494 * Protocol management and registration routines
495 *
496 *******************************************************************************/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498
499/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500 * Add a protocol ID to the list. Now that the input handler is
501 * smarter we can dispense with all the messy stuff that used to be
502 * here.
503 *
504 * BEWARE!!! Protocol handlers, mangling input packets,
505 * MUST BE last in hash buckets and checking protocol handlers
506 * MUST start from promiscuous ptype_all chain in net_bh.
507 * It is true now, do not change it.
508 * Explanation follows: if protocol handler, mangling packet, will
509 * be the first on list, it is not able to sense, that packet
510 * is cloned and should be copied-on-write, so that it will
511 * change it and subsequent readers will get broken packet.
512 * --ANK (980803)
513 */
514
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000515static inline struct list_head *ptype_head(const struct packet_type *pt)
516{
517 if (pt->type == htons(ETH_P_ALL))
Salam Noureddine7866a622015-01-27 11:35:48 -0800518 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000519 else
Salam Noureddine7866a622015-01-27 11:35:48 -0800520 return pt->dev ? &pt->dev->ptype_specific :
521 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000522}
523
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524/**
525 * dev_add_pack - add packet handler
526 * @pt: packet type declaration
527 *
528 * Add a protocol handler to the networking stack. The passed &packet_type
529 * is linked into kernel lists and may not be freed until it has been
530 * removed from the kernel lists.
531 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900532 * This call does not sleep therefore it can not
Linus Torvalds1da177e2005-04-16 15:20:36 -0700533 * guarantee all CPU's that are in middle of receiving packets
534 * will see the new packet type (until the next received packet).
535 */
536
537void dev_add_pack(struct packet_type *pt)
538{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000539 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700540
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000541 spin_lock(&ptype_lock);
542 list_add_rcu(&pt->list, head);
543 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700544}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700545EXPORT_SYMBOL(dev_add_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700546
Linus Torvalds1da177e2005-04-16 15:20:36 -0700547/**
548 * __dev_remove_pack - remove packet handler
549 * @pt: packet type declaration
550 *
551 * Remove a protocol handler that was previously added to the kernel
552 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
553 * from the kernel lists and can be freed or reused once this function
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900554 * returns.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700555 *
556 * The packet type might still be in use by receivers
557 * and must not be freed until after all the CPU's have gone
558 * through a quiescent state.
559 */
560void __dev_remove_pack(struct packet_type *pt)
561{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000562 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563 struct packet_type *pt1;
564
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000565 spin_lock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700566
567 list_for_each_entry(pt1, head, list) {
568 if (pt == pt1) {
569 list_del_rcu(&pt->list);
570 goto out;
571 }
572 }
573
Joe Perches7b6cd1c2012-02-01 10:54:43 +0000574 pr_warn("dev_remove_pack: %p not found\n", pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700575out:
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000576 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700577}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700578EXPORT_SYMBOL(__dev_remove_pack);
579
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580/**
581 * dev_remove_pack - remove packet handler
582 * @pt: packet type declaration
583 *
584 * Remove a protocol handler that was previously added to the kernel
585 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
586 * from the kernel lists and can be freed or reused once this function
587 * returns.
588 *
589 * This call sleeps to guarantee that no CPU is looking at the packet
590 * type after return.
591 */
592void dev_remove_pack(struct packet_type *pt)
593{
594 __dev_remove_pack(pt);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900595
Linus Torvalds1da177e2005-04-16 15:20:36 -0700596 synchronize_net();
597}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700598EXPORT_SYMBOL(dev_remove_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700599
Vlad Yasevich62532da2012-11-15 08:49:10 +0000600
601/**
602 * dev_add_offload - register offload handlers
603 * @po: protocol offload declaration
604 *
605 * Add protocol offload handlers to the networking stack. The passed
606 * &proto_offload is linked into kernel lists and may not be freed until
607 * it has been removed from the kernel lists.
608 *
609 * This call does not sleep therefore it can not
610 * guarantee all CPU's that are in middle of receiving packets
611 * will see the new offload handlers (until the next received packet).
612 */
613void dev_add_offload(struct packet_offload *po)
614{
David S. Millerbdef7de2015-06-01 14:56:09 -0700615 struct packet_offload *elem;
Vlad Yasevich62532da2012-11-15 08:49:10 +0000616
617 spin_lock(&offload_lock);
David S. Millerbdef7de2015-06-01 14:56:09 -0700618 list_for_each_entry(elem, &offload_base, list) {
619 if (po->priority < elem->priority)
620 break;
621 }
622 list_add_rcu(&po->list, elem->list.prev);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000623 spin_unlock(&offload_lock);
624}
625EXPORT_SYMBOL(dev_add_offload);
626
627/**
628 * __dev_remove_offload - remove offload handler
629 * @po: packet offload declaration
630 *
631 * Remove a protocol offload handler that was previously added to the
632 * kernel offload handlers by dev_add_offload(). The passed &offload_type
633 * is removed from the kernel lists and can be freed or reused once this
634 * function returns.
635 *
636 * The packet type might still be in use by receivers
637 * and must not be freed until after all the CPU's have gone
638 * through a quiescent state.
639 */
stephen hemminger1d143d92013-12-29 14:01:29 -0800640static void __dev_remove_offload(struct packet_offload *po)
Vlad Yasevich62532da2012-11-15 08:49:10 +0000641{
642 struct list_head *head = &offload_base;
643 struct packet_offload *po1;
644
Eric Dumazetc53aa502012-11-16 08:08:23 +0000645 spin_lock(&offload_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000646
647 list_for_each_entry(po1, head, list) {
648 if (po == po1) {
649 list_del_rcu(&po->list);
650 goto out;
651 }
652 }
653
654 pr_warn("dev_remove_offload: %p not found\n", po);
655out:
Eric Dumazetc53aa502012-11-16 08:08:23 +0000656 spin_unlock(&offload_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000657}
Vlad Yasevich62532da2012-11-15 08:49:10 +0000658
659/**
660 * dev_remove_offload - remove packet offload handler
661 * @po: packet offload declaration
662 *
663 * Remove a packet offload handler that was previously added to the kernel
664 * offload handlers by dev_add_offload(). The passed &offload_type is
665 * removed from the kernel lists and can be freed or reused once this
666 * function returns.
667 *
668 * This call sleeps to guarantee that no CPU is looking at the packet
669 * type after return.
670 */
671void dev_remove_offload(struct packet_offload *po)
672{
673 __dev_remove_offload(po);
674
675 synchronize_net();
676}
677EXPORT_SYMBOL(dev_remove_offload);
678
Linus Torvalds1da177e2005-04-16 15:20:36 -0700679/******************************************************************************
tchardingeb13da12017-02-09 17:56:06 +1100680 *
681 * Device Boot-time Settings Routines
682 *
683 ******************************************************************************/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700684
685/* Boot time configuration table */
686static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
687
688/**
689 * netdev_boot_setup_add - add new setup entry
690 * @name: name of the device
691 * @map: configured settings for the device
692 *
693 * Adds new setup entry to the dev_boot_setup list. The function
694 * returns 0 on error and 1 on success. This is a generic routine to
695 * all netdevices.
696 */
697static int netdev_boot_setup_add(char *name, struct ifmap *map)
698{
699 struct netdev_boot_setup *s;
700 int i;
701
702 s = dev_boot_setup;
703 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
704 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
705 memset(s[i].name, 0, sizeof(s[i].name));
Wang Chen93b3cff92008-07-01 19:57:19 -0700706 strlcpy(s[i].name, name, IFNAMSIZ);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700707 memcpy(&s[i].map, map, sizeof(s[i].map));
708 break;
709 }
710 }
711
712 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
713}
714
715/**
tcharding722c9a02017-02-09 17:56:04 +1100716 * netdev_boot_setup_check - check boot time settings
717 * @dev: the netdevice
Linus Torvalds1da177e2005-04-16 15:20:36 -0700718 *
tcharding722c9a02017-02-09 17:56:04 +1100719 * Check boot time settings for the device.
720 * The found settings are set for the device to be used
721 * later in the device probing.
722 * Returns 0 if no settings found, 1 if they are.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700723 */
724int netdev_boot_setup_check(struct net_device *dev)
725{
726 struct netdev_boot_setup *s = dev_boot_setup;
727 int i;
728
729 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
730 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
Wang Chen93b3cff92008-07-01 19:57:19 -0700731 !strcmp(dev->name, s[i].name)) {
tcharding722c9a02017-02-09 17:56:04 +1100732 dev->irq = s[i].map.irq;
733 dev->base_addr = s[i].map.base_addr;
734 dev->mem_start = s[i].map.mem_start;
735 dev->mem_end = s[i].map.mem_end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736 return 1;
737 }
738 }
739 return 0;
740}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700741EXPORT_SYMBOL(netdev_boot_setup_check);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700742
743
744/**
tcharding722c9a02017-02-09 17:56:04 +1100745 * netdev_boot_base - get address from boot time settings
746 * @prefix: prefix for network device
747 * @unit: id for network device
Linus Torvalds1da177e2005-04-16 15:20:36 -0700748 *
tcharding722c9a02017-02-09 17:56:04 +1100749 * Check boot time settings for the base address of device.
750 * The found settings are set for the device to be used
751 * later in the device probing.
752 * Returns 0 if no settings found.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700753 */
754unsigned long netdev_boot_base(const char *prefix, int unit)
755{
756 const struct netdev_boot_setup *s = dev_boot_setup;
757 char name[IFNAMSIZ];
758 int i;
759
760 sprintf(name, "%s%d", prefix, unit);
761
762 /*
763 * If device already registered then return base of 1
764 * to indicate not to probe for this interface
765 */
Eric W. Biederman881d9662007-09-17 11:56:21 -0700766 if (__dev_get_by_name(&init_net, name))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700767 return 1;
768
769 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
770 if (!strcmp(name, s[i].name))
771 return s[i].map.base_addr;
772 return 0;
773}
774
775/*
776 * Saves at boot time configured settings for any netdevice.
777 */
778int __init netdev_boot_setup(char *str)
779{
780 int ints[5];
781 struct ifmap map;
782
783 str = get_options(str, ARRAY_SIZE(ints), ints);
784 if (!str || !*str)
785 return 0;
786
787 /* Save settings */
788 memset(&map, 0, sizeof(map));
789 if (ints[0] > 0)
790 map.irq = ints[1];
791 if (ints[0] > 1)
792 map.base_addr = ints[2];
793 if (ints[0] > 2)
794 map.mem_start = ints[3];
795 if (ints[0] > 3)
796 map.mem_end = ints[4];
797
798 /* Add new entry to the list */
799 return netdev_boot_setup_add(str, &map);
800}
801
802__setup("netdev=", netdev_boot_setup);
803
804/*******************************************************************************
tchardingeb13da12017-02-09 17:56:06 +1100805 *
806 * Device Interface Subroutines
807 *
808 *******************************************************************************/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700809
810/**
Nicolas Dichtela54acb32015-04-02 17:07:00 +0200811 * dev_get_iflink - get 'iflink' value of a interface
812 * @dev: targeted interface
813 *
814 * Indicates the ifindex the interface is linked to.
815 * Physical interfaces have the same 'ifindex' and 'iflink' values.
816 */
817
818int dev_get_iflink(const struct net_device *dev)
819{
820 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
821 return dev->netdev_ops->ndo_get_iflink(dev);
822
Nicolas Dichtel7a66bbc2015-04-02 17:07:09 +0200823 return dev->ifindex;
Nicolas Dichtela54acb32015-04-02 17:07:00 +0200824}
825EXPORT_SYMBOL(dev_get_iflink);
826
827/**
Pravin B Shelarfc4099f2015-10-22 18:17:16 -0700828 * dev_fill_metadata_dst - Retrieve tunnel egress information.
829 * @dev: targeted interface
830 * @skb: The packet.
831 *
832 * For better visibility of tunnel traffic OVS needs to retrieve
833 * egress tunnel information for a packet. Following API allows
834 * user to get this info.
835 */
836int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
837{
838 struct ip_tunnel_info *info;
839
840 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst)
841 return -EINVAL;
842
843 info = skb_tunnel_info_unclone(skb);
844 if (!info)
845 return -ENOMEM;
846 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
847 return -EINVAL;
848
849 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
850}
851EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
852
Pablo Neira Ayusoddb94ea2021-03-24 02:30:32 +0100853static struct net_device_path *dev_fwd_path(struct net_device_path_stack *stack)
854{
855 int k = stack->num_paths++;
856
857 if (WARN_ON_ONCE(k >= NET_DEVICE_PATH_STACK_MAX))
858 return NULL;
859
860 return &stack->path[k];
861}
862
863int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr,
864 struct net_device_path_stack *stack)
865{
866 const struct net_device *last_dev;
867 struct net_device_path_ctx ctx = {
868 .dev = dev,
869 .daddr = daddr,
870 };
871 struct net_device_path *path;
872 int ret = 0;
873
874 stack->num_paths = 0;
875 while (ctx.dev && ctx.dev->netdev_ops->ndo_fill_forward_path) {
876 last_dev = ctx.dev;
877 path = dev_fwd_path(stack);
878 if (!path)
879 return -1;
880
881 memset(path, 0, sizeof(struct net_device_path));
882 ret = ctx.dev->netdev_ops->ndo_fill_forward_path(&ctx, path);
883 if (ret < 0)
884 return -1;
885
886 if (WARN_ON_ONCE(last_dev == ctx.dev))
887 return -1;
888 }
889 path = dev_fwd_path(stack);
890 if (!path)
891 return -1;
892 path->type = DEV_PATH_ETHERNET;
893 path->dev = ctx.dev;
894
895 return ret;
896}
897EXPORT_SYMBOL_GPL(dev_fill_forward_path);
898
Pravin B Shelarfc4099f2015-10-22 18:17:16 -0700899/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700900 * __dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700901 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700902 * @name: name to find
903 *
904 * Find an interface by name. Must be called under RTNL semaphore
905 * or @dev_base_lock. If the name is found a pointer to the device
906 * is returned. If the name is not found then %NULL is returned. The
907 * reference counters are not incremented so the caller must be
908 * careful with locks.
909 */
910
Eric W. Biederman881d9662007-09-17 11:56:21 -0700911struct net_device *__dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700912{
Jiri Pirkoff927412019-09-30 11:48:15 +0200913 struct netdev_name_node *node_name;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914
Jiri Pirkoff927412019-09-30 11:48:15 +0200915 node_name = netdev_name_node_lookup(net, name);
916 return node_name ? node_name->dev : NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700917}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700918EXPORT_SYMBOL(__dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700919
920/**
tcharding722c9a02017-02-09 17:56:04 +1100921 * dev_get_by_name_rcu - find a device by its name
922 * @net: the applicable net namespace
923 * @name: name to find
Eric Dumazet72c95282009-10-30 07:11:27 +0000924 *
tcharding722c9a02017-02-09 17:56:04 +1100925 * Find an interface by name.
926 * If the name is found a pointer to the device is returned.
927 * If the name is not found then %NULL is returned.
928 * The reference counters are not incremented so the caller must be
929 * careful with locks. The caller must hold RCU lock.
Eric Dumazet72c95282009-10-30 07:11:27 +0000930 */
931
932struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
933{
Jiri Pirkoff927412019-09-30 11:48:15 +0200934 struct netdev_name_node *node_name;
Eric Dumazet72c95282009-10-30 07:11:27 +0000935
Jiri Pirkoff927412019-09-30 11:48:15 +0200936 node_name = netdev_name_node_lookup_rcu(net, name);
937 return node_name ? node_name->dev : NULL;
Eric Dumazet72c95282009-10-30 07:11:27 +0000938}
939EXPORT_SYMBOL(dev_get_by_name_rcu);
940
941/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700942 * dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700943 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700944 * @name: name to find
945 *
946 * Find an interface by name. This can be called from any
947 * context and does its own locking. The returned handle has
948 * the usage count incremented and the caller must use dev_put() to
949 * release it when it is no longer needed. %NULL is returned if no
950 * matching device is found.
951 */
952
Eric W. Biederman881d9662007-09-17 11:56:21 -0700953struct net_device *dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700954{
955 struct net_device *dev;
956
Eric Dumazet72c95282009-10-30 07:11:27 +0000957 rcu_read_lock();
958 dev = dev_get_by_name_rcu(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700959 if (dev)
960 dev_hold(dev);
Eric Dumazet72c95282009-10-30 07:11:27 +0000961 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962 return dev;
963}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700964EXPORT_SYMBOL(dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700965
966/**
967 * __dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700968 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969 * @ifindex: index of device
970 *
971 * Search for an interface by index. Returns %NULL if the device
972 * is not found or a pointer to the device. The device has not
973 * had its reference counter increased so the caller must be careful
974 * about locking. The caller must hold either the RTNL semaphore
975 * or @dev_base_lock.
976 */
977
Eric W. Biederman881d9662007-09-17 11:56:21 -0700978struct net_device *__dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700979{
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700980 struct net_device *dev;
981 struct hlist_head *head = dev_index_hash(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982
Sasha Levinb67bfe02013-02-27 17:06:00 -0800983 hlist_for_each_entry(dev, head, index_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700984 if (dev->ifindex == ifindex)
985 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700986
Linus Torvalds1da177e2005-04-16 15:20:36 -0700987 return NULL;
988}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700989EXPORT_SYMBOL(__dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700990
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000991/**
992 * dev_get_by_index_rcu - find a device by its ifindex
993 * @net: the applicable net namespace
994 * @ifindex: index of device
995 *
996 * Search for an interface by index. Returns %NULL if the device
997 * is not found or a pointer to the device. The device has not
998 * had its reference counter increased so the caller must be careful
999 * about locking. The caller must hold RCU lock.
1000 */
1001
1002struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
1003{
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00001004 struct net_device *dev;
1005 struct hlist_head *head = dev_index_hash(net, ifindex);
1006
Sasha Levinb67bfe02013-02-27 17:06:00 -08001007 hlist_for_each_entry_rcu(dev, head, index_hlist)
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00001008 if (dev->ifindex == ifindex)
1009 return dev;
1010
1011 return NULL;
1012}
1013EXPORT_SYMBOL(dev_get_by_index_rcu);
1014
Linus Torvalds1da177e2005-04-16 15:20:36 -07001015
1016/**
1017 * dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001018 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07001019 * @ifindex: index of device
1020 *
1021 * Search for an interface by index. Returns NULL if the device
1022 * is not found or a pointer to the device. The device returned has
1023 * had a reference added and the pointer is safe until the user calls
1024 * dev_put to indicate they have finished with it.
1025 */
1026
Eric W. Biederman881d9662007-09-17 11:56:21 -07001027struct net_device *dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001028{
1029 struct net_device *dev;
1030
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00001031 rcu_read_lock();
1032 dev = dev_get_by_index_rcu(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001033 if (dev)
1034 dev_hold(dev);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00001035 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001036 return dev;
1037}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001038EXPORT_SYMBOL(dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001039
1040/**
Miroslav Lichvar90b602f2017-05-19 17:52:37 +02001041 * dev_get_by_napi_id - find a device by napi_id
1042 * @napi_id: ID of the NAPI struct
1043 *
1044 * Search for an interface by NAPI ID. Returns %NULL if the device
1045 * is not found or a pointer to the device. The device has not had
1046 * its reference counter increased so the caller must be careful
1047 * about locking. The caller must hold RCU lock.
1048 */
1049
1050struct net_device *dev_get_by_napi_id(unsigned int napi_id)
1051{
1052 struct napi_struct *napi;
1053
1054 WARN_ON_ONCE(!rcu_read_lock_held());
1055
1056 if (napi_id < MIN_NAPI_ID)
1057 return NULL;
1058
1059 napi = napi_by_id(napi_id);
1060
1061 return napi ? napi->dev : NULL;
1062}
1063EXPORT_SYMBOL(dev_get_by_napi_id);
1064
1065/**
Nicolas Schichan5dbe7c12013-06-26 17:23:42 +02001066 * netdev_get_name - get a netdevice name, knowing its ifindex.
1067 * @net: network namespace
1068 * @name: a pointer to the buffer where the name will be stored.
1069 * @ifindex: the ifindex of the interface to get the name from.
Nicolas Schichan5dbe7c12013-06-26 17:23:42 +02001070 */
1071int netdev_get_name(struct net *net, char *name, int ifindex)
1072{
1073 struct net_device *dev;
Ahmed S. Darwish11d60112020-06-03 16:49:44 +02001074 int ret;
Nicolas Schichan5dbe7c12013-06-26 17:23:42 +02001075
Ahmed S. Darwish11d60112020-06-03 16:49:44 +02001076 down_read(&devnet_rename_sem);
Nicolas Schichan5dbe7c12013-06-26 17:23:42 +02001077 rcu_read_lock();
Ahmed S. Darwish11d60112020-06-03 16:49:44 +02001078
Nicolas Schichan5dbe7c12013-06-26 17:23:42 +02001079 dev = dev_get_by_index_rcu(net, ifindex);
1080 if (!dev) {
Ahmed S. Darwish11d60112020-06-03 16:49:44 +02001081 ret = -ENODEV;
1082 goto out;
Nicolas Schichan5dbe7c12013-06-26 17:23:42 +02001083 }
1084
1085 strcpy(name, dev->name);
Nicolas Schichan5dbe7c12013-06-26 17:23:42 +02001086
Ahmed S. Darwish11d60112020-06-03 16:49:44 +02001087 ret = 0;
1088out:
1089 rcu_read_unlock();
1090 up_read(&devnet_rename_sem);
1091 return ret;
Nicolas Schichan5dbe7c12013-06-26 17:23:42 +02001092}
1093
1094/**
Eric Dumazet941666c2010-12-05 01:23:53 +00001095 * dev_getbyhwaddr_rcu - find a device by its hardware address
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001096 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07001097 * @type: media type of device
1098 * @ha: hardware address
1099 *
1100 * Search for an interface by MAC address. Returns NULL if the device
Eric Dumazetc5066532011-01-24 13:16:16 -08001101 * is not found or a pointer to the device.
1102 * The caller must hold RCU or RTNL.
Eric Dumazet941666c2010-12-05 01:23:53 +00001103 * The returned device has not had its ref count increased
Linus Torvalds1da177e2005-04-16 15:20:36 -07001104 * and the caller must therefore be careful about locking
1105 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001106 */
1107
Eric Dumazet941666c2010-12-05 01:23:53 +00001108struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
1109 const char *ha)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001110{
1111 struct net_device *dev;
1112
Eric Dumazet941666c2010-12-05 01:23:53 +00001113 for_each_netdev_rcu(net, dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001114 if (dev->type == type &&
1115 !memcmp(dev->dev_addr, ha, dev->addr_len))
Pavel Emelianov7562f872007-05-03 15:13:45 -07001116 return dev;
1117
1118 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001119}
Eric Dumazet941666c2010-12-05 01:23:53 +00001120EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
Jochen Friedrichcf309e32005-09-22 04:44:55 -03001121
Eric W. Biederman881d9662007-09-17 11:56:21 -07001122struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001123{
Eric Dumazet99fe3c32010-03-18 11:27:25 +00001124 struct net_device *dev, *ret = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001125
Eric Dumazet99fe3c32010-03-18 11:27:25 +00001126 rcu_read_lock();
1127 for_each_netdev_rcu(net, dev)
1128 if (dev->type == type) {
1129 dev_hold(dev);
1130 ret = dev;
1131 break;
1132 }
1133 rcu_read_unlock();
1134 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001135}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001136EXPORT_SYMBOL(dev_getfirstbyhwtype);
1137
1138/**
WANG Cong6c555492014-09-11 15:35:09 -07001139 * __dev_get_by_flags - find any device with given flags
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001140 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07001141 * @if_flags: IFF_* values
1142 * @mask: bitmask of bits in if_flags to check
1143 *
1144 * Search for any interface with the given flags. Returns NULL if a device
Eric Dumazetbb69ae02010-06-07 11:42:13 +00001145 * is not found or a pointer to the device. Must be called inside
WANG Cong6c555492014-09-11 15:35:09 -07001146 * rtnl_lock(), and result refcount is unchanged.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001147 */
1148
WANG Cong6c555492014-09-11 15:35:09 -07001149struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
1150 unsigned short mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001151{
Pavel Emelianov7562f872007-05-03 15:13:45 -07001152 struct net_device *dev, *ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001153
WANG Cong6c555492014-09-11 15:35:09 -07001154 ASSERT_RTNL();
1155
Pavel Emelianov7562f872007-05-03 15:13:45 -07001156 ret = NULL;
WANG Cong6c555492014-09-11 15:35:09 -07001157 for_each_netdev(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001158 if (((dev->flags ^ if_flags) & mask) == 0) {
Pavel Emelianov7562f872007-05-03 15:13:45 -07001159 ret = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001160 break;
1161 }
1162 }
Pavel Emelianov7562f872007-05-03 15:13:45 -07001163 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001164}
WANG Cong6c555492014-09-11 15:35:09 -07001165EXPORT_SYMBOL(__dev_get_by_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001166
1167/**
1168 * dev_valid_name - check if name is okay for network device
1169 * @name: name string
1170 *
1171 * Network device names need to be valid file names to
Randy Dunlap4250b752020-09-17 21:35:15 -07001172 * allow sysfs to work. We also disallow any kind of
David S. Millerc7fa9d12006-08-15 16:34:13 -07001173 * whitespace.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001174 */
David S. Miller95f050b2012-03-06 16:12:15 -05001175bool dev_valid_name(const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001176{
David S. Millerc7fa9d12006-08-15 16:34:13 -07001177 if (*name == '\0')
David S. Miller95f050b2012-03-06 16:12:15 -05001178 return false;
Eric Dumazeta9d48202018-04-05 06:39:26 -07001179 if (strnlen(name, IFNAMSIZ) == IFNAMSIZ)
David S. Miller95f050b2012-03-06 16:12:15 -05001180 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -07001181 if (!strcmp(name, ".") || !strcmp(name, ".."))
David S. Miller95f050b2012-03-06 16:12:15 -05001182 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -07001183
1184 while (*name) {
Matthew Thodea4176a92015-02-17 18:31:57 -06001185 if (*name == '/' || *name == ':' || isspace(*name))
David S. Miller95f050b2012-03-06 16:12:15 -05001186 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -07001187 name++;
1188 }
David S. Miller95f050b2012-03-06 16:12:15 -05001189 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001190}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001191EXPORT_SYMBOL(dev_valid_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001192
1193/**
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001194 * __dev_alloc_name - allocate a name for a device
1195 * @net: network namespace to allocate the device name in
Linus Torvalds1da177e2005-04-16 15:20:36 -07001196 * @name: name format string
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001197 * @buf: scratch buffer and result name string
Linus Torvalds1da177e2005-04-16 15:20:36 -07001198 *
1199 * Passed a format string - eg "lt%d" it will try and find a suitable
Stephen Hemminger3041a062006-05-26 13:25:24 -07001200 * id. It scans list of devices to build up a free map, then chooses
1201 * the first empty slot. The caller must hold the dev_base or rtnl lock
1202 * while allocating the name and adding the device in order to avoid
1203 * duplicates.
1204 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1205 * Returns the number of the unit assigned or a negative errno code.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001206 */
1207
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001208static int __dev_alloc_name(struct net *net, const char *name, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001209{
1210 int i = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001211 const char *p;
1212 const int max_netdevices = 8*PAGE_SIZE;
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -07001213 unsigned long *inuse;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001214 struct net_device *d;
1215
Rasmus Villemoes93809102017-11-13 00:15:08 +01001216 if (!dev_valid_name(name))
1217 return -EINVAL;
1218
Rasmus Villemoes51f299d2017-11-13 00:15:04 +01001219 p = strchr(name, '%');
Linus Torvalds1da177e2005-04-16 15:20:36 -07001220 if (p) {
1221 /*
1222 * Verify the string as this thing may have come from
1223 * the user. There must be either one "%d" and no other "%"
1224 * characters.
1225 */
1226 if (p[1] != 'd' || strchr(p + 2, '%'))
1227 return -EINVAL;
1228
1229 /* Use one page as a bit array of possible slots */
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -07001230 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001231 if (!inuse)
1232 return -ENOMEM;
1233
Eric W. Biederman881d9662007-09-17 11:56:21 -07001234 for_each_netdev(net, d) {
Jiri Bohac6c015a22021-03-18 04:42:53 +01001235 struct netdev_name_node *name_node;
1236 list_for_each_entry(name_node, &d->name_node->list, list) {
1237 if (!sscanf(name_node->name, name, &i))
1238 continue;
1239 if (i < 0 || i >= max_netdevices)
1240 continue;
1241
1242 /* avoid cases where sscanf is not exact inverse of printf */
1243 snprintf(buf, IFNAMSIZ, name, i);
1244 if (!strncmp(buf, name_node->name, IFNAMSIZ))
1245 set_bit(i, inuse);
1246 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001247 if (!sscanf(d->name, name, &i))
1248 continue;
1249 if (i < 0 || i >= max_netdevices)
1250 continue;
1251
1252 /* avoid cases where sscanf is not exact inverse of printf */
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001253 snprintf(buf, IFNAMSIZ, name, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001254 if (!strncmp(buf, d->name, IFNAMSIZ))
1255 set_bit(i, inuse);
1256 }
1257
1258 i = find_first_zero_bit(inuse, max_netdevices);
1259 free_page((unsigned long) inuse);
1260 }
1261
Rasmus Villemoes6224abd2017-11-13 00:15:07 +01001262 snprintf(buf, IFNAMSIZ, name, i);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001263 if (!__dev_get_by_name(net, buf))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001264 return i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001265
1266 /* It is possible to run out of possible slots
1267 * when the name is long and there isn't enough space left
1268 * for the digits, or if all bits are used.
1269 */
Johannes Berg029b6d12017-12-02 08:41:55 +01001270 return -ENFILE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001271}
1272
Rasmus Villemoes2c88b852017-11-13 00:15:05 +01001273static int dev_alloc_name_ns(struct net *net,
1274 struct net_device *dev,
1275 const char *name)
1276{
1277 char buf[IFNAMSIZ];
1278 int ret;
1279
Rasmus Villemoesc46d7642017-11-13 00:15:06 +01001280 BUG_ON(!net);
Rasmus Villemoes2c88b852017-11-13 00:15:05 +01001281 ret = __dev_alloc_name(net, name, buf);
1282 if (ret >= 0)
1283 strlcpy(dev->name, buf, IFNAMSIZ);
1284 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001285}
1286
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001287/**
1288 * dev_alloc_name - allocate a name for a device
1289 * @dev: device
1290 * @name: name format string
1291 *
1292 * Passed a format string - eg "lt%d" it will try and find a suitable
1293 * id. It scans list of devices to build up a free map, then chooses
1294 * the first empty slot. The caller must hold the dev_base or rtnl lock
1295 * while allocating the name and adding the device in order to avoid
1296 * duplicates.
1297 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1298 * Returns the number of the unit assigned or a negative errno code.
1299 */
1300
1301int dev_alloc_name(struct net_device *dev, const char *name)
1302{
Rasmus Villemoesc46d7642017-11-13 00:15:06 +01001303 return dev_alloc_name_ns(dev_net(dev), dev, name);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001304}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001305EXPORT_SYMBOL(dev_alloc_name);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001306
Eric Dumazetbacb7e12019-10-08 14:20:34 -07001307static int dev_get_valid_name(struct net *net, struct net_device *dev,
1308 const char *name)
Gao feng828de4f2012-09-13 20:58:27 +00001309{
David S. Miller55a5ec92018-01-02 11:45:07 -05001310 BUG_ON(!net);
1311
1312 if (!dev_valid_name(name))
1313 return -EINVAL;
1314
1315 if (strchr(name, '%'))
1316 return dev_alloc_name_ns(net, dev, name);
1317 else if (__dev_get_by_name(net, name))
1318 return -EEXIST;
1319 else if (dev->name != name)
1320 strlcpy(dev->name, name, IFNAMSIZ);
1321
1322 return 0;
Octavian Purdilad9031022009-11-18 02:36:59 +00001323}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001324
1325/**
1326 * dev_change_name - change name of a device
1327 * @dev: device
1328 * @newname: name (or format string) must be at least IFNAMSIZ
1329 *
1330 * Change name of a device, can pass format strings "eth%d".
1331 * for wildcarding.
1332 */
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07001333int dev_change_name(struct net_device *dev, const char *newname)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001334{
Tom Gundersen238fa362014-07-14 16:37:23 +02001335 unsigned char old_assign_type;
Herbert Xufcc5a032007-07-30 17:03:38 -07001336 char oldname[IFNAMSIZ];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001337 int err = 0;
Herbert Xufcc5a032007-07-30 17:03:38 -07001338 int ret;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001339 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001340
1341 ASSERT_RTNL();
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001342 BUG_ON(!dev_net(dev));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001343
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001344 net = dev_net(dev);
Si-Wei Liu8065a772019-04-08 19:45:27 -04001345
1346 /* Some auto-enslaved devices e.g. failover slaves are
1347 * special, as userspace might rename the device after
1348 * the interface had been brought up and running since
1349 * the point kernel initiated auto-enslavement. Allow
1350 * live name change even when these slave devices are
1351 * up and running.
1352 *
1353 * Typically, users of these auto-enslaving devices
1354 * don't actually care about slave name change, as
1355 * they are supposed to operate on master interface
1356 * directly.
1357 */
1358 if (dev->flags & IFF_UP &&
1359 likely(!(dev->priv_flags & IFF_LIVE_RENAME_OK)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001360 return -EBUSY;
1361
Ahmed S. Darwish11d60112020-06-03 16:49:44 +02001362 down_write(&devnet_rename_sem);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001363
1364 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
Ahmed S. Darwish11d60112020-06-03 16:49:44 +02001365 up_write(&devnet_rename_sem);
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001366 return 0;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001367 }
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001368
Herbert Xufcc5a032007-07-30 17:03:38 -07001369 memcpy(oldname, dev->name, IFNAMSIZ);
1370
Gao feng828de4f2012-09-13 20:58:27 +00001371 err = dev_get_valid_name(net, dev, newname);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001372 if (err < 0) {
Ahmed S. Darwish11d60112020-06-03 16:49:44 +02001373 up_write(&devnet_rename_sem);
Octavian Purdilad9031022009-11-18 02:36:59 +00001374 return err;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001375 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001376
Veaceslav Falico6fe82a32014-07-17 20:33:32 +02001377 if (oldname[0] && !strchr(oldname, '%'))
1378 netdev_info(dev, "renamed from %s\n", oldname);
1379
Tom Gundersen238fa362014-07-14 16:37:23 +02001380 old_assign_type = dev->name_assign_type;
1381 dev->name_assign_type = NET_NAME_RENAMED;
1382
Herbert Xufcc5a032007-07-30 17:03:38 -07001383rollback:
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001384 ret = device_rename(&dev->dev, dev->name);
1385 if (ret) {
1386 memcpy(dev->name, oldname, IFNAMSIZ);
Tom Gundersen238fa362014-07-14 16:37:23 +02001387 dev->name_assign_type = old_assign_type;
Ahmed S. Darwish11d60112020-06-03 16:49:44 +02001388 up_write(&devnet_rename_sem);
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001389 return ret;
Stephen Hemmingerdcc99772008-05-14 22:33:38 -07001390 }
Herbert Xu7f988ea2007-07-30 16:35:46 -07001391
Ahmed S. Darwish11d60112020-06-03 16:49:44 +02001392 up_write(&devnet_rename_sem);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001393
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01001394 netdev_adjacent_rename_links(dev, oldname);
1395
Herbert Xu7f988ea2007-07-30 16:35:46 -07001396 write_lock_bh(&dev_base_lock);
Jiri Pirkoff927412019-09-30 11:48:15 +02001397 netdev_name_node_del(dev->name_node);
Eric Dumazet72c95282009-10-30 07:11:27 +00001398 write_unlock_bh(&dev_base_lock);
1399
1400 synchronize_rcu();
1401
1402 write_lock_bh(&dev_base_lock);
Jiri Pirkoff927412019-09-30 11:48:15 +02001403 netdev_name_node_add(net, dev->name_node);
Herbert Xu7f988ea2007-07-30 16:35:46 -07001404 write_unlock_bh(&dev_base_lock);
1405
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001406 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001407 ret = notifier_to_errno(ret);
1408
1409 if (ret) {
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001410 /* err >= 0 after dev_alloc_name() or stores the first errno */
1411 if (err >= 0) {
Herbert Xufcc5a032007-07-30 17:03:38 -07001412 err = ret;
Ahmed S. Darwish11d60112020-06-03 16:49:44 +02001413 down_write(&devnet_rename_sem);
Herbert Xufcc5a032007-07-30 17:03:38 -07001414 memcpy(dev->name, oldname, IFNAMSIZ);
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01001415 memcpy(oldname, newname, IFNAMSIZ);
Tom Gundersen238fa362014-07-14 16:37:23 +02001416 dev->name_assign_type = old_assign_type;
1417 old_assign_type = NET_NAME_RENAMED;
Herbert Xufcc5a032007-07-30 17:03:38 -07001418 goto rollback;
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001419 } else {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001420 pr_err("%s: name change rollback failed: %d\n",
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001421 dev->name, ret);
Herbert Xufcc5a032007-07-30 17:03:38 -07001422 }
1423 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001424
1425 return err;
1426}
1427
1428/**
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001429 * dev_set_alias - change ifalias of a device
1430 * @dev: device
1431 * @alias: name up to IFALIASZ
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07001432 * @len: limit of bytes to copy from info
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001433 *
1434 * Set ifalias for a device,
1435 */
1436int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1437{
Florian Westphal6c557002017-10-02 23:50:05 +02001438 struct dev_ifalias *new_alias = NULL;
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001439
1440 if (len >= IFALIASZ)
1441 return -EINVAL;
1442
Florian Westphal6c557002017-10-02 23:50:05 +02001443 if (len) {
1444 new_alias = kmalloc(sizeof(*new_alias) + len + 1, GFP_KERNEL);
1445 if (!new_alias)
1446 return -ENOMEM;
1447
1448 memcpy(new_alias->ifalias, alias, len);
1449 new_alias->ifalias[len] = 0;
Oliver Hartkopp96ca4a2c2008-09-23 21:23:19 -07001450 }
1451
Florian Westphal6c557002017-10-02 23:50:05 +02001452 mutex_lock(&ifalias_mutex);
Paul E. McKenneye3f0d762019-09-23 15:42:28 -07001453 new_alias = rcu_replace_pointer(dev->ifalias, new_alias,
1454 mutex_is_locked(&ifalias_mutex));
Florian Westphal6c557002017-10-02 23:50:05 +02001455 mutex_unlock(&ifalias_mutex);
1456
1457 if (new_alias)
1458 kfree_rcu(new_alias, rcuhead);
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001459
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001460 return len;
1461}
Stephen Hemminger0fe554a2018-04-17 14:25:30 -07001462EXPORT_SYMBOL(dev_set_alias);
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001463
Florian Westphal6c557002017-10-02 23:50:05 +02001464/**
1465 * dev_get_alias - get ifalias of a device
1466 * @dev: device
Florian Westphal20e88322017-10-04 13:56:50 +02001467 * @name: buffer to store name of ifalias
Florian Westphal6c557002017-10-02 23:50:05 +02001468 * @len: size of buffer
1469 *
1470 * get ifalias for a device. Caller must make sure dev cannot go
1471 * away, e.g. rcu read lock or own a reference count to device.
1472 */
1473int dev_get_alias(const struct net_device *dev, char *name, size_t len)
1474{
1475 const struct dev_ifalias *alias;
1476 int ret = 0;
1477
1478 rcu_read_lock();
1479 alias = rcu_dereference(dev->ifalias);
1480 if (alias)
1481 ret = snprintf(name, len, "%s", alias->ifalias);
1482 rcu_read_unlock();
1483
1484 return ret;
1485}
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001486
1487/**
Stephen Hemminger3041a062006-05-26 13:25:24 -07001488 * netdev_features_change - device changes features
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001489 * @dev: device to cause notification
1490 *
1491 * Called to indicate a device has changed features.
1492 */
1493void netdev_features_change(struct net_device *dev)
1494{
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001495 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001496}
1497EXPORT_SYMBOL(netdev_features_change);
1498
1499/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001500 * netdev_state_change - device changes state
1501 * @dev: device to cause notification
1502 *
1503 * Called to indicate a device has changed state. This function calls
1504 * the notifier chains for netdev_chain and sends a NEWLINK message
1505 * to the routing socket.
1506 */
1507void netdev_state_change(struct net_device *dev)
1508{
1509 if (dev->flags & IFF_UP) {
David Ahern51d0c0472017-10-04 17:48:45 -07001510 struct netdev_notifier_change_info change_info = {
1511 .info.dev = dev,
1512 };
Loic Prylli54951192014-07-01 21:39:43 -07001513
David Ahern51d0c0472017-10-04 17:48:45 -07001514 call_netdevice_notifiers_info(NETDEV_CHANGE,
Loic Prylli54951192014-07-01 21:39:43 -07001515 &change_info.info);
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001516 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001517 }
1518}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001519EXPORT_SYMBOL(netdev_state_change);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001520
Amerigo Wangee89bab2012-08-09 22:14:56 +00001521/**
Lijun Pan7061eb82020-12-14 15:19:28 -06001522 * __netdev_notify_peers - notify network peers about existence of @dev,
1523 * to be called when rtnl lock is already held.
1524 * @dev: network device
1525 *
1526 * Generate traffic such that interested network peers are aware of
1527 * @dev, such as by generating a gratuitous ARP. This may be used when
1528 * a device wants to inform the rest of the network about some sort of
1529 * reconfiguration such as a failover event or virtual machine
1530 * migration.
1531 */
1532void __netdev_notify_peers(struct net_device *dev)
1533{
1534 ASSERT_RTNL();
1535 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1536 call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
1537}
1538EXPORT_SYMBOL(__netdev_notify_peers);
1539
1540/**
tcharding722c9a02017-02-09 17:56:04 +11001541 * netdev_notify_peers - notify network peers about existence of @dev
1542 * @dev: network device
Amerigo Wangee89bab2012-08-09 22:14:56 +00001543 *
1544 * Generate traffic such that interested network peers are aware of
1545 * @dev, such as by generating a gratuitous ARP. This may be used when
1546 * a device wants to inform the rest of the network about some sort of
1547 * reconfiguration such as a failover event or virtual machine
1548 * migration.
1549 */
1550void netdev_notify_peers(struct net_device *dev)
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001551{
Amerigo Wangee89bab2012-08-09 22:14:56 +00001552 rtnl_lock();
Lijun Pan7061eb82020-12-14 15:19:28 -06001553 __netdev_notify_peers(dev);
Amerigo Wangee89bab2012-08-09 22:14:56 +00001554 rtnl_unlock();
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001555}
Amerigo Wangee89bab2012-08-09 22:14:56 +00001556EXPORT_SYMBOL(netdev_notify_peers);
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001557
Wei Wang29863d42021-02-08 11:34:09 -08001558static int napi_threaded_poll(void *data);
1559
1560static int napi_kthread_create(struct napi_struct *n)
1561{
1562 int err = 0;
1563
1564 /* Create and wake up the kthread once to put it in
1565 * TASK_INTERRUPTIBLE mode to avoid the blocked task
1566 * warning and work with loadavg.
1567 */
1568 n->thread = kthread_run(napi_threaded_poll, n, "napi/%s-%d",
1569 n->dev->name, n->napi_id);
1570 if (IS_ERR(n->thread)) {
1571 err = PTR_ERR(n->thread);
1572 pr_err("kthread_run failed with err %d\n", err);
1573 n->thread = NULL;
1574 }
1575
1576 return err;
1577}
1578
Petr Machata40c900a2018-12-06 17:05:47 +00001579static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001580{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001581 const struct net_device_ops *ops = dev->netdev_ops;
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001582 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001583
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001584 ASSERT_RTNL();
1585
Heiner Kallweitbd869242020-06-20 22:35:42 +02001586 if (!netif_device_present(dev)) {
1587 /* may be detached because parent is runtime-suspended */
1588 if (dev->dev.parent)
1589 pm_runtime_resume(dev->dev.parent);
1590 if (!netif_device_present(dev))
1591 return -ENODEV;
1592 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001593
Neil Hormanca99ca12013-02-05 08:05:43 +00001594 /* Block netpoll from trying to do any rx path servicing.
1595 * If we don't do this there is a chance ndo_poll_controller
1596 * or ndo_poll may be running while we open the device
1597 */
Eric W. Biederman66b55522014-03-27 15:39:03 -07001598 netpoll_poll_disable(dev);
Neil Hormanca99ca12013-02-05 08:05:43 +00001599
Petr Machata40c900a2018-12-06 17:05:47 +00001600 ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack);
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001601 ret = notifier_to_errno(ret);
1602 if (ret)
1603 return ret;
1604
Linus Torvalds1da177e2005-04-16 15:20:36 -07001605 set_bit(__LINK_STATE_START, &dev->state);
Jeff Garzikbada3392007-10-23 20:19:37 -07001606
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001607 if (ops->ndo_validate_addr)
1608 ret = ops->ndo_validate_addr(dev);
Jeff Garzikbada3392007-10-23 20:19:37 -07001609
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001610 if (!ret && ops->ndo_open)
1611 ret = ops->ndo_open(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001612
Eric W. Biederman66b55522014-03-27 15:39:03 -07001613 netpoll_poll_enable(dev);
Neil Hormanca99ca12013-02-05 08:05:43 +00001614
Jeff Garzikbada3392007-10-23 20:19:37 -07001615 if (ret)
1616 clear_bit(__LINK_STATE_START, &dev->state);
1617 else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001618 dev->flags |= IFF_UP;
Patrick McHardy4417da62007-06-27 01:28:10 -07001619 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001620 dev_activate(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04001621 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001622 }
Jeff Garzikbada3392007-10-23 20:19:37 -07001623
Linus Torvalds1da177e2005-04-16 15:20:36 -07001624 return ret;
1625}
Patrick McHardybd380812010-02-26 06:34:53 +00001626
1627/**
1628 * dev_open - prepare an interface for use.
Petr Machata00f54e62018-12-06 17:05:36 +00001629 * @dev: device to open
1630 * @extack: netlink extended ack
Patrick McHardybd380812010-02-26 06:34:53 +00001631 *
1632 * Takes a device from down to up state. The device's private open
1633 * function is invoked and then the multicast lists are loaded. Finally
1634 * the device is moved into the up state and a %NETDEV_UP message is
1635 * sent to the netdev notifier chain.
1636 *
1637 * Calling this function on an active interface is a nop. On a failure
1638 * a negative errno code is returned.
1639 */
Petr Machata00f54e62018-12-06 17:05:36 +00001640int dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
Patrick McHardybd380812010-02-26 06:34:53 +00001641{
1642 int ret;
1643
Patrick McHardybd380812010-02-26 06:34:53 +00001644 if (dev->flags & IFF_UP)
1645 return 0;
1646
Petr Machata40c900a2018-12-06 17:05:47 +00001647 ret = __dev_open(dev, extack);
Patrick McHardybd380812010-02-26 06:34:53 +00001648 if (ret < 0)
1649 return ret;
1650
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001651 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
Patrick McHardybd380812010-02-26 06:34:53 +00001652 call_netdevice_notifiers(NETDEV_UP, dev);
1653
1654 return ret;
1655}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001656EXPORT_SYMBOL(dev_open);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001657
stephen hemminger7051b882017-07-18 15:59:27 -07001658static void __dev_close_many(struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001659{
Octavian Purdila44345722010-12-13 12:44:07 +00001660 struct net_device *dev;
Patrick McHardybd380812010-02-26 06:34:53 +00001661
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001662 ASSERT_RTNL();
David S. Miller9d5010d2007-09-12 14:33:25 +02001663 might_sleep();
1664
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001665 list_for_each_entry(dev, head, close_list) {
Eric W. Biederman3f4df202014-03-27 15:38:17 -07001666 /* Temporarily disable netpoll until the interface is down */
Eric W. Biederman66b55522014-03-27 15:39:03 -07001667 netpoll_poll_disable(dev);
Eric W. Biederman3f4df202014-03-27 15:38:17 -07001668
Octavian Purdila44345722010-12-13 12:44:07 +00001669 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001670
Octavian Purdila44345722010-12-13 12:44:07 +00001671 clear_bit(__LINK_STATE_START, &dev->state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001672
Octavian Purdila44345722010-12-13 12:44:07 +00001673 /* Synchronize to scheduled poll. We cannot touch poll list, it
1674 * can be even on different cpu. So just clear netif_running().
1675 *
1676 * dev->stop() will invoke napi_disable() on all of it's
1677 * napi_struct instances on this device.
1678 */
Peter Zijlstra4e857c52014-03-17 18:06:10 +01001679 smp_mb__after_atomic(); /* Commit netif_running(). */
Octavian Purdila44345722010-12-13 12:44:07 +00001680 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001681
Octavian Purdila44345722010-12-13 12:44:07 +00001682 dev_deactivate_many(head);
1683
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001684 list_for_each_entry(dev, head, close_list) {
Octavian Purdila44345722010-12-13 12:44:07 +00001685 const struct net_device_ops *ops = dev->netdev_ops;
1686
1687 /*
1688 * Call the device specific close. This cannot fail.
1689 * Only if device is UP
1690 *
1691 * We allow it to be called even after a DETACH hot-plug
1692 * event.
1693 */
1694 if (ops->ndo_stop)
1695 ops->ndo_stop(dev);
1696
Octavian Purdila44345722010-12-13 12:44:07 +00001697 dev->flags &= ~IFF_UP;
Eric W. Biederman66b55522014-03-27 15:39:03 -07001698 netpoll_poll_enable(dev);
Octavian Purdila44345722010-12-13 12:44:07 +00001699 }
Octavian Purdila44345722010-12-13 12:44:07 +00001700}
1701
stephen hemminger7051b882017-07-18 15:59:27 -07001702static void __dev_close(struct net_device *dev)
Octavian Purdila44345722010-12-13 12:44:07 +00001703{
1704 LIST_HEAD(single);
1705
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001706 list_add(&dev->close_list, &single);
stephen hemminger7051b882017-07-18 15:59:27 -07001707 __dev_close_many(&single);
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001708 list_del(&single);
Octavian Purdila44345722010-12-13 12:44:07 +00001709}
1710
stephen hemminger7051b882017-07-18 15:59:27 -07001711void dev_close_many(struct list_head *head, bool unlink)
Octavian Purdila44345722010-12-13 12:44:07 +00001712{
1713 struct net_device *dev, *tmp;
Octavian Purdila44345722010-12-13 12:44:07 +00001714
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001715 /* Remove the devices that don't need to be closed */
1716 list_for_each_entry_safe(dev, tmp, head, close_list)
Octavian Purdila44345722010-12-13 12:44:07 +00001717 if (!(dev->flags & IFF_UP))
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001718 list_del_init(&dev->close_list);
Octavian Purdila44345722010-12-13 12:44:07 +00001719
1720 __dev_close_many(head);
Matti Linnanvuorid8b2a4d2008-02-12 23:10:11 -08001721
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001722 list_for_each_entry_safe(dev, tmp, head, close_list) {
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001723 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
Octavian Purdila44345722010-12-13 12:44:07 +00001724 call_netdevice_notifiers(NETDEV_DOWN, dev);
David S. Miller99c4a262015-03-18 22:52:33 -04001725 if (unlink)
1726 list_del_init(&dev->close_list);
Octavian Purdila44345722010-12-13 12:44:07 +00001727 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001728}
David S. Miller99c4a262015-03-18 22:52:33 -04001729EXPORT_SYMBOL(dev_close_many);
Patrick McHardybd380812010-02-26 06:34:53 +00001730
1731/**
1732 * dev_close - shutdown an interface.
1733 * @dev: device to shutdown
1734 *
1735 * This function moves an active device into down state. A
1736 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1737 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1738 * chain.
1739 */
stephen hemminger7051b882017-07-18 15:59:27 -07001740void dev_close(struct net_device *dev)
Patrick McHardybd380812010-02-26 06:34:53 +00001741{
Eric Dumazete14a5992011-05-10 12:26:06 -07001742 if (dev->flags & IFF_UP) {
1743 LIST_HEAD(single);
Patrick McHardybd380812010-02-26 06:34:53 +00001744
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001745 list_add(&dev->close_list, &single);
David S. Miller99c4a262015-03-18 22:52:33 -04001746 dev_close_many(&single, true);
Eric Dumazete14a5992011-05-10 12:26:06 -07001747 list_del(&single);
1748 }
Patrick McHardybd380812010-02-26 06:34:53 +00001749}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001750EXPORT_SYMBOL(dev_close);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001751
1752
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001753/**
1754 * dev_disable_lro - disable Large Receive Offload on a device
1755 * @dev: device
1756 *
1757 * Disable Large Receive Offload (LRO) on a net device. Must be
1758 * called under RTNL. This is needed if received packets may be
1759 * forwarded to another interface.
1760 */
1761void dev_disable_lro(struct net_device *dev)
1762{
Michal Kubečekfbe168b2014-11-13 07:54:50 +01001763 struct net_device *lower_dev;
1764 struct list_head *iter;
Michal Kubeček529d0482013-11-15 06:18:50 +01001765
Michał Mirosławbc5787c62011-11-15 15:29:55 +00001766 dev->wanted_features &= ~NETIF_F_LRO;
1767 netdev_update_features(dev);
Michał Mirosław27660512011-03-18 16:56:34 +00001768
Michał Mirosław22d59692011-04-21 12:42:15 +00001769 if (unlikely(dev->features & NETIF_F_LRO))
1770 netdev_WARN(dev, "failed to disable LRO!\n");
Michal Kubečekfbe168b2014-11-13 07:54:50 +01001771
1772 netdev_for_each_lower_dev(dev, lower_dev, iter)
1773 dev_disable_lro(lower_dev);
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001774}
1775EXPORT_SYMBOL(dev_disable_lro);
1776
Michael Chan56f5aa72017-12-16 03:09:41 -05001777/**
1778 * dev_disable_gro_hw - disable HW Generic Receive Offload on a device
1779 * @dev: device
1780 *
1781 * Disable HW Generic Receive Offload (GRO_HW) on a net device. Must be
1782 * called under RTNL. This is needed if Generic XDP is installed on
1783 * the device.
1784 */
1785static void dev_disable_gro_hw(struct net_device *dev)
1786{
1787 dev->wanted_features &= ~NETIF_F_GRO_HW;
1788 netdev_update_features(dev);
1789
1790 if (unlikely(dev->features & NETIF_F_GRO_HW))
1791 netdev_WARN(dev, "failed to disable GRO_HW!\n");
1792}
1793
Kirill Tkhaiede27622018-03-23 19:47:19 +03001794const char *netdev_cmd_to_name(enum netdev_cmd cmd)
1795{
1796#define N(val) \
1797 case NETDEV_##val: \
1798 return "NETDEV_" __stringify(val);
1799 switch (cmd) {
1800 N(UP) N(DOWN) N(REBOOT) N(CHANGE) N(REGISTER) N(UNREGISTER)
1801 N(CHANGEMTU) N(CHANGEADDR) N(GOING_DOWN) N(CHANGENAME) N(FEAT_CHANGE)
1802 N(BONDING_FAILOVER) N(PRE_UP) N(PRE_TYPE_CHANGE) N(POST_TYPE_CHANGE)
1803 N(POST_INIT) N(RELEASE) N(NOTIFY_PEERS) N(JOIN) N(CHANGEUPPER)
1804 N(RESEND_IGMP) N(PRECHANGEMTU) N(CHANGEINFODATA) N(BONDING_INFO)
1805 N(PRECHANGEUPPER) N(CHANGELOWERSTATE) N(UDP_TUNNEL_PUSH_INFO)
1806 N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN)
Gal Pressman9daae9b2018-03-28 17:46:54 +03001807 N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO)
1808 N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO)
Petr Machata15704152018-12-13 11:54:33 +00001809 N(PRE_CHANGEADDR)
Kirill Tkhai3f5ecd82018-04-26 15:18:38 +03001810 }
Kirill Tkhaiede27622018-03-23 19:47:19 +03001811#undef N
1812 return "UNKNOWN_NETDEV_EVENT";
1813}
1814EXPORT_SYMBOL_GPL(netdev_cmd_to_name);
1815
Jiri Pirko351638e2013-05-28 01:30:21 +00001816static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1817 struct net_device *dev)
1818{
David Ahern51d0c0472017-10-04 17:48:45 -07001819 struct netdev_notifier_info info = {
1820 .dev = dev,
1821 };
Jiri Pirko351638e2013-05-28 01:30:21 +00001822
Jiri Pirko351638e2013-05-28 01:30:21 +00001823 return nb->notifier_call(nb, val, &info);
1824}
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001825
Jiri Pirkoafa0df52019-09-30 10:15:09 +02001826static int call_netdevice_register_notifiers(struct notifier_block *nb,
1827 struct net_device *dev)
1828{
1829 int err;
1830
1831 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1832 err = notifier_to_errno(err);
1833 if (err)
1834 return err;
1835
1836 if (!(dev->flags & IFF_UP))
1837 return 0;
1838
1839 call_netdevice_notifier(nb, NETDEV_UP, dev);
1840 return 0;
1841}
1842
1843static void call_netdevice_unregister_notifiers(struct notifier_block *nb,
1844 struct net_device *dev)
1845{
1846 if (dev->flags & IFF_UP) {
1847 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1848 dev);
1849 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1850 }
1851 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1852}
1853
1854static int call_netdevice_register_net_notifiers(struct notifier_block *nb,
1855 struct net *net)
1856{
1857 struct net_device *dev;
1858 int err;
1859
1860 for_each_netdev(net, dev) {
1861 err = call_netdevice_register_notifiers(nb, dev);
1862 if (err)
1863 goto rollback;
1864 }
1865 return 0;
1866
1867rollback:
1868 for_each_netdev_continue_reverse(net, dev)
1869 call_netdevice_unregister_notifiers(nb, dev);
1870 return err;
1871}
1872
1873static void call_netdevice_unregister_net_notifiers(struct notifier_block *nb,
1874 struct net *net)
1875{
1876 struct net_device *dev;
1877
1878 for_each_netdev(net, dev)
1879 call_netdevice_unregister_notifiers(nb, dev);
1880}
1881
Eric W. Biederman881d9662007-09-17 11:56:21 -07001882static int dev_boot_phase = 1;
1883
Linus Torvalds1da177e2005-04-16 15:20:36 -07001884/**
tcharding722c9a02017-02-09 17:56:04 +11001885 * register_netdevice_notifier - register a network notifier block
1886 * @nb: notifier
Linus Torvalds1da177e2005-04-16 15:20:36 -07001887 *
tcharding722c9a02017-02-09 17:56:04 +11001888 * Register a notifier to be called when network device events occur.
1889 * The notifier passed is linked into the kernel structures and must
1890 * not be reused until it has been unregistered. A negative errno code
1891 * is returned on a failure.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001892 *
tcharding722c9a02017-02-09 17:56:04 +11001893 * When registered all registration and up events are replayed
1894 * to the new notifier to allow device to have a race free
1895 * view of the network device list.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001896 */
1897
1898int register_netdevice_notifier(struct notifier_block *nb)
1899{
Eric W. Biederman881d9662007-09-17 11:56:21 -07001900 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001901 int err;
1902
Kirill Tkhai328fbe72018-03-29 17:03:45 +03001903 /* Close race with setup_net() and cleanup_net() */
1904 down_write(&pernet_ops_rwsem);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001905 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001906 err = raw_notifier_chain_register(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001907 if (err)
1908 goto unlock;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001909 if (dev_boot_phase)
1910 goto unlock;
1911 for_each_net(net) {
Jiri Pirkoafa0df52019-09-30 10:15:09 +02001912 err = call_netdevice_register_net_notifiers(nb, net);
1913 if (err)
1914 goto rollback;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001915 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001916
1917unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001918 rtnl_unlock();
Kirill Tkhai328fbe72018-03-29 17:03:45 +03001919 up_write(&pernet_ops_rwsem);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001920 return err;
Herbert Xufcc5a032007-07-30 17:03:38 -07001921
1922rollback:
Jiri Pirkoafa0df52019-09-30 10:15:09 +02001923 for_each_net_continue_reverse(net)
1924 call_netdevice_unregister_net_notifiers(nb, net);
Herbert Xufcc5a032007-07-30 17:03:38 -07001925
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001926 raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001927 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001928}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001929EXPORT_SYMBOL(register_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001930
1931/**
tcharding722c9a02017-02-09 17:56:04 +11001932 * unregister_netdevice_notifier - unregister a network notifier block
1933 * @nb: notifier
Linus Torvalds1da177e2005-04-16 15:20:36 -07001934 *
tcharding722c9a02017-02-09 17:56:04 +11001935 * Unregister a notifier previously registered by
1936 * register_netdevice_notifier(). The notifier is unlinked into the
1937 * kernel structures and may then be reused. A negative errno code
1938 * is returned on a failure.
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001939 *
tcharding722c9a02017-02-09 17:56:04 +11001940 * After unregistering unregister and down device events are synthesized
1941 * for all devices on the device list to the removed notifier to remove
1942 * the need for special case cleanup code.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001943 */
1944
1945int unregister_netdevice_notifier(struct notifier_block *nb)
1946{
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001947 struct net *net;
Herbert Xu9f514952006-03-25 01:24:25 -08001948 int err;
1949
Kirill Tkhai328fbe72018-03-29 17:03:45 +03001950 /* Close race with setup_net() and cleanup_net() */
1951 down_write(&pernet_ops_rwsem);
Herbert Xu9f514952006-03-25 01:24:25 -08001952 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001953 err = raw_notifier_chain_unregister(&netdev_chain, nb);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001954 if (err)
1955 goto unlock;
1956
Jiri Pirko48b3a132020-01-25 12:17:06 +01001957 for_each_net(net)
1958 call_netdevice_unregister_net_notifiers(nb, net);
1959
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001960unlock:
Herbert Xu9f514952006-03-25 01:24:25 -08001961 rtnl_unlock();
Kirill Tkhai328fbe72018-03-29 17:03:45 +03001962 up_write(&pernet_ops_rwsem);
Herbert Xu9f514952006-03-25 01:24:25 -08001963 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001964}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001965EXPORT_SYMBOL(unregister_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001966
Jiri Pirko1f637702020-01-25 12:17:07 +01001967static int __register_netdevice_notifier_net(struct net *net,
1968 struct notifier_block *nb,
1969 bool ignore_call_fail)
1970{
1971 int err;
1972
1973 err = raw_notifier_chain_register(&net->netdev_chain, nb);
1974 if (err)
1975 return err;
1976 if (dev_boot_phase)
1977 return 0;
1978
1979 err = call_netdevice_register_net_notifiers(nb, net);
1980 if (err && !ignore_call_fail)
1981 goto chain_unregister;
1982
1983 return 0;
1984
1985chain_unregister:
1986 raw_notifier_chain_unregister(&net->netdev_chain, nb);
1987 return err;
1988}
1989
1990static int __unregister_netdevice_notifier_net(struct net *net,
1991 struct notifier_block *nb)
1992{
1993 int err;
1994
1995 err = raw_notifier_chain_unregister(&net->netdev_chain, nb);
1996 if (err)
1997 return err;
1998
1999 call_netdevice_unregister_net_notifiers(nb, net);
2000 return 0;
2001}
2002
Linus Torvalds1da177e2005-04-16 15:20:36 -07002003/**
Jiri Pirkoa30c7b42019-09-30 10:15:10 +02002004 * register_netdevice_notifier_net - register a per-netns network notifier block
2005 * @net: network namespace
2006 * @nb: notifier
2007 *
2008 * Register a notifier to be called when network device events occur.
2009 * The notifier passed is linked into the kernel structures and must
2010 * not be reused until it has been unregistered. A negative errno code
2011 * is returned on a failure.
2012 *
2013 * When registered all registration and up events are replayed
2014 * to the new notifier to allow device to have a race free
2015 * view of the network device list.
2016 */
2017
2018int register_netdevice_notifier_net(struct net *net, struct notifier_block *nb)
2019{
2020 int err;
2021
2022 rtnl_lock();
Jiri Pirko1f637702020-01-25 12:17:07 +01002023 err = __register_netdevice_notifier_net(net, nb, false);
Jiri Pirkoa30c7b42019-09-30 10:15:10 +02002024 rtnl_unlock();
2025 return err;
Jiri Pirkoa30c7b42019-09-30 10:15:10 +02002026}
2027EXPORT_SYMBOL(register_netdevice_notifier_net);
2028
2029/**
2030 * unregister_netdevice_notifier_net - unregister a per-netns
2031 * network notifier block
2032 * @net: network namespace
2033 * @nb: notifier
2034 *
2035 * Unregister a notifier previously registered by
2036 * register_netdevice_notifier(). The notifier is unlinked into the
2037 * kernel structures and may then be reused. A negative errno code
2038 * is returned on a failure.
2039 *
2040 * After unregistering unregister and down device events are synthesized
2041 * for all devices on the device list to the removed notifier to remove
2042 * the need for special case cleanup code.
2043 */
2044
2045int unregister_netdevice_notifier_net(struct net *net,
2046 struct notifier_block *nb)
2047{
2048 int err;
2049
2050 rtnl_lock();
Jiri Pirko1f637702020-01-25 12:17:07 +01002051 err = __unregister_netdevice_notifier_net(net, nb);
Jiri Pirkoa30c7b42019-09-30 10:15:10 +02002052 rtnl_unlock();
2053 return err;
2054}
2055EXPORT_SYMBOL(unregister_netdevice_notifier_net);
2056
Jiri Pirko93642e12020-01-25 12:17:08 +01002057int register_netdevice_notifier_dev_net(struct net_device *dev,
2058 struct notifier_block *nb,
2059 struct netdev_net_notifier *nn)
2060{
2061 int err;
2062
2063 rtnl_lock();
2064 err = __register_netdevice_notifier_net(dev_net(dev), nb, false);
2065 if (!err) {
2066 nn->nb = nb;
2067 list_add(&nn->list, &dev->net_notifier_list);
2068 }
2069 rtnl_unlock();
2070 return err;
2071}
2072EXPORT_SYMBOL(register_netdevice_notifier_dev_net);
2073
2074int unregister_netdevice_notifier_dev_net(struct net_device *dev,
2075 struct notifier_block *nb,
2076 struct netdev_net_notifier *nn)
2077{
2078 int err;
2079
2080 rtnl_lock();
2081 list_del(&nn->list);
2082 err = __unregister_netdevice_notifier_net(dev_net(dev), nb);
2083 rtnl_unlock();
2084 return err;
2085}
2086EXPORT_SYMBOL(unregister_netdevice_notifier_dev_net);
2087
2088static void move_netdevice_notifiers_dev_net(struct net_device *dev,
2089 struct net *net)
2090{
2091 struct netdev_net_notifier *nn;
2092
2093 list_for_each_entry(nn, &dev->net_notifier_list, list) {
2094 __unregister_netdevice_notifier_net(dev_net(dev), nn->nb);
2095 __register_netdevice_notifier_net(net, nn->nb, true);
2096 }
2097}
2098
Jiri Pirkoa30c7b42019-09-30 10:15:10 +02002099/**
Jiri Pirko351638e2013-05-28 01:30:21 +00002100 * call_netdevice_notifiers_info - call all network notifier blocks
2101 * @val: value passed unmodified to notifier function
Jiri Pirko351638e2013-05-28 01:30:21 +00002102 * @info: notifier information data
2103 *
2104 * Call all network notifier blocks. Parameters and return value
2105 * are as for raw_notifier_call_chain().
2106 */
2107
stephen hemminger1d143d92013-12-29 14:01:29 -08002108static int call_netdevice_notifiers_info(unsigned long val,
stephen hemminger1d143d92013-12-29 14:01:29 -08002109 struct netdev_notifier_info *info)
Jiri Pirko351638e2013-05-28 01:30:21 +00002110{
Jiri Pirkoa30c7b42019-09-30 10:15:10 +02002111 struct net *net = dev_net(info->dev);
2112 int ret;
2113
Jiri Pirko351638e2013-05-28 01:30:21 +00002114 ASSERT_RTNL();
Jiri Pirkoa30c7b42019-09-30 10:15:10 +02002115
2116 /* Run per-netns notifier block chain first, then run the global one.
2117 * Hopefully, one day, the global one is going to be removed after
2118 * all notifier block registrators get converted to be per-netns.
2119 */
2120 ret = raw_notifier_call_chain(&net->netdev_chain, val, info);
2121 if (ret & NOTIFY_STOP_MASK)
2122 return ret;
Jiri Pirko351638e2013-05-28 01:30:21 +00002123 return raw_notifier_call_chain(&netdev_chain, val, info);
2124}
Jiri Pirko351638e2013-05-28 01:30:21 +00002125
Petr Machata26372602018-12-06 17:05:45 +00002126static int call_netdevice_notifiers_extack(unsigned long val,
2127 struct net_device *dev,
2128 struct netlink_ext_ack *extack)
2129{
2130 struct netdev_notifier_info info = {
2131 .dev = dev,
2132 .extack = extack,
2133 };
2134
2135 return call_netdevice_notifiers_info(val, &info);
2136}
2137
Jiri Pirko351638e2013-05-28 01:30:21 +00002138/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07002139 * call_netdevice_notifiers - call all network notifier blocks
2140 * @val: value passed unmodified to notifier function
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07002141 * @dev: net_device pointer passed unmodified to notifier function
Linus Torvalds1da177e2005-04-16 15:20:36 -07002142 *
2143 * Call all network notifier blocks. Parameters and return value
Alan Sternf07d5b92006-05-09 15:23:03 -07002144 * are as for raw_notifier_call_chain().
Linus Torvalds1da177e2005-04-16 15:20:36 -07002145 */
2146
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07002147int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002148{
Petr Machata26372602018-12-06 17:05:45 +00002149 return call_netdevice_notifiers_extack(val, dev, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002150}
stephen hemmingeredf947f2011-03-24 13:24:01 +00002151EXPORT_SYMBOL(call_netdevice_notifiers);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002152
Sabrina Dubrocaaf7d6cc2018-10-09 17:48:14 +02002153/**
2154 * call_netdevice_notifiers_mtu - call all network notifier blocks
2155 * @val: value passed unmodified to notifier function
2156 * @dev: net_device pointer passed unmodified to notifier function
2157 * @arg: additional u32 argument passed to the notifier function
2158 *
2159 * Call all network notifier blocks. Parameters and return value
2160 * are as for raw_notifier_call_chain().
2161 */
2162static int call_netdevice_notifiers_mtu(unsigned long val,
2163 struct net_device *dev, u32 arg)
2164{
2165 struct netdev_notifier_info_ext info = {
2166 .info.dev = dev,
2167 .ext.mtu = arg,
2168 };
2169
2170 BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != 0);
2171
2172 return call_netdevice_notifiers_info(val, &info.info);
2173}
2174
Pablo Neira1cf519002015-05-13 18:19:37 +02002175#ifdef CONFIG_NET_INGRESS
Davidlohr Buesoaabf6772018-05-08 09:07:00 -07002176static DEFINE_STATIC_KEY_FALSE(ingress_needed_key);
Daniel Borkmann45771392015-04-10 23:07:54 +02002177
2178void net_inc_ingress_queue(void)
2179{
Davidlohr Buesoaabf6772018-05-08 09:07:00 -07002180 static_branch_inc(&ingress_needed_key);
Daniel Borkmann45771392015-04-10 23:07:54 +02002181}
2182EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
2183
2184void net_dec_ingress_queue(void)
2185{
Davidlohr Buesoaabf6772018-05-08 09:07:00 -07002186 static_branch_dec(&ingress_needed_key);
Daniel Borkmann45771392015-04-10 23:07:54 +02002187}
2188EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
2189#endif
2190
Daniel Borkmann1f211a12016-01-07 22:29:47 +01002191#ifdef CONFIG_NET_EGRESS
Davidlohr Buesoaabf6772018-05-08 09:07:00 -07002192static DEFINE_STATIC_KEY_FALSE(egress_needed_key);
Daniel Borkmann1f211a12016-01-07 22:29:47 +01002193
2194void net_inc_egress_queue(void)
2195{
Davidlohr Buesoaabf6772018-05-08 09:07:00 -07002196 static_branch_inc(&egress_needed_key);
Daniel Borkmann1f211a12016-01-07 22:29:47 +01002197}
2198EXPORT_SYMBOL_GPL(net_inc_egress_queue);
2199
2200void net_dec_egress_queue(void)
2201{
Davidlohr Buesoaabf6772018-05-08 09:07:00 -07002202 static_branch_dec(&egress_needed_key);
Daniel Borkmann1f211a12016-01-07 22:29:47 +01002203}
2204EXPORT_SYMBOL_GPL(net_dec_egress_queue);
2205#endif
2206
Davidlohr Bueso39e83922018-05-08 09:07:01 -07002207static DEFINE_STATIC_KEY_FALSE(netstamp_needed_key);
Masahiro Yamadae9666d12018-12-31 00:14:15 +09002208#ifdef CONFIG_JUMP_LABEL
Eric Dumazetb90e5792011-11-28 11:16:50 +00002209static atomic_t netstamp_needed_deferred;
Eric Dumazet13baa002017-03-01 14:28:39 -08002210static atomic_t netstamp_wanted;
Eric Dumazet5fa8bbd2017-02-02 10:31:35 -08002211static void netstamp_clear(struct work_struct *work)
2212{
2213 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
Eric Dumazet13baa002017-03-01 14:28:39 -08002214 int wanted;
Eric Dumazet5fa8bbd2017-02-02 10:31:35 -08002215
Eric Dumazet13baa002017-03-01 14:28:39 -08002216 wanted = atomic_add_return(deferred, &netstamp_wanted);
2217 if (wanted > 0)
Davidlohr Bueso39e83922018-05-08 09:07:01 -07002218 static_branch_enable(&netstamp_needed_key);
Eric Dumazet13baa002017-03-01 14:28:39 -08002219 else
Davidlohr Bueso39e83922018-05-08 09:07:01 -07002220 static_branch_disable(&netstamp_needed_key);
Eric Dumazet5fa8bbd2017-02-02 10:31:35 -08002221}
2222static DECLARE_WORK(netstamp_work, netstamp_clear);
Eric Dumazetb90e5792011-11-28 11:16:50 +00002223#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002224
2225void net_enable_timestamp(void)
2226{
Masahiro Yamadae9666d12018-12-31 00:14:15 +09002227#ifdef CONFIG_JUMP_LABEL
Eric Dumazet13baa002017-03-01 14:28:39 -08002228 int wanted;
2229
2230 while (1) {
2231 wanted = atomic_read(&netstamp_wanted);
2232 if (wanted <= 0)
2233 break;
2234 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted)
2235 return;
2236 }
2237 atomic_inc(&netstamp_needed_deferred);
2238 schedule_work(&netstamp_work);
2239#else
Davidlohr Bueso39e83922018-05-08 09:07:01 -07002240 static_branch_inc(&netstamp_needed_key);
Eric Dumazet13baa002017-03-01 14:28:39 -08002241#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002242}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002243EXPORT_SYMBOL(net_enable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002244
2245void net_disable_timestamp(void)
2246{
Masahiro Yamadae9666d12018-12-31 00:14:15 +09002247#ifdef CONFIG_JUMP_LABEL
Eric Dumazet13baa002017-03-01 14:28:39 -08002248 int wanted;
2249
2250 while (1) {
2251 wanted = atomic_read(&netstamp_wanted);
2252 if (wanted <= 1)
2253 break;
2254 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted)
2255 return;
2256 }
2257 atomic_dec(&netstamp_needed_deferred);
Eric Dumazet5fa8bbd2017-02-02 10:31:35 -08002258 schedule_work(&netstamp_work);
2259#else
Davidlohr Bueso39e83922018-05-08 09:07:01 -07002260 static_branch_dec(&netstamp_needed_key);
Eric Dumazet5fa8bbd2017-02-02 10:31:35 -08002261#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002262}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002263EXPORT_SYMBOL(net_disable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002264
Eric Dumazet3b098e22010-05-15 23:57:10 -07002265static inline void net_timestamp_set(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002266{
Thomas Gleixner2456e852016-12-25 11:38:40 +01002267 skb->tstamp = 0;
Davidlohr Bueso39e83922018-05-08 09:07:01 -07002268 if (static_branch_unlikely(&netstamp_needed_key))
Patrick McHardya61bbcf2005-08-14 17:24:31 -07002269 __net_timestamp(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002270}
2271
Davidlohr Bueso39e83922018-05-08 09:07:01 -07002272#define net_timestamp_check(COND, SKB) \
2273 if (static_branch_unlikely(&netstamp_needed_key)) { \
2274 if ((COND) && !(SKB)->tstamp) \
2275 __net_timestamp(SKB); \
2276 } \
Eric Dumazet3b098e22010-05-15 23:57:10 -07002277
Nikolay Aleksandrovf4b05d22016-04-28 17:59:28 +02002278bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
Daniel Lezcano79b569f2011-03-30 02:42:17 -07002279{
Jesper Dangaard Brouer5f7d5722021-02-09 14:38:29 +01002280 return __is_skb_forwardable(dev, skb, true);
Daniel Lezcano79b569f2011-03-30 02:42:17 -07002281}
Vlad Yasevich1ee481f2014-03-27 17:32:29 -04002282EXPORT_SYMBOL_GPL(is_skb_forwardable);
Daniel Lezcano79b569f2011-03-30 02:42:17 -07002283
Jesper Dangaard Brouer5f7d5722021-02-09 14:38:29 +01002284static int __dev_forward_skb2(struct net_device *dev, struct sk_buff *skb,
2285 bool check_mtu)
Herbert Xua0265d22014-04-17 13:45:03 +08002286{
Jesper Dangaard Brouer5f7d5722021-02-09 14:38:29 +01002287 int ret = ____dev_forward_skb(dev, skb, check_mtu);
Martin KaFai Lau4e3264d2016-11-09 15:36:33 -08002288
2289 if (likely(!ret)) {
2290 skb->protocol = eth_type_trans(skb, dev);
2291 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
Herbert Xua0265d22014-04-17 13:45:03 +08002292 }
2293
Martin KaFai Lau4e3264d2016-11-09 15:36:33 -08002294 return ret;
Herbert Xua0265d22014-04-17 13:45:03 +08002295}
Jesper Dangaard Brouer5f7d5722021-02-09 14:38:29 +01002296
2297int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2298{
2299 return __dev_forward_skb2(dev, skb, true);
2300}
Herbert Xua0265d22014-04-17 13:45:03 +08002301EXPORT_SYMBOL_GPL(__dev_forward_skb);
2302
Arnd Bergmann44540962009-11-26 06:07:08 +00002303/**
2304 * dev_forward_skb - loopback an skb to another netif
2305 *
2306 * @dev: destination network device
2307 * @skb: buffer to forward
2308 *
2309 * return values:
2310 * NET_RX_SUCCESS (no congestion)
Eric Dumazet6ec82562010-05-06 00:53:53 -07002311 * NET_RX_DROP (packet was dropped, but freed)
Arnd Bergmann44540962009-11-26 06:07:08 +00002312 *
2313 * dev_forward_skb can be used for injecting an skb from the
2314 * start_xmit function of one device into the receive queue
2315 * of another device.
2316 *
2317 * The receiving device may be in another namespace, so
2318 * we have to clear all information in the skb that could
2319 * impact namespace isolation.
2320 */
2321int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2322{
Herbert Xua0265d22014-04-17 13:45:03 +08002323 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00002324}
2325EXPORT_SYMBOL_GPL(dev_forward_skb);
2326
Jesper Dangaard Brouer5f7d5722021-02-09 14:38:29 +01002327int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb)
2328{
2329 return __dev_forward_skb2(dev, skb, false) ?: netif_rx_internal(skb);
2330}
2331
Changli Gao71d9dec2010-12-15 19:57:25 +00002332static inline int deliver_skb(struct sk_buff *skb,
2333 struct packet_type *pt_prev,
2334 struct net_device *orig_dev)
2335{
Willem de Bruijn1f8b9772017-08-03 16:29:41 -04002336 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00002337 return -ENOMEM;
Reshetova, Elena63354792017-06-30 13:07:58 +03002338 refcount_inc(&skb->users);
Changli Gao71d9dec2010-12-15 19:57:25 +00002339 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
2340}
2341
Salam Noureddine7866a622015-01-27 11:35:48 -08002342static inline void deliver_ptype_list_skb(struct sk_buff *skb,
2343 struct packet_type **pt,
Jiri Pirkofbcb2172015-03-30 16:56:01 +02002344 struct net_device *orig_dev,
2345 __be16 type,
Salam Noureddine7866a622015-01-27 11:35:48 -08002346 struct list_head *ptype_list)
2347{
2348 struct packet_type *ptype, *pt_prev = *pt;
2349
2350 list_for_each_entry_rcu(ptype, ptype_list, list) {
2351 if (ptype->type != type)
2352 continue;
2353 if (pt_prev)
Jiri Pirkofbcb2172015-03-30 16:56:01 +02002354 deliver_skb(skb, pt_prev, orig_dev);
Salam Noureddine7866a622015-01-27 11:35:48 -08002355 pt_prev = ptype;
2356 }
2357 *pt = pt_prev;
2358}
2359
Eric Leblondc0de08d2012-08-16 22:02:58 +00002360static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
2361{
Eric Leblonda3d744e2012-11-06 02:10:10 +00002362 if (!ptype->af_packet_priv || !skb->sk)
Eric Leblondc0de08d2012-08-16 22:02:58 +00002363 return false;
2364
2365 if (ptype->id_match)
2366 return ptype->id_match(ptype, skb->sk);
2367 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
2368 return true;
2369
2370 return false;
2371}
2372
Maciej W. Rozycki9f9a7422018-10-09 23:57:49 +01002373/**
2374 * dev_nit_active - return true if any network interface taps are in use
2375 *
2376 * @dev: network device to check for the presence of taps
2377 */
2378bool dev_nit_active(struct net_device *dev)
2379{
2380 return !list_empty(&ptype_all) || !list_empty(&dev->ptype_all);
2381}
2382EXPORT_SYMBOL_GPL(dev_nit_active);
2383
Linus Torvalds1da177e2005-04-16 15:20:36 -07002384/*
2385 * Support routine. Sends outgoing frames to any network
2386 * taps currently in use.
2387 */
2388
David Ahern74b20582016-05-10 11:19:50 -07002389void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002390{
2391 struct packet_type *ptype;
Changli Gao71d9dec2010-12-15 19:57:25 +00002392 struct sk_buff *skb2 = NULL;
2393 struct packet_type *pt_prev = NULL;
Salam Noureddine7866a622015-01-27 11:35:48 -08002394 struct list_head *ptype_list = &ptype_all;
Patrick McHardya61bbcf2005-08-14 17:24:31 -07002395
Linus Torvalds1da177e2005-04-16 15:20:36 -07002396 rcu_read_lock();
Salam Noureddine7866a622015-01-27 11:35:48 -08002397again:
2398 list_for_each_entry_rcu(ptype, ptype_list, list) {
Vincent Whitchurchfa788d92018-09-03 16:23:36 +02002399 if (ptype->ignore_outgoing)
2400 continue;
2401
Linus Torvalds1da177e2005-04-16 15:20:36 -07002402 /* Never send packets back to the socket
2403 * they originated from - MvS (miquels@drinkel.ow.org)
2404 */
Salam Noureddine7866a622015-01-27 11:35:48 -08002405 if (skb_loop_sk(ptype, skb))
2406 continue;
Changli Gao71d9dec2010-12-15 19:57:25 +00002407
Salam Noureddine7866a622015-01-27 11:35:48 -08002408 if (pt_prev) {
2409 deliver_skb(skb2, pt_prev, skb->dev);
Changli Gao71d9dec2010-12-15 19:57:25 +00002410 pt_prev = ptype;
Salam Noureddine7866a622015-01-27 11:35:48 -08002411 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002412 }
Salam Noureddine7866a622015-01-27 11:35:48 -08002413
2414 /* need to clone skb, done only once */
2415 skb2 = skb_clone(skb, GFP_ATOMIC);
2416 if (!skb2)
2417 goto out_unlock;
2418
2419 net_timestamp_set(skb2);
2420
2421 /* skb->nh should be correctly
2422 * set by sender, so that the second statement is
2423 * just protection against buggy protocols.
2424 */
2425 skb_reset_mac_header(skb2);
2426
2427 if (skb_network_header(skb2) < skb2->data ||
2428 skb_network_header(skb2) > skb_tail_pointer(skb2)) {
2429 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
2430 ntohs(skb2->protocol),
2431 dev->name);
2432 skb_reset_network_header(skb2);
2433 }
2434
2435 skb2->transport_header = skb2->network_header;
2436 skb2->pkt_type = PACKET_OUTGOING;
2437 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002438 }
Salam Noureddine7866a622015-01-27 11:35:48 -08002439
2440 if (ptype_list == &ptype_all) {
2441 ptype_list = &dev->ptype_all;
2442 goto again;
2443 }
2444out_unlock:
Willem de Bruijn581fe0e2017-09-22 19:42:37 -04002445 if (pt_prev) {
2446 if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC))
2447 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
2448 else
2449 kfree_skb(skb2);
2450 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002451 rcu_read_unlock();
2452}
David Ahern74b20582016-05-10 11:19:50 -07002453EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002454
Ben Hutchings2c530402012-07-10 10:55:09 +00002455/**
2456 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
John Fastabend4f57c082011-01-17 08:06:04 +00002457 * @dev: Network device
2458 * @txq: number of queues available
2459 *
2460 * If real_num_tx_queues is changed the tc mappings may no longer be
2461 * valid. To resolve this verify the tc mapping remains valid and if
2462 * not NULL the mapping. With no priorities mapping to this
2463 * offset/count pair it will no longer be used. In the worst case TC0
2464 * is invalid nothing can be done so disable priority mappings. If is
2465 * expected that drivers will fix this mapping if they can before
2466 * calling netif_set_real_num_tx_queues.
2467 */
Eric Dumazetbb134d22011-01-20 19:18:08 +00002468static void netif_setup_tc(struct net_device *dev, unsigned int txq)
John Fastabend4f57c082011-01-17 08:06:04 +00002469{
2470 int i;
2471 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2472
2473 /* If TC0 is invalidated disable TC mapping */
2474 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00002475 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
John Fastabend4f57c082011-01-17 08:06:04 +00002476 dev->num_tc = 0;
2477 return;
2478 }
2479
2480 /* Invalidated prio to tc mappings set to TC0 */
2481 for (i = 1; i < TC_BITMASK + 1; i++) {
2482 int q = netdev_get_prio_tc_map(dev, i);
2483
2484 tc = &dev->tc_to_txq[q];
2485 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00002486 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
2487 i, q);
John Fastabend4f57c082011-01-17 08:06:04 +00002488 netdev_set_prio_tc_map(dev, i, 0);
2489 }
2490 }
2491}
2492
Alexander Duyck8d059b02016-10-28 11:43:49 -04002493int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
2494{
2495 if (dev->num_tc) {
2496 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2497 int i;
2498
Alexander Duyckffcfe252018-07-09 12:19:38 -04002499 /* walk through the TCs and see if it falls into any of them */
Alexander Duyck8d059b02016-10-28 11:43:49 -04002500 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
2501 if ((txq - tc->offset) < tc->count)
2502 return i;
2503 }
2504
Alexander Duyckffcfe252018-07-09 12:19:38 -04002505 /* didn't find it, just return -1 to indicate no match */
Alexander Duyck8d059b02016-10-28 11:43:49 -04002506 return -1;
2507 }
2508
2509 return 0;
2510}
Henrik Austad8a5f2162017-10-17 12:10:10 +02002511EXPORT_SYMBOL(netdev_txq_to_tc);
Alexander Duyck8d059b02016-10-28 11:43:49 -04002512
Alexander Duyck537c00d2013-01-10 08:57:02 +00002513#ifdef CONFIG_XPS
Vladimir Oltean5da9ace32021-03-22 13:30:19 +02002514static struct static_key xps_needed __read_mostly;
2515static struct static_key xps_rxqs_needed __read_mostly;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002516static DEFINE_MUTEX(xps_map_mutex);
2517#define xmap_dereference(P) \
2518 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
2519
Alexander Duyck6234f872016-10-28 11:46:49 -04002520static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
Antoine Tenart2d05bf02021-03-18 19:37:51 +01002521 struct xps_dev_maps *old_maps, int tci, u16 index)
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00002522{
2523 struct xps_map *map = NULL;
2524 int pos;
2525
2526 if (dev_maps)
Amritha Nambiar80d19662018-06-29 21:26:41 -07002527 map = xmap_dereference(dev_maps->attr_map[tci]);
Alexander Duyck6234f872016-10-28 11:46:49 -04002528 if (!map)
2529 return false;
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00002530
Alexander Duyck6234f872016-10-28 11:46:49 -04002531 for (pos = map->len; pos--;) {
2532 if (map->queues[pos] != index)
2533 continue;
2534
2535 if (map->len > 1) {
2536 map->queues[pos] = map->queues[--map->len];
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00002537 break;
2538 }
Alexander Duyck6234f872016-10-28 11:46:49 -04002539
Antoine Tenart2d05bf02021-03-18 19:37:51 +01002540 if (old_maps)
2541 RCU_INIT_POINTER(old_maps->attr_map[tci], NULL);
Amritha Nambiar80d19662018-06-29 21:26:41 -07002542 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
Alexander Duyck6234f872016-10-28 11:46:49 -04002543 kfree_rcu(map, rcu);
2544 return false;
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00002545 }
2546
Alexander Duyck6234f872016-10-28 11:46:49 -04002547 return true;
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00002548}
2549
Alexander Duyck6234f872016-10-28 11:46:49 -04002550static bool remove_xps_queue_cpu(struct net_device *dev,
2551 struct xps_dev_maps *dev_maps,
2552 int cpu, u16 offset, u16 count)
2553{
Antoine Tenart255c04a2021-03-18 19:37:43 +01002554 int num_tc = dev_maps->num_tc;
Alexander Duyck184c4492016-10-28 11:50:13 -04002555 bool active = false;
2556 int tci;
Alexander Duyck6234f872016-10-28 11:46:49 -04002557
Alexander Duyck184c4492016-10-28 11:50:13 -04002558 for (tci = cpu * num_tc; num_tc--; tci++) {
2559 int i, j;
2560
2561 for (i = count, j = offset; i--; j++) {
Antoine Tenart2d05bf02021-03-18 19:37:51 +01002562 if (!remove_xps_queue(dev_maps, NULL, tci, j))
Alexander Duyck184c4492016-10-28 11:50:13 -04002563 break;
2564 }
2565
2566 active |= i < 0;
Alexander Duyck6234f872016-10-28 11:46:49 -04002567 }
2568
Alexander Duyck184c4492016-10-28 11:50:13 -04002569 return active;
Alexander Duyck6234f872016-10-28 11:46:49 -04002570}
2571
Sabrina Dubroca867d0ad2018-11-29 14:14:49 +01002572static void reset_xps_maps(struct net_device *dev,
2573 struct xps_dev_maps *dev_maps,
Antoine Tenart044ab862021-03-18 19:37:46 +01002574 enum xps_map_type type)
Sabrina Dubroca867d0ad2018-11-29 14:14:49 +01002575{
Sabrina Dubroca867d0ad2018-11-29 14:14:49 +01002576 static_key_slow_dec_cpuslocked(&xps_needed);
Antoine Tenart044ab862021-03-18 19:37:46 +01002577 if (type == XPS_RXQS)
2578 static_key_slow_dec_cpuslocked(&xps_rxqs_needed);
2579
2580 RCU_INIT_POINTER(dev->xps_maps[type], NULL);
2581
Sabrina Dubroca867d0ad2018-11-29 14:14:49 +01002582 kfree_rcu(dev_maps, rcu);
2583}
2584
Antoine Tenart044ab862021-03-18 19:37:46 +01002585static void clean_xps_maps(struct net_device *dev, enum xps_map_type type,
2586 u16 offset, u16 count)
Amritha Nambiar80d19662018-06-29 21:26:41 -07002587{
Antoine Tenart044ab862021-03-18 19:37:46 +01002588 struct xps_dev_maps *dev_maps;
Amritha Nambiar80d19662018-06-29 21:26:41 -07002589 bool active = false;
2590 int i, j;
2591
Antoine Tenart044ab862021-03-18 19:37:46 +01002592 dev_maps = xmap_dereference(dev->xps_maps[type]);
2593 if (!dev_maps)
2594 return;
2595
Antoine Tenart6f361582021-03-18 19:37:45 +01002596 for (j = 0; j < dev_maps->nr_ids; j++)
2597 active |= remove_xps_queue_cpu(dev, dev_maps, j, offset, count);
Sabrina Dubroca867d0ad2018-11-29 14:14:49 +01002598 if (!active)
Antoine Tenart044ab862021-03-18 19:37:46 +01002599 reset_xps_maps(dev, dev_maps, type);
Amritha Nambiar80d19662018-06-29 21:26:41 -07002600
Antoine Tenart044ab862021-03-18 19:37:46 +01002601 if (type == XPS_CPUS) {
Antoine Tenart6f361582021-03-18 19:37:45 +01002602 for (i = offset + (count - 1); count--; i--)
Sabrina Dubrocaf28c0202018-11-29 14:14:48 +01002603 netdev_queue_numa_node_write(
Antoine Tenart6f361582021-03-18 19:37:45 +01002604 netdev_get_tx_queue(dev, i), NUMA_NO_NODE);
Amritha Nambiar80d19662018-06-29 21:26:41 -07002605 }
2606}
2607
Alexander Duyck6234f872016-10-28 11:46:49 -04002608static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2609 u16 count)
Alexander Duyck537c00d2013-01-10 08:57:02 +00002610{
Amritha Nambiar04157462018-06-29 21:26:46 -07002611 if (!static_key_false(&xps_needed))
2612 return;
2613
Andrei Vagin4d99f662018-08-08 20:07:35 -07002614 cpus_read_lock();
Alexander Duyck537c00d2013-01-10 08:57:02 +00002615 mutex_lock(&xps_map_mutex);
Alexander Duyck537c00d2013-01-10 08:57:02 +00002616
Antoine Tenart044ab862021-03-18 19:37:46 +01002617 if (static_key_false(&xps_rxqs_needed))
2618 clean_xps_maps(dev, XPS_RXQS, offset, count);
Amritha Nambiar80d19662018-06-29 21:26:41 -07002619
Antoine Tenart044ab862021-03-18 19:37:46 +01002620 clean_xps_maps(dev, XPS_CPUS, offset, count);
Alexander Duyck537c00d2013-01-10 08:57:02 +00002621
Alexander Duyck537c00d2013-01-10 08:57:02 +00002622 mutex_unlock(&xps_map_mutex);
Andrei Vagin4d99f662018-08-08 20:07:35 -07002623 cpus_read_unlock();
Alexander Duyck537c00d2013-01-10 08:57:02 +00002624}
2625
Alexander Duyck6234f872016-10-28 11:46:49 -04002626static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2627{
2628 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2629}
2630
Amritha Nambiar80d19662018-06-29 21:26:41 -07002631static struct xps_map *expand_xps_map(struct xps_map *map, int attr_index,
2632 u16 index, bool is_rxqs_map)
Alexander Duyck01c5f862013-01-10 08:57:35 +00002633{
2634 struct xps_map *new_map;
2635 int alloc_len = XPS_MIN_MAP_ALLOC;
2636 int i, pos;
2637
2638 for (pos = 0; map && pos < map->len; pos++) {
2639 if (map->queues[pos] != index)
2640 continue;
2641 return map;
2642 }
2643
Amritha Nambiar80d19662018-06-29 21:26:41 -07002644 /* Need to add tx-queue to this CPU's/rx-queue's existing map */
Alexander Duyck01c5f862013-01-10 08:57:35 +00002645 if (map) {
2646 if (pos < map->alloc_len)
2647 return map;
2648
2649 alloc_len = map->alloc_len * 2;
2650 }
2651
Amritha Nambiar80d19662018-06-29 21:26:41 -07002652 /* Need to allocate new map to store tx-queue on this CPU's/rx-queue's
2653 * map
2654 */
2655 if (is_rxqs_map)
2656 new_map = kzalloc(XPS_MAP_SIZE(alloc_len), GFP_KERNEL);
2657 else
2658 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2659 cpu_to_node(attr_index));
Alexander Duyck01c5f862013-01-10 08:57:35 +00002660 if (!new_map)
2661 return NULL;
2662
2663 for (i = 0; i < pos; i++)
2664 new_map->queues[i] = map->queues[i];
2665 new_map->alloc_len = alloc_len;
2666 new_map->len = pos;
2667
2668 return new_map;
2669}
2670
Antoine Tenart402fbb92021-03-18 19:37:47 +01002671/* Copy xps maps at a given index */
2672static void xps_copy_dev_maps(struct xps_dev_maps *dev_maps,
2673 struct xps_dev_maps *new_dev_maps, int index,
2674 int tc, bool skip_tc)
2675{
2676 int i, tci = index * dev_maps->num_tc;
2677 struct xps_map *map;
2678
2679 /* copy maps belonging to foreign traffic classes */
2680 for (i = 0; i < dev_maps->num_tc; i++, tci++) {
2681 if (i == tc && skip_tc)
2682 continue;
2683
2684 /* fill in the new device map from the old device map */
2685 map = xmap_dereference(dev_maps->attr_map[tci]);
2686 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2687 }
2688}
2689
Andrei Vagin4d99f662018-08-08 20:07:35 -07002690/* Must be called under cpus_read_lock */
Amritha Nambiar80d19662018-06-29 21:26:41 -07002691int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
Antoine Tenart044ab862021-03-18 19:37:46 +01002692 u16 index, enum xps_map_type type)
Alexander Duyck537c00d2013-01-10 08:57:02 +00002693{
Antoine Tenart2d05bf02021-03-18 19:37:51 +01002694 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL, *old_dev_maps = NULL;
Antoine Tenart6f361582021-03-18 19:37:45 +01002695 const unsigned long *online_mask = NULL;
Antoine Tenart255c04a2021-03-18 19:37:43 +01002696 bool active = false, copy = false;
Amritha Nambiar80d19662018-06-29 21:26:41 -07002697 int i, j, tci, numa_node_id = -2;
Alexander Duyck184c4492016-10-28 11:50:13 -04002698 int maps_sz, num_tc = 1, tc = 0;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002699 struct xps_map *map, *new_map;
Amritha Nambiar80d19662018-06-29 21:26:41 -07002700 unsigned int nr_ids;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002701
Alexander Duyck184c4492016-10-28 11:50:13 -04002702 if (dev->num_tc) {
Alexander Duyckffcfe252018-07-09 12:19:38 -04002703 /* Do not allow XPS on subordinate device directly */
Alexander Duyck184c4492016-10-28 11:50:13 -04002704 num_tc = dev->num_tc;
Alexander Duyckffcfe252018-07-09 12:19:38 -04002705 if (num_tc < 0)
2706 return -EINVAL;
2707
2708 /* If queue belongs to subordinate dev use its map */
2709 dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;
2710
Alexander Duyck184c4492016-10-28 11:50:13 -04002711 tc = netdev_txq_to_tc(dev, index);
2712 if (tc < 0)
2713 return -EINVAL;
2714 }
2715
Amritha Nambiar80d19662018-06-29 21:26:41 -07002716 mutex_lock(&xps_map_mutex);
Antoine Tenart044ab862021-03-18 19:37:46 +01002717
2718 dev_maps = xmap_dereference(dev->xps_maps[type]);
2719 if (type == XPS_RXQS) {
Amritha Nambiar80d19662018-06-29 21:26:41 -07002720 maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues);
Amritha Nambiar80d19662018-06-29 21:26:41 -07002721 nr_ids = dev->num_rx_queues;
2722 } else {
2723 maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc);
Antoine Tenart6f361582021-03-18 19:37:45 +01002724 if (num_possible_cpus() > 1)
Amritha Nambiar80d19662018-06-29 21:26:41 -07002725 online_mask = cpumask_bits(cpu_online_mask);
Amritha Nambiar80d19662018-06-29 21:26:41 -07002726 nr_ids = nr_cpu_ids;
2727 }
2728
Alexander Duyck184c4492016-10-28 11:50:13 -04002729 if (maps_sz < L1_CACHE_BYTES)
2730 maps_sz = L1_CACHE_BYTES;
2731
Antoine Tenart255c04a2021-03-18 19:37:43 +01002732 /* The old dev_maps could be larger or smaller than the one we're
Antoine Tenart5478fcd2021-03-18 19:37:44 +01002733 * setting up now, as dev->num_tc or nr_ids could have been updated in
2734 * between. We could try to be smart, but let's be safe instead and only
2735 * copy foreign traffic classes if the two map sizes match.
Antoine Tenart255c04a2021-03-18 19:37:43 +01002736 */
Antoine Tenart5478fcd2021-03-18 19:37:44 +01002737 if (dev_maps &&
2738 dev_maps->num_tc == num_tc && dev_maps->nr_ids == nr_ids)
Antoine Tenart255c04a2021-03-18 19:37:43 +01002739 copy = true;
2740
Alexander Duyck01c5f862013-01-10 08:57:35 +00002741 /* allocate memory for queue storage */
Amritha Nambiar80d19662018-06-29 21:26:41 -07002742 for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids),
2743 j < nr_ids;) {
Alexander Duyck2bb60cb2013-02-22 06:38:44 +00002744 if (!new_dev_maps) {
Antoine Tenart255c04a2021-03-18 19:37:43 +01002745 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2746 if (!new_dev_maps) {
2747 mutex_unlock(&xps_map_mutex);
2748 return -ENOMEM;
2749 }
2750
Antoine Tenart5478fcd2021-03-18 19:37:44 +01002751 new_dev_maps->nr_ids = nr_ids;
Antoine Tenart255c04a2021-03-18 19:37:43 +01002752 new_dev_maps->num_tc = num_tc;
Alexander Duyck2bb60cb2013-02-22 06:38:44 +00002753 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002754
Amritha Nambiar80d19662018-06-29 21:26:41 -07002755 tci = j * num_tc + tc;
Antoine Tenart255c04a2021-03-18 19:37:43 +01002756 map = copy ? xmap_dereference(dev_maps->attr_map[tci]) : NULL;
Alexander Duyck01c5f862013-01-10 08:57:35 +00002757
Antoine Tenart044ab862021-03-18 19:37:46 +01002758 map = expand_xps_map(map, j, index, type == XPS_RXQS);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002759 if (!map)
2760 goto error;
2761
Amritha Nambiar80d19662018-06-29 21:26:41 -07002762 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002763 }
2764
2765 if (!new_dev_maps)
2766 goto out_no_new_maps;
2767
Sabrina Dubroca867d0ad2018-11-29 14:14:49 +01002768 if (!dev_maps) {
2769 /* Increment static keys at most once per type */
2770 static_key_slow_inc_cpuslocked(&xps_needed);
Antoine Tenart044ab862021-03-18 19:37:46 +01002771 if (type == XPS_RXQS)
Sabrina Dubroca867d0ad2018-11-29 14:14:49 +01002772 static_key_slow_inc_cpuslocked(&xps_rxqs_needed);
2773 }
Amritha Nambiar04157462018-06-29 21:26:46 -07002774
Antoine Tenart6f361582021-03-18 19:37:45 +01002775 for (j = 0; j < nr_ids; j++) {
Antoine Tenart402fbb92021-03-18 19:37:47 +01002776 bool skip_tc = false;
Alexander Duyck184c4492016-10-28 11:50:13 -04002777
Amritha Nambiar80d19662018-06-29 21:26:41 -07002778 tci = j * num_tc + tc;
Amritha Nambiar80d19662018-06-29 21:26:41 -07002779 if (netif_attr_test_mask(j, mask, nr_ids) &&
2780 netif_attr_test_online(j, online_mask, nr_ids)) {
2781 /* add tx-queue to CPU/rx-queue maps */
Alexander Duyck01c5f862013-01-10 08:57:35 +00002782 int pos = 0;
2783
Antoine Tenart402fbb92021-03-18 19:37:47 +01002784 skip_tc = true;
2785
Amritha Nambiar80d19662018-06-29 21:26:41 -07002786 map = xmap_dereference(new_dev_maps->attr_map[tci]);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002787 while ((pos < map->len) && (map->queues[pos] != index))
2788 pos++;
2789
2790 if (pos == map->len)
2791 map->queues[map->len++] = index;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002792#ifdef CONFIG_NUMA
Antoine Tenart044ab862021-03-18 19:37:46 +01002793 if (type == XPS_CPUS) {
Amritha Nambiar80d19662018-06-29 21:26:41 -07002794 if (numa_node_id == -2)
2795 numa_node_id = cpu_to_node(j);
2796 else if (numa_node_id != cpu_to_node(j))
2797 numa_node_id = -1;
2798 }
Alexander Duyck537c00d2013-01-10 08:57:02 +00002799#endif
Alexander Duyck537c00d2013-01-10 08:57:02 +00002800 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002801
Antoine Tenart402fbb92021-03-18 19:37:47 +01002802 if (copy)
2803 xps_copy_dev_maps(dev_maps, new_dev_maps, j, tc,
2804 skip_tc);
Alexander Duyck537c00d2013-01-10 08:57:02 +00002805 }
2806
Antoine Tenart044ab862021-03-18 19:37:46 +01002807 rcu_assign_pointer(dev->xps_maps[type], new_dev_maps);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002808
Alexander Duyck537c00d2013-01-10 08:57:02 +00002809 /* Cleanup old maps */
Alexander Duyck184c4492016-10-28 11:50:13 -04002810 if (!dev_maps)
2811 goto out_no_old_maps;
2812
Antoine Tenart6f361582021-03-18 19:37:45 +01002813 for (j = 0; j < dev_maps->nr_ids; j++) {
Antoine Tenart255c04a2021-03-18 19:37:43 +01002814 for (i = num_tc, tci = j * dev_maps->num_tc; i--; tci++) {
Amritha Nambiar80d19662018-06-29 21:26:41 -07002815 map = xmap_dereference(dev_maps->attr_map[tci]);
Antoine Tenart255c04a2021-03-18 19:37:43 +01002816 if (!map)
2817 continue;
2818
2819 if (copy) {
2820 new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2821 if (map == new_map)
2822 continue;
2823 }
2824
Antoine Tenart75b27582021-03-18 19:37:52 +01002825 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
Antoine Tenart255c04a2021-03-18 19:37:43 +01002826 kfree_rcu(map, rcu);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002827 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002828 }
Alexander Duyck537c00d2013-01-10 08:57:02 +00002829
Antoine Tenart2d05bf02021-03-18 19:37:51 +01002830 old_dev_maps = dev_maps;
Alexander Duyck184c4492016-10-28 11:50:13 -04002831
2832out_no_old_maps:
Alexander Duyck01c5f862013-01-10 08:57:35 +00002833 dev_maps = new_dev_maps;
2834 active = true;
2835
2836out_no_new_maps:
Antoine Tenart044ab862021-03-18 19:37:46 +01002837 if (type == XPS_CPUS)
Amritha Nambiar80d19662018-06-29 21:26:41 -07002838 /* update Tx queue numa node */
2839 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2840 (numa_node_id >= 0) ?
2841 numa_node_id : NUMA_NO_NODE);
Alexander Duyck537c00d2013-01-10 08:57:02 +00002842
Alexander Duyck01c5f862013-01-10 08:57:35 +00002843 if (!dev_maps)
2844 goto out_no_maps;
2845
Amritha Nambiar80d19662018-06-29 21:26:41 -07002846 /* removes tx-queue from unused CPUs/rx-queues */
Antoine Tenart6f361582021-03-18 19:37:45 +01002847 for (j = 0; j < dev_maps->nr_ids; j++) {
Antoine Tenart132f7432021-03-18 19:37:48 +01002848 tci = j * dev_maps->num_tc;
2849
2850 for (i = 0; i < dev_maps->num_tc; i++, tci++) {
2851 if (i == tc &&
2852 netif_attr_test_mask(j, mask, dev_maps->nr_ids) &&
2853 netif_attr_test_online(j, online_mask, dev_maps->nr_ids))
2854 continue;
2855
Antoine Tenart2d05bf02021-03-18 19:37:51 +01002856 active |= remove_xps_queue(dev_maps,
2857 copy ? old_dev_maps : NULL,
2858 tci, index);
Antoine Tenart132f7432021-03-18 19:37:48 +01002859 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002860 }
2861
Antoine Tenart2d05bf02021-03-18 19:37:51 +01002862 if (old_dev_maps)
2863 kfree_rcu(old_dev_maps, rcu);
2864
Alexander Duyck01c5f862013-01-10 08:57:35 +00002865 /* free map if not active */
Sabrina Dubroca867d0ad2018-11-29 14:14:49 +01002866 if (!active)
Antoine Tenart044ab862021-03-18 19:37:46 +01002867 reset_xps_maps(dev, dev_maps, type);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002868
2869out_no_maps:
Alexander Duyck537c00d2013-01-10 08:57:02 +00002870 mutex_unlock(&xps_map_mutex);
2871
2872 return 0;
2873error:
Alexander Duyck01c5f862013-01-10 08:57:35 +00002874 /* remove any maps that we added */
Antoine Tenart6f361582021-03-18 19:37:45 +01002875 for (j = 0; j < nr_ids; j++) {
Amritha Nambiar80d19662018-06-29 21:26:41 -07002876 for (i = num_tc, tci = j * num_tc; i--; tci++) {
2877 new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
Antoine Tenart255c04a2021-03-18 19:37:43 +01002878 map = copy ?
Amritha Nambiar80d19662018-06-29 21:26:41 -07002879 xmap_dereference(dev_maps->attr_map[tci]) :
Alexander Duyck184c4492016-10-28 11:50:13 -04002880 NULL;
2881 if (new_map && new_map != map)
2882 kfree(new_map);
2883 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002884 }
2885
Alexander Duyck537c00d2013-01-10 08:57:02 +00002886 mutex_unlock(&xps_map_mutex);
2887
Alexander Duyck537c00d2013-01-10 08:57:02 +00002888 kfree(new_dev_maps);
2889 return -ENOMEM;
2890}
Andrei Vagin4d99f662018-08-08 20:07:35 -07002891EXPORT_SYMBOL_GPL(__netif_set_xps_queue);
Amritha Nambiar80d19662018-06-29 21:26:41 -07002892
2893int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2894 u16 index)
2895{
Andrei Vagin4d99f662018-08-08 20:07:35 -07002896 int ret;
2897
2898 cpus_read_lock();
Antoine Tenart044ab862021-03-18 19:37:46 +01002899 ret = __netif_set_xps_queue(dev, cpumask_bits(mask), index, XPS_CPUS);
Andrei Vagin4d99f662018-08-08 20:07:35 -07002900 cpus_read_unlock();
2901
2902 return ret;
Amritha Nambiar80d19662018-06-29 21:26:41 -07002903}
Alexander Duyck537c00d2013-01-10 08:57:02 +00002904EXPORT_SYMBOL(netif_set_xps_queue);
2905
2906#endif
Alexander Duyckffcfe252018-07-09 12:19:38 -04002907static void netdev_unbind_all_sb_channels(struct net_device *dev)
2908{
2909 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2910
2911 /* Unbind any subordinate channels */
2912 while (txq-- != &dev->_tx[0]) {
2913 if (txq->sb_dev)
2914 netdev_unbind_sb_channel(dev, txq->sb_dev);
2915 }
2916}
2917
Alexander Duyck9cf1f6a2016-10-28 11:43:20 -04002918void netdev_reset_tc(struct net_device *dev)
2919{
Alexander Duyck6234f872016-10-28 11:46:49 -04002920#ifdef CONFIG_XPS
2921 netif_reset_xps_queues_gt(dev, 0);
2922#endif
Alexander Duyckffcfe252018-07-09 12:19:38 -04002923 netdev_unbind_all_sb_channels(dev);
2924
2925 /* Reset TC configuration of device */
Alexander Duyck9cf1f6a2016-10-28 11:43:20 -04002926 dev->num_tc = 0;
2927 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2928 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2929}
2930EXPORT_SYMBOL(netdev_reset_tc);
2931
2932int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2933{
2934 if (tc >= dev->num_tc)
2935 return -EINVAL;
2936
Alexander Duyck6234f872016-10-28 11:46:49 -04002937#ifdef CONFIG_XPS
2938 netif_reset_xps_queues(dev, offset, count);
2939#endif
Alexander Duyck9cf1f6a2016-10-28 11:43:20 -04002940 dev->tc_to_txq[tc].count = count;
2941 dev->tc_to_txq[tc].offset = offset;
2942 return 0;
2943}
2944EXPORT_SYMBOL(netdev_set_tc_queue);
2945
2946int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2947{
2948 if (num_tc > TC_MAX_QUEUE)
2949 return -EINVAL;
2950
Alexander Duyck6234f872016-10-28 11:46:49 -04002951#ifdef CONFIG_XPS
2952 netif_reset_xps_queues_gt(dev, 0);
2953#endif
Alexander Duyckffcfe252018-07-09 12:19:38 -04002954 netdev_unbind_all_sb_channels(dev);
2955
Alexander Duyck9cf1f6a2016-10-28 11:43:20 -04002956 dev->num_tc = num_tc;
2957 return 0;
2958}
2959EXPORT_SYMBOL(netdev_set_num_tc);
2960
Alexander Duyckffcfe252018-07-09 12:19:38 -04002961void netdev_unbind_sb_channel(struct net_device *dev,
2962 struct net_device *sb_dev)
2963{
2964 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2965
2966#ifdef CONFIG_XPS
2967 netif_reset_xps_queues_gt(sb_dev, 0);
2968#endif
2969 memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq));
2970 memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map));
2971
2972 while (txq-- != &dev->_tx[0]) {
2973 if (txq->sb_dev == sb_dev)
2974 txq->sb_dev = NULL;
2975 }
2976}
2977EXPORT_SYMBOL(netdev_unbind_sb_channel);
2978
2979int netdev_bind_sb_channel_queue(struct net_device *dev,
2980 struct net_device *sb_dev,
2981 u8 tc, u16 count, u16 offset)
2982{
2983 /* Make certain the sb_dev and dev are already configured */
2984 if (sb_dev->num_tc >= 0 || tc >= dev->num_tc)
2985 return -EINVAL;
2986
2987 /* We cannot hand out queues we don't have */
2988 if ((offset + count) > dev->real_num_tx_queues)
2989 return -EINVAL;
2990
2991 /* Record the mapping */
2992 sb_dev->tc_to_txq[tc].count = count;
2993 sb_dev->tc_to_txq[tc].offset = offset;
2994
2995 /* Provide a way for Tx queue to find the tc_to_txq map or
2996 * XPS map for itself.
2997 */
2998 while (count--)
2999 netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev;
3000
3001 return 0;
3002}
3003EXPORT_SYMBOL(netdev_bind_sb_channel_queue);
3004
3005int netdev_set_sb_channel(struct net_device *dev, u16 channel)
3006{
3007 /* Do not use a multiqueue device to represent a subordinate channel */
3008 if (netif_is_multiqueue(dev))
3009 return -ENODEV;
3010
3011 /* We allow channels 1 - 32767 to be used for subordinate channels.
3012 * Channel 0 is meant to be "native" mode and used only to represent
3013 * the main root device. We allow writing 0 to reset the device back
3014 * to normal mode after being used as a subordinate channel.
3015 */
3016 if (channel > S16_MAX)
3017 return -EINVAL;
3018
3019 dev->num_tc = -channel;
3020
3021 return 0;
3022}
3023EXPORT_SYMBOL(netdev_set_sb_channel);
3024
John Fastabendf0796d52010-07-01 13:21:57 +00003025/*
3026 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
Gal Pressman3a053b12018-02-28 15:59:15 +02003027 * greater than real_num_tx_queues stale skbs on the qdisc must be flushed.
John Fastabendf0796d52010-07-01 13:21:57 +00003028 */
Tom Herberte6484932010-10-18 18:04:39 +00003029int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
John Fastabendf0796d52010-07-01 13:21:57 +00003030{
Jakub Kicinskiac5b7012018-02-12 21:35:31 -08003031 bool disabling;
Tom Herbert1d24eb42010-11-21 13:17:27 +00003032 int rc;
3033
Jakub Kicinskiac5b7012018-02-12 21:35:31 -08003034 disabling = txq < dev->real_num_tx_queues;
3035
Tom Herberte6484932010-10-18 18:04:39 +00003036 if (txq < 1 || txq > dev->num_tx_queues)
3037 return -EINVAL;
John Fastabendf0796d52010-07-01 13:21:57 +00003038
Ben Hutchings5c565802011-02-15 19:39:21 +00003039 if (dev->reg_state == NETREG_REGISTERED ||
3040 dev->reg_state == NETREG_UNREGISTERING) {
Tom Herberte6484932010-10-18 18:04:39 +00003041 ASSERT_RTNL();
3042
Tom Herbert1d24eb42010-11-21 13:17:27 +00003043 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
3044 txq);
Tom Herbertbf264142010-11-26 08:36:09 +00003045 if (rc)
3046 return rc;
3047
John Fastabend4f57c082011-01-17 08:06:04 +00003048 if (dev->num_tc)
3049 netif_setup_tc(dev, txq);
3050
Jakub Kicinskiac5b7012018-02-12 21:35:31 -08003051 dev->real_num_tx_queues = txq;
3052
3053 if (disabling) {
3054 synchronize_net();
Tom Herberte6484932010-10-18 18:04:39 +00003055 qdisc_reset_all_tx_gt(dev, txq);
Alexander Duyck024e9672013-01-10 08:57:46 +00003056#ifdef CONFIG_XPS
3057 netif_reset_xps_queues_gt(dev, txq);
3058#endif
3059 }
Jakub Kicinskiac5b7012018-02-12 21:35:31 -08003060 } else {
3061 dev->real_num_tx_queues = txq;
John Fastabendf0796d52010-07-01 13:21:57 +00003062 }
Tom Herberte6484932010-10-18 18:04:39 +00003063
Tom Herberte6484932010-10-18 18:04:39 +00003064 return 0;
John Fastabendf0796d52010-07-01 13:21:57 +00003065}
3066EXPORT_SYMBOL(netif_set_real_num_tx_queues);
Denis Vlasenko56079432006-03-29 15:57:29 -08003067
Michael Daltona953be52014-01-16 22:23:28 -08003068#ifdef CONFIG_SYSFS
Ben Hutchings62fe0b42010-09-27 08:24:33 +00003069/**
3070 * netif_set_real_num_rx_queues - set actual number of RX queues used
3071 * @dev: Network device
3072 * @rxq: Actual number of RX queues
3073 *
3074 * This must be called either with the rtnl_lock held or before
3075 * registration of the net device. Returns 0 on success, or a
Ben Hutchings4e7f7952010-10-08 10:33:39 -07003076 * negative error code. If called before registration, it always
3077 * succeeds.
Ben Hutchings62fe0b42010-09-27 08:24:33 +00003078 */
3079int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
3080{
3081 int rc;
3082
Tom Herbertbd25fa72010-10-18 18:00:16 +00003083 if (rxq < 1 || rxq > dev->num_rx_queues)
3084 return -EINVAL;
3085
Ben Hutchings62fe0b42010-09-27 08:24:33 +00003086 if (dev->reg_state == NETREG_REGISTERED) {
3087 ASSERT_RTNL();
3088
Ben Hutchings62fe0b42010-09-27 08:24:33 +00003089 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
3090 rxq);
3091 if (rc)
3092 return rc;
Ben Hutchings62fe0b42010-09-27 08:24:33 +00003093 }
3094
3095 dev->real_num_rx_queues = rxq;
3096 return 0;
3097}
3098EXPORT_SYMBOL(netif_set_real_num_rx_queues);
3099#endif
3100
Ben Hutchings2c530402012-07-10 10:55:09 +00003101/**
3102 * netif_get_num_default_rss_queues - default number of RSS queues
Yuval Mintz16917b82012-07-01 03:18:50 +00003103 *
3104 * This routine should set an upper limit on the number of RSS queues
3105 * used by default by multiqueue devices.
3106 */
Ben Hutchingsa55b1382012-07-10 10:54:38 +00003107int netif_get_num_default_rss_queues(void)
Yuval Mintz16917b82012-07-01 03:18:50 +00003108{
Hariprasad Shenai40e4e712016-06-08 18:09:08 +05303109 return is_kdump_kernel() ?
3110 1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
Yuval Mintz16917b82012-07-01 03:18:50 +00003111}
3112EXPORT_SYMBOL(netif_get_num_default_rss_queues);
3113
Eric Dumazet3bcb8462016-06-04 20:02:28 -07003114static void __netif_reschedule(struct Qdisc *q)
Jarek Poplawskidef82a12008-08-17 21:54:43 -07003115{
3116 struct softnet_data *sd;
3117 unsigned long flags;
3118
3119 local_irq_save(flags);
Christoph Lameter903ceff2014-08-17 12:30:35 -05003120 sd = this_cpu_ptr(&softnet_data);
Changli Gaoa9cbd582010-04-26 23:06:24 +00003121 q->next_sched = NULL;
3122 *sd->output_queue_tailp = q;
3123 sd->output_queue_tailp = &q->next_sched;
Jarek Poplawskidef82a12008-08-17 21:54:43 -07003124 raise_softirq_irqoff(NET_TX_SOFTIRQ);
3125 local_irq_restore(flags);
3126}
3127
David S. Miller37437bb2008-07-16 02:15:04 -07003128void __netif_schedule(struct Qdisc *q)
Denis Vlasenko56079432006-03-29 15:57:29 -08003129{
Jarek Poplawskidef82a12008-08-17 21:54:43 -07003130 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
3131 __netif_reschedule(q);
Denis Vlasenko56079432006-03-29 15:57:29 -08003132}
3133EXPORT_SYMBOL(__netif_schedule);
3134
Eric Dumazete6247022013-12-05 04:45:08 -08003135struct dev_kfree_skb_cb {
3136 enum skb_free_reason reason;
3137};
3138
3139static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
Denis Vlasenko56079432006-03-29 15:57:29 -08003140{
Eric Dumazete6247022013-12-05 04:45:08 -08003141 return (struct dev_kfree_skb_cb *)skb->cb;
Denis Vlasenko56079432006-03-29 15:57:29 -08003142}
Denis Vlasenko56079432006-03-29 15:57:29 -08003143
John Fastabend46e5da40a2014-09-12 20:04:52 -07003144void netif_schedule_queue(struct netdev_queue *txq)
3145{
3146 rcu_read_lock();
Julio Faracco5be55152019-10-01 11:39:04 -03003147 if (!netif_xmit_stopped(txq)) {
John Fastabend46e5da40a2014-09-12 20:04:52 -07003148 struct Qdisc *q = rcu_dereference(txq->qdisc);
3149
3150 __netif_schedule(q);
3151 }
3152 rcu_read_unlock();
3153}
3154EXPORT_SYMBOL(netif_schedule_queue);
3155
John Fastabend46e5da40a2014-09-12 20:04:52 -07003156void netif_tx_wake_queue(struct netdev_queue *dev_queue)
3157{
3158 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
3159 struct Qdisc *q;
3160
3161 rcu_read_lock();
3162 q = rcu_dereference(dev_queue->qdisc);
3163 __netif_schedule(q);
3164 rcu_read_unlock();
3165 }
3166}
3167EXPORT_SYMBOL(netif_tx_wake_queue);
3168
Eric Dumazete6247022013-12-05 04:45:08 -08003169void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
3170{
3171 unsigned long flags;
3172
Myungho Jung98998862017-04-25 11:58:15 -07003173 if (unlikely(!skb))
3174 return;
3175
Reshetova, Elena63354792017-06-30 13:07:58 +03003176 if (likely(refcount_read(&skb->users) == 1)) {
Eric Dumazete6247022013-12-05 04:45:08 -08003177 smp_rmb();
Reshetova, Elena63354792017-06-30 13:07:58 +03003178 refcount_set(&skb->users, 0);
3179 } else if (likely(!refcount_dec_and_test(&skb->users))) {
Eric Dumazete6247022013-12-05 04:45:08 -08003180 return;
3181 }
3182 get_kfree_skb_cb(skb)->reason = reason;
3183 local_irq_save(flags);
3184 skb->next = __this_cpu_read(softnet_data.completion_queue);
3185 __this_cpu_write(softnet_data.completion_queue, skb);
3186 raise_softirq_irqoff(NET_TX_SOFTIRQ);
3187 local_irq_restore(flags);
3188}
3189EXPORT_SYMBOL(__dev_kfree_skb_irq);
3190
3191void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
Denis Vlasenko56079432006-03-29 15:57:29 -08003192{
3193 if (in_irq() || irqs_disabled())
Eric Dumazete6247022013-12-05 04:45:08 -08003194 __dev_kfree_skb_irq(skb, reason);
Denis Vlasenko56079432006-03-29 15:57:29 -08003195 else
3196 dev_kfree_skb(skb);
3197}
Eric Dumazete6247022013-12-05 04:45:08 -08003198EXPORT_SYMBOL(__dev_kfree_skb_any);
Denis Vlasenko56079432006-03-29 15:57:29 -08003199
3200
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003201/**
3202 * netif_device_detach - mark device as removed
3203 * @dev: network device
3204 *
3205 * Mark device as removed from system and therefore no longer available.
3206 */
Denis Vlasenko56079432006-03-29 15:57:29 -08003207void netif_device_detach(struct net_device *dev)
3208{
3209 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
3210 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00003211 netif_tx_stop_all_queues(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08003212 }
3213}
3214EXPORT_SYMBOL(netif_device_detach);
3215
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003216/**
3217 * netif_device_attach - mark device as attached
3218 * @dev: network device
3219 *
3220 * Mark device as attached from system and restart if needed.
3221 */
Denis Vlasenko56079432006-03-29 15:57:29 -08003222void netif_device_attach(struct net_device *dev)
3223{
3224 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
3225 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00003226 netif_tx_wake_all_queues(dev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003227 __netdev_watchdog_up(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08003228 }
3229}
3230EXPORT_SYMBOL(netif_device_attach);
3231
Jiri Pirko5605c762015-05-12 14:56:12 +02003232/*
3233 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
3234 * to be used as a distribution range.
3235 */
Alexander Duyckeadec8772018-07-09 12:19:48 -04003236static u16 skb_tx_hash(const struct net_device *dev,
3237 const struct net_device *sb_dev,
3238 struct sk_buff *skb)
Jiri Pirko5605c762015-05-12 14:56:12 +02003239{
3240 u32 hash;
3241 u16 qoffset = 0;
Alexander Duyck1b837d42018-04-27 14:06:53 -04003242 u16 qcount = dev->real_num_tx_queues;
Jiri Pirko5605c762015-05-12 14:56:12 +02003243
Alexander Duyckeadec8772018-07-09 12:19:48 -04003244 if (dev->num_tc) {
3245 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
3246
3247 qoffset = sb_dev->tc_to_txq[tc].offset;
3248 qcount = sb_dev->tc_to_txq[tc].count;
3249 }
3250
Jiri Pirko5605c762015-05-12 14:56:12 +02003251 if (skb_rx_queue_recorded(skb)) {
3252 hash = skb_get_rx_queue(skb);
Amritha Nambiar6e11d152020-02-24 10:56:00 -08003253 if (hash >= qoffset)
3254 hash -= qoffset;
Alexander Duyck1b837d42018-04-27 14:06:53 -04003255 while (unlikely(hash >= qcount))
3256 hash -= qcount;
Alexander Duyckeadec8772018-07-09 12:19:48 -04003257 return hash + qoffset;
Jiri Pirko5605c762015-05-12 14:56:12 +02003258 }
3259
3260 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
3261}
Jiri Pirko5605c762015-05-12 14:56:12 +02003262
Ben Hutchings36c92472012-01-17 07:57:56 +00003263static void skb_warn_bad_offload(const struct sk_buff *skb)
3264{
Wei Tang84d15ae2016-06-16 21:17:49 +08003265 static const netdev_features_t null_features;
Ben Hutchings36c92472012-01-17 07:57:56 +00003266 struct net_device *dev = skb->dev;
Bjørn Mork88ad4172015-11-16 19:16:40 +01003267 const char *name = "";
Ben Hutchings36c92472012-01-17 07:57:56 +00003268
Ben Greearc846ad92013-04-19 10:45:52 +00003269 if (!net_ratelimit())
3270 return;
3271
Bjørn Mork88ad4172015-11-16 19:16:40 +01003272 if (dev) {
3273 if (dev->dev.parent)
3274 name = dev_driver_string(dev->dev.parent);
3275 else
3276 name = netdev_name(dev);
3277 }
Willem de Bruijn64131392019-07-07 05:51:55 -04003278 skb_dump(KERN_WARNING, skb, false);
3279 WARN(1, "%s: caps=(%pNF, %pNF)\n",
Bjørn Mork88ad4172015-11-16 19:16:40 +01003280 name, dev ? &dev->features : &null_features,
Willem de Bruijn64131392019-07-07 05:51:55 -04003281 skb->sk ? &skb->sk->sk_route_caps : &null_features);
Ben Hutchings36c92472012-01-17 07:57:56 +00003282}
3283
Linus Torvalds1da177e2005-04-16 15:20:36 -07003284/*
3285 * Invalidate hardware checksum when packet is to be mangled, and
3286 * complete checksum manually on outgoing path.
3287 */
Patrick McHardy84fa7932006-08-29 16:44:56 -07003288int skb_checksum_help(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003289{
Al Virod3bc23e2006-11-14 21:24:49 -08003290 __wsum csum;
Herbert Xu663ead32007-04-09 11:59:07 -07003291 int ret = 0, offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003292
Patrick McHardy84fa7932006-08-29 16:44:56 -07003293 if (skb->ip_summed == CHECKSUM_COMPLETE)
Herbert Xua430a432006-07-08 13:34:56 -07003294 goto out_set_summed;
3295
Yi Li3aefd7d2020-10-27 13:59:04 +08003296 if (unlikely(skb_is_gso(skb))) {
Ben Hutchings36c92472012-01-17 07:57:56 +00003297 skb_warn_bad_offload(skb);
3298 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003299 }
3300
Eric Dumazetcef401d2013-01-25 20:34:37 +00003301 /* Before computing a checksum, we should make sure no frag could
3302 * be modified by an external entity : checksum could be wrong.
3303 */
3304 if (skb_has_shared_frag(skb)) {
3305 ret = __skb_linearize(skb);
3306 if (ret)
3307 goto out;
3308 }
3309
Michał Mirosław55508d62010-12-14 15:24:08 +00003310 offset = skb_checksum_start_offset(skb);
Herbert Xua0308472007-10-15 01:47:15 -07003311 BUG_ON(offset >= skb_headlen(skb));
3312 csum = skb_checksum(skb, offset, skb->len - offset, 0);
3313
3314 offset += skb->csum_offset;
3315 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
3316
Heiner Kallweit8211fbf2019-10-06 18:52:43 +02003317 ret = skb_ensure_writable(skb, offset + sizeof(__sum16));
3318 if (ret)
3319 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003320
Eric Dumazet4f2e4ad2016-10-29 11:02:36 -07003321 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
Herbert Xua430a432006-07-08 13:34:56 -07003322out_set_summed:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003323 skb->ip_summed = CHECKSUM_NONE;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003324out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003325 return ret;
3326}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003327EXPORT_SYMBOL(skb_checksum_help);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003328
Davide Carattib72b5bf2017-05-18 15:44:38 +02003329int skb_crc32c_csum_help(struct sk_buff *skb)
3330{
3331 __le32 crc32c_csum;
3332 int ret = 0, offset, start;
3333
3334 if (skb->ip_summed != CHECKSUM_PARTIAL)
3335 goto out;
3336
3337 if (unlikely(skb_is_gso(skb)))
3338 goto out;
3339
3340 /* Before computing a checksum, we should make sure no frag could
3341 * be modified by an external entity : checksum could be wrong.
3342 */
3343 if (unlikely(skb_has_shared_frag(skb))) {
3344 ret = __skb_linearize(skb);
3345 if (ret)
3346 goto out;
3347 }
3348 start = skb_checksum_start_offset(skb);
3349 offset = start + offsetof(struct sctphdr, checksum);
3350 if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
3351 ret = -EINVAL;
3352 goto out;
3353 }
Heiner Kallweit8211fbf2019-10-06 18:52:43 +02003354
3355 ret = skb_ensure_writable(skb, offset + sizeof(__le32));
3356 if (ret)
3357 goto out;
3358
Davide Carattib72b5bf2017-05-18 15:44:38 +02003359 crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start,
3360 skb->len - start, ~(__u32)0,
3361 crc32c_csum_stub));
3362 *(__le32 *)(skb->data + offset) = crc32c_csum;
3363 skb->ip_summed = CHECKSUM_NONE;
Davide Carattidba00302017-05-18 15:44:40 +02003364 skb->csum_not_inet = 0;
Davide Carattib72b5bf2017-05-18 15:44:38 +02003365out:
3366 return ret;
3367}
3368
Vlad Yasevich53d64712014-03-27 17:26:18 -04003369__be16 skb_network_protocol(struct sk_buff *skb, int *depth)
Pravin B Shelarec5f0612013-03-07 09:28:01 +00003370{
3371 __be16 type = skb->protocol;
3372
Pravin B Shelar19acc322013-05-07 20:41:07 +00003373 /* Tunnel gso handlers can set protocol to ethernet. */
3374 if (type == htons(ETH_P_TEB)) {
3375 struct ethhdr *eth;
3376
3377 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
3378 return 0;
3379
Eric Dumazet1dfe82e2018-03-26 08:08:07 -07003380 eth = (struct ethhdr *)skb->data;
Pravin B Shelar19acc322013-05-07 20:41:07 +00003381 type = eth->h_proto;
3382 }
3383
Toshiaki Makitad4bcef32015-01-29 20:37:07 +09003384 return __vlan_get_protocol(skb, type, depth);
Pravin B Shelarec5f0612013-03-07 09:28:01 +00003385}
3386
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00003387/**
3388 * skb_mac_gso_segment - mac layer segmentation handler.
3389 * @skb: buffer to segment
3390 * @features: features for the output path (see dev->features)
3391 */
3392struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
3393 netdev_features_t features)
3394{
3395 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
3396 struct packet_offload *ptype;
Vlad Yasevich53d64712014-03-27 17:26:18 -04003397 int vlan_depth = skb->mac_len;
3398 __be16 type = skb_network_protocol(skb, &vlan_depth);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00003399
Pravin B Shelarec5f0612013-03-07 09:28:01 +00003400 if (unlikely(!type))
3401 return ERR_PTR(-EINVAL);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00003402
Vlad Yasevich53d64712014-03-27 17:26:18 -04003403 __skb_pull(skb, vlan_depth);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00003404
3405 rcu_read_lock();
3406 list_for_each_entry_rcu(ptype, &offload_base, list) {
3407 if (ptype->type == type && ptype->callbacks.gso_segment) {
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00003408 segs = ptype->callbacks.gso_segment(skb, features);
3409 break;
3410 }
3411 }
3412 rcu_read_unlock();
3413
3414 __skb_push(skb, skb->data - skb_mac_header(skb));
3415
3416 return segs;
3417}
3418EXPORT_SYMBOL(skb_mac_gso_segment);
3419
3420
Cong Wang12b00042013-02-05 16:36:38 +00003421/* openvswitch calls this on rx path, so we need a different check.
3422 */
3423static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
3424{
3425 if (tx_path)
Willem de Bruijn0c19f8462017-11-21 10:22:25 -05003426 return skb->ip_summed != CHECKSUM_PARTIAL &&
3427 skb->ip_summed != CHECKSUM_UNNECESSARY;
Eric Dumazet6e7bc472017-02-03 14:29:42 -08003428
3429 return skb->ip_summed == CHECKSUM_NONE;
Cong Wang12b00042013-02-05 16:36:38 +00003430}
3431
Herbert Xuf6a78bf2006-06-22 02:57:17 -07003432/**
Cong Wang12b00042013-02-05 16:36:38 +00003433 * __skb_gso_segment - Perform segmentation on skb.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07003434 * @skb: buffer to segment
Herbert Xu576a30e2006-06-27 13:22:38 -07003435 * @features: features for the output path (see dev->features)
Cong Wang12b00042013-02-05 16:36:38 +00003436 * @tx_path: whether it is called in TX path
Herbert Xuf6a78bf2006-06-22 02:57:17 -07003437 *
3438 * This function segments the given skb and returns a list of segments.
Herbert Xu576a30e2006-06-27 13:22:38 -07003439 *
3440 * It may return NULL if the skb requires no segmentation. This is
3441 * only possible when GSO is used for verifying header integrity.
Konstantin Khlebnikov9207f9d2016-01-08 15:21:46 +03003442 *
Cambda Zhua08e7fd2020-03-26 15:33:14 +08003443 * Segmentation preserves SKB_GSO_CB_OFFSET bytes of previous skb cb.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07003444 */
Cong Wang12b00042013-02-05 16:36:38 +00003445struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
3446 netdev_features_t features, bool tx_path)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07003447{
Eric Dumazetb2504a52017-01-31 10:20:32 -08003448 struct sk_buff *segs;
3449
Cong Wang12b00042013-02-05 16:36:38 +00003450 if (unlikely(skb_needs_check(skb, tx_path))) {
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00003451 int err;
3452
Eric Dumazetb2504a52017-01-31 10:20:32 -08003453 /* We're going to init ->check field in TCP or UDP header */
françois romieua40e0a62014-07-15 23:55:35 +02003454 err = skb_cow_head(skb, 0);
3455 if (err < 0)
Herbert Xua430a432006-07-08 13:34:56 -07003456 return ERR_PTR(err);
3457 }
3458
Alexander Duyck802ab552016-04-10 21:45:03 -04003459 /* Only report GSO partial support if it will enable us to
3460 * support segmentation on this frame without needing additional
3461 * work.
3462 */
3463 if (features & NETIF_F_GSO_PARTIAL) {
3464 netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
3465 struct net_device *dev = skb->dev;
3466
3467 partial_features |= dev->features & dev->gso_partial_features;
3468 if (!skb_gso_ok(skb, features | partial_features))
3469 features &= ~NETIF_F_GSO_PARTIAL;
3470 }
3471
Cambda Zhua08e7fd2020-03-26 15:33:14 +08003472 BUILD_BUG_ON(SKB_GSO_CB_OFFSET +
Konstantin Khlebnikov9207f9d2016-01-08 15:21:46 +03003473 sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
3474
Pravin B Shelar68c33162013-02-14 14:02:41 +00003475 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
Eric Dumazet3347c962013-10-19 11:42:56 -07003476 SKB_GSO_CB(skb)->encap_level = 0;
3477
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00003478 skb_reset_mac_header(skb);
3479 skb_reset_mac_len(skb);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07003480
Eric Dumazetb2504a52017-01-31 10:20:32 -08003481 segs = skb_mac_gso_segment(skb, features);
3482
Steffen Klassert3a1296a2020-01-25 11:26:44 +01003483 if (segs != skb && unlikely(skb_needs_check(skb, tx_path) && !IS_ERR(segs)))
Eric Dumazetb2504a52017-01-31 10:20:32 -08003484 skb_warn_bad_offload(skb);
3485
3486 return segs;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07003487}
Cong Wang12b00042013-02-05 16:36:38 +00003488EXPORT_SYMBOL(__skb_gso_segment);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07003489
Herbert Xufb286bb2005-11-10 13:01:24 -08003490/* Take action when hardware reception checksum errors are detected. */
3491#ifdef CONFIG_BUG
Tanner Love127d7352021-06-28 09:50:07 -04003492static void do_netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
3493{
3494 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
3495 skb_dump(KERN_ERR, skb, true);
3496 dump_stack();
3497}
3498
Cong Wang7fe50ac2018-11-12 14:47:18 -08003499void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
Herbert Xufb286bb2005-11-10 13:01:24 -08003500{
Tanner Love127d7352021-06-28 09:50:07 -04003501 DO_ONCE_LITE(do_netdev_rx_csum_fault, dev, skb);
Herbert Xufb286bb2005-11-10 13:01:24 -08003502}
3503EXPORT_SYMBOL(netdev_rx_csum_fault);
3504#endif
3505
Christoph Hellwigab74cfe2018-04-03 20:31:35 +02003506/* XXX: check that highmem exists at all on the given machine. */
Florian Westphalc1e756b2014-05-05 15:00:44 +02003507static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003508{
Herbert Xu3d3a8532006-06-27 13:33:10 -07003509#ifdef CONFIG_HIGHMEM
Linus Torvalds1da177e2005-04-16 15:20:36 -07003510 int i;
tchardingf4563a72017-02-09 17:56:07 +11003511
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00003512 if (!(dev->features & NETIF_F_HIGHDMA)) {
Ian Campbellea2ab692011-08-22 23:44:58 +00003513 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3514 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
tchardingf4563a72017-02-09 17:56:07 +11003515
Ian Campbellea2ab692011-08-22 23:44:58 +00003516 if (PageHighMem(skb_frag_page(frag)))
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00003517 return 1;
Ian Campbellea2ab692011-08-22 23:44:58 +00003518 }
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00003519 }
Herbert Xu3d3a8532006-06-27 13:33:10 -07003520#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07003521 return 0;
3522}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003523
Simon Horman3b392dd2014-06-04 08:53:17 +09003524/* If MPLS offload request, verify we are testing hardware MPLS features
3525 * instead of standard features for the netdev.
3526 */
Pravin B Shelard0edc7b2014-12-23 16:20:11 -08003527#if IS_ENABLED(CONFIG_NET_MPLS_GSO)
Simon Horman3b392dd2014-06-04 08:53:17 +09003528static netdev_features_t net_mpls_features(struct sk_buff *skb,
3529 netdev_features_t features,
3530 __be16 type)
3531{
Simon Horman25cd9ba2014-10-06 05:05:13 -07003532 if (eth_p_mpls(type))
Simon Horman3b392dd2014-06-04 08:53:17 +09003533 features &= skb->dev->mpls_features;
3534
3535 return features;
3536}
3537#else
3538static netdev_features_t net_mpls_features(struct sk_buff *skb,
3539 netdev_features_t features,
3540 __be16 type)
3541{
3542 return features;
3543}
3544#endif
3545
Michał Mirosławc8f44af2011-11-15 15:29:55 +00003546static netdev_features_t harmonize_features(struct sk_buff *skb,
Florian Westphalc1e756b2014-05-05 15:00:44 +02003547 netdev_features_t features)
Jesse Grossf01a5232011-01-09 06:23:31 +00003548{
Simon Horman3b392dd2014-06-04 08:53:17 +09003549 __be16 type;
3550
Miaohe Lin9fc95f52020-07-30 19:02:36 +08003551 type = skb_network_protocol(skb, NULL);
Simon Horman3b392dd2014-06-04 08:53:17 +09003552 features = net_mpls_features(skb, features, type);
Vlad Yasevich53d64712014-03-27 17:26:18 -04003553
Ed Cashinc0d680e2012-09-19 15:49:00 +00003554 if (skb->ip_summed != CHECKSUM_NONE &&
Simon Horman3b392dd2014-06-04 08:53:17 +09003555 !can_checksum_protocol(features, type)) {
Alexander Duyck996e8022016-05-02 09:25:10 -07003556 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
Jesse Grossf01a5232011-01-09 06:23:31 +00003557 }
Eric Dumazet7be2c822017-01-18 12:12:17 -08003558 if (illegal_highdma(skb->dev, skb))
3559 features &= ~NETIF_F_SG;
Jesse Grossf01a5232011-01-09 06:23:31 +00003560
3561 return features;
3562}
3563
Toshiaki Makitae38f3022015-03-27 14:31:13 +09003564netdev_features_t passthru_features_check(struct sk_buff *skb,
3565 struct net_device *dev,
3566 netdev_features_t features)
3567{
3568 return features;
3569}
3570EXPORT_SYMBOL(passthru_features_check);
3571
Toshiaki Makita7ce23672018-04-17 18:46:14 +09003572static netdev_features_t dflt_features_check(struct sk_buff *skb,
Toshiaki Makita8cb65d02015-03-27 14:31:12 +09003573 struct net_device *dev,
3574 netdev_features_t features)
3575{
3576 return vlan_features_check(skb, features);
3577}
3578
Alexander Duyckcbc53e02016-04-10 21:44:51 -04003579static netdev_features_t gso_features_check(const struct sk_buff *skb,
3580 struct net_device *dev,
3581 netdev_features_t features)
3582{
3583 u16 gso_segs = skb_shinfo(skb)->gso_segs;
3584
3585 if (gso_segs > dev->gso_max_segs)
3586 return features & ~NETIF_F_GSO_MASK;
3587
Heiner Kallweit1d155df2020-11-21 00:22:20 +01003588 if (!skb_shinfo(skb)->gso_type) {
3589 skb_warn_bad_offload(skb);
3590 return features & ~NETIF_F_GSO_MASK;
3591 }
3592
Alexander Duyck802ab552016-04-10 21:45:03 -04003593 /* Support for GSO partial features requires software
3594 * intervention before we can actually process the packets
3595 * so we need to strip support for any partial features now
3596 * and we can pull them back in after we have partially
3597 * segmented the frame.
3598 */
3599 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
3600 features &= ~dev->gso_partial_features;
3601
3602 /* Make sure to clear the IPv4 ID mangling feature if the
3603 * IPv4 header has the potential to be fragmented.
Alexander Duyckcbc53e02016-04-10 21:44:51 -04003604 */
3605 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
3606 struct iphdr *iph = skb->encapsulation ?
3607 inner_ip_hdr(skb) : ip_hdr(skb);
3608
3609 if (!(iph->frag_off & htons(IP_DF)))
3610 features &= ~NETIF_F_TSO_MANGLEID;
3611 }
3612
3613 return features;
3614}
3615
Florian Westphalc1e756b2014-05-05 15:00:44 +02003616netdev_features_t netif_skb_features(struct sk_buff *skb)
Jesse Gross58e998c2010-10-29 12:14:55 +00003617{
Jesse Gross5f352272014-12-23 22:37:26 -08003618 struct net_device *dev = skb->dev;
Eric Dumazetfcbeb972014-10-05 10:11:27 -07003619 netdev_features_t features = dev->features;
Jesse Gross58e998c2010-10-29 12:14:55 +00003620
Alexander Duyckcbc53e02016-04-10 21:44:51 -04003621 if (skb_is_gso(skb))
3622 features = gso_features_check(skb, dev, features);
Ben Hutchings30b678d2012-07-30 15:57:00 +00003623
Jesse Gross5f352272014-12-23 22:37:26 -08003624 /* If encapsulation offload request, verify we are testing
3625 * hardware encapsulation features instead of standard
3626 * features for the netdev
3627 */
3628 if (skb->encapsulation)
3629 features &= dev->hw_enc_features;
3630
Toshiaki Makitaf5a7fb82015-03-27 14:31:11 +09003631 if (skb_vlan_tagged(skb))
3632 features = netdev_intersect_features(features,
3633 dev->vlan_features |
3634 NETIF_F_HW_VLAN_CTAG_TX |
3635 NETIF_F_HW_VLAN_STAG_TX);
Jesse Gross58e998c2010-10-29 12:14:55 +00003636
Jesse Gross5f352272014-12-23 22:37:26 -08003637 if (dev->netdev_ops->ndo_features_check)
3638 features &= dev->netdev_ops->ndo_features_check(skb, dev,
3639 features);
Toshiaki Makita8cb65d02015-03-27 14:31:12 +09003640 else
3641 features &= dflt_features_check(skb, dev, features);
Jesse Gross5f352272014-12-23 22:37:26 -08003642
Florian Westphalc1e756b2014-05-05 15:00:44 +02003643 return harmonize_features(skb, features);
Jesse Gross58e998c2010-10-29 12:14:55 +00003644}
Florian Westphalc1e756b2014-05-05 15:00:44 +02003645EXPORT_SYMBOL(netif_skb_features);
Jesse Gross58e998c2010-10-29 12:14:55 +00003646
David S. Miller2ea25512014-08-29 21:10:01 -07003647static int xmit_one(struct sk_buff *skb, struct net_device *dev,
David S. Miller95f6b3d2014-08-29 21:57:30 -07003648 struct netdev_queue *txq, bool more)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07003649{
David S. Miller2ea25512014-08-29 21:10:01 -07003650 unsigned int len;
3651 int rc;
Stephen Hemminger00829822008-11-20 20:14:53 -08003652
Maciej W. Rozycki9f9a7422018-10-09 23:57:49 +01003653 if (dev_nit_active(dev))
David S. Miller2ea25512014-08-29 21:10:01 -07003654 dev_queue_xmit_nit(skb, dev);
Jesse Grossfc741212011-01-09 06:23:32 +00003655
David S. Miller2ea25512014-08-29 21:10:01 -07003656 len = skb->len;
Willy Tarreau37447412020-08-10 10:27:42 +02003657 PRANDOM_ADD_NOISE(skb, dev, txq, len + jiffies);
David S. Miller2ea25512014-08-29 21:10:01 -07003658 trace_net_dev_start_xmit(skb, dev);
David S. Miller95f6b3d2014-08-29 21:57:30 -07003659 rc = netdev_start_xmit(skb, dev, txq, more);
David S. Miller2ea25512014-08-29 21:10:01 -07003660 trace_net_dev_xmit(skb, rc, dev, len);
Eric Dumazetadf30902009-06-02 05:19:30 +00003661
Patrick McHardy572a9d72009-11-10 06:14:14 +00003662 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07003663}
David S. Miller2ea25512014-08-29 21:10:01 -07003664
David S. Miller8dcda222014-09-01 15:06:40 -07003665struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
3666 struct netdev_queue *txq, int *ret)
David S. Miller7f2e8702014-08-29 21:19:14 -07003667{
3668 struct sk_buff *skb = first;
3669 int rc = NETDEV_TX_OK;
3670
3671 while (skb) {
3672 struct sk_buff *next = skb->next;
3673
David S. Millera8305bf2018-07-29 20:42:53 -07003674 skb_mark_not_on_list(skb);
David S. Miller95f6b3d2014-08-29 21:57:30 -07003675 rc = xmit_one(skb, dev, txq, next != NULL);
David S. Miller7f2e8702014-08-29 21:19:14 -07003676 if (unlikely(!dev_xmit_complete(rc))) {
3677 skb->next = next;
3678 goto out;
3679 }
3680
3681 skb = next;
Eric Dumazetfe60faa2018-10-31 08:39:13 -07003682 if (netif_tx_queue_stopped(txq) && skb) {
David S. Miller7f2e8702014-08-29 21:19:14 -07003683 rc = NETDEV_TX_BUSY;
3684 break;
3685 }
3686 }
3687
3688out:
3689 *ret = rc;
3690 return skb;
3691}
3692
Eric Dumazet1ff0dc92014-10-06 11:26:27 -07003693static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
3694 netdev_features_t features)
David S. Millereae3f882014-08-30 15:17:13 -07003695{
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01003696 if (skb_vlan_tag_present(skb) &&
Jiri Pirko59682502014-11-19 14:04:59 +01003697 !vlan_hw_offload_capable(features, skb->vlan_proto))
3698 skb = __vlan_hwaccel_push_inside(skb);
David S. Millereae3f882014-08-30 15:17:13 -07003699 return skb;
3700}
3701
Davide Caratti43c26a12017-05-18 15:44:41 +02003702int skb_csum_hwoffload_help(struct sk_buff *skb,
3703 const netdev_features_t features)
3704{
Xin Longfa821172021-01-16 14:13:37 +08003705 if (unlikely(skb_csum_is_sctp(skb)))
Davide Caratti43c26a12017-05-18 15:44:41 +02003706 return !!(features & NETIF_F_SCTP_CRC) ? 0 :
3707 skb_crc32c_csum_help(skb);
3708
Xin Long62fafcd2021-01-28 17:18:31 +08003709 if (features & NETIF_F_HW_CSUM)
3710 return 0;
3711
3712 if (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
3713 switch (skb->csum_offset) {
3714 case offsetof(struct tcphdr, check):
3715 case offsetof(struct udphdr, check):
3716 return 0;
3717 }
3718 }
3719
3720 return skb_checksum_help(skb);
Davide Caratti43c26a12017-05-18 15:44:41 +02003721}
3722EXPORT_SYMBOL(skb_csum_hwoffload_help);
3723
Steffen Klassertf53c7232017-12-20 10:41:36 +01003724static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again)
David S. Millereae3f882014-08-30 15:17:13 -07003725{
3726 netdev_features_t features;
3727
David S. Millereae3f882014-08-30 15:17:13 -07003728 features = netif_skb_features(skb);
3729 skb = validate_xmit_vlan(skb, features);
3730 if (unlikely(!skb))
3731 goto out_null;
3732
Ilya Lesokhinebf4e802018-04-30 10:16:12 +03003733 skb = sk_validate_xmit_skb(skb, dev);
3734 if (unlikely(!skb))
3735 goto out_null;
3736
Johannes Berg8b86a612015-04-17 15:45:04 +02003737 if (netif_needs_gso(skb, features)) {
David S. Millerce937182014-08-30 19:22:20 -07003738 struct sk_buff *segs;
3739
3740 segs = skb_gso_segment(skb, features);
Jason Wangcecda692014-09-19 16:04:38 +08003741 if (IS_ERR(segs)) {
Jason Wangaf6dabc2014-12-19 11:09:13 +08003742 goto out_kfree_skb;
Jason Wangcecda692014-09-19 16:04:38 +08003743 } else if (segs) {
3744 consume_skb(skb);
3745 skb = segs;
3746 }
David S. Millereae3f882014-08-30 15:17:13 -07003747 } else {
3748 if (skb_needs_linearize(skb, features) &&
3749 __skb_linearize(skb))
3750 goto out_kfree_skb;
3751
3752 /* If packet is not checksummed and device does not
3753 * support checksumming for this protocol, complete
3754 * checksumming here.
3755 */
3756 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3757 if (skb->encapsulation)
3758 skb_set_inner_transport_header(skb,
3759 skb_checksum_start_offset(skb));
3760 else
3761 skb_set_transport_header(skb,
3762 skb_checksum_start_offset(skb));
Davide Caratti43c26a12017-05-18 15:44:41 +02003763 if (skb_csum_hwoffload_help(skb, features))
David S. Millereae3f882014-08-30 15:17:13 -07003764 goto out_kfree_skb;
3765 }
3766 }
3767
Steffen Klassertf53c7232017-12-20 10:41:36 +01003768 skb = validate_xmit_xfrm(skb, features, again);
Steffen Klassert3dca3f32017-12-20 10:41:31 +01003769
David S. Millereae3f882014-08-30 15:17:13 -07003770 return skb;
3771
3772out_kfree_skb:
3773 kfree_skb(skb);
3774out_null:
Eric Dumazetd21fd632016-04-12 21:50:07 -07003775 atomic_long_inc(&dev->tx_dropped);
David S. Millereae3f882014-08-30 15:17:13 -07003776 return NULL;
3777}
3778
Steffen Klassertf53c7232017-12-20 10:41:36 +01003779struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again)
Eric Dumazet55a93b32014-10-03 15:31:07 -07003780{
3781 struct sk_buff *next, *head = NULL, *tail;
3782
Eric Dumazetbec3cfd2014-10-03 20:59:19 -07003783 for (; skb != NULL; skb = next) {
Eric Dumazet55a93b32014-10-03 15:31:07 -07003784 next = skb->next;
David S. Millera8305bf2018-07-29 20:42:53 -07003785 skb_mark_not_on_list(skb);
Eric Dumazet55a93b32014-10-03 15:31:07 -07003786
Eric Dumazetbec3cfd2014-10-03 20:59:19 -07003787 /* in case skb wont be segmented, point to itself */
3788 skb->prev = skb;
3789
Steffen Klassertf53c7232017-12-20 10:41:36 +01003790 skb = validate_xmit_skb(skb, dev, again);
Eric Dumazetbec3cfd2014-10-03 20:59:19 -07003791 if (!skb)
3792 continue;
3793
3794 if (!head)
3795 head = skb;
3796 else
3797 tail->next = skb;
3798 /* If skb was segmented, skb->prev points to
3799 * the last segment. If not, it still contains skb.
3800 */
3801 tail = skb->prev;
Eric Dumazet55a93b32014-10-03 15:31:07 -07003802 }
3803 return head;
3804}
Willem de Bruijn104ba782016-10-26 11:23:07 -04003805EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07003806
Eric Dumazet1def9232013-01-10 12:36:42 +00003807static void qdisc_pkt_len_init(struct sk_buff *skb)
3808{
3809 const struct skb_shared_info *shinfo = skb_shinfo(skb);
3810
3811 qdisc_skb_cb(skb)->pkt_len = skb->len;
3812
3813 /* To get more precise estimation of bytes sent on wire,
3814 * we add to pkt_len the headers size of all segments
3815 */
Maxim Mikityanskiya0dce872019-02-22 12:55:22 +00003816 if (shinfo->gso_size && skb_transport_header_was_set(skb)) {
Eric Dumazet757b8b12013-01-15 21:14:21 -08003817 unsigned int hdr_len;
Jason Wang15e5a032013-03-25 20:19:59 +00003818 u16 gso_segs = shinfo->gso_segs;
Eric Dumazet1def9232013-01-10 12:36:42 +00003819
Eric Dumazet757b8b12013-01-15 21:14:21 -08003820 /* mac layer + network layer */
3821 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3822
3823 /* + transport layer */
Eric Dumazet7c68d1a2018-01-18 19:59:19 -08003824 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
3825 const struct tcphdr *th;
3826 struct tcphdr _tcphdr;
3827
3828 th = skb_header_pointer(skb, skb_transport_offset(skb),
3829 sizeof(_tcphdr), &_tcphdr);
3830 if (likely(th))
3831 hdr_len += __tcp_hdrlen(th);
3832 } else {
3833 struct udphdr _udphdr;
3834
3835 if (skb_header_pointer(skb, skb_transport_offset(skb),
3836 sizeof(_udphdr), &_udphdr))
3837 hdr_len += sizeof(struct udphdr);
3838 }
Jason Wang15e5a032013-03-25 20:19:59 +00003839
3840 if (shinfo->gso_type & SKB_GSO_DODGY)
3841 gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3842 shinfo->gso_size);
3843
3844 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
Eric Dumazet1def9232013-01-10 12:36:42 +00003845 }
3846}
3847
Qitao Xu70713dd2021-07-14 23:03:24 -07003848static int dev_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *q,
3849 struct sk_buff **to_free,
3850 struct netdev_queue *txq)
3851{
3852 int rc;
3853
3854 rc = q->enqueue(skb, q, to_free) & NET_XMIT_MASK;
3855 if (rc == NET_XMIT_SUCCESS)
3856 trace_qdisc_enqueue(q, txq, skb);
3857 return rc;
3858}
3859
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003860static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3861 struct net_device *dev,
3862 struct netdev_queue *txq)
3863{
3864 spinlock_t *root_lock = qdisc_lock(q);
Eric Dumazet520ac302016-06-21 23:16:49 -07003865 struct sk_buff *to_free = NULL;
Eric Dumazeta2da5702011-01-20 03:48:19 +00003866 bool contended;
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003867 int rc;
3868
Eric Dumazeta2da5702011-01-20 03:48:19 +00003869 qdisc_calculate_pkt_len(skb, q);
John Fastabend6b3ba912017-12-07 09:54:25 -08003870
3871 if (q->flags & TCQ_F_NOLOCK) {
Yunsheng Linc4fef012021-06-22 14:49:56 +08003872 if (q->flags & TCQ_F_CAN_BYPASS && nolock_qdisc_is_empty(q) &&
3873 qdisc_run_begin(q)) {
3874 /* Retest nolock_qdisc_is_empty() within the protection
3875 * of q->seqlock to protect from racing with requeuing.
3876 */
3877 if (unlikely(!nolock_qdisc_is_empty(q))) {
Qitao Xu70713dd2021-07-14 23:03:24 -07003878 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
Yunsheng Linc4fef012021-06-22 14:49:56 +08003879 __qdisc_run(q);
3880 qdisc_run_end(q);
John Fastabend6b3ba912017-12-07 09:54:25 -08003881
Yunsheng Linc4fef012021-06-22 14:49:56 +08003882 goto no_lock_out;
3883 }
3884
3885 qdisc_bstats_cpu_update(q, skb);
3886 if (sch_direct_xmit(skb, q, dev, txq, NULL, true) &&
3887 !nolock_qdisc_is_empty(q))
3888 __qdisc_run(q);
3889
3890 qdisc_run_end(q);
3891 return NET_XMIT_SUCCESS;
3892 }
3893
Qitao Xu70713dd2021-07-14 23:03:24 -07003894 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
Yunsheng Linc4fef012021-06-22 14:49:56 +08003895 qdisc_run(q);
3896
3897no_lock_out:
John Fastabend6b3ba912017-12-07 09:54:25 -08003898 if (unlikely(to_free))
3899 kfree_skb_list(to_free);
3900 return rc;
3901 }
3902
Eric Dumazet79640a42010-06-02 05:09:29 -07003903 /*
3904 * Heuristic to force contended enqueues to serialize on a
3905 * separate lock before trying to get qdisc main lock.
Eric Dumazetf9eb8ae2016-06-06 09:37:15 -07003906 * This permits qdisc->running owner to get the lock more
Ying Xue9bf2b8c2014-06-26 15:56:31 +08003907 * often and dequeue packets faster.
Eric Dumazet79640a42010-06-02 05:09:29 -07003908 */
Eric Dumazeta2da5702011-01-20 03:48:19 +00003909 contended = qdisc_is_running(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07003910 if (unlikely(contended))
3911 spin_lock(&q->busylock);
3912
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003913 spin_lock(root_lock);
3914 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
Eric Dumazet520ac302016-06-21 23:16:49 -07003915 __qdisc_drop(skb, &to_free);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003916 rc = NET_XMIT_DROP;
3917 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
Eric Dumazetbc135b22010-06-02 03:23:51 -07003918 qdisc_run_begin(q)) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003919 /*
3920 * This is a work-conserving queue; there are no old skbs
3921 * waiting to be sent out; and the qdisc is not running -
3922 * xmit the skb directly.
3923 */
Eric Dumazetbfe0d022011-01-09 08:30:54 +00003924
Eric Dumazetbfe0d022011-01-09 08:30:54 +00003925 qdisc_bstats_update(q, skb);
3926
Eric Dumazet55a93b32014-10-03 15:31:07 -07003927 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
Eric Dumazet79640a42010-06-02 05:09:29 -07003928 if (unlikely(contended)) {
3929 spin_unlock(&q->busylock);
3930 contended = false;
3931 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003932 __qdisc_run(q);
John Fastabend6c148182017-12-07 09:54:06 -08003933 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003934
John Fastabend6c148182017-12-07 09:54:06 -08003935 qdisc_run_end(q);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003936 rc = NET_XMIT_SUCCESS;
3937 } else {
Qitao Xu70713dd2021-07-14 23:03:24 -07003938 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
Eric Dumazet79640a42010-06-02 05:09:29 -07003939 if (qdisc_run_begin(q)) {
3940 if (unlikely(contended)) {
3941 spin_unlock(&q->busylock);
3942 contended = false;
3943 }
3944 __qdisc_run(q);
John Fastabend6c148182017-12-07 09:54:06 -08003945 qdisc_run_end(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07003946 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003947 }
3948 spin_unlock(root_lock);
Eric Dumazet520ac302016-06-21 23:16:49 -07003949 if (unlikely(to_free))
3950 kfree_skb_list(to_free);
Eric Dumazet79640a42010-06-02 05:09:29 -07003951 if (unlikely(contended))
3952 spin_unlock(&q->busylock);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003953 return rc;
3954}
3955
Daniel Borkmann86f85152013-12-29 17:27:11 +01003956#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
Neil Horman5bc14212011-11-22 05:10:51 +00003957static void skb_update_prio(struct sk_buff *skb)
3958{
Eric Dumazet4dcb31d2018-03-14 09:04:16 -07003959 const struct netprio_map *map;
3960 const struct sock *sk;
3961 unsigned int prioidx;
Neil Horman5bc14212011-11-22 05:10:51 +00003962
Eric Dumazet4dcb31d2018-03-14 09:04:16 -07003963 if (skb->priority)
3964 return;
3965 map = rcu_dereference_bh(skb->dev->priomap);
3966 if (!map)
3967 return;
3968 sk = skb_to_full_sk(skb);
3969 if (!sk)
3970 return;
Eric Dumazet91c68ce2012-07-08 21:45:10 +00003971
Eric Dumazet4dcb31d2018-03-14 09:04:16 -07003972 prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data);
3973
3974 if (prioidx < map->priomap_len)
3975 skb->priority = map->priomap[prioidx];
Neil Horman5bc14212011-11-22 05:10:51 +00003976}
3977#else
3978#define skb_update_prio(skb)
3979#endif
3980
Dave Jonesd29f7492008-07-22 14:09:06 -07003981/**
Michel Machado95603e22012-06-12 10:16:35 +00003982 * dev_loopback_xmit - loop back @skb
Eric W. Biederman0c4b51f2015-09-15 20:04:18 -05003983 * @net: network namespace this loopback is happening in
3984 * @sk: sk needed to be a netfilter okfn
Michel Machado95603e22012-06-12 10:16:35 +00003985 * @skb: buffer to transmit
3986 */
Eric W. Biederman0c4b51f2015-09-15 20:04:18 -05003987int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
Michel Machado95603e22012-06-12 10:16:35 +00003988{
3989 skb_reset_mac_header(skb);
3990 __skb_pull(skb, skb_network_offset(skb));
3991 skb->pkt_type = PACKET_LOOPBACK;
3992 skb->ip_summed = CHECKSUM_UNNECESSARY;
3993 WARN_ON(!skb_dst(skb));
3994 skb_dst_force(skb);
3995 netif_rx_ni(skb);
3996 return 0;
3997}
3998EXPORT_SYMBOL(dev_loopback_xmit);
3999
Daniel Borkmann1f211a12016-01-07 22:29:47 +01004000#ifdef CONFIG_NET_EGRESS
4001static struct sk_buff *
4002sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
4003{
Jiri Pirko46209402017-11-03 11:46:25 +01004004 struct mini_Qdisc *miniq = rcu_dereference_bh(dev->miniq_egress);
Daniel Borkmann1f211a12016-01-07 22:29:47 +01004005 struct tcf_result cl_res;
4006
Jiri Pirko46209402017-11-03 11:46:25 +01004007 if (!miniq)
Daniel Borkmann1f211a12016-01-07 22:29:47 +01004008 return skb;
4009
Willem de Bruijn8dc07fd2017-01-07 17:06:37 -05004010 /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
wenxuaadaca92020-11-25 12:01:21 +08004011 qdisc_skb_cb(skb)->mru = 0;
wenxu7baf2422021-01-19 16:31:50 +08004012 qdisc_skb_cb(skb)->post_ct = false;
Jiri Pirko46209402017-11-03 11:46:25 +01004013 mini_qdisc_bstats_cpu_update(miniq, skb);
Daniel Borkmann1f211a12016-01-07 22:29:47 +01004014
Jiri Pirko46209402017-11-03 11:46:25 +01004015 switch (tcf_classify(skb, miniq->filter_list, &cl_res, false)) {
Daniel Borkmann1f211a12016-01-07 22:29:47 +01004016 case TC_ACT_OK:
4017 case TC_ACT_RECLASSIFY:
4018 skb->tc_index = TC_H_MIN(cl_res.classid);
4019 break;
4020 case TC_ACT_SHOT:
Jiri Pirko46209402017-11-03 11:46:25 +01004021 mini_qdisc_qstats_cpu_drop(miniq);
Daniel Borkmann1f211a12016-01-07 22:29:47 +01004022 *ret = NET_XMIT_DROP;
Daniel Borkmann7e2c3ae2016-05-15 23:28:29 +02004023 kfree_skb(skb);
4024 return NULL;
Daniel Borkmann1f211a12016-01-07 22:29:47 +01004025 case TC_ACT_STOLEN:
4026 case TC_ACT_QUEUED:
Jiri Pirkoe25ea212017-06-06 14:12:02 +02004027 case TC_ACT_TRAP:
Daniel Borkmann1f211a12016-01-07 22:29:47 +01004028 *ret = NET_XMIT_SUCCESS;
Daniel Borkmann7e2c3ae2016-05-15 23:28:29 +02004029 consume_skb(skb);
Daniel Borkmann1f211a12016-01-07 22:29:47 +01004030 return NULL;
4031 case TC_ACT_REDIRECT:
4032 /* No need to push/pop skb's mac_header here on egress! */
4033 skb_do_redirect(skb);
4034 *ret = NET_XMIT_SUCCESS;
4035 return NULL;
4036 default:
4037 break;
4038 }
Daniel Borkmann357b6cc2020-03-18 10:33:22 +01004039
Daniel Borkmann1f211a12016-01-07 22:29:47 +01004040 return skb;
4041}
4042#endif /* CONFIG_NET_EGRESS */
4043
Amritha Nambiarfc9bab22018-06-29 21:27:02 -07004044#ifdef CONFIG_XPS
4045static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb,
4046 struct xps_dev_maps *dev_maps, unsigned int tci)
4047{
Antoine Tenart255c04a2021-03-18 19:37:43 +01004048 int tc = netdev_get_prio_tc_map(dev, skb->priority);
Amritha Nambiarfc9bab22018-06-29 21:27:02 -07004049 struct xps_map *map;
4050 int queue_index = -1;
4051
Antoine Tenart5478fcd2021-03-18 19:37:44 +01004052 if (tc >= dev_maps->num_tc || tci >= dev_maps->nr_ids)
Antoine Tenart255c04a2021-03-18 19:37:43 +01004053 return queue_index;
4054
4055 tci *= dev_maps->num_tc;
4056 tci += tc;
Amritha Nambiarfc9bab22018-06-29 21:27:02 -07004057
4058 map = rcu_dereference(dev_maps->attr_map[tci]);
4059 if (map) {
4060 if (map->len == 1)
4061 queue_index = map->queues[0];
4062 else
4063 queue_index = map->queues[reciprocal_scale(
4064 skb_get_hash(skb), map->len)];
4065 if (unlikely(queue_index >= dev->real_num_tx_queues))
4066 queue_index = -1;
4067 }
4068 return queue_index;
4069}
4070#endif
4071
Alexander Duyckeadec8772018-07-09 12:19:48 -04004072static int get_xps_queue(struct net_device *dev, struct net_device *sb_dev,
4073 struct sk_buff *skb)
Jiri Pirko638b2a62015-05-12 14:56:13 +02004074{
4075#ifdef CONFIG_XPS
4076 struct xps_dev_maps *dev_maps;
Amritha Nambiarfc9bab22018-06-29 21:27:02 -07004077 struct sock *sk = skb->sk;
Jiri Pirko638b2a62015-05-12 14:56:13 +02004078 int queue_index = -1;
4079
Amritha Nambiar04157462018-06-29 21:26:46 -07004080 if (!static_key_false(&xps_needed))
4081 return -1;
4082
Jiri Pirko638b2a62015-05-12 14:56:13 +02004083 rcu_read_lock();
Amritha Nambiarfc9bab22018-06-29 21:27:02 -07004084 if (!static_key_false(&xps_rxqs_needed))
4085 goto get_cpus_map;
4086
Antoine Tenart044ab862021-03-18 19:37:46 +01004087 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_RXQS]);
Jiri Pirko638b2a62015-05-12 14:56:13 +02004088 if (dev_maps) {
Amritha Nambiarfc9bab22018-06-29 21:27:02 -07004089 int tci = sk_rx_queue_get(sk);
Alexander Duyck184c4492016-10-28 11:50:13 -04004090
Antoine Tenart5478fcd2021-03-18 19:37:44 +01004091 if (tci >= 0)
Amritha Nambiarfc9bab22018-06-29 21:27:02 -07004092 queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4093 tci);
4094 }
Alexander Duyck184c4492016-10-28 11:50:13 -04004095
Amritha Nambiarfc9bab22018-06-29 21:27:02 -07004096get_cpus_map:
4097 if (queue_index < 0) {
Antoine Tenart044ab862021-03-18 19:37:46 +01004098 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_CPUS]);
Amritha Nambiarfc9bab22018-06-29 21:27:02 -07004099 if (dev_maps) {
4100 unsigned int tci = skb->sender_cpu - 1;
4101
4102 queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4103 tci);
Jiri Pirko638b2a62015-05-12 14:56:13 +02004104 }
4105 }
4106 rcu_read_unlock();
4107
4108 return queue_index;
4109#else
4110 return -1;
4111#endif
4112}
4113
Alexander Duycka4ea8a32018-07-09 12:19:54 -04004114u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb,
Paolo Abenia350ecc2019-03-20 11:02:06 +01004115 struct net_device *sb_dev)
Alexander Duycka4ea8a32018-07-09 12:19:54 -04004116{
4117 return 0;
4118}
4119EXPORT_SYMBOL(dev_pick_tx_zero);
4120
4121u16 dev_pick_tx_cpu_id(struct net_device *dev, struct sk_buff *skb,
Paolo Abenia350ecc2019-03-20 11:02:06 +01004122 struct net_device *sb_dev)
Alexander Duycka4ea8a32018-07-09 12:19:54 -04004123{
4124 return (u16)raw_smp_processor_id() % dev->real_num_tx_queues;
4125}
4126EXPORT_SYMBOL(dev_pick_tx_cpu_id);
4127
Paolo Abenib71b5832019-03-20 11:02:05 +01004128u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb,
4129 struct net_device *sb_dev)
Jiri Pirko638b2a62015-05-12 14:56:13 +02004130{
4131 struct sock *sk = skb->sk;
4132 int queue_index = sk_tx_queue_get(sk);
4133
Alexander Duyckeadec8772018-07-09 12:19:48 -04004134 sb_dev = sb_dev ? : dev;
4135
Jiri Pirko638b2a62015-05-12 14:56:13 +02004136 if (queue_index < 0 || skb->ooo_okay ||
4137 queue_index >= dev->real_num_tx_queues) {
Alexander Duyckeadec8772018-07-09 12:19:48 -04004138 int new_index = get_xps_queue(dev, sb_dev, skb);
tchardingf4563a72017-02-09 17:56:07 +11004139
Jiri Pirko638b2a62015-05-12 14:56:13 +02004140 if (new_index < 0)
Alexander Duyckeadec8772018-07-09 12:19:48 -04004141 new_index = skb_tx_hash(dev, sb_dev, skb);
Jiri Pirko638b2a62015-05-12 14:56:13 +02004142
4143 if (queue_index != new_index && sk &&
Eric Dumazet004a5d02015-10-04 21:08:10 -07004144 sk_fullsock(sk) &&
Jiri Pirko638b2a62015-05-12 14:56:13 +02004145 rcu_access_pointer(sk->sk_dst_cache))
4146 sk_tx_queue_set(sk, new_index);
4147
4148 queue_index = new_index;
4149 }
4150
4151 return queue_index;
4152}
Paolo Abenib71b5832019-03-20 11:02:05 +01004153EXPORT_SYMBOL(netdev_pick_tx);
Jiri Pirko638b2a62015-05-12 14:56:13 +02004154
Paolo Abeni4bd97d52019-03-20 11:02:04 +01004155struct netdev_queue *netdev_core_pick_tx(struct net_device *dev,
4156 struct sk_buff *skb,
4157 struct net_device *sb_dev)
Jiri Pirko638b2a62015-05-12 14:56:13 +02004158{
4159 int queue_index = 0;
4160
4161#ifdef CONFIG_XPS
Eric Dumazet52bd2d62015-11-18 06:30:50 -08004162 u32 sender_cpu = skb->sender_cpu - 1;
4163
4164 if (sender_cpu >= (u32)NR_CPUS)
Jiri Pirko638b2a62015-05-12 14:56:13 +02004165 skb->sender_cpu = raw_smp_processor_id() + 1;
4166#endif
4167
4168 if (dev->real_num_tx_queues != 1) {
4169 const struct net_device_ops *ops = dev->netdev_ops;
tchardingf4563a72017-02-09 17:56:07 +11004170
Jiri Pirko638b2a62015-05-12 14:56:13 +02004171 if (ops->ndo_select_queue)
Paolo Abenia350ecc2019-03-20 11:02:06 +01004172 queue_index = ops->ndo_select_queue(dev, skb, sb_dev);
Jiri Pirko638b2a62015-05-12 14:56:13 +02004173 else
Paolo Abeni4bd97d52019-03-20 11:02:04 +01004174 queue_index = netdev_pick_tx(dev, skb, sb_dev);
Jiri Pirko638b2a62015-05-12 14:56:13 +02004175
Alexander Duyckd5845272017-11-22 10:57:41 -08004176 queue_index = netdev_cap_txqueue(dev, queue_index);
Jiri Pirko638b2a62015-05-12 14:56:13 +02004177 }
4178
4179 skb_set_queue_mapping(skb, queue_index);
4180 return netdev_get_tx_queue(dev, queue_index);
4181}
4182
Michel Machado95603e22012-06-12 10:16:35 +00004183/**
Jason Wang9d08dd32014-01-20 11:25:13 +08004184 * __dev_queue_xmit - transmit a buffer
Dave Jonesd29f7492008-07-22 14:09:06 -07004185 * @skb: buffer to transmit
Alexander Duyckeadec8772018-07-09 12:19:48 -04004186 * @sb_dev: suboordinate device used for L2 forwarding offload
Dave Jonesd29f7492008-07-22 14:09:06 -07004187 *
4188 * Queue a buffer for transmission to a network device. The caller must
4189 * have set the device and priority and built the buffer before calling
4190 * this function. The function can be called from an interrupt.
4191 *
4192 * A negative errno code is returned on a failure. A success does not
4193 * guarantee the frame will be transmitted as it may be dropped due
4194 * to congestion or traffic shaping.
4195 *
4196 * -----------------------------------------------------------------------------------
4197 * I notice this method can also return errors from the queue disciplines,
4198 * including NET_XMIT_DROP, which is a positive value. So, errors can also
4199 * be positive.
4200 *
4201 * Regardless of the return value, the skb is consumed, so it is currently
4202 * difficult to retry a send to this method. (You can bump the ref count
4203 * before sending to hold a reference for retry if you are careful.)
4204 *
4205 * When calling this method, interrupts MUST be enabled. This is because
4206 * the BH enable code must have IRQs enabled so that it will not deadlock.
4207 * --BLG
4208 */
Alexander Duyckeadec8772018-07-09 12:19:48 -04004209static int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004210{
4211 struct net_device *dev = skb->dev;
David S. Millerdc2b4842008-07-08 17:18:23 -07004212 struct netdev_queue *txq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004213 struct Qdisc *q;
4214 int rc = -ENOMEM;
Steffen Klassertf53c7232017-12-20 10:41:36 +01004215 bool again = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004216
Eric Dumazet6d1ccff2013-02-05 20:22:20 +00004217 skb_reset_mac_header(skb);
4218
Willem de Bruijne7fd2882014-08-04 22:11:48 -04004219 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
Yousuk Seunge7ed11e2021-01-20 12:41:55 -08004220 __skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED);
Willem de Bruijne7fd2882014-08-04 22:11:48 -04004221
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004222 /* Disable soft irqs for various locks below. Also
4223 * stops preemption for RCU.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004224 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004225 rcu_read_lock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004226
Neil Horman5bc14212011-11-22 05:10:51 +00004227 skb_update_prio(skb);
4228
Daniel Borkmann1f211a12016-01-07 22:29:47 +01004229 qdisc_pkt_len_init(skb);
4230#ifdef CONFIG_NET_CLS_ACT
Willem de Bruijn8dc07fd2017-01-07 17:06:37 -05004231 skb->tc_at_ingress = 0;
Daniel Borkmann357b6cc2020-03-18 10:33:22 +01004232# ifdef CONFIG_NET_EGRESS
Davidlohr Buesoaabf6772018-05-08 09:07:00 -07004233 if (static_branch_unlikely(&egress_needed_key)) {
Daniel Borkmann1f211a12016-01-07 22:29:47 +01004234 skb = sch_handle_egress(skb, &rc, dev);
4235 if (!skb)
4236 goto out;
4237 }
Daniel Borkmann357b6cc2020-03-18 10:33:22 +01004238# endif
Daniel Borkmann1f211a12016-01-07 22:29:47 +01004239#endif
Eric Dumazet02875872014-10-05 18:38:35 -07004240 /* If device/qdisc don't need skb->dst, release it right now while
4241 * its hot in this cpu cache.
4242 */
4243 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
4244 skb_dst_drop(skb);
4245 else
4246 skb_dst_force(skb);
4247
Paolo Abeni4bd97d52019-03-20 11:02:04 +01004248 txq = netdev_core_pick_tx(dev, skb, sb_dev);
Paul E. McKenneya898def2010-02-22 17:04:49 -08004249 q = rcu_dereference_bh(txq->qdisc);
David S. Miller37437bb2008-07-16 02:15:04 -07004250
Koki Sanagicf66ba52010-08-23 18:45:02 +09004251 trace_net_dev_queue(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004252 if (q->enqueue) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00004253 rc = __dev_xmit_skb(skb, q, dev, txq);
David S. Miller37437bb2008-07-16 02:15:04 -07004254 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004255 }
4256
4257 /* The device has no queue. Common case for software devices:
tchardingeb13da12017-02-09 17:56:06 +11004258 * loopback, all the sorts of tunnels...
Linus Torvalds1da177e2005-04-16 15:20:36 -07004259
tchardingeb13da12017-02-09 17:56:06 +11004260 * Really, it is unlikely that netif_tx_lock protection is necessary
4261 * here. (f.e. loopback and IP tunnels are clean ignoring statistics
4262 * counters.)
4263 * However, it is possible, that they rely on protection
4264 * made by us here.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004265
tchardingeb13da12017-02-09 17:56:06 +11004266 * Check this and shot the lock. It is not prone from deadlocks.
4267 *Either shot noqueue qdisc, it is even simpler 8)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004268 */
4269 if (dev->flags & IFF_UP) {
4270 int cpu = smp_processor_id(); /* ok because BHs are off */
4271
David S. Millerc773e842008-07-08 23:13:53 -07004272 if (txq->xmit_lock_owner != cpu) {
Florian Westphal97cdcf32019-04-01 16:42:13 +02004273 if (dev_xmit_recursion())
Eric Dumazet745e20f2010-09-29 13:23:09 -07004274 goto recursion_alert;
4275
Steffen Klassertf53c7232017-12-20 10:41:36 +01004276 skb = validate_xmit_skb(skb, dev, &again);
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02004277 if (!skb)
Eric Dumazetd21fd632016-04-12 21:50:07 -07004278 goto out;
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02004279
Willy Tarreau37447412020-08-10 10:27:42 +02004280 PRANDOM_ADD_NOISE(skb, dev, txq, jiffies);
David S. Millerc773e842008-07-08 23:13:53 -07004281 HARD_TX_LOCK(dev, txq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004282
Tom Herbert734664982011-11-28 16:32:44 +00004283 if (!netif_xmit_stopped(txq)) {
Florian Westphal97cdcf32019-04-01 16:42:13 +02004284 dev_xmit_recursion_inc();
David S. Millerce937182014-08-30 19:22:20 -07004285 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
Florian Westphal97cdcf32019-04-01 16:42:13 +02004286 dev_xmit_recursion_dec();
Patrick McHardy572a9d72009-11-10 06:14:14 +00004287 if (dev_xmit_complete(rc)) {
David S. Millerc773e842008-07-08 23:13:53 -07004288 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004289 goto out;
4290 }
4291 }
David S. Millerc773e842008-07-08 23:13:53 -07004292 HARD_TX_UNLOCK(dev, txq);
Joe Perchese87cc472012-05-13 21:56:26 +00004293 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
4294 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004295 } else {
4296 /* Recursion is detected! It is possible,
Eric Dumazet745e20f2010-09-29 13:23:09 -07004297 * unfortunately
4298 */
4299recursion_alert:
Joe Perchese87cc472012-05-13 21:56:26 +00004300 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
4301 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004302 }
4303 }
4304
4305 rc = -ENETDOWN;
Herbert Xud4828d82006-06-22 02:28:18 -07004306 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004307
Eric Dumazet015f0682014-03-27 08:45:56 -07004308 atomic_long_inc(&dev->tx_dropped);
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02004309 kfree_skb_list(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004310 return rc;
4311out:
Herbert Xud4828d82006-06-22 02:28:18 -07004312 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004313 return rc;
4314}
Jason Wangf663dd92014-01-10 16:18:26 +08004315
Eric W. Biederman2b4aa3c2015-09-15 20:04:07 -05004316int dev_queue_xmit(struct sk_buff *skb)
Jason Wangf663dd92014-01-10 16:18:26 +08004317{
4318 return __dev_queue_xmit(skb, NULL);
4319}
Eric W. Biederman2b4aa3c2015-09-15 20:04:07 -05004320EXPORT_SYMBOL(dev_queue_xmit);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004321
Alexander Duyckeadec8772018-07-09 12:19:48 -04004322int dev_queue_xmit_accel(struct sk_buff *skb, struct net_device *sb_dev)
Jason Wangf663dd92014-01-10 16:18:26 +08004323{
Alexander Duyckeadec8772018-07-09 12:19:48 -04004324 return __dev_queue_xmit(skb, sb_dev);
Jason Wangf663dd92014-01-10 16:18:26 +08004325}
4326EXPORT_SYMBOL(dev_queue_xmit_accel);
4327
Björn Töpel36ccdf82020-11-23 18:56:00 +01004328int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
Magnus Karlsson865b03f2018-05-02 13:01:33 +02004329{
4330 struct net_device *dev = skb->dev;
4331 struct sk_buff *orig_skb = skb;
4332 struct netdev_queue *txq;
4333 int ret = NETDEV_TX_BUSY;
4334 bool again = false;
4335
4336 if (unlikely(!netif_running(dev) ||
4337 !netif_carrier_ok(dev)))
4338 goto drop;
4339
4340 skb = validate_xmit_skb_list(skb, dev, &again);
4341 if (skb != orig_skb)
4342 goto drop;
4343
4344 skb_set_queue_mapping(skb, queue_id);
4345 txq = skb_get_tx_queue(dev, skb);
Willy Tarreau37447412020-08-10 10:27:42 +02004346 PRANDOM_ADD_NOISE(skb, dev, txq, jiffies);
Magnus Karlsson865b03f2018-05-02 13:01:33 +02004347
4348 local_bh_disable();
4349
Eric Dumazet0ad6f6e2020-06-17 22:23:25 -07004350 dev_xmit_recursion_inc();
Magnus Karlsson865b03f2018-05-02 13:01:33 +02004351 HARD_TX_LOCK(dev, txq, smp_processor_id());
4352 if (!netif_xmit_frozen_or_drv_stopped(txq))
4353 ret = netdev_start_xmit(skb, dev, txq, false);
4354 HARD_TX_UNLOCK(dev, txq);
Eric Dumazet0ad6f6e2020-06-17 22:23:25 -07004355 dev_xmit_recursion_dec();
Magnus Karlsson865b03f2018-05-02 13:01:33 +02004356
4357 local_bh_enable();
Magnus Karlsson865b03f2018-05-02 13:01:33 +02004358 return ret;
4359drop:
4360 atomic_long_inc(&dev->tx_dropped);
4361 kfree_skb_list(skb);
4362 return NET_XMIT_DROP;
4363}
Björn Töpel36ccdf82020-11-23 18:56:00 +01004364EXPORT_SYMBOL(__dev_direct_xmit);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004365
tchardingeb13da12017-02-09 17:56:06 +11004366/*************************************************************************
4367 * Receiver routines
4368 *************************************************************************/
Linus Torvalds1da177e2005-04-16 15:20:36 -07004369
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07004370int netdev_max_backlog __read_mostly = 1000;
Eric Dumazetc9e6bc62012-09-27 19:29:05 +00004371EXPORT_SYMBOL(netdev_max_backlog);
4372
Eric Dumazet3b098e22010-05-15 23:57:10 -07004373int netdev_tstamp_prequeue __read_mostly = 1;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07004374int netdev_budget __read_mostly = 300;
Konstantin Khlebnikova48379802020-04-06 14:39:32 +03004375/* Must be at least 2 jiffes to guarantee 1 jiffy timeout */
4376unsigned int __read_mostly netdev_budget_usecs = 2 * USEC_PER_SEC / HZ;
Matthias Tafelmeier3d48b532016-12-29 21:37:21 +01004377int weight_p __read_mostly = 64; /* old backlog weight */
4378int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */
4379int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */
4380int dev_rx_weight __read_mostly = 64;
4381int dev_tx_weight __read_mostly = 64;
Edward Cree323ebb62019-08-06 14:53:55 +01004382/* Maximum number of GRO_NORMAL skbs to batch up for list-RX */
4383int gro_normal_batch __read_mostly = 8;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004384
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004385/* Called with irq disabled */
4386static inline void ____napi_schedule(struct softnet_data *sd,
4387 struct napi_struct *napi)
4388{
Wei Wang29863d42021-02-08 11:34:09 -08004389 struct task_struct *thread;
4390
4391 if (test_bit(NAPI_STATE_THREADED, &napi->state)) {
4392 /* Paired with smp_mb__before_atomic() in
Wei Wang5fdd2f02021-02-08 11:34:10 -08004393 * napi_enable()/dev_set_threaded().
4394 * Use READ_ONCE() to guarantee a complete
4395 * read on napi->thread. Only call
Wei Wang29863d42021-02-08 11:34:09 -08004396 * wake_up_process() when it's not NULL.
4397 */
4398 thread = READ_ONCE(napi->thread);
4399 if (thread) {
Wei Wangcb038352021-03-16 15:36:47 -07004400 /* Avoid doing set_bit() if the thread is in
4401 * INTERRUPTIBLE state, cause napi_thread_wait()
4402 * makes sure to proceed with napi polling
4403 * if the thread is explicitly woken from here.
4404 */
Peter Zijlstra2f064a52021-06-11 10:28:17 +02004405 if (READ_ONCE(thread->__state) != TASK_INTERRUPTIBLE)
Wei Wangcb038352021-03-16 15:36:47 -07004406 set_bit(NAPI_STATE_SCHED_THREADED, &napi->state);
Wei Wang29863d42021-02-08 11:34:09 -08004407 wake_up_process(thread);
4408 return;
4409 }
4410 }
4411
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004412 list_add_tail(&napi->poll_list, &sd->poll_list);
4413 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4414}
4415
Eric Dumazetdf334542010-03-24 19:13:54 +00004416#ifdef CONFIG_RPS
Tom Herbertfec5e652010-04-16 16:01:27 -07004417
4418/* One global table that all flow-based protocols share. */
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00004419struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
Tom Herbertfec5e652010-04-16 16:01:27 -07004420EXPORT_SYMBOL(rps_sock_flow_table);
Eric Dumazet567e4b72015-02-06 12:59:01 -08004421u32 rps_cpu_mask __read_mostly;
4422EXPORT_SYMBOL(rps_cpu_mask);
Tom Herbertfec5e652010-04-16 16:01:27 -07004423
Eric Dumazetdc053602019-03-22 08:56:38 -07004424struct static_key_false rps_needed __read_mostly;
Jason Wang3df97ba2016-04-25 23:13:42 -04004425EXPORT_SYMBOL(rps_needed);
Eric Dumazetdc053602019-03-22 08:56:38 -07004426struct static_key_false rfs_needed __read_mostly;
Eric Dumazet13bfff22016-12-07 08:29:10 -08004427EXPORT_SYMBOL(rfs_needed);
Eric Dumazetadc93002011-11-17 03:13:26 +00004428
Ben Hutchingsc4454772011-01-19 11:03:53 +00004429static struct rps_dev_flow *
4430set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4431 struct rps_dev_flow *rflow, u16 next_cpu)
4432{
Eric Dumazeta31196b2015-04-25 09:35:24 -07004433 if (next_cpu < nr_cpu_ids) {
Ben Hutchingsc4454772011-01-19 11:03:53 +00004434#ifdef CONFIG_RFS_ACCEL
4435 struct netdev_rx_queue *rxqueue;
4436 struct rps_dev_flow_table *flow_table;
4437 struct rps_dev_flow *old_rflow;
4438 u32 flow_id;
4439 u16 rxq_index;
4440 int rc;
4441
4442 /* Should we steer this flow to a different hardware queue? */
Ben Hutchings69a19ee2011-02-15 20:32:04 +00004443 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
4444 !(dev->features & NETIF_F_NTUPLE))
Ben Hutchingsc4454772011-01-19 11:03:53 +00004445 goto out;
4446 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
4447 if (rxq_index == skb_get_rx_queue(skb))
4448 goto out;
4449
4450 rxqueue = dev->_rx + rxq_index;
4451 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4452 if (!flow_table)
4453 goto out;
Tom Herbert61b905d2014-03-24 15:34:47 -07004454 flow_id = skb_get_hash(skb) & flow_table->mask;
Ben Hutchingsc4454772011-01-19 11:03:53 +00004455 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
4456 rxq_index, flow_id);
4457 if (rc < 0)
4458 goto out;
4459 old_rflow = rflow;
4460 rflow = &flow_table->flows[flow_id];
Ben Hutchingsc4454772011-01-19 11:03:53 +00004461 rflow->filter = rc;
4462 if (old_rflow->filter == rflow->filter)
4463 old_rflow->filter = RPS_NO_FILTER;
4464 out:
4465#endif
4466 rflow->last_qtail =
Ben Hutchings09994d12011-10-03 04:42:46 +00004467 per_cpu(softnet_data, next_cpu).input_queue_head;
Ben Hutchingsc4454772011-01-19 11:03:53 +00004468 }
4469
Ben Hutchings09994d12011-10-03 04:42:46 +00004470 rflow->cpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00004471 return rflow;
4472}
4473
Tom Herbert0a9627f2010-03-16 08:03:29 +00004474/*
4475 * get_rps_cpu is called from netif_receive_skb and returns the target
4476 * CPU from the RPS map of the receiving queue for a given skb.
Eric Dumazetb0e28f12010-04-15 00:14:07 -07004477 * rcu_read_lock must be held on entry.
Tom Herbert0a9627f2010-03-16 08:03:29 +00004478 */
Tom Herbertfec5e652010-04-16 16:01:27 -07004479static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4480 struct rps_dev_flow **rflowp)
Tom Herbert0a9627f2010-03-16 08:03:29 +00004481{
Eric Dumazet567e4b72015-02-06 12:59:01 -08004482 const struct rps_sock_flow_table *sock_flow_table;
4483 struct netdev_rx_queue *rxqueue = dev->_rx;
Tom Herbertfec5e652010-04-16 16:01:27 -07004484 struct rps_dev_flow_table *flow_table;
Eric Dumazet567e4b72015-02-06 12:59:01 -08004485 struct rps_map *map;
Tom Herbert0a9627f2010-03-16 08:03:29 +00004486 int cpu = -1;
Eric Dumazet567e4b72015-02-06 12:59:01 -08004487 u32 tcpu;
Tom Herbert61b905d2014-03-24 15:34:47 -07004488 u32 hash;
Tom Herbert0a9627f2010-03-16 08:03:29 +00004489
Tom Herbert0a9627f2010-03-16 08:03:29 +00004490 if (skb_rx_queue_recorded(skb)) {
4491 u16 index = skb_get_rx_queue(skb);
Eric Dumazet567e4b72015-02-06 12:59:01 -08004492
Ben Hutchings62fe0b42010-09-27 08:24:33 +00004493 if (unlikely(index >= dev->real_num_rx_queues)) {
4494 WARN_ONCE(dev->real_num_rx_queues > 1,
4495 "%s received packet on queue %u, but number "
4496 "of RX queues is %u\n",
4497 dev->name, index, dev->real_num_rx_queues);
Tom Herbert0a9627f2010-03-16 08:03:29 +00004498 goto done;
4499 }
Eric Dumazet567e4b72015-02-06 12:59:01 -08004500 rxqueue += index;
Tom Herbert0a9627f2010-03-16 08:03:29 +00004501 }
4502
Eric Dumazet567e4b72015-02-06 12:59:01 -08004503 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
4504
4505 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4506 map = rcu_dereference(rxqueue->rps_map);
4507 if (!flow_table && !map)
4508 goto done;
4509
Changli Gao2d47b452010-08-17 19:00:56 +00004510 skb_reset_network_header(skb);
Tom Herbert61b905d2014-03-24 15:34:47 -07004511 hash = skb_get_hash(skb);
4512 if (!hash)
Tom Herbert0a9627f2010-03-16 08:03:29 +00004513 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00004514
Tom Herbertfec5e652010-04-16 16:01:27 -07004515 sock_flow_table = rcu_dereference(rps_sock_flow_table);
4516 if (flow_table && sock_flow_table) {
Tom Herbertfec5e652010-04-16 16:01:27 -07004517 struct rps_dev_flow *rflow;
Eric Dumazet567e4b72015-02-06 12:59:01 -08004518 u32 next_cpu;
4519 u32 ident;
Tom Herbertfec5e652010-04-16 16:01:27 -07004520
Eric Dumazet567e4b72015-02-06 12:59:01 -08004521 /* First check into global flow table if there is a match */
4522 ident = sock_flow_table->ents[hash & sock_flow_table->mask];
4523 if ((ident ^ hash) & ~rps_cpu_mask)
4524 goto try_rps;
4525
4526 next_cpu = ident & rps_cpu_mask;
4527
4528 /* OK, now we know there is a match,
4529 * we can look at the local (per receive queue) flow table
4530 */
Tom Herbert61b905d2014-03-24 15:34:47 -07004531 rflow = &flow_table->flows[hash & flow_table->mask];
Tom Herbertfec5e652010-04-16 16:01:27 -07004532 tcpu = rflow->cpu;
4533
Tom Herbertfec5e652010-04-16 16:01:27 -07004534 /*
4535 * If the desired CPU (where last recvmsg was done) is
4536 * different from current CPU (one in the rx-queue flow
4537 * table entry), switch if one of the following holds:
Eric Dumazeta31196b2015-04-25 09:35:24 -07004538 * - Current CPU is unset (>= nr_cpu_ids).
Tom Herbertfec5e652010-04-16 16:01:27 -07004539 * - Current CPU is offline.
4540 * - The current CPU's queue tail has advanced beyond the
4541 * last packet that was enqueued using this table entry.
4542 * This guarantees that all previous packets for the flow
4543 * have been dequeued, thus preserving in order delivery.
4544 */
4545 if (unlikely(tcpu != next_cpu) &&
Eric Dumazeta31196b2015-04-25 09:35:24 -07004546 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
Tom Herbertfec5e652010-04-16 16:01:27 -07004547 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
Tom Herbertbaefa312012-11-16 09:04:15 +00004548 rflow->last_qtail)) >= 0)) {
4549 tcpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00004550 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
Tom Herbertbaefa312012-11-16 09:04:15 +00004551 }
Ben Hutchingsc4454772011-01-19 11:03:53 +00004552
Eric Dumazeta31196b2015-04-25 09:35:24 -07004553 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
Tom Herbertfec5e652010-04-16 16:01:27 -07004554 *rflowp = rflow;
4555 cpu = tcpu;
4556 goto done;
4557 }
4558 }
4559
Eric Dumazet567e4b72015-02-06 12:59:01 -08004560try_rps:
4561
Tom Herbert0a9627f2010-03-16 08:03:29 +00004562 if (map) {
Daniel Borkmann8fc54f62014-08-23 20:58:54 +02004563 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
Tom Herbert0a9627f2010-03-16 08:03:29 +00004564 if (cpu_online(tcpu)) {
4565 cpu = tcpu;
4566 goto done;
4567 }
4568 }
4569
4570done:
Tom Herbert0a9627f2010-03-16 08:03:29 +00004571 return cpu;
4572}
4573
Ben Hutchingsc4454772011-01-19 11:03:53 +00004574#ifdef CONFIG_RFS_ACCEL
4575
4576/**
4577 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
4578 * @dev: Device on which the filter was set
4579 * @rxq_index: RX queue index
4580 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
4581 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
4582 *
4583 * Drivers that implement ndo_rx_flow_steer() should periodically call
4584 * this function for each installed filter and remove the filters for
4585 * which it returns %true.
4586 */
4587bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
4588 u32 flow_id, u16 filter_id)
4589{
4590 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
4591 struct rps_dev_flow_table *flow_table;
4592 struct rps_dev_flow *rflow;
4593 bool expire = true;
Eric Dumazeta31196b2015-04-25 09:35:24 -07004594 unsigned int cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00004595
4596 rcu_read_lock();
4597 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4598 if (flow_table && flow_id <= flow_table->mask) {
4599 rflow = &flow_table->flows[flow_id];
Mark Rutland6aa7de02017-10-23 14:07:29 -07004600 cpu = READ_ONCE(rflow->cpu);
Eric Dumazeta31196b2015-04-25 09:35:24 -07004601 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
Ben Hutchingsc4454772011-01-19 11:03:53 +00004602 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
4603 rflow->last_qtail) <
4604 (int)(10 * flow_table->mask)))
4605 expire = false;
4606 }
4607 rcu_read_unlock();
4608 return expire;
4609}
4610EXPORT_SYMBOL(rps_may_expire_flow);
4611
4612#endif /* CONFIG_RFS_ACCEL */
4613
Tom Herbert0a9627f2010-03-16 08:03:29 +00004614/* Called from hardirq (IPI) context */
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004615static void rps_trigger_softirq(void *data)
Tom Herbert0a9627f2010-03-16 08:03:29 +00004616{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004617 struct softnet_data *sd = data;
4618
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004619 ____napi_schedule(sd, &sd->backlog);
Changli Gaodee42872010-05-02 05:42:16 +00004620 sd->received_rps++;
Tom Herbert0a9627f2010-03-16 08:03:29 +00004621}
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004622
Tom Herbertfec5e652010-04-16 16:01:27 -07004623#endif /* CONFIG_RPS */
Tom Herbert0a9627f2010-03-16 08:03:29 +00004624
4625/*
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004626 * Check if this softnet_data structure is another cpu one
4627 * If yes, queue it to our IPI list and return 1
4628 * If no, return 0
4629 */
4630static int rps_ipi_queued(struct softnet_data *sd)
4631{
4632#ifdef CONFIG_RPS
Christoph Lameter903ceff2014-08-17 12:30:35 -05004633 struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004634
4635 if (sd != mysd) {
4636 sd->rps_ipi_next = mysd->rps_ipi_list;
4637 mysd->rps_ipi_list = sd;
4638
4639 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4640 return 1;
4641 }
4642#endif /* CONFIG_RPS */
4643 return 0;
4644}
4645
Willem de Bruijn99bbc702013-05-20 04:02:32 +00004646#ifdef CONFIG_NET_FLOW_LIMIT
4647int netdev_flow_limit_table_len __read_mostly = (1 << 12);
4648#endif
4649
4650static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
4651{
4652#ifdef CONFIG_NET_FLOW_LIMIT
4653 struct sd_flow_limit *fl;
4654 struct softnet_data *sd;
4655 unsigned int old_flow, new_flow;
4656
4657 if (qlen < (netdev_max_backlog >> 1))
4658 return false;
4659
Christoph Lameter903ceff2014-08-17 12:30:35 -05004660 sd = this_cpu_ptr(&softnet_data);
Willem de Bruijn99bbc702013-05-20 04:02:32 +00004661
4662 rcu_read_lock();
4663 fl = rcu_dereference(sd->flow_limit);
4664 if (fl) {
Tom Herbert3958afa1b2013-12-15 22:12:06 -08004665 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
Willem de Bruijn99bbc702013-05-20 04:02:32 +00004666 old_flow = fl->history[fl->history_head];
4667 fl->history[fl->history_head] = new_flow;
4668
4669 fl->history_head++;
4670 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
4671
4672 if (likely(fl->buckets[old_flow]))
4673 fl->buckets[old_flow]--;
4674
4675 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
4676 fl->count++;
4677 rcu_read_unlock();
4678 return true;
4679 }
4680 }
4681 rcu_read_unlock();
4682#endif
4683 return false;
4684}
4685
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004686/*
Tom Herbert0a9627f2010-03-16 08:03:29 +00004687 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
4688 * queue (may be a remote CPU queue).
4689 */
Tom Herbertfec5e652010-04-16 16:01:27 -07004690static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
4691 unsigned int *qtail)
Tom Herbert0a9627f2010-03-16 08:03:29 +00004692{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004693 struct softnet_data *sd;
Tom Herbert0a9627f2010-03-16 08:03:29 +00004694 unsigned long flags;
Willem de Bruijn99bbc702013-05-20 04:02:32 +00004695 unsigned int qlen;
Tom Herbert0a9627f2010-03-16 08:03:29 +00004696
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004697 sd = &per_cpu(softnet_data, cpu);
Tom Herbert0a9627f2010-03-16 08:03:29 +00004698
4699 local_irq_save(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00004700
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004701 rps_lock(sd);
Julian Anastasove9e4dd32015-07-09 09:59:09 +03004702 if (!netif_running(skb->dev))
4703 goto drop;
Willem de Bruijn99bbc702013-05-20 04:02:32 +00004704 qlen = skb_queue_len(&sd->input_pkt_queue);
4705 if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
Li RongQinge008f3f2014-12-08 09:42:55 +08004706 if (qlen) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00004707enqueue:
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004708 __skb_queue_tail(&sd->input_pkt_queue, skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00004709 input_queue_tail_incr_save(sd, qtail);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004710 rps_unlock(sd);
Changli Gao152102c2010-03-30 20:16:22 +00004711 local_irq_restore(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00004712 return NET_RX_SUCCESS;
4713 }
4714
Eric Dumazetebda37c22010-05-06 23:51:21 +00004715 /* Schedule NAPI for backlog device
4716 * We can use non atomic operation since we own the queue lock
4717 */
4718 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004719 if (!rps_ipi_queued(sd))
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004720 ____napi_schedule(sd, &sd->backlog);
Tom Herbert0a9627f2010-03-16 08:03:29 +00004721 }
4722 goto enqueue;
4723 }
4724
Julian Anastasove9e4dd32015-07-09 09:59:09 +03004725drop:
Changli Gaodee42872010-05-02 05:42:16 +00004726 sd->dropped++;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004727 rps_unlock(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00004728
Tom Herbert0a9627f2010-03-16 08:03:29 +00004729 local_irq_restore(flags);
4730
Eric Dumazetcaf586e2010-09-30 21:06:55 +00004731 atomic_long_inc(&skb->dev->rx_dropped);
Tom Herbert0a9627f2010-03-16 08:03:29 +00004732 kfree_skb(skb);
4733 return NET_RX_DROP;
4734}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004735
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +01004736static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb)
4737{
4738 struct net_device *dev = skb->dev;
4739 struct netdev_rx_queue *rxqueue;
4740
4741 rxqueue = dev->_rx;
4742
4743 if (skb_rx_queue_recorded(skb)) {
4744 u16 index = skb_get_rx_queue(skb);
4745
4746 if (unlikely(index >= dev->real_num_rx_queues)) {
4747 WARN_ONCE(dev->real_num_rx_queues > 1,
4748 "%s received packet on queue %u, but number "
4749 "of RX queues is %u\n",
4750 dev->name, index, dev->real_num_rx_queues);
4751
4752 return rxqueue; /* Return first rxqueue */
4753 }
4754 rxqueue += index;
4755 }
4756 return rxqueue;
4757}
4758
Kumar Kartikeya Dwivedife21cb92021-07-02 16:48:21 +05304759u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,
4760 struct bpf_prog *xdp_prog)
John Fastabendd4455162017-07-17 09:26:45 -07004761{
Lorenzo Bianconibe9df4a2020-12-22 22:09:29 +01004762 void *orig_data, *orig_data_end, *hard_start;
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +01004763 struct netdev_rx_queue *rxqueue;
Martin Willi22b60342021-04-19 16:15:59 +02004764 bool orig_bcast, orig_host;
Lorenzo Bianconi43b51692020-12-22 22:09:28 +01004765 u32 mac_len, frame_sz;
Jesper Dangaard Brouer29724952018-10-09 12:04:43 +02004766 __be16 orig_eth_type;
4767 struct ethhdr *eth;
Kumar Kartikeya Dwivedife21cb92021-07-02 16:48:21 +05304768 u32 metalen, act;
Lorenzo Bianconibe9df4a2020-12-22 22:09:29 +01004769 int off;
John Fastabendd4455162017-07-17 09:26:45 -07004770
John Fastabendd4455162017-07-17 09:26:45 -07004771 /* The XDP program wants to see the packet starting at the MAC
4772 * header.
4773 */
4774 mac_len = skb->data - skb_mac_header(skb);
Lorenzo Bianconibe9df4a2020-12-22 22:09:29 +01004775 hard_start = skb->data - skb_headroom(skb);
Jesper Dangaard Brouera0757672020-05-14 12:49:28 +02004776
4777 /* SKB "head" area always have tailroom for skb_shared_info */
Lorenzo Bianconibe9df4a2020-12-22 22:09:29 +01004778 frame_sz = (void *)skb_end_pointer(skb) - hard_start;
Lorenzo Bianconi43b51692020-12-22 22:09:28 +01004779 frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
Jesper Dangaard Brouera0757672020-05-14 12:49:28 +02004780
Lorenzo Bianconibe9df4a2020-12-22 22:09:29 +01004781 rxqueue = netif_get_rxqueue(skb);
4782 xdp_init_buff(xdp, frame_sz, &rxqueue->xdp_rxq);
4783 xdp_prepare_buff(xdp, hard_start, skb_headroom(skb) - mac_len,
4784 skb_headlen(skb) + mac_len, true);
Björn Töpel02671e22018-05-02 13:01:30 +02004785
4786 orig_data_end = xdp->data_end;
4787 orig_data = xdp->data;
Jesper Dangaard Brouer29724952018-10-09 12:04:43 +02004788 eth = (struct ethhdr *)xdp->data;
Martin Willi22b60342021-04-19 16:15:59 +02004789 orig_host = ether_addr_equal_64bits(eth->h_dest, skb->dev->dev_addr);
Jesper Dangaard Brouer29724952018-10-09 12:04:43 +02004790 orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest);
4791 orig_eth_type = eth->h_proto;
John Fastabendd4455162017-07-17 09:26:45 -07004792
Björn Töpel02671e22018-05-02 13:01:30 +02004793 act = bpf_prog_run_xdp(xdp_prog, xdp);
John Fastabendd4455162017-07-17 09:26:45 -07004794
Jesper Dangaard Brouer065af352019-08-01 20:00:31 +02004795 /* check if bpf_xdp_adjust_head was used */
Björn Töpel02671e22018-05-02 13:01:30 +02004796 off = xdp->data - orig_data;
Jesper Dangaard Brouer065af352019-08-01 20:00:31 +02004797 if (off) {
4798 if (off > 0)
4799 __skb_pull(skb, off);
4800 else if (off < 0)
4801 __skb_push(skb, -off);
4802
4803 skb->mac_header += off;
4804 skb_reset_network_header(skb);
4805 }
John Fastabendd4455162017-07-17 09:26:45 -07004806
Jesper Dangaard Brouera0757672020-05-14 12:49:28 +02004807 /* check if bpf_xdp_adjust_tail was used */
4808 off = xdp->data_end - orig_data_end;
Nikita V. Shirokovf7613122018-04-25 07:15:03 -07004809 if (off != 0) {
Björn Töpel02671e22018-05-02 13:01:30 +02004810 skb_set_tail_pointer(skb, xdp->data_end - xdp->data);
Jesper Dangaard Brouera0757672020-05-14 12:49:28 +02004811 skb->len += off; /* positive on grow, negative on shrink */
Nikita V. Shirokovf7613122018-04-25 07:15:03 -07004812 }
Nikita V. Shirokov198d83b2018-04-17 21:42:14 -07004813
Jesper Dangaard Brouer29724952018-10-09 12:04:43 +02004814 /* check if XDP changed eth hdr such SKB needs update */
4815 eth = (struct ethhdr *)xdp->data;
4816 if ((orig_eth_type != eth->h_proto) ||
Martin Willi22b60342021-04-19 16:15:59 +02004817 (orig_host != ether_addr_equal_64bits(eth->h_dest,
4818 skb->dev->dev_addr)) ||
Jesper Dangaard Brouer29724952018-10-09 12:04:43 +02004819 (orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) {
4820 __skb_push(skb, ETH_HLEN);
Martin Willi22b60342021-04-19 16:15:59 +02004821 skb->pkt_type = PACKET_HOST;
Jesper Dangaard Brouer29724952018-10-09 12:04:43 +02004822 skb->protocol = eth_type_trans(skb, skb->dev);
4823 }
4824
Kumar Kartikeya Dwivedife21cb92021-07-02 16:48:21 +05304825 /* Redirect/Tx gives L2 packet, code that will reuse skb must __skb_pull
4826 * before calling us again on redirect path. We do not call do_redirect
4827 * as we leave that up to the caller.
4828 *
4829 * Caller is responsible for managing lifetime of skb (i.e. calling
4830 * kfree_skb in response to actions it cannot handle/XDP_DROP).
4831 */
John Fastabendd4455162017-07-17 09:26:45 -07004832 switch (act) {
John Fastabend6103aa92017-07-17 09:27:50 -07004833 case XDP_REDIRECT:
John Fastabendd4455162017-07-17 09:26:45 -07004834 case XDP_TX:
4835 __skb_push(skb, mac_len);
John Fastabendd4455162017-07-17 09:26:45 -07004836 break;
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02004837 case XDP_PASS:
Björn Töpel02671e22018-05-02 13:01:30 +02004838 metalen = xdp->data - xdp->data_meta;
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02004839 if (metalen)
4840 skb_metadata_set(skb, metalen);
4841 break;
Kumar Kartikeya Dwivedife21cb92021-07-02 16:48:21 +05304842 }
4843
4844 return act;
4845}
4846
4847static u32 netif_receive_generic_xdp(struct sk_buff *skb,
4848 struct xdp_buff *xdp,
4849 struct bpf_prog *xdp_prog)
4850{
4851 u32 act = XDP_DROP;
4852
4853 /* Reinjected packets coming from act_mirred or similar should
4854 * not get XDP generic processing.
4855 */
4856 if (skb_is_redirected(skb))
4857 return XDP_PASS;
4858
4859 /* XDP packets must be linear and must have sufficient headroom
4860 * of XDP_PACKET_HEADROOM bytes. This is the guarantee that also
4861 * native XDP provides, thus we need to do it here as well.
4862 */
4863 if (skb_cloned(skb) || skb_is_nonlinear(skb) ||
4864 skb_headroom(skb) < XDP_PACKET_HEADROOM) {
4865 int hroom = XDP_PACKET_HEADROOM - skb_headroom(skb);
4866 int troom = skb->tail + skb->data_len - skb->end;
4867
4868 /* In case we have to go down the path and also linearize,
4869 * then lets do the pskb_expand_head() work just once here.
4870 */
4871 if (pskb_expand_head(skb,
4872 hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0,
4873 troom > 0 ? troom + 128 : 0, GFP_ATOMIC))
4874 goto do_drop;
4875 if (skb_linearize(skb))
4876 goto do_drop;
4877 }
4878
4879 act = bpf_prog_run_generic_xdp(skb, xdp, xdp_prog);
4880 switch (act) {
4881 case XDP_REDIRECT:
4882 case XDP_TX:
4883 case XDP_PASS:
4884 break;
John Fastabendd4455162017-07-17 09:26:45 -07004885 default:
4886 bpf_warn_invalid_xdp_action(act);
Gustavo A. R. Silvadf561f662020-08-23 17:36:59 -05004887 fallthrough;
John Fastabendd4455162017-07-17 09:26:45 -07004888 case XDP_ABORTED:
4889 trace_xdp_exception(skb->dev, xdp_prog, act);
Gustavo A. R. Silvadf561f662020-08-23 17:36:59 -05004890 fallthrough;
John Fastabendd4455162017-07-17 09:26:45 -07004891 case XDP_DROP:
4892 do_drop:
4893 kfree_skb(skb);
4894 break;
4895 }
4896
4897 return act;
4898}
4899
4900/* When doing generic XDP we have to bypass the qdisc layer and the
4901 * network taps in order to match in-driver-XDP behavior.
4902 */
Jason Wang7c497472017-08-11 19:41:17 +08004903void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)
John Fastabendd4455162017-07-17 09:26:45 -07004904{
4905 struct net_device *dev = skb->dev;
4906 struct netdev_queue *txq;
4907 bool free_skb = true;
4908 int cpu, rc;
4909
Paolo Abeni4bd97d52019-03-20 11:02:04 +01004910 txq = netdev_core_pick_tx(dev, skb, NULL);
John Fastabendd4455162017-07-17 09:26:45 -07004911 cpu = smp_processor_id();
4912 HARD_TX_LOCK(dev, txq, cpu);
4913 if (!netif_xmit_stopped(txq)) {
4914 rc = netdev_start_xmit(skb, dev, txq, 0);
4915 if (dev_xmit_complete(rc))
4916 free_skb = false;
4917 }
4918 HARD_TX_UNLOCK(dev, txq);
4919 if (free_skb) {
4920 trace_xdp_exception(dev, xdp_prog, XDP_TX);
4921 kfree_skb(skb);
4922 }
4923}
4924
Davidlohr Bueso02786472018-05-08 09:07:02 -07004925static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key);
John Fastabendd4455162017-07-17 09:26:45 -07004926
Jason Wang7c497472017-08-11 19:41:17 +08004927int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb)
John Fastabendd4455162017-07-17 09:26:45 -07004928{
John Fastabendd4455162017-07-17 09:26:45 -07004929 if (xdp_prog) {
Björn Töpel02671e22018-05-02 13:01:30 +02004930 struct xdp_buff xdp;
4931 u32 act;
John Fastabend6103aa92017-07-17 09:27:50 -07004932 int err;
John Fastabendd4455162017-07-17 09:26:45 -07004933
Björn Töpel02671e22018-05-02 13:01:30 +02004934 act = netif_receive_generic_xdp(skb, &xdp, xdp_prog);
John Fastabendd4455162017-07-17 09:26:45 -07004935 if (act != XDP_PASS) {
John Fastabend6103aa92017-07-17 09:27:50 -07004936 switch (act) {
4937 case XDP_REDIRECT:
Jesper Dangaard Brouer2facaad2017-08-24 12:33:08 +02004938 err = xdp_do_generic_redirect(skb->dev, skb,
Björn Töpel02671e22018-05-02 13:01:30 +02004939 &xdp, xdp_prog);
John Fastabend6103aa92017-07-17 09:27:50 -07004940 if (err)
4941 goto out_redir;
Björn Töpel02671e22018-05-02 13:01:30 +02004942 break;
John Fastabend6103aa92017-07-17 09:27:50 -07004943 case XDP_TX:
John Fastabendd4455162017-07-17 09:26:45 -07004944 generic_xdp_tx(skb, xdp_prog);
John Fastabend6103aa92017-07-17 09:27:50 -07004945 break;
4946 }
John Fastabendd4455162017-07-17 09:26:45 -07004947 return XDP_DROP;
4948 }
4949 }
4950 return XDP_PASS;
John Fastabend6103aa92017-07-17 09:27:50 -07004951out_redir:
John Fastabend6103aa92017-07-17 09:27:50 -07004952 kfree_skb(skb);
4953 return XDP_DROP;
John Fastabendd4455162017-07-17 09:26:45 -07004954}
Jason Wang7c497472017-08-11 19:41:17 +08004955EXPORT_SYMBOL_GPL(do_xdp_generic);
John Fastabendd4455162017-07-17 09:26:45 -07004956
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004957static int netif_rx_internal(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004958{
Eric Dumazetb0e28f12010-04-15 00:14:07 -07004959 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004960
Eric Dumazet588f0332011-11-15 04:12:55 +00004961 net_timestamp_check(netdev_tstamp_prequeue, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004962
Koki Sanagicf66ba52010-08-23 18:45:02 +09004963 trace_netif_rx(skb);
John Fastabendd4455162017-07-17 09:26:45 -07004964
Eric Dumazetdf334542010-03-24 19:13:54 +00004965#ifdef CONFIG_RPS
Eric Dumazetdc053602019-03-22 08:56:38 -07004966 if (static_branch_unlikely(&rps_needed)) {
Tom Herbertfec5e652010-04-16 16:01:27 -07004967 struct rps_dev_flow voidflow, *rflow = &voidflow;
Eric Dumazetb0e28f12010-04-15 00:14:07 -07004968 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004969
Changli Gaocece1942010-08-07 20:35:43 -07004970 preempt_disable();
Eric Dumazetb0e28f12010-04-15 00:14:07 -07004971 rcu_read_lock();
Tom Herbertfec5e652010-04-16 16:01:27 -07004972
4973 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Eric Dumazetb0e28f12010-04-15 00:14:07 -07004974 if (cpu < 0)
4975 cpu = smp_processor_id();
Tom Herbertfec5e652010-04-16 16:01:27 -07004976
4977 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4978
Eric Dumazetb0e28f12010-04-15 00:14:07 -07004979 rcu_read_unlock();
Changli Gaocece1942010-08-07 20:35:43 -07004980 preempt_enable();
Eric Dumazetadc93002011-11-17 03:13:26 +00004981 } else
4982#endif
Tom Herbertfec5e652010-04-16 16:01:27 -07004983 {
4984 unsigned int qtail;
tchardingf4563a72017-02-09 17:56:07 +11004985
Tom Herbertfec5e652010-04-16 16:01:27 -07004986 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
4987 put_cpu();
4988 }
Eric Dumazetb0e28f12010-04-15 00:14:07 -07004989 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004990}
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004991
4992/**
4993 * netif_rx - post buffer to the network code
4994 * @skb: buffer to post
4995 *
4996 * This function receives a packet from a device driver and queues it for
4997 * the upper (protocol) levels to process. It always succeeds. The buffer
4998 * may be dropped during processing for congestion control or by the
4999 * protocol layers.
5000 *
5001 * return values:
5002 * NET_RX_SUCCESS (no congestion)
5003 * NET_RX_DROP (packet was dropped)
5004 *
5005 */
5006
5007int netif_rx(struct sk_buff *skb)
5008{
Geneviève Bastienb0e3f1b2018-11-27 12:52:39 -05005009 int ret;
5010
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00005011 trace_netif_rx_entry(skb);
5012
Geneviève Bastienb0e3f1b2018-11-27 12:52:39 -05005013 ret = netif_rx_internal(skb);
5014 trace_netif_rx_exit(ret);
5015
5016 return ret;
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00005017}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005018EXPORT_SYMBOL(netif_rx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005019
5020int netif_rx_ni(struct sk_buff *skb)
5021{
5022 int err;
5023
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00005024 trace_netif_rx_ni_entry(skb);
5025
Linus Torvalds1da177e2005-04-16 15:20:36 -07005026 preempt_disable();
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00005027 err = netif_rx_internal(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005028 if (local_softirq_pending())
5029 do_softirq();
5030 preempt_enable();
Geneviève Bastienb0e3f1b2018-11-27 12:52:39 -05005031 trace_netif_rx_ni_exit(err);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005032
5033 return err;
5034}
Linus Torvalds1da177e2005-04-16 15:20:36 -07005035EXPORT_SYMBOL(netif_rx_ni);
5036
Sebastian Andrzej Siewiorc11171a2020-09-29 22:25:12 +02005037int netif_rx_any_context(struct sk_buff *skb)
5038{
5039 /*
5040 * If invoked from contexts which do not invoke bottom half
5041 * processing either at return from interrupt or when softrqs are
5042 * reenabled, use netif_rx_ni() which invokes bottomhalf processing
5043 * directly.
5044 */
5045 if (in_interrupt())
5046 return netif_rx(skb);
5047 else
5048 return netif_rx_ni(skb);
5049}
5050EXPORT_SYMBOL(netif_rx_any_context);
5051
Emese Revfy0766f782016-06-20 20:42:34 +02005052static __latent_entropy void net_tx_action(struct softirq_action *h)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005053{
Christoph Lameter903ceff2014-08-17 12:30:35 -05005054 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005055
5056 if (sd->completion_queue) {
5057 struct sk_buff *clist;
5058
5059 local_irq_disable();
5060 clist = sd->completion_queue;
5061 sd->completion_queue = NULL;
5062 local_irq_enable();
5063
5064 while (clist) {
5065 struct sk_buff *skb = clist;
tchardingf4563a72017-02-09 17:56:07 +11005066
Linus Torvalds1da177e2005-04-16 15:20:36 -07005067 clist = clist->next;
5068
Reshetova, Elena63354792017-06-30 13:07:58 +03005069 WARN_ON(refcount_read(&skb->users));
Eric Dumazete6247022013-12-05 04:45:08 -08005070 if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
5071 trace_consume_skb(skb);
5072 else
5073 trace_kfree_skb(skb, net_tx_action);
Jesper Dangaard Brouer15fad712016-02-08 13:15:04 +01005074
5075 if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
5076 __kfree_skb(skb);
5077 else
5078 __kfree_skb_defer(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005079 }
5080 }
5081
5082 if (sd->output_queue) {
David S. Miller37437bb2008-07-16 02:15:04 -07005083 struct Qdisc *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005084
5085 local_irq_disable();
5086 head = sd->output_queue;
5087 sd->output_queue = NULL;
Changli Gaoa9cbd582010-04-26 23:06:24 +00005088 sd->output_queue_tailp = &sd->output_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005089 local_irq_enable();
5090
Yunsheng Lin102b55e2021-05-14 11:17:00 +08005091 rcu_read_lock();
5092
Linus Torvalds1da177e2005-04-16 15:20:36 -07005093 while (head) {
David S. Miller37437bb2008-07-16 02:15:04 -07005094 struct Qdisc *q = head;
John Fastabend6b3ba912017-12-07 09:54:25 -08005095 spinlock_t *root_lock = NULL;
David S. Miller37437bb2008-07-16 02:15:04 -07005096
Linus Torvalds1da177e2005-04-16 15:20:36 -07005097 head = head->next_sched;
5098
Eric Dumazet3bcb8462016-06-04 20:02:28 -07005099 /* We need to make sure head->next_sched is read
5100 * before clearing __QDISC_STATE_SCHED
5101 */
5102 smp_mb__before_atomic();
Yunsheng Lin102b55e2021-05-14 11:17:00 +08005103
5104 if (!(q->flags & TCQ_F_NOLOCK)) {
5105 root_lock = qdisc_lock(q);
5106 spin_lock(root_lock);
5107 } else if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED,
5108 &q->state))) {
5109 /* There is a synchronize_net() between
5110 * STATE_DEACTIVATED flag being set and
5111 * qdisc_reset()/some_qdisc_is_busy() in
5112 * dev_deactivate(), so we can safely bail out
5113 * early here to avoid data race between
5114 * qdisc_deactivate() and some_qdisc_is_busy()
5115 * for lockless qdisc.
5116 */
5117 clear_bit(__QDISC_STATE_SCHED, &q->state);
5118 continue;
5119 }
5120
Eric Dumazet3bcb8462016-06-04 20:02:28 -07005121 clear_bit(__QDISC_STATE_SCHED, &q->state);
5122 qdisc_run(q);
John Fastabend6b3ba912017-12-07 09:54:25 -08005123 if (root_lock)
5124 spin_unlock(root_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005125 }
Yunsheng Lin102b55e2021-05-14 11:17:00 +08005126
5127 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005128 }
Steffen Klassertf53c7232017-12-20 10:41:36 +01005129
5130 xfrm_dev_backlog(sd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005131}
5132
Javier Martinez Canillas181402a2016-09-09 08:43:15 -04005133#if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
Michał Mirosławda678292009-06-05 05:35:28 +00005134/* This hook is defined here for ATM LANE */
5135int (*br_fdb_test_addr_hook)(struct net_device *dev,
5136 unsigned char *addr) __read_mostly;
Stephen Hemminger4fb019a2009-09-11 11:50:08 -07005137EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
Michał Mirosławda678292009-06-05 05:35:28 +00005138#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07005139
Daniel Borkmann1f211a12016-01-07 22:29:47 +01005140static inline struct sk_buff *
5141sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
Daniel Borkmann9aa12062020-10-11 01:40:02 +02005142 struct net_device *orig_dev, bool *another)
Herbert Xuf697c3e2007-10-14 00:38:47 -07005143{
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02005144#ifdef CONFIG_NET_CLS_ACT
Jiri Pirko46209402017-11-03 11:46:25 +01005145 struct mini_Qdisc *miniq = rcu_dereference_bh(skb->dev->miniq_ingress);
Daniel Borkmannd2788d32015-05-09 22:51:32 +02005146 struct tcf_result cl_res;
Eric Dumazet24824a02010-10-02 06:11:55 +00005147
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02005148 /* If there's at least one ingress present somewhere (so
5149 * we get here via enabled static key), remaining devices
5150 * that are not configured with an ingress qdisc will bail
Daniel Borkmannd2788d32015-05-09 22:51:32 +02005151 * out here.
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02005152 */
Jiri Pirko46209402017-11-03 11:46:25 +01005153 if (!miniq)
Daniel Borkmann45771392015-04-10 23:07:54 +02005154 return skb;
Jiri Pirko46209402017-11-03 11:46:25 +01005155
Herbert Xuf697c3e2007-10-14 00:38:47 -07005156 if (*pt_prev) {
5157 *ret = deliver_skb(skb, *pt_prev, orig_dev);
5158 *pt_prev = NULL;
Herbert Xuf697c3e2007-10-14 00:38:47 -07005159 }
5160
Florian Westphal33654952015-05-14 00:36:28 +02005161 qdisc_skb_cb(skb)->pkt_len = skb->len;
wenxuaadaca92020-11-25 12:01:21 +08005162 qdisc_skb_cb(skb)->mru = 0;
wenxu7baf2422021-01-19 16:31:50 +08005163 qdisc_skb_cb(skb)->post_ct = false;
Willem de Bruijn8dc07fd2017-01-07 17:06:37 -05005164 skb->tc_at_ingress = 1;
Jiri Pirko46209402017-11-03 11:46:25 +01005165 mini_qdisc_bstats_cpu_update(miniq, skb);
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02005166
Paul Blakey7d17c542020-02-16 12:01:22 +02005167 switch (tcf_classify_ingress(skb, miniq->block, miniq->filter_list,
5168 &cl_res, false)) {
Daniel Borkmannd2788d32015-05-09 22:51:32 +02005169 case TC_ACT_OK:
5170 case TC_ACT_RECLASSIFY:
5171 skb->tc_index = TC_H_MIN(cl_res.classid);
5172 break;
5173 case TC_ACT_SHOT:
Jiri Pirko46209402017-11-03 11:46:25 +01005174 mini_qdisc_qstats_cpu_drop(miniq);
Eric Dumazet8a3a4c62016-05-06 15:55:50 -07005175 kfree_skb(skb);
5176 return NULL;
Daniel Borkmannd2788d32015-05-09 22:51:32 +02005177 case TC_ACT_STOLEN:
5178 case TC_ACT_QUEUED:
Jiri Pirkoe25ea212017-06-06 14:12:02 +02005179 case TC_ACT_TRAP:
Eric Dumazet8a3a4c62016-05-06 15:55:50 -07005180 consume_skb(skb);
Daniel Borkmannd2788d32015-05-09 22:51:32 +02005181 return NULL;
Alexei Starovoitov27b29f62015-09-15 23:05:43 -07005182 case TC_ACT_REDIRECT:
5183 /* skb_mac_header check was done by cls/act_bpf, so
5184 * we can safely push the L2 header back before
5185 * redirecting to another netdev
5186 */
5187 __skb_push(skb, skb->mac_len);
Daniel Borkmann9aa12062020-10-11 01:40:02 +02005188 if (skb_do_redirect(skb) == -EAGAIN) {
5189 __skb_pull(skb, skb->mac_len);
5190 *another = true;
5191 break;
5192 }
Alexei Starovoitov27b29f62015-09-15 23:05:43 -07005193 return NULL;
John Hurley720f22f2019-06-24 23:13:35 +01005194 case TC_ACT_CONSUMED:
Paolo Abenicd11b1642018-07-30 14:30:44 +02005195 return NULL;
Daniel Borkmannd2788d32015-05-09 22:51:32 +02005196 default:
5197 break;
Herbert Xuf697c3e2007-10-14 00:38:47 -07005198 }
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02005199#endif /* CONFIG_NET_CLS_ACT */
Herbert Xuf697c3e2007-10-14 00:38:47 -07005200 return skb;
5201}
Linus Torvalds1da177e2005-04-16 15:20:36 -07005202
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00005203/**
Mahesh Bandewar24b27fc2016-09-01 22:18:34 -07005204 * netdev_is_rx_handler_busy - check if receive handler is registered
5205 * @dev: device to check
5206 *
5207 * Check if a receive handler is already registered for a given device.
5208 * Return true if there one.
5209 *
5210 * The caller must hold the rtnl_mutex.
5211 */
5212bool netdev_is_rx_handler_busy(struct net_device *dev)
5213{
5214 ASSERT_RTNL();
5215 return dev && rtnl_dereference(dev->rx_handler);
5216}
5217EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
5218
5219/**
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00005220 * netdev_rx_handler_register - register receive handler
5221 * @dev: device to register a handler for
5222 * @rx_handler: receive handler to register
Jiri Pirko93e2c322010-06-10 03:34:59 +00005223 * @rx_handler_data: data pointer that is used by rx handler
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00005224 *
Masanari Iidae2278672014-02-18 22:54:36 +09005225 * Register a receive handler for a device. This handler will then be
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00005226 * called from __netif_receive_skb. A negative errno code is returned
5227 * on a failure.
5228 *
5229 * The caller must hold the rtnl_mutex.
Jiri Pirko8a4eb572011-03-12 03:14:39 +00005230 *
5231 * For a general description of rx_handler, see enum rx_handler_result.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00005232 */
5233int netdev_rx_handler_register(struct net_device *dev,
Jiri Pirko93e2c322010-06-10 03:34:59 +00005234 rx_handler_func_t *rx_handler,
5235 void *rx_handler_data)
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00005236{
Mahesh Bandewar1b7cd002017-01-18 15:02:49 -08005237 if (netdev_is_rx_handler_busy(dev))
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00005238 return -EBUSY;
5239
Paolo Abenif54262502018-03-09 10:39:24 +01005240 if (dev->priv_flags & IFF_NO_RX_HANDLER)
5241 return -EINVAL;
5242
Eric Dumazet00cfec32013-03-29 03:01:22 +00005243 /* Note: rx_handler_data must be set before rx_handler */
Jiri Pirko93e2c322010-06-10 03:34:59 +00005244 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00005245 rcu_assign_pointer(dev->rx_handler, rx_handler);
5246
5247 return 0;
5248}
5249EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
5250
5251/**
5252 * netdev_rx_handler_unregister - unregister receive handler
5253 * @dev: device to unregister a handler from
5254 *
Kusanagi Kouichi166ec362013-03-18 02:59:52 +00005255 * Unregister a receive handler from a device.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00005256 *
5257 * The caller must hold the rtnl_mutex.
5258 */
5259void netdev_rx_handler_unregister(struct net_device *dev)
5260{
5261
5262 ASSERT_RTNL();
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00005263 RCU_INIT_POINTER(dev->rx_handler, NULL);
Eric Dumazet00cfec32013-03-29 03:01:22 +00005264 /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
5265 * section has a guarantee to see a non NULL rx_handler_data
5266 * as well.
5267 */
5268 synchronize_net();
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00005269 RCU_INIT_POINTER(dev->rx_handler_data, NULL);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00005270}
5271EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
5272
Mel Gormanb4b9e352012-07-31 16:44:26 -07005273/*
5274 * Limit the use of PFMEMALLOC reserves to those protocols that implement
5275 * the special handling of PFMEMALLOC skbs.
5276 */
5277static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
5278{
5279 switch (skb->protocol) {
Joe Perches2b8837a2014-03-12 10:04:17 -07005280 case htons(ETH_P_ARP):
5281 case htons(ETH_P_IP):
5282 case htons(ETH_P_IPV6):
5283 case htons(ETH_P_8021Q):
5284 case htons(ETH_P_8021AD):
Mel Gormanb4b9e352012-07-31 16:44:26 -07005285 return true;
5286 default:
5287 return false;
5288 }
5289}
5290
Pablo Neirae687ad62015-05-13 18:19:38 +02005291static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
5292 int *ret, struct net_device *orig_dev)
5293{
5294 if (nf_hook_ingress_active(skb)) {
Aaron Conole2c1e2702016-09-21 11:35:03 -04005295 int ingress_retval;
5296
Pablo Neirae687ad62015-05-13 18:19:38 +02005297 if (*pt_prev) {
5298 *ret = deliver_skb(skb, *pt_prev, orig_dev);
5299 *pt_prev = NULL;
5300 }
5301
Aaron Conole2c1e2702016-09-21 11:35:03 -04005302 rcu_read_lock();
5303 ingress_retval = nf_hook_ingress(skb);
5304 rcu_read_unlock();
5305 return ingress_retval;
Pablo Neirae687ad62015-05-13 18:19:38 +02005306 }
5307 return 0;
5308}
Pablo Neirae687ad62015-05-13 18:19:38 +02005309
Boris Sukholitkoc0bbbdc2020-05-19 10:32:37 +03005310static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc,
Edward Cree88eb1942018-07-02 16:13:56 +01005311 struct packet_type **ppt_prev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005312{
5313 struct packet_type *ptype, *pt_prev;
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00005314 rx_handler_func_t *rx_handler;
Boris Sukholitkoc0bbbdc2020-05-19 10:32:37 +03005315 struct sk_buff *skb = *pskb;
David S. Millerf2ccd8f2005-08-09 19:34:12 -07005316 struct net_device *orig_dev;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00005317 bool deliver_exact = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005318 int ret = NET_RX_DROP;
Al Viro252e33462006-11-14 20:48:11 -08005319 __be16 type;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005320
Eric Dumazet588f0332011-11-15 04:12:55 +00005321 net_timestamp_check(!netdev_tstamp_prequeue, skb);
Eric Dumazet81bbb3d2009-09-30 16:42:42 -07005322
Koki Sanagicf66ba52010-08-23 18:45:02 +09005323 trace_netif_receive_skb(skb);
Patrick McHardy9b22ea52008-11-04 14:49:57 -08005324
Joe Eykholtcc9bd5c2008-07-02 18:22:00 -07005325 orig_dev = skb->dev;
Jiri Pirko1765a572011-02-12 06:48:36 +00005326
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07005327 skb_reset_network_header(skb);
Eric Dumazetfda55ec2013-01-07 09:28:21 +00005328 if (!skb_transport_header_was_set(skb))
5329 skb_reset_transport_header(skb);
Jiri Pirko0b5c9db2011-06-10 06:56:58 +00005330 skb_reset_mac_len(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005331
5332 pt_prev = NULL;
5333
David S. Miller63d8ea72011-02-28 10:48:59 -08005334another_round:
David S. Millerb6858172012-07-23 16:27:54 -07005335 skb->skb_iif = skb->dev->ifindex;
David S. Miller63d8ea72011-02-28 10:48:59 -08005336
5337 __this_cpu_inc(softnet_data.processed);
5338
Stephen Hemminger458bf2f2019-05-28 11:47:31 -07005339 if (static_branch_unlikely(&generic_xdp_needed_key)) {
5340 int ret2;
5341
Sebastian Andrzej Siewior2b4cd142021-06-17 09:38:17 +02005342 migrate_disable();
Stephen Hemminger458bf2f2019-05-28 11:47:31 -07005343 ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
Sebastian Andrzej Siewior2b4cd142021-06-17 09:38:17 +02005344 migrate_enable();
Stephen Hemminger458bf2f2019-05-28 11:47:31 -07005345
Boris Sukholitkoc0bbbdc2020-05-19 10:32:37 +03005346 if (ret2 != XDP_PASS) {
5347 ret = NET_RX_DROP;
5348 goto out;
5349 }
Stephen Hemminger458bf2f2019-05-28 11:47:31 -07005350 }
5351
Menglong Dong324cefa2021-01-11 02:42:21 -08005352 if (eth_type_vlan(skb->protocol)) {
Vlad Yasevich0d5501c2014-08-08 14:42:13 -04005353 skb = skb_vlan_untag(skb);
Jiri Pirkobcc6d472011-04-07 19:48:33 +00005354 if (unlikely(!skb))
Julian Anastasov2c17d272015-07-09 09:59:10 +03005355 goto out;
Jiri Pirkobcc6d472011-04-07 19:48:33 +00005356 }
5357
Willem de Bruijne7246e12017-01-07 17:06:35 -05005358 if (skb_skip_tc_classify(skb))
5359 goto skip_classify;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005360
David S. Miller9754e292013-02-14 15:57:38 -05005361 if (pfmemalloc)
Mel Gormanb4b9e352012-07-31 16:44:26 -07005362 goto skip_taps;
5363
Linus Torvalds1da177e2005-04-16 15:20:36 -07005364 list_for_each_entry_rcu(ptype, &ptype_all, list) {
Salam Noureddine7866a622015-01-27 11:35:48 -08005365 if (pt_prev)
5366 ret = deliver_skb(skb, pt_prev, orig_dev);
5367 pt_prev = ptype;
5368 }
5369
5370 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
5371 if (pt_prev)
5372 ret = deliver_skb(skb, pt_prev, orig_dev);
5373 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005374 }
5375
Mel Gormanb4b9e352012-07-31 16:44:26 -07005376skip_taps:
Pablo Neira1cf519002015-05-13 18:19:37 +02005377#ifdef CONFIG_NET_INGRESS
Davidlohr Buesoaabf6772018-05-08 09:07:00 -07005378 if (static_branch_unlikely(&ingress_needed_key)) {
Daniel Borkmann9aa12062020-10-11 01:40:02 +02005379 bool another = false;
5380
5381 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev,
5382 &another);
5383 if (another)
5384 goto another_round;
Daniel Borkmann45771392015-04-10 23:07:54 +02005385 if (!skb)
Julian Anastasov2c17d272015-07-09 09:59:10 +03005386 goto out;
Pablo Neirae687ad62015-05-13 18:19:38 +02005387
5388 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
Julian Anastasov2c17d272015-07-09 09:59:10 +03005389 goto out;
Daniel Borkmann45771392015-04-10 23:07:54 +02005390 }
Pablo Neira1cf519002015-05-13 18:19:37 +02005391#endif
Pablo Neira Ayuso2c646052020-03-25 13:47:18 +01005392 skb_reset_redirect(skb);
Willem de Bruijne7246e12017-01-07 17:06:35 -05005393skip_classify:
David S. Miller9754e292013-02-14 15:57:38 -05005394 if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07005395 goto drop;
5396
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01005397 if (skb_vlan_tag_present(skb)) {
John Fastabend24257172011-10-10 09:16:41 +00005398 if (pt_prev) {
5399 ret = deliver_skb(skb, pt_prev, orig_dev);
5400 pt_prev = NULL;
5401 }
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00005402 if (vlan_do_receive(&skb))
John Fastabend24257172011-10-10 09:16:41 +00005403 goto another_round;
5404 else if (unlikely(!skb))
Julian Anastasov2c17d272015-07-09 09:59:10 +03005405 goto out;
John Fastabend24257172011-10-10 09:16:41 +00005406 }
5407
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00005408 rx_handler = rcu_dereference(skb->dev->rx_handler);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00005409 if (rx_handler) {
5410 if (pt_prev) {
5411 ret = deliver_skb(skb, pt_prev, orig_dev);
5412 pt_prev = NULL;
5413 }
Jiri Pirko8a4eb572011-03-12 03:14:39 +00005414 switch (rx_handler(&skb)) {
5415 case RX_HANDLER_CONSUMED:
Cristian Bercaru3bc1b1a2013-03-08 07:03:38 +00005416 ret = NET_RX_SUCCESS;
Julian Anastasov2c17d272015-07-09 09:59:10 +03005417 goto out;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00005418 case RX_HANDLER_ANOTHER:
David S. Miller63d8ea72011-02-28 10:48:59 -08005419 goto another_round;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00005420 case RX_HANDLER_EXACT:
5421 deliver_exact = true;
Gustavo A. R. Silvab1866bf2021-03-09 23:42:43 -06005422 break;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00005423 case RX_HANDLER_PASS:
5424 break;
5425 default:
5426 BUG();
5427 }
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00005428 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005429
Vladimir Olteanb14a9fc2020-09-12 02:26:07 +03005430 if (unlikely(skb_vlan_tag_present(skb)) && !netdev_uses_dsa(skb->dev)) {
Govindarajulu Varadarajan36b2f612019-06-14 06:13:54 -07005431check_vlan_id:
5432 if (skb_vlan_tag_get_id(skb)) {
5433 /* Vlan id is non 0 and vlan_do_receive() above couldn't
5434 * find vlan device.
5435 */
Eric Dumazetd4b812d2013-07-18 07:19:26 -07005436 skb->pkt_type = PACKET_OTHERHOST;
Menglong Dong324cefa2021-01-11 02:42:21 -08005437 } else if (eth_type_vlan(skb->protocol)) {
Govindarajulu Varadarajan36b2f612019-06-14 06:13:54 -07005438 /* Outer header is 802.1P with vlan 0, inner header is
5439 * 802.1Q or 802.1AD and vlan_do_receive() above could
5440 * not find vlan dev for vlan id 0.
5441 */
5442 __vlan_hwaccel_clear_tag(skb);
5443 skb = skb_vlan_untag(skb);
5444 if (unlikely(!skb))
5445 goto out;
5446 if (vlan_do_receive(&skb))
5447 /* After stripping off 802.1P header with vlan 0
5448 * vlan dev is found for inner header.
5449 */
5450 goto another_round;
5451 else if (unlikely(!skb))
5452 goto out;
5453 else
5454 /* We have stripped outer 802.1P vlan 0 header.
5455 * But could not find vlan dev.
5456 * check again for vlan id to set OTHERHOST.
5457 */
5458 goto check_vlan_id;
5459 }
Eric Dumazetd4b812d2013-07-18 07:19:26 -07005460 /* Note: we might in the future use prio bits
5461 * and set skb->priority like in vlan_do_receive()
5462 * For the time being, just ignore Priority Code Point
5463 */
Michał Mirosławb18175242018-11-09 00:18:02 +01005464 __vlan_hwaccel_clear_tag(skb);
Eric Dumazetd4b812d2013-07-18 07:19:26 -07005465 }
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00005466
Linus Torvalds1da177e2005-04-16 15:20:36 -07005467 type = skb->protocol;
Salam Noureddine7866a622015-01-27 11:35:48 -08005468
5469 /* deliver only exact match when indicated */
5470 if (likely(!deliver_exact)) {
5471 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5472 &ptype_base[ntohs(type) &
5473 PTYPE_HASH_MASK]);
5474 }
5475
5476 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5477 &orig_dev->ptype_specific);
5478
5479 if (unlikely(skb->dev != orig_dev)) {
5480 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5481 &skb->dev->ptype_specific);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005482 }
5483
5484 if (pt_prev) {
Willem de Bruijn1f8b9772017-08-03 16:29:41 -04005485 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
Michael S. Tsirkin0e698bf2012-09-15 22:44:16 +00005486 goto drop;
Edward Cree88eb1942018-07-02 16:13:56 +01005487 *ppt_prev = pt_prev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005488 } else {
Mel Gormanb4b9e352012-07-31 16:44:26 -07005489drop:
Jarod Wilson6e7333d2016-02-01 18:51:05 -05005490 if (!deliver_exact)
5491 atomic_long_inc(&skb->dev->rx_dropped);
5492 else
5493 atomic_long_inc(&skb->dev->rx_nohandler);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005494 kfree_skb(skb);
5495 /* Jamal, now you will not able to escape explaining
5496 * me how you were going to use this. :-)
5497 */
5498 ret = NET_RX_DROP;
5499 }
5500
Julian Anastasov2c17d272015-07-09 09:59:10 +03005501out:
Boris Sukholitkoc0bbbdc2020-05-19 10:32:37 +03005502 /* The invariant here is that if *ppt_prev is not NULL
5503 * then skb should also be non-NULL.
5504 *
5505 * Apparently *ppt_prev assignment above holds this invariant due to
5506 * skb dereferencing near it.
5507 */
5508 *pskb = skb;
David S. Miller9754e292013-02-14 15:57:38 -05005509 return ret;
5510}
5511
Edward Cree88eb1942018-07-02 16:13:56 +01005512static int __netif_receive_skb_one_core(struct sk_buff *skb, bool pfmemalloc)
5513{
5514 struct net_device *orig_dev = skb->dev;
5515 struct packet_type *pt_prev = NULL;
5516 int ret;
5517
Boris Sukholitkoc0bbbdc2020-05-19 10:32:37 +03005518 ret = __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
Edward Cree88eb1942018-07-02 16:13:56 +01005519 if (pt_prev)
Paolo Abenif5737cb2019-05-03 17:01:36 +02005520 ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb,
5521 skb->dev, pt_prev, orig_dev);
Edward Cree88eb1942018-07-02 16:13:56 +01005522 return ret;
5523}
5524
Jesper Dangaard Brouer1c601d82017-10-16 12:19:39 +02005525/**
5526 * netif_receive_skb_core - special purpose version of netif_receive_skb
5527 * @skb: buffer to process
5528 *
5529 * More direct receive version of netif_receive_skb(). It should
5530 * only be used by callers that have a need to skip RPS and Generic XDP.
Mauro Carvalho Chehab2de97802020-03-17 15:54:20 +01005531 * Caller must also take care of handling if ``(page_is_)pfmemalloc``.
Jesper Dangaard Brouer1c601d82017-10-16 12:19:39 +02005532 *
5533 * This function may only be called from softirq context and interrupts
5534 * should be enabled.
5535 *
5536 * Return values (usually ignored):
5537 * NET_RX_SUCCESS: no congestion
5538 * NET_RX_DROP: packet was dropped
5539 */
5540int netif_receive_skb_core(struct sk_buff *skb)
5541{
5542 int ret;
5543
5544 rcu_read_lock();
Edward Cree88eb1942018-07-02 16:13:56 +01005545 ret = __netif_receive_skb_one_core(skb, false);
Jesper Dangaard Brouer1c601d82017-10-16 12:19:39 +02005546 rcu_read_unlock();
5547
5548 return ret;
5549}
5550EXPORT_SYMBOL(netif_receive_skb_core);
5551
Edward Cree88eb1942018-07-02 16:13:56 +01005552static inline void __netif_receive_skb_list_ptype(struct list_head *head,
5553 struct packet_type *pt_prev,
5554 struct net_device *orig_dev)
Edward Cree4ce00172018-07-02 16:13:40 +01005555{
5556 struct sk_buff *skb, *next;
5557
Edward Cree88eb1942018-07-02 16:13:56 +01005558 if (!pt_prev)
5559 return;
5560 if (list_empty(head))
5561 return;
Edward Cree17266ee2018-07-02 16:14:12 +01005562 if (pt_prev->list_func != NULL)
Paolo Abenifdf71422019-06-04 11:44:06 +02005563 INDIRECT_CALL_INET(pt_prev->list_func, ipv6_list_rcv,
5564 ip_list_rcv, head, pt_prev, orig_dev);
Edward Cree17266ee2018-07-02 16:14:12 +01005565 else
Alexander Lobakin9a5a90d2019-03-28 18:23:04 +03005566 list_for_each_entry_safe(skb, next, head, list) {
5567 skb_list_del_init(skb);
Paolo Abenifdf71422019-06-04 11:44:06 +02005568 pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
Alexander Lobakin9a5a90d2019-03-28 18:23:04 +03005569 }
Edward Cree88eb1942018-07-02 16:13:56 +01005570}
5571
5572static void __netif_receive_skb_list_core(struct list_head *head, bool pfmemalloc)
5573{
5574 /* Fast-path assumptions:
5575 * - There is no RX handler.
5576 * - Only one packet_type matches.
5577 * If either of these fails, we will end up doing some per-packet
5578 * processing in-line, then handling the 'last ptype' for the whole
5579 * sublist. This can't cause out-of-order delivery to any single ptype,
5580 * because the 'last ptype' must be constant across the sublist, and all
5581 * other ptypes are handled per-packet.
5582 */
5583 /* Current (common) ptype of sublist */
5584 struct packet_type *pt_curr = NULL;
5585 /* Current (common) orig_dev of sublist */
5586 struct net_device *od_curr = NULL;
5587 struct list_head sublist;
5588 struct sk_buff *skb, *next;
5589
Edward Cree9af86f92018-07-09 18:10:19 +01005590 INIT_LIST_HEAD(&sublist);
Edward Cree88eb1942018-07-02 16:13:56 +01005591 list_for_each_entry_safe(skb, next, head, list) {
5592 struct net_device *orig_dev = skb->dev;
5593 struct packet_type *pt_prev = NULL;
5594
Edward Cree22f6bbb2018-12-04 17:37:57 +00005595 skb_list_del_init(skb);
Boris Sukholitkoc0bbbdc2020-05-19 10:32:37 +03005596 __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
Edward Cree9af86f92018-07-09 18:10:19 +01005597 if (!pt_prev)
5598 continue;
Edward Cree88eb1942018-07-02 16:13:56 +01005599 if (pt_curr != pt_prev || od_curr != orig_dev) {
5600 /* dispatch old sublist */
Edward Cree88eb1942018-07-02 16:13:56 +01005601 __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
5602 /* start new sublist */
Edward Cree9af86f92018-07-09 18:10:19 +01005603 INIT_LIST_HEAD(&sublist);
Edward Cree88eb1942018-07-02 16:13:56 +01005604 pt_curr = pt_prev;
5605 od_curr = orig_dev;
5606 }
Edward Cree9af86f92018-07-09 18:10:19 +01005607 list_add_tail(&skb->list, &sublist);
Edward Cree88eb1942018-07-02 16:13:56 +01005608 }
5609
5610 /* dispatch final sublist */
Edward Cree9af86f92018-07-09 18:10:19 +01005611 __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
Edward Cree4ce00172018-07-02 16:13:40 +01005612}
5613
David S. Miller9754e292013-02-14 15:57:38 -05005614static int __netif_receive_skb(struct sk_buff *skb)
5615{
5616 int ret;
5617
5618 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
Vlastimil Babkaf1083042017-05-08 15:59:53 -07005619 unsigned int noreclaim_flag;
David S. Miller9754e292013-02-14 15:57:38 -05005620
5621 /*
5622 * PFMEMALLOC skbs are special, they should
5623 * - be delivered to SOCK_MEMALLOC sockets only
5624 * - stay away from userspace
5625 * - have bounded memory usage
5626 *
5627 * Use PF_MEMALLOC as this saves us from propagating the allocation
5628 * context down to all allocation sites.
5629 */
Vlastimil Babkaf1083042017-05-08 15:59:53 -07005630 noreclaim_flag = memalloc_noreclaim_save();
Edward Cree88eb1942018-07-02 16:13:56 +01005631 ret = __netif_receive_skb_one_core(skb, true);
Vlastimil Babkaf1083042017-05-08 15:59:53 -07005632 memalloc_noreclaim_restore(noreclaim_flag);
David S. Miller9754e292013-02-14 15:57:38 -05005633 } else
Edward Cree88eb1942018-07-02 16:13:56 +01005634 ret = __netif_receive_skb_one_core(skb, false);
David S. Miller9754e292013-02-14 15:57:38 -05005635
Linus Torvalds1da177e2005-04-16 15:20:36 -07005636 return ret;
5637}
Tom Herbert0a9627f2010-03-16 08:03:29 +00005638
Edward Cree4ce00172018-07-02 16:13:40 +01005639static void __netif_receive_skb_list(struct list_head *head)
5640{
5641 unsigned long noreclaim_flag = 0;
5642 struct sk_buff *skb, *next;
5643 bool pfmemalloc = false; /* Is current sublist PF_MEMALLOC? */
5644
5645 list_for_each_entry_safe(skb, next, head, list) {
5646 if ((sk_memalloc_socks() && skb_pfmemalloc(skb)) != pfmemalloc) {
5647 struct list_head sublist;
5648
5649 /* Handle the previous sublist */
5650 list_cut_before(&sublist, head, &skb->list);
Edward Creeb9f463d2018-07-02 16:14:44 +01005651 if (!list_empty(&sublist))
5652 __netif_receive_skb_list_core(&sublist, pfmemalloc);
Edward Cree4ce00172018-07-02 16:13:40 +01005653 pfmemalloc = !pfmemalloc;
5654 /* See comments in __netif_receive_skb */
5655 if (pfmemalloc)
5656 noreclaim_flag = memalloc_noreclaim_save();
5657 else
5658 memalloc_noreclaim_restore(noreclaim_flag);
5659 }
5660 }
5661 /* Handle the remaining sublist */
Edward Creeb9f463d2018-07-02 16:14:44 +01005662 if (!list_empty(head))
5663 __netif_receive_skb_list_core(head, pfmemalloc);
Edward Cree4ce00172018-07-02 16:13:40 +01005664 /* Restore pflags */
5665 if (pfmemalloc)
5666 memalloc_noreclaim_restore(noreclaim_flag);
5667}
5668
Jakub Kicinskif4e63522017-11-03 13:56:16 -07005669static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp)
David S. Millerb5cdae32017-04-18 15:36:58 -04005670{
Martin KaFai Lau58038692017-06-15 17:29:09 -07005671 struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
David S. Millerb5cdae32017-04-18 15:36:58 -04005672 struct bpf_prog *new = xdp->prog;
5673 int ret = 0;
5674
5675 switch (xdp->command) {
Martin KaFai Lau58038692017-06-15 17:29:09 -07005676 case XDP_SETUP_PROG:
David S. Millerb5cdae32017-04-18 15:36:58 -04005677 rcu_assign_pointer(dev->xdp_prog, new);
5678 if (old)
5679 bpf_prog_put(old);
5680
5681 if (old && !new) {
Davidlohr Bueso02786472018-05-08 09:07:02 -07005682 static_branch_dec(&generic_xdp_needed_key);
David S. Millerb5cdae32017-04-18 15:36:58 -04005683 } else if (new && !old) {
Davidlohr Bueso02786472018-05-08 09:07:02 -07005684 static_branch_inc(&generic_xdp_needed_key);
David S. Millerb5cdae32017-04-18 15:36:58 -04005685 dev_disable_lro(dev);
Michael Chan56f5aa72017-12-16 03:09:41 -05005686 dev_disable_gro_hw(dev);
David S. Millerb5cdae32017-04-18 15:36:58 -04005687 }
5688 break;
David S. Millerb5cdae32017-04-18 15:36:58 -04005689
David S. Millerb5cdae32017-04-18 15:36:58 -04005690 default:
5691 ret = -EINVAL;
5692 break;
5693 }
5694
5695 return ret;
5696}
5697
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00005698static int netif_receive_skb_internal(struct sk_buff *skb)
Tom Herbert0a9627f2010-03-16 08:03:29 +00005699{
Julian Anastasov2c17d272015-07-09 09:59:10 +03005700 int ret;
5701
Eric Dumazet588f0332011-11-15 04:12:55 +00005702 net_timestamp_check(netdev_tstamp_prequeue, skb);
Eric Dumazet3b098e22010-05-15 23:57:10 -07005703
Richard Cochranc1f19b52010-07-17 08:49:36 +00005704 if (skb_defer_rx_timestamp(skb))
5705 return NET_RX_SUCCESS;
5706
John Fastabendbbbe2112017-09-08 14:00:30 -07005707 rcu_read_lock();
Eric Dumazetdf334542010-03-24 19:13:54 +00005708#ifdef CONFIG_RPS
Eric Dumazetdc053602019-03-22 08:56:38 -07005709 if (static_branch_unlikely(&rps_needed)) {
Eric Dumazet3b098e22010-05-15 23:57:10 -07005710 struct rps_dev_flow voidflow, *rflow = &voidflow;
Julian Anastasov2c17d272015-07-09 09:59:10 +03005711 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
Tom Herbertfec5e652010-04-16 16:01:27 -07005712
Eric Dumazet3b098e22010-05-15 23:57:10 -07005713 if (cpu >= 0) {
5714 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5715 rcu_read_unlock();
Eric Dumazetadc93002011-11-17 03:13:26 +00005716 return ret;
Eric Dumazet3b098e22010-05-15 23:57:10 -07005717 }
Tom Herbertfec5e652010-04-16 16:01:27 -07005718 }
Tom Herbert1e94d722010-03-18 17:45:44 -07005719#endif
Julian Anastasov2c17d272015-07-09 09:59:10 +03005720 ret = __netif_receive_skb(skb);
5721 rcu_read_unlock();
5722 return ret;
Tom Herbert0a9627f2010-03-16 08:03:29 +00005723}
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00005724
Edward Cree7da517a2018-07-02 16:13:24 +01005725static void netif_receive_skb_list_internal(struct list_head *head)
5726{
Edward Cree7da517a2018-07-02 16:13:24 +01005727 struct sk_buff *skb, *next;
Edward Cree8c057ef2018-07-09 18:09:54 +01005728 struct list_head sublist;
Edward Cree7da517a2018-07-02 16:13:24 +01005729
Edward Cree8c057ef2018-07-09 18:09:54 +01005730 INIT_LIST_HEAD(&sublist);
Edward Cree7da517a2018-07-02 16:13:24 +01005731 list_for_each_entry_safe(skb, next, head, list) {
5732 net_timestamp_check(netdev_tstamp_prequeue, skb);
Edward Cree22f6bbb2018-12-04 17:37:57 +00005733 skb_list_del_init(skb);
Edward Cree8c057ef2018-07-09 18:09:54 +01005734 if (!skb_defer_rx_timestamp(skb))
5735 list_add_tail(&skb->list, &sublist);
Edward Cree7da517a2018-07-02 16:13:24 +01005736 }
Edward Cree8c057ef2018-07-09 18:09:54 +01005737 list_splice_init(&sublist, head);
Edward Cree7da517a2018-07-02 16:13:24 +01005738
Edward Cree7da517a2018-07-02 16:13:24 +01005739 rcu_read_lock();
5740#ifdef CONFIG_RPS
Eric Dumazetdc053602019-03-22 08:56:38 -07005741 if (static_branch_unlikely(&rps_needed)) {
Edward Cree7da517a2018-07-02 16:13:24 +01005742 list_for_each_entry_safe(skb, next, head, list) {
5743 struct rps_dev_flow voidflow, *rflow = &voidflow;
5744 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
5745
5746 if (cpu >= 0) {
Edward Cree8c057ef2018-07-09 18:09:54 +01005747 /* Will be handled, remove from list */
Edward Cree22f6bbb2018-12-04 17:37:57 +00005748 skb_list_del_init(skb);
Edward Cree8c057ef2018-07-09 18:09:54 +01005749 enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
Edward Cree7da517a2018-07-02 16:13:24 +01005750 }
5751 }
5752 }
5753#endif
5754 __netif_receive_skb_list(head);
5755 rcu_read_unlock();
5756}
5757
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00005758/**
5759 * netif_receive_skb - process receive buffer from network
5760 * @skb: buffer to process
5761 *
5762 * netif_receive_skb() is the main receive data processing function.
5763 * It always succeeds. The buffer may be dropped during processing
5764 * for congestion control or by the protocol layers.
5765 *
5766 * This function may only be called from softirq context and interrupts
5767 * should be enabled.
5768 *
5769 * Return values (usually ignored):
5770 * NET_RX_SUCCESS: no congestion
5771 * NET_RX_DROP: packet was dropped
5772 */
Eric W. Biederman04eb4482015-09-15 20:04:15 -05005773int netif_receive_skb(struct sk_buff *skb)
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00005774{
Geneviève Bastienb0e3f1b2018-11-27 12:52:39 -05005775 int ret;
5776
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00005777 trace_netif_receive_skb_entry(skb);
5778
Geneviève Bastienb0e3f1b2018-11-27 12:52:39 -05005779 ret = netif_receive_skb_internal(skb);
5780 trace_netif_receive_skb_exit(ret);
5781
5782 return ret;
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00005783}
Eric W. Biederman04eb4482015-09-15 20:04:15 -05005784EXPORT_SYMBOL(netif_receive_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005785
Edward Creef6ad8c12018-07-02 16:12:45 +01005786/**
5787 * netif_receive_skb_list - process many receive buffers from network
5788 * @head: list of skbs to process.
5789 *
Edward Cree7da517a2018-07-02 16:13:24 +01005790 * Since return value of netif_receive_skb() is normally ignored, and
5791 * wouldn't be meaningful for a list, this function returns void.
Edward Creef6ad8c12018-07-02 16:12:45 +01005792 *
5793 * This function may only be called from softirq context and interrupts
5794 * should be enabled.
5795 */
5796void netif_receive_skb_list(struct list_head *head)
5797{
Edward Cree7da517a2018-07-02 16:13:24 +01005798 struct sk_buff *skb;
Edward Creef6ad8c12018-07-02 16:12:45 +01005799
Edward Creeb9f463d2018-07-02 16:14:44 +01005800 if (list_empty(head))
5801 return;
Geneviève Bastienb0e3f1b2018-11-27 12:52:39 -05005802 if (trace_netif_receive_skb_list_entry_enabled()) {
5803 list_for_each_entry(skb, head, list)
5804 trace_netif_receive_skb_list_entry(skb);
5805 }
Edward Cree7da517a2018-07-02 16:13:24 +01005806 netif_receive_skb_list_internal(head);
Geneviève Bastienb0e3f1b2018-11-27 12:52:39 -05005807 trace_netif_receive_skb_list_exit(0);
Edward Creef6ad8c12018-07-02 16:12:45 +01005808}
5809EXPORT_SYMBOL(netif_receive_skb_list);
5810
Wei Yongjunce1e2a72020-07-13 22:23:44 +08005811static DEFINE_PER_CPU(struct work_struct, flush_works);
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005812
5813/* Network device is going away, flush any packets still pending */
5814static void flush_backlog(struct work_struct *work)
5815{
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005816 struct sk_buff *skb, *tmp;
5817 struct softnet_data *sd;
5818
5819 local_bh_disable();
5820 sd = this_cpu_ptr(&softnet_data);
5821
5822 local_irq_disable();
Eric Dumazete36fa2f2010-04-19 21:17:14 +00005823 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07005824 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
Eric Dumazet41852492016-08-26 12:50:39 -07005825 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00005826 __skb_unlink(skb, &sd->input_pkt_queue);
Subash Abhinov Kasiviswanathan7df5cb752020-07-23 11:31:48 -06005827 dev_kfree_skb_irq(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00005828 input_queue_head_incr(sd);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07005829 }
Changli Gao6e7676c2010-04-27 15:07:33 -07005830 }
Eric Dumazete36fa2f2010-04-19 21:17:14 +00005831 rps_unlock(sd);
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005832 local_irq_enable();
Changli Gao6e7676c2010-04-27 15:07:33 -07005833
5834 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
Eric Dumazet41852492016-08-26 12:50:39 -07005835 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
Changli Gao6e7676c2010-04-27 15:07:33 -07005836 __skb_unlink(skb, &sd->process_queue);
5837 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00005838 input_queue_head_incr(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07005839 }
5840 }
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005841 local_bh_enable();
5842}
5843
Paolo Abeni2de79ee2020-09-10 23:33:18 +02005844static bool flush_required(int cpu)
5845{
5846#if IS_ENABLED(CONFIG_RPS)
5847 struct softnet_data *sd = &per_cpu(softnet_data, cpu);
5848 bool do_flush;
5849
5850 local_irq_disable();
5851 rps_lock(sd);
5852
5853 /* as insertion into process_queue happens with the rps lock held,
5854 * process_queue access may race only with dequeue
5855 */
5856 do_flush = !skb_queue_empty(&sd->input_pkt_queue) ||
5857 !skb_queue_empty_lockless(&sd->process_queue);
5858 rps_unlock(sd);
5859 local_irq_enable();
5860
5861 return do_flush;
5862#endif
5863 /* without RPS we can't safely check input_pkt_queue: during a
5864 * concurrent remote skb_queue_splice() we can detect as empty both
5865 * input_pkt_queue and process_queue even if the latter could end-up
5866 * containing a lot of packets.
5867 */
5868 return true;
5869}
5870
Eric Dumazet41852492016-08-26 12:50:39 -07005871static void flush_all_backlogs(void)
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005872{
Paolo Abeni2de79ee2020-09-10 23:33:18 +02005873 static cpumask_t flush_cpus;
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005874 unsigned int cpu;
5875
Paolo Abeni2de79ee2020-09-10 23:33:18 +02005876 /* since we are under rtnl lock protection we can use static data
5877 * for the cpumask and avoid allocating on stack the possibly
5878 * large mask
5879 */
5880 ASSERT_RTNL();
5881
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005882 get_online_cpus();
5883
Paolo Abeni2de79ee2020-09-10 23:33:18 +02005884 cpumask_clear(&flush_cpus);
5885 for_each_online_cpu(cpu) {
5886 if (flush_required(cpu)) {
5887 queue_work_on(cpu, system_highpri_wq,
5888 per_cpu_ptr(&flush_works, cpu));
5889 cpumask_set_cpu(cpu, &flush_cpus);
5890 }
5891 }
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005892
Paolo Abeni2de79ee2020-09-10 23:33:18 +02005893 /* we can have in flight packet[s] on the cpus we are not flushing,
Jakub Kicinski0cbe1e52021-01-19 12:25:21 -08005894 * synchronize_net() in unregister_netdevice_many() will take care of
Paolo Abeni2de79ee2020-09-10 23:33:18 +02005895 * them
5896 */
5897 for_each_cpu(cpu, &flush_cpus)
Eric Dumazet41852492016-08-26 12:50:39 -07005898 flush_work(per_cpu_ptr(&flush_works, cpu));
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005899
5900 put_online_cpus();
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07005901}
5902
Maxim Mikityanskiyc8079432020-01-21 15:09:40 +00005903/* Pass the currently batched GRO_NORMAL SKBs up to the stack. */
5904static void gro_normal_list(struct napi_struct *napi)
5905{
5906 if (!napi->rx_count)
5907 return;
5908 netif_receive_skb_list_internal(&napi->rx_list);
5909 INIT_LIST_HEAD(&napi->rx_list);
5910 napi->rx_count = 0;
5911}
5912
5913/* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded,
5914 * pass the whole batch up to the stack.
5915 */
Eric Dumazet8dc1c442021-02-04 13:31:46 -08005916static void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb, int segs)
Maxim Mikityanskiyc8079432020-01-21 15:09:40 +00005917{
5918 list_add_tail(&skb->list, &napi->rx_list);
Eric Dumazet8dc1c442021-02-04 13:31:46 -08005919 napi->rx_count += segs;
5920 if (napi->rx_count >= gro_normal_batch)
Maxim Mikityanskiyc8079432020-01-21 15:09:40 +00005921 gro_normal_list(napi);
5922}
5923
Maxim Mikityanskiyc8079432020-01-21 15:09:40 +00005924static int napi_gro_complete(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xud565b0a2008-12-15 23:38:52 -08005925{
Vlad Yasevich22061d82012-11-15 08:49:11 +00005926 struct packet_offload *ptype;
Herbert Xud565b0a2008-12-15 23:38:52 -08005927 __be16 type = skb->protocol;
Vlad Yasevich22061d82012-11-15 08:49:11 +00005928 struct list_head *head = &offload_base;
Herbert Xud565b0a2008-12-15 23:38:52 -08005929 int err = -ENOENT;
5930
Eric Dumazetc3c7c252012-12-06 13:54:59 +00005931 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
5932
Herbert Xufc59f9a2009-04-14 15:11:06 -07005933 if (NAPI_GRO_CB(skb)->count == 1) {
5934 skb_shinfo(skb)->gso_size = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08005935 goto out;
Herbert Xufc59f9a2009-04-14 15:11:06 -07005936 }
Herbert Xud565b0a2008-12-15 23:38:52 -08005937
5938 rcu_read_lock();
5939 list_for_each_entry_rcu(ptype, head, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00005940 if (ptype->type != type || !ptype->callbacks.gro_complete)
Herbert Xud565b0a2008-12-15 23:38:52 -08005941 continue;
5942
Paolo Abeniaaa5d902018-12-14 11:51:58 +01005943 err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete,
5944 ipv6_gro_complete, inet_gro_complete,
5945 skb, 0);
Herbert Xud565b0a2008-12-15 23:38:52 -08005946 break;
5947 }
5948 rcu_read_unlock();
5949
5950 if (err) {
5951 WARN_ON(&ptype->list == head);
5952 kfree_skb(skb);
5953 return NET_RX_SUCCESS;
5954 }
5955
5956out:
Eric Dumazet8dc1c442021-02-04 13:31:46 -08005957 gro_normal_one(napi, skb, NAPI_GRO_CB(skb)->count);
Maxim Mikityanskiyc8079432020-01-21 15:09:40 +00005958 return NET_RX_SUCCESS;
Herbert Xud565b0a2008-12-15 23:38:52 -08005959}
5960
Li RongQing6312fe72018-07-05 14:34:32 +08005961static void __napi_gro_flush_chain(struct napi_struct *napi, u32 index,
David Miller07d78362018-06-24 14:14:02 +09005962 bool flush_old)
Herbert Xud565b0a2008-12-15 23:38:52 -08005963{
Li RongQing6312fe72018-07-05 14:34:32 +08005964 struct list_head *head = &napi->gro_hash[index].list;
David Millerd4546c22018-06-24 14:13:49 +09005965 struct sk_buff *skb, *p;
Herbert Xud565b0a2008-12-15 23:38:52 -08005966
David Miller07d78362018-06-24 14:14:02 +09005967 list_for_each_entry_safe_reverse(skb, p, head, list) {
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00005968 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
5969 return;
David S. Miller992cba72018-07-31 15:27:56 -07005970 skb_list_del_init(skb);
Maxim Mikityanskiyc8079432020-01-21 15:09:40 +00005971 napi_gro_complete(napi, skb);
Li RongQing6312fe72018-07-05 14:34:32 +08005972 napi->gro_hash[index].count--;
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00005973 }
Li RongQingd9f37d02018-07-13 14:41:36 +08005974
5975 if (!napi->gro_hash[index].count)
5976 __clear_bit(index, &napi->gro_bitmask);
Herbert Xud565b0a2008-12-15 23:38:52 -08005977}
David Miller07d78362018-06-24 14:14:02 +09005978
Li RongQing6312fe72018-07-05 14:34:32 +08005979/* napi->gro_hash[].list contains packets ordered by age.
David Miller07d78362018-06-24 14:14:02 +09005980 * youngest packets at the head of it.
5981 * Complete skbs in reverse order to reduce latencies.
5982 */
5983void napi_gro_flush(struct napi_struct *napi, bool flush_old)
5984{
Eric Dumazet42519ed2018-11-21 11:39:28 -08005985 unsigned long bitmask = napi->gro_bitmask;
5986 unsigned int i, base = ~0U;
David Miller07d78362018-06-24 14:14:02 +09005987
Eric Dumazet42519ed2018-11-21 11:39:28 -08005988 while ((i = ffs(bitmask)) != 0) {
5989 bitmask >>= i;
5990 base += i;
5991 __napi_gro_flush_chain(napi, base, flush_old);
Li RongQingd9f37d02018-07-13 14:41:36 +08005992 }
David Miller07d78362018-06-24 14:14:02 +09005993}
Eric Dumazet86cac582010-08-31 18:25:32 +00005994EXPORT_SYMBOL(napi_gro_flush);
Herbert Xud565b0a2008-12-15 23:38:52 -08005995
Alexander Lobakin0ccf4d52021-03-13 20:30:05 +00005996static void gro_list_prepare(const struct list_head *head,
5997 const struct sk_buff *skb)
Eric Dumazet89c5fa32012-12-10 13:28:16 +00005998{
Eric Dumazet89c5fa32012-12-10 13:28:16 +00005999 unsigned int maclen = skb->dev->hard_header_len;
Tom Herbert0b4cec82014-01-15 08:58:06 -08006000 u32 hash = skb_get_hash_raw(skb);
David Millerd4546c22018-06-24 14:13:49 +09006001 struct sk_buff *p;
Eric Dumazet89c5fa32012-12-10 13:28:16 +00006002
David Miller07d78362018-06-24 14:14:02 +09006003 list_for_each_entry(p, head, list) {
Eric Dumazet89c5fa32012-12-10 13:28:16 +00006004 unsigned long diffs;
6005
Tom Herbert0b4cec82014-01-15 08:58:06 -08006006 NAPI_GRO_CB(p)->flush = 0;
6007
6008 if (hash != skb_get_hash_raw(p)) {
6009 NAPI_GRO_CB(p)->same_flow = 0;
6010 continue;
6011 }
6012
Eric Dumazet89c5fa32012-12-10 13:28:16 +00006013 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
Michał Mirosławb18175242018-11-09 00:18:02 +01006014 diffs |= skb_vlan_tag_present(p) ^ skb_vlan_tag_present(skb);
6015 if (skb_vlan_tag_present(p))
Tonghao Zhangfc5141c2019-11-22 20:38:01 +08006016 diffs |= skb_vlan_tag_get(p) ^ skb_vlan_tag_get(skb);
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02006017 diffs |= skb_metadata_differs(p, skb);
Eric Dumazet89c5fa32012-12-10 13:28:16 +00006018 if (maclen == ETH_HLEN)
6019 diffs |= compare_ether_header(skb_mac_header(p),
Eric Dumazeta50e2332014-03-29 21:28:21 -07006020 skb_mac_header(skb));
Eric Dumazet89c5fa32012-12-10 13:28:16 +00006021 else if (!diffs)
6022 diffs = memcmp(skb_mac_header(p),
Eric Dumazeta50e2332014-03-29 21:28:21 -07006023 skb_mac_header(skb),
Eric Dumazet89c5fa32012-12-10 13:28:16 +00006024 maclen);
Paul Blakey8550ff82021-07-05 13:54:51 +03006025
Paolo Abeni9efb4b52021-07-28 18:24:02 +02006026 /* in most common scenarions _state is 0
6027 * otherwise we are already on some slower paths
6028 * either skip all the infrequent tests altogether or
6029 * avoid trying too hard to skip each of them individually
6030 */
6031 if (!diffs && unlikely(skb->slow_gro | p->slow_gro)) {
Florian Fainelli9615fe32021-07-07 21:10:51 -07006032#if IS_ENABLED(CONFIG_SKB_EXTENSIONS) && IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
Paolo Abeni9efb4b52021-07-28 18:24:02 +02006033 struct tc_skb_ext *skb_ext;
6034 struct tc_skb_ext *p_ext;
6035#endif
6036
6037 diffs |= skb_metadata_dst_cmp(p, skb);
6038 diffs |= skb_get_nfct(p) ^ skb_get_nfct(skb);
6039
6040#if IS_ENABLED(CONFIG_SKB_EXTENSIONS) && IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
6041 skb_ext = skb_ext_find(skb, TC_SKB_EXT);
6042 p_ext = skb_ext_find(p, TC_SKB_EXT);
Paul Blakey8550ff82021-07-05 13:54:51 +03006043
6044 diffs |= (!!p_ext) ^ (!!skb_ext);
6045 if (!diffs && unlikely(skb_ext))
6046 diffs |= p_ext->chain ^ skb_ext->chain;
Florian Fainelli9615fe32021-07-07 21:10:51 -07006047#endif
Paolo Abeni9efb4b52021-07-28 18:24:02 +02006048 }
Paul Blakey8550ff82021-07-05 13:54:51 +03006049
Eric Dumazet89c5fa32012-12-10 13:28:16 +00006050 NAPI_GRO_CB(p)->same_flow = !diffs;
Eric Dumazet89c5fa32012-12-10 13:28:16 +00006051 }
6052}
6053
Alexander Lobakin7ad18ff2021-04-19 12:53:06 +00006054static inline void skb_gro_reset_offset(struct sk_buff *skb, u32 nhoff)
Jerry Chu299603e82013-12-11 20:53:45 -08006055{
6056 const struct skb_shared_info *pinfo = skb_shinfo(skb);
6057 const skb_frag_t *frag0 = &pinfo->frags[0];
6058
6059 NAPI_GRO_CB(skb)->data_offset = 0;
6060 NAPI_GRO_CB(skb)->frag0 = NULL;
6061 NAPI_GRO_CB(skb)->frag0_len = 0;
6062
Alexander Lobakin8aef9982019-11-15 12:11:35 +03006063 if (!skb_headlen(skb) && pinfo->nr_frags &&
Eric Dumazet38ec4942021-04-13 05:41:35 -07006064 !PageHighMem(skb_frag_page(frag0)) &&
Alexander Lobakin7ad18ff2021-04-19 12:53:06 +00006065 (!NET_IP_ALIGN || !((skb_frag_off(frag0) + nhoff) & 3))) {
Jerry Chu299603e82013-12-11 20:53:45 -08006066 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
Eric Dumazet7cfd5fd2017-01-10 19:52:43 -08006067 NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,
6068 skb_frag_size(frag0),
6069 skb->end - skb->tail);
Herbert Xud565b0a2008-12-15 23:38:52 -08006070 }
6071}
6072
Eric Dumazeta50e2332014-03-29 21:28:21 -07006073static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
6074{
6075 struct skb_shared_info *pinfo = skb_shinfo(skb);
6076
6077 BUG_ON(skb->end - skb->tail < grow);
6078
6079 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
6080
6081 skb->data_len -= grow;
6082 skb->tail += grow;
6083
Jonathan Lemonb54c9d52019-07-30 07:40:33 -07006084 skb_frag_off_add(&pinfo->frags[0], grow);
Eric Dumazeta50e2332014-03-29 21:28:21 -07006085 skb_frag_size_sub(&pinfo->frags[0], grow);
6086
6087 if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
6088 skb_frag_unref(skb, 0);
6089 memmove(pinfo->frags, pinfo->frags + 1,
6090 --pinfo->nr_frags * sizeof(pinfo->frags[0]));
6091 }
6092}
6093
Maxim Mikityanskiyc8079432020-01-21 15:09:40 +00006094static void gro_flush_oldest(struct napi_struct *napi, struct list_head *head)
David Miller07d78362018-06-24 14:14:02 +09006095{
Li RongQing6312fe72018-07-05 14:34:32 +08006096 struct sk_buff *oldest;
David Miller07d78362018-06-24 14:14:02 +09006097
Li RongQing6312fe72018-07-05 14:34:32 +08006098 oldest = list_last_entry(head, struct sk_buff, list);
David Miller07d78362018-06-24 14:14:02 +09006099
Li RongQing6312fe72018-07-05 14:34:32 +08006100 /* We are called with head length >= MAX_GRO_SKBS, so this is
David Miller07d78362018-06-24 14:14:02 +09006101 * impossible.
6102 */
6103 if (WARN_ON_ONCE(!oldest))
6104 return;
6105
Li RongQingd9f37d02018-07-13 14:41:36 +08006106 /* Do not adjust napi->gro_hash[].count, caller is adding a new
6107 * SKB to the chain.
David Miller07d78362018-06-24 14:14:02 +09006108 */
David S. Millerece23712018-10-28 10:35:12 -07006109 skb_list_del_init(oldest);
Maxim Mikityanskiyc8079432020-01-21 15:09:40 +00006110 napi_gro_complete(napi, oldest);
David Miller07d78362018-06-24 14:14:02 +09006111}
6112
Rami Rosenbb728822012-11-28 21:55:25 +00006113static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xud565b0a2008-12-15 23:38:52 -08006114{
Alexander Lobakind0eed5c2021-03-13 20:30:14 +00006115 u32 bucket = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1);
6116 struct gro_list *gro_list = &napi->gro_hash[bucket];
David Millerd4546c22018-06-24 14:13:49 +09006117 struct list_head *head = &offload_base;
Vlad Yasevich22061d82012-11-15 08:49:11 +00006118 struct packet_offload *ptype;
Herbert Xud565b0a2008-12-15 23:38:52 -08006119 __be16 type = skb->protocol;
David Millerd4546c22018-06-24 14:13:49 +09006120 struct sk_buff *pp = NULL;
Ben Hutchings5b252f02009-10-29 07:17:09 +00006121 enum gro_result ret;
David Millerd4546c22018-06-24 14:13:49 +09006122 int same_flow;
Eric Dumazeta50e2332014-03-29 21:28:21 -07006123 int grow;
Herbert Xud565b0a2008-12-15 23:38:52 -08006124
David S. Millerb5cdae32017-04-18 15:36:58 -04006125 if (netif_elide_gro(skb->dev))
Herbert Xud565b0a2008-12-15 23:38:52 -08006126 goto normal;
6127
Alexander Lobakin9dc2c312021-03-13 20:30:10 +00006128 gro_list_prepare(&gro_list->list, skb);
Eric Dumazet89c5fa32012-12-10 13:28:16 +00006129
Herbert Xud565b0a2008-12-15 23:38:52 -08006130 rcu_read_lock();
6131 list_for_each_entry_rcu(ptype, head, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00006132 if (ptype->type != type || !ptype->callbacks.gro_receive)
Herbert Xud565b0a2008-12-15 23:38:52 -08006133 continue;
6134
Herbert Xu86911732009-01-29 14:19:50 +00006135 skb_set_network_header(skb, skb_gro_offset(skb));
Eric Dumazetefd94502013-02-14 17:31:48 +00006136 skb_reset_mac_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08006137 NAPI_GRO_CB(skb)->same_flow = 0;
Eric Dumazetd61d0722016-11-07 11:12:27 -08006138 NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb);
Herbert Xu5d38a072009-01-04 16:13:40 -08006139 NAPI_GRO_CB(skb)->free = 0;
Jesse Grossfac8e0f2016-03-19 09:32:01 -07006140 NAPI_GRO_CB(skb)->encap_mark = 0;
Sabrina Dubrocafcd91dd2016-10-20 15:58:02 +02006141 NAPI_GRO_CB(skb)->recursion_counter = 0;
Alexander Duycka0ca1532016-04-05 09:13:39 -07006142 NAPI_GRO_CB(skb)->is_fou = 0;
Alexander Duyck15305452016-04-10 21:44:57 -04006143 NAPI_GRO_CB(skb)->is_atomic = 1;
Tom Herbert15e23962015-02-10 16:30:31 -08006144 NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08006145
Tom Herbert662880f2014-08-27 21:26:56 -07006146 /* Setup for GRO checksum validation */
6147 switch (skb->ip_summed) {
6148 case CHECKSUM_COMPLETE:
6149 NAPI_GRO_CB(skb)->csum = skb->csum;
6150 NAPI_GRO_CB(skb)->csum_valid = 1;
6151 NAPI_GRO_CB(skb)->csum_cnt = 0;
6152 break;
6153 case CHECKSUM_UNNECESSARY:
6154 NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
6155 NAPI_GRO_CB(skb)->csum_valid = 0;
6156 break;
6157 default:
6158 NAPI_GRO_CB(skb)->csum_cnt = 0;
6159 NAPI_GRO_CB(skb)->csum_valid = 0;
6160 }
Herbert Xud565b0a2008-12-15 23:38:52 -08006161
Paolo Abeniaaa5d902018-12-14 11:51:58 +01006162 pp = INDIRECT_CALL_INET(ptype->callbacks.gro_receive,
6163 ipv6_gro_receive, inet_gro_receive,
Alexander Lobakin9dc2c312021-03-13 20:30:10 +00006164 &gro_list->list, skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08006165 break;
6166 }
6167 rcu_read_unlock();
6168
6169 if (&ptype->list == head)
6170 goto normal;
6171
Masahiro Yamada45586c72020-02-03 17:37:45 -08006172 if (PTR_ERR(pp) == -EINPROGRESS) {
Steffen Klassert25393d32017-02-15 09:39:44 +01006173 ret = GRO_CONSUMED;
6174 goto ok;
6175 }
6176
Herbert Xu0da2afd52008-12-26 14:57:42 -08006177 same_flow = NAPI_GRO_CB(skb)->same_flow;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00006178 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
Herbert Xu0da2afd52008-12-26 14:57:42 -08006179
Herbert Xud565b0a2008-12-15 23:38:52 -08006180 if (pp) {
David S. Miller992cba72018-07-31 15:27:56 -07006181 skb_list_del_init(pp);
Maxim Mikityanskiyc8079432020-01-21 15:09:40 +00006182 napi_gro_complete(napi, pp);
Alexander Lobakin9dc2c312021-03-13 20:30:10 +00006183 gro_list->count--;
Herbert Xud565b0a2008-12-15 23:38:52 -08006184 }
6185
Herbert Xu0da2afd52008-12-26 14:57:42 -08006186 if (same_flow)
Herbert Xud565b0a2008-12-15 23:38:52 -08006187 goto ok;
6188
Eric Dumazet600adc12014-01-09 14:12:19 -08006189 if (NAPI_GRO_CB(skb)->flush)
Herbert Xud565b0a2008-12-15 23:38:52 -08006190 goto normal;
Herbert Xud565b0a2008-12-15 23:38:52 -08006191
Alexander Lobakin9dc2c312021-03-13 20:30:10 +00006192 if (unlikely(gro_list->count >= MAX_GRO_SKBS))
6193 gro_flush_oldest(napi, &gro_list->list);
6194 else
6195 gro_list->count++;
6196
Herbert Xud565b0a2008-12-15 23:38:52 -08006197 NAPI_GRO_CB(skb)->count = 1;
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00006198 NAPI_GRO_CB(skb)->age = jiffies;
Eric Dumazet29e98242014-05-16 11:34:37 -07006199 NAPI_GRO_CB(skb)->last = skb;
Herbert Xu86911732009-01-29 14:19:50 +00006200 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
Alexander Lobakin9dc2c312021-03-13 20:30:10 +00006201 list_add(&skb->list, &gro_list->list);
Herbert Xu5d0d9be2009-01-29 14:19:48 +00006202 ret = GRO_HELD;
Herbert Xud565b0a2008-12-15 23:38:52 -08006203
Herbert Xuad0f99042009-02-01 01:24:55 -08006204pull:
Eric Dumazeta50e2332014-03-29 21:28:21 -07006205 grow = skb_gro_offset(skb) - skb_headlen(skb);
6206 if (grow > 0)
6207 gro_pull_from_frag0(skb, grow);
Herbert Xud565b0a2008-12-15 23:38:52 -08006208ok:
Alexander Lobakin9dc2c312021-03-13 20:30:10 +00006209 if (gro_list->count) {
Alexander Lobakind0eed5c2021-03-13 20:30:14 +00006210 if (!test_bit(bucket, &napi->gro_bitmask))
6211 __set_bit(bucket, &napi->gro_bitmask);
6212 } else if (test_bit(bucket, &napi->gro_bitmask)) {
6213 __clear_bit(bucket, &napi->gro_bitmask);
Li RongQingd9f37d02018-07-13 14:41:36 +08006214 }
6215
Herbert Xu5d0d9be2009-01-29 14:19:48 +00006216 return ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08006217
6218normal:
Herbert Xuad0f99042009-02-01 01:24:55 -08006219 ret = GRO_NORMAL;
6220 goto pull;
Herbert Xu5d38a072009-01-04 16:13:40 -08006221}
Herbert Xu96e93ea2009-01-06 10:49:34 -08006222
Jerry Chubf5a7552014-01-07 10:23:19 -08006223struct packet_offload *gro_find_receive_by_type(__be16 type)
6224{
6225 struct list_head *offload_head = &offload_base;
6226 struct packet_offload *ptype;
6227
6228 list_for_each_entry_rcu(ptype, offload_head, list) {
6229 if (ptype->type != type || !ptype->callbacks.gro_receive)
6230 continue;
6231 return ptype;
6232 }
6233 return NULL;
6234}
Or Gerlitze27a2f82014-01-20 13:59:20 +02006235EXPORT_SYMBOL(gro_find_receive_by_type);
Jerry Chubf5a7552014-01-07 10:23:19 -08006236
6237struct packet_offload *gro_find_complete_by_type(__be16 type)
6238{
6239 struct list_head *offload_head = &offload_base;
6240 struct packet_offload *ptype;
6241
6242 list_for_each_entry_rcu(ptype, offload_head, list) {
6243 if (ptype->type != type || !ptype->callbacks.gro_complete)
6244 continue;
6245 return ptype;
6246 }
6247 return NULL;
6248}
Or Gerlitze27a2f82014-01-20 13:59:20 +02006249EXPORT_SYMBOL(gro_find_complete_by_type);
Herbert Xu96e93ea2009-01-06 10:49:34 -08006250
Alexander Lobakin6570bc72019-10-14 11:00:33 +03006251static gro_result_t napi_skb_finish(struct napi_struct *napi,
6252 struct sk_buff *skb,
6253 gro_result_t ret)
Herbert Xu5d38a072009-01-04 16:13:40 -08006254{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00006255 switch (ret) {
6256 case GRO_NORMAL:
Eric Dumazet8dc1c442021-02-04 13:31:46 -08006257 gro_normal_one(napi, skb, 1);
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07006258 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08006259
Eric Dumazetdaa86542012-04-19 07:07:40 +00006260 case GRO_MERGED_FREE:
Michal Kubečeke44699d2017-06-29 11:13:36 +02006261 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
6262 napi_skb_free_stolen_head(skb);
Antoine Tenart28b34f02021-07-09 18:16:09 +02006263 else if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
6264 __kfree_skb(skb);
Michal Kubečeke44699d2017-06-29 11:13:36 +02006265 else
Alexander Lobakin9243adf2021-02-13 14:13:09 +00006266 __kfree_skb_defer(skb);
Eric Dumazetdaa86542012-04-19 07:07:40 +00006267 break;
6268
Ben Hutchings5b252f02009-10-29 07:17:09 +00006269 case GRO_HELD:
6270 case GRO_MERGED:
Steffen Klassert25393d32017-02-15 09:39:44 +01006271 case GRO_CONSUMED:
Ben Hutchings5b252f02009-10-29 07:17:09 +00006272 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08006273 }
6274
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07006275 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00006276}
Herbert Xu5d0d9be2009-01-29 14:19:48 +00006277
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07006278gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00006279{
Geneviève Bastienb0e3f1b2018-11-27 12:52:39 -05006280 gro_result_t ret;
6281
Eric Dumazet93f93a42015-11-18 06:30:59 -08006282 skb_mark_napi_id(skb, napi);
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00006283 trace_napi_gro_receive_entry(skb);
Herbert Xu86911732009-01-29 14:19:50 +00006284
Alexander Lobakin7ad18ff2021-04-19 12:53:06 +00006285 skb_gro_reset_offset(skb, 0);
Eric Dumazeta50e2332014-03-29 21:28:21 -07006286
Alexander Lobakin6570bc72019-10-14 11:00:33 +03006287 ret = napi_skb_finish(napi, skb, dev_gro_receive(napi, skb));
Geneviève Bastienb0e3f1b2018-11-27 12:52:39 -05006288 trace_napi_gro_receive_exit(ret);
6289
6290 return ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08006291}
6292EXPORT_SYMBOL(napi_gro_receive);
6293
stephen hemmingerd0c2b0d2010-10-19 07:12:10 +00006294static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08006295{
Eric Dumazet93a35f52014-10-23 06:30:30 -07006296 if (unlikely(skb->pfmemalloc)) {
6297 consume_skb(skb);
6298 return;
6299 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08006300 __skb_pull(skb, skb_headlen(skb));
Eric Dumazet2a2a4592012-03-21 06:58:03 +00006301 /* restore the reserve we had after netdev_alloc_skb_ip_align() */
6302 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
Michał Mirosławb18175242018-11-09 00:18:02 +01006303 __vlan_hwaccel_clear_tag(skb);
Herbert Xu66c46d72011-01-29 20:44:54 -08006304 skb->dev = napi->dev;
Andy Gospodarek6d152e22011-02-02 14:53:25 -08006305 skb->skb_iif = 0;
Eric Dumazet33d9a2c2018-11-17 21:57:02 -08006306
6307 /* eth_type_trans() assumes pkt_type is PACKET_HOST */
6308 skb->pkt_type = PACKET_HOST;
6309
Jerry Chuc3caf112014-07-14 15:54:46 -07006310 skb->encapsulation = 0;
6311 skb_shinfo(skb)->gso_type = 0;
Eric Dumazete33d0ba2014-04-03 09:28:10 -07006312 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
Paolo Abeni9efb4b52021-07-28 18:24:02 +02006313 if (unlikely(skb->slow_gro)) {
6314 skb_ext_reset(skb);
6315 nf_reset_ct(skb);
6316 skb->slow_gro = 0;
6317 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08006318
6319 napi->skb = skb;
6320}
Herbert Xu96e93ea2009-01-06 10:49:34 -08006321
Herbert Xu76620aa2009-04-16 02:02:07 -07006322struct sk_buff *napi_get_frags(struct napi_struct *napi)
Herbert Xu5d38a072009-01-04 16:13:40 -08006323{
Herbert Xu5d38a072009-01-04 16:13:40 -08006324 struct sk_buff *skb = napi->skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08006325
6326 if (!skb) {
Alexander Duyckfd11a832014-12-09 19:40:49 -08006327 skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
Eric Dumazete2f9dc32015-11-19 12:11:23 -08006328 if (skb) {
6329 napi->skb = skb;
6330 skb_mark_napi_id(skb, napi);
6331 }
Herbert Xu5d38a072009-01-04 16:13:40 -08006332 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08006333 return skb;
6334}
Herbert Xu76620aa2009-04-16 02:02:07 -07006335EXPORT_SYMBOL(napi_get_frags);
Herbert Xu96e93ea2009-01-06 10:49:34 -08006336
Eric Dumazeta50e2332014-03-29 21:28:21 -07006337static gro_result_t napi_frags_finish(struct napi_struct *napi,
6338 struct sk_buff *skb,
6339 gro_result_t ret)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00006340{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00006341 switch (ret) {
6342 case GRO_NORMAL:
Eric Dumazeta50e2332014-03-29 21:28:21 -07006343 case GRO_HELD:
6344 __skb_push(skb, ETH_HLEN);
6345 skb->protocol = eth_type_trans(skb, skb->dev);
Edward Cree323ebb62019-08-06 14:53:55 +01006346 if (ret == GRO_NORMAL)
Eric Dumazet8dc1c442021-02-04 13:31:46 -08006347 gro_normal_one(napi, skb, 1);
Herbert Xu86911732009-01-29 14:19:50 +00006348 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00006349
Michal Kubečeke44699d2017-06-29 11:13:36 +02006350 case GRO_MERGED_FREE:
6351 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
6352 napi_skb_free_stolen_head(skb);
6353 else
6354 napi_reuse_skb(napi, skb);
6355 break;
6356
Ben Hutchings5b252f02009-10-29 07:17:09 +00006357 case GRO_MERGED:
Steffen Klassert25393d32017-02-15 09:39:44 +01006358 case GRO_CONSUMED:
Ben Hutchings5b252f02009-10-29 07:17:09 +00006359 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00006360 }
6361
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07006362 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00006363}
Herbert Xu5d0d9be2009-01-29 14:19:48 +00006364
Eric Dumazeta50e2332014-03-29 21:28:21 -07006365/* Upper GRO stack assumes network header starts at gro_offset=0
6366 * Drivers could call both napi_gro_frags() and napi_gro_receive()
6367 * We copy ethernet header into skb->data to have a common layout.
6368 */
Eric Dumazet4adb9c42012-05-18 20:49:06 +00006369static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
Herbert Xu96e93ea2009-01-06 10:49:34 -08006370{
Herbert Xu76620aa2009-04-16 02:02:07 -07006371 struct sk_buff *skb = napi->skb;
Eric Dumazeta50e2332014-03-29 21:28:21 -07006372 const struct ethhdr *eth;
6373 unsigned int hlen = sizeof(*eth);
Herbert Xu76620aa2009-04-16 02:02:07 -07006374
6375 napi->skb = NULL;
6376
Eric Dumazeta50e2332014-03-29 21:28:21 -07006377 skb_reset_mac_header(skb);
Alexander Lobakin7ad18ff2021-04-19 12:53:06 +00006378 skb_gro_reset_offset(skb, hlen);
Eric Dumazeta50e2332014-03-29 21:28:21 -07006379
Eric Dumazeta50e2332014-03-29 21:28:21 -07006380 if (unlikely(skb_gro_header_hard(skb, hlen))) {
6381 eth = skb_gro_header_slow(skb, hlen, 0);
6382 if (unlikely(!eth)) {
Aaron Conole4da46ce2016-04-02 15:26:43 -04006383 net_warn_ratelimited("%s: dropping impossible skb from %s\n",
6384 __func__, napi->dev->name);
Eric Dumazeta50e2332014-03-29 21:28:21 -07006385 napi_reuse_skb(napi, skb);
6386 return NULL;
6387 }
6388 } else {
Eric Dumazeta4270d62019-05-29 15:36:10 -07006389 eth = (const struct ethhdr *)skb->data;
Eric Dumazeta50e2332014-03-29 21:28:21 -07006390 gro_pull_from_frag0(skb, hlen);
6391 NAPI_GRO_CB(skb)->frag0 += hlen;
6392 NAPI_GRO_CB(skb)->frag0_len -= hlen;
Herbert Xu76620aa2009-04-16 02:02:07 -07006393 }
Eric Dumazeta50e2332014-03-29 21:28:21 -07006394 __skb_pull(skb, hlen);
6395
6396 /*
6397 * This works because the only protocols we care about don't require
6398 * special handling.
6399 * We'll fix it up properly in napi_frags_finish()
6400 */
6401 skb->protocol = eth->h_proto;
Herbert Xu76620aa2009-04-16 02:02:07 -07006402
Herbert Xu76620aa2009-04-16 02:02:07 -07006403 return skb;
6404}
Herbert Xu76620aa2009-04-16 02:02:07 -07006405
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07006406gro_result_t napi_gro_frags(struct napi_struct *napi)
Herbert Xu76620aa2009-04-16 02:02:07 -07006407{
Geneviève Bastienb0e3f1b2018-11-27 12:52:39 -05006408 gro_result_t ret;
Herbert Xu76620aa2009-04-16 02:02:07 -07006409 struct sk_buff *skb = napi_frags_skb(napi);
Herbert Xu96e93ea2009-01-06 10:49:34 -08006410
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00006411 trace_napi_gro_frags_entry(skb);
6412
Geneviève Bastienb0e3f1b2018-11-27 12:52:39 -05006413 ret = napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
6414 trace_napi_gro_frags_exit(ret);
6415
6416 return ret;
Herbert Xu5d38a072009-01-04 16:13:40 -08006417}
6418EXPORT_SYMBOL(napi_gro_frags);
6419
Tom Herbert573e8fc2014-08-22 13:33:47 -07006420/* Compute the checksum from gro_offset and return the folded value
6421 * after adding in any pseudo checksum.
6422 */
6423__sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
6424{
6425 __wsum wsum;
6426 __sum16 sum;
6427
6428 wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
6429
6430 /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
6431 sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
Cong Wang14641932018-11-26 09:31:26 -08006432 /* See comments in __skb_checksum_complete(). */
Tom Herbert573e8fc2014-08-22 13:33:47 -07006433 if (likely(!sum)) {
6434 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
6435 !skb->csum_complete_sw)
Cong Wang7fe50ac2018-11-12 14:47:18 -08006436 netdev_rx_csum_fault(skb->dev, skb);
Tom Herbert573e8fc2014-08-22 13:33:47 -07006437 }
6438
6439 NAPI_GRO_CB(skb)->csum = wsum;
6440 NAPI_GRO_CB(skb)->csum_valid = 1;
6441
6442 return sum;
6443}
6444EXPORT_SYMBOL(__skb_gro_checksum_complete);
6445
ashwanth@codeaurora.org773fc8f2017-06-09 14:24:58 +05306446static void net_rps_send_ipi(struct softnet_data *remsd)
6447{
6448#ifdef CONFIG_RPS
6449 while (remsd) {
6450 struct softnet_data *next = remsd->rps_ipi_next;
6451
6452 if (cpu_online(remsd->cpu))
6453 smp_call_function_single_async(remsd->cpu, &remsd->csd);
6454 remsd = next;
6455 }
6456#endif
6457}
6458
Eric Dumazete326bed2010-04-22 00:22:45 -07006459/*
Zhi Yong Wu855abcf2014-01-01 04:34:50 +08006460 * net_rps_action_and_irq_enable sends any pending IPI's for rps.
Eric Dumazete326bed2010-04-22 00:22:45 -07006461 * Note: called with local irq disabled, but exits with local irq enabled.
6462 */
6463static void net_rps_action_and_irq_enable(struct softnet_data *sd)
6464{
6465#ifdef CONFIG_RPS
6466 struct softnet_data *remsd = sd->rps_ipi_list;
6467
6468 if (remsd) {
6469 sd->rps_ipi_list = NULL;
6470
6471 local_irq_enable();
6472
6473 /* Send pending IPI's to kick RPS processing on remote cpus. */
ashwanth@codeaurora.org773fc8f2017-06-09 14:24:58 +05306474 net_rps_send_ipi(remsd);
Eric Dumazete326bed2010-04-22 00:22:45 -07006475 } else
6476#endif
6477 local_irq_enable();
6478}
6479
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08006480static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
6481{
6482#ifdef CONFIG_RPS
6483 return sd->rps_ipi_list != NULL;
6484#else
6485 return false;
6486#endif
6487}
6488
Stephen Hemmingerbea33482007-10-03 16:41:36 -07006489static int process_backlog(struct napi_struct *napi, int quota)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006490{
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07006491 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
Paolo Abeni145dd5f2016-08-25 15:58:44 +02006492 bool again = true;
6493 int work = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006494
Eric Dumazete326bed2010-04-22 00:22:45 -07006495 /* Check if we have pending ipi, its better to send them now,
6496 * not waiting net_rx_action() end.
6497 */
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08006498 if (sd_has_rps_ipi_waiting(sd)) {
Eric Dumazete326bed2010-04-22 00:22:45 -07006499 local_irq_disable();
6500 net_rps_action_and_irq_enable(sd);
6501 }
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08006502
Matthias Tafelmeier3d48b532016-12-29 21:37:21 +01006503 napi->weight = dev_rx_weight;
Paolo Abeni145dd5f2016-08-25 15:58:44 +02006504 while (again) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07006505 struct sk_buff *skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006506
Changli Gao6e7676c2010-04-27 15:07:33 -07006507 while ((skb = __skb_dequeue(&sd->process_queue))) {
Julian Anastasov2c17d272015-07-09 09:59:10 +03006508 rcu_read_lock();
Changli Gao6e7676c2010-04-27 15:07:33 -07006509 __netif_receive_skb(skb);
Julian Anastasov2c17d272015-07-09 09:59:10 +03006510 rcu_read_unlock();
Tom Herbert76cc8b12010-05-20 18:37:59 +00006511 input_queue_head_incr(sd);
Paolo Abeni145dd5f2016-08-25 15:58:44 +02006512 if (++work >= quota)
Tom Herbert76cc8b12010-05-20 18:37:59 +00006513 return work;
Paolo Abeni145dd5f2016-08-25 15:58:44 +02006514
Stephen Hemmingerbea33482007-10-03 16:41:36 -07006515 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006516
Paolo Abeni145dd5f2016-08-25 15:58:44 +02006517 local_irq_disable();
Changli Gao6e7676c2010-04-27 15:07:33 -07006518 rps_lock(sd);
Tom Herbert11ef7a82014-06-30 09:50:40 -07006519 if (skb_queue_empty(&sd->input_pkt_queue)) {
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07006520 /*
6521 * Inline a custom version of __napi_complete().
6522 * only current cpu owns and manipulates this napi,
Tom Herbert11ef7a82014-06-30 09:50:40 -07006523 * and NAPI_STATE_SCHED is the only possible flag set
6524 * on backlog.
6525 * We can use a plain write instead of clear_bit(),
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07006526 * and we dont need an smp_mb() memory barrier.
6527 */
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07006528 napi->state = 0;
Paolo Abeni145dd5f2016-08-25 15:58:44 +02006529 again = false;
6530 } else {
6531 skb_queue_splice_tail_init(&sd->input_pkt_queue,
6532 &sd->process_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07006533 }
6534 rps_unlock(sd);
Paolo Abeni145dd5f2016-08-25 15:58:44 +02006535 local_irq_enable();
Changli Gao6e7676c2010-04-27 15:07:33 -07006536 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006537
Stephen Hemmingerbea33482007-10-03 16:41:36 -07006538 return work;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006539}
6540
Stephen Hemmingerbea33482007-10-03 16:41:36 -07006541/**
6542 * __napi_schedule - schedule for receive
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07006543 * @n: entry to schedule
Stephen Hemmingerbea33482007-10-03 16:41:36 -07006544 *
Eric Dumazetbc9ad162014-10-28 18:05:13 -07006545 * The entry's receive function will be scheduled to run.
6546 * Consider using __napi_schedule_irqoff() if hard irqs are masked.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07006547 */
Harvey Harrisonb5606c22008-02-13 15:03:16 -08006548void __napi_schedule(struct napi_struct *n)
Stephen Hemmingerbea33482007-10-03 16:41:36 -07006549{
6550 unsigned long flags;
6551
6552 local_irq_save(flags);
Christoph Lameter903ceff2014-08-17 12:30:35 -05006553 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07006554 local_irq_restore(flags);
6555}
6556EXPORT_SYMBOL(__napi_schedule);
6557
Eric Dumazetbc9ad162014-10-28 18:05:13 -07006558/**
Eric Dumazet39e6c822017-02-28 10:34:50 -08006559 * napi_schedule_prep - check if napi can be scheduled
6560 * @n: napi context
6561 *
6562 * Test if NAPI routine is already running, and if not mark
Jonathan Neuschäferee1a4c82020-09-05 20:32:18 +02006563 * it as running. This is used as a condition variable to
Eric Dumazet39e6c822017-02-28 10:34:50 -08006564 * insure only one NAPI poll instance runs. We also make
6565 * sure there is no pending NAPI disable.
6566 */
6567bool napi_schedule_prep(struct napi_struct *n)
6568{
6569 unsigned long val, new;
6570
6571 do {
6572 val = READ_ONCE(n->state);
6573 if (unlikely(val & NAPIF_STATE_DISABLE))
6574 return false;
6575 new = val | NAPIF_STATE_SCHED;
6576
6577 /* Sets STATE_MISSED bit if STATE_SCHED was already set
6578 * This was suggested by Alexander Duyck, as compiler
6579 * emits better code than :
6580 * if (val & NAPIF_STATE_SCHED)
6581 * new |= NAPIF_STATE_MISSED;
6582 */
6583 new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED *
6584 NAPIF_STATE_MISSED;
6585 } while (cmpxchg(&n->state, val, new) != val);
6586
6587 return !(val & NAPIF_STATE_SCHED);
6588}
6589EXPORT_SYMBOL(napi_schedule_prep);
6590
6591/**
Eric Dumazetbc9ad162014-10-28 18:05:13 -07006592 * __napi_schedule_irqoff - schedule for receive
6593 * @n: entry to schedule
6594 *
Sebastian Andrzej Siewior8380c812021-05-12 23:43:24 +02006595 * Variant of __napi_schedule() assuming hard irqs are masked.
6596 *
6597 * On PREEMPT_RT enabled kernels this maps to __napi_schedule()
6598 * because the interrupt disabled assumption might not be true
6599 * due to force-threaded interrupts and spinlock substitution.
Eric Dumazetbc9ad162014-10-28 18:05:13 -07006600 */
6601void __napi_schedule_irqoff(struct napi_struct *n)
6602{
Sebastian Andrzej Siewior8380c812021-05-12 23:43:24 +02006603 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
6604 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
6605 else
6606 __napi_schedule(n);
Eric Dumazetbc9ad162014-10-28 18:05:13 -07006607}
6608EXPORT_SYMBOL(__napi_schedule_irqoff);
6609
Eric Dumazet364b6052016-11-15 10:15:13 -08006610bool napi_complete_done(struct napi_struct *n, int work_done)
Herbert Xud565b0a2008-12-15 23:38:52 -08006611{
Eric Dumazet6f8b12d2020-04-22 09:13:27 -07006612 unsigned long flags, val, new, timeout = 0;
6613 bool ret = true;
Herbert Xud565b0a2008-12-15 23:38:52 -08006614
6615 /*
Eric Dumazet217f6972016-11-15 10:15:11 -08006616 * 1) Don't let napi dequeue from the cpu poll list
6617 * just in case its running on a different cpu.
6618 * 2) If we are busy polling, do nothing here, we have
6619 * the guarantee we will be called later.
Herbert Xud565b0a2008-12-15 23:38:52 -08006620 */
Eric Dumazet217f6972016-11-15 10:15:11 -08006621 if (unlikely(n->state & (NAPIF_STATE_NPSVC |
6622 NAPIF_STATE_IN_BUSY_POLL)))
Eric Dumazet364b6052016-11-15 10:15:13 -08006623 return false;
Herbert Xud565b0a2008-12-15 23:38:52 -08006624
Eric Dumazet6f8b12d2020-04-22 09:13:27 -07006625 if (work_done) {
6626 if (n->gro_bitmask)
Eric Dumazet7e417a62020-04-22 09:13:28 -07006627 timeout = READ_ONCE(n->dev->gro_flush_timeout);
6628 n->defer_hard_irqs_count = READ_ONCE(n->dev->napi_defer_hard_irqs);
Eric Dumazet6f8b12d2020-04-22 09:13:27 -07006629 }
6630 if (n->defer_hard_irqs_count > 0) {
6631 n->defer_hard_irqs_count--;
Eric Dumazet7e417a62020-04-22 09:13:28 -07006632 timeout = READ_ONCE(n->dev->gro_flush_timeout);
Eric Dumazet6f8b12d2020-04-22 09:13:27 -07006633 if (timeout)
6634 ret = false;
6635 }
6636 if (n->gro_bitmask) {
Paolo Abeni605108a2018-11-21 18:21:35 +01006637 /* When the NAPI instance uses a timeout and keeps postponing
6638 * it, we need to bound somehow the time packets are kept in
6639 * the GRO layer
6640 */
6641 napi_gro_flush(n, !!timeout);
Eric Dumazet3b47d302014-11-06 21:09:44 -08006642 }
Maxim Mikityanskiyc8079432020-01-21 15:09:40 +00006643
6644 gro_normal_list(n);
6645
Eric Dumazet02c16022017-02-04 15:25:02 -08006646 if (unlikely(!list_empty(&n->poll_list))) {
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08006647 /* If n->poll_list is not empty, we need to mask irqs */
6648 local_irq_save(flags);
Eric Dumazet02c16022017-02-04 15:25:02 -08006649 list_del_init(&n->poll_list);
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08006650 local_irq_restore(flags);
6651 }
Eric Dumazet39e6c822017-02-28 10:34:50 -08006652
6653 do {
6654 val = READ_ONCE(n->state);
6655
6656 WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED));
6657
Björn Töpel7fd32532020-11-30 19:51:56 +01006658 new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED |
Wei Wangcb038352021-03-16 15:36:47 -07006659 NAPIF_STATE_SCHED_THREADED |
Björn Töpel7fd32532020-11-30 19:51:56 +01006660 NAPIF_STATE_PREFER_BUSY_POLL);
Eric Dumazet39e6c822017-02-28 10:34:50 -08006661
6662 /* If STATE_MISSED was set, leave STATE_SCHED set,
6663 * because we will call napi->poll() one more time.
6664 * This C code was suggested by Alexander Duyck to help gcc.
6665 */
6666 new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED *
6667 NAPIF_STATE_SCHED;
6668 } while (cmpxchg(&n->state, val, new) != val);
6669
6670 if (unlikely(val & NAPIF_STATE_MISSED)) {
6671 __napi_schedule(n);
6672 return false;
6673 }
6674
Eric Dumazet6f8b12d2020-04-22 09:13:27 -07006675 if (timeout)
6676 hrtimer_start(&n->timer, ns_to_ktime(timeout),
6677 HRTIMER_MODE_REL_PINNED);
6678 return ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08006679}
Eric Dumazet3b47d302014-11-06 21:09:44 -08006680EXPORT_SYMBOL(napi_complete_done);
Herbert Xud565b0a2008-12-15 23:38:52 -08006681
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006682/* must be called under rcu_read_lock(), as we dont take a reference */
Eric Dumazet02d62e82015-11-18 06:30:52 -08006683static struct napi_struct *napi_by_id(unsigned int napi_id)
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006684{
6685 unsigned int hash = napi_id % HASH_SIZE(napi_hash);
6686 struct napi_struct *napi;
6687
6688 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
6689 if (napi->napi_id == napi_id)
6690 return napi;
6691
6692 return NULL;
6693}
Eric Dumazet02d62e82015-11-18 06:30:52 -08006694
6695#if defined(CONFIG_NET_RX_BUSY_POLL)
Eric Dumazet217f6972016-11-15 10:15:11 -08006696
Björn Töpel7fd32532020-11-30 19:51:56 +01006697static void __busy_poll_stop(struct napi_struct *napi, bool skip_schedule)
Eric Dumazet217f6972016-11-15 10:15:11 -08006698{
Björn Töpel7fd32532020-11-30 19:51:56 +01006699 if (!skip_schedule) {
6700 gro_normal_list(napi);
6701 __napi_schedule(napi);
6702 return;
6703 }
6704
6705 if (napi->gro_bitmask) {
6706 /* flush too old packets
6707 * If HZ < 1000, flush all packets.
6708 */
6709 napi_gro_flush(napi, HZ >= 1000);
6710 }
6711
6712 gro_normal_list(napi);
6713 clear_bit(NAPI_STATE_SCHED, &napi->state);
6714}
6715
Björn Töpel7c951caf2020-11-30 19:51:57 +01006716static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, bool prefer_busy_poll,
6717 u16 budget)
Björn Töpel7fd32532020-11-30 19:51:56 +01006718{
6719 bool skip_schedule = false;
6720 unsigned long timeout;
Eric Dumazet217f6972016-11-15 10:15:11 -08006721 int rc;
6722
Eric Dumazet39e6c822017-02-28 10:34:50 -08006723 /* Busy polling means there is a high chance device driver hard irq
6724 * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was
6725 * set in napi_schedule_prep().
6726 * Since we are about to call napi->poll() once more, we can safely
6727 * clear NAPI_STATE_MISSED.
6728 *
6729 * Note: x86 could use a single "lock and ..." instruction
6730 * to perform these two clear_bit()
6731 */
6732 clear_bit(NAPI_STATE_MISSED, &napi->state);
Eric Dumazet217f6972016-11-15 10:15:11 -08006733 clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
6734
6735 local_bh_disable();
6736
Björn Töpel7fd32532020-11-30 19:51:56 +01006737 if (prefer_busy_poll) {
6738 napi->defer_hard_irqs_count = READ_ONCE(napi->dev->napi_defer_hard_irqs);
6739 timeout = READ_ONCE(napi->dev->gro_flush_timeout);
6740 if (napi->defer_hard_irqs_count && timeout) {
6741 hrtimer_start(&napi->timer, ns_to_ktime(timeout), HRTIMER_MODE_REL_PINNED);
6742 skip_schedule = true;
6743 }
6744 }
6745
Eric Dumazet217f6972016-11-15 10:15:11 -08006746 /* All we really want here is to re-enable device interrupts.
6747 * Ideally, a new ndo_busy_poll_stop() could avoid another round.
6748 */
Björn Töpel7c951caf2020-11-30 19:51:57 +01006749 rc = napi->poll(napi, budget);
Edward Cree323ebb62019-08-06 14:53:55 +01006750 /* We can't gro_normal_list() here, because napi->poll() might have
6751 * rearmed the napi (napi_complete_done()) in which case it could
6752 * already be running on another CPU.
6753 */
Björn Töpel7c951caf2020-11-30 19:51:57 +01006754 trace_napi_poll(napi, rc, budget);
Eric Dumazet217f6972016-11-15 10:15:11 -08006755 netpoll_poll_unlock(have_poll_lock);
Björn Töpel7c951caf2020-11-30 19:51:57 +01006756 if (rc == budget)
Björn Töpel7fd32532020-11-30 19:51:56 +01006757 __busy_poll_stop(napi, skip_schedule);
Eric Dumazet217f6972016-11-15 10:15:11 -08006758 local_bh_enable();
Eric Dumazet217f6972016-11-15 10:15:11 -08006759}
6760
Sridhar Samudrala7db6b042017-03-24 10:08:24 -07006761void napi_busy_loop(unsigned int napi_id,
6762 bool (*loop_end)(void *, unsigned long),
Björn Töpel7c951caf2020-11-30 19:51:57 +01006763 void *loop_end_arg, bool prefer_busy_poll, u16 budget)
Eric Dumazet02d62e82015-11-18 06:30:52 -08006764{
Sridhar Samudrala7db6b042017-03-24 10:08:24 -07006765 unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
Eric Dumazet217f6972016-11-15 10:15:11 -08006766 int (*napi_poll)(struct napi_struct *napi, int budget);
Eric Dumazet217f6972016-11-15 10:15:11 -08006767 void *have_poll_lock = NULL;
Eric Dumazet02d62e82015-11-18 06:30:52 -08006768 struct napi_struct *napi;
Eric Dumazet217f6972016-11-15 10:15:11 -08006769
6770restart:
Eric Dumazet217f6972016-11-15 10:15:11 -08006771 napi_poll = NULL;
Eric Dumazet02d62e82015-11-18 06:30:52 -08006772
Eric Dumazet2a028ec2015-11-18 06:30:53 -08006773 rcu_read_lock();
Eric Dumazet02d62e82015-11-18 06:30:52 -08006774
Alexander Duyck545cd5e2017-03-24 10:07:53 -07006775 napi = napi_by_id(napi_id);
Eric Dumazet02d62e82015-11-18 06:30:52 -08006776 if (!napi)
6777 goto out;
6778
Eric Dumazet217f6972016-11-15 10:15:11 -08006779 preempt_disable();
6780 for (;;) {
Alexander Duyck2b5cd0d2017-03-24 10:08:12 -07006781 int work = 0;
6782
Eric Dumazet2a028ec2015-11-18 06:30:53 -08006783 local_bh_disable();
Eric Dumazet217f6972016-11-15 10:15:11 -08006784 if (!napi_poll) {
6785 unsigned long val = READ_ONCE(napi->state);
6786
6787 /* If multiple threads are competing for this napi,
6788 * we avoid dirtying napi->state as much as we can.
6789 */
6790 if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
Björn Töpel7fd32532020-11-30 19:51:56 +01006791 NAPIF_STATE_IN_BUSY_POLL)) {
6792 if (prefer_busy_poll)
6793 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
Eric Dumazet217f6972016-11-15 10:15:11 -08006794 goto count;
Björn Töpel7fd32532020-11-30 19:51:56 +01006795 }
Eric Dumazet217f6972016-11-15 10:15:11 -08006796 if (cmpxchg(&napi->state, val,
6797 val | NAPIF_STATE_IN_BUSY_POLL |
Björn Töpel7fd32532020-11-30 19:51:56 +01006798 NAPIF_STATE_SCHED) != val) {
6799 if (prefer_busy_poll)
6800 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
Eric Dumazet217f6972016-11-15 10:15:11 -08006801 goto count;
Björn Töpel7fd32532020-11-30 19:51:56 +01006802 }
Eric Dumazet217f6972016-11-15 10:15:11 -08006803 have_poll_lock = netpoll_poll_lock(napi);
6804 napi_poll = napi->poll;
6805 }
Björn Töpel7c951caf2020-11-30 19:51:57 +01006806 work = napi_poll(napi, budget);
6807 trace_napi_poll(napi, work, budget);
Edward Cree323ebb62019-08-06 14:53:55 +01006808 gro_normal_list(napi);
Eric Dumazet217f6972016-11-15 10:15:11 -08006809count:
Alexander Duyck2b5cd0d2017-03-24 10:08:12 -07006810 if (work > 0)
Sridhar Samudrala7db6b042017-03-24 10:08:24 -07006811 __NET_ADD_STATS(dev_net(napi->dev),
Alexander Duyck2b5cd0d2017-03-24 10:08:12 -07006812 LINUX_MIB_BUSYPOLLRXPACKETS, work);
Eric Dumazet2a028ec2015-11-18 06:30:53 -08006813 local_bh_enable();
Eric Dumazet02d62e82015-11-18 06:30:52 -08006814
Sridhar Samudrala7db6b042017-03-24 10:08:24 -07006815 if (!loop_end || loop_end(loop_end_arg, start_time))
Eric Dumazet217f6972016-11-15 10:15:11 -08006816 break;
Eric Dumazet02d62e82015-11-18 06:30:52 -08006817
Eric Dumazet217f6972016-11-15 10:15:11 -08006818 if (unlikely(need_resched())) {
6819 if (napi_poll)
Björn Töpel7c951caf2020-11-30 19:51:57 +01006820 busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget);
Eric Dumazet217f6972016-11-15 10:15:11 -08006821 preempt_enable();
6822 rcu_read_unlock();
6823 cond_resched();
Sridhar Samudrala7db6b042017-03-24 10:08:24 -07006824 if (loop_end(loop_end_arg, start_time))
Alexander Duyck2b5cd0d2017-03-24 10:08:12 -07006825 return;
Eric Dumazet217f6972016-11-15 10:15:11 -08006826 goto restart;
6827 }
Linus Torvalds6cdf89b2016-12-12 10:48:02 -08006828 cpu_relax();
Eric Dumazet217f6972016-11-15 10:15:11 -08006829 }
6830 if (napi_poll)
Björn Töpel7c951caf2020-11-30 19:51:57 +01006831 busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget);
Eric Dumazet217f6972016-11-15 10:15:11 -08006832 preempt_enable();
Eric Dumazet02d62e82015-11-18 06:30:52 -08006833out:
Eric Dumazet2a028ec2015-11-18 06:30:53 -08006834 rcu_read_unlock();
Eric Dumazet02d62e82015-11-18 06:30:52 -08006835}
Sridhar Samudrala7db6b042017-03-24 10:08:24 -07006836EXPORT_SYMBOL(napi_busy_loop);
Eric Dumazet02d62e82015-11-18 06:30:52 -08006837
6838#endif /* CONFIG_NET_RX_BUSY_POLL */
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006839
Eric Dumazet149d6ad2016-11-08 11:07:28 -08006840static void napi_hash_add(struct napi_struct *napi)
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006841{
Jakub Kicinski4d092dd2020-09-09 10:37:52 -07006842 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state))
Eric Dumazet52bd2d62015-11-18 06:30:50 -08006843 return;
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006844
Eric Dumazet52bd2d62015-11-18 06:30:50 -08006845 spin_lock(&napi_hash_lock);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006846
Alexander Duyck545cd5e2017-03-24 10:07:53 -07006847 /* 0..NR_CPUS range is reserved for sender_cpu use */
Eric Dumazet52bd2d62015-11-18 06:30:50 -08006848 do {
Alexander Duyck545cd5e2017-03-24 10:07:53 -07006849 if (unlikely(++napi_gen_id < MIN_NAPI_ID))
6850 napi_gen_id = MIN_NAPI_ID;
Eric Dumazet52bd2d62015-11-18 06:30:50 -08006851 } while (napi_by_id(napi_gen_id));
6852 napi->napi_id = napi_gen_id;
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006853
Eric Dumazet52bd2d62015-11-18 06:30:50 -08006854 hlist_add_head_rcu(&napi->napi_hash_node,
6855 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006856
Eric Dumazet52bd2d62015-11-18 06:30:50 -08006857 spin_unlock(&napi_hash_lock);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006858}
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006859
6860/* Warning : caller is responsible to make sure rcu grace period
6861 * is respected before freeing memory containing @napi
6862 */
Jakub Kicinski5198d5452020-09-09 10:37:51 -07006863static void napi_hash_del(struct napi_struct *napi)
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006864{
6865 spin_lock(&napi_hash_lock);
6866
Jakub Kicinski4d092dd2020-09-09 10:37:52 -07006867 hlist_del_init_rcu(&napi->napi_hash_node);
Jakub Kicinski5198d5452020-09-09 10:37:51 -07006868
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006869 spin_unlock(&napi_hash_lock);
6870}
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006871
Eric Dumazet3b47d302014-11-06 21:09:44 -08006872static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
6873{
6874 struct napi_struct *napi;
6875
6876 napi = container_of(timer, struct napi_struct, timer);
Eric Dumazet39e6c822017-02-28 10:34:50 -08006877
6878 /* Note : we use a relaxed variant of napi_schedule_prep() not setting
6879 * NAPI_STATE_MISSED, since we do not react to a device IRQ.
6880 */
Eric Dumazet6f8b12d2020-04-22 09:13:27 -07006881 if (!napi_disable_pending(napi) &&
Björn Töpel7fd32532020-11-30 19:51:56 +01006882 !test_and_set_bit(NAPI_STATE_SCHED, &napi->state)) {
6883 clear_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
Eric Dumazet39e6c822017-02-28 10:34:50 -08006884 __napi_schedule_irqoff(napi);
Björn Töpel7fd32532020-11-30 19:51:56 +01006885 }
Eric Dumazet3b47d302014-11-06 21:09:44 -08006886
6887 return HRTIMER_NORESTART;
6888}
6889
David S. Miller7c4ec742018-07-20 23:37:55 -07006890static void init_gro_hash(struct napi_struct *napi)
Herbert Xud565b0a2008-12-15 23:38:52 -08006891{
David Miller07d78362018-06-24 14:14:02 +09006892 int i;
6893
Li RongQing6312fe72018-07-05 14:34:32 +08006894 for (i = 0; i < GRO_HASH_BUCKETS; i++) {
6895 INIT_LIST_HEAD(&napi->gro_hash[i].list);
6896 napi->gro_hash[i].count = 0;
6897 }
David S. Miller7c4ec742018-07-20 23:37:55 -07006898 napi->gro_bitmask = 0;
6899}
6900
Wei Wang5fdd2f02021-02-08 11:34:10 -08006901int dev_set_threaded(struct net_device *dev, bool threaded)
6902{
6903 struct napi_struct *napi;
6904 int err = 0;
6905
6906 if (dev->threaded == threaded)
6907 return 0;
6908
6909 if (threaded) {
6910 list_for_each_entry(napi, &dev->napi_list, dev_list) {
6911 if (!napi->thread) {
6912 err = napi_kthread_create(napi);
6913 if (err) {
6914 threaded = false;
6915 break;
6916 }
6917 }
6918 }
6919 }
6920
6921 dev->threaded = threaded;
6922
6923 /* Make sure kthread is created before THREADED bit
6924 * is set.
6925 */
6926 smp_mb__before_atomic();
6927
6928 /* Setting/unsetting threaded mode on a napi might not immediately
6929 * take effect, if the current napi instance is actively being
6930 * polled. In this case, the switch between threaded mode and
6931 * softirq mode will happen in the next round of napi_schedule().
6932 * This should not cause hiccups/stalls to the live traffic.
6933 */
6934 list_for_each_entry(napi, &dev->napi_list, dev_list) {
6935 if (threaded)
6936 set_bit(NAPI_STATE_THREADED, &napi->state);
6937 else
6938 clear_bit(NAPI_STATE_THREADED, &napi->state);
6939 }
6940
6941 return err;
6942}
Lorenzo Bianconi8f648602021-03-14 15:49:19 +01006943EXPORT_SYMBOL(dev_set_threaded);
Wei Wang5fdd2f02021-02-08 11:34:10 -08006944
David S. Miller7c4ec742018-07-20 23:37:55 -07006945void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
6946 int (*poll)(struct napi_struct *, int), int weight)
6947{
Jakub Kicinski4d092dd2020-09-09 10:37:52 -07006948 if (WARN_ON(test_and_set_bit(NAPI_STATE_LISTED, &napi->state)))
6949 return;
6950
David S. Miller7c4ec742018-07-20 23:37:55 -07006951 INIT_LIST_HEAD(&napi->poll_list);
Jakub Kicinski4d092dd2020-09-09 10:37:52 -07006952 INIT_HLIST_NODE(&napi->napi_hash_node);
David S. Miller7c4ec742018-07-20 23:37:55 -07006953 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
6954 napi->timer.function = napi_watchdog;
6955 init_gro_hash(napi);
Herbert Xu5d38a072009-01-04 16:13:40 -08006956 napi->skb = NULL;
Edward Cree323ebb62019-08-06 14:53:55 +01006957 INIT_LIST_HEAD(&napi->rx_list);
6958 napi->rx_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08006959 napi->poll = poll;
Eric Dumazet82dc3c63c2013-03-05 15:57:22 +00006960 if (weight > NAPI_POLL_WEIGHT)
Qian Caibf29e9e2018-12-01 21:11:19 -05006961 netdev_err_once(dev, "%s() called with weight %d\n", __func__,
6962 weight);
Herbert Xud565b0a2008-12-15 23:38:52 -08006963 napi->weight = weight;
Herbert Xud565b0a2008-12-15 23:38:52 -08006964 napi->dev = dev;
Herbert Xu5d38a072009-01-04 16:13:40 -08006965#ifdef CONFIG_NETPOLL
Herbert Xud565b0a2008-12-15 23:38:52 -08006966 napi->poll_owner = -1;
6967#endif
6968 set_bit(NAPI_STATE_SCHED, &napi->state);
Jakub Kicinski96e97bc2020-08-26 12:40:06 -07006969 set_bit(NAPI_STATE_NPSVC, &napi->state);
6970 list_add_rcu(&napi->dev_list, &dev->napi_list);
Eric Dumazet93d05d42015-11-18 06:31:03 -08006971 napi_hash_add(napi);
Wei Wang29863d42021-02-08 11:34:09 -08006972 /* Create kthread for this napi if dev->threaded is set.
6973 * Clear dev->threaded if kthread creation failed so that
6974 * threaded mode will not be enabled in napi_enable().
6975 */
6976 if (dev->threaded && napi_kthread_create(napi))
6977 dev->threaded = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08006978}
6979EXPORT_SYMBOL(netif_napi_add);
6980
Eric Dumazet3b47d302014-11-06 21:09:44 -08006981void napi_disable(struct napi_struct *n)
6982{
6983 might_sleep();
6984 set_bit(NAPI_STATE_DISABLE, &n->state);
6985
6986 while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
6987 msleep(1);
Neil Horman2d8bff1262015-09-23 14:57:58 -04006988 while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state))
6989 msleep(1);
Eric Dumazet3b47d302014-11-06 21:09:44 -08006990
6991 hrtimer_cancel(&n->timer);
6992
Björn Töpel7fd32532020-11-30 19:51:56 +01006993 clear_bit(NAPI_STATE_PREFER_BUSY_POLL, &n->state);
Eric Dumazet3b47d302014-11-06 21:09:44 -08006994 clear_bit(NAPI_STATE_DISABLE, &n->state);
Wei Wang29863d42021-02-08 11:34:09 -08006995 clear_bit(NAPI_STATE_THREADED, &n->state);
Eric Dumazet3b47d302014-11-06 21:09:44 -08006996}
6997EXPORT_SYMBOL(napi_disable);
6998
Wei Wang29863d42021-02-08 11:34:09 -08006999/**
7000 * napi_enable - enable NAPI scheduling
7001 * @n: NAPI context
7002 *
7003 * Resume NAPI from being scheduled on this context.
7004 * Must be paired with napi_disable.
7005 */
7006void napi_enable(struct napi_struct *n)
7007{
7008 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
7009 smp_mb__before_atomic();
7010 clear_bit(NAPI_STATE_SCHED, &n->state);
7011 clear_bit(NAPI_STATE_NPSVC, &n->state);
7012 if (n->dev->threaded && n->thread)
7013 set_bit(NAPI_STATE_THREADED, &n->state);
7014}
7015EXPORT_SYMBOL(napi_enable);
7016
David Miller07d78362018-06-24 14:14:02 +09007017static void flush_gro_hash(struct napi_struct *napi)
David Millerd4546c22018-06-24 14:13:49 +09007018{
David Miller07d78362018-06-24 14:14:02 +09007019 int i;
David Millerd4546c22018-06-24 14:13:49 +09007020
David Miller07d78362018-06-24 14:14:02 +09007021 for (i = 0; i < GRO_HASH_BUCKETS; i++) {
7022 struct sk_buff *skb, *n;
7023
Li RongQing6312fe72018-07-05 14:34:32 +08007024 list_for_each_entry_safe(skb, n, &napi->gro_hash[i].list, list)
David Miller07d78362018-06-24 14:14:02 +09007025 kfree_skb(skb);
Li RongQing6312fe72018-07-05 14:34:32 +08007026 napi->gro_hash[i].count = 0;
David Miller07d78362018-06-24 14:14:02 +09007027 }
David Millerd4546c22018-06-24 14:13:49 +09007028}
7029
Eric Dumazet93d05d42015-11-18 06:31:03 -08007030/* Must be called in process context */
Jakub Kicinski5198d5452020-09-09 10:37:51 -07007031void __netif_napi_del(struct napi_struct *napi)
Herbert Xud565b0a2008-12-15 23:38:52 -08007032{
Jakub Kicinski4d092dd2020-09-09 10:37:52 -07007033 if (!test_and_clear_bit(NAPI_STATE_LISTED, &napi->state))
7034 return;
7035
Jakub Kicinski5198d5452020-09-09 10:37:51 -07007036 napi_hash_del(napi);
Jakub Kicinski5251ef82020-09-09 10:37:53 -07007037 list_del_rcu(&napi->dev_list);
Herbert Xu76620aa2009-04-16 02:02:07 -07007038 napi_free_frags(napi);
Herbert Xud565b0a2008-12-15 23:38:52 -08007039
David Miller07d78362018-06-24 14:14:02 +09007040 flush_gro_hash(napi);
Li RongQingd9f37d02018-07-13 14:41:36 +08007041 napi->gro_bitmask = 0;
Wei Wang29863d42021-02-08 11:34:09 -08007042
7043 if (napi->thread) {
7044 kthread_stop(napi->thread);
7045 napi->thread = NULL;
7046 }
Herbert Xud565b0a2008-12-15 23:38:52 -08007047}
Jakub Kicinski5198d5452020-09-09 10:37:51 -07007048EXPORT_SYMBOL(__netif_napi_del);
Herbert Xud565b0a2008-12-15 23:38:52 -08007049
Felix Fietkau898f8012021-02-08 11:34:08 -08007050static int __napi_poll(struct napi_struct *n, bool *repoll)
Herbert Xu726ce702014-12-21 07:16:21 +11007051{
Herbert Xu726ce702014-12-21 07:16:21 +11007052 int work, weight;
7053
Herbert Xu726ce702014-12-21 07:16:21 +11007054 weight = n->weight;
7055
7056 /* This NAPI_STATE_SCHED test is for avoiding a race
7057 * with netpoll's poll_napi(). Only the entity which
7058 * obtains the lock and sees NAPI_STATE_SCHED set will
7059 * actually make the ->poll() call. Therefore we avoid
7060 * accidentally calling ->poll() when NAPI is not scheduled.
7061 */
7062 work = 0;
7063 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
7064 work = n->poll(n, weight);
Jesper Dangaard Brouer1db19db2016-07-07 18:01:32 +02007065 trace_napi_poll(n, work, weight);
Herbert Xu726ce702014-12-21 07:16:21 +11007066 }
7067
Eric Dumazet427d5832020-06-17 09:40:51 -07007068 if (unlikely(work > weight))
7069 pr_err_once("NAPI poll function %pS returned %d, exceeding its budget of %d.\n",
7070 n->poll, work, weight);
Herbert Xu726ce702014-12-21 07:16:21 +11007071
7072 if (likely(work < weight))
Felix Fietkau898f8012021-02-08 11:34:08 -08007073 return work;
Herbert Xu726ce702014-12-21 07:16:21 +11007074
7075 /* Drivers must not modify the NAPI state if they
7076 * consume the entire weight. In such cases this code
7077 * still "owns" the NAPI instance and therefore can
7078 * move the instance around on the list at-will.
7079 */
7080 if (unlikely(napi_disable_pending(n))) {
7081 napi_complete(n);
Felix Fietkau898f8012021-02-08 11:34:08 -08007082 return work;
Herbert Xu726ce702014-12-21 07:16:21 +11007083 }
7084
Björn Töpel7fd32532020-11-30 19:51:56 +01007085 /* The NAPI context has more processing work, but busy-polling
7086 * is preferred. Exit early.
7087 */
7088 if (napi_prefer_busy_poll(n)) {
7089 if (napi_complete_done(n, work)) {
7090 /* If timeout is not set, we need to make sure
7091 * that the NAPI is re-scheduled.
7092 */
7093 napi_schedule(n);
7094 }
Felix Fietkau898f8012021-02-08 11:34:08 -08007095 return work;
Björn Töpel7fd32532020-11-30 19:51:56 +01007096 }
7097
Li RongQingd9f37d02018-07-13 14:41:36 +08007098 if (n->gro_bitmask) {
Herbert Xu726ce702014-12-21 07:16:21 +11007099 /* flush too old packets
7100 * If HZ < 1000, flush all packets.
7101 */
7102 napi_gro_flush(n, HZ >= 1000);
7103 }
7104
Maxim Mikityanskiyc8079432020-01-21 15:09:40 +00007105 gro_normal_list(n);
7106
Herbert Xu001ce542014-12-21 07:16:22 +11007107 /* Some drivers may have called napi_schedule
7108 * prior to exhausting their budget.
7109 */
7110 if (unlikely(!list_empty(&n->poll_list))) {
7111 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
7112 n->dev ? n->dev->name : "backlog");
Felix Fietkau898f8012021-02-08 11:34:08 -08007113 return work;
Herbert Xu001ce542014-12-21 07:16:22 +11007114 }
7115
Felix Fietkau898f8012021-02-08 11:34:08 -08007116 *repoll = true;
Herbert Xu726ce702014-12-21 07:16:21 +11007117
Felix Fietkau898f8012021-02-08 11:34:08 -08007118 return work;
7119}
7120
7121static int napi_poll(struct napi_struct *n, struct list_head *repoll)
7122{
7123 bool do_repoll = false;
7124 void *have;
7125 int work;
7126
7127 list_del_init(&n->poll_list);
7128
7129 have = netpoll_poll_lock(n);
7130
7131 work = __napi_poll(n, &do_repoll);
7132
7133 if (do_repoll)
7134 list_add_tail(&n->poll_list, repoll);
7135
Herbert Xu726ce702014-12-21 07:16:21 +11007136 netpoll_poll_unlock(have);
7137
7138 return work;
7139}
7140
Wei Wang29863d42021-02-08 11:34:09 -08007141static int napi_thread_wait(struct napi_struct *napi)
7142{
Wei Wangcb038352021-03-16 15:36:47 -07007143 bool woken = false;
7144
Wei Wang29863d42021-02-08 11:34:09 -08007145 set_current_state(TASK_INTERRUPTIBLE);
7146
Paolo Abeni27f0ad72021-04-09 17:24:17 +02007147 while (!kthread_should_stop()) {
Wei Wangcb038352021-03-16 15:36:47 -07007148 /* Testing SCHED_THREADED bit here to make sure the current
7149 * kthread owns this napi and could poll on this napi.
7150 * Testing SCHED bit is not enough because SCHED bit might be
7151 * set by some other busy poll thread or by napi_disable().
7152 */
7153 if (test_bit(NAPI_STATE_SCHED_THREADED, &napi->state) || woken) {
Wei Wang29863d42021-02-08 11:34:09 -08007154 WARN_ON(!list_empty(&napi->poll_list));
7155 __set_current_state(TASK_RUNNING);
7156 return 0;
7157 }
7158
7159 schedule();
Wei Wangcb038352021-03-16 15:36:47 -07007160 /* woken being true indicates this thread owns this napi. */
7161 woken = true;
Wei Wang29863d42021-02-08 11:34:09 -08007162 set_current_state(TASK_INTERRUPTIBLE);
7163 }
7164 __set_current_state(TASK_RUNNING);
Paolo Abeni27f0ad72021-04-09 17:24:17 +02007165
Wei Wang29863d42021-02-08 11:34:09 -08007166 return -1;
7167}
7168
7169static int napi_threaded_poll(void *data)
7170{
7171 struct napi_struct *napi = data;
7172 void *have;
7173
7174 while (!napi_thread_wait(napi)) {
7175 for (;;) {
7176 bool repoll = false;
7177
7178 local_bh_disable();
7179
7180 have = netpoll_poll_lock(napi);
7181 __napi_poll(napi, &repoll);
7182 netpoll_poll_unlock(have);
7183
Wei Wang29863d42021-02-08 11:34:09 -08007184 local_bh_enable();
7185
7186 if (!repoll)
7187 break;
7188
7189 cond_resched();
7190 }
7191 }
7192 return 0;
7193}
7194
Emese Revfy0766f782016-06-20 20:42:34 +02007195static __latent_entropy void net_rx_action(struct softirq_action *h)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007196{
Christoph Lameter903ceff2014-08-17 12:30:35 -05007197 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
Matthew Whitehead7acf8a12017-04-19 12:37:10 -04007198 unsigned long time_limit = jiffies +
7199 usecs_to_jiffies(netdev_budget_usecs);
Stephen Hemminger51b0bde2005-06-23 20:14:40 -07007200 int budget = netdev_budget;
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08007201 LIST_HEAD(list);
7202 LIST_HEAD(repoll);
Matt Mackall53fb95d2005-08-11 19:27:43 -07007203
Linus Torvalds1da177e2005-04-16 15:20:36 -07007204 local_irq_disable();
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08007205 list_splice_init(&sd->poll_list, &list);
7206 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007207
Herbert Xuceb8d5b2014-12-21 07:16:25 +11007208 for (;;) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07007209 struct napi_struct *n;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007210
Herbert Xuceb8d5b2014-12-21 07:16:25 +11007211 if (list_empty(&list)) {
7212 if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
Alexander Lobakinfec6e492021-02-13 14:12:02 +00007213 return;
Herbert Xuceb8d5b2014-12-21 07:16:25 +11007214 break;
7215 }
7216
Herbert Xu6bd373e2014-12-21 07:16:24 +11007217 n = list_first_entry(&list, struct napi_struct, poll_list);
7218 budget -= napi_poll(n, &repoll);
7219
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08007220 /* If softirq window is exhausted then punt.
Stephen Hemminger24f8b232008-11-03 17:14:38 -08007221 * Allow this to run for 2 jiffies since which will allow
7222 * an average latency of 1.5/HZ.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07007223 */
Herbert Xuceb8d5b2014-12-21 07:16:25 +11007224 if (unlikely(budget <= 0 ||
7225 time_after_eq(jiffies, time_limit))) {
7226 sd->time_squeeze++;
7227 break;
7228 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007229 }
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08007230
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08007231 local_irq_disable();
7232
7233 list_splice_tail_init(&sd->poll_list, &list);
7234 list_splice_tail(&repoll, &list);
7235 list_splice(&list, &sd->poll_list);
7236 if (!list_empty(&sd->poll_list))
7237 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
7238
Eric Dumazete326bed2010-04-22 00:22:45 -07007239 net_rps_action_and_irq_enable(sd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007240}
7241
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02007242struct netdev_adjacent {
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007243 struct net_device *dev;
Veaceslav Falico5d261912013-08-28 23:25:05 +02007244
7245 /* upper master flag, there can only be one master device per list */
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007246 bool master;
Veaceslav Falico5d261912013-08-28 23:25:05 +02007247
Taehee Yoo32b6d342019-10-21 18:47:56 +00007248 /* lookup ignore flag */
7249 bool ignore;
7250
Veaceslav Falico5d261912013-08-28 23:25:05 +02007251 /* counter for the number of times this device was added to us */
7252 u16 ref_nr;
7253
Veaceslav Falico402dae92013-09-25 09:20:09 +02007254 /* private field for the users */
7255 void *private;
7256
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007257 struct list_head list;
7258 struct rcu_head rcu;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007259};
7260
Michal Kubeček6ea29da2015-09-24 10:59:05 +02007261static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
Veaceslav Falico2f268f12013-09-25 09:20:07 +02007262 struct list_head *adj_list)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007263{
Veaceslav Falico5d261912013-08-28 23:25:05 +02007264 struct netdev_adjacent *adj;
Veaceslav Falico5d261912013-08-28 23:25:05 +02007265
Veaceslav Falico2f268f12013-09-25 09:20:07 +02007266 list_for_each_entry(adj, adj_list, list) {
Veaceslav Falico5d261912013-08-28 23:25:05 +02007267 if (adj->dev == adj_dev)
7268 return adj;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007269 }
7270 return NULL;
7271}
7272
Taehee Yooeff74232020-09-25 18:13:12 +00007273static int ____netdev_has_upper_dev(struct net_device *upper_dev,
7274 struct netdev_nested_priv *priv)
David Ahernf1170fd2016-10-17 19:15:51 -07007275{
Taehee Yooeff74232020-09-25 18:13:12 +00007276 struct net_device *dev = (struct net_device *)priv->data;
David Ahernf1170fd2016-10-17 19:15:51 -07007277
7278 return upper_dev == dev;
7279}
7280
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007281/**
7282 * netdev_has_upper_dev - Check if device is linked to an upper device
7283 * @dev: device
7284 * @upper_dev: upper device to check
7285 *
7286 * Find out if a device is linked to specified upper device and return true
7287 * in case it is. Note that this checks only immediate upper device,
7288 * not through a complete stack of devices. The caller must hold the RTNL lock.
7289 */
7290bool netdev_has_upper_dev(struct net_device *dev,
7291 struct net_device *upper_dev)
7292{
Taehee Yooeff74232020-09-25 18:13:12 +00007293 struct netdev_nested_priv priv = {
7294 .data = (void *)upper_dev,
7295 };
7296
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007297 ASSERT_RTNL();
7298
Taehee Yoo32b6d342019-10-21 18:47:56 +00007299 return netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
Taehee Yooeff74232020-09-25 18:13:12 +00007300 &priv);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007301}
7302EXPORT_SYMBOL(netdev_has_upper_dev);
7303
7304/**
Mauro Carvalho Chehabc1639be2020-11-16 11:17:58 +01007305 * netdev_has_upper_dev_all_rcu - Check if device is linked to an upper device
David Ahern1a3f0602016-10-17 19:15:44 -07007306 * @dev: device
7307 * @upper_dev: upper device to check
7308 *
7309 * Find out if a device is linked to specified upper device and return true
7310 * in case it is. Note that this checks the entire upper device chain.
7311 * The caller must hold rcu lock.
7312 */
7313
David Ahern1a3f0602016-10-17 19:15:44 -07007314bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
7315 struct net_device *upper_dev)
7316{
Taehee Yooeff74232020-09-25 18:13:12 +00007317 struct netdev_nested_priv priv = {
7318 .data = (void *)upper_dev,
7319 };
7320
Taehee Yoo32b6d342019-10-21 18:47:56 +00007321 return !!netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
Taehee Yooeff74232020-09-25 18:13:12 +00007322 &priv);
David Ahern1a3f0602016-10-17 19:15:44 -07007323}
7324EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu);
7325
7326/**
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007327 * netdev_has_any_upper_dev - Check if device is linked to some device
7328 * @dev: device
7329 *
7330 * Find out if a device is linked to an upper device and return true in case
7331 * it is. The caller must hold the RTNL lock.
7332 */
Ido Schimmel25cc72a2017-09-01 10:52:31 +02007333bool netdev_has_any_upper_dev(struct net_device *dev)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007334{
7335 ASSERT_RTNL();
7336
David Ahernf1170fd2016-10-17 19:15:51 -07007337 return !list_empty(&dev->adj_list.upper);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007338}
Ido Schimmel25cc72a2017-09-01 10:52:31 +02007339EXPORT_SYMBOL(netdev_has_any_upper_dev);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007340
7341/**
7342 * netdev_master_upper_dev_get - Get master upper device
7343 * @dev: device
7344 *
7345 * Find a master upper device and return pointer to it or NULL in case
7346 * it's not there. The caller must hold the RTNL lock.
7347 */
7348struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
7349{
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02007350 struct netdev_adjacent *upper;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007351
7352 ASSERT_RTNL();
7353
Veaceslav Falico2f268f12013-09-25 09:20:07 +02007354 if (list_empty(&dev->adj_list.upper))
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007355 return NULL;
7356
Veaceslav Falico2f268f12013-09-25 09:20:07 +02007357 upper = list_first_entry(&dev->adj_list.upper,
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02007358 struct netdev_adjacent, list);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007359 if (likely(upper->master))
7360 return upper->dev;
7361 return NULL;
7362}
7363EXPORT_SYMBOL(netdev_master_upper_dev_get);
7364
Taehee Yoo32b6d342019-10-21 18:47:56 +00007365static struct net_device *__netdev_master_upper_dev_get(struct net_device *dev)
7366{
7367 struct netdev_adjacent *upper;
7368
7369 ASSERT_RTNL();
7370
7371 if (list_empty(&dev->adj_list.upper))
7372 return NULL;
7373
7374 upper = list_first_entry(&dev->adj_list.upper,
7375 struct netdev_adjacent, list);
7376 if (likely(upper->master) && !upper->ignore)
7377 return upper->dev;
7378 return NULL;
7379}
7380
David Ahern0f524a82016-10-17 19:15:52 -07007381/**
7382 * netdev_has_any_lower_dev - Check if device is linked to some device
7383 * @dev: device
7384 *
7385 * Find out if a device is linked to a lower device and return true in case
7386 * it is. The caller must hold the RTNL lock.
7387 */
7388static bool netdev_has_any_lower_dev(struct net_device *dev)
7389{
7390 ASSERT_RTNL();
7391
7392 return !list_empty(&dev->adj_list.lower);
7393}
7394
Veaceslav Falicob6ccba42013-09-25 09:20:23 +02007395void *netdev_adjacent_get_private(struct list_head *adj_list)
7396{
7397 struct netdev_adjacent *adj;
7398
7399 adj = list_entry(adj_list, struct netdev_adjacent, list);
7400
7401 return adj->private;
7402}
7403EXPORT_SYMBOL(netdev_adjacent_get_private);
7404
Veaceslav Falico31088a12013-09-25 09:20:12 +02007405/**
Vlad Yasevich44a40852014-05-16 17:20:38 -04007406 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
7407 * @dev: device
7408 * @iter: list_head ** of the current position
7409 *
7410 * Gets the next device from the dev's upper list, starting from iter
7411 * position. The caller must hold RCU read lock.
7412 */
7413struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
7414 struct list_head **iter)
7415{
7416 struct netdev_adjacent *upper;
7417
7418 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
7419
7420 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7421
7422 if (&upper->list == &dev->adj_list.upper)
7423 return NULL;
7424
7425 *iter = &upper->list;
7426
7427 return upper->dev;
7428}
7429EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
7430
Taehee Yoo32b6d342019-10-21 18:47:56 +00007431static struct net_device *__netdev_next_upper_dev(struct net_device *dev,
7432 struct list_head **iter,
7433 bool *ignore)
Taehee Yoo5343da42019-10-21 18:47:50 +00007434{
7435 struct netdev_adjacent *upper;
7436
7437 upper = list_entry((*iter)->next, struct netdev_adjacent, list);
7438
7439 if (&upper->list == &dev->adj_list.upper)
7440 return NULL;
7441
7442 *iter = &upper->list;
Taehee Yoo32b6d342019-10-21 18:47:56 +00007443 *ignore = upper->ignore;
Taehee Yoo5343da42019-10-21 18:47:50 +00007444
7445 return upper->dev;
7446}
7447
David Ahern1a3f0602016-10-17 19:15:44 -07007448static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev,
7449 struct list_head **iter)
7450{
7451 struct netdev_adjacent *upper;
7452
7453 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
7454
7455 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7456
7457 if (&upper->list == &dev->adj_list.upper)
7458 return NULL;
7459
7460 *iter = &upper->list;
7461
7462 return upper->dev;
7463}
7464
Taehee Yoo32b6d342019-10-21 18:47:56 +00007465static int __netdev_walk_all_upper_dev(struct net_device *dev,
7466 int (*fn)(struct net_device *dev,
Taehee Yooeff74232020-09-25 18:13:12 +00007467 struct netdev_nested_priv *priv),
7468 struct netdev_nested_priv *priv)
Taehee Yoo5343da42019-10-21 18:47:50 +00007469{
7470 struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7471 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7472 int ret, cur = 0;
Taehee Yoo32b6d342019-10-21 18:47:56 +00007473 bool ignore;
Taehee Yoo5343da42019-10-21 18:47:50 +00007474
7475 now = dev;
7476 iter = &dev->adj_list.upper;
7477
7478 while (1) {
7479 if (now != dev) {
Taehee Yooeff74232020-09-25 18:13:12 +00007480 ret = fn(now, priv);
Taehee Yoo5343da42019-10-21 18:47:50 +00007481 if (ret)
7482 return ret;
7483 }
7484
7485 next = NULL;
7486 while (1) {
Taehee Yoo32b6d342019-10-21 18:47:56 +00007487 udev = __netdev_next_upper_dev(now, &iter, &ignore);
Taehee Yoo5343da42019-10-21 18:47:50 +00007488 if (!udev)
7489 break;
Taehee Yoo32b6d342019-10-21 18:47:56 +00007490 if (ignore)
7491 continue;
Taehee Yoo5343da42019-10-21 18:47:50 +00007492
7493 next = udev;
7494 niter = &udev->adj_list.upper;
7495 dev_stack[cur] = now;
7496 iter_stack[cur++] = iter;
7497 break;
7498 }
7499
7500 if (!next) {
7501 if (!cur)
7502 return 0;
7503 next = dev_stack[--cur];
7504 niter = iter_stack[cur];
7505 }
7506
7507 now = next;
7508 iter = niter;
7509 }
7510
7511 return 0;
7512}
7513
David Ahern1a3f0602016-10-17 19:15:44 -07007514int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
7515 int (*fn)(struct net_device *dev,
Taehee Yooeff74232020-09-25 18:13:12 +00007516 struct netdev_nested_priv *priv),
7517 struct netdev_nested_priv *priv)
David Ahern1a3f0602016-10-17 19:15:44 -07007518{
Taehee Yoo5343da42019-10-21 18:47:50 +00007519 struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7520 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7521 int ret, cur = 0;
David Ahern1a3f0602016-10-17 19:15:44 -07007522
Taehee Yoo5343da42019-10-21 18:47:50 +00007523 now = dev;
7524 iter = &dev->adj_list.upper;
David Ahern1a3f0602016-10-17 19:15:44 -07007525
Taehee Yoo5343da42019-10-21 18:47:50 +00007526 while (1) {
7527 if (now != dev) {
Taehee Yooeff74232020-09-25 18:13:12 +00007528 ret = fn(now, priv);
Taehee Yoo5343da42019-10-21 18:47:50 +00007529 if (ret)
7530 return ret;
7531 }
7532
7533 next = NULL;
7534 while (1) {
7535 udev = netdev_next_upper_dev_rcu(now, &iter);
7536 if (!udev)
7537 break;
7538
7539 next = udev;
7540 niter = &udev->adj_list.upper;
7541 dev_stack[cur] = now;
7542 iter_stack[cur++] = iter;
7543 break;
7544 }
7545
7546 if (!next) {
7547 if (!cur)
7548 return 0;
7549 next = dev_stack[--cur];
7550 niter = iter_stack[cur];
7551 }
7552
7553 now = next;
7554 iter = niter;
David Ahern1a3f0602016-10-17 19:15:44 -07007555 }
7556
7557 return 0;
7558}
7559EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu);
7560
Taehee Yoo32b6d342019-10-21 18:47:56 +00007561static bool __netdev_has_upper_dev(struct net_device *dev,
7562 struct net_device *upper_dev)
7563{
Taehee Yooeff74232020-09-25 18:13:12 +00007564 struct netdev_nested_priv priv = {
Taehee Yoo1fc70ed2020-09-25 18:13:29 +00007565 .flags = 0,
Taehee Yooeff74232020-09-25 18:13:12 +00007566 .data = (void *)upper_dev,
7567 };
7568
Taehee Yoo32b6d342019-10-21 18:47:56 +00007569 ASSERT_RTNL();
7570
7571 return __netdev_walk_all_upper_dev(dev, ____netdev_has_upper_dev,
Taehee Yooeff74232020-09-25 18:13:12 +00007572 &priv);
Taehee Yoo32b6d342019-10-21 18:47:56 +00007573}
7574
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007575/**
Veaceslav Falico31088a12013-09-25 09:20:12 +02007576 * netdev_lower_get_next_private - Get the next ->private from the
7577 * lower neighbour list
7578 * @dev: device
7579 * @iter: list_head ** of the current position
7580 *
7581 * Gets the next netdev_adjacent->private from the dev's lower neighbour
7582 * list, starting from iter position. The caller must hold either hold the
7583 * RTNL lock or its own locking that guarantees that the neighbour lower
subashab@codeaurora.orgb4691392015-07-24 03:03:29 +00007584 * list will remain unchanged.
Veaceslav Falico31088a12013-09-25 09:20:12 +02007585 */
7586void *netdev_lower_get_next_private(struct net_device *dev,
7587 struct list_head **iter)
7588{
7589 struct netdev_adjacent *lower;
7590
7591 lower = list_entry(*iter, struct netdev_adjacent, list);
7592
7593 if (&lower->list == &dev->adj_list.lower)
7594 return NULL;
7595
Veaceslav Falico6859e7d2014-04-07 11:25:12 +02007596 *iter = lower->list.next;
Veaceslav Falico31088a12013-09-25 09:20:12 +02007597
7598 return lower->private;
7599}
7600EXPORT_SYMBOL(netdev_lower_get_next_private);
7601
7602/**
7603 * netdev_lower_get_next_private_rcu - Get the next ->private from the
7604 * lower neighbour list, RCU
7605 * variant
7606 * @dev: device
7607 * @iter: list_head ** of the current position
7608 *
7609 * Gets the next netdev_adjacent->private from the dev's lower neighbour
7610 * list, starting from iter position. The caller must hold RCU read lock.
7611 */
7612void *netdev_lower_get_next_private_rcu(struct net_device *dev,
7613 struct list_head **iter)
7614{
7615 struct netdev_adjacent *lower;
7616
7617 WARN_ON_ONCE(!rcu_read_lock_held());
7618
7619 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7620
7621 if (&lower->list == &dev->adj_list.lower)
7622 return NULL;
7623
Veaceslav Falico6859e7d2014-04-07 11:25:12 +02007624 *iter = &lower->list;
Veaceslav Falico31088a12013-09-25 09:20:12 +02007625
7626 return lower->private;
7627}
7628EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
7629
7630/**
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04007631 * netdev_lower_get_next - Get the next device from the lower neighbour
7632 * list
7633 * @dev: device
7634 * @iter: list_head ** of the current position
7635 *
7636 * Gets the next netdev_adjacent from the dev's lower neighbour
7637 * list, starting from iter position. The caller must hold RTNL lock or
7638 * its own locking that guarantees that the neighbour lower
subashab@codeaurora.orgb4691392015-07-24 03:03:29 +00007639 * list will remain unchanged.
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04007640 */
7641void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
7642{
7643 struct netdev_adjacent *lower;
7644
Nikolay Aleksandrovcfdd28b2016-02-17 18:00:31 +01007645 lower = list_entry(*iter, struct netdev_adjacent, list);
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04007646
7647 if (&lower->list == &dev->adj_list.lower)
7648 return NULL;
7649
Nikolay Aleksandrovcfdd28b2016-02-17 18:00:31 +01007650 *iter = lower->list.next;
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04007651
7652 return lower->dev;
7653}
7654EXPORT_SYMBOL(netdev_lower_get_next);
7655
David Ahern1a3f0602016-10-17 19:15:44 -07007656static struct net_device *netdev_next_lower_dev(struct net_device *dev,
7657 struct list_head **iter)
7658{
7659 struct netdev_adjacent *lower;
7660
David Ahern46b5ab12016-10-26 13:21:33 -07007661 lower = list_entry((*iter)->next, struct netdev_adjacent, list);
David Ahern1a3f0602016-10-17 19:15:44 -07007662
7663 if (&lower->list == &dev->adj_list.lower)
7664 return NULL;
7665
David Ahern46b5ab12016-10-26 13:21:33 -07007666 *iter = &lower->list;
David Ahern1a3f0602016-10-17 19:15:44 -07007667
7668 return lower->dev;
7669}
7670
Taehee Yoo32b6d342019-10-21 18:47:56 +00007671static struct net_device *__netdev_next_lower_dev(struct net_device *dev,
7672 struct list_head **iter,
7673 bool *ignore)
7674{
7675 struct netdev_adjacent *lower;
7676
7677 lower = list_entry((*iter)->next, struct netdev_adjacent, list);
7678
7679 if (&lower->list == &dev->adj_list.lower)
7680 return NULL;
7681
7682 *iter = &lower->list;
7683 *ignore = lower->ignore;
7684
7685 return lower->dev;
7686}
7687
David Ahern1a3f0602016-10-17 19:15:44 -07007688int netdev_walk_all_lower_dev(struct net_device *dev,
7689 int (*fn)(struct net_device *dev,
Taehee Yooeff74232020-09-25 18:13:12 +00007690 struct netdev_nested_priv *priv),
7691 struct netdev_nested_priv *priv)
David Ahern1a3f0602016-10-17 19:15:44 -07007692{
Taehee Yoo5343da42019-10-21 18:47:50 +00007693 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7694 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7695 int ret, cur = 0;
David Ahern1a3f0602016-10-17 19:15:44 -07007696
Taehee Yoo5343da42019-10-21 18:47:50 +00007697 now = dev;
7698 iter = &dev->adj_list.lower;
David Ahern1a3f0602016-10-17 19:15:44 -07007699
Taehee Yoo5343da42019-10-21 18:47:50 +00007700 while (1) {
7701 if (now != dev) {
Taehee Yooeff74232020-09-25 18:13:12 +00007702 ret = fn(now, priv);
Taehee Yoo5343da42019-10-21 18:47:50 +00007703 if (ret)
7704 return ret;
7705 }
7706
7707 next = NULL;
7708 while (1) {
7709 ldev = netdev_next_lower_dev(now, &iter);
7710 if (!ldev)
7711 break;
7712
7713 next = ldev;
7714 niter = &ldev->adj_list.lower;
7715 dev_stack[cur] = now;
7716 iter_stack[cur++] = iter;
7717 break;
7718 }
7719
7720 if (!next) {
7721 if (!cur)
7722 return 0;
7723 next = dev_stack[--cur];
7724 niter = iter_stack[cur];
7725 }
7726
7727 now = next;
7728 iter = niter;
David Ahern1a3f0602016-10-17 19:15:44 -07007729 }
7730
7731 return 0;
7732}
7733EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev);
7734
Taehee Yoo32b6d342019-10-21 18:47:56 +00007735static int __netdev_walk_all_lower_dev(struct net_device *dev,
7736 int (*fn)(struct net_device *dev,
Taehee Yooeff74232020-09-25 18:13:12 +00007737 struct netdev_nested_priv *priv),
7738 struct netdev_nested_priv *priv)
Taehee Yoo32b6d342019-10-21 18:47:56 +00007739{
7740 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7741 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7742 int ret, cur = 0;
7743 bool ignore;
7744
7745 now = dev;
7746 iter = &dev->adj_list.lower;
7747
7748 while (1) {
7749 if (now != dev) {
Taehee Yooeff74232020-09-25 18:13:12 +00007750 ret = fn(now, priv);
Taehee Yoo32b6d342019-10-21 18:47:56 +00007751 if (ret)
7752 return ret;
7753 }
7754
7755 next = NULL;
7756 while (1) {
7757 ldev = __netdev_next_lower_dev(now, &iter, &ignore);
7758 if (!ldev)
7759 break;
7760 if (ignore)
7761 continue;
7762
7763 next = ldev;
7764 niter = &ldev->adj_list.lower;
7765 dev_stack[cur] = now;
7766 iter_stack[cur++] = iter;
7767 break;
7768 }
7769
7770 if (!next) {
7771 if (!cur)
7772 return 0;
7773 next = dev_stack[--cur];
7774 niter = iter_stack[cur];
7775 }
7776
7777 now = next;
7778 iter = niter;
7779 }
7780
7781 return 0;
7782}
7783
Taehee Yoo7151aff2020-02-15 10:50:21 +00007784struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
7785 struct list_head **iter)
David Ahern1a3f0602016-10-17 19:15:44 -07007786{
7787 struct netdev_adjacent *lower;
7788
7789 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7790 if (&lower->list == &dev->adj_list.lower)
7791 return NULL;
7792
7793 *iter = &lower->list;
7794
7795 return lower->dev;
7796}
Taehee Yoo7151aff2020-02-15 10:50:21 +00007797EXPORT_SYMBOL(netdev_next_lower_dev_rcu);
David Ahern1a3f0602016-10-17 19:15:44 -07007798
Taehee Yoo5343da42019-10-21 18:47:50 +00007799static u8 __netdev_upper_depth(struct net_device *dev)
7800{
7801 struct net_device *udev;
7802 struct list_head *iter;
7803 u8 max_depth = 0;
Taehee Yoo32b6d342019-10-21 18:47:56 +00007804 bool ignore;
Taehee Yoo5343da42019-10-21 18:47:50 +00007805
7806 for (iter = &dev->adj_list.upper,
Taehee Yoo32b6d342019-10-21 18:47:56 +00007807 udev = __netdev_next_upper_dev(dev, &iter, &ignore);
Taehee Yoo5343da42019-10-21 18:47:50 +00007808 udev;
Taehee Yoo32b6d342019-10-21 18:47:56 +00007809 udev = __netdev_next_upper_dev(dev, &iter, &ignore)) {
7810 if (ignore)
7811 continue;
Taehee Yoo5343da42019-10-21 18:47:50 +00007812 if (max_depth < udev->upper_level)
7813 max_depth = udev->upper_level;
7814 }
7815
7816 return max_depth;
7817}
7818
7819static u8 __netdev_lower_depth(struct net_device *dev)
7820{
7821 struct net_device *ldev;
7822 struct list_head *iter;
7823 u8 max_depth = 0;
Taehee Yoo32b6d342019-10-21 18:47:56 +00007824 bool ignore;
Taehee Yoo5343da42019-10-21 18:47:50 +00007825
7826 for (iter = &dev->adj_list.lower,
Taehee Yoo32b6d342019-10-21 18:47:56 +00007827 ldev = __netdev_next_lower_dev(dev, &iter, &ignore);
Taehee Yoo5343da42019-10-21 18:47:50 +00007828 ldev;
Taehee Yoo32b6d342019-10-21 18:47:56 +00007829 ldev = __netdev_next_lower_dev(dev, &iter, &ignore)) {
7830 if (ignore)
7831 continue;
Taehee Yoo5343da42019-10-21 18:47:50 +00007832 if (max_depth < ldev->lower_level)
7833 max_depth = ldev->lower_level;
7834 }
7835
7836 return max_depth;
7837}
7838
Taehee Yooeff74232020-09-25 18:13:12 +00007839static int __netdev_update_upper_level(struct net_device *dev,
7840 struct netdev_nested_priv *__unused)
Taehee Yoo5343da42019-10-21 18:47:50 +00007841{
7842 dev->upper_level = __netdev_upper_depth(dev) + 1;
7843 return 0;
7844}
7845
Taehee Yooeff74232020-09-25 18:13:12 +00007846static int __netdev_update_lower_level(struct net_device *dev,
Taehee Yoo1fc70ed2020-09-25 18:13:29 +00007847 struct netdev_nested_priv *priv)
Taehee Yoo5343da42019-10-21 18:47:50 +00007848{
7849 dev->lower_level = __netdev_lower_depth(dev) + 1;
Taehee Yoo1fc70ed2020-09-25 18:13:29 +00007850
7851#ifdef CONFIG_LOCKDEP
7852 if (!priv)
7853 return 0;
7854
7855 if (priv->flags & NESTED_SYNC_IMM)
7856 dev->nested_level = dev->lower_level - 1;
7857 if (priv->flags & NESTED_SYNC_TODO)
7858 net_unlink_todo(dev);
7859#endif
Taehee Yoo5343da42019-10-21 18:47:50 +00007860 return 0;
7861}
7862
David Ahern1a3f0602016-10-17 19:15:44 -07007863int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
7864 int (*fn)(struct net_device *dev,
Taehee Yooeff74232020-09-25 18:13:12 +00007865 struct netdev_nested_priv *priv),
7866 struct netdev_nested_priv *priv)
David Ahern1a3f0602016-10-17 19:15:44 -07007867{
Taehee Yoo5343da42019-10-21 18:47:50 +00007868 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7869 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7870 int ret, cur = 0;
David Ahern1a3f0602016-10-17 19:15:44 -07007871
Taehee Yoo5343da42019-10-21 18:47:50 +00007872 now = dev;
7873 iter = &dev->adj_list.lower;
David Ahern1a3f0602016-10-17 19:15:44 -07007874
Taehee Yoo5343da42019-10-21 18:47:50 +00007875 while (1) {
7876 if (now != dev) {
Taehee Yooeff74232020-09-25 18:13:12 +00007877 ret = fn(now, priv);
Taehee Yoo5343da42019-10-21 18:47:50 +00007878 if (ret)
7879 return ret;
7880 }
7881
7882 next = NULL;
7883 while (1) {
7884 ldev = netdev_next_lower_dev_rcu(now, &iter);
7885 if (!ldev)
7886 break;
7887
7888 next = ldev;
7889 niter = &ldev->adj_list.lower;
7890 dev_stack[cur] = now;
7891 iter_stack[cur++] = iter;
7892 break;
7893 }
7894
7895 if (!next) {
7896 if (!cur)
7897 return 0;
7898 next = dev_stack[--cur];
7899 niter = iter_stack[cur];
7900 }
7901
7902 now = next;
7903 iter = niter;
David Ahern1a3f0602016-10-17 19:15:44 -07007904 }
7905
7906 return 0;
7907}
7908EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu);
7909
Jiri Pirko7ce856a2016-07-04 08:23:12 +02007910/**
dingtianhonge001bfa2013-12-13 10:19:55 +08007911 * netdev_lower_get_first_private_rcu - Get the first ->private from the
7912 * lower neighbour list, RCU
7913 * variant
7914 * @dev: device
7915 *
7916 * Gets the first netdev_adjacent->private from the dev's lower neighbour
7917 * list. The caller must hold RCU read lock.
7918 */
7919void *netdev_lower_get_first_private_rcu(struct net_device *dev)
7920{
7921 struct netdev_adjacent *lower;
7922
7923 lower = list_first_or_null_rcu(&dev->adj_list.lower,
7924 struct netdev_adjacent, list);
7925 if (lower)
7926 return lower->private;
7927 return NULL;
7928}
7929EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
7930
7931/**
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007932 * netdev_master_upper_dev_get_rcu - Get master upper device
7933 * @dev: device
7934 *
7935 * Find a master upper device and return pointer to it or NULL in case
7936 * it's not there. The caller must hold the RCU read lock.
7937 */
7938struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
7939{
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02007940 struct netdev_adjacent *upper;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007941
Veaceslav Falico2f268f12013-09-25 09:20:07 +02007942 upper = list_first_or_null_rcu(&dev->adj_list.upper,
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02007943 struct netdev_adjacent, list);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007944 if (upper && likely(upper->master))
7945 return upper->dev;
7946 return NULL;
7947}
7948EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
7949
Rashika Kheria0a59f3a2014-02-09 20:26:25 +05307950static int netdev_adjacent_sysfs_add(struct net_device *dev,
Veaceslav Falico3ee32702014-01-14 21:58:50 +01007951 struct net_device *adj_dev,
7952 struct list_head *dev_list)
7953{
7954 char linkname[IFNAMSIZ+7];
tchardingf4563a72017-02-09 17:56:07 +11007955
Veaceslav Falico3ee32702014-01-14 21:58:50 +01007956 sprintf(linkname, dev_list == &dev->adj_list.upper ?
7957 "upper_%s" : "lower_%s", adj_dev->name);
7958 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
7959 linkname);
7960}
Rashika Kheria0a59f3a2014-02-09 20:26:25 +05307961static void netdev_adjacent_sysfs_del(struct net_device *dev,
Veaceslav Falico3ee32702014-01-14 21:58:50 +01007962 char *name,
7963 struct list_head *dev_list)
7964{
7965 char linkname[IFNAMSIZ+7];
tchardingf4563a72017-02-09 17:56:07 +11007966
Veaceslav Falico3ee32702014-01-14 21:58:50 +01007967 sprintf(linkname, dev_list == &dev->adj_list.upper ?
7968 "upper_%s" : "lower_%s", name);
7969 sysfs_remove_link(&(dev->dev.kobj), linkname);
7970}
7971
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04007972static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
7973 struct net_device *adj_dev,
7974 struct list_head *dev_list)
7975{
7976 return (dev_list == &dev->adj_list.upper ||
7977 dev_list == &dev->adj_list.lower) &&
7978 net_eq(dev_net(dev), dev_net(adj_dev));
7979}
Veaceslav Falico3ee32702014-01-14 21:58:50 +01007980
Veaceslav Falico5d261912013-08-28 23:25:05 +02007981static int __netdev_adjacent_dev_insert(struct net_device *dev,
7982 struct net_device *adj_dev,
Veaceslav Falico7863c052013-09-25 09:20:06 +02007983 struct list_head *dev_list,
Veaceslav Falico402dae92013-09-25 09:20:09 +02007984 void *private, bool master)
Veaceslav Falico5d261912013-08-28 23:25:05 +02007985{
7986 struct netdev_adjacent *adj;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02007987 int ret;
Veaceslav Falico5d261912013-08-28 23:25:05 +02007988
Michal Kubeček6ea29da2015-09-24 10:59:05 +02007989 adj = __netdev_find_adj(adj_dev, dev_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02007990
7991 if (adj) {
David Ahern790510d2016-10-17 19:15:43 -07007992 adj->ref_nr += 1;
David Ahern67b62f92016-10-17 19:15:53 -07007993 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n",
7994 dev->name, adj_dev->name, adj->ref_nr);
7995
Veaceslav Falico5d261912013-08-28 23:25:05 +02007996 return 0;
7997 }
7998
7999 adj = kmalloc(sizeof(*adj), GFP_KERNEL);
8000 if (!adj)
8001 return -ENOMEM;
8002
8003 adj->dev = adj_dev;
8004 adj->master = master;
David Ahern790510d2016-10-17 19:15:43 -07008005 adj->ref_nr = 1;
Veaceslav Falico402dae92013-09-25 09:20:09 +02008006 adj->private = private;
Taehee Yoo32b6d342019-10-21 18:47:56 +00008007 adj->ignore = false;
Veaceslav Falico5d261912013-08-28 23:25:05 +02008008 dev_hold(adj_dev);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02008009
David Ahern67b62f92016-10-17 19:15:53 -07008010 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n",
8011 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02008012
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04008013 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
Veaceslav Falico3ee32702014-01-14 21:58:50 +01008014 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
Veaceslav Falico5831d662013-09-25 09:20:32 +02008015 if (ret)
8016 goto free_adj;
8017 }
8018
Veaceslav Falico7863c052013-09-25 09:20:06 +02008019 /* Ensure that master link is always the first item in list. */
Veaceslav Falico842d67a2013-09-25 09:20:31 +02008020 if (master) {
8021 ret = sysfs_create_link(&(dev->dev.kobj),
8022 &(adj_dev->dev.kobj), "master");
8023 if (ret)
Veaceslav Falico5831d662013-09-25 09:20:32 +02008024 goto remove_symlinks;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02008025
Veaceslav Falico7863c052013-09-25 09:20:06 +02008026 list_add_rcu(&adj->list, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02008027 } else {
Veaceslav Falico7863c052013-09-25 09:20:06 +02008028 list_add_tail_rcu(&adj->list, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02008029 }
Veaceslav Falico5d261912013-08-28 23:25:05 +02008030
8031 return 0;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02008032
Veaceslav Falico5831d662013-09-25 09:20:32 +02008033remove_symlinks:
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04008034 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
Veaceslav Falico3ee32702014-01-14 21:58:50 +01008035 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02008036free_adj:
8037 kfree(adj);
Nikolay Aleksandrov974daef2013-10-23 15:28:56 +02008038 dev_put(adj_dev);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02008039
8040 return ret;
Veaceslav Falico5d261912013-08-28 23:25:05 +02008041}
8042
stephen hemminger1d143d92013-12-29 14:01:29 -08008043static void __netdev_adjacent_dev_remove(struct net_device *dev,
8044 struct net_device *adj_dev,
Andrew Collins93409032016-10-03 13:43:02 -06008045 u16 ref_nr,
stephen hemminger1d143d92013-12-29 14:01:29 -08008046 struct list_head *dev_list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02008047{
8048 struct netdev_adjacent *adj;
8049
David Ahern67b62f92016-10-17 19:15:53 -07008050 pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n",
8051 dev->name, adj_dev->name, ref_nr);
8052
Michal Kubeček6ea29da2015-09-24 10:59:05 +02008053 adj = __netdev_find_adj(adj_dev, dev_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02008054
Veaceslav Falico2f268f12013-09-25 09:20:07 +02008055 if (!adj) {
David Ahern67b62f92016-10-17 19:15:53 -07008056 pr_err("Adjacency does not exist for device %s from %s\n",
Veaceslav Falico2f268f12013-09-25 09:20:07 +02008057 dev->name, adj_dev->name);
David Ahern67b62f92016-10-17 19:15:53 -07008058 WARN_ON(1);
8059 return;
Veaceslav Falico2f268f12013-09-25 09:20:07 +02008060 }
Veaceslav Falico5d261912013-08-28 23:25:05 +02008061
Andrew Collins93409032016-10-03 13:43:02 -06008062 if (adj->ref_nr > ref_nr) {
David Ahern67b62f92016-10-17 19:15:53 -07008063 pr_debug("adjacency: %s to %s ref_nr - %d = %d\n",
8064 dev->name, adj_dev->name, ref_nr,
8065 adj->ref_nr - ref_nr);
Andrew Collins93409032016-10-03 13:43:02 -06008066 adj->ref_nr -= ref_nr;
Veaceslav Falico5d261912013-08-28 23:25:05 +02008067 return;
8068 }
8069
Veaceslav Falico842d67a2013-09-25 09:20:31 +02008070 if (adj->master)
8071 sysfs_remove_link(&(dev->dev.kobj), "master");
8072
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04008073 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
Veaceslav Falico3ee32702014-01-14 21:58:50 +01008074 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
Veaceslav Falico5831d662013-09-25 09:20:32 +02008075
Veaceslav Falico5d261912013-08-28 23:25:05 +02008076 list_del_rcu(&adj->list);
David Ahern67b62f92016-10-17 19:15:53 -07008077 pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n",
Veaceslav Falico2f268f12013-09-25 09:20:07 +02008078 adj_dev->name, dev->name, adj_dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02008079 dev_put(adj_dev);
8080 kfree_rcu(adj, rcu);
8081}
8082
stephen hemminger1d143d92013-12-29 14:01:29 -08008083static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
8084 struct net_device *upper_dev,
8085 struct list_head *up_list,
8086 struct list_head *down_list,
8087 void *private, bool master)
Veaceslav Falico5d261912013-08-28 23:25:05 +02008088{
8089 int ret;
8090
David Ahern790510d2016-10-17 19:15:43 -07008091 ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list,
Andrew Collins93409032016-10-03 13:43:02 -06008092 private, master);
Veaceslav Falico5d261912013-08-28 23:25:05 +02008093 if (ret)
8094 return ret;
8095
David Ahern790510d2016-10-17 19:15:43 -07008096 ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list,
Andrew Collins93409032016-10-03 13:43:02 -06008097 private, false);
Veaceslav Falico5d261912013-08-28 23:25:05 +02008098 if (ret) {
David Ahern790510d2016-10-17 19:15:43 -07008099 __netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02008100 return ret;
8101 }
8102
8103 return 0;
8104}
8105
stephen hemminger1d143d92013-12-29 14:01:29 -08008106static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
8107 struct net_device *upper_dev,
Andrew Collins93409032016-10-03 13:43:02 -06008108 u16 ref_nr,
stephen hemminger1d143d92013-12-29 14:01:29 -08008109 struct list_head *up_list,
8110 struct list_head *down_list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02008111{
Andrew Collins93409032016-10-03 13:43:02 -06008112 __netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list);
8113 __netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02008114}
8115
stephen hemminger1d143d92013-12-29 14:01:29 -08008116static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
8117 struct net_device *upper_dev,
8118 void *private, bool master)
Veaceslav Falico2f268f12013-09-25 09:20:07 +02008119{
David Ahernf1170fd2016-10-17 19:15:51 -07008120 return __netdev_adjacent_dev_link_lists(dev, upper_dev,
8121 &dev->adj_list.upper,
8122 &upper_dev->adj_list.lower,
8123 private, master);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02008124}
8125
stephen hemminger1d143d92013-12-29 14:01:29 -08008126static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
8127 struct net_device *upper_dev)
Veaceslav Falico2f268f12013-09-25 09:20:07 +02008128{
Andrew Collins93409032016-10-03 13:43:02 -06008129 __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1,
Veaceslav Falico2f268f12013-09-25 09:20:07 +02008130 &dev->adj_list.upper,
8131 &upper_dev->adj_list.lower);
8132}
Veaceslav Falico5d261912013-08-28 23:25:05 +02008133
Jiri Pirko9ff162a2013-01-03 22:48:49 +00008134static int __netdev_upper_dev_link(struct net_device *dev,
Veaceslav Falico402dae92013-09-25 09:20:09 +02008135 struct net_device *upper_dev, bool master,
David Ahern42ab19e2017-10-04 17:48:47 -07008136 void *upper_priv, void *upper_info,
Taehee Yoo1fc70ed2020-09-25 18:13:29 +00008137 struct netdev_nested_priv *priv,
David Ahern42ab19e2017-10-04 17:48:47 -07008138 struct netlink_ext_ack *extack)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00008139{
David Ahern51d0c0472017-10-04 17:48:45 -07008140 struct netdev_notifier_changeupper_info changeupper_info = {
8141 .info = {
8142 .dev = dev,
David Ahern42ab19e2017-10-04 17:48:47 -07008143 .extack = extack,
David Ahern51d0c0472017-10-04 17:48:45 -07008144 },
8145 .upper_dev = upper_dev,
8146 .master = master,
8147 .linking = true,
8148 .upper_info = upper_info,
8149 };
Mike Manning50d629e2018-02-26 23:49:30 +00008150 struct net_device *master_dev;
Veaceslav Falico5d261912013-08-28 23:25:05 +02008151 int ret = 0;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00008152
8153 ASSERT_RTNL();
8154
8155 if (dev == upper_dev)
8156 return -EBUSY;
8157
8158 /* To prevent loops, check if dev is not upper device to upper_dev. */
Taehee Yoo32b6d342019-10-21 18:47:56 +00008159 if (__netdev_has_upper_dev(upper_dev, dev))
Jiri Pirko9ff162a2013-01-03 22:48:49 +00008160 return -EBUSY;
8161
Taehee Yoo5343da42019-10-21 18:47:50 +00008162 if ((dev->lower_level + upper_dev->upper_level) > MAX_NEST_DEV)
8163 return -EMLINK;
8164
Mike Manning50d629e2018-02-26 23:49:30 +00008165 if (!master) {
Taehee Yoo32b6d342019-10-21 18:47:56 +00008166 if (__netdev_has_upper_dev(dev, upper_dev))
Mike Manning50d629e2018-02-26 23:49:30 +00008167 return -EEXIST;
8168 } else {
Taehee Yoo32b6d342019-10-21 18:47:56 +00008169 master_dev = __netdev_master_upper_dev_get(dev);
Mike Manning50d629e2018-02-26 23:49:30 +00008170 if (master_dev)
8171 return master_dev == upper_dev ? -EEXIST : -EBUSY;
8172 }
Jiri Pirko9ff162a2013-01-03 22:48:49 +00008173
David Ahern51d0c0472017-10-04 17:48:45 -07008174 ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
Jiri Pirko573c7ba2015-10-16 14:01:22 +02008175 &changeupper_info.info);
8176 ret = notifier_to_errno(ret);
8177 if (ret)
8178 return ret;
8179
Jiri Pirko6dffb042015-12-03 12:12:10 +01008180 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
Veaceslav Falico402dae92013-09-25 09:20:09 +02008181 master);
Veaceslav Falico5d261912013-08-28 23:25:05 +02008182 if (ret)
8183 return ret;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00008184
David Ahern51d0c0472017-10-04 17:48:45 -07008185 ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
Ido Schimmelb03804e2015-12-03 12:12:03 +01008186 &changeupper_info.info);
8187 ret = notifier_to_errno(ret);
8188 if (ret)
David Ahernf1170fd2016-10-17 19:15:51 -07008189 goto rollback;
Ido Schimmelb03804e2015-12-03 12:12:03 +01008190
Taehee Yoo5343da42019-10-21 18:47:50 +00008191 __netdev_update_upper_level(dev, NULL);
Taehee Yoo32b6d342019-10-21 18:47:56 +00008192 __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
Taehee Yoo5343da42019-10-21 18:47:50 +00008193
Taehee Yoo1fc70ed2020-09-25 18:13:29 +00008194 __netdev_update_lower_level(upper_dev, priv);
Taehee Yoo32b6d342019-10-21 18:47:56 +00008195 __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
Taehee Yoo1fc70ed2020-09-25 18:13:29 +00008196 priv);
Taehee Yoo5343da42019-10-21 18:47:50 +00008197
Jiri Pirko9ff162a2013-01-03 22:48:49 +00008198 return 0;
Veaceslav Falico5d261912013-08-28 23:25:05 +02008199
David Ahernf1170fd2016-10-17 19:15:51 -07008200rollback:
Veaceslav Falico2f268f12013-09-25 09:20:07 +02008201 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
Veaceslav Falico5d261912013-08-28 23:25:05 +02008202
8203 return ret;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00008204}
8205
8206/**
8207 * netdev_upper_dev_link - Add a link to the upper device
8208 * @dev: device
8209 * @upper_dev: new upper device
Florian Fainelli7a006d52018-01-22 19:14:28 -08008210 * @extack: netlink extended ack
Jiri Pirko9ff162a2013-01-03 22:48:49 +00008211 *
8212 * Adds a link to device which is upper to this one. The caller must hold
8213 * the RTNL lock. On a failure a negative errno code is returned.
8214 * On success the reference counts are adjusted and the function
8215 * returns zero.
8216 */
8217int netdev_upper_dev_link(struct net_device *dev,
David Ahern42ab19e2017-10-04 17:48:47 -07008218 struct net_device *upper_dev,
8219 struct netlink_ext_ack *extack)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00008220{
Taehee Yoo1fc70ed2020-09-25 18:13:29 +00008221 struct netdev_nested_priv priv = {
8222 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
8223 .data = NULL,
8224 };
8225
David Ahern42ab19e2017-10-04 17:48:47 -07008226 return __netdev_upper_dev_link(dev, upper_dev, false,
Taehee Yoo1fc70ed2020-09-25 18:13:29 +00008227 NULL, NULL, &priv, extack);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00008228}
8229EXPORT_SYMBOL(netdev_upper_dev_link);
8230
8231/**
8232 * netdev_master_upper_dev_link - Add a master link to the upper device
8233 * @dev: device
8234 * @upper_dev: new upper device
Jiri Pirko6dffb042015-12-03 12:12:10 +01008235 * @upper_priv: upper device private
Jiri Pirko29bf24a2015-12-03 12:12:11 +01008236 * @upper_info: upper info to be passed down via notifier
Florian Fainelli7a006d52018-01-22 19:14:28 -08008237 * @extack: netlink extended ack
Jiri Pirko9ff162a2013-01-03 22:48:49 +00008238 *
8239 * Adds a link to device which is upper to this one. In this case, only
8240 * one master upper device can be linked, although other non-master devices
8241 * might be linked as well. The caller must hold the RTNL lock.
8242 * On a failure a negative errno code is returned. On success the reference
8243 * counts are adjusted and the function returns zero.
8244 */
8245int netdev_master_upper_dev_link(struct net_device *dev,
Jiri Pirko6dffb042015-12-03 12:12:10 +01008246 struct net_device *upper_dev,
David Ahern42ab19e2017-10-04 17:48:47 -07008247 void *upper_priv, void *upper_info,
8248 struct netlink_ext_ack *extack)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00008249{
Taehee Yoo1fc70ed2020-09-25 18:13:29 +00008250 struct netdev_nested_priv priv = {
8251 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
8252 .data = NULL,
8253 };
8254
Jiri Pirko29bf24a2015-12-03 12:12:11 +01008255 return __netdev_upper_dev_link(dev, upper_dev, true,
Taehee Yoo1fc70ed2020-09-25 18:13:29 +00008256 upper_priv, upper_info, &priv, extack);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00008257}
8258EXPORT_SYMBOL(netdev_master_upper_dev_link);
8259
Taehee Yoofe8300f2020-09-25 18:13:02 +00008260static void __netdev_upper_dev_unlink(struct net_device *dev,
Taehee Yoo1fc70ed2020-09-25 18:13:29 +00008261 struct net_device *upper_dev,
8262 struct netdev_nested_priv *priv)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00008263{
David Ahern51d0c0472017-10-04 17:48:45 -07008264 struct netdev_notifier_changeupper_info changeupper_info = {
8265 .info = {
8266 .dev = dev,
8267 },
8268 .upper_dev = upper_dev,
8269 .linking = false,
8270 };
tchardingf4563a72017-02-09 17:56:07 +11008271
Jiri Pirko9ff162a2013-01-03 22:48:49 +00008272 ASSERT_RTNL();
8273
Jiri Pirko0e4ead92015-08-27 09:31:18 +02008274 changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
Jiri Pirko0e4ead92015-08-27 09:31:18 +02008275
David Ahern51d0c0472017-10-04 17:48:45 -07008276 call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
Jiri Pirko573c7ba2015-10-16 14:01:22 +02008277 &changeupper_info.info);
8278
Veaceslav Falico2f268f12013-09-25 09:20:07 +02008279 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
Veaceslav Falico5d261912013-08-28 23:25:05 +02008280
David Ahern51d0c0472017-10-04 17:48:45 -07008281 call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
Jiri Pirko0e4ead92015-08-27 09:31:18 +02008282 &changeupper_info.info);
Taehee Yoo5343da42019-10-21 18:47:50 +00008283
8284 __netdev_update_upper_level(dev, NULL);
Taehee Yoo32b6d342019-10-21 18:47:56 +00008285 __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
Taehee Yoo5343da42019-10-21 18:47:50 +00008286
Taehee Yoo1fc70ed2020-09-25 18:13:29 +00008287 __netdev_update_lower_level(upper_dev, priv);
Taehee Yoo32b6d342019-10-21 18:47:56 +00008288 __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
Taehee Yoo1fc70ed2020-09-25 18:13:29 +00008289 priv);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00008290}
Taehee Yoofe8300f2020-09-25 18:13:02 +00008291
8292/**
8293 * netdev_upper_dev_unlink - Removes a link to upper device
8294 * @dev: device
8295 * @upper_dev: new upper device
8296 *
8297 * Removes a link to device which is upper to this one. The caller must hold
8298 * the RTNL lock.
8299 */
8300void netdev_upper_dev_unlink(struct net_device *dev,
8301 struct net_device *upper_dev)
8302{
Taehee Yoo1fc70ed2020-09-25 18:13:29 +00008303 struct netdev_nested_priv priv = {
8304 .flags = NESTED_SYNC_TODO,
8305 .data = NULL,
8306 };
8307
8308 __netdev_upper_dev_unlink(dev, upper_dev, &priv);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00008309}
8310EXPORT_SYMBOL(netdev_upper_dev_unlink);
8311
Taehee Yoo32b6d342019-10-21 18:47:56 +00008312static void __netdev_adjacent_dev_set(struct net_device *upper_dev,
8313 struct net_device *lower_dev,
8314 bool val)
8315{
8316 struct netdev_adjacent *adj;
8317
8318 adj = __netdev_find_adj(lower_dev, &upper_dev->adj_list.lower);
8319 if (adj)
8320 adj->ignore = val;
8321
8322 adj = __netdev_find_adj(upper_dev, &lower_dev->adj_list.upper);
8323 if (adj)
8324 adj->ignore = val;
8325}
8326
8327static void netdev_adjacent_dev_disable(struct net_device *upper_dev,
8328 struct net_device *lower_dev)
8329{
8330 __netdev_adjacent_dev_set(upper_dev, lower_dev, true);
8331}
8332
8333static void netdev_adjacent_dev_enable(struct net_device *upper_dev,
8334 struct net_device *lower_dev)
8335{
8336 __netdev_adjacent_dev_set(upper_dev, lower_dev, false);
8337}
8338
8339int netdev_adjacent_change_prepare(struct net_device *old_dev,
8340 struct net_device *new_dev,
8341 struct net_device *dev,
8342 struct netlink_ext_ack *extack)
8343{
Taehee Yoo1fc70ed2020-09-25 18:13:29 +00008344 struct netdev_nested_priv priv = {
8345 .flags = 0,
8346 .data = NULL,
8347 };
Taehee Yoo32b6d342019-10-21 18:47:56 +00008348 int err;
8349
8350 if (!new_dev)
8351 return 0;
8352
8353 if (old_dev && new_dev != old_dev)
8354 netdev_adjacent_dev_disable(dev, old_dev);
Taehee Yoo1fc70ed2020-09-25 18:13:29 +00008355 err = __netdev_upper_dev_link(new_dev, dev, false, NULL, NULL, &priv,
8356 extack);
Taehee Yoo32b6d342019-10-21 18:47:56 +00008357 if (err) {
8358 if (old_dev && new_dev != old_dev)
8359 netdev_adjacent_dev_enable(dev, old_dev);
8360 return err;
8361 }
8362
8363 return 0;
8364}
8365EXPORT_SYMBOL(netdev_adjacent_change_prepare);
8366
8367void netdev_adjacent_change_commit(struct net_device *old_dev,
8368 struct net_device *new_dev,
8369 struct net_device *dev)
8370{
Taehee Yoo1fc70ed2020-09-25 18:13:29 +00008371 struct netdev_nested_priv priv = {
8372 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
8373 .data = NULL,
8374 };
8375
Taehee Yoo32b6d342019-10-21 18:47:56 +00008376 if (!new_dev || !old_dev)
8377 return;
8378
8379 if (new_dev == old_dev)
8380 return;
8381
8382 netdev_adjacent_dev_enable(dev, old_dev);
Taehee Yoo1fc70ed2020-09-25 18:13:29 +00008383 __netdev_upper_dev_unlink(old_dev, dev, &priv);
Taehee Yoo32b6d342019-10-21 18:47:56 +00008384}
8385EXPORT_SYMBOL(netdev_adjacent_change_commit);
8386
8387void netdev_adjacent_change_abort(struct net_device *old_dev,
8388 struct net_device *new_dev,
8389 struct net_device *dev)
8390{
Taehee Yoo1fc70ed2020-09-25 18:13:29 +00008391 struct netdev_nested_priv priv = {
8392 .flags = 0,
8393 .data = NULL,
8394 };
8395
Taehee Yoo32b6d342019-10-21 18:47:56 +00008396 if (!new_dev)
8397 return;
8398
8399 if (old_dev && new_dev != old_dev)
8400 netdev_adjacent_dev_enable(dev, old_dev);
8401
Taehee Yoo1fc70ed2020-09-25 18:13:29 +00008402 __netdev_upper_dev_unlink(new_dev, dev, &priv);
Taehee Yoo32b6d342019-10-21 18:47:56 +00008403}
8404EXPORT_SYMBOL(netdev_adjacent_change_abort);
8405
Moni Shoua61bd3852015-02-03 16:48:29 +02008406/**
8407 * netdev_bonding_info_change - Dispatch event about slave change
8408 * @dev: device
Masanari Iida4a26e4532015-02-14 22:26:34 +09008409 * @bonding_info: info to dispatch
Moni Shoua61bd3852015-02-03 16:48:29 +02008410 *
8411 * Send NETDEV_BONDING_INFO to netdev notifiers with info.
8412 * The caller must hold the RTNL lock.
8413 */
8414void netdev_bonding_info_change(struct net_device *dev,
8415 struct netdev_bonding_info *bonding_info)
8416{
David Ahern51d0c0472017-10-04 17:48:45 -07008417 struct netdev_notifier_bonding_info info = {
8418 .info.dev = dev,
8419 };
Moni Shoua61bd3852015-02-03 16:48:29 +02008420
8421 memcpy(&info.bonding_info, bonding_info,
8422 sizeof(struct netdev_bonding_info));
David Ahern51d0c0472017-10-04 17:48:45 -07008423 call_netdevice_notifiers_info(NETDEV_BONDING_INFO,
Moni Shoua61bd3852015-02-03 16:48:29 +02008424 &info.info);
8425}
8426EXPORT_SYMBOL(netdev_bonding_info_change);
8427
Maor Gottliebcff9f122020-04-30 22:21:31 +03008428/**
8429 * netdev_get_xmit_slave - Get the xmit slave of master device
Andrew Lunn88425002020-07-13 01:14:59 +02008430 * @dev: device
Maor Gottliebcff9f122020-04-30 22:21:31 +03008431 * @skb: The packet
8432 * @all_slaves: assume all the slaves are active
8433 *
8434 * The reference counters are not incremented so the caller must be
8435 * careful with locks. The caller must hold RCU lock.
8436 * %NULL is returned if no slave is found.
8437 */
8438
8439struct net_device *netdev_get_xmit_slave(struct net_device *dev,
8440 struct sk_buff *skb,
8441 bool all_slaves)
8442{
8443 const struct net_device_ops *ops = dev->netdev_ops;
8444
8445 if (!ops->ndo_get_xmit_slave)
8446 return NULL;
8447 return ops->ndo_get_xmit_slave(dev, skb, all_slaves);
8448}
8449EXPORT_SYMBOL(netdev_get_xmit_slave);
8450
Tariq Toukan719a4022021-01-17 16:59:42 +02008451static struct net_device *netdev_sk_get_lower_dev(struct net_device *dev,
8452 struct sock *sk)
8453{
8454 const struct net_device_ops *ops = dev->netdev_ops;
8455
8456 if (!ops->ndo_sk_get_lower_dev)
8457 return NULL;
8458 return ops->ndo_sk_get_lower_dev(dev, sk);
8459}
8460
8461/**
8462 * netdev_sk_get_lowest_dev - Get the lowest device in chain given device and socket
8463 * @dev: device
8464 * @sk: the socket
8465 *
8466 * %NULL is returned if no lower device is found.
8467 */
8468
8469struct net_device *netdev_sk_get_lowest_dev(struct net_device *dev,
8470 struct sock *sk)
8471{
8472 struct net_device *lower;
8473
8474 lower = netdev_sk_get_lower_dev(dev, sk);
8475 while (lower) {
8476 dev = lower;
8477 lower = netdev_sk_get_lower_dev(dev, sk);
8478 }
8479
8480 return dev;
8481}
8482EXPORT_SYMBOL(netdev_sk_get_lowest_dev);
8483
Eric Dumazet2ce1ee12015-02-04 13:37:44 -08008484static void netdev_adjacent_add_links(struct net_device *dev)
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04008485{
8486 struct netdev_adjacent *iter;
8487
8488 struct net *net = dev_net(dev);
8489
8490 list_for_each_entry(iter, &dev->adj_list.upper, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08008491 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04008492 continue;
8493 netdev_adjacent_sysfs_add(iter->dev, dev,
8494 &iter->dev->adj_list.lower);
8495 netdev_adjacent_sysfs_add(dev, iter->dev,
8496 &dev->adj_list.upper);
8497 }
8498
8499 list_for_each_entry(iter, &dev->adj_list.lower, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08008500 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04008501 continue;
8502 netdev_adjacent_sysfs_add(iter->dev, dev,
8503 &iter->dev->adj_list.upper);
8504 netdev_adjacent_sysfs_add(dev, iter->dev,
8505 &dev->adj_list.lower);
8506 }
8507}
8508
Eric Dumazet2ce1ee12015-02-04 13:37:44 -08008509static void netdev_adjacent_del_links(struct net_device *dev)
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04008510{
8511 struct netdev_adjacent *iter;
8512
8513 struct net *net = dev_net(dev);
8514
8515 list_for_each_entry(iter, &dev->adj_list.upper, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08008516 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04008517 continue;
8518 netdev_adjacent_sysfs_del(iter->dev, dev->name,
8519 &iter->dev->adj_list.lower);
8520 netdev_adjacent_sysfs_del(dev, iter->dev->name,
8521 &dev->adj_list.upper);
8522 }
8523
8524 list_for_each_entry(iter, &dev->adj_list.lower, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08008525 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04008526 continue;
8527 netdev_adjacent_sysfs_del(iter->dev, dev->name,
8528 &iter->dev->adj_list.upper);
8529 netdev_adjacent_sysfs_del(dev, iter->dev->name,
8530 &dev->adj_list.lower);
8531 }
8532}
8533
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01008534void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
Veaceslav Falico402dae92013-09-25 09:20:09 +02008535{
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01008536 struct netdev_adjacent *iter;
Veaceslav Falico402dae92013-09-25 09:20:09 +02008537
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04008538 struct net *net = dev_net(dev);
8539
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01008540 list_for_each_entry(iter, &dev->adj_list.upper, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08008541 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04008542 continue;
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01008543 netdev_adjacent_sysfs_del(iter->dev, oldname,
8544 &iter->dev->adj_list.lower);
8545 netdev_adjacent_sysfs_add(iter->dev, dev,
8546 &iter->dev->adj_list.lower);
8547 }
Veaceslav Falico402dae92013-09-25 09:20:09 +02008548
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01008549 list_for_each_entry(iter, &dev->adj_list.lower, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08008550 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04008551 continue;
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01008552 netdev_adjacent_sysfs_del(iter->dev, oldname,
8553 &iter->dev->adj_list.upper);
8554 netdev_adjacent_sysfs_add(iter->dev, dev,
8555 &iter->dev->adj_list.upper);
8556 }
Veaceslav Falico402dae92013-09-25 09:20:09 +02008557}
Veaceslav Falico402dae92013-09-25 09:20:09 +02008558
8559void *netdev_lower_dev_get_private(struct net_device *dev,
8560 struct net_device *lower_dev)
8561{
8562 struct netdev_adjacent *lower;
8563
8564 if (!lower_dev)
8565 return NULL;
Michal Kubeček6ea29da2015-09-24 10:59:05 +02008566 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
Veaceslav Falico402dae92013-09-25 09:20:09 +02008567 if (!lower)
8568 return NULL;
8569
8570 return lower->private;
8571}
8572EXPORT_SYMBOL(netdev_lower_dev_get_private);
8573
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04008574
Jiri Pirko04d48262015-12-03 12:12:15 +01008575/**
Mauro Carvalho Chehabc1639be2020-11-16 11:17:58 +01008576 * netdev_lower_state_changed - Dispatch event about lower device state change
Jiri Pirko04d48262015-12-03 12:12:15 +01008577 * @lower_dev: device
8578 * @lower_state_info: state to dispatch
8579 *
8580 * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info.
8581 * The caller must hold the RTNL lock.
8582 */
8583void netdev_lower_state_changed(struct net_device *lower_dev,
8584 void *lower_state_info)
8585{
David Ahern51d0c0472017-10-04 17:48:45 -07008586 struct netdev_notifier_changelowerstate_info changelowerstate_info = {
8587 .info.dev = lower_dev,
8588 };
Jiri Pirko04d48262015-12-03 12:12:15 +01008589
8590 ASSERT_RTNL();
8591 changelowerstate_info.lower_state_info = lower_state_info;
David Ahern51d0c0472017-10-04 17:48:45 -07008592 call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE,
Jiri Pirko04d48262015-12-03 12:12:15 +01008593 &changelowerstate_info.info);
8594}
8595EXPORT_SYMBOL(netdev_lower_state_changed);
8596
Patrick McHardyb6c40d62008-10-07 15:26:48 -07008597static void dev_change_rx_flags(struct net_device *dev, int flags)
8598{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08008599 const struct net_device_ops *ops = dev->netdev_ops;
8600
Vlad Yasevichd2615bf2013-11-19 20:47:15 -05008601 if (ops->ndo_change_rx_flags)
Stephen Hemmingerd3147742008-11-19 21:32:24 -08008602 ops->ndo_change_rx_flags(dev, flags);
Patrick McHardyb6c40d62008-10-07 15:26:48 -07008603}
8604
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02008605static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
Patrick McHardy4417da62007-06-27 01:28:10 -07008606{
Eric Dumazetb536db92011-11-30 21:42:26 +00008607 unsigned int old_flags = dev->flags;
Eric W. Biedermand04a48b2012-05-23 17:01:57 -06008608 kuid_t uid;
8609 kgid_t gid;
Patrick McHardy4417da62007-06-27 01:28:10 -07008610
Patrick McHardy24023452007-07-14 18:51:31 -07008611 ASSERT_RTNL();
8612
Wang Chendad9b332008-06-18 01:48:28 -07008613 dev->flags |= IFF_PROMISC;
8614 dev->promiscuity += inc;
8615 if (dev->promiscuity == 0) {
8616 /*
8617 * Avoid overflow.
8618 * If inc causes overflow, untouch promisc and return error.
8619 */
8620 if (inc < 0)
8621 dev->flags &= ~IFF_PROMISC;
8622 else {
8623 dev->promiscuity -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00008624 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
8625 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07008626 return -EOVERFLOW;
8627 }
8628 }
Patrick McHardy4417da62007-06-27 01:28:10 -07008629 if (dev->flags != old_flags) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00008630 pr_info("device %s %s promiscuous mode\n",
8631 dev->name,
8632 dev->flags & IFF_PROMISC ? "entered" : "left");
David Howells8192b0c2008-11-14 10:39:10 +11008633 if (audit_enabled) {
8634 current_uid_gid(&uid, &gid);
Richard Guy Briggscdfb6b32018-05-12 21:58:20 -04008635 audit_log(audit_context(), GFP_ATOMIC,
8636 AUDIT_ANOM_PROMISCUOUS,
8637 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
8638 dev->name, (dev->flags & IFF_PROMISC),
8639 (old_flags & IFF_PROMISC),
8640 from_kuid(&init_user_ns, audit_get_loginuid(current)),
8641 from_kuid(&init_user_ns, uid),
8642 from_kgid(&init_user_ns, gid),
8643 audit_get_sessionid(current));
David Howells8192b0c2008-11-14 10:39:10 +11008644 }
Patrick McHardy24023452007-07-14 18:51:31 -07008645
Patrick McHardyb6c40d62008-10-07 15:26:48 -07008646 dev_change_rx_flags(dev, IFF_PROMISC);
Patrick McHardy4417da62007-06-27 01:28:10 -07008647 }
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02008648 if (notify)
8649 __dev_notify_flags(dev, old_flags, IFF_PROMISC);
Wang Chendad9b332008-06-18 01:48:28 -07008650 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07008651}
8652
Linus Torvalds1da177e2005-04-16 15:20:36 -07008653/**
8654 * dev_set_promiscuity - update promiscuity count on a device
8655 * @dev: device
8656 * @inc: modifier
8657 *
Stephen Hemminger3041a062006-05-26 13:25:24 -07008658 * Add or remove promiscuity from a device. While the count in the device
Linus Torvalds1da177e2005-04-16 15:20:36 -07008659 * remains above zero the interface remains promiscuous. Once it hits zero
8660 * the device reverts back to normal filtering operation. A negative inc
8661 * value is used to drop promiscuity on the device.
Wang Chendad9b332008-06-18 01:48:28 -07008662 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07008663 */
Wang Chendad9b332008-06-18 01:48:28 -07008664int dev_set_promiscuity(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008665{
Eric Dumazetb536db92011-11-30 21:42:26 +00008666 unsigned int old_flags = dev->flags;
Wang Chendad9b332008-06-18 01:48:28 -07008667 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008668
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02008669 err = __dev_set_promiscuity(dev, inc, true);
Patrick McHardy4b5a6982008-07-06 15:49:08 -07008670 if (err < 0)
Wang Chendad9b332008-06-18 01:48:28 -07008671 return err;
Patrick McHardy4417da62007-06-27 01:28:10 -07008672 if (dev->flags != old_flags)
8673 dev_set_rx_mode(dev);
Wang Chendad9b332008-06-18 01:48:28 -07008674 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008675}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07008676EXPORT_SYMBOL(dev_set_promiscuity);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008677
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02008678static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008679{
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02008680 unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008681
Patrick McHardy24023452007-07-14 18:51:31 -07008682 ASSERT_RTNL();
8683
Linus Torvalds1da177e2005-04-16 15:20:36 -07008684 dev->flags |= IFF_ALLMULTI;
Wang Chendad9b332008-06-18 01:48:28 -07008685 dev->allmulti += inc;
8686 if (dev->allmulti == 0) {
8687 /*
8688 * Avoid overflow.
8689 * If inc causes overflow, untouch allmulti and return error.
8690 */
8691 if (inc < 0)
8692 dev->flags &= ~IFF_ALLMULTI;
8693 else {
8694 dev->allmulti -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00008695 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
8696 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07008697 return -EOVERFLOW;
8698 }
8699 }
Patrick McHardy24023452007-07-14 18:51:31 -07008700 if (dev->flags ^ old_flags) {
Patrick McHardyb6c40d62008-10-07 15:26:48 -07008701 dev_change_rx_flags(dev, IFF_ALLMULTI);
Patrick McHardy4417da62007-06-27 01:28:10 -07008702 dev_set_rx_mode(dev);
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02008703 if (notify)
8704 __dev_notify_flags(dev, old_flags,
8705 dev->gflags ^ old_gflags);
Patrick McHardy24023452007-07-14 18:51:31 -07008706 }
Wang Chendad9b332008-06-18 01:48:28 -07008707 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07008708}
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02008709
8710/**
8711 * dev_set_allmulti - update allmulti count on a device
8712 * @dev: device
8713 * @inc: modifier
8714 *
8715 * Add or remove reception of all multicast frames to a device. While the
8716 * count in the device remains above zero the interface remains listening
8717 * to all interfaces. Once it hits zero the device reverts back to normal
8718 * filtering operation. A negative @inc value is used to drop the counter
8719 * when releasing a resource needing all multicasts.
8720 * Return 0 if successful or a negative errno code on error.
8721 */
8722
8723int dev_set_allmulti(struct net_device *dev, int inc)
8724{
8725 return __dev_set_allmulti(dev, inc, true);
8726}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07008727EXPORT_SYMBOL(dev_set_allmulti);
Patrick McHardy4417da62007-06-27 01:28:10 -07008728
8729/*
8730 * Upload unicast and multicast address lists to device and
8731 * configure RX filtering. When the device doesn't support unicast
Joe Perches53ccaae2007-12-20 14:02:06 -08008732 * filtering it is put in promiscuous mode while unicast addresses
Patrick McHardy4417da62007-06-27 01:28:10 -07008733 * are present.
8734 */
8735void __dev_set_rx_mode(struct net_device *dev)
8736{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08008737 const struct net_device_ops *ops = dev->netdev_ops;
8738
Patrick McHardy4417da62007-06-27 01:28:10 -07008739 /* dev_open will call this function so the list will stay sane. */
8740 if (!(dev->flags&IFF_UP))
8741 return;
8742
8743 if (!netif_device_present(dev))
YOSHIFUJI Hideaki40b77c92007-07-19 10:43:23 +09008744 return;
Patrick McHardy4417da62007-06-27 01:28:10 -07008745
Jiri Pirko01789342011-08-16 06:29:00 +00008746 if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
Patrick McHardy4417da62007-06-27 01:28:10 -07008747 /* Unicast addresses changes may only happen under the rtnl,
8748 * therefore calling __dev_set_promiscuity here is safe.
8749 */
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08008750 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02008751 __dev_set_promiscuity(dev, 1, false);
Joe Perches2d348d12011-07-25 16:17:35 -07008752 dev->uc_promisc = true;
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08008753 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02008754 __dev_set_promiscuity(dev, -1, false);
Joe Perches2d348d12011-07-25 16:17:35 -07008755 dev->uc_promisc = false;
Patrick McHardy4417da62007-06-27 01:28:10 -07008756 }
Patrick McHardy4417da62007-06-27 01:28:10 -07008757 }
Jiri Pirko01789342011-08-16 06:29:00 +00008758
8759 if (ops->ndo_set_rx_mode)
8760 ops->ndo_set_rx_mode(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07008761}
8762
8763void dev_set_rx_mode(struct net_device *dev)
8764{
David S. Millerb9e40852008-07-15 00:15:08 -07008765 netif_addr_lock_bh(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07008766 __dev_set_rx_mode(dev);
David S. Millerb9e40852008-07-15 00:15:08 -07008767 netif_addr_unlock_bh(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008768}
8769
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07008770/**
8771 * dev_get_flags - get flags reported to userspace
8772 * @dev: device
8773 *
8774 * Get the combination of flag bits exported through APIs to userspace.
8775 */
Eric Dumazet95c96172012-04-15 05:58:06 +00008776unsigned int dev_get_flags(const struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008777{
Eric Dumazet95c96172012-04-15 05:58:06 +00008778 unsigned int flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008779
8780 flags = (dev->flags & ~(IFF_PROMISC |
8781 IFF_ALLMULTI |
Stefan Rompfb00055a2006-03-20 17:09:11 -08008782 IFF_RUNNING |
8783 IFF_LOWER_UP |
8784 IFF_DORMANT)) |
Linus Torvalds1da177e2005-04-16 15:20:36 -07008785 (dev->gflags & (IFF_PROMISC |
8786 IFF_ALLMULTI));
8787
Stefan Rompfb00055a2006-03-20 17:09:11 -08008788 if (netif_running(dev)) {
8789 if (netif_oper_up(dev))
8790 flags |= IFF_RUNNING;
8791 if (netif_carrier_ok(dev))
8792 flags |= IFF_LOWER_UP;
8793 if (netif_dormant(dev))
8794 flags |= IFF_DORMANT;
8795 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07008796
8797 return flags;
8798}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07008799EXPORT_SYMBOL(dev_get_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008800
Petr Machata6d040322018-12-06 17:05:43 +00008801int __dev_change_flags(struct net_device *dev, unsigned int flags,
8802 struct netlink_ext_ack *extack)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008803{
Eric Dumazetb536db92011-11-30 21:42:26 +00008804 unsigned int old_flags = dev->flags;
Patrick McHardybd380812010-02-26 06:34:53 +00008805 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008806
Patrick McHardy24023452007-07-14 18:51:31 -07008807 ASSERT_RTNL();
8808
Linus Torvalds1da177e2005-04-16 15:20:36 -07008809 /*
8810 * Set the flags on our device.
8811 */
8812
8813 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
8814 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
8815 IFF_AUTOMEDIA)) |
8816 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
8817 IFF_ALLMULTI));
8818
8819 /*
8820 * Load in the correct multicast list now the flags have changed.
8821 */
8822
Patrick McHardyb6c40d62008-10-07 15:26:48 -07008823 if ((old_flags ^ flags) & IFF_MULTICAST)
8824 dev_change_rx_flags(dev, IFF_MULTICAST);
Patrick McHardy24023452007-07-14 18:51:31 -07008825
Patrick McHardy4417da62007-06-27 01:28:10 -07008826 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008827
8828 /*
8829 * Have we downed the interface. We handle IFF_UP ourselves
8830 * according to user attempts to set it, rather than blindly
8831 * setting it.
8832 */
8833
8834 ret = 0;
stephen hemminger7051b882017-07-18 15:59:27 -07008835 if ((old_flags ^ flags) & IFF_UP) {
8836 if (old_flags & IFF_UP)
8837 __dev_close(dev);
8838 else
Petr Machata40c900a2018-12-06 17:05:47 +00008839 ret = __dev_open(dev, extack);
stephen hemminger7051b882017-07-18 15:59:27 -07008840 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07008841
Linus Torvalds1da177e2005-04-16 15:20:36 -07008842 if ((flags ^ dev->gflags) & IFF_PROMISC) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07008843 int inc = (flags & IFF_PROMISC) ? 1 : -1;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02008844 unsigned int old_flags = dev->flags;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07008845
Linus Torvalds1da177e2005-04-16 15:20:36 -07008846 dev->gflags ^= IFF_PROMISC;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02008847
8848 if (__dev_set_promiscuity(dev, inc, false) >= 0)
8849 if (dev->flags != old_flags)
8850 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008851 }
8852
8853 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
tchardingeb13da12017-02-09 17:56:06 +11008854 * is important. Some (broken) drivers set IFF_PROMISC, when
8855 * IFF_ALLMULTI is requested not asking us and not reporting.
Linus Torvalds1da177e2005-04-16 15:20:36 -07008856 */
8857 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07008858 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
8859
Linus Torvalds1da177e2005-04-16 15:20:36 -07008860 dev->gflags ^= IFF_ALLMULTI;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02008861 __dev_set_allmulti(dev, inc, false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008862 }
8863
Patrick McHardybd380812010-02-26 06:34:53 +00008864 return ret;
8865}
8866
Nicolas Dichtela528c212013-09-25 12:02:44 +02008867void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
8868 unsigned int gchanges)
Patrick McHardybd380812010-02-26 06:34:53 +00008869{
8870 unsigned int changes = dev->flags ^ old_flags;
8871
Nicolas Dichtela528c212013-09-25 12:02:44 +02008872 if (gchanges)
Alexei Starovoitov7f294052013-10-23 16:02:42 -07008873 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC);
Nicolas Dichtela528c212013-09-25 12:02:44 +02008874
Patrick McHardybd380812010-02-26 06:34:53 +00008875 if (changes & IFF_UP) {
8876 if (dev->flags & IFF_UP)
8877 call_netdevice_notifiers(NETDEV_UP, dev);
8878 else
8879 call_netdevice_notifiers(NETDEV_DOWN, dev);
8880 }
8881
8882 if (dev->flags & IFF_UP &&
Jiri Pirkobe9efd32013-05-28 01:30:22 +00008883 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
David Ahern51d0c0472017-10-04 17:48:45 -07008884 struct netdev_notifier_change_info change_info = {
8885 .info = {
8886 .dev = dev,
8887 },
8888 .flags_changed = changes,
8889 };
Jiri Pirkobe9efd32013-05-28 01:30:22 +00008890
David Ahern51d0c0472017-10-04 17:48:45 -07008891 call_netdevice_notifiers_info(NETDEV_CHANGE, &change_info.info);
Jiri Pirkobe9efd32013-05-28 01:30:22 +00008892 }
Patrick McHardybd380812010-02-26 06:34:53 +00008893}
8894
8895/**
8896 * dev_change_flags - change device settings
8897 * @dev: device
8898 * @flags: device state flags
Petr Machata567c5e12018-12-06 17:05:42 +00008899 * @extack: netlink extended ack
Patrick McHardybd380812010-02-26 06:34:53 +00008900 *
8901 * Change settings on device based state flags. The flags are
8902 * in the userspace exported format.
8903 */
Petr Machata567c5e12018-12-06 17:05:42 +00008904int dev_change_flags(struct net_device *dev, unsigned int flags,
8905 struct netlink_ext_ack *extack)
Patrick McHardybd380812010-02-26 06:34:53 +00008906{
Eric Dumazetb536db92011-11-30 21:42:26 +00008907 int ret;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02008908 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
Patrick McHardybd380812010-02-26 06:34:53 +00008909
Petr Machata6d040322018-12-06 17:05:43 +00008910 ret = __dev_change_flags(dev, flags, extack);
Patrick McHardybd380812010-02-26 06:34:53 +00008911 if (ret < 0)
8912 return ret;
8913
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02008914 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
Nicolas Dichtela528c212013-09-25 12:02:44 +02008915 __dev_notify_flags(dev, old_flags, changes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008916 return ret;
8917}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07008918EXPORT_SYMBOL(dev_change_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008919
WANG Congf51048c2017-07-06 15:01:57 -07008920int __dev_set_mtu(struct net_device *dev, int new_mtu)
Veaceslav Falico2315dc92014-01-10 16:56:25 +01008921{
8922 const struct net_device_ops *ops = dev->netdev_ops;
8923
8924 if (ops->ndo_change_mtu)
8925 return ops->ndo_change_mtu(dev, new_mtu);
8926
Eric Dumazet501a90c2019-12-05 20:43:46 -08008927 /* Pairs with all the lockless reads of dev->mtu in the stack */
8928 WRITE_ONCE(dev->mtu, new_mtu);
Veaceslav Falico2315dc92014-01-10 16:56:25 +01008929 return 0;
8930}
WANG Congf51048c2017-07-06 15:01:57 -07008931EXPORT_SYMBOL(__dev_set_mtu);
Veaceslav Falico2315dc92014-01-10 16:56:25 +01008932
Eric Dumazetd836f5c2020-01-21 22:47:29 -08008933int dev_validate_mtu(struct net_device *dev, int new_mtu,
8934 struct netlink_ext_ack *extack)
8935{
8936 /* MTU must be positive, and in range */
8937 if (new_mtu < 0 || new_mtu < dev->min_mtu) {
8938 NL_SET_ERR_MSG(extack, "mtu less than device minimum");
8939 return -EINVAL;
8940 }
8941
8942 if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) {
8943 NL_SET_ERR_MSG(extack, "mtu greater than device maximum");
8944 return -EINVAL;
8945 }
8946 return 0;
8947}
8948
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07008949/**
Stephen Hemminger7a4c53b2018-07-27 13:43:23 -07008950 * dev_set_mtu_ext - Change maximum transfer unit
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07008951 * @dev: device
8952 * @new_mtu: new transfer unit
Stephen Hemminger7a4c53b2018-07-27 13:43:23 -07008953 * @extack: netlink extended ack
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07008954 *
8955 * Change the maximum transfer size of the network device.
8956 */
Stephen Hemminger7a4c53b2018-07-27 13:43:23 -07008957int dev_set_mtu_ext(struct net_device *dev, int new_mtu,
8958 struct netlink_ext_ack *extack)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008959{
Veaceslav Falico2315dc92014-01-10 16:56:25 +01008960 int err, orig_mtu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008961
8962 if (new_mtu == dev->mtu)
8963 return 0;
8964
Eric Dumazetd836f5c2020-01-21 22:47:29 -08008965 err = dev_validate_mtu(dev, new_mtu, extack);
8966 if (err)
8967 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008968
8969 if (!netif_device_present(dev))
8970 return -ENODEV;
8971
Veaceslav Falico1d486bf2014-01-16 00:02:18 +01008972 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
8973 err = notifier_to_errno(err);
8974 if (err)
8975 return err;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08008976
Veaceslav Falico2315dc92014-01-10 16:56:25 +01008977 orig_mtu = dev->mtu;
8978 err = __dev_set_mtu(dev, new_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008979
Veaceslav Falico2315dc92014-01-10 16:56:25 +01008980 if (!err) {
Sabrina Dubrocaaf7d6cc2018-10-09 17:48:14 +02008981 err = call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
8982 orig_mtu);
Veaceslav Falico2315dc92014-01-10 16:56:25 +01008983 err = notifier_to_errno(err);
8984 if (err) {
8985 /* setting mtu back and notifying everyone again,
8986 * so that they have a chance to revert changes.
8987 */
8988 __dev_set_mtu(dev, orig_mtu);
Sabrina Dubrocaaf7d6cc2018-10-09 17:48:14 +02008989 call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
8990 new_mtu);
Veaceslav Falico2315dc92014-01-10 16:56:25 +01008991 }
8992 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07008993 return err;
8994}
Stephen Hemminger7a4c53b2018-07-27 13:43:23 -07008995
8996int dev_set_mtu(struct net_device *dev, int new_mtu)
8997{
8998 struct netlink_ext_ack extack;
8999 int err;
9000
Li RongQinga6bcfc82018-08-03 15:45:21 +08009001 memset(&extack, 0, sizeof(extack));
Stephen Hemminger7a4c53b2018-07-27 13:43:23 -07009002 err = dev_set_mtu_ext(dev, new_mtu, &extack);
Li RongQinga6bcfc82018-08-03 15:45:21 +08009003 if (err && extack._msg)
Stephen Hemminger7a4c53b2018-07-27 13:43:23 -07009004 net_err_ratelimited("%s: %s\n", dev->name, extack._msg);
9005 return err;
9006}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07009007EXPORT_SYMBOL(dev_set_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07009008
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07009009/**
Cong Wang6a643dd2018-01-25 18:26:22 -08009010 * dev_change_tx_queue_len - Change TX queue length of a netdevice
9011 * @dev: device
9012 * @new_len: new tx queue length
9013 */
9014int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len)
9015{
9016 unsigned int orig_len = dev->tx_queue_len;
9017 int res;
9018
9019 if (new_len != (unsigned int)new_len)
9020 return -ERANGE;
9021
9022 if (new_len != orig_len) {
9023 dev->tx_queue_len = new_len;
9024 res = call_netdevice_notifiers(NETDEV_CHANGE_TX_QUEUE_LEN, dev);
9025 res = notifier_to_errno(res);
Tariq Toukan7effaf02018-07-24 14:12:20 +03009026 if (res)
9027 goto err_rollback;
9028 res = dev_qdisc_change_tx_queue_len(dev);
9029 if (res)
9030 goto err_rollback;
Cong Wang6a643dd2018-01-25 18:26:22 -08009031 }
9032
9033 return 0;
Tariq Toukan7effaf02018-07-24 14:12:20 +03009034
9035err_rollback:
9036 netdev_err(dev, "refused to change device tx_queue_len\n");
9037 dev->tx_queue_len = orig_len;
9038 return res;
Cong Wang6a643dd2018-01-25 18:26:22 -08009039}
9040
9041/**
Vlad Dogarucbda10f2011-01-13 23:38:30 +00009042 * dev_set_group - Change group this device belongs to
9043 * @dev: device
9044 * @new_group: group this device should belong to
9045 */
9046void dev_set_group(struct net_device *dev, int new_group)
9047{
9048 dev->group = new_group;
9049}
9050EXPORT_SYMBOL(dev_set_group);
9051
9052/**
Petr Machatad59cdf92018-12-13 11:54:35 +00009053 * dev_pre_changeaddr_notify - Call NETDEV_PRE_CHANGEADDR.
9054 * @dev: device
9055 * @addr: new address
9056 * @extack: netlink extended ack
9057 */
9058int dev_pre_changeaddr_notify(struct net_device *dev, const char *addr,
9059 struct netlink_ext_ack *extack)
9060{
9061 struct netdev_notifier_pre_changeaddr_info info = {
9062 .info.dev = dev,
9063 .info.extack = extack,
9064 .dev_addr = addr,
9065 };
9066 int rc;
9067
9068 rc = call_netdevice_notifiers_info(NETDEV_PRE_CHANGEADDR, &info.info);
9069 return notifier_to_errno(rc);
9070}
9071EXPORT_SYMBOL(dev_pre_changeaddr_notify);
9072
9073/**
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07009074 * dev_set_mac_address - Change Media Access Control Address
9075 * @dev: device
9076 * @sa: new address
Petr Machata3a37a962018-12-13 11:54:30 +00009077 * @extack: netlink extended ack
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07009078 *
9079 * Change the hardware (MAC) address of the device
9080 */
Petr Machata3a37a962018-12-13 11:54:30 +00009081int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa,
9082 struct netlink_ext_ack *extack)
Linus Torvalds1da177e2005-04-16 15:20:36 -07009083{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08009084 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07009085 int err;
9086
Stephen Hemmingerd3147742008-11-19 21:32:24 -08009087 if (!ops->ndo_set_mac_address)
Linus Torvalds1da177e2005-04-16 15:20:36 -07009088 return -EOPNOTSUPP;
9089 if (sa->sa_family != dev->type)
9090 return -EINVAL;
9091 if (!netif_device_present(dev))
9092 return -ENODEV;
Petr Machatad59cdf92018-12-13 11:54:35 +00009093 err = dev_pre_changeaddr_notify(dev, sa->sa_data, extack);
9094 if (err)
9095 return err;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08009096 err = ops->ndo_set_mac_address(dev, sa);
Jiri Pirkof6521512013-01-01 03:30:14 +00009097 if (err)
9098 return err;
Jiri Pirkofbdeca22013-01-01 03:30:16 +00009099 dev->addr_assign_type = NET_ADDR_SET;
Jiri Pirkof6521512013-01-01 03:30:14 +00009100 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04009101 add_device_randomness(dev->dev_addr, dev->addr_len);
Jiri Pirkof6521512013-01-01 03:30:14 +00009102 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07009103}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07009104EXPORT_SYMBOL(dev_set_mac_address);
Linus Torvalds1da177e2005-04-16 15:20:36 -07009105
Cong Wang3b23a322021-02-11 11:34:10 -08009106static DECLARE_RWSEM(dev_addr_sem);
9107
9108int dev_set_mac_address_user(struct net_device *dev, struct sockaddr *sa,
9109 struct netlink_ext_ack *extack)
9110{
9111 int ret;
9112
9113 down_write(&dev_addr_sem);
9114 ret = dev_set_mac_address(dev, sa, extack);
9115 up_write(&dev_addr_sem);
9116 return ret;
9117}
9118EXPORT_SYMBOL(dev_set_mac_address_user);
9119
9120int dev_get_mac_address(struct sockaddr *sa, struct net *net, char *dev_name)
9121{
9122 size_t size = sizeof(sa->sa_data);
9123 struct net_device *dev;
9124 int ret = 0;
9125
9126 down_read(&dev_addr_sem);
9127 rcu_read_lock();
9128
9129 dev = dev_get_by_name_rcu(net, dev_name);
9130 if (!dev) {
9131 ret = -ENODEV;
9132 goto unlock;
9133 }
9134 if (!dev->addr_len)
9135 memset(sa->sa_data, 0, size);
9136 else
9137 memcpy(sa->sa_data, dev->dev_addr,
9138 min_t(size_t, size, dev->addr_len));
9139 sa->sa_family = dev->type;
9140
9141unlock:
9142 rcu_read_unlock();
9143 up_read(&dev_addr_sem);
9144 return ret;
9145}
9146EXPORT_SYMBOL(dev_get_mac_address);
9147
Jiri Pirko4bf84c32012-12-27 23:49:37 +00009148/**
9149 * dev_change_carrier - Change device carrier
9150 * @dev: device
Randy Dunlap691b3b72013-03-04 12:32:43 +00009151 * @new_carrier: new value
Jiri Pirko4bf84c32012-12-27 23:49:37 +00009152 *
9153 * Change device carrier
9154 */
9155int dev_change_carrier(struct net_device *dev, bool new_carrier)
9156{
9157 const struct net_device_ops *ops = dev->netdev_ops;
9158
9159 if (!ops->ndo_change_carrier)
9160 return -EOPNOTSUPP;
9161 if (!netif_device_present(dev))
9162 return -ENODEV;
9163 return ops->ndo_change_carrier(dev, new_carrier);
9164}
9165EXPORT_SYMBOL(dev_change_carrier);
9166
Linus Torvalds1da177e2005-04-16 15:20:36 -07009167/**
Jiri Pirko66b52b02013-07-29 18:16:49 +02009168 * dev_get_phys_port_id - Get device physical port ID
9169 * @dev: device
9170 * @ppid: port ID
9171 *
9172 * Get device physical port ID
9173 */
9174int dev_get_phys_port_id(struct net_device *dev,
Jiri Pirko02637fc2014-11-28 14:34:16 +01009175 struct netdev_phys_item_id *ppid)
Jiri Pirko66b52b02013-07-29 18:16:49 +02009176{
9177 const struct net_device_ops *ops = dev->netdev_ops;
9178
9179 if (!ops->ndo_get_phys_port_id)
9180 return -EOPNOTSUPP;
9181 return ops->ndo_get_phys_port_id(dev, ppid);
9182}
9183EXPORT_SYMBOL(dev_get_phys_port_id);
9184
9185/**
David Aherndb24a902015-03-17 20:23:15 -06009186 * dev_get_phys_port_name - Get device physical port name
9187 * @dev: device
9188 * @name: port name
Luis de Bethencourted49e652016-03-21 16:31:14 +00009189 * @len: limit of bytes to copy to name
David Aherndb24a902015-03-17 20:23:15 -06009190 *
9191 * Get device physical port name
9192 */
9193int dev_get_phys_port_name(struct net_device *dev,
9194 char *name, size_t len)
9195{
9196 const struct net_device_ops *ops = dev->netdev_ops;
Jiri Pirkoaf3836d2019-03-28 13:56:37 +01009197 int err;
David Aherndb24a902015-03-17 20:23:15 -06009198
Jiri Pirkoaf3836d2019-03-28 13:56:37 +01009199 if (ops->ndo_get_phys_port_name) {
9200 err = ops->ndo_get_phys_port_name(dev, name, len);
9201 if (err != -EOPNOTSUPP)
9202 return err;
9203 }
9204 return devlink_compat_phys_port_name_get(dev, name, len);
David Aherndb24a902015-03-17 20:23:15 -06009205}
9206EXPORT_SYMBOL(dev_get_phys_port_name);
9207
9208/**
Florian Fainellid6abc5962019-02-06 09:45:35 -08009209 * dev_get_port_parent_id - Get the device's port parent identifier
9210 * @dev: network device
9211 * @ppid: pointer to a storage for the port's parent identifier
9212 * @recurse: allow/disallow recursion to lower devices
9213 *
9214 * Get the devices's port parent identifier
9215 */
9216int dev_get_port_parent_id(struct net_device *dev,
9217 struct netdev_phys_item_id *ppid,
9218 bool recurse)
9219{
9220 const struct net_device_ops *ops = dev->netdev_ops;
9221 struct netdev_phys_item_id first = { };
9222 struct net_device *lower_dev;
9223 struct list_head *iter;
Jiri Pirko7e1146e2019-04-03 14:24:17 +02009224 int err;
Florian Fainellid6abc5962019-02-06 09:45:35 -08009225
Jiri Pirko7e1146e2019-04-03 14:24:17 +02009226 if (ops->ndo_get_port_parent_id) {
9227 err = ops->ndo_get_port_parent_id(dev, ppid);
9228 if (err != -EOPNOTSUPP)
9229 return err;
9230 }
9231
9232 err = devlink_compat_switch_id_get(dev, ppid);
9233 if (!err || err != -EOPNOTSUPP)
9234 return err;
Florian Fainellid6abc5962019-02-06 09:45:35 -08009235
9236 if (!recurse)
Jiri Pirko7e1146e2019-04-03 14:24:17 +02009237 return -EOPNOTSUPP;
Florian Fainellid6abc5962019-02-06 09:45:35 -08009238
9239 netdev_for_each_lower_dev(dev, lower_dev, iter) {
9240 err = dev_get_port_parent_id(lower_dev, ppid, recurse);
9241 if (err)
9242 break;
9243 if (!first.id_len)
9244 first = *ppid;
9245 else if (memcmp(&first, ppid, sizeof(*ppid)))
Ido Schimmele1b9efe2020-09-10 14:01:26 +03009246 return -EOPNOTSUPP;
Florian Fainellid6abc5962019-02-06 09:45:35 -08009247 }
9248
9249 return err;
9250}
9251EXPORT_SYMBOL(dev_get_port_parent_id);
9252
9253/**
9254 * netdev_port_same_parent_id - Indicate if two network devices have
9255 * the same port parent identifier
9256 * @a: first network device
9257 * @b: second network device
9258 */
9259bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b)
9260{
9261 struct netdev_phys_item_id a_id = { };
9262 struct netdev_phys_item_id b_id = { };
9263
9264 if (dev_get_port_parent_id(a, &a_id, true) ||
9265 dev_get_port_parent_id(b, &b_id, true))
9266 return false;
9267
9268 return netdev_phys_item_id_same(&a_id, &b_id);
9269}
9270EXPORT_SYMBOL(netdev_port_same_parent_id);
9271
9272/**
Anuradha Karuppiahd746d702015-07-14 13:43:19 -07009273 * dev_change_proto_down - update protocol port state information
9274 * @dev: device
9275 * @proto_down: new value
9276 *
9277 * This info can be used by switch drivers to set the phys state of the
9278 * port.
9279 */
9280int dev_change_proto_down(struct net_device *dev, bool proto_down)
9281{
9282 const struct net_device_ops *ops = dev->netdev_ops;
9283
9284 if (!ops->ndo_change_proto_down)
9285 return -EOPNOTSUPP;
9286 if (!netif_device_present(dev))
9287 return -ENODEV;
9288 return ops->ndo_change_proto_down(dev, proto_down);
9289}
9290EXPORT_SYMBOL(dev_change_proto_down);
9291
Andy Roulinb5899672019-02-22 18:06:36 +00009292/**
9293 * dev_change_proto_down_generic - generic implementation for
9294 * ndo_change_proto_down that sets carrier according to
9295 * proto_down.
9296 *
9297 * @dev: device
9298 * @proto_down: new value
9299 */
9300int dev_change_proto_down_generic(struct net_device *dev, bool proto_down)
9301{
9302 if (proto_down)
9303 netif_carrier_off(dev);
9304 else
9305 netif_carrier_on(dev);
9306 dev->proto_down = proto_down;
9307 return 0;
9308}
9309EXPORT_SYMBOL(dev_change_proto_down_generic);
9310
Roopa Prabhu829eb202020-07-31 17:34:01 -07009311/**
9312 * dev_change_proto_down_reason - proto down reason
9313 *
9314 * @dev: device
9315 * @mask: proto down mask
9316 * @value: proto down value
9317 */
9318void dev_change_proto_down_reason(struct net_device *dev, unsigned long mask,
9319 u32 value)
9320{
9321 int b;
9322
9323 if (!mask) {
9324 dev->proto_down_reason = value;
9325 } else {
9326 for_each_set_bit(b, &mask, 32) {
9327 if (value & (1 << b))
9328 dev->proto_down_reason |= BIT(b);
9329 else
9330 dev->proto_down_reason &= ~BIT(b);
9331 }
9332 }
9333}
9334EXPORT_SYMBOL(dev_change_proto_down_reason);
9335
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009336struct bpf_xdp_link {
9337 struct bpf_link link;
9338 struct net_device *dev; /* protected by rtnl_lock, no refcnt held */
9339 int flags;
9340};
9341
Andrii Nakryikoc8a36f12020-08-19 22:28:41 -07009342static enum bpf_xdp_mode dev_xdp_mode(struct net_device *dev, u32 flags)
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02009343{
Andrii Nakryiko7f0a8382020-07-21 23:45:55 -07009344 if (flags & XDP_FLAGS_HW_MODE)
9345 return XDP_MODE_HW;
9346 if (flags & XDP_FLAGS_DRV_MODE)
9347 return XDP_MODE_DRV;
Andrii Nakryikoc8a36f12020-08-19 22:28:41 -07009348 if (flags & XDP_FLAGS_SKB_MODE)
9349 return XDP_MODE_SKB;
9350 return dev->netdev_ops->ndo_bpf ? XDP_MODE_DRV : XDP_MODE_SKB;
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02009351}
9352
Andrii Nakryiko7f0a8382020-07-21 23:45:55 -07009353static bpf_op_t dev_xdp_bpf_op(struct net_device *dev, enum bpf_xdp_mode mode)
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02009354{
Andrii Nakryiko7f0a8382020-07-21 23:45:55 -07009355 switch (mode) {
9356 case XDP_MODE_SKB:
9357 return generic_xdp_install;
9358 case XDP_MODE_DRV:
9359 case XDP_MODE_HW:
9360 return dev->netdev_ops->ndo_bpf;
9361 default:
9362 return NULL;
Tom Rix5d867242020-11-01 07:36:47 -08009363 }
Andrii Nakryiko7f0a8382020-07-21 23:45:55 -07009364}
9365
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009366static struct bpf_xdp_link *dev_xdp_link(struct net_device *dev,
9367 enum bpf_xdp_mode mode)
9368{
9369 return dev->xdp_state[mode].link;
9370}
9371
Andrii Nakryiko7f0a8382020-07-21 23:45:55 -07009372static struct bpf_prog *dev_xdp_prog(struct net_device *dev,
9373 enum bpf_xdp_mode mode)
9374{
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009375 struct bpf_xdp_link *link = dev_xdp_link(dev, mode);
9376
9377 if (link)
9378 return link->link.prog;
Andrii Nakryiko7f0a8382020-07-21 23:45:55 -07009379 return dev->xdp_state[mode].prog;
9380}
9381
Toke Høiland-Jørgensen998f1722020-12-09 14:57:37 +01009382static u8 dev_xdp_prog_count(struct net_device *dev)
9383{
9384 u8 count = 0;
9385 int i;
9386
9387 for (i = 0; i < __MAX_XDP_MODE; i++)
9388 if (dev->xdp_state[i].prog || dev->xdp_state[i].link)
9389 count++;
9390 return count;
9391}
9392
Andrii Nakryiko7f0a8382020-07-21 23:45:55 -07009393u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode)
9394{
9395 struct bpf_prog *prog = dev_xdp_prog(dev, mode);
9396
9397 return prog ? prog->aux->id : 0;
9398}
9399
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009400static void dev_xdp_set_link(struct net_device *dev, enum bpf_xdp_mode mode,
9401 struct bpf_xdp_link *link)
9402{
9403 dev->xdp_state[mode].link = link;
9404 dev->xdp_state[mode].prog = NULL;
9405}
9406
Andrii Nakryiko7f0a8382020-07-21 23:45:55 -07009407static void dev_xdp_set_prog(struct net_device *dev, enum bpf_xdp_mode mode,
9408 struct bpf_prog *prog)
9409{
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009410 dev->xdp_state[mode].link = NULL;
Andrii Nakryiko7f0a8382020-07-21 23:45:55 -07009411 dev->xdp_state[mode].prog = prog;
9412}
9413
9414static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode,
9415 bpf_op_t bpf_op, struct netlink_ext_ack *extack,
9416 u32 flags, struct bpf_prog *prog)
9417{
Jakub Kicinskif4e63522017-11-03 13:56:16 -07009418 struct netdev_bpf xdp;
Björn Töpel7e6897f2019-12-13 18:51:09 +01009419 int err;
9420
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02009421 memset(&xdp, 0, sizeof(xdp));
Andrii Nakryiko7f0a8382020-07-21 23:45:55 -07009422 xdp.command = mode == XDP_MODE_HW ? XDP_SETUP_PROG_HW : XDP_SETUP_PROG;
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02009423 xdp.extack = extack;
Jakub Kicinski32d60272017-06-21 18:25:03 -07009424 xdp.flags = flags;
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02009425 xdp.prog = prog;
9426
Andrii Nakryiko7f0a8382020-07-21 23:45:55 -07009427 /* Drivers assume refcnt is already incremented (i.e, prog pointer is
9428 * "moved" into driver), so they don't increment it on their own, but
9429 * they do decrement refcnt when program is detached or replaced.
9430 * Given net_device also owns link/prog, we need to bump refcnt here
9431 * to prevent drivers from underflowing it.
9432 */
9433 if (prog)
9434 bpf_prog_inc(prog);
Björn Töpel7e6897f2019-12-13 18:51:09 +01009435 err = bpf_op(dev, &xdp);
Andrii Nakryiko7f0a8382020-07-21 23:45:55 -07009436 if (err) {
9437 if (prog)
9438 bpf_prog_put(prog);
9439 return err;
9440 }
Björn Töpel7e6897f2019-12-13 18:51:09 +01009441
Andrii Nakryiko7f0a8382020-07-21 23:45:55 -07009442 if (mode != XDP_MODE_HW)
9443 bpf_prog_change_xdp(dev_xdp_prog(dev, mode), prog);
Björn Töpel7e6897f2019-12-13 18:51:09 +01009444
Andrii Nakryiko7f0a8382020-07-21 23:45:55 -07009445 return 0;
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02009446}
9447
Jakub Kicinskibd0b2e72017-12-01 15:08:57 -08009448static void dev_xdp_uninstall(struct net_device *dev)
9449{
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009450 struct bpf_xdp_link *link;
Andrii Nakryiko7f0a8382020-07-21 23:45:55 -07009451 struct bpf_prog *prog;
9452 enum bpf_xdp_mode mode;
9453 bpf_op_t bpf_op;
Jakub Kicinskibd0b2e72017-12-01 15:08:57 -08009454
Andrii Nakryiko7f0a8382020-07-21 23:45:55 -07009455 ASSERT_RTNL();
Jakub Kicinskibd0b2e72017-12-01 15:08:57 -08009456
Andrii Nakryiko7f0a8382020-07-21 23:45:55 -07009457 for (mode = XDP_MODE_SKB; mode < __MAX_XDP_MODE; mode++) {
9458 prog = dev_xdp_prog(dev, mode);
9459 if (!prog)
9460 continue;
Jakub Kicinskibd0b2e72017-12-01 15:08:57 -08009461
Andrii Nakryiko7f0a8382020-07-21 23:45:55 -07009462 bpf_op = dev_xdp_bpf_op(dev, mode);
9463 if (!bpf_op)
9464 continue;
Jakub Kicinskibd0b2e72017-12-01 15:08:57 -08009465
Andrii Nakryiko7f0a8382020-07-21 23:45:55 -07009466 WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
9467
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009468 /* auto-detach link from net device */
9469 link = dev_xdp_link(dev, mode);
9470 if (link)
9471 link->dev = NULL;
9472 else
9473 bpf_prog_put(prog);
9474
9475 dev_xdp_set_link(dev, mode, NULL);
Andrii Nakryiko7f0a8382020-07-21 23:45:55 -07009476 }
Jakub Kicinskibd0b2e72017-12-01 15:08:57 -08009477}
9478
Andrii Nakryikod4baa932020-07-21 23:45:56 -07009479static int dev_xdp_attach(struct net_device *dev, struct netlink_ext_ack *extack,
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009480 struct bpf_xdp_link *link, struct bpf_prog *new_prog,
9481 struct bpf_prog *old_prog, u32 flags)
Andrii Nakryikod4baa932020-07-21 23:45:56 -07009482{
Toke Høiland-Jørgensen998f1722020-12-09 14:57:37 +01009483 unsigned int num_modes = hweight32(flags & XDP_FLAGS_MODES);
Andrii Nakryikod4baa932020-07-21 23:45:56 -07009484 struct bpf_prog *cur_prog;
9485 enum bpf_xdp_mode mode;
9486 bpf_op_t bpf_op;
9487 int err;
9488
9489 ASSERT_RTNL();
9490
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009491 /* either link or prog attachment, never both */
9492 if (link && (new_prog || old_prog))
9493 return -EINVAL;
9494 /* link supports only XDP mode flags */
9495 if (link && (flags & ~XDP_FLAGS_MODES)) {
9496 NL_SET_ERR_MSG(extack, "Invalid XDP flags for BPF link attachment");
9497 return -EINVAL;
9498 }
Toke Høiland-Jørgensen998f1722020-12-09 14:57:37 +01009499 /* just one XDP mode bit should be set, zero defaults to drv/skb mode */
9500 if (num_modes > 1) {
Andrii Nakryikod4baa932020-07-21 23:45:56 -07009501 NL_SET_ERR_MSG(extack, "Only one XDP mode flag can be set");
9502 return -EINVAL;
9503 }
Toke Høiland-Jørgensen998f1722020-12-09 14:57:37 +01009504 /* avoid ambiguity if offload + drv/skb mode progs are both loaded */
9505 if (!num_modes && dev_xdp_prog_count(dev) > 1) {
9506 NL_SET_ERR_MSG(extack,
9507 "More than one program loaded, unset mode is ambiguous");
9508 return -EINVAL;
9509 }
Andrii Nakryikod4baa932020-07-21 23:45:56 -07009510 /* old_prog != NULL implies XDP_FLAGS_REPLACE is set */
9511 if (old_prog && !(flags & XDP_FLAGS_REPLACE)) {
9512 NL_SET_ERR_MSG(extack, "XDP_FLAGS_REPLACE is not specified");
9513 return -EINVAL;
9514 }
9515
Andrii Nakryikoc8a36f12020-08-19 22:28:41 -07009516 mode = dev_xdp_mode(dev, flags);
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009517 /* can't replace attached link */
9518 if (dev_xdp_link(dev, mode)) {
9519 NL_SET_ERR_MSG(extack, "Can't replace active BPF XDP link");
9520 return -EBUSY;
9521 }
9522
Andrii Nakryikod4baa932020-07-21 23:45:56 -07009523 cur_prog = dev_xdp_prog(dev, mode);
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009524 /* can't replace attached prog with link */
9525 if (link && cur_prog) {
9526 NL_SET_ERR_MSG(extack, "Can't replace active XDP program with BPF link");
9527 return -EBUSY;
9528 }
Andrii Nakryikod4baa932020-07-21 23:45:56 -07009529 if ((flags & XDP_FLAGS_REPLACE) && cur_prog != old_prog) {
9530 NL_SET_ERR_MSG(extack, "Active program does not match expected");
9531 return -EEXIST;
9532 }
Andrii Nakryikod4baa932020-07-21 23:45:56 -07009533
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009534 /* put effective new program into new_prog */
9535 if (link)
9536 new_prog = link->link.prog;
9537
Andrii Nakryikod4baa932020-07-21 23:45:56 -07009538 if (new_prog) {
9539 bool offload = mode == XDP_MODE_HW;
9540 enum bpf_xdp_mode other_mode = mode == XDP_MODE_SKB
9541 ? XDP_MODE_DRV : XDP_MODE_SKB;
9542
Andrii Nakryiko068d9d12020-08-11 19:29:23 -07009543 if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) && cur_prog) {
9544 NL_SET_ERR_MSG(extack, "XDP program already attached");
9545 return -EBUSY;
9546 }
Andrii Nakryikod4baa932020-07-21 23:45:56 -07009547 if (!offload && dev_xdp_prog(dev, other_mode)) {
9548 NL_SET_ERR_MSG(extack, "Native and generic XDP can't be active at the same time");
9549 return -EEXIST;
9550 }
9551 if (!offload && bpf_prog_is_dev_bound(new_prog->aux)) {
9552 NL_SET_ERR_MSG(extack, "Using device-bound program without HW_MODE flag is not supported");
9553 return -EINVAL;
9554 }
9555 if (new_prog->expected_attach_type == BPF_XDP_DEVMAP) {
9556 NL_SET_ERR_MSG(extack, "BPF_XDP_DEVMAP programs can not be attached to a device");
9557 return -EINVAL;
9558 }
9559 if (new_prog->expected_attach_type == BPF_XDP_CPUMAP) {
9560 NL_SET_ERR_MSG(extack, "BPF_XDP_CPUMAP programs can not be attached to a device");
9561 return -EINVAL;
9562 }
9563 }
9564
9565 /* don't call drivers if the effective program didn't change */
9566 if (new_prog != cur_prog) {
9567 bpf_op = dev_xdp_bpf_op(dev, mode);
9568 if (!bpf_op) {
9569 NL_SET_ERR_MSG(extack, "Underlying driver does not support XDP in native mode");
9570 return -EOPNOTSUPP;
9571 }
9572
9573 err = dev_xdp_install(dev, mode, bpf_op, extack, flags, new_prog);
9574 if (err)
9575 return err;
9576 }
9577
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009578 if (link)
9579 dev_xdp_set_link(dev, mode, link);
9580 else
9581 dev_xdp_set_prog(dev, mode, new_prog);
Andrii Nakryikod4baa932020-07-21 23:45:56 -07009582 if (cur_prog)
9583 bpf_prog_put(cur_prog);
9584
9585 return 0;
9586}
9587
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009588static int dev_xdp_attach_link(struct net_device *dev,
9589 struct netlink_ext_ack *extack,
9590 struct bpf_xdp_link *link)
9591{
9592 return dev_xdp_attach(dev, extack, link, NULL, NULL, link->flags);
9593}
9594
9595static int dev_xdp_detach_link(struct net_device *dev,
9596 struct netlink_ext_ack *extack,
9597 struct bpf_xdp_link *link)
9598{
9599 enum bpf_xdp_mode mode;
9600 bpf_op_t bpf_op;
9601
9602 ASSERT_RTNL();
9603
Andrii Nakryikoc8a36f12020-08-19 22:28:41 -07009604 mode = dev_xdp_mode(dev, link->flags);
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009605 if (dev_xdp_link(dev, mode) != link)
9606 return -EINVAL;
9607
9608 bpf_op = dev_xdp_bpf_op(dev, mode);
9609 WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
9610 dev_xdp_set_link(dev, mode, NULL);
9611 return 0;
9612}
9613
9614static void bpf_xdp_link_release(struct bpf_link *link)
9615{
9616 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9617
9618 rtnl_lock();
9619
9620 /* if racing with net_device's tear down, xdp_link->dev might be
9621 * already NULL, in which case link was already auto-detached
9622 */
Andrii Nakryiko73b11c2a2020-07-31 11:28:26 -07009623 if (xdp_link->dev) {
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009624 WARN_ON(dev_xdp_detach_link(xdp_link->dev, NULL, xdp_link));
Andrii Nakryiko73b11c2a2020-07-31 11:28:26 -07009625 xdp_link->dev = NULL;
9626 }
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009627
9628 rtnl_unlock();
9629}
9630
Andrii Nakryiko73b11c2a2020-07-31 11:28:26 -07009631static int bpf_xdp_link_detach(struct bpf_link *link)
9632{
9633 bpf_xdp_link_release(link);
9634 return 0;
9635}
9636
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009637static void bpf_xdp_link_dealloc(struct bpf_link *link)
9638{
9639 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9640
9641 kfree(xdp_link);
9642}
9643
Andrii Nakryikoc1931c92020-07-21 23:45:59 -07009644static void bpf_xdp_link_show_fdinfo(const struct bpf_link *link,
9645 struct seq_file *seq)
9646{
9647 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9648 u32 ifindex = 0;
9649
9650 rtnl_lock();
9651 if (xdp_link->dev)
9652 ifindex = xdp_link->dev->ifindex;
9653 rtnl_unlock();
9654
9655 seq_printf(seq, "ifindex:\t%u\n", ifindex);
9656}
9657
9658static int bpf_xdp_link_fill_link_info(const struct bpf_link *link,
9659 struct bpf_link_info *info)
9660{
9661 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9662 u32 ifindex = 0;
9663
9664 rtnl_lock();
9665 if (xdp_link->dev)
9666 ifindex = xdp_link->dev->ifindex;
9667 rtnl_unlock();
9668
9669 info->xdp.ifindex = ifindex;
9670 return 0;
9671}
9672
Andrii Nakryiko026a4c22020-07-21 23:45:58 -07009673static int bpf_xdp_link_update(struct bpf_link *link, struct bpf_prog *new_prog,
9674 struct bpf_prog *old_prog)
9675{
9676 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9677 enum bpf_xdp_mode mode;
9678 bpf_op_t bpf_op;
9679 int err = 0;
9680
9681 rtnl_lock();
9682
9683 /* link might have been auto-released already, so fail */
9684 if (!xdp_link->dev) {
9685 err = -ENOLINK;
9686 goto out_unlock;
9687 }
9688
9689 if (old_prog && link->prog != old_prog) {
9690 err = -EPERM;
9691 goto out_unlock;
9692 }
9693 old_prog = link->prog;
9694 if (old_prog == new_prog) {
9695 /* no-op, don't disturb drivers */
9696 bpf_prog_put(new_prog);
9697 goto out_unlock;
9698 }
9699
Andrii Nakryikoc8a36f12020-08-19 22:28:41 -07009700 mode = dev_xdp_mode(xdp_link->dev, xdp_link->flags);
Andrii Nakryiko026a4c22020-07-21 23:45:58 -07009701 bpf_op = dev_xdp_bpf_op(xdp_link->dev, mode);
9702 err = dev_xdp_install(xdp_link->dev, mode, bpf_op, NULL,
9703 xdp_link->flags, new_prog);
9704 if (err)
9705 goto out_unlock;
9706
9707 old_prog = xchg(&link->prog, new_prog);
9708 bpf_prog_put(old_prog);
9709
9710out_unlock:
9711 rtnl_unlock();
9712 return err;
9713}
9714
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009715static const struct bpf_link_ops bpf_xdp_link_lops = {
9716 .release = bpf_xdp_link_release,
9717 .dealloc = bpf_xdp_link_dealloc,
Andrii Nakryiko73b11c2a2020-07-31 11:28:26 -07009718 .detach = bpf_xdp_link_detach,
Andrii Nakryikoc1931c92020-07-21 23:45:59 -07009719 .show_fdinfo = bpf_xdp_link_show_fdinfo,
9720 .fill_link_info = bpf_xdp_link_fill_link_info,
Andrii Nakryiko026a4c22020-07-21 23:45:58 -07009721 .update_prog = bpf_xdp_link_update,
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009722};
9723
9724int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
9725{
9726 struct net *net = current->nsproxy->net_ns;
9727 struct bpf_link_primer link_primer;
9728 struct bpf_xdp_link *link;
9729 struct net_device *dev;
9730 int err, fd;
9731
Xuan Zhuo5acc7d32021-07-10 11:16:35 +08009732 rtnl_lock();
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009733 dev = dev_get_by_index(net, attr->link_create.target_ifindex);
Xuan Zhuo5acc7d32021-07-10 11:16:35 +08009734 if (!dev) {
9735 rtnl_unlock();
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009736 return -EINVAL;
Xuan Zhuo5acc7d32021-07-10 11:16:35 +08009737 }
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009738
9739 link = kzalloc(sizeof(*link), GFP_USER);
9740 if (!link) {
9741 err = -ENOMEM;
Xuan Zhuo5acc7d32021-07-10 11:16:35 +08009742 goto unlock;
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009743 }
9744
9745 bpf_link_init(&link->link, BPF_LINK_TYPE_XDP, &bpf_xdp_link_lops, prog);
9746 link->dev = dev;
9747 link->flags = attr->link_create.flags;
9748
9749 err = bpf_link_prime(&link->link, &link_primer);
9750 if (err) {
9751 kfree(link);
Xuan Zhuo5acc7d32021-07-10 11:16:35 +08009752 goto unlock;
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009753 }
9754
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009755 err = dev_xdp_attach_link(dev, NULL, link);
9756 rtnl_unlock();
9757
9758 if (err) {
Xuan Zhuo5acc7d32021-07-10 11:16:35 +08009759 link->dev = NULL;
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009760 bpf_link_cleanup(&link_primer);
9761 goto out_put_dev;
9762 }
9763
9764 fd = bpf_link_settle(&link_primer);
9765 /* link itself doesn't hold dev's refcnt to not complicate shutdown */
9766 dev_put(dev);
9767 return fd;
9768
Xuan Zhuo5acc7d32021-07-10 11:16:35 +08009769unlock:
9770 rtnl_unlock();
9771
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009772out_put_dev:
9773 dev_put(dev);
9774 return err;
Anuradha Karuppiahd746d702015-07-14 13:43:19 -07009775}
9776
9777/**
Brenden Blancoa7862b42016-07-19 12:16:48 -07009778 * dev_change_xdp_fd - set or clear a bpf program for a device rx path
9779 * @dev: device
Jakub Kicinskib5d60982017-05-01 15:53:43 -07009780 * @extack: netlink extended ack
Brenden Blancoa7862b42016-07-19 12:16:48 -07009781 * @fd: new program fd or negative value to clear
Toke Høiland-Jørgensen92234c82020-03-25 18:23:26 +01009782 * @expected_fd: old program fd that userspace expects to replace or clear
Daniel Borkmann85de8572016-11-28 23:16:54 +01009783 * @flags: xdp-related flags
Brenden Blancoa7862b42016-07-19 12:16:48 -07009784 *
9785 * Set or clear a bpf program for a device
9786 */
Jakub Kicinskiddf9f972017-04-30 21:46:46 -07009787int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
Toke Høiland-Jørgensen92234c82020-03-25 18:23:26 +01009788 int fd, int expected_fd, u32 flags)
Brenden Blancoa7862b42016-07-19 12:16:48 -07009789{
Andrii Nakryikoc8a36f12020-08-19 22:28:41 -07009790 enum bpf_xdp_mode mode = dev_xdp_mode(dev, flags);
Andrii Nakryikod4baa932020-07-21 23:45:56 -07009791 struct bpf_prog *new_prog = NULL, *old_prog = NULL;
Brenden Blancoa7862b42016-07-19 12:16:48 -07009792 int err;
9793
Daniel Borkmann85de8572016-11-28 23:16:54 +01009794 ASSERT_RTNL();
9795
Toke Høiland-Jørgensen92234c82020-03-25 18:23:26 +01009796 if (fd >= 0) {
Andrii Nakryikod4baa932020-07-21 23:45:56 -07009797 new_prog = bpf_prog_get_type_dev(fd, BPF_PROG_TYPE_XDP,
9798 mode != XDP_MODE_SKB);
9799 if (IS_ERR(new_prog))
9800 return PTR_ERR(new_prog);
Brenden Blancoa7862b42016-07-19 12:16:48 -07009801 }
9802
Andrii Nakryikod4baa932020-07-21 23:45:56 -07009803 if (expected_fd >= 0) {
9804 old_prog = bpf_prog_get_type_dev(expected_fd, BPF_PROG_TYPE_XDP,
9805 mode != XDP_MODE_SKB);
9806 if (IS_ERR(old_prog)) {
9807 err = PTR_ERR(old_prog);
9808 old_prog = NULL;
9809 goto err_out;
9810 }
Andrii Nakryiko7f0a8382020-07-21 23:45:55 -07009811 }
Brenden Blancoa7862b42016-07-19 12:16:48 -07009812
Andrii Nakryikoaa8d3a72020-07-21 23:45:57 -07009813 err = dev_xdp_attach(dev, extack, NULL, new_prog, old_prog, flags);
Andrii Nakryikod4baa932020-07-21 23:45:56 -07009814
9815err_out:
9816 if (err && new_prog)
9817 bpf_prog_put(new_prog);
9818 if (old_prog)
9819 bpf_prog_put(old_prog);
Brenden Blancoa7862b42016-07-19 12:16:48 -07009820 return err;
9821}
Brenden Blancoa7862b42016-07-19 12:16:48 -07009822
9823/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07009824 * dev_new_index - allocate an ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07009825 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07009826 *
9827 * Returns a suitable unique value for a new device interface
9828 * number. The caller must hold the rtnl semaphore or the
9829 * dev_base_lock to be sure it remains unique.
9830 */
Eric W. Biederman881d9662007-09-17 11:56:21 -07009831static int dev_new_index(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07009832{
Pavel Emelyanovaa79e662012-08-08 21:53:19 +00009833 int ifindex = net->ifindex;
tchardingf4563a72017-02-09 17:56:07 +11009834
Linus Torvalds1da177e2005-04-16 15:20:36 -07009835 for (;;) {
9836 if (++ifindex <= 0)
9837 ifindex = 1;
Eric W. Biederman881d9662007-09-17 11:56:21 -07009838 if (!__dev_get_by_index(net, ifindex))
Pavel Emelyanovaa79e662012-08-08 21:53:19 +00009839 return net->ifindex = ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07009840 }
9841}
9842
Linus Torvalds1da177e2005-04-16 15:20:36 -07009843/* Delayed registration/unregisteration */
Denis Cheng3b5b34f2007-12-07 00:49:17 -08009844static LIST_HEAD(net_todo_list);
Cong Wang200b9162014-05-12 15:11:20 -07009845DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07009846
Stephen Hemminger6f05f622007-03-08 20:46:03 -08009847static void net_set_todo(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07009848{
Linus Torvalds1da177e2005-04-16 15:20:36 -07009849 list_add_tail(&dev->todo_list, &net_todo_list);
Eric W. Biederman50624c92013-09-23 21:19:49 -07009850 dev_net(dev)->dev_unreg_count++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07009851}
9852
Jarod Wilsonfd867d52015-11-02 21:55:59 -05009853static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
9854 struct net_device *upper, netdev_features_t features)
9855{
9856 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
9857 netdev_features_t feature;
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05009858 int feature_bit;
Jarod Wilsonfd867d52015-11-02 21:55:59 -05009859
Hauke Mehrtens3b89ea92019-02-15 17:58:54 +01009860 for_each_netdev_feature(upper_disables, feature_bit) {
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05009861 feature = __NETIF_F_BIT(feature_bit);
Jarod Wilsonfd867d52015-11-02 21:55:59 -05009862 if (!(upper->wanted_features & feature)
9863 && (features & feature)) {
9864 netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
9865 &feature, upper->name);
9866 features &= ~feature;
9867 }
9868 }
9869
9870 return features;
9871}
9872
9873static void netdev_sync_lower_features(struct net_device *upper,
9874 struct net_device *lower, netdev_features_t features)
9875{
9876 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
9877 netdev_features_t feature;
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05009878 int feature_bit;
Jarod Wilsonfd867d52015-11-02 21:55:59 -05009879
Hauke Mehrtens3b89ea92019-02-15 17:58:54 +01009880 for_each_netdev_feature(upper_disables, feature_bit) {
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05009881 feature = __NETIF_F_BIT(feature_bit);
Jarod Wilsonfd867d52015-11-02 21:55:59 -05009882 if (!(features & feature) && (lower->features & feature)) {
9883 netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
9884 &feature, lower->name);
9885 lower->wanted_features &= ~feature;
Cong Wangdd912302020-05-07 12:19:03 -07009886 __netdev_update_features(lower);
Jarod Wilsonfd867d52015-11-02 21:55:59 -05009887
9888 if (unlikely(lower->features & feature))
9889 netdev_WARN(upper, "failed to disable %pNF on %s!\n",
9890 &feature, lower->name);
Cong Wangdd912302020-05-07 12:19:03 -07009891 else
9892 netdev_features_change(lower);
Jarod Wilsonfd867d52015-11-02 21:55:59 -05009893 }
9894 }
9895}
9896
Michał Mirosławc8f44af2011-11-15 15:29:55 +00009897static netdev_features_t netdev_fix_features(struct net_device *dev,
9898 netdev_features_t features)
Herbert Xub63365a2008-10-23 01:11:29 -07009899{
Michał Mirosław57422dc2011-01-22 12:14:12 +00009900 /* Fix illegal checksum combinations */
9901 if ((features & NETIF_F_HW_CSUM) &&
9902 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04009903 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
Michał Mirosław57422dc2011-01-22 12:14:12 +00009904 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
9905 }
9906
Herbert Xub63365a2008-10-23 01:11:29 -07009907 /* TSO requires that SG is present as well. */
Ben Hutchingsea2d3682011-04-12 14:38:37 +00009908 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04009909 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
Ben Hutchingsea2d3682011-04-12 14:38:37 +00009910 features &= ~NETIF_F_ALL_TSO;
Herbert Xub63365a2008-10-23 01:11:29 -07009911 }
9912
Pravin B Shelarec5f0612013-03-07 09:28:01 +00009913 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
9914 !(features & NETIF_F_IP_CSUM)) {
9915 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
9916 features &= ~NETIF_F_TSO;
9917 features &= ~NETIF_F_TSO_ECN;
9918 }
9919
9920 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
9921 !(features & NETIF_F_IPV6_CSUM)) {
9922 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
9923 features &= ~NETIF_F_TSO6;
9924 }
9925
Alexander Duyckb1dc4972016-05-02 09:38:24 -07009926 /* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */
9927 if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO))
9928 features &= ~NETIF_F_TSO_MANGLEID;
9929
Ben Hutchings31d8b9e2011-04-12 14:47:15 +00009930 /* TSO ECN requires that TSO is present as well. */
9931 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
9932 features &= ~NETIF_F_TSO_ECN;
9933
Michał Mirosław212b5732011-02-15 16:59:16 +00009934 /* Software GSO depends on SG. */
9935 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04009936 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
Michał Mirosław212b5732011-02-15 16:59:16 +00009937 features &= ~NETIF_F_GSO;
9938 }
9939
Alexander Duyck802ab552016-04-10 21:45:03 -04009940 /* GSO partial features require GSO partial be set */
9941 if ((features & dev->gso_partial_features) &&
9942 !(features & NETIF_F_GSO_PARTIAL)) {
9943 netdev_dbg(dev,
9944 "Dropping partially supported GSO features since no GSO partial.\n");
9945 features &= ~dev->gso_partial_features;
9946 }
9947
Michael Chanfb1f5f72017-12-16 03:09:40 -05009948 if (!(features & NETIF_F_RXCSUM)) {
9949 /* NETIF_F_GRO_HW implies doing RXCSUM since every packet
9950 * successfully merged by hardware must also have the
9951 * checksum verified by hardware. If the user does not
9952 * want to enable RXCSUM, logically, we should disable GRO_HW.
9953 */
9954 if (features & NETIF_F_GRO_HW) {
9955 netdev_dbg(dev, "Dropping NETIF_F_GRO_HW since no RXCSUM feature.\n");
9956 features &= ~NETIF_F_GRO_HW;
9957 }
9958 }
9959
Gal Pressmande8d5ab2018-03-12 11:48:49 +02009960 /* LRO/HW-GRO features cannot be combined with RX-FCS */
9961 if (features & NETIF_F_RXFCS) {
9962 if (features & NETIF_F_LRO) {
9963 netdev_dbg(dev, "Dropping LRO feature since RX-FCS is requested.\n");
9964 features &= ~NETIF_F_LRO;
9965 }
9966
9967 if (features & NETIF_F_GRO_HW) {
9968 netdev_dbg(dev, "Dropping HW-GRO feature since RX-FCS is requested.\n");
9969 features &= ~NETIF_F_GRO_HW;
9970 }
Gal Pressmane6c6a922018-03-04 14:12:04 +02009971 }
9972
Tariq Toukan25537d72021-01-14 17:12:15 +02009973 if (features & NETIF_F_HW_TLS_TX) {
9974 bool ip_csum = (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) ==
9975 (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
9976 bool hw_csum = features & NETIF_F_HW_CSUM;
9977
9978 if (!ip_csum && !hw_csum) {
9979 netdev_dbg(dev, "Dropping TLS TX HW offload feature since no CSUM feature.\n");
9980 features &= ~NETIF_F_HW_TLS_TX;
9981 }
Tariq Toukanae0b04b2020-12-13 16:39:29 +02009982 }
9983
Tariq Toukana3eb4e92021-01-17 17:15:38 +02009984 if ((features & NETIF_F_HW_TLS_RX) && !(features & NETIF_F_RXCSUM)) {
9985 netdev_dbg(dev, "Dropping TLS RX HW offload feature since no RXCSUM feature.\n");
9986 features &= ~NETIF_F_HW_TLS_RX;
9987 }
9988
Herbert Xub63365a2008-10-23 01:11:29 -07009989 return features;
9990}
Herbert Xub63365a2008-10-23 01:11:29 -07009991
Michał Mirosław6cb6a272011-04-02 22:48:47 -07009992int __netdev_update_features(struct net_device *dev)
Michał Mirosław5455c692011-02-15 16:59:17 +00009993{
Jarod Wilsonfd867d52015-11-02 21:55:59 -05009994 struct net_device *upper, *lower;
Michał Mirosławc8f44af2011-11-15 15:29:55 +00009995 netdev_features_t features;
Jarod Wilsonfd867d52015-11-02 21:55:59 -05009996 struct list_head *iter;
Jarod Wilsone7868a82015-11-03 23:09:32 -05009997 int err = -1;
Michał Mirosław5455c692011-02-15 16:59:17 +00009998
Michał Mirosław87267482011-04-12 09:56:38 +00009999 ASSERT_RTNL();
10000
Michał Mirosław5455c692011-02-15 16:59:17 +000010001 features = netdev_get_wanted_features(dev);
10002
10003 if (dev->netdev_ops->ndo_fix_features)
10004 features = dev->netdev_ops->ndo_fix_features(dev, features);
10005
10006 /* driver might be less strict about feature dependencies */
10007 features = netdev_fix_features(dev, features);
10008
Randy Dunlap4250b752020-09-17 21:35:15 -070010009 /* some features can't be enabled if they're off on an upper device */
Jarod Wilsonfd867d52015-11-02 21:55:59 -050010010 netdev_for_each_upper_dev_rcu(dev, upper, iter)
10011 features = netdev_sync_upper_features(dev, upper, features);
10012
Michał Mirosław5455c692011-02-15 16:59:17 +000010013 if (dev->features == features)
Jarod Wilsone7868a82015-11-03 23:09:32 -050010014 goto sync_lower;
Michał Mirosław5455c692011-02-15 16:59:17 +000010015
Michał Mirosławc8f44af2011-11-15 15:29:55 +000010016 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
10017 &dev->features, &features);
Michał Mirosław5455c692011-02-15 16:59:17 +000010018
10019 if (dev->netdev_ops->ndo_set_features)
10020 err = dev->netdev_ops->ndo_set_features(dev, features);
Nikolay Aleksandrov5f8dc332015-11-13 14:54:01 +010010021 else
10022 err = 0;
Michał Mirosław5455c692011-02-15 16:59:17 +000010023
Michał Mirosław6cb6a272011-04-02 22:48:47 -070010024 if (unlikely(err < 0)) {
Michał Mirosław5455c692011-02-15 16:59:17 +000010025 netdev_err(dev,
Michał Mirosławc8f44af2011-11-15 15:29:55 +000010026 "set_features() failed (%d); wanted %pNF, left %pNF\n",
10027 err, &features, &dev->features);
Nikolay Aleksandrov17b85d22015-11-17 15:49:06 +010010028 /* return non-0 since some features might have changed and
10029 * it's better to fire a spurious notification than miss it
10030 */
10031 return -1;
Michał Mirosław6cb6a272011-04-02 22:48:47 -070010032 }
10033
Jarod Wilsone7868a82015-11-03 23:09:32 -050010034sync_lower:
Jarod Wilsonfd867d52015-11-02 21:55:59 -050010035 /* some features must be disabled on lower devices when disabled
10036 * on an upper device (think: bonding master or bridge)
10037 */
10038 netdev_for_each_lower_dev(dev, lower, iter)
10039 netdev_sync_lower_features(dev, lower, features);
10040
Sabrina Dubrocaae847f42017-07-21 12:49:31 +020010041 if (!err) {
10042 netdev_features_t diff = features ^ dev->features;
10043
10044 if (diff & NETIF_F_RX_UDP_TUNNEL_PORT) {
10045 /* udp_tunnel_{get,drop}_rx_info both need
10046 * NETIF_F_RX_UDP_TUNNEL_PORT enabled on the
10047 * device, or they won't do anything.
10048 * Thus we need to update dev->features
10049 * *before* calling udp_tunnel_get_rx_info,
10050 * but *after* calling udp_tunnel_drop_rx_info.
10051 */
10052 if (features & NETIF_F_RX_UDP_TUNNEL_PORT) {
10053 dev->features = features;
10054 udp_tunnel_get_rx_info(dev);
10055 } else {
10056 udp_tunnel_drop_rx_info(dev);
10057 }
10058 }
10059
Gal Pressman9daae9b2018-03-28 17:46:54 +030010060 if (diff & NETIF_F_HW_VLAN_CTAG_FILTER) {
10061 if (features & NETIF_F_HW_VLAN_CTAG_FILTER) {
10062 dev->features = features;
10063 err |= vlan_get_rx_ctag_filter_info(dev);
10064 } else {
10065 vlan_drop_rx_ctag_filter_info(dev);
10066 }
10067 }
10068
10069 if (diff & NETIF_F_HW_VLAN_STAG_FILTER) {
10070 if (features & NETIF_F_HW_VLAN_STAG_FILTER) {
10071 dev->features = features;
10072 err |= vlan_get_rx_stag_filter_info(dev);
10073 } else {
10074 vlan_drop_rx_stag_filter_info(dev);
10075 }
10076 }
10077
Michał Mirosław6cb6a272011-04-02 22:48:47 -070010078 dev->features = features;
Sabrina Dubrocaae847f42017-07-21 12:49:31 +020010079 }
Michał Mirosław6cb6a272011-04-02 22:48:47 -070010080
Jarod Wilsone7868a82015-11-03 23:09:32 -050010081 return err < 0 ? 0 : 1;
Michał Mirosław6cb6a272011-04-02 22:48:47 -070010082}
10083
Michał Mirosławafe12cc2011-05-07 03:22:17 +000010084/**
10085 * netdev_update_features - recalculate device features
10086 * @dev: the device to check
10087 *
10088 * Recalculate dev->features set and send notifications if it
10089 * has changed. Should be called after driver or hardware dependent
10090 * conditions might have changed that influence the features.
10091 */
Michał Mirosław6cb6a272011-04-02 22:48:47 -070010092void netdev_update_features(struct net_device *dev)
10093{
10094 if (__netdev_update_features(dev))
10095 netdev_features_change(dev);
Michał Mirosław5455c692011-02-15 16:59:17 +000010096}
10097EXPORT_SYMBOL(netdev_update_features);
10098
Linus Torvalds1da177e2005-04-16 15:20:36 -070010099/**
Michał Mirosławafe12cc2011-05-07 03:22:17 +000010100 * netdev_change_features - recalculate device features
10101 * @dev: the device to check
10102 *
10103 * Recalculate dev->features set and send notifications even
10104 * if they have not changed. Should be called instead of
10105 * netdev_update_features() if also dev->vlan_features might
10106 * have changed to allow the changes to be propagated to stacked
10107 * VLAN devices.
10108 */
10109void netdev_change_features(struct net_device *dev)
10110{
10111 __netdev_update_features(dev);
10112 netdev_features_change(dev);
10113}
10114EXPORT_SYMBOL(netdev_change_features);
10115
10116/**
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -080010117 * netif_stacked_transfer_operstate - transfer operstate
10118 * @rootdev: the root or lower level device to transfer state from
10119 * @dev: the device to transfer operstate to
10120 *
10121 * Transfer operational state from root to device. This is normally
10122 * called when a stacking relationship exists between the root
10123 * device and the device(a leaf device).
10124 */
10125void netif_stacked_transfer_operstate(const struct net_device *rootdev,
10126 struct net_device *dev)
10127{
10128 if (rootdev->operstate == IF_OPER_DORMANT)
10129 netif_dormant_on(dev);
10130 else
10131 netif_dormant_off(dev);
10132
Andrew Lunneec517cd2020-04-20 00:11:50 +020010133 if (rootdev->operstate == IF_OPER_TESTING)
10134 netif_testing_on(dev);
10135 else
10136 netif_testing_off(dev);
10137
Zhang Shengju0575c862017-04-26 17:49:38 +080010138 if (netif_carrier_ok(rootdev))
10139 netif_carrier_on(dev);
10140 else
10141 netif_carrier_off(dev);
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -080010142}
10143EXPORT_SYMBOL(netif_stacked_transfer_operstate);
10144
Eric Dumazet1b4bf462010-09-23 17:26:35 +000010145static int netif_alloc_rx_queues(struct net_device *dev)
10146{
Eric Dumazet1b4bf462010-09-23 17:26:35 +000010147 unsigned int i, count = dev->num_rx_queues;
Tom Herbertbd25fa72010-10-18 18:00:16 +000010148 struct netdev_rx_queue *rx;
Pankaj Gupta10595902015-01-12 11:41:28 +053010149 size_t sz = count * sizeof(*rx);
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +010010150 int err = 0;
Eric Dumazet1b4bf462010-09-23 17:26:35 +000010151
Tom Herbertbd25fa72010-10-18 18:00:16 +000010152 BUG_ON(count < 1);
Eric Dumazet1b4bf462010-09-23 17:26:35 +000010153
Vasily Averinc948f512021-07-19 13:44:23 +030010154 rx = kvzalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
Michal Hockoda6bc572017-05-08 15:57:31 -070010155 if (!rx)
10156 return -ENOMEM;
10157
Tom Herbertbd25fa72010-10-18 18:00:16 +000010158 dev->_rx = rx;
10159
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +010010160 for (i = 0; i < count; i++) {
Tom Herbertfe822242010-11-09 10:47:38 +000010161 rx[i].dev = dev;
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +010010162
10163 /* XDP RX-queue setup */
Björn Töpelb02e5a02020-11-30 19:52:01 +010010164 err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i, 0);
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +010010165 if (err < 0)
10166 goto err_rxq_info;
10167 }
Eric Dumazet1b4bf462010-09-23 17:26:35 +000010168 return 0;
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +010010169
10170err_rxq_info:
10171 /* Rollback successful reg's and free other resources */
10172 while (i--)
10173 xdp_rxq_info_unreg(&rx[i].xdp_rxq);
Jakub Kicinski141b52a2018-01-10 01:20:01 -080010174 kvfree(dev->_rx);
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +010010175 dev->_rx = NULL;
10176 return err;
Eric Dumazet1b4bf462010-09-23 17:26:35 +000010177}
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +010010178
10179static void netif_free_rx_queues(struct net_device *dev)
10180{
10181 unsigned int i, count = dev->num_rx_queues;
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +010010182
10183 /* netif_alloc_rx_queues alloc failed, resources have been unreg'ed */
10184 if (!dev->_rx)
10185 return;
10186
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +010010187 for (i = 0; i < count; i++)
Jakub Kicinski82aaff22018-01-10 01:20:02 -080010188 xdp_rxq_info_unreg(&dev->_rx[i].xdp_rxq);
10189
10190 kvfree(dev->_rx);
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +010010191}
Eric Dumazet1b4bf462010-09-23 17:26:35 +000010192
Changli Gaoaa942102010-12-04 02:31:41 +000010193static void netdev_init_one_queue(struct net_device *dev,
10194 struct netdev_queue *queue, void *_unused)
10195{
10196 /* Initialize queue lock */
10197 spin_lock_init(&queue->_xmit_lock);
Cong Wang1a33e102020-05-02 22:22:19 -070010198 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
Changli Gaoaa942102010-12-04 02:31:41 +000010199 queue->xmit_lock_owner = -1;
Changli Gaob236da62010-12-14 03:09:15 +000010200 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
Changli Gaoaa942102010-12-04 02:31:41 +000010201 queue->dev = dev;
Tom Herbert114cf582011-11-28 16:33:09 +000010202#ifdef CONFIG_BQL
10203 dql_init(&queue->dql, HZ);
10204#endif
Changli Gaoaa942102010-12-04 02:31:41 +000010205}
10206
Eric Dumazet60877a32013-06-20 01:15:51 -070010207static void netif_free_tx_queues(struct net_device *dev)
10208{
WANG Cong4cb28972014-06-02 15:55:22 -070010209 kvfree(dev->_tx);
Eric Dumazet60877a32013-06-20 01:15:51 -070010210}
10211
Tom Herberte6484932010-10-18 18:04:39 +000010212static int netif_alloc_netdev_queues(struct net_device *dev)
10213{
10214 unsigned int count = dev->num_tx_queues;
10215 struct netdev_queue *tx;
Eric Dumazet60877a32013-06-20 01:15:51 -070010216 size_t sz = count * sizeof(*tx);
Tom Herberte6484932010-10-18 18:04:39 +000010217
Eric Dumazetd3397272015-07-06 17:13:26 +020010218 if (count < 1 || count > 0xffff)
10219 return -EINVAL;
Tom Herberte6484932010-10-18 18:04:39 +000010220
Vasily Averinc948f512021-07-19 13:44:23 +030010221 tx = kvzalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
Michal Hockoda6bc572017-05-08 15:57:31 -070010222 if (!tx)
10223 return -ENOMEM;
10224
Tom Herberte6484932010-10-18 18:04:39 +000010225 dev->_tx = tx;
Tom Herbert1d24eb42010-11-21 13:17:27 +000010226
Tom Herberte6484932010-10-18 18:04:39 +000010227 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
10228 spin_lock_init(&dev->tx_global_lock);
Changli Gaoaa942102010-12-04 02:31:41 +000010229
10230 return 0;
Tom Herberte6484932010-10-18 18:04:39 +000010231}
10232
Denys Vlasenkoa2029242015-05-11 21:17:53 +020010233void netif_tx_stop_all_queues(struct net_device *dev)
10234{
10235 unsigned int i;
10236
10237 for (i = 0; i < dev->num_tx_queues; i++) {
10238 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
tchardingf4563a72017-02-09 17:56:07 +110010239
Denys Vlasenkoa2029242015-05-11 21:17:53 +020010240 netif_tx_stop_queue(txq);
10241 }
10242}
10243EXPORT_SYMBOL(netif_tx_stop_all_queues);
10244
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -080010245/**
Linus Torvalds1da177e2005-04-16 15:20:36 -070010246 * register_netdevice - register a network device
10247 * @dev: device to register
10248 *
10249 * Take a completed network device structure and add it to the kernel
10250 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
10251 * chain. 0 is returned on success. A negative errno code is returned
10252 * on a failure to set up the device, or if the name is a duplicate.
10253 *
10254 * Callers must hold the rtnl semaphore. You may want
10255 * register_netdev() instead of this.
10256 *
10257 * BUGS:
10258 * The locking appears insufficient to guarantee two parallel registers
10259 * will not get the same name.
10260 */
10261
10262int register_netdevice(struct net_device *dev)
10263{
Linus Torvalds1da177e2005-04-16 15:20:36 -070010264 int ret;
Stephen Hemmingerd3147742008-11-19 21:32:24 -080010265 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -070010266
Florian Fainellie283de32018-04-30 14:20:05 -070010267 BUILD_BUG_ON(sizeof(netdev_features_t) * BITS_PER_BYTE <
10268 NETDEV_FEATURE_COUNT);
Linus Torvalds1da177e2005-04-16 15:20:36 -070010269 BUG_ON(dev_boot_phase);
10270 ASSERT_RTNL();
10271
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -070010272 might_sleep();
10273
Linus Torvalds1da177e2005-04-16 15:20:36 -070010274 /* When net_device's are persistent, this will be fatal. */
10275 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
Stephen Hemmingerd3147742008-11-19 21:32:24 -080010276 BUG_ON(!net);
Linus Torvalds1da177e2005-04-16 15:20:36 -070010277
Jakub Kicinski9000edb2020-03-16 13:47:12 -070010278 ret = ethtool_check_ops(dev->ethtool_ops);
10279 if (ret)
10280 return ret;
10281
David S. Millerf1f28aa2008-07-15 00:08:33 -070010282 spin_lock_init(&dev->addr_list_lock);
Cong Wang845e0eb2020-06-08 14:53:01 -070010283 netdev_set_addr_lockdep_class(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -070010284
Gao feng828de4f2012-09-13 20:58:27 +000010285 ret = dev_get_valid_name(net, dev, dev->name);
Peter Pan(潘卫平)0696c3a2011-05-12 15:46:56 +000010286 if (ret < 0)
10287 goto out;
10288
Eric Dumazet9077f052019-10-03 08:59:24 -070010289 ret = -ENOMEM;
Jiri Pirkoff927412019-09-30 11:48:15 +020010290 dev->name_node = netdev_name_node_head_alloc(dev);
10291 if (!dev->name_node)
10292 goto out;
10293
Linus Torvalds1da177e2005-04-16 15:20:36 -070010294 /* Init, if this function is available */
Stephen Hemmingerd3147742008-11-19 21:32:24 -080010295 if (dev->netdev_ops->ndo_init) {
10296 ret = dev->netdev_ops->ndo_init(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -070010297 if (ret) {
10298 if (ret > 0)
10299 ret = -EIO;
Dan Carpenter42c17fa2019-12-03 17:12:39 +030010300 goto err_free_name;
Linus Torvalds1da177e2005-04-16 15:20:36 -070010301 }
10302 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +090010303
Patrick McHardyf6469682013-04-19 02:04:27 +000010304 if (((dev->hw_features | dev->features) &
10305 NETIF_F_HW_VLAN_CTAG_FILTER) &&
Michał Mirosławd2ed2732013-01-29 15:14:16 +000010306 (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
10307 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
10308 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
10309 ret = -EINVAL;
10310 goto err_uninit;
10311 }
10312
Pavel Emelyanov9c7dafb2012-08-08 21:52:46 +000010313 ret = -EBUSY;
10314 if (!dev->ifindex)
10315 dev->ifindex = dev_new_index(net);
10316 else if (__dev_get_by_index(net, dev->ifindex))
10317 goto err_uninit;
10318
Michał Mirosław5455c692011-02-15 16:59:17 +000010319 /* Transfer changeable features to wanted_features and enable
10320 * software offloads (GSO and GRO).
10321 */
Steffen Klassert1a3c9982020-01-25 11:26:43 +010010322 dev->hw_features |= (NETIF_F_SOFT_FEATURES | NETIF_F_SOFT_FEATURES_OFF);
Michał Mirosław14d12322011-02-22 16:52:28 +000010323 dev->features |= NETIF_F_SOFT_FEATURES;
Sabrina Dubrocad764a122017-07-21 12:49:28 +020010324
Jakub Kicinski876c4382021-01-06 13:06:34 -080010325 if (dev->udp_tunnel_nic_info) {
Sabrina Dubrocad764a122017-07-21 12:49:28 +020010326 dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT;
10327 dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT;
10328 }
10329
Michał Mirosław14d12322011-02-22 16:52:28 +000010330 dev->wanted_features = dev->features & dev->hw_features;
Linus Torvalds1da177e2005-04-16 15:20:36 -070010331
Alexander Duyckcbc53e02016-04-10 21:44:51 -040010332 if (!(dev->flags & IFF_LOOPBACK))
Michał Mirosław34324dc2011-11-15 15:29:55 +000010333 dev->hw_features |= NETIF_F_NOCACHE_COPY;
Alexander Duyckcbc53e02016-04-10 21:44:51 -040010334
Alexander Duyck7f348a62016-04-20 16:51:00 -040010335 /* If IPv4 TCP segmentation offload is supported we should also
10336 * allow the device to enable segmenting the frame with the option
10337 * of ignoring a static IP ID value. This doesn't enable the
10338 * feature itself but allows the user to enable it later.
10339 */
Alexander Duyckcbc53e02016-04-10 21:44:51 -040010340 if (dev->hw_features & NETIF_F_TSO)
10341 dev->hw_features |= NETIF_F_TSO_MANGLEID;
Alexander Duyck7f348a62016-04-20 16:51:00 -040010342 if (dev->vlan_features & NETIF_F_TSO)
10343 dev->vlan_features |= NETIF_F_TSO_MANGLEID;
10344 if (dev->mpls_features & NETIF_F_TSO)
10345 dev->mpls_features |= NETIF_F_TSO_MANGLEID;
10346 if (dev->hw_enc_features & NETIF_F_TSO)
10347 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
Tom Herbertc6e1a0d2011-04-04 22:30:30 -070010348
Michał Mirosław1180e7d2011-07-14 14:41:11 -070010349 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
Brandon Philips16c3ea72010-09-15 09:24:24 +000010350 */
Michał Mirosław1180e7d2011-07-14 14:41:11 -070010351 dev->vlan_features |= NETIF_F_HIGHDMA;
Brandon Philips16c3ea72010-09-15 09:24:24 +000010352
Pravin B Shelaree579672013-03-07 09:28:08 +000010353 /* Make NETIF_F_SG inheritable to tunnel devices.
10354 */
Alexander Duyck802ab552016-04-10 21:45:03 -040010355 dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL;
Pravin B Shelaree579672013-03-07 09:28:08 +000010356
Simon Horman0d89d202013-05-23 21:02:52 +000010357 /* Make NETIF_F_SG inheritable to MPLS.
10358 */
10359 dev->mpls_features |= NETIF_F_SG;
10360
Johannes Berg7ffbe3f2009-10-02 05:15:27 +000010361 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
10362 ret = notifier_to_errno(ret);
10363 if (ret)
10364 goto err_uninit;
10365
Eric W. Biederman8b41d182007-09-26 22:02:53 -070010366 ret = netdev_register_kobject(dev);
Jouni Hogandercb626bf2020-01-20 09:51:03 +020010367 if (ret) {
10368 dev->reg_state = NETREG_UNREGISTERED;
Herbert Xu7ce1b0e2007-07-30 16:29:40 -070010369 goto err_uninit;
Jouni Hogandercb626bf2020-01-20 09:51:03 +020010370 }
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -070010371 dev->reg_state = NETREG_REGISTERED;
10372
Michał Mirosław6cb6a272011-04-02 22:48:47 -070010373 __netdev_update_features(dev);
Michał Mirosław8e9b59b2011-02-22 16:52:28 +000010374
Linus Torvalds1da177e2005-04-16 15:20:36 -070010375 /*
10376 * Default initial state at registry is that the
10377 * device is present.
10378 */
10379
10380 set_bit(__LINK_STATE_PRESENT, &dev->state);
10381
Ben Hutchings8f4cccb2012-08-20 22:16:51 +010010382 linkwatch_init_dev(dev);
10383
Linus Torvalds1da177e2005-04-16 15:20:36 -070010384 dev_init_scheduler(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -070010385 dev_hold(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +020010386 list_netdevice(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -040010387 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -070010388
Jiri Pirko948b3372013-01-08 01:38:25 +000010389 /* If the device has permanent device address, driver should
10390 * set dev_addr and also addr_assign_type should be set to
10391 * NET_ADDR_PERM (default value).
10392 */
10393 if (dev->addr_assign_type == NET_ADDR_PERM)
10394 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
10395
Linus Torvalds1da177e2005-04-16 15:20:36 -070010396 /* Notify protocols, that a new device appeared. */
Pavel Emelyanov056925a2007-09-16 15:42:43 -070010397 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -070010398 ret = notifier_to_errno(ret);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -070010399 if (ret) {
Jakub Kicinski766b0512021-01-06 10:40:07 -080010400 /* Expect explicit free_netdev() on failure */
10401 dev->needs_free_netdev = false;
Jakub Kicinski037e56b2021-01-19 12:25:19 -080010402 unregister_netdevice_queue(dev, NULL);
Jakub Kicinski766b0512021-01-06 10:40:07 -080010403 goto out;
Daniel Lezcano93ee31f2007-10-30 15:38:18 -070010404 }
Eric W. Biedermand90a9092009-12-12 22:11:15 +000010405 /*
10406 * Prevent userspace races by waiting until the network
10407 * device is fully setup before sending notifications.
10408 */
Patrick McHardya2835762010-02-26 06:34:51 +000010409 if (!dev->rtnl_link_ops ||
10410 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
Alexei Starovoitov7f294052013-10-23 16:02:42 -070010411 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -070010412
10413out:
10414 return ret;
Herbert Xu7ce1b0e2007-07-30 16:29:40 -070010415
10416err_uninit:
Stephen Hemmingerd3147742008-11-19 21:32:24 -080010417 if (dev->netdev_ops->ndo_uninit)
10418 dev->netdev_ops->ndo_uninit(dev);
David S. Millercf124db2017-05-08 12:52:56 -040010419 if (dev->priv_destructor)
10420 dev->priv_destructor(dev);
Dan Carpenter42c17fa2019-12-03 17:12:39 +030010421err_free_name:
10422 netdev_name_node_free(dev->name_node);
Herbert Xu7ce1b0e2007-07-30 16:29:40 -070010423 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -070010424}
Eric Dumazetd1b19df2009-09-03 01:29:39 -070010425EXPORT_SYMBOL(register_netdevice);
Linus Torvalds1da177e2005-04-16 15:20:36 -070010426
10427/**
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -080010428 * init_dummy_netdev - init a dummy network device for NAPI
10429 * @dev: device to init
10430 *
10431 * This takes a network device structure and initialize the minimum
10432 * amount of fields so it can be used to schedule NAPI polls without
10433 * registering a full blown interface. This is to be used by drivers
10434 * that need to tie several hardware interfaces to a single NAPI
10435 * poll scheduler due to HW limitations.
10436 */
10437int init_dummy_netdev(struct net_device *dev)
10438{
10439 /* Clear everything. Note we don't initialize spinlocks
10440 * are they aren't supposed to be taken by any of the
10441 * NAPI code and this dummy netdev is supposed to be
10442 * only ever used for NAPI polls
10443 */
10444 memset(dev, 0, sizeof(struct net_device));
10445
10446 /* make sure we BUG if trying to hit standard
10447 * register/unregister code path
10448 */
10449 dev->reg_state = NETREG_DUMMY;
10450
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -080010451 /* NAPI wants this */
10452 INIT_LIST_HEAD(&dev->napi_list);
10453
10454 /* a dummy interface is started by default */
10455 set_bit(__LINK_STATE_PRESENT, &dev->state);
10456 set_bit(__LINK_STATE_START, &dev->state);
10457
Josh Elsasser35edfdc2019-01-26 14:38:33 -080010458 /* napi_busy_loop stats accounting wants this */
10459 dev_net_set(dev, &init_net);
10460
Eric Dumazet29b44332010-10-11 10:22:12 +000010461 /* Note : We dont allocate pcpu_refcnt for dummy devices,
10462 * because users of this 'device' dont need to change
10463 * its refcount.
10464 */
10465
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -080010466 return 0;
10467}
10468EXPORT_SYMBOL_GPL(init_dummy_netdev);
10469
10470
10471/**
Linus Torvalds1da177e2005-04-16 15:20:36 -070010472 * register_netdev - register a network device
10473 * @dev: device to register
10474 *
10475 * Take a completed network device structure and add it to the kernel
10476 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
10477 * chain. 0 is returned on success. A negative errno code is returned
10478 * on a failure to set up the device, or if the name is a duplicate.
10479 *
Borislav Petkov38b4da382007-04-20 22:14:10 -070010480 * This is a wrapper around register_netdevice that takes the rtnl semaphore
Linus Torvalds1da177e2005-04-16 15:20:36 -070010481 * and expands the device name if you passed a format string to
10482 * alloc_netdev.
10483 */
10484int register_netdev(struct net_device *dev)
10485{
10486 int err;
10487
Kirill Tkhaib0f3deb2018-03-14 22:17:28 +030010488 if (rtnl_lock_killable())
10489 return -EINTR;
Linus Torvalds1da177e2005-04-16 15:20:36 -070010490 err = register_netdevice(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -070010491 rtnl_unlock();
10492 return err;
10493}
10494EXPORT_SYMBOL(register_netdev);
10495
Eric Dumazet29b44332010-10-11 10:22:12 +000010496int netdev_refcnt_read(const struct net_device *dev)
10497{
Eric Dumazet919067c2021-03-19 10:39:33 -070010498#ifdef CONFIG_PCPU_DEV_REFCNT
Eric Dumazet29b44332010-10-11 10:22:12 +000010499 int i, refcnt = 0;
10500
10501 for_each_possible_cpu(i)
10502 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
10503 return refcnt;
Eric Dumazet919067c2021-03-19 10:39:33 -070010504#else
10505 return refcount_read(&dev->dev_refcnt);
10506#endif
Eric Dumazet29b44332010-10-11 10:22:12 +000010507}
10508EXPORT_SYMBOL(netdev_refcnt_read);
10509
Dmitry Vyukov5aa3afe2021-03-23 07:49:23 +010010510int netdev_unregister_timeout_secs __read_mostly = 10;
10511
Mauro Carvalho Chehabde2b5412020-09-22 13:22:52 +020010512#define WAIT_REFS_MIN_MSECS 1
10513#define WAIT_REFS_MAX_MSECS 250
Ben Hutchings2c530402012-07-10 10:55:09 +000010514/**
Linus Torvalds1da177e2005-04-16 15:20:36 -070010515 * netdev_wait_allrefs - wait until all references are gone.
Randy Dunlap3de7a372012-08-18 14:36:44 +000010516 * @dev: target net_device
Linus Torvalds1da177e2005-04-16 15:20:36 -070010517 *
10518 * This is called when unregistering network devices.
10519 *
10520 * Any protocol or device that holds a reference should register
10521 * for netdevice notification, and cleanup and put back the
10522 * reference if they receive an UNREGISTER event.
10523 * We can get stuck here if buggy protocols don't correctly
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +090010524 * call dev_put.
Linus Torvalds1da177e2005-04-16 15:20:36 -070010525 */
10526static void netdev_wait_allrefs(struct net_device *dev)
10527{
10528 unsigned long rebroadcast_time, warning_time;
Francesco Ruggeri0e4be9e2020-09-18 13:19:01 -070010529 int wait = 0, refcnt;
Linus Torvalds1da177e2005-04-16 15:20:36 -070010530
Eric Dumazete014deb2009-11-17 05:59:21 +000010531 linkwatch_forget_dev(dev);
10532
Linus Torvalds1da177e2005-04-16 15:20:36 -070010533 rebroadcast_time = warning_time = jiffies;
Eric Dumazet29b44332010-10-11 10:22:12 +000010534 refcnt = netdev_refcnt_read(dev);
10535
Eric Dumazetadd2d732021-03-22 11:21:45 -070010536 while (refcnt != 1) {
Linus Torvalds1da177e2005-04-16 15:20:36 -070010537 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -080010538 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -070010539
10540 /* Rebroadcast unregister notification */
Pavel Emelyanov056925a2007-09-16 15:42:43 -070010541 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -070010542
Eric Dumazet748e2d92012-08-22 21:50:59 +000010543 __rtnl_unlock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +000010544 rcu_barrier();
Eric Dumazet748e2d92012-08-22 21:50:59 +000010545 rtnl_lock();
10546
Linus Torvalds1da177e2005-04-16 15:20:36 -070010547 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
10548 &dev->state)) {
10549 /* We must not have linkwatch events
10550 * pending on unregister. If this
10551 * happens, we simply run the queue
10552 * unscheduled, resulting in a noop
10553 * for this device.
10554 */
10555 linkwatch_run_queue();
10556 }
10557
Stephen Hemminger6756ae42006-03-20 22:23:58 -080010558 __rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -070010559
10560 rebroadcast_time = jiffies;
10561 }
10562
Francesco Ruggeri0e4be9e2020-09-18 13:19:01 -070010563 if (!wait) {
10564 rcu_barrier();
10565 wait = WAIT_REFS_MIN_MSECS;
10566 } else {
10567 msleep(wait);
10568 wait = min(wait << 1, WAIT_REFS_MAX_MSECS);
10569 }
Linus Torvalds1da177e2005-04-16 15:20:36 -070010570
Eric Dumazet29b44332010-10-11 10:22:12 +000010571 refcnt = netdev_refcnt_read(dev);
10572
Dmitry Vyukov6c996e12021-03-25 15:52:45 +010010573 if (refcnt != 1 &&
Dmitry Vyukov5aa3afe2021-03-23 07:49:23 +010010574 time_after(jiffies, warning_time +
10575 netdev_unregister_timeout_secs * HZ)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +000010576 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
10577 dev->name, refcnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -070010578 warning_time = jiffies;
10579 }
10580 }
10581}
10582
10583/* The sequence is:
10584 *
10585 * rtnl_lock();
10586 * ...
10587 * register_netdevice(x1);
10588 * register_netdevice(x2);
10589 * ...
10590 * unregister_netdevice(y1);
10591 * unregister_netdevice(y2);
10592 * ...
10593 * rtnl_unlock();
10594 * free_netdev(y1);
10595 * free_netdev(y2);
10596 *
Herbert Xu58ec3b42008-10-07 15:50:03 -070010597 * We are invoked by rtnl_unlock().
Linus Torvalds1da177e2005-04-16 15:20:36 -070010598 * This allows us to deal with problems:
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -070010599 * 1) We can delete sysfs objects which invoke hotplug
Linus Torvalds1da177e2005-04-16 15:20:36 -070010600 * without deadlocking with linkwatch via keventd.
10601 * 2) Since we run with the RTNL semaphore not held, we can sleep
10602 * safely in order to wait for the netdev refcnt to drop to zero.
Herbert Xu58ec3b42008-10-07 15:50:03 -070010603 *
10604 * We must not return until all unregister events added during
10605 * the interval the lock was held have been completed.
Linus Torvalds1da177e2005-04-16 15:20:36 -070010606 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070010607void netdev_run_todo(void)
10608{
Oleg Nesterov626ab0e2006-06-23 02:05:55 -070010609 struct list_head list;
Taehee Yoo1fc70ed2020-09-25 18:13:29 +000010610#ifdef CONFIG_LOCKDEP
10611 struct list_head unlink_list;
10612
10613 list_replace_init(&net_unlink_list, &unlink_list);
10614
10615 while (!list_empty(&unlink_list)) {
10616 struct net_device *dev = list_first_entry(&unlink_list,
10617 struct net_device,
10618 unlink_list);
Taehee Yoo0e8b8d62020-10-15 16:26:06 +000010619 list_del_init(&dev->unlink_list);
Taehee Yoo1fc70ed2020-09-25 18:13:29 +000010620 dev->nested_level = dev->lower_level - 1;
10621 }
10622#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -070010623
Linus Torvalds1da177e2005-04-16 15:20:36 -070010624 /* Snapshot list, allow later requests */
Oleg Nesterov626ab0e2006-06-23 02:05:55 -070010625 list_replace_init(&net_todo_list, &list);
Herbert Xu58ec3b42008-10-07 15:50:03 -070010626
10627 __rtnl_unlock();
Oleg Nesterov626ab0e2006-06-23 02:05:55 -070010628
Eric Dumazet0115e8e2012-08-22 17:19:46 +000010629
10630 /* Wait for rcu callbacks to finish before next phase */
Eric W. Biederman850a5452011-10-13 22:25:23 +000010631 if (!list_empty(&list))
10632 rcu_barrier();
10633
Linus Torvalds1da177e2005-04-16 15:20:36 -070010634 while (!list_empty(&list)) {
10635 struct net_device *dev
stephen hemmingere5e26d72010-02-24 14:01:38 +000010636 = list_first_entry(&list, struct net_device, todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -070010637 list_del(&dev->todo_list);
10638
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -070010639 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +000010640 pr_err("network todo '%s' but state %d\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -070010641 dev->name, dev->reg_state);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -070010642 dump_stack();
10643 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070010644 }
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -070010645
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -070010646 dev->reg_state = NETREG_UNREGISTERED;
10647
10648 netdev_wait_allrefs(dev);
10649
10650 /* paranoia */
Eric Dumazetadd2d732021-03-22 11:21:45 -070010651 BUG_ON(netdev_refcnt_read(dev) != 1);
Salam Noureddine7866a622015-01-27 11:35:48 -080010652 BUG_ON(!list_empty(&dev->ptype_all));
10653 BUG_ON(!list_empty(&dev->ptype_specific));
Eric Dumazet33d480c2011-08-11 19:30:52 +000010654 WARN_ON(rcu_access_pointer(dev->ip_ptr));
10655 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
David Ahern330c7272018-02-13 08:52:00 -080010656#if IS_ENABLED(CONFIG_DECNET)
Ilpo Järvinen547b7922008-07-25 21:43:18 -070010657 WARN_ON(dev->dn_ptr);
David Ahern330c7272018-02-13 08:52:00 -080010658#endif
David S. Millercf124db2017-05-08 12:52:56 -040010659 if (dev->priv_destructor)
10660 dev->priv_destructor(dev);
10661 if (dev->needs_free_netdev)
10662 free_netdev(dev);
Stephen Hemminger9093bbb2007-05-19 15:39:25 -070010663
Eric W. Biederman50624c92013-09-23 21:19:49 -070010664 /* Report a network device has been unregistered */
10665 rtnl_lock();
10666 dev_net(dev)->dev_unreg_count--;
10667 __rtnl_unlock();
10668 wake_up(&netdev_unregistering_wq);
10669
Stephen Hemminger9093bbb2007-05-19 15:39:25 -070010670 /* Free network device */
10671 kobject_put(&dev->dev.kobj);
Linus Torvalds1da177e2005-04-16 15:20:36 -070010672 }
Linus Torvalds1da177e2005-04-16 15:20:36 -070010673}
10674
Jarod Wilson92566452016-02-01 18:51:04 -050010675/* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has
10676 * all the same fields in the same order as net_device_stats, with only
10677 * the type differing, but rtnl_link_stats64 may have additional fields
10678 * at the end for newer counters.
Ben Hutchings3cfde792010-07-09 09:11:52 +000010679 */
Eric Dumazet77a1abf2012-03-05 04:50:09 +000010680void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
10681 const struct net_device_stats *netdev_stats)
Ben Hutchings3cfde792010-07-09 09:11:52 +000010682{
10683#if BITS_PER_LONG == 64
Jarod Wilson92566452016-02-01 18:51:04 -050010684 BUILD_BUG_ON(sizeof(*stats64) < sizeof(*netdev_stats));
Alban Browaeys9af99592017-07-03 03:20:13 +020010685 memcpy(stats64, netdev_stats, sizeof(*netdev_stats));
Jarod Wilson92566452016-02-01 18:51:04 -050010686 /* zero out counters that only exist in rtnl_link_stats64 */
10687 memset((char *)stats64 + sizeof(*netdev_stats), 0,
10688 sizeof(*stats64) - sizeof(*netdev_stats));
Ben Hutchings3cfde792010-07-09 09:11:52 +000010689#else
Jarod Wilson92566452016-02-01 18:51:04 -050010690 size_t i, n = sizeof(*netdev_stats) / sizeof(unsigned long);
Ben Hutchings3cfde792010-07-09 09:11:52 +000010691 const unsigned long *src = (const unsigned long *)netdev_stats;
10692 u64 *dst = (u64 *)stats64;
10693
Jarod Wilson92566452016-02-01 18:51:04 -050010694 BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64));
Ben Hutchings3cfde792010-07-09 09:11:52 +000010695 for (i = 0; i < n; i++)
10696 dst[i] = src[i];
Jarod Wilson92566452016-02-01 18:51:04 -050010697 /* zero out counters that only exist in rtnl_link_stats64 */
10698 memset((char *)stats64 + n * sizeof(u64), 0,
10699 sizeof(*stats64) - n * sizeof(u64));
Ben Hutchings3cfde792010-07-09 09:11:52 +000010700#endif
10701}
Eric Dumazet77a1abf2012-03-05 04:50:09 +000010702EXPORT_SYMBOL(netdev_stats_to_stats64);
Ben Hutchings3cfde792010-07-09 09:11:52 +000010703
Eric Dumazetd83345a2009-11-16 03:36:51 +000010704/**
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -080010705 * dev_get_stats - get network device statistics
10706 * @dev: device to get statistics from
Eric Dumazet28172732010-07-07 14:58:56 -070010707 * @storage: place to store stats
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -080010708 *
Ben Hutchingsd7753512010-07-09 09:12:41 +000010709 * Get network statistics from device. Return @storage.
10710 * The device driver may provide its own method by setting
10711 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
10712 * otherwise the internal statistics structure is used.
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -080010713 */
Ben Hutchingsd7753512010-07-09 09:12:41 +000010714struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
10715 struct rtnl_link_stats64 *storage)
Eric Dumazet7004bf22009-05-18 00:34:33 +000010716{
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -080010717 const struct net_device_ops *ops = dev->netdev_ops;
10718
Eric Dumazet28172732010-07-07 14:58:56 -070010719 if (ops->ndo_get_stats64) {
10720 memset(storage, 0, sizeof(*storage));
Eric Dumazetcaf586e2010-09-30 21:06:55 +000010721 ops->ndo_get_stats64(dev, storage);
10722 } else if (ops->ndo_get_stats) {
Ben Hutchings3cfde792010-07-09 09:11:52 +000010723 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
Eric Dumazetcaf586e2010-09-30 21:06:55 +000010724 } else {
10725 netdev_stats_to_stats64(storage, &dev->stats);
Eric Dumazet28172732010-07-07 14:58:56 -070010726 }
Eric Dumazet6f64ec72017-06-27 07:02:20 -070010727 storage->rx_dropped += (unsigned long)atomic_long_read(&dev->rx_dropped);
10728 storage->tx_dropped += (unsigned long)atomic_long_read(&dev->tx_dropped);
10729 storage->rx_nohandler += (unsigned long)atomic_long_read(&dev->rx_nohandler);
Eric Dumazet28172732010-07-07 14:58:56 -070010730 return storage;
Rusty Russellc45d2862007-03-28 14:29:08 -070010731}
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -080010732EXPORT_SYMBOL(dev_get_stats);
Rusty Russellc45d2862007-03-28 14:29:08 -070010733
Heiner Kallweit44fa32f2020-10-12 10:01:27 +020010734/**
10735 * dev_fetch_sw_netstats - get per-cpu network device statistics
10736 * @s: place to store stats
10737 * @netstats: per-cpu network stats to read from
10738 *
10739 * Read per-cpu network statistics and populate the related fields in @s.
10740 */
10741void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s,
10742 const struct pcpu_sw_netstats __percpu *netstats)
10743{
10744 int cpu;
10745
10746 for_each_possible_cpu(cpu) {
10747 const struct pcpu_sw_netstats *stats;
10748 struct pcpu_sw_netstats tmp;
10749 unsigned int start;
10750
10751 stats = per_cpu_ptr(netstats, cpu);
10752 do {
10753 start = u64_stats_fetch_begin_irq(&stats->syncp);
10754 tmp.rx_packets = stats->rx_packets;
10755 tmp.rx_bytes = stats->rx_bytes;
10756 tmp.tx_packets = stats->tx_packets;
10757 tmp.tx_bytes = stats->tx_bytes;
10758 } while (u64_stats_fetch_retry_irq(&stats->syncp, start));
10759
10760 s->rx_packets += tmp.rx_packets;
10761 s->rx_bytes += tmp.rx_bytes;
10762 s->tx_packets += tmp.tx_packets;
10763 s->tx_bytes += tmp.tx_bytes;
10764 }
10765}
10766EXPORT_SYMBOL_GPL(dev_fetch_sw_netstats);
10767
Heiner Kallweita1839422020-11-07 21:49:07 +010010768/**
10769 * dev_get_tstats64 - ndo_get_stats64 implementation
10770 * @dev: device to get statistics from
10771 * @s: place to store stats
10772 *
10773 * Populate @s from dev->stats and dev->tstats. Can be used as
10774 * ndo_get_stats64() callback.
10775 */
10776void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s)
10777{
10778 netdev_stats_to_stats64(s, &dev->stats);
10779 dev_fetch_sw_netstats(s, dev->tstats);
10780}
10781EXPORT_SYMBOL_GPL(dev_get_tstats64);
10782
Eric Dumazet24824a02010-10-02 06:11:55 +000010783struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
David S. Millerdc2b4842008-07-08 17:18:23 -070010784{
Eric Dumazet24824a02010-10-02 06:11:55 +000010785 struct netdev_queue *queue = dev_ingress_queue(dev);
David S. Millerdc2b4842008-07-08 17:18:23 -070010786
Eric Dumazet24824a02010-10-02 06:11:55 +000010787#ifdef CONFIG_NET_CLS_ACT
10788 if (queue)
10789 return queue;
10790 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
10791 if (!queue)
10792 return NULL;
10793 netdev_init_one_queue(dev, queue, NULL);
Eric Dumazet2ce1ee12015-02-04 13:37:44 -080010794 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
Eric Dumazet24824a02010-10-02 06:11:55 +000010795 queue->qdisc_sleeping = &noop_qdisc;
10796 rcu_assign_pointer(dev->ingress_queue, queue);
10797#endif
10798 return queue;
David S. Millerbb949fb2008-07-08 16:55:56 -070010799}
10800
Eric Dumazet2c60db02012-09-16 09:17:26 +000010801static const struct ethtool_ops default_ethtool_ops;
10802
Stanislaw Gruszkad07d7502013-01-10 23:19:10 +000010803void netdev_set_default_ethtool_ops(struct net_device *dev,
10804 const struct ethtool_ops *ops)
10805{
10806 if (dev->ethtool_ops == &default_ethtool_ops)
10807 dev->ethtool_ops = ops;
10808}
10809EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
10810
Eric Dumazet74d332c2013-10-30 13:10:44 -070010811void netdev_freemem(struct net_device *dev)
10812{
10813 char *addr = (char *)dev - dev->padded;
10814
WANG Cong4cb28972014-06-02 15:55:22 -070010815 kvfree(addr);
Eric Dumazet74d332c2013-10-30 13:10:44 -070010816}
10817
Linus Torvalds1da177e2005-04-16 15:20:36 -070010818/**
tcharding722c9a02017-02-09 17:56:04 +110010819 * alloc_netdev_mqs - allocate network device
10820 * @sizeof_priv: size of private data to allocate space for
10821 * @name: device name format string
10822 * @name_assign_type: origin of device name
10823 * @setup: callback to initialize device
10824 * @txqs: the number of TX subqueues to allocate
10825 * @rxqs: the number of RX subqueues to allocate
Linus Torvalds1da177e2005-04-16 15:20:36 -070010826 *
tcharding722c9a02017-02-09 17:56:04 +110010827 * Allocates a struct net_device with private data area for driver use
10828 * and performs basic initialization. Also allocates subqueue structs
10829 * for each queue on the device.
Linus Torvalds1da177e2005-04-16 15:20:36 -070010830 */
Tom Herbert36909ea2011-01-09 19:36:31 +000010831struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
Tom Gundersenc835a672014-07-14 16:37:24 +020010832 unsigned char name_assign_type,
Tom Herbert36909ea2011-01-09 19:36:31 +000010833 void (*setup)(struct net_device *),
10834 unsigned int txqs, unsigned int rxqs)
Linus Torvalds1da177e2005-04-16 15:20:36 -070010835{
Linus Torvalds1da177e2005-04-16 15:20:36 -070010836 struct net_device *dev;
Alexey Dobriyan52a59bd2017-09-21 23:33:29 +030010837 unsigned int alloc_size;
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +000010838 struct net_device *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -070010839
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -070010840 BUG_ON(strlen(name) >= sizeof(dev->name));
10841
Tom Herbert36909ea2011-01-09 19:36:31 +000010842 if (txqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +000010843 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
Tom Herbert55513fb2010-10-18 17:55:58 +000010844 return NULL;
10845 }
10846
Tom Herbert36909ea2011-01-09 19:36:31 +000010847 if (rxqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +000010848 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
Tom Herbert36909ea2011-01-09 19:36:31 +000010849 return NULL;
10850 }
Tom Herbert36909ea2011-01-09 19:36:31 +000010851
David S. Millerfd2ea0a2008-07-17 01:56:23 -070010852 alloc_size = sizeof(struct net_device);
Alexey Dobriyand1643d22008-04-18 15:43:32 -070010853 if (sizeof_priv) {
10854 /* ensure 32-byte alignment of private area */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +000010855 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
Alexey Dobriyand1643d22008-04-18 15:43:32 -070010856 alloc_size += sizeof_priv;
10857 }
10858 /* ensure 32-byte alignment of whole construct */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +000010859 alloc_size += NETDEV_ALIGN - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -070010860
Vasily Averinc948f512021-07-19 13:44:23 +030010861 p = kvzalloc(alloc_size, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
Joe Perches62b59422013-02-04 16:48:16 +000010862 if (!p)
Linus Torvalds1da177e2005-04-16 15:20:36 -070010863 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -070010864
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +000010865 dev = PTR_ALIGN(p, NETDEV_ALIGN);
Linus Torvalds1da177e2005-04-16 15:20:36 -070010866 dev->padded = (char *)dev - (char *)p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +000010867
Eric Dumazet919067c2021-03-19 10:39:33 -070010868#ifdef CONFIG_PCPU_DEV_REFCNT
Eric Dumazet29b44332010-10-11 10:22:12 +000010869 dev->pcpu_refcnt = alloc_percpu(int);
10870 if (!dev->pcpu_refcnt)
Eric Dumazet74d332c2013-10-30 13:10:44 -070010871 goto free_dev;
Eric Dumazetadd2d732021-03-22 11:21:45 -070010872 dev_hold(dev);
10873#else
10874 refcount_set(&dev->dev_refcnt, 1);
Eric Dumazet919067c2021-03-19 10:39:33 -070010875#endif
Jiri Pirkoab9c73c2009-05-08 13:30:17 +000010876
Linus Torvalds1da177e2005-04-16 15:20:36 -070010877 if (dev_addr_init(dev))
Eric Dumazet29b44332010-10-11 10:22:12 +000010878 goto free_pcpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -070010879
Jiri Pirko22bedad32010-04-01 21:22:57 +000010880 dev_mc_init(dev);
Jiri Pirkoa748ee22010-04-01 21:22:09 +000010881 dev_uc_init(dev);
Jiri Pirkoccffad252009-05-22 23:22:17 +000010882
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +090010883 dev_net_set(dev, &init_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -070010884
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -070010885 dev->gso_max_size = GSO_MAX_SIZE;
Ben Hutchings30b678d2012-07-30 15:57:00 +000010886 dev->gso_max_segs = GSO_MAX_SEGS;
Taehee Yoo5343da42019-10-21 18:47:50 +000010887 dev->upper_level = 1;
10888 dev->lower_level = 1;
Taehee Yoo1fc70ed2020-09-25 18:13:29 +000010889#ifdef CONFIG_LOCKDEP
10890 dev->nested_level = 0;
10891 INIT_LIST_HEAD(&dev->unlink_list);
10892#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -070010893
Herbert Xud565b0a2008-12-15 23:38:52 -080010894 INIT_LIST_HEAD(&dev->napi_list);
Eric W. Biederman9fdce092009-10-30 14:51:13 +000010895 INIT_LIST_HEAD(&dev->unreg_list);
Eric W. Biederman5cde2822013-10-05 19:26:05 -070010896 INIT_LIST_HEAD(&dev->close_list);
Eric Dumazete014deb2009-11-17 05:59:21 +000010897 INIT_LIST_HEAD(&dev->link_watch_list);
Veaceslav Falico2f268f12013-09-25 09:20:07 +020010898 INIT_LIST_HEAD(&dev->adj_list.upper);
10899 INIT_LIST_HEAD(&dev->adj_list.lower);
Salam Noureddine7866a622015-01-27 11:35:48 -080010900 INIT_LIST_HEAD(&dev->ptype_all);
10901 INIT_LIST_HEAD(&dev->ptype_specific);
Jiri Pirko93642e12020-01-25 12:17:08 +010010902 INIT_LIST_HEAD(&dev->net_notifier_list);
Jiri Kosina59cc1f62016-08-10 11:05:15 +020010903#ifdef CONFIG_NET_SCHED
10904 hash_init(dev->qdisc_hash);
10905#endif
Eric Dumazet02875872014-10-05 18:38:35 -070010906 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
Linus Torvalds1da177e2005-04-16 15:20:36 -070010907 setup(dev);
David S. Miller8d3bdbd2011-02-08 15:02:50 -080010908
Phil Suttera8131042016-02-17 15:37:43 +010010909 if (!dev->tx_queue_len) {
Phil Sutterf84bb1e2015-08-27 21:21:36 +020010910 dev->priv_flags |= IFF_NO_QUEUE;
Jesper Dangaard Brouer11597082016-11-03 14:56:06 +010010911 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
Phil Suttera8131042016-02-17 15:37:43 +010010912 }
Phil Sutter906470c2015-08-18 10:30:48 +020010913
David S. Miller8d3bdbd2011-02-08 15:02:50 -080010914 dev->num_tx_queues = txqs;
10915 dev->real_num_tx_queues = txqs;
10916 if (netif_alloc_netdev_queues(dev))
10917 goto free_all;
10918
David S. Miller8d3bdbd2011-02-08 15:02:50 -080010919 dev->num_rx_queues = rxqs;
10920 dev->real_num_rx_queues = rxqs;
10921 if (netif_alloc_rx_queues(dev))
10922 goto free_all;
David S. Miller8d3bdbd2011-02-08 15:02:50 -080010923
Linus Torvalds1da177e2005-04-16 15:20:36 -070010924 strcpy(dev->name, name);
Tom Gundersenc835a672014-07-14 16:37:24 +020010925 dev->name_assign_type = name_assign_type;
Vlad Dogarucbda10f2011-01-13 23:38:30 +000010926 dev->group = INIT_NETDEV_GROUP;
Eric Dumazet2c60db02012-09-16 09:17:26 +000010927 if (!dev->ethtool_ops)
10928 dev->ethtool_ops = &default_ethtool_ops;
Pablo Neirae687ad62015-05-13 18:19:38 +020010929
Daniel Borkmann357b6cc2020-03-18 10:33:22 +010010930 nf_hook_ingress_init(dev);
Pablo Neirae687ad62015-05-13 18:19:38 +020010931
Linus Torvalds1da177e2005-04-16 15:20:36 -070010932 return dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +000010933
David S. Miller8d3bdbd2011-02-08 15:02:50 -080010934free_all:
10935 free_netdev(dev);
10936 return NULL;
10937
Eric Dumazet29b44332010-10-11 10:22:12 +000010938free_pcpu:
Eric Dumazet919067c2021-03-19 10:39:33 -070010939#ifdef CONFIG_PCPU_DEV_REFCNT
Eric Dumazet29b44332010-10-11 10:22:12 +000010940 free_percpu(dev->pcpu_refcnt);
Eric Dumazet74d332c2013-10-30 13:10:44 -070010941free_dev:
Eric Dumazet919067c2021-03-19 10:39:33 -070010942#endif
Eric Dumazet74d332c2013-10-30 13:10:44 -070010943 netdev_freemem(dev);
Jiri Pirkoab9c73c2009-05-08 13:30:17 +000010944 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -070010945}
Tom Herbert36909ea2011-01-09 19:36:31 +000010946EXPORT_SYMBOL(alloc_netdev_mqs);
Linus Torvalds1da177e2005-04-16 15:20:36 -070010947
10948/**
tcharding722c9a02017-02-09 17:56:04 +110010949 * free_netdev - free network device
10950 * @dev: device
Linus Torvalds1da177e2005-04-16 15:20:36 -070010951 *
tcharding722c9a02017-02-09 17:56:04 +110010952 * This function does the last stage of destroying an allocated device
10953 * interface. The reference to the device object is released. If this
10954 * is the last reference then it will be freed.Must be called in process
10955 * context.
Linus Torvalds1da177e2005-04-16 15:20:36 -070010956 */
10957void free_netdev(struct net_device *dev)
10958{
Herbert Xud565b0a2008-12-15 23:38:52 -080010959 struct napi_struct *p, *n;
10960
Eric Dumazet93d05d42015-11-18 06:31:03 -080010961 might_sleep();
Jakub Kicinskic269a242021-01-06 10:40:06 -080010962
10963 /* When called immediately after register_netdevice() failed the unwind
10964 * handling may still be dismantling the device. Handle that case by
10965 * deferring the free.
10966 */
10967 if (dev->reg_state == NETREG_UNREGISTERING) {
10968 ASSERT_RTNL();
10969 dev->needs_free_netdev = true;
10970 return;
10971 }
10972
Eric Dumazet60877a32013-06-20 01:15:51 -070010973 netif_free_tx_queues(dev);
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +010010974 netif_free_rx_queues(dev);
David S. Millere8a04642008-07-17 00:34:19 -070010975
Eric Dumazet33d480c2011-08-11 19:30:52 +000010976 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
Eric Dumazet24824a02010-10-02 06:11:55 +000010977
Jiri Pirkof001fde2009-05-05 02:48:28 +000010978 /* Flush device addresses */
10979 dev_addr_flush(dev);
10980
Herbert Xud565b0a2008-12-15 23:38:52 -080010981 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
10982 netif_napi_del(p);
10983
Eric Dumazet919067c2021-03-19 10:39:33 -070010984#ifdef CONFIG_PCPU_DEV_REFCNT
Eric Dumazet29b44332010-10-11 10:22:12 +000010985 free_percpu(dev->pcpu_refcnt);
10986 dev->pcpu_refcnt = NULL;
Eric Dumazet919067c2021-03-19 10:39:33 -070010987#endif
Toke Høiland-Jørgensen75ccae62020-01-16 16:14:44 +010010988 free_percpu(dev->xdp_bulkq);
10989 dev->xdp_bulkq = NULL;
Eric Dumazet29b44332010-10-11 10:22:12 +000010990
Stephen Hemminger3041a062006-05-26 13:25:24 -070010991 /* Compatibility with error handling in drivers */
Linus Torvalds1da177e2005-04-16 15:20:36 -070010992 if (dev->reg_state == NETREG_UNINITIALIZED) {
Eric Dumazet74d332c2013-10-30 13:10:44 -070010993 netdev_freemem(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -070010994 return;
10995 }
10996
10997 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
10998 dev->reg_state = NETREG_RELEASED;
10999
Greg Kroah-Hartman43cb76d2002-04-09 12:14:34 -070011000 /* will free via device release */
11001 put_device(&dev->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -070011002}
Eric Dumazetd1b19df2009-09-03 01:29:39 -070011003EXPORT_SYMBOL(free_netdev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +090011004
Stephen Hemmingerf0db2752008-09-30 02:23:58 -070011005/**
11006 * synchronize_net - Synchronize with packet receive processing
11007 *
11008 * Wait for packets currently being received to be done.
11009 * Does not block later packets from starting.
11010 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +090011011void synchronize_net(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -070011012{
11013 might_sleep();
Eric Dumazetbe3fc412011-05-23 23:07:32 +000011014 if (rtnl_is_locked())
11015 synchronize_rcu_expedited();
11016 else
11017 synchronize_rcu();
Linus Torvalds1da177e2005-04-16 15:20:36 -070011018}
Eric Dumazetd1b19df2009-09-03 01:29:39 -070011019EXPORT_SYMBOL(synchronize_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -070011020
11021/**
Eric Dumazet44a08732009-10-27 07:03:04 +000011022 * unregister_netdevice_queue - remove device from the kernel
Linus Torvalds1da177e2005-04-16 15:20:36 -070011023 * @dev: device
Eric Dumazet44a08732009-10-27 07:03:04 +000011024 * @head: list
Jaswinder Singh Rajput6ebfbc02009-11-22 20:43:13 -080011025 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070011026 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -080011027 * from the kernel tables.
Eric Dumazet44a08732009-10-27 07:03:04 +000011028 * If head not NULL, device is queued to be unregistered later.
Linus Torvalds1da177e2005-04-16 15:20:36 -070011029 *
11030 * Callers must hold the rtnl semaphore. You may want
11031 * unregister_netdev() instead of this.
11032 */
11033
Eric Dumazet44a08732009-10-27 07:03:04 +000011034void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -070011035{
Herbert Xua6620712007-12-12 19:21:56 -080011036 ASSERT_RTNL();
11037
Eric Dumazet44a08732009-10-27 07:03:04 +000011038 if (head) {
Eric W. Biederman9fdce092009-10-30 14:51:13 +000011039 list_move_tail(&dev->unreg_list, head);
Eric Dumazet44a08732009-10-27 07:03:04 +000011040 } else {
Jakub Kicinski037e56b2021-01-19 12:25:19 -080011041 LIST_HEAD(single);
11042
11043 list_add(&dev->unreg_list, &single);
Jakub Kicinski0cbe1e52021-01-19 12:25:21 -080011044 unregister_netdevice_many(&single);
Eric Dumazet44a08732009-10-27 07:03:04 +000011045 }
Linus Torvalds1da177e2005-04-16 15:20:36 -070011046}
Eric Dumazet44a08732009-10-27 07:03:04 +000011047EXPORT_SYMBOL(unregister_netdevice_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -070011048
11049/**
Eric Dumazet9b5e3832009-10-27 07:04:19 +000011050 * unregister_netdevice_many - unregister many devices
11051 * @head: list of devices
Eric Dumazet87757a92014-06-06 06:44:03 -070011052 *
11053 * Note: As most callers use a stack allocated list_head,
11054 * we force a list_del() to make sure stack wont be corrupted later.
Eric Dumazet9b5e3832009-10-27 07:04:19 +000011055 */
11056void unregister_netdevice_many(struct list_head *head)
11057{
Jakub Kicinskibcfe2f12021-01-19 12:25:20 -080011058 struct net_device *dev, *tmp;
11059 LIST_HEAD(close_head);
11060
11061 BUG_ON(dev_boot_phase);
11062 ASSERT_RTNL();
11063
Jakub Kicinski0cbe1e52021-01-19 12:25:21 -080011064 if (list_empty(head))
11065 return;
11066
Jakub Kicinskibcfe2f12021-01-19 12:25:20 -080011067 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
11068 /* Some devices call without registering
11069 * for initialization unwind. Remove those
11070 * devices and proceed with the remaining.
11071 */
11072 if (dev->reg_state == NETREG_UNINITIALIZED) {
11073 pr_debug("unregister_netdevice: device %s/%p never was registered\n",
11074 dev->name, dev);
11075
11076 WARN_ON(1);
11077 list_del(&dev->unreg_list);
11078 continue;
11079 }
11080 dev->dismantle = true;
11081 BUG_ON(dev->reg_state != NETREG_REGISTERED);
11082 }
11083
11084 /* If device is running, close it first. */
11085 list_for_each_entry(dev, head, unreg_list)
11086 list_add_tail(&dev->close_list, &close_head);
11087 dev_close_many(&close_head, true);
11088
11089 list_for_each_entry(dev, head, unreg_list) {
11090 /* And unlink it from device chain. */
11091 unlist_netdevice(dev);
11092
11093 dev->reg_state = NETREG_UNREGISTERING;
11094 }
11095 flush_all_backlogs();
11096
11097 synchronize_net();
11098
11099 list_for_each_entry(dev, head, unreg_list) {
11100 struct sk_buff *skb = NULL;
11101
11102 /* Shutdown queueing discipline. */
11103 dev_shutdown(dev);
11104
11105 dev_xdp_uninstall(dev);
11106
11107 /* Notify protocols, that we are about to destroy
11108 * this device. They should clean all the things.
11109 */
11110 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
11111
11112 if (!dev->rtnl_link_ops ||
11113 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
11114 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0,
11115 GFP_KERNEL, NULL, 0);
11116
11117 /*
11118 * Flush the unicast and multicast chains
11119 */
11120 dev_uc_flush(dev);
11121 dev_mc_flush(dev);
11122
11123 netdev_name_node_alt_flush(dev);
11124 netdev_name_node_free(dev->name_node);
11125
11126 if (dev->netdev_ops->ndo_uninit)
11127 dev->netdev_ops->ndo_uninit(dev);
11128
11129 if (skb)
11130 rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
11131
11132 /* Notifier chain MUST detach us all upper devices. */
11133 WARN_ON(netdev_has_any_upper_dev(dev));
11134 WARN_ON(netdev_has_any_lower_dev(dev));
11135
11136 /* Remove entries from kobject tree */
11137 netdev_unregister_kobject(dev);
11138#ifdef CONFIG_XPS
11139 /* Remove XPS queueing entries */
11140 netif_reset_xps_queues_gt(dev, 0);
11141#endif
11142 }
11143
11144 synchronize_net();
11145
11146 list_for_each_entry(dev, head, unreg_list) {
11147 dev_put(dev);
11148 net_set_todo(dev);
11149 }
Jakub Kicinski0cbe1e52021-01-19 12:25:21 -080011150
11151 list_del(head);
Jakub Kicinskibcfe2f12021-01-19 12:25:20 -080011152}
Jakub Kicinski0cbe1e52021-01-19 12:25:21 -080011153EXPORT_SYMBOL(unregister_netdevice_many);
Jakub Kicinskibcfe2f12021-01-19 12:25:20 -080011154
Eric Dumazet9b5e3832009-10-27 07:04:19 +000011155/**
Linus Torvalds1da177e2005-04-16 15:20:36 -070011156 * unregister_netdev - remove device from the kernel
11157 * @dev: device
11158 *
11159 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -080011160 * from the kernel tables.
Linus Torvalds1da177e2005-04-16 15:20:36 -070011161 *
11162 * This is just a wrapper for unregister_netdevice that takes
11163 * the rtnl semaphore. In general you want to use this and not
11164 * unregister_netdevice.
11165 */
11166void unregister_netdev(struct net_device *dev)
11167{
11168 rtnl_lock();
11169 unregister_netdevice(dev);
11170 rtnl_unlock();
11171}
Linus Torvalds1da177e2005-04-16 15:20:36 -070011172EXPORT_SYMBOL(unregister_netdev);
11173
Eric W. Biedermance286d32007-09-12 13:53:49 +020011174/**
Andrei Vagin0854fa82021-04-06 23:40:51 -070011175 * __dev_change_net_namespace - move device to different nethost namespace
Eric W. Biedermance286d32007-09-12 13:53:49 +020011176 * @dev: device
11177 * @net: network namespace
11178 * @pat: If not NULL name pattern to try if the current device name
11179 * is already taken in the destination network namespace.
Andrei Vagineeb85a12021-04-05 00:12:23 -070011180 * @new_ifindex: If not zero, specifies device index in the target
11181 * namespace.
Eric W. Biedermance286d32007-09-12 13:53:49 +020011182 *
11183 * This function shuts down a device interface and moves it
11184 * to a new network namespace. On success 0 is returned, on
11185 * a failure a netagive errno code is returned.
11186 *
11187 * Callers must hold the rtnl semaphore.
11188 */
11189
Andrei Vagin0854fa82021-04-06 23:40:51 -070011190int __dev_change_net_namespace(struct net_device *dev, struct net *net,
11191 const char *pat, int new_ifindex)
Eric W. Biedermance286d32007-09-12 13:53:49 +020011192{
Christian Brauneref6a4c82020-02-27 04:37:19 +010011193 struct net *net_old = dev_net(dev);
Andrei Vagineeb85a12021-04-05 00:12:23 -070011194 int err, new_nsid;
Eric W. Biedermance286d32007-09-12 13:53:49 +020011195
11196 ASSERT_RTNL();
11197
11198 /* Don't allow namespace local devices to be moved. */
11199 err = -EINVAL;
11200 if (dev->features & NETIF_F_NETNS_LOCAL)
11201 goto out;
11202
11203 /* Ensure the device has been registrered */
Eric W. Biedermance286d32007-09-12 13:53:49 +020011204 if (dev->reg_state != NETREG_REGISTERED)
11205 goto out;
11206
11207 /* Get out if there is nothing todo */
11208 err = 0;
Christian Brauneref6a4c82020-02-27 04:37:19 +010011209 if (net_eq(net_old, net))
Eric W. Biedermance286d32007-09-12 13:53:49 +020011210 goto out;
11211
11212 /* Pick the destination device name, and ensure
11213 * we can use it in the destination network namespace.
11214 */
11215 err = -EEXIST;
Octavian Purdilad9031022009-11-18 02:36:59 +000011216 if (__dev_get_by_name(net, dev->name)) {
Eric W. Biedermance286d32007-09-12 13:53:49 +020011217 /* We get here if we can't use the current device name */
11218 if (!pat)
11219 goto out;
Li RongQing7892bd02018-06-19 17:23:17 +080011220 err = dev_get_valid_name(net, dev, pat);
11221 if (err < 0)
Eric W. Biedermance286d32007-09-12 13:53:49 +020011222 goto out;
11223 }
11224
Andrei Vagineeb85a12021-04-05 00:12:23 -070011225 /* Check that new_ifindex isn't used yet. */
11226 err = -EBUSY;
11227 if (new_ifindex && __dev_get_by_index(net, new_ifindex))
11228 goto out;
11229
Eric W. Biedermance286d32007-09-12 13:53:49 +020011230 /*
11231 * And now a mini version of register_netdevice unregister_netdevice.
11232 */
11233
11234 /* If device is running close it first. */
Pavel Emelyanov9b772652007-10-10 02:49:09 -070011235 dev_close(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +020011236
11237 /* And unlink it from device chain */
Eric W. Biedermance286d32007-09-12 13:53:49 +020011238 unlist_netdevice(dev);
11239
11240 synchronize_net();
11241
11242 /* Shutdown queueing discipline. */
11243 dev_shutdown(dev);
11244
11245 /* Notify protocols, that we are about to destroy
tchardingeb13da12017-02-09 17:56:06 +110011246 * this device. They should clean all the things.
11247 *
11248 * Note that dev->reg_state stays at NETREG_REGISTERED.
11249 * This is wanted because this way 8021q and macvlan know
11250 * the device is just moving and can keep their slaves up.
11251 */
Eric W. Biedermance286d32007-09-12 13:53:49 +020011252 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Gao feng6549dd42012-08-23 15:36:55 +000011253 rcu_barrier();
Nicolas Dichtel38e01b32018-01-25 15:01:39 +010011254
Guillaume Naultd4e4fdf2019-10-23 18:39:04 +020011255 new_nsid = peernet2id_alloc(dev_net(dev), net, GFP_KERNEL);
Nicolas Dichtel38e01b32018-01-25 15:01:39 +010011256 /* If there is an ifindex conflict assign a new one */
Andrei Vagineeb85a12021-04-05 00:12:23 -070011257 if (!new_ifindex) {
11258 if (__dev_get_by_index(net, dev->ifindex))
11259 new_ifindex = dev_new_index(net);
11260 else
11261 new_ifindex = dev->ifindex;
11262 }
Nicolas Dichtel38e01b32018-01-25 15:01:39 +010011263
11264 rtmsg_ifinfo_newnet(RTM_DELLINK, dev, ~0U, GFP_KERNEL, &new_nsid,
11265 new_ifindex);
Eric W. Biedermance286d32007-09-12 13:53:49 +020011266
11267 /*
11268 * Flush the unicast and multicast chains
11269 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +000011270 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +000011271 dev_mc_flush(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +020011272
Serge Hallyn4e66ae22012-12-03 16:17:12 +000011273 /* Send a netdev-removed uevent to the old namespace */
11274 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +040011275 netdev_adjacent_del_links(dev);
Serge Hallyn4e66ae22012-12-03 16:17:12 +000011276
Jiri Pirko93642e12020-01-25 12:17:08 +010011277 /* Move per-net netdevice notifiers that are following the netdevice */
11278 move_netdevice_notifiers_dev_net(dev, net);
11279
Eric W. Biedermance286d32007-09-12 13:53:49 +020011280 /* Actually switch the network namespace */
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +090011281 dev_net_set(dev, net);
Nicolas Dichtel38e01b32018-01-25 15:01:39 +010011282 dev->ifindex = new_ifindex;
Eric W. Biedermance286d32007-09-12 13:53:49 +020011283
Serge Hallyn4e66ae22012-12-03 16:17:12 +000011284 /* Send a netdev-add uevent to the new namespace */
11285 kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +040011286 netdev_adjacent_add_links(dev);
Serge Hallyn4e66ae22012-12-03 16:17:12 +000011287
Eric W. Biederman8b41d182007-09-26 22:02:53 -070011288 /* Fixup kobjects */
Eric W. Biedermana1b3f592010-05-04 17:36:49 -070011289 err = device_rename(&dev->dev, dev->name);
Eric W. Biederman8b41d182007-09-26 22:02:53 -070011290 WARN_ON(err);
Eric W. Biedermance286d32007-09-12 13:53:49 +020011291
Christian Brauneref6a4c82020-02-27 04:37:19 +010011292 /* Adapt owner in case owning user namespace of target network
11293 * namespace is different from the original one.
11294 */
11295 err = netdev_change_owner(dev, net_old, net);
11296 WARN_ON(err);
11297
Eric W. Biedermance286d32007-09-12 13:53:49 +020011298 /* Add the device back in the hashes */
11299 list_netdevice(dev);
11300
11301 /* Notify protocols, that a new device appeared. */
11302 call_netdevice_notifiers(NETDEV_REGISTER, dev);
11303
Eric W. Biedermand90a9092009-12-12 22:11:15 +000011304 /*
11305 * Prevent userspace races by waiting until the network
11306 * device is fully setup before sending notifications.
11307 */
Alexei Starovoitov7f294052013-10-23 16:02:42 -070011308 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
Eric W. Biedermand90a9092009-12-12 22:11:15 +000011309
Eric W. Biedermance286d32007-09-12 13:53:49 +020011310 synchronize_net();
11311 err = 0;
11312out:
11313 return err;
11314}
Andrei Vagin0854fa82021-04-06 23:40:51 -070011315EXPORT_SYMBOL_GPL(__dev_change_net_namespace);
Eric W. Biedermance286d32007-09-12 13:53:49 +020011316
Sebastian Andrzej Siewiorf0bf90d2016-11-03 15:50:04 +010011317static int dev_cpu_dead(unsigned int oldcpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -070011318{
11319 struct sk_buff **list_skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -070011320 struct sk_buff *skb;
Sebastian Andrzej Siewiorf0bf90d2016-11-03 15:50:04 +010011321 unsigned int cpu;
Ashwanth Goli97d8b6e2017-06-13 16:54:55 +053011322 struct softnet_data *sd, *oldsd, *remsd = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -070011323
Linus Torvalds1da177e2005-04-16 15:20:36 -070011324 local_irq_disable();
11325 cpu = smp_processor_id();
11326 sd = &per_cpu(softnet_data, cpu);
11327 oldsd = &per_cpu(softnet_data, oldcpu);
11328
11329 /* Find end of our completion_queue. */
11330 list_skb = &sd->completion_queue;
11331 while (*list_skb)
11332 list_skb = &(*list_skb)->next;
11333 /* Append completion queue from offline CPU. */
11334 *list_skb = oldsd->completion_queue;
11335 oldsd->completion_queue = NULL;
11336
Linus Torvalds1da177e2005-04-16 15:20:36 -070011337 /* Append output queue from offline CPU. */
Changli Gaoa9cbd582010-04-26 23:06:24 +000011338 if (oldsd->output_queue) {
11339 *sd->output_queue_tailp = oldsd->output_queue;
11340 sd->output_queue_tailp = oldsd->output_queue_tailp;
11341 oldsd->output_queue = NULL;
11342 oldsd->output_queue_tailp = &oldsd->output_queue;
11343 }
Eric Dumazetac64da02015-01-15 17:04:22 -080011344 /* Append NAPI poll list from offline CPU, with one exception :
11345 * process_backlog() must be called by cpu owning percpu backlog.
11346 * We properly handle process_queue & input_pkt_queue later.
11347 */
11348 while (!list_empty(&oldsd->poll_list)) {
11349 struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
11350 struct napi_struct,
11351 poll_list);
11352
11353 list_del_init(&napi->poll_list);
11354 if (napi->poll == process_backlog)
11355 napi->state = 0;
11356 else
11357 ____napi_schedule(sd, napi);
Heiko Carstens264524d2011-06-06 20:50:03 +000011358 }
Linus Torvalds1da177e2005-04-16 15:20:36 -070011359
11360 raise_softirq_irqoff(NET_TX_SOFTIRQ);
11361 local_irq_enable();
11362
ashwanth@codeaurora.org773fc8f2017-06-09 14:24:58 +053011363#ifdef CONFIG_RPS
11364 remsd = oldsd->rps_ipi_list;
11365 oldsd->rps_ipi_list = NULL;
11366#endif
11367 /* send out pending IPI's on offline CPU */
11368 net_rps_send_ipi(remsd);
11369
Linus Torvalds1da177e2005-04-16 15:20:36 -070011370 /* Process offline CPU's input_pkt_queue */
Tom Herbert76cc8b12010-05-20 18:37:59 +000011371 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
Eric Dumazet91e83132015-02-05 14:58:14 -080011372 netif_rx_ni(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +000011373 input_queue_head_incr(oldsd);
11374 }
Eric Dumazetac64da02015-01-15 17:04:22 -080011375 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
Eric Dumazet91e83132015-02-05 14:58:14 -080011376 netif_rx_ni(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +000011377 input_queue_head_incr(oldsd);
Tom Herbertfec5e652010-04-16 16:01:27 -070011378 }
Linus Torvalds1da177e2005-04-16 15:20:36 -070011379
Sebastian Andrzej Siewiorf0bf90d2016-11-03 15:50:04 +010011380 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -070011381}
Linus Torvalds1da177e2005-04-16 15:20:36 -070011382
Herbert Xu7f353bf2007-08-10 15:47:58 -070011383/**
Herbert Xub63365a2008-10-23 01:11:29 -070011384 * netdev_increment_features - increment feature set by one
11385 * @all: current feature set
11386 * @one: new feature set
11387 * @mask: mask feature set
Herbert Xu7f353bf2007-08-10 15:47:58 -070011388 *
11389 * Computes a new feature set after adding a device with feature set
Herbert Xub63365a2008-10-23 01:11:29 -070011390 * @one to the master device with current feature set @all. Will not
11391 * enable anything that is off in @mask. Returns the new feature set.
Herbert Xu7f353bf2007-08-10 15:47:58 -070011392 */
Michał Mirosławc8f44af2011-11-15 15:29:55 +000011393netdev_features_t netdev_increment_features(netdev_features_t all,
11394 netdev_features_t one, netdev_features_t mask)
Herbert Xu7f353bf2007-08-10 15:47:58 -070011395{
Tom Herbertc8cd0982015-12-14 11:19:44 -080011396 if (mask & NETIF_F_HW_CSUM)
Tom Herberta1882222015-12-14 11:19:43 -080011397 mask |= NETIF_F_CSUM_MASK;
Michał Mirosław1742f182011-04-22 06:31:16 +000011398 mask |= NETIF_F_VLAN_CHALLENGED;
11399
Tom Herberta1882222015-12-14 11:19:43 -080011400 all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask;
Michał Mirosław1742f182011-04-22 06:31:16 +000011401 all &= one | ~NETIF_F_ALL_FOR_ALL;
11402
Michał Mirosław1742f182011-04-22 06:31:16 +000011403 /* If one device supports hw checksumming, set for all. */
Tom Herbertc8cd0982015-12-14 11:19:44 -080011404 if (all & NETIF_F_HW_CSUM)
11405 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM);
Herbert Xu7f353bf2007-08-10 15:47:58 -070011406
11407 return all;
11408}
Herbert Xub63365a2008-10-23 01:11:29 -070011409EXPORT_SYMBOL(netdev_increment_features);
Herbert Xu7f353bf2007-08-10 15:47:58 -070011410
Baruch Siach430f03c2013-06-02 20:43:55 +000011411static struct hlist_head * __net_init netdev_create_hash(void)
Pavel Emelyanov30d97d32007-09-16 15:40:33 -070011412{
11413 int i;
11414 struct hlist_head *hash;
11415
Kees Cook6da2ec52018-06-12 13:55:00 -070011416 hash = kmalloc_array(NETDEV_HASHENTRIES, sizeof(*hash), GFP_KERNEL);
Pavel Emelyanov30d97d32007-09-16 15:40:33 -070011417 if (hash != NULL)
11418 for (i = 0; i < NETDEV_HASHENTRIES; i++)
11419 INIT_HLIST_HEAD(&hash[i]);
11420
11421 return hash;
11422}
11423
Eric W. Biederman881d9662007-09-17 11:56:21 -070011424/* Initialize per network namespace state */
Pavel Emelyanov46650792007-10-08 20:38:39 -070011425static int __net_init netdev_init(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -070011426{
Li RongQingd9f37d02018-07-13 14:41:36 +080011427 BUILD_BUG_ON(GRO_HASH_BUCKETS >
Pankaj Bharadiyac5936422019-12-09 10:31:43 -080011428 8 * sizeof_field(struct napi_struct, gro_bitmask));
Li RongQingd9f37d02018-07-13 14:41:36 +080011429
Rustad, Mark D734b6542012-07-18 09:06:07 +000011430 if (net != &init_net)
11431 INIT_LIST_HEAD(&net->dev_base_head);
Eric W. Biederman881d9662007-09-17 11:56:21 -070011432
Pavel Emelyanov30d97d32007-09-16 15:40:33 -070011433 net->dev_name_head = netdev_create_hash();
11434 if (net->dev_name_head == NULL)
11435 goto err_name;
Eric W. Biederman881d9662007-09-17 11:56:21 -070011436
Pavel Emelyanov30d97d32007-09-16 15:40:33 -070011437 net->dev_index_head = netdev_create_hash();
11438 if (net->dev_index_head == NULL)
11439 goto err_idx;
Eric W. Biederman881d9662007-09-17 11:56:21 -070011440
Jiri Pirkoa30c7b42019-09-30 10:15:10 +020011441 RAW_INIT_NOTIFIER_HEAD(&net->netdev_chain);
11442
Eric W. Biederman881d9662007-09-17 11:56:21 -070011443 return 0;
Pavel Emelyanov30d97d32007-09-16 15:40:33 -070011444
11445err_idx:
11446 kfree(net->dev_name_head);
11447err_name:
11448 return -ENOMEM;
Eric W. Biederman881d9662007-09-17 11:56:21 -070011449}
11450
Stephen Hemmingerf0db2752008-09-30 02:23:58 -070011451/**
11452 * netdev_drivername - network driver for the device
11453 * @dev: network device
Stephen Hemmingerf0db2752008-09-30 02:23:58 -070011454 *
11455 * Determine network driver for device.
11456 */
David S. Miller3019de12011-06-06 16:41:33 -070011457const char *netdev_drivername(const struct net_device *dev)
Arjan van de Ven6579e572008-07-21 13:31:48 -070011458{
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -070011459 const struct device_driver *driver;
11460 const struct device *parent;
David S. Miller3019de12011-06-06 16:41:33 -070011461 const char *empty = "";
Arjan van de Ven6579e572008-07-21 13:31:48 -070011462
11463 parent = dev->dev.parent;
Arjan van de Ven6579e572008-07-21 13:31:48 -070011464 if (!parent)
David S. Miller3019de12011-06-06 16:41:33 -070011465 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -070011466
11467 driver = parent->driver;
11468 if (driver && driver->name)
David S. Miller3019de12011-06-06 16:41:33 -070011469 return driver->name;
11470 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -070011471}
11472
Joe Perches6ea754e2014-09-22 11:10:50 -070011473static void __netdev_printk(const char *level, const struct net_device *dev,
11474 struct va_format *vaf)
Joe Perches256df2f2010-06-27 01:02:35 +000011475{
Joe Perchesb004ff42012-09-12 20:12:19 -070011476 if (dev && dev->dev.parent) {
Joe Perches6ea754e2014-09-22 11:10:50 -070011477 dev_printk_emit(level[1] - '0',
11478 dev->dev.parent,
11479 "%s %s %s%s: %pV",
11480 dev_driver_string(dev->dev.parent),
11481 dev_name(dev->dev.parent),
11482 netdev_name(dev), netdev_reg_state(dev),
11483 vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -070011484 } else if (dev) {
Joe Perches6ea754e2014-09-22 11:10:50 -070011485 printk("%s%s%s: %pV",
11486 level, netdev_name(dev), netdev_reg_state(dev), vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -070011487 } else {
Joe Perches6ea754e2014-09-22 11:10:50 -070011488 printk("%s(NULL net_device): %pV", level, vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -070011489 }
Joe Perches256df2f2010-06-27 01:02:35 +000011490}
11491
Joe Perches6ea754e2014-09-22 11:10:50 -070011492void netdev_printk(const char *level, const struct net_device *dev,
11493 const char *format, ...)
Joe Perches256df2f2010-06-27 01:02:35 +000011494{
11495 struct va_format vaf;
11496 va_list args;
Joe Perches256df2f2010-06-27 01:02:35 +000011497
11498 va_start(args, format);
11499
11500 vaf.fmt = format;
11501 vaf.va = &args;
11502
Joe Perches6ea754e2014-09-22 11:10:50 -070011503 __netdev_printk(level, dev, &vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -070011504
Joe Perches256df2f2010-06-27 01:02:35 +000011505 va_end(args);
Joe Perches256df2f2010-06-27 01:02:35 +000011506}
11507EXPORT_SYMBOL(netdev_printk);
11508
11509#define define_netdev_printk_level(func, level) \
Joe Perches6ea754e2014-09-22 11:10:50 -070011510void func(const struct net_device *dev, const char *fmt, ...) \
Joe Perches256df2f2010-06-27 01:02:35 +000011511{ \
Joe Perches256df2f2010-06-27 01:02:35 +000011512 struct va_format vaf; \
11513 va_list args; \
11514 \
11515 va_start(args, fmt); \
11516 \
11517 vaf.fmt = fmt; \
11518 vaf.va = &args; \
11519 \
Joe Perches6ea754e2014-09-22 11:10:50 -070011520 __netdev_printk(level, dev, &vaf); \
Joe Perchesb004ff42012-09-12 20:12:19 -070011521 \
Joe Perches256df2f2010-06-27 01:02:35 +000011522 va_end(args); \
Joe Perches256df2f2010-06-27 01:02:35 +000011523} \
11524EXPORT_SYMBOL(func);
11525
11526define_netdev_printk_level(netdev_emerg, KERN_EMERG);
11527define_netdev_printk_level(netdev_alert, KERN_ALERT);
11528define_netdev_printk_level(netdev_crit, KERN_CRIT);
11529define_netdev_printk_level(netdev_err, KERN_ERR);
11530define_netdev_printk_level(netdev_warn, KERN_WARNING);
11531define_netdev_printk_level(netdev_notice, KERN_NOTICE);
11532define_netdev_printk_level(netdev_info, KERN_INFO);
11533
Pavel Emelyanov46650792007-10-08 20:38:39 -070011534static void __net_exit netdev_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -070011535{
11536 kfree(net->dev_name_head);
11537 kfree(net->dev_index_head);
Vasily Averinee21b18b2017-11-12 22:28:46 +030011538 if (net != &init_net)
11539 WARN_ON_ONCE(!list_empty(&net->dev_base_head));
Eric W. Biederman881d9662007-09-17 11:56:21 -070011540}
11541
Denis V. Lunev022cbae2007-11-13 03:23:50 -080011542static struct pernet_operations __net_initdata netdev_net_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -070011543 .init = netdev_init,
11544 .exit = netdev_exit,
11545};
11546
Pavel Emelyanov46650792007-10-08 20:38:39 -070011547static void __net_exit default_device_exit(struct net *net)
Eric W. Biedermance286d32007-09-12 13:53:49 +020011548{
Eric W. Biedermane008b5f2009-11-29 22:25:30 +000011549 struct net_device *dev, *aux;
Eric W. Biedermance286d32007-09-12 13:53:49 +020011550 /*
Eric W. Biedermane008b5f2009-11-29 22:25:30 +000011551 * Push all migratable network devices back to the
Eric W. Biedermance286d32007-09-12 13:53:49 +020011552 * initial network namespace
11553 */
11554 rtnl_lock();
Eric W. Biedermane008b5f2009-11-29 22:25:30 +000011555 for_each_netdev_safe(net, dev, aux) {
Eric W. Biedermance286d32007-09-12 13:53:49 +020011556 int err;
Pavel Emelyanovaca51392008-05-08 01:24:25 -070011557 char fb_name[IFNAMSIZ];
Eric W. Biedermance286d32007-09-12 13:53:49 +020011558
11559 /* Ignore unmoveable devices (i.e. loopback) */
11560 if (dev->features & NETIF_F_NETNS_LOCAL)
11561 continue;
11562
Eric W. Biedermane008b5f2009-11-29 22:25:30 +000011563 /* Leave virtual devices for the generic cleanup */
Martin Willi3a5ca852021-03-02 13:24:23 +010011564 if (dev->rtnl_link_ops && !dev->rtnl_link_ops->netns_refund)
Eric W. Biedermane008b5f2009-11-29 22:25:30 +000011565 continue;
Eric W. Biedermand0c082c2008-11-05 15:59:38 -080011566
Lucas De Marchi25985ed2011-03-30 22:57:33 -030011567 /* Push remaining network devices to init_net */
Pavel Emelyanovaca51392008-05-08 01:24:25 -070011568 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
Jiri Pirko55b40db2019-07-28 14:56:36 +020011569 if (__dev_get_by_name(&init_net, fb_name))
11570 snprintf(fb_name, IFNAMSIZ, "dev%%d");
Andrei Vagin0854fa82021-04-06 23:40:51 -070011571 err = dev_change_net_namespace(dev, &init_net, fb_name);
Eric W. Biedermance286d32007-09-12 13:53:49 +020011572 if (err) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +000011573 pr_emerg("%s: failed to move %s to init_net: %d\n",
11574 __func__, dev->name, err);
Pavel Emelyanovaca51392008-05-08 01:24:25 -070011575 BUG();
Eric W. Biedermance286d32007-09-12 13:53:49 +020011576 }
11577 }
11578 rtnl_unlock();
11579}
11580
Eric W. Biederman50624c92013-09-23 21:19:49 -070011581static void __net_exit rtnl_lock_unregistering(struct list_head *net_list)
11582{
11583 /* Return with the rtnl_lock held when there are no network
11584 * devices unregistering in any network namespace in net_list.
11585 */
11586 struct net *net;
11587 bool unregistering;
Peter Zijlstraff960a72014-10-29 17:04:56 +010011588 DEFINE_WAIT_FUNC(wait, woken_wake_function);
Eric W. Biederman50624c92013-09-23 21:19:49 -070011589
Peter Zijlstraff960a72014-10-29 17:04:56 +010011590 add_wait_queue(&netdev_unregistering_wq, &wait);
Eric W. Biederman50624c92013-09-23 21:19:49 -070011591 for (;;) {
Eric W. Biederman50624c92013-09-23 21:19:49 -070011592 unregistering = false;
11593 rtnl_lock();
11594 list_for_each_entry(net, net_list, exit_list) {
11595 if (net->dev_unreg_count > 0) {
11596 unregistering = true;
11597 break;
11598 }
11599 }
11600 if (!unregistering)
11601 break;
11602 __rtnl_unlock();
Peter Zijlstraff960a72014-10-29 17:04:56 +010011603
11604 wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
Eric W. Biederman50624c92013-09-23 21:19:49 -070011605 }
Peter Zijlstraff960a72014-10-29 17:04:56 +010011606 remove_wait_queue(&netdev_unregistering_wq, &wait);
Eric W. Biederman50624c92013-09-23 21:19:49 -070011607}
11608
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +000011609static void __net_exit default_device_exit_batch(struct list_head *net_list)
11610{
11611 /* At exit all network devices most be removed from a network
Uwe Kleine-Königb5950762010-11-01 15:38:34 -040011612 * namespace. Do this in the reverse order of registration.
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +000011613 * Do this across as many network namespaces as possible to
11614 * improve batching efficiency.
11615 */
11616 struct net_device *dev;
11617 struct net *net;
11618 LIST_HEAD(dev_kill_list);
11619
Eric W. Biederman50624c92013-09-23 21:19:49 -070011620 /* To prevent network device cleanup code from dereferencing
11621 * loopback devices or network devices that have been freed
11622 * wait here for all pending unregistrations to complete,
11623 * before unregistring the loopback device and allowing the
11624 * network namespace be freed.
11625 *
11626 * The netdev todo list containing all network devices
11627 * unregistrations that happen in default_device_exit_batch
11628 * will run in the rtnl_unlock() at the end of
11629 * default_device_exit_batch.
11630 */
11631 rtnl_lock_unregistering(net_list);
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +000011632 list_for_each_entry(net, net_list, exit_list) {
11633 for_each_netdev_reverse(net, dev) {
Jiri Pirkob0ab2fa2014-06-26 09:58:25 +020011634 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +000011635 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
11636 else
11637 unregister_netdevice_queue(dev, &dev_kill_list);
11638 }
11639 }
11640 unregister_netdevice_many(&dev_kill_list);
11641 rtnl_unlock();
11642}
11643
Denis V. Lunev022cbae2007-11-13 03:23:50 -080011644static struct pernet_operations __net_initdata default_device_ops = {
Eric W. Biedermance286d32007-09-12 13:53:49 +020011645 .exit = default_device_exit,
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +000011646 .exit_batch = default_device_exit_batch,
Eric W. Biedermance286d32007-09-12 13:53:49 +020011647};
11648
Linus Torvalds1da177e2005-04-16 15:20:36 -070011649/*
11650 * Initialize the DEV module. At boot time this walks the device list and
11651 * unhooks any devices that fail to initialise (normally hardware not
11652 * present) and leaves us with a valid list of present and active devices.
11653 *
11654 */
11655
11656/*
11657 * This is called single threaded during boot, so no need
11658 * to take the rtnl semaphore.
11659 */
11660static int __init net_dev_init(void)
11661{
11662 int i, rc = -ENOMEM;
11663
11664 BUG_ON(!dev_boot_phase);
11665
Linus Torvalds1da177e2005-04-16 15:20:36 -070011666 if (dev_proc_init())
11667 goto out;
11668
Eric W. Biederman8b41d182007-09-26 22:02:53 -070011669 if (netdev_kobject_init())
Linus Torvalds1da177e2005-04-16 15:20:36 -070011670 goto out;
11671
11672 INIT_LIST_HEAD(&ptype_all);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +080011673 for (i = 0; i < PTYPE_HASH_SIZE; i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -070011674 INIT_LIST_HEAD(&ptype_base[i]);
11675
Vlad Yasevich62532da2012-11-15 08:49:10 +000011676 INIT_LIST_HEAD(&offload_base);
11677
Eric W. Biederman881d9662007-09-17 11:56:21 -070011678 if (register_pernet_subsys(&netdev_net_ops))
11679 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -070011680
11681 /*
11682 * Initialise the packet receive queues.
11683 */
11684
KAMEZAWA Hiroyuki6f912042006-04-10 22:52:50 -070011685 for_each_possible_cpu(i) {
Eric Dumazet41852492016-08-26 12:50:39 -070011686 struct work_struct *flush = per_cpu_ptr(&flush_works, i);
Eric Dumazete36fa2f2010-04-19 21:17:14 +000011687 struct softnet_data *sd = &per_cpu(softnet_data, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -070011688
Eric Dumazet41852492016-08-26 12:50:39 -070011689 INIT_WORK(flush, flush_backlog);
11690
Eric Dumazete36fa2f2010-04-19 21:17:14 +000011691 skb_queue_head_init(&sd->input_pkt_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -070011692 skb_queue_head_init(&sd->process_queue);
Steffen Klassertf53c7232017-12-20 10:41:36 +010011693#ifdef CONFIG_XFRM_OFFLOAD
11694 skb_queue_head_init(&sd->xfrm_backlog);
11695#endif
Eric Dumazete36fa2f2010-04-19 21:17:14 +000011696 INIT_LIST_HEAD(&sd->poll_list);
Changli Gaoa9cbd582010-04-26 23:06:24 +000011697 sd->output_queue_tailp = &sd->output_queue;
Eric Dumazetdf334542010-03-24 19:13:54 +000011698#ifdef CONFIG_RPS
Peter Zijlstra545b8c82020-06-15 11:29:31 +020011699 INIT_CSD(&sd->csd, rps_trigger_softirq, sd);
Eric Dumazete36fa2f2010-04-19 21:17:14 +000011700 sd->cpu = i;
Tom Herbert1e94d722010-03-18 17:45:44 -070011701#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +000011702
David S. Miller7c4ec742018-07-20 23:37:55 -070011703 init_gro_hash(&sd->backlog);
Eric Dumazete36fa2f2010-04-19 21:17:14 +000011704 sd->backlog.poll = process_backlog;
11705 sd->backlog.weight = weight_p;
Linus Torvalds1da177e2005-04-16 15:20:36 -070011706 }
11707
Linus Torvalds1da177e2005-04-16 15:20:36 -070011708 dev_boot_phase = 0;
11709
Eric W. Biederman505d4f72008-11-07 22:54:20 -080011710 /* The loopback device is special if any other network devices
11711 * is present in a network namespace the loopback device must
11712 * be present. Since we now dynamically allocate and free the
11713 * loopback device ensure this invariant is maintained by
11714 * keeping the loopback device as the first device on the
11715 * list of network devices. Ensuring the loopback devices
11716 * is the first device that appears and the last network device
11717 * that disappears.
11718 */
11719 if (register_pernet_device(&loopback_net_ops))
11720 goto out;
11721
11722 if (register_pernet_device(&default_device_ops))
11723 goto out;
11724
Carlos R. Mafra962cf362008-05-15 11:15:37 -030011725 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
11726 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -070011727
Sebastian Andrzej Siewiorf0bf90d2016-11-03 15:50:04 +010011728 rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead",
11729 NULL, dev_cpu_dead);
11730 WARN_ON(rc < 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -070011731 rc = 0;
11732out:
11733 return rc;
11734}
11735
11736subsys_initcall(net_dev_init);