blob: 6d60149287a1868cd65fcc55a4e29edd7611def3 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
tcharding722c9a02017-02-09 17:56:04 +11002 * NET3 Protocol independent device support routines.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
tcharding722c9a02017-02-09 17:56:04 +110010 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
tcharding722c9a02017-02-09 17:56:04 +110024 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
Linus Torvalds1da177e2005-04-16 15:20:36 -070027 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
tcharding722c9a02017-02-09 17:56:04 +110039 * Alan Cox : Fix ETH_P_ALL echoback lengths.
Linus Torvalds1da177e2005-04-16 15:20:36 -070040 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
tcharding722c9a02017-02-09 17:56:04 +110049 * Alan Cox : Fixed nasty side effect of device close
Linus Torvalds1da177e2005-04-16 15:20:36 -070050 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
tcharding722c9a02017-02-09 17:56:04 +110070 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
Linus Torvalds1da177e2005-04-16 15:20:36 -070072 * - netif_rx() feedback
73 */
74
Linus Torvalds7c0f6ba2016-12-24 11:46:01 -080075#include <linux/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#include <linux/bitops.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080077#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070078#include <linux/cpu.h>
79#include <linux/types.h>
80#include <linux/kernel.h>
stephen hemminger08e98972009-11-10 07:20:34 +000081#include <linux/hash.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090082#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070083#include <linux/sched.h>
Vlastimil Babkaf1083042017-05-08 15:59:53 -070084#include <linux/sched/mm.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080085#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070086#include <linux/string.h>
87#include <linux/mm.h>
88#include <linux/socket.h>
89#include <linux/sockios.h>
90#include <linux/errno.h>
91#include <linux/interrupt.h>
92#include <linux/if_ether.h>
93#include <linux/netdevice.h>
94#include <linux/etherdevice.h>
Ben Hutchings0187bdf2008-06-19 16:15:47 -070095#include <linux/ethtool.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070096#include <linux/notifier.h>
97#include <linux/skbuff.h>
Brenden Blancoa7862b42016-07-19 12:16:48 -070098#include <linux/bpf.h>
David S. Millerb5cdae32017-04-18 15:36:58 -040099#include <linux/bpf_trace.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +0200100#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101#include <net/sock.h>
Eric Dumazet02d62e82015-11-18 06:30:52 -0800102#include <net/busy_poll.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103#include <linux/rtnetlink.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104#include <linux/stat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105#include <net/dst.h>
Pravin B Shelarfc4099f2015-10-22 18:17:16 -0700106#include <net/dst_metadata.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107#include <net/pkt_sched.h>
108#include <net/checksum.h>
Arnd Bergmann44540962009-11-26 06:07:08 +0000109#include <net/xfrm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110#include <linux/highmem.h>
111#include <linux/init.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113#include <linux/netpoll.h>
114#include <linux/rcupdate.h>
115#include <linux/delay.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116#include <net/iw_handler.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117#include <asm/current.h>
Steve Grubb5bdb9882005-12-03 08:39:35 -0500118#include <linux/audit.h>
Chris Leechdb217332006-06-17 21:24:58 -0700119#include <linux/dmaengine.h>
Herbert Xuf6a78bf2006-06-22 02:57:17 -0700120#include <linux/err.h>
David S. Millerc7fa9d12006-08-15 16:34:13 -0700121#include <linux/ctype.h>
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700122#include <linux/if_arp.h>
Ben Hutchings6de329e2008-06-16 17:02:28 -0700123#include <linux/if_vlan.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700124#include <linux/ip.h>
Alexander Duyckad55dca2008-09-20 22:05:50 -0700125#include <net/ip.h>
Simon Horman25cd9ba2014-10-06 05:05:13 -0700126#include <net/mpls.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700127#include <linux/ipv6.h>
128#include <linux/in.h>
David S. Millerb6b2fed2008-07-21 09:48:06 -0700129#include <linux/jhash.h>
130#include <linux/random.h>
David S. Miller9cbc1cb2009-06-15 03:02:23 -0700131#include <trace/events/napi.h>
Koki Sanagicf66ba52010-08-23 18:45:02 +0900132#include <trace/events/net.h>
Koki Sanagi07dc22e2010-08-23 18:46:12 +0900133#include <trace/events/skb.h>
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +0000134#include <linux/pci.h>
Stephen Rothwellcaeda9b2010-09-16 21:39:16 -0700135#include <linux/inetdevice.h>
Ben Hutchingsc4454772011-01-19 11:03:53 +0000136#include <linux/cpu_rmap.h>
Ingo Molnarc5905af2012-02-24 08:31:31 +0100137#include <linux/static_key.h>
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300138#include <linux/hashtable.h>
Eric Dumazet60877a32013-06-20 01:15:51 -0700139#include <linux/vmalloc.h>
Michal Kubeček529d0482013-11-15 06:18:50 +0100140#include <linux/if_macvlan.h>
Willem de Bruijne7fd2882014-08-04 22:11:48 -0400141#include <linux/errqueue.h>
Eric Dumazet3b47d302014-11-06 21:09:44 -0800142#include <linux/hrtimer.h>
Pablo Neirae687ad62015-05-13 18:19:38 +0200143#include <linux/netfilter_ingress.h>
Hariprasad Shenai40e4e712016-06-08 18:09:08 +0530144#include <linux/crash_dump.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700145
Pavel Emelyanov342709e2007-10-23 21:14:45 -0700146#include "net-sysfs.h"
147
Herbert Xud565b0a2008-12-15 23:38:52 -0800148/* Instead of increasing this, you should create a hash table. */
149#define MAX_GRO_SKBS 8
150
Herbert Xu5d38a072009-01-04 16:13:40 -0800151/* This should be increased if a protocol with a bigger head is added. */
152#define GRO_MAX_HEAD (MAX_HEADER + 128)
153
Linus Torvalds1da177e2005-04-16 15:20:36 -0700154static DEFINE_SPINLOCK(ptype_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000155static DEFINE_SPINLOCK(offload_lock);
Cong Wang900ff8c2013-02-18 19:20:33 +0000156struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
157struct list_head ptype_all __read_mostly; /* Taps */
Vlad Yasevich62532da2012-11-15 08:49:10 +0000158static struct list_head offload_base __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159
Ben Hutchingsae78dbf2014-01-10 22:17:24 +0000160static int netif_rx_internal(struct sk_buff *skb);
Loic Prylli54951192014-07-01 21:39:43 -0700161static int call_netdevice_notifiers_info(unsigned long val,
162 struct net_device *dev,
163 struct netdev_notifier_info *info);
Ben Hutchingsae78dbf2014-01-10 22:17:24 +0000164
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165/*
Pavel Emelianov7562f872007-05-03 15:13:45 -0700166 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167 * semaphore.
168 *
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800169 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700170 *
171 * Writers must hold the rtnl semaphore while they loop through the
Pavel Emelianov7562f872007-05-03 15:13:45 -0700172 * dev_base_head list, and hold dev_base_lock for writing when they do the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173 * actual updates. This allows pure readers to access the list even
174 * while a writer is preparing to update it.
175 *
176 * To put it another way, dev_base_lock is held for writing only to
177 * protect against pure readers; the rtnl semaphore provides the
178 * protection against other writers.
179 *
180 * See, for example usages, register_netdevice() and
181 * unregister_netdevice(), which must be called with the rtnl
182 * semaphore held.
183 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184DEFINE_RWLOCK(dev_base_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185EXPORT_SYMBOL(dev_base_lock);
186
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300187/* protects napi_hash addition/deletion and napi_gen_id */
188static DEFINE_SPINLOCK(napi_hash_lock);
189
Eric Dumazet52bd2d62015-11-18 06:30:50 -0800190static unsigned int napi_gen_id = NR_CPUS;
Eric Dumazet6180d9d2015-11-18 06:31:01 -0800191static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300192
Thomas Gleixner18afa4b2013-07-23 16:13:17 +0200193static seqcount_t devnet_rename_seq;
Brian Haleyc91f6df2012-11-26 05:21:08 +0000194
Thomas Graf4e985ad2011-06-21 03:11:20 +0000195static inline void dev_base_seq_inc(struct net *net)
196{
tcharding643aa9c2017-02-09 17:56:05 +1100197 while (++net->dev_base_seq == 0)
198 ;
Thomas Graf4e985ad2011-06-21 03:11:20 +0000199}
200
Eric W. Biederman881d9662007-09-17 11:56:21 -0700201static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202{
Linus Torvalds8387ff22016-06-10 07:51:30 -0700203 unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
Eric Dumazet95c96172012-04-15 05:58:06 +0000204
stephen hemminger08e98972009-11-10 07:20:34 +0000205 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700206}
207
Eric W. Biederman881d9662007-09-17 11:56:21 -0700208static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700209{
Eric Dumazet7c28bd02009-10-24 06:13:17 -0700210 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211}
212
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000213static inline void rps_lock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000214{
215#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000216 spin_lock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000217#endif
218}
219
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000220static inline void rps_unlock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000221{
222#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000223 spin_unlock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000224#endif
225}
226
Eric W. Biedermance286d32007-09-12 13:53:49 +0200227/* Device list insertion */
dingtianhong53759be2013-04-17 22:17:50 +0000228static void list_netdevice(struct net_device *dev)
Eric W. Biedermance286d32007-09-12 13:53:49 +0200229{
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900230 struct net *net = dev_net(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200231
232 ASSERT_RTNL();
233
234 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800235 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
Eric Dumazet72c95282009-10-30 07:11:27 +0000236 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000237 hlist_add_head_rcu(&dev->index_hlist,
238 dev_index_hash(net, dev->ifindex));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200239 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000240
241 dev_base_seq_inc(net);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200242}
243
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000244/* Device list removal
245 * caller must respect a RCU grace period before freeing/reusing dev
246 */
Eric W. Biedermance286d32007-09-12 13:53:49 +0200247static void unlist_netdevice(struct net_device *dev)
248{
249 ASSERT_RTNL();
250
251 /* Unlink dev from the device chain */
252 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800253 list_del_rcu(&dev->dev_list);
Eric Dumazet72c95282009-10-30 07:11:27 +0000254 hlist_del_rcu(&dev->name_hlist);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000255 hlist_del_rcu(&dev->index_hlist);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200256 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000257
258 dev_base_seq_inc(dev_net(dev));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200259}
260
Linus Torvalds1da177e2005-04-16 15:20:36 -0700261/*
262 * Our notifier list
263 */
264
Alan Sternf07d5b92006-05-09 15:23:03 -0700265static RAW_NOTIFIER_HEAD(netdev_chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266
267/*
268 * Device drivers call our routines to queue packets here. We empty the
269 * queue in the local softnet handler.
270 */
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700271
Eric Dumazet9958da02010-04-17 04:17:02 +0000272DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700273EXPORT_PER_CPU_SYMBOL(softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700274
David S. Millercf508b12008-07-22 14:16:42 -0700275#ifdef CONFIG_LOCKDEP
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700276/*
David S. Millerc773e842008-07-08 23:13:53 -0700277 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700278 * according to dev->type
279 */
tcharding643aa9c2017-02-09 17:56:05 +1100280static const unsigned short netdev_lock_type[] = {
281 ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700282 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
283 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
284 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
285 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
286 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
287 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
288 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
289 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
290 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
291 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
292 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
Paul Gortmaker211ed862012-05-10 17:14:35 -0400293 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
294 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
295 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700296
tcharding643aa9c2017-02-09 17:56:05 +1100297static const char *const netdev_lock_name[] = {
298 "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
299 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
300 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
301 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
302 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
303 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
304 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
305 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
306 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
307 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
308 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
309 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
310 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
311 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
312 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700313
314static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
David S. Millercf508b12008-07-22 14:16:42 -0700315static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700316
317static inline unsigned short netdev_lock_pos(unsigned short dev_type)
318{
319 int i;
320
321 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
322 if (netdev_lock_type[i] == dev_type)
323 return i;
324 /* the last key is used by default */
325 return ARRAY_SIZE(netdev_lock_type) - 1;
326}
327
David S. Millercf508b12008-07-22 14:16:42 -0700328static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
329 unsigned short dev_type)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700330{
331 int i;
332
333 i = netdev_lock_pos(dev_type);
334 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
335 netdev_lock_name[i]);
336}
David S. Millercf508b12008-07-22 14:16:42 -0700337
338static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
339{
340 int i;
341
342 i = netdev_lock_pos(dev->type);
343 lockdep_set_class_and_name(&dev->addr_list_lock,
344 &netdev_addr_lock_key[i],
345 netdev_lock_name[i]);
346}
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700347#else
David S. Millercf508b12008-07-22 14:16:42 -0700348static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
349 unsigned short dev_type)
350{
351}
352static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700353{
354}
355#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700356
357/*******************************************************************************
tchardingeb13da12017-02-09 17:56:06 +1100358 *
359 * Protocol management and registration routines
360 *
361 *******************************************************************************/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362
Linus Torvalds1da177e2005-04-16 15:20:36 -0700363
364/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700365 * Add a protocol ID to the list. Now that the input handler is
366 * smarter we can dispense with all the messy stuff that used to be
367 * here.
368 *
369 * BEWARE!!! Protocol handlers, mangling input packets,
370 * MUST BE last in hash buckets and checking protocol handlers
371 * MUST start from promiscuous ptype_all chain in net_bh.
372 * It is true now, do not change it.
373 * Explanation follows: if protocol handler, mangling packet, will
374 * be the first on list, it is not able to sense, that packet
375 * is cloned and should be copied-on-write, so that it will
376 * change it and subsequent readers will get broken packet.
377 * --ANK (980803)
378 */
379
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000380static inline struct list_head *ptype_head(const struct packet_type *pt)
381{
382 if (pt->type == htons(ETH_P_ALL))
Salam Noureddine7866a622015-01-27 11:35:48 -0800383 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000384 else
Salam Noureddine7866a622015-01-27 11:35:48 -0800385 return pt->dev ? &pt->dev->ptype_specific :
386 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000387}
388
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389/**
390 * dev_add_pack - add packet handler
391 * @pt: packet type declaration
392 *
393 * Add a protocol handler to the networking stack. The passed &packet_type
394 * is linked into kernel lists and may not be freed until it has been
395 * removed from the kernel lists.
396 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900397 * This call does not sleep therefore it can not
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398 * guarantee all CPU's that are in middle of receiving packets
399 * will see the new packet type (until the next received packet).
400 */
401
402void dev_add_pack(struct packet_type *pt)
403{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000404 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700405
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000406 spin_lock(&ptype_lock);
407 list_add_rcu(&pt->list, head);
408 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700410EXPORT_SYMBOL(dev_add_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700411
Linus Torvalds1da177e2005-04-16 15:20:36 -0700412/**
413 * __dev_remove_pack - remove packet handler
414 * @pt: packet type declaration
415 *
416 * Remove a protocol handler that was previously added to the kernel
417 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
418 * from the kernel lists and can be freed or reused once this function
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900419 * returns.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420 *
421 * The packet type might still be in use by receivers
422 * and must not be freed until after all the CPU's have gone
423 * through a quiescent state.
424 */
425void __dev_remove_pack(struct packet_type *pt)
426{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000427 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700428 struct packet_type *pt1;
429
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000430 spin_lock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700431
432 list_for_each_entry(pt1, head, list) {
433 if (pt == pt1) {
434 list_del_rcu(&pt->list);
435 goto out;
436 }
437 }
438
Joe Perches7b6cd1c2012-02-01 10:54:43 +0000439 pr_warn("dev_remove_pack: %p not found\n", pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700440out:
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000441 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700442}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700443EXPORT_SYMBOL(__dev_remove_pack);
444
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445/**
446 * dev_remove_pack - remove packet handler
447 * @pt: packet type declaration
448 *
449 * Remove a protocol handler that was previously added to the kernel
450 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
451 * from the kernel lists and can be freed or reused once this function
452 * returns.
453 *
454 * This call sleeps to guarantee that no CPU is looking at the packet
455 * type after return.
456 */
457void dev_remove_pack(struct packet_type *pt)
458{
459 __dev_remove_pack(pt);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900460
Linus Torvalds1da177e2005-04-16 15:20:36 -0700461 synchronize_net();
462}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700463EXPORT_SYMBOL(dev_remove_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700464
Vlad Yasevich62532da2012-11-15 08:49:10 +0000465
466/**
467 * dev_add_offload - register offload handlers
468 * @po: protocol offload declaration
469 *
470 * Add protocol offload handlers to the networking stack. The passed
471 * &proto_offload is linked into kernel lists and may not be freed until
472 * it has been removed from the kernel lists.
473 *
474 * This call does not sleep therefore it can not
475 * guarantee all CPU's that are in middle of receiving packets
476 * will see the new offload handlers (until the next received packet).
477 */
478void dev_add_offload(struct packet_offload *po)
479{
David S. Millerbdef7de2015-06-01 14:56:09 -0700480 struct packet_offload *elem;
Vlad Yasevich62532da2012-11-15 08:49:10 +0000481
482 spin_lock(&offload_lock);
David S. Millerbdef7de2015-06-01 14:56:09 -0700483 list_for_each_entry(elem, &offload_base, list) {
484 if (po->priority < elem->priority)
485 break;
486 }
487 list_add_rcu(&po->list, elem->list.prev);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000488 spin_unlock(&offload_lock);
489}
490EXPORT_SYMBOL(dev_add_offload);
491
492/**
493 * __dev_remove_offload - remove offload handler
494 * @po: packet offload declaration
495 *
496 * Remove a protocol offload handler that was previously added to the
497 * kernel offload handlers by dev_add_offload(). The passed &offload_type
498 * is removed from the kernel lists and can be freed or reused once this
499 * function returns.
500 *
501 * The packet type might still be in use by receivers
502 * and must not be freed until after all the CPU's have gone
503 * through a quiescent state.
504 */
stephen hemminger1d143d92013-12-29 14:01:29 -0800505static void __dev_remove_offload(struct packet_offload *po)
Vlad Yasevich62532da2012-11-15 08:49:10 +0000506{
507 struct list_head *head = &offload_base;
508 struct packet_offload *po1;
509
Eric Dumazetc53aa502012-11-16 08:08:23 +0000510 spin_lock(&offload_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000511
512 list_for_each_entry(po1, head, list) {
513 if (po == po1) {
514 list_del_rcu(&po->list);
515 goto out;
516 }
517 }
518
519 pr_warn("dev_remove_offload: %p not found\n", po);
520out:
Eric Dumazetc53aa502012-11-16 08:08:23 +0000521 spin_unlock(&offload_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000522}
Vlad Yasevich62532da2012-11-15 08:49:10 +0000523
524/**
525 * dev_remove_offload - remove packet offload handler
526 * @po: packet offload declaration
527 *
528 * Remove a packet offload handler that was previously added to the kernel
529 * offload handlers by dev_add_offload(). The passed &offload_type is
530 * removed from the kernel lists and can be freed or reused once this
531 * function returns.
532 *
533 * This call sleeps to guarantee that no CPU is looking at the packet
534 * type after return.
535 */
536void dev_remove_offload(struct packet_offload *po)
537{
538 __dev_remove_offload(po);
539
540 synchronize_net();
541}
542EXPORT_SYMBOL(dev_remove_offload);
543
Linus Torvalds1da177e2005-04-16 15:20:36 -0700544/******************************************************************************
tchardingeb13da12017-02-09 17:56:06 +1100545 *
546 * Device Boot-time Settings Routines
547 *
548 ******************************************************************************/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700549
550/* Boot time configuration table */
551static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
552
553/**
554 * netdev_boot_setup_add - add new setup entry
555 * @name: name of the device
556 * @map: configured settings for the device
557 *
558 * Adds new setup entry to the dev_boot_setup list. The function
559 * returns 0 on error and 1 on success. This is a generic routine to
560 * all netdevices.
561 */
562static int netdev_boot_setup_add(char *name, struct ifmap *map)
563{
564 struct netdev_boot_setup *s;
565 int i;
566
567 s = dev_boot_setup;
568 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
569 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
570 memset(s[i].name, 0, sizeof(s[i].name));
Wang Chen93b3cff2008-07-01 19:57:19 -0700571 strlcpy(s[i].name, name, IFNAMSIZ);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572 memcpy(&s[i].map, map, sizeof(s[i].map));
573 break;
574 }
575 }
576
577 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
578}
579
580/**
tcharding722c9a02017-02-09 17:56:04 +1100581 * netdev_boot_setup_check - check boot time settings
582 * @dev: the netdevice
Linus Torvalds1da177e2005-04-16 15:20:36 -0700583 *
tcharding722c9a02017-02-09 17:56:04 +1100584 * Check boot time settings for the device.
585 * The found settings are set for the device to be used
586 * later in the device probing.
587 * Returns 0 if no settings found, 1 if they are.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700588 */
589int netdev_boot_setup_check(struct net_device *dev)
590{
591 struct netdev_boot_setup *s = dev_boot_setup;
592 int i;
593
594 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
595 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
Wang Chen93b3cff2008-07-01 19:57:19 -0700596 !strcmp(dev->name, s[i].name)) {
tcharding722c9a02017-02-09 17:56:04 +1100597 dev->irq = s[i].map.irq;
598 dev->base_addr = s[i].map.base_addr;
599 dev->mem_start = s[i].map.mem_start;
600 dev->mem_end = s[i].map.mem_end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700601 return 1;
602 }
603 }
604 return 0;
605}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700606EXPORT_SYMBOL(netdev_boot_setup_check);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700607
608
609/**
tcharding722c9a02017-02-09 17:56:04 +1100610 * netdev_boot_base - get address from boot time settings
611 * @prefix: prefix for network device
612 * @unit: id for network device
Linus Torvalds1da177e2005-04-16 15:20:36 -0700613 *
tcharding722c9a02017-02-09 17:56:04 +1100614 * Check boot time settings for the base address of device.
615 * The found settings are set for the device to be used
616 * later in the device probing.
617 * Returns 0 if no settings found.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700618 */
619unsigned long netdev_boot_base(const char *prefix, int unit)
620{
621 const struct netdev_boot_setup *s = dev_boot_setup;
622 char name[IFNAMSIZ];
623 int i;
624
625 sprintf(name, "%s%d", prefix, unit);
626
627 /*
628 * If device already registered then return base of 1
629 * to indicate not to probe for this interface
630 */
Eric W. Biederman881d9662007-09-17 11:56:21 -0700631 if (__dev_get_by_name(&init_net, name))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700632 return 1;
633
634 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
635 if (!strcmp(name, s[i].name))
636 return s[i].map.base_addr;
637 return 0;
638}
639
640/*
641 * Saves at boot time configured settings for any netdevice.
642 */
643int __init netdev_boot_setup(char *str)
644{
645 int ints[5];
646 struct ifmap map;
647
648 str = get_options(str, ARRAY_SIZE(ints), ints);
649 if (!str || !*str)
650 return 0;
651
652 /* Save settings */
653 memset(&map, 0, sizeof(map));
654 if (ints[0] > 0)
655 map.irq = ints[1];
656 if (ints[0] > 1)
657 map.base_addr = ints[2];
658 if (ints[0] > 2)
659 map.mem_start = ints[3];
660 if (ints[0] > 3)
661 map.mem_end = ints[4];
662
663 /* Add new entry to the list */
664 return netdev_boot_setup_add(str, &map);
665}
666
667__setup("netdev=", netdev_boot_setup);
668
669/*******************************************************************************
tchardingeb13da12017-02-09 17:56:06 +1100670 *
671 * Device Interface Subroutines
672 *
673 *******************************************************************************/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700674
675/**
Nicolas Dichtela54acb32015-04-02 17:07:00 +0200676 * dev_get_iflink - get 'iflink' value of a interface
677 * @dev: targeted interface
678 *
679 * Indicates the ifindex the interface is linked to.
680 * Physical interfaces have the same 'ifindex' and 'iflink' values.
681 */
682
683int dev_get_iflink(const struct net_device *dev)
684{
685 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
686 return dev->netdev_ops->ndo_get_iflink(dev);
687
Nicolas Dichtel7a66bbc2015-04-02 17:07:09 +0200688 return dev->ifindex;
Nicolas Dichtela54acb32015-04-02 17:07:00 +0200689}
690EXPORT_SYMBOL(dev_get_iflink);
691
692/**
Pravin B Shelarfc4099f2015-10-22 18:17:16 -0700693 * dev_fill_metadata_dst - Retrieve tunnel egress information.
694 * @dev: targeted interface
695 * @skb: The packet.
696 *
697 * For better visibility of tunnel traffic OVS needs to retrieve
698 * egress tunnel information for a packet. Following API allows
699 * user to get this info.
700 */
701int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
702{
703 struct ip_tunnel_info *info;
704
705 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst)
706 return -EINVAL;
707
708 info = skb_tunnel_info_unclone(skb);
709 if (!info)
710 return -ENOMEM;
711 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
712 return -EINVAL;
713
714 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
715}
716EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
717
718/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700719 * __dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700720 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700721 * @name: name to find
722 *
723 * Find an interface by name. Must be called under RTNL semaphore
724 * or @dev_base_lock. If the name is found a pointer to the device
725 * is returned. If the name is not found then %NULL is returned. The
726 * reference counters are not incremented so the caller must be
727 * careful with locks.
728 */
729
Eric W. Biederman881d9662007-09-17 11:56:21 -0700730struct net_device *__dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700731{
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700732 struct net_device *dev;
733 struct hlist_head *head = dev_name_hash(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700734
Sasha Levinb67bfe02013-02-27 17:06:00 -0800735 hlist_for_each_entry(dev, head, name_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736 if (!strncmp(dev->name, name, IFNAMSIZ))
737 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700738
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739 return NULL;
740}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700741EXPORT_SYMBOL(__dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700742
743/**
tcharding722c9a02017-02-09 17:56:04 +1100744 * dev_get_by_name_rcu - find a device by its name
745 * @net: the applicable net namespace
746 * @name: name to find
Eric Dumazet72c95282009-10-30 07:11:27 +0000747 *
tcharding722c9a02017-02-09 17:56:04 +1100748 * Find an interface by name.
749 * If the name is found a pointer to the device is returned.
750 * If the name is not found then %NULL is returned.
751 * The reference counters are not incremented so the caller must be
752 * careful with locks. The caller must hold RCU lock.
Eric Dumazet72c95282009-10-30 07:11:27 +0000753 */
754
755struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
756{
Eric Dumazet72c95282009-10-30 07:11:27 +0000757 struct net_device *dev;
758 struct hlist_head *head = dev_name_hash(net, name);
759
Sasha Levinb67bfe02013-02-27 17:06:00 -0800760 hlist_for_each_entry_rcu(dev, head, name_hlist)
Eric Dumazet72c95282009-10-30 07:11:27 +0000761 if (!strncmp(dev->name, name, IFNAMSIZ))
762 return dev;
763
764 return NULL;
765}
766EXPORT_SYMBOL(dev_get_by_name_rcu);
767
768/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700769 * dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700770 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700771 * @name: name to find
772 *
773 * Find an interface by name. This can be called from any
774 * context and does its own locking. The returned handle has
775 * the usage count incremented and the caller must use dev_put() to
776 * release it when it is no longer needed. %NULL is returned if no
777 * matching device is found.
778 */
779
Eric W. Biederman881d9662007-09-17 11:56:21 -0700780struct net_device *dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700781{
782 struct net_device *dev;
783
Eric Dumazet72c95282009-10-30 07:11:27 +0000784 rcu_read_lock();
785 dev = dev_get_by_name_rcu(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700786 if (dev)
787 dev_hold(dev);
Eric Dumazet72c95282009-10-30 07:11:27 +0000788 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700789 return dev;
790}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700791EXPORT_SYMBOL(dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700792
793/**
794 * __dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700795 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700796 * @ifindex: index of device
797 *
798 * Search for an interface by index. Returns %NULL if the device
799 * is not found or a pointer to the device. The device has not
800 * had its reference counter increased so the caller must be careful
801 * about locking. The caller must hold either the RTNL semaphore
802 * or @dev_base_lock.
803 */
804
Eric W. Biederman881d9662007-09-17 11:56:21 -0700805struct net_device *__dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700806{
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700807 struct net_device *dev;
808 struct hlist_head *head = dev_index_hash(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700809
Sasha Levinb67bfe02013-02-27 17:06:00 -0800810 hlist_for_each_entry(dev, head, index_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700811 if (dev->ifindex == ifindex)
812 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700813
Linus Torvalds1da177e2005-04-16 15:20:36 -0700814 return NULL;
815}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700816EXPORT_SYMBOL(__dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700817
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000818/**
819 * dev_get_by_index_rcu - find a device by its ifindex
820 * @net: the applicable net namespace
821 * @ifindex: index of device
822 *
823 * Search for an interface by index. Returns %NULL if the device
824 * is not found or a pointer to the device. The device has not
825 * had its reference counter increased so the caller must be careful
826 * about locking. The caller must hold RCU lock.
827 */
828
829struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
830{
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000831 struct net_device *dev;
832 struct hlist_head *head = dev_index_hash(net, ifindex);
833
Sasha Levinb67bfe02013-02-27 17:06:00 -0800834 hlist_for_each_entry_rcu(dev, head, index_hlist)
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000835 if (dev->ifindex == ifindex)
836 return dev;
837
838 return NULL;
839}
840EXPORT_SYMBOL(dev_get_by_index_rcu);
841
Linus Torvalds1da177e2005-04-16 15:20:36 -0700842
843/**
844 * dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700845 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700846 * @ifindex: index of device
847 *
848 * Search for an interface by index. Returns NULL if the device
849 * is not found or a pointer to the device. The device returned has
850 * had a reference added and the pointer is safe until the user calls
851 * dev_put to indicate they have finished with it.
852 */
853
Eric W. Biederman881d9662007-09-17 11:56:21 -0700854struct net_device *dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700855{
856 struct net_device *dev;
857
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000858 rcu_read_lock();
859 dev = dev_get_by_index_rcu(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700860 if (dev)
861 dev_hold(dev);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000862 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700863 return dev;
864}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700865EXPORT_SYMBOL(dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700866
867/**
Nicolas Schichan5dbe7c12013-06-26 17:23:42 +0200868 * netdev_get_name - get a netdevice name, knowing its ifindex.
869 * @net: network namespace
870 * @name: a pointer to the buffer where the name will be stored.
871 * @ifindex: the ifindex of the interface to get the name from.
872 *
873 * The use of raw_seqcount_begin() and cond_resched() before
874 * retrying is required as we want to give the writers a chance
875 * to complete when CONFIG_PREEMPT is not set.
876 */
877int netdev_get_name(struct net *net, char *name, int ifindex)
878{
879 struct net_device *dev;
880 unsigned int seq;
881
882retry:
883 seq = raw_seqcount_begin(&devnet_rename_seq);
884 rcu_read_lock();
885 dev = dev_get_by_index_rcu(net, ifindex);
886 if (!dev) {
887 rcu_read_unlock();
888 return -ENODEV;
889 }
890
891 strcpy(name, dev->name);
892 rcu_read_unlock();
893 if (read_seqcount_retry(&devnet_rename_seq, seq)) {
894 cond_resched();
895 goto retry;
896 }
897
898 return 0;
899}
900
901/**
Eric Dumazet941666c2010-12-05 01:23:53 +0000902 * dev_getbyhwaddr_rcu - find a device by its hardware address
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700903 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700904 * @type: media type of device
905 * @ha: hardware address
906 *
907 * Search for an interface by MAC address. Returns NULL if the device
Eric Dumazetc5066532011-01-24 13:16:16 -0800908 * is not found or a pointer to the device.
909 * The caller must hold RCU or RTNL.
Eric Dumazet941666c2010-12-05 01:23:53 +0000910 * The returned device has not had its ref count increased
Linus Torvalds1da177e2005-04-16 15:20:36 -0700911 * and the caller must therefore be careful about locking
912 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700913 */
914
Eric Dumazet941666c2010-12-05 01:23:53 +0000915struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
916 const char *ha)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700917{
918 struct net_device *dev;
919
Eric Dumazet941666c2010-12-05 01:23:53 +0000920 for_each_netdev_rcu(net, dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700921 if (dev->type == type &&
922 !memcmp(dev->dev_addr, ha, dev->addr_len))
Pavel Emelianov7562f872007-05-03 15:13:45 -0700923 return dev;
924
925 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700926}
Eric Dumazet941666c2010-12-05 01:23:53 +0000927EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
Jochen Friedrichcf309e32005-09-22 04:44:55 -0300928
Eric W. Biederman881d9662007-09-17 11:56:21 -0700929struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700930{
931 struct net_device *dev;
932
933 ASSERT_RTNL();
Eric W. Biederman881d9662007-09-17 11:56:21 -0700934 for_each_netdev(net, dev)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700935 if (dev->type == type)
Pavel Emelianov7562f872007-05-03 15:13:45 -0700936 return dev;
937
938 return NULL;
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700939}
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700940EXPORT_SYMBOL(__dev_getfirstbyhwtype);
941
Eric W. Biederman881d9662007-09-17 11:56:21 -0700942struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700943{
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000944 struct net_device *dev, *ret = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700945
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000946 rcu_read_lock();
947 for_each_netdev_rcu(net, dev)
948 if (dev->type == type) {
949 dev_hold(dev);
950 ret = dev;
951 break;
952 }
953 rcu_read_unlock();
954 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700955}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700956EXPORT_SYMBOL(dev_getfirstbyhwtype);
957
958/**
WANG Cong6c555492014-09-11 15:35:09 -0700959 * __dev_get_by_flags - find any device with given flags
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700960 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700961 * @if_flags: IFF_* values
962 * @mask: bitmask of bits in if_flags to check
963 *
964 * Search for any interface with the given flags. Returns NULL if a device
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000965 * is not found or a pointer to the device. Must be called inside
WANG Cong6c555492014-09-11 15:35:09 -0700966 * rtnl_lock(), and result refcount is unchanged.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700967 */
968
WANG Cong6c555492014-09-11 15:35:09 -0700969struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
970 unsigned short mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700971{
Pavel Emelianov7562f872007-05-03 15:13:45 -0700972 struct net_device *dev, *ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700973
WANG Cong6c555492014-09-11 15:35:09 -0700974 ASSERT_RTNL();
975
Pavel Emelianov7562f872007-05-03 15:13:45 -0700976 ret = NULL;
WANG Cong6c555492014-09-11 15:35:09 -0700977 for_each_netdev(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700978 if (((dev->flags ^ if_flags) & mask) == 0) {
Pavel Emelianov7562f872007-05-03 15:13:45 -0700979 ret = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700980 break;
981 }
982 }
Pavel Emelianov7562f872007-05-03 15:13:45 -0700983 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700984}
WANG Cong6c555492014-09-11 15:35:09 -0700985EXPORT_SYMBOL(__dev_get_by_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700986
987/**
988 * dev_valid_name - check if name is okay for network device
989 * @name: name string
990 *
991 * Network device names need to be valid file names to
David S. Millerc7fa9d12006-08-15 16:34:13 -0700992 * to allow sysfs to work. We also disallow any kind of
993 * whitespace.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700994 */
David S. Miller95f050b2012-03-06 16:12:15 -0500995bool dev_valid_name(const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700996{
David S. Millerc7fa9d12006-08-15 16:34:13 -0700997 if (*name == '\0')
David S. Miller95f050b2012-03-06 16:12:15 -0500998 return false;
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -0700999 if (strlen(name) >= IFNAMSIZ)
David S. Miller95f050b2012-03-06 16:12:15 -05001000 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -07001001 if (!strcmp(name, ".") || !strcmp(name, ".."))
David S. Miller95f050b2012-03-06 16:12:15 -05001002 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -07001003
1004 while (*name) {
Matthew Thodea4176a92015-02-17 18:31:57 -06001005 if (*name == '/' || *name == ':' || isspace(*name))
David S. Miller95f050b2012-03-06 16:12:15 -05001006 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -07001007 name++;
1008 }
David S. Miller95f050b2012-03-06 16:12:15 -05001009 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001010}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001011EXPORT_SYMBOL(dev_valid_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001012
1013/**
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001014 * __dev_alloc_name - allocate a name for a device
1015 * @net: network namespace to allocate the device name in
Linus Torvalds1da177e2005-04-16 15:20:36 -07001016 * @name: name format string
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001017 * @buf: scratch buffer and result name string
Linus Torvalds1da177e2005-04-16 15:20:36 -07001018 *
1019 * Passed a format string - eg "lt%d" it will try and find a suitable
Stephen Hemminger3041a062006-05-26 13:25:24 -07001020 * id. It scans list of devices to build up a free map, then chooses
1021 * the first empty slot. The caller must hold the dev_base or rtnl lock
1022 * while allocating the name and adding the device in order to avoid
1023 * duplicates.
1024 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1025 * Returns the number of the unit assigned or a negative errno code.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001026 */
1027
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001028static int __dev_alloc_name(struct net *net, const char *name, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001029{
1030 int i = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001031 const char *p;
1032 const int max_netdevices = 8*PAGE_SIZE;
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -07001033 unsigned long *inuse;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001034 struct net_device *d;
1035
1036 p = strnchr(name, IFNAMSIZ-1, '%');
1037 if (p) {
1038 /*
1039 * Verify the string as this thing may have come from
1040 * the user. There must be either one "%d" and no other "%"
1041 * characters.
1042 */
1043 if (p[1] != 'd' || strchr(p + 2, '%'))
1044 return -EINVAL;
1045
1046 /* Use one page as a bit array of possible slots */
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -07001047 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001048 if (!inuse)
1049 return -ENOMEM;
1050
Eric W. Biederman881d9662007-09-17 11:56:21 -07001051 for_each_netdev(net, d) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001052 if (!sscanf(d->name, name, &i))
1053 continue;
1054 if (i < 0 || i >= max_netdevices)
1055 continue;
1056
1057 /* avoid cases where sscanf is not exact inverse of printf */
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001058 snprintf(buf, IFNAMSIZ, name, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001059 if (!strncmp(buf, d->name, IFNAMSIZ))
1060 set_bit(i, inuse);
1061 }
1062
1063 i = find_first_zero_bit(inuse, max_netdevices);
1064 free_page((unsigned long) inuse);
1065 }
1066
Octavian Purdilad9031022009-11-18 02:36:59 +00001067 if (buf != name)
1068 snprintf(buf, IFNAMSIZ, name, i);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001069 if (!__dev_get_by_name(net, buf))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001070 return i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001071
1072 /* It is possible to run out of possible slots
1073 * when the name is long and there isn't enough space left
1074 * for the digits, or if all bits are used.
1075 */
1076 return -ENFILE;
1077}
1078
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001079/**
1080 * dev_alloc_name - allocate a name for a device
1081 * @dev: device
1082 * @name: name format string
1083 *
1084 * Passed a format string - eg "lt%d" it will try and find a suitable
1085 * id. It scans list of devices to build up a free map, then chooses
1086 * the first empty slot. The caller must hold the dev_base or rtnl lock
1087 * while allocating the name and adding the device in order to avoid
1088 * duplicates.
1089 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1090 * Returns the number of the unit assigned or a negative errno code.
1091 */
1092
1093int dev_alloc_name(struct net_device *dev, const char *name)
1094{
1095 char buf[IFNAMSIZ];
1096 struct net *net;
1097 int ret;
1098
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001099 BUG_ON(!dev_net(dev));
1100 net = dev_net(dev);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001101 ret = __dev_alloc_name(net, name, buf);
1102 if (ret >= 0)
1103 strlcpy(dev->name, buf, IFNAMSIZ);
1104 return ret;
1105}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001106EXPORT_SYMBOL(dev_alloc_name);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001107
Gao feng828de4f2012-09-13 20:58:27 +00001108static int dev_alloc_name_ns(struct net *net,
1109 struct net_device *dev,
1110 const char *name)
Octavian Purdilad9031022009-11-18 02:36:59 +00001111{
Gao feng828de4f2012-09-13 20:58:27 +00001112 char buf[IFNAMSIZ];
1113 int ret;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001114
Gao feng828de4f2012-09-13 20:58:27 +00001115 ret = __dev_alloc_name(net, name, buf);
1116 if (ret >= 0)
1117 strlcpy(dev->name, buf, IFNAMSIZ);
1118 return ret;
1119}
1120
1121static int dev_get_valid_name(struct net *net,
1122 struct net_device *dev,
1123 const char *name)
1124{
1125 BUG_ON(!net);
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001126
Octavian Purdilad9031022009-11-18 02:36:59 +00001127 if (!dev_valid_name(name))
1128 return -EINVAL;
1129
Jiri Pirko1c5cae82011-04-30 01:21:32 +00001130 if (strchr(name, '%'))
Gao feng828de4f2012-09-13 20:58:27 +00001131 return dev_alloc_name_ns(net, dev, name);
Octavian Purdilad9031022009-11-18 02:36:59 +00001132 else if (__dev_get_by_name(net, name))
1133 return -EEXIST;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001134 else if (dev->name != name)
1135 strlcpy(dev->name, name, IFNAMSIZ);
Octavian Purdilad9031022009-11-18 02:36:59 +00001136
1137 return 0;
1138}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001139
1140/**
1141 * dev_change_name - change name of a device
1142 * @dev: device
1143 * @newname: name (or format string) must be at least IFNAMSIZ
1144 *
1145 * Change name of a device, can pass format strings "eth%d".
1146 * for wildcarding.
1147 */
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07001148int dev_change_name(struct net_device *dev, const char *newname)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001149{
Tom Gundersen238fa362014-07-14 16:37:23 +02001150 unsigned char old_assign_type;
Herbert Xufcc5a032007-07-30 17:03:38 -07001151 char oldname[IFNAMSIZ];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001152 int err = 0;
Herbert Xufcc5a032007-07-30 17:03:38 -07001153 int ret;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001154 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001155
1156 ASSERT_RTNL();
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001157 BUG_ON(!dev_net(dev));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001158
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001159 net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001160 if (dev->flags & IFF_UP)
1161 return -EBUSY;
1162
Eric Dumazet30e6c9fa2012-12-20 17:25:08 +00001163 write_seqcount_begin(&devnet_rename_seq);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001164
1165 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
Eric Dumazet30e6c9fa2012-12-20 17:25:08 +00001166 write_seqcount_end(&devnet_rename_seq);
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001167 return 0;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001168 }
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001169
Herbert Xufcc5a032007-07-30 17:03:38 -07001170 memcpy(oldname, dev->name, IFNAMSIZ);
1171
Gao feng828de4f2012-09-13 20:58:27 +00001172 err = dev_get_valid_name(net, dev, newname);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001173 if (err < 0) {
Eric Dumazet30e6c9fa2012-12-20 17:25:08 +00001174 write_seqcount_end(&devnet_rename_seq);
Octavian Purdilad9031022009-11-18 02:36:59 +00001175 return err;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001176 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001177
Veaceslav Falico6fe82a32014-07-17 20:33:32 +02001178 if (oldname[0] && !strchr(oldname, '%'))
1179 netdev_info(dev, "renamed from %s\n", oldname);
1180
Tom Gundersen238fa362014-07-14 16:37:23 +02001181 old_assign_type = dev->name_assign_type;
1182 dev->name_assign_type = NET_NAME_RENAMED;
1183
Herbert Xufcc5a032007-07-30 17:03:38 -07001184rollback:
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001185 ret = device_rename(&dev->dev, dev->name);
1186 if (ret) {
1187 memcpy(dev->name, oldname, IFNAMSIZ);
Tom Gundersen238fa362014-07-14 16:37:23 +02001188 dev->name_assign_type = old_assign_type;
Eric Dumazet30e6c9fa2012-12-20 17:25:08 +00001189 write_seqcount_end(&devnet_rename_seq);
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001190 return ret;
Stephen Hemmingerdcc99772008-05-14 22:33:38 -07001191 }
Herbert Xu7f988ea2007-07-30 16:35:46 -07001192
Eric Dumazet30e6c9fa2012-12-20 17:25:08 +00001193 write_seqcount_end(&devnet_rename_seq);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001194
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01001195 netdev_adjacent_rename_links(dev, oldname);
1196
Herbert Xu7f988ea2007-07-30 16:35:46 -07001197 write_lock_bh(&dev_base_lock);
Eric Dumazet372b2312011-05-17 13:56:59 -04001198 hlist_del_rcu(&dev->name_hlist);
Eric Dumazet72c95282009-10-30 07:11:27 +00001199 write_unlock_bh(&dev_base_lock);
1200
1201 synchronize_rcu();
1202
1203 write_lock_bh(&dev_base_lock);
1204 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Herbert Xu7f988ea2007-07-30 16:35:46 -07001205 write_unlock_bh(&dev_base_lock);
1206
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001207 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001208 ret = notifier_to_errno(ret);
1209
1210 if (ret) {
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001211 /* err >= 0 after dev_alloc_name() or stores the first errno */
1212 if (err >= 0) {
Herbert Xufcc5a032007-07-30 17:03:38 -07001213 err = ret;
Eric Dumazet30e6c9fa2012-12-20 17:25:08 +00001214 write_seqcount_begin(&devnet_rename_seq);
Herbert Xufcc5a032007-07-30 17:03:38 -07001215 memcpy(dev->name, oldname, IFNAMSIZ);
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01001216 memcpy(oldname, newname, IFNAMSIZ);
Tom Gundersen238fa362014-07-14 16:37:23 +02001217 dev->name_assign_type = old_assign_type;
1218 old_assign_type = NET_NAME_RENAMED;
Herbert Xufcc5a032007-07-30 17:03:38 -07001219 goto rollback;
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001220 } else {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001221 pr_err("%s: name change rollback failed: %d\n",
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001222 dev->name, ret);
Herbert Xufcc5a032007-07-30 17:03:38 -07001223 }
1224 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001225
1226 return err;
1227}
1228
1229/**
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001230 * dev_set_alias - change ifalias of a device
1231 * @dev: device
1232 * @alias: name up to IFALIASZ
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07001233 * @len: limit of bytes to copy from info
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001234 *
1235 * Set ifalias for a device,
1236 */
1237int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1238{
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001239 char *new_ifalias;
1240
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001241 ASSERT_RTNL();
1242
1243 if (len >= IFALIASZ)
1244 return -EINVAL;
1245
Oliver Hartkopp96ca4a2c2008-09-23 21:23:19 -07001246 if (!len) {
Sachin Kamat388dfc22012-11-20 00:57:04 +00001247 kfree(dev->ifalias);
1248 dev->ifalias = NULL;
Oliver Hartkopp96ca4a2c2008-09-23 21:23:19 -07001249 return 0;
1250 }
1251
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001252 new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1253 if (!new_ifalias)
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001254 return -ENOMEM;
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001255 dev->ifalias = new_ifalias;
Alexander Potapenkoc28294b2017-06-06 15:56:54 +02001256 memcpy(dev->ifalias, alias, len);
1257 dev->ifalias[len] = 0;
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001258
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001259 return len;
1260}
1261
1262
1263/**
Stephen Hemminger3041a062006-05-26 13:25:24 -07001264 * netdev_features_change - device changes features
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001265 * @dev: device to cause notification
1266 *
1267 * Called to indicate a device has changed features.
1268 */
1269void netdev_features_change(struct net_device *dev)
1270{
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001271 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001272}
1273EXPORT_SYMBOL(netdev_features_change);
1274
1275/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001276 * netdev_state_change - device changes state
1277 * @dev: device to cause notification
1278 *
1279 * Called to indicate a device has changed state. This function calls
1280 * the notifier chains for netdev_chain and sends a NEWLINK message
1281 * to the routing socket.
1282 */
1283void netdev_state_change(struct net_device *dev)
1284{
1285 if (dev->flags & IFF_UP) {
Loic Prylli54951192014-07-01 21:39:43 -07001286 struct netdev_notifier_change_info change_info;
1287
1288 change_info.flags_changed = 0;
1289 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
1290 &change_info.info);
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001291 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001292 }
1293}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001294EXPORT_SYMBOL(netdev_state_change);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001295
Amerigo Wangee89bab2012-08-09 22:14:56 +00001296/**
tcharding722c9a02017-02-09 17:56:04 +11001297 * netdev_notify_peers - notify network peers about existence of @dev
1298 * @dev: network device
Amerigo Wangee89bab2012-08-09 22:14:56 +00001299 *
1300 * Generate traffic such that interested network peers are aware of
1301 * @dev, such as by generating a gratuitous ARP. This may be used when
1302 * a device wants to inform the rest of the network about some sort of
1303 * reconfiguration such as a failover event or virtual machine
1304 * migration.
1305 */
1306void netdev_notify_peers(struct net_device *dev)
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001307{
Amerigo Wangee89bab2012-08-09 22:14:56 +00001308 rtnl_lock();
1309 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
Vlad Yasevich37c343b2017-03-14 08:58:08 -04001310 call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
Amerigo Wangee89bab2012-08-09 22:14:56 +00001311 rtnl_unlock();
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001312}
Amerigo Wangee89bab2012-08-09 22:14:56 +00001313EXPORT_SYMBOL(netdev_notify_peers);
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001314
Patrick McHardybd380812010-02-26 06:34:53 +00001315static int __dev_open(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001316{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001317 const struct net_device_ops *ops = dev->netdev_ops;
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001318 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001319
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001320 ASSERT_RTNL();
1321
Linus Torvalds1da177e2005-04-16 15:20:36 -07001322 if (!netif_device_present(dev))
1323 return -ENODEV;
1324
Neil Hormanca99ca12013-02-05 08:05:43 +00001325 /* Block netpoll from trying to do any rx path servicing.
1326 * If we don't do this there is a chance ndo_poll_controller
1327 * or ndo_poll may be running while we open the device
1328 */
Eric W. Biederman66b55522014-03-27 15:39:03 -07001329 netpoll_poll_disable(dev);
Neil Hormanca99ca12013-02-05 08:05:43 +00001330
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001331 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1332 ret = notifier_to_errno(ret);
1333 if (ret)
1334 return ret;
1335
Linus Torvalds1da177e2005-04-16 15:20:36 -07001336 set_bit(__LINK_STATE_START, &dev->state);
Jeff Garzikbada3392007-10-23 20:19:37 -07001337
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001338 if (ops->ndo_validate_addr)
1339 ret = ops->ndo_validate_addr(dev);
Jeff Garzikbada3392007-10-23 20:19:37 -07001340
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001341 if (!ret && ops->ndo_open)
1342 ret = ops->ndo_open(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001343
Eric W. Biederman66b55522014-03-27 15:39:03 -07001344 netpoll_poll_enable(dev);
Neil Hormanca99ca12013-02-05 08:05:43 +00001345
Jeff Garzikbada3392007-10-23 20:19:37 -07001346 if (ret)
1347 clear_bit(__LINK_STATE_START, &dev->state);
1348 else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001349 dev->flags |= IFF_UP;
Patrick McHardy4417da62007-06-27 01:28:10 -07001350 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001351 dev_activate(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04001352 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001353 }
Jeff Garzikbada3392007-10-23 20:19:37 -07001354
Linus Torvalds1da177e2005-04-16 15:20:36 -07001355 return ret;
1356}
Patrick McHardybd380812010-02-26 06:34:53 +00001357
1358/**
1359 * dev_open - prepare an interface for use.
1360 * @dev: device to open
1361 *
1362 * Takes a device from down to up state. The device's private open
1363 * function is invoked and then the multicast lists are loaded. Finally
1364 * the device is moved into the up state and a %NETDEV_UP message is
1365 * sent to the netdev notifier chain.
1366 *
1367 * Calling this function on an active interface is a nop. On a failure
1368 * a negative errno code is returned.
1369 */
1370int dev_open(struct net_device *dev)
1371{
1372 int ret;
1373
Patrick McHardybd380812010-02-26 06:34:53 +00001374 if (dev->flags & IFF_UP)
1375 return 0;
1376
Patrick McHardybd380812010-02-26 06:34:53 +00001377 ret = __dev_open(dev);
1378 if (ret < 0)
1379 return ret;
1380
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001381 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
Patrick McHardybd380812010-02-26 06:34:53 +00001382 call_netdevice_notifiers(NETDEV_UP, dev);
1383
1384 return ret;
1385}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001386EXPORT_SYMBOL(dev_open);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001387
Octavian Purdila44345722010-12-13 12:44:07 +00001388static int __dev_close_many(struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001389{
Octavian Purdila44345722010-12-13 12:44:07 +00001390 struct net_device *dev;
Patrick McHardybd380812010-02-26 06:34:53 +00001391
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001392 ASSERT_RTNL();
David S. Miller9d5010d2007-09-12 14:33:25 +02001393 might_sleep();
1394
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001395 list_for_each_entry(dev, head, close_list) {
Eric W. Biederman3f4df202014-03-27 15:38:17 -07001396 /* Temporarily disable netpoll until the interface is down */
Eric W. Biederman66b55522014-03-27 15:39:03 -07001397 netpoll_poll_disable(dev);
Eric W. Biederman3f4df202014-03-27 15:38:17 -07001398
Octavian Purdila44345722010-12-13 12:44:07 +00001399 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001400
Octavian Purdila44345722010-12-13 12:44:07 +00001401 clear_bit(__LINK_STATE_START, &dev->state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001402
Octavian Purdila44345722010-12-13 12:44:07 +00001403 /* Synchronize to scheduled poll. We cannot touch poll list, it
1404 * can be even on different cpu. So just clear netif_running().
1405 *
1406 * dev->stop() will invoke napi_disable() on all of it's
1407 * napi_struct instances on this device.
1408 */
Peter Zijlstra4e857c52014-03-17 18:06:10 +01001409 smp_mb__after_atomic(); /* Commit netif_running(). */
Octavian Purdila44345722010-12-13 12:44:07 +00001410 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001411
Octavian Purdila44345722010-12-13 12:44:07 +00001412 dev_deactivate_many(head);
1413
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001414 list_for_each_entry(dev, head, close_list) {
Octavian Purdila44345722010-12-13 12:44:07 +00001415 const struct net_device_ops *ops = dev->netdev_ops;
1416
1417 /*
1418 * Call the device specific close. This cannot fail.
1419 * Only if device is UP
1420 *
1421 * We allow it to be called even after a DETACH hot-plug
1422 * event.
1423 */
1424 if (ops->ndo_stop)
1425 ops->ndo_stop(dev);
1426
Octavian Purdila44345722010-12-13 12:44:07 +00001427 dev->flags &= ~IFF_UP;
Eric W. Biederman66b55522014-03-27 15:39:03 -07001428 netpoll_poll_enable(dev);
Octavian Purdila44345722010-12-13 12:44:07 +00001429 }
1430
1431 return 0;
1432}
1433
1434static int __dev_close(struct net_device *dev)
1435{
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001436 int retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001437 LIST_HEAD(single);
1438
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001439 list_add(&dev->close_list, &single);
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001440 retval = __dev_close_many(&single);
1441 list_del(&single);
Neil Hormanca99ca12013-02-05 08:05:43 +00001442
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001443 return retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001444}
1445
David S. Miller99c4a262015-03-18 22:52:33 -04001446int dev_close_many(struct list_head *head, bool unlink)
Octavian Purdila44345722010-12-13 12:44:07 +00001447{
1448 struct net_device *dev, *tmp;
Octavian Purdila44345722010-12-13 12:44:07 +00001449
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001450 /* Remove the devices that don't need to be closed */
1451 list_for_each_entry_safe(dev, tmp, head, close_list)
Octavian Purdila44345722010-12-13 12:44:07 +00001452 if (!(dev->flags & IFF_UP))
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001453 list_del_init(&dev->close_list);
Octavian Purdila44345722010-12-13 12:44:07 +00001454
1455 __dev_close_many(head);
Matti Linnanvuorid8b2a4d2008-02-12 23:10:11 -08001456
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001457 list_for_each_entry_safe(dev, tmp, head, close_list) {
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001458 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
Octavian Purdila44345722010-12-13 12:44:07 +00001459 call_netdevice_notifiers(NETDEV_DOWN, dev);
David S. Miller99c4a262015-03-18 22:52:33 -04001460 if (unlink)
1461 list_del_init(&dev->close_list);
Octavian Purdila44345722010-12-13 12:44:07 +00001462 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001463
Linus Torvalds1da177e2005-04-16 15:20:36 -07001464 return 0;
1465}
David S. Miller99c4a262015-03-18 22:52:33 -04001466EXPORT_SYMBOL(dev_close_many);
Patrick McHardybd380812010-02-26 06:34:53 +00001467
1468/**
1469 * dev_close - shutdown an interface.
1470 * @dev: device to shutdown
1471 *
1472 * This function moves an active device into down state. A
1473 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1474 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1475 * chain.
1476 */
1477int dev_close(struct net_device *dev)
1478{
Eric Dumazete14a5992011-05-10 12:26:06 -07001479 if (dev->flags & IFF_UP) {
1480 LIST_HEAD(single);
Patrick McHardybd380812010-02-26 06:34:53 +00001481
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001482 list_add(&dev->close_list, &single);
David S. Miller99c4a262015-03-18 22:52:33 -04001483 dev_close_many(&single, true);
Eric Dumazete14a5992011-05-10 12:26:06 -07001484 list_del(&single);
1485 }
dingtianhongda6e3782013-05-27 19:53:31 +00001486 return 0;
Patrick McHardybd380812010-02-26 06:34:53 +00001487}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001488EXPORT_SYMBOL(dev_close);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001489
1490
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001491/**
1492 * dev_disable_lro - disable Large Receive Offload on a device
1493 * @dev: device
1494 *
1495 * Disable Large Receive Offload (LRO) on a net device. Must be
1496 * called under RTNL. This is needed if received packets may be
1497 * forwarded to another interface.
1498 */
1499void dev_disable_lro(struct net_device *dev)
1500{
Michal Kubečekfbe168b2014-11-13 07:54:50 +01001501 struct net_device *lower_dev;
1502 struct list_head *iter;
Michal Kubeček529d0482013-11-15 06:18:50 +01001503
Michał Mirosławbc5787c62011-11-15 15:29:55 +00001504 dev->wanted_features &= ~NETIF_F_LRO;
1505 netdev_update_features(dev);
Michał Mirosław27660512011-03-18 16:56:34 +00001506
Michał Mirosław22d59692011-04-21 12:42:15 +00001507 if (unlikely(dev->features & NETIF_F_LRO))
1508 netdev_WARN(dev, "failed to disable LRO!\n");
Michal Kubečekfbe168b2014-11-13 07:54:50 +01001509
1510 netdev_for_each_lower_dev(dev, lower_dev, iter)
1511 dev_disable_lro(lower_dev);
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001512}
1513EXPORT_SYMBOL(dev_disable_lro);
1514
Jiri Pirko351638e2013-05-28 01:30:21 +00001515static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1516 struct net_device *dev)
1517{
1518 struct netdev_notifier_info info;
1519
1520 netdev_notifier_info_init(&info, dev);
1521 return nb->notifier_call(nb, val, &info);
1522}
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001523
Eric W. Biederman881d9662007-09-17 11:56:21 -07001524static int dev_boot_phase = 1;
1525
Linus Torvalds1da177e2005-04-16 15:20:36 -07001526/**
tcharding722c9a02017-02-09 17:56:04 +11001527 * register_netdevice_notifier - register a network notifier block
1528 * @nb: notifier
Linus Torvalds1da177e2005-04-16 15:20:36 -07001529 *
tcharding722c9a02017-02-09 17:56:04 +11001530 * Register a notifier to be called when network device events occur.
1531 * The notifier passed is linked into the kernel structures and must
1532 * not be reused until it has been unregistered. A negative errno code
1533 * is returned on a failure.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001534 *
tcharding722c9a02017-02-09 17:56:04 +11001535 * When registered all registration and up events are replayed
1536 * to the new notifier to allow device to have a race free
1537 * view of the network device list.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001538 */
1539
1540int register_netdevice_notifier(struct notifier_block *nb)
1541{
1542 struct net_device *dev;
Herbert Xufcc5a032007-07-30 17:03:38 -07001543 struct net_device *last;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001544 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001545 int err;
1546
1547 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001548 err = raw_notifier_chain_register(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001549 if (err)
1550 goto unlock;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001551 if (dev_boot_phase)
1552 goto unlock;
1553 for_each_net(net) {
1554 for_each_netdev(net, dev) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001555 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001556 err = notifier_to_errno(err);
1557 if (err)
1558 goto rollback;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001559
Eric W. Biederman881d9662007-09-17 11:56:21 -07001560 if (!(dev->flags & IFF_UP))
1561 continue;
Herbert Xufcc5a032007-07-30 17:03:38 -07001562
Jiri Pirko351638e2013-05-28 01:30:21 +00001563 call_netdevice_notifier(nb, NETDEV_UP, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001564 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001565 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001566
1567unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001568 rtnl_unlock();
1569 return err;
Herbert Xufcc5a032007-07-30 17:03:38 -07001570
1571rollback:
1572 last = dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001573 for_each_net(net) {
1574 for_each_netdev(net, dev) {
1575 if (dev == last)
RongQing.Li8f891482011-11-30 23:43:07 -05001576 goto outroll;
Herbert Xufcc5a032007-07-30 17:03:38 -07001577
Eric W. Biederman881d9662007-09-17 11:56:21 -07001578 if (dev->flags & IFF_UP) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001579 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1580 dev);
1581 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001582 }
Jiri Pirko351638e2013-05-28 01:30:21 +00001583 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001584 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001585 }
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001586
RongQing.Li8f891482011-11-30 23:43:07 -05001587outroll:
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001588 raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001589 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001590}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001591EXPORT_SYMBOL(register_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001592
1593/**
tcharding722c9a02017-02-09 17:56:04 +11001594 * unregister_netdevice_notifier - unregister a network notifier block
1595 * @nb: notifier
Linus Torvalds1da177e2005-04-16 15:20:36 -07001596 *
tcharding722c9a02017-02-09 17:56:04 +11001597 * Unregister a notifier previously registered by
1598 * register_netdevice_notifier(). The notifier is unlinked into the
1599 * kernel structures and may then be reused. A negative errno code
1600 * is returned on a failure.
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001601 *
tcharding722c9a02017-02-09 17:56:04 +11001602 * After unregistering unregister and down device events are synthesized
1603 * for all devices on the device list to the removed notifier to remove
1604 * the need for special case cleanup code.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001605 */
1606
1607int unregister_netdevice_notifier(struct notifier_block *nb)
1608{
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001609 struct net_device *dev;
1610 struct net *net;
Herbert Xu9f514952006-03-25 01:24:25 -08001611 int err;
1612
1613 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001614 err = raw_notifier_chain_unregister(&netdev_chain, nb);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001615 if (err)
1616 goto unlock;
1617
1618 for_each_net(net) {
1619 for_each_netdev(net, dev) {
1620 if (dev->flags & IFF_UP) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001621 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1622 dev);
1623 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001624 }
Jiri Pirko351638e2013-05-28 01:30:21 +00001625 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001626 }
1627 }
1628unlock:
Herbert Xu9f514952006-03-25 01:24:25 -08001629 rtnl_unlock();
1630 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001631}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001632EXPORT_SYMBOL(unregister_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001633
1634/**
Jiri Pirko351638e2013-05-28 01:30:21 +00001635 * call_netdevice_notifiers_info - call all network notifier blocks
1636 * @val: value passed unmodified to notifier function
1637 * @dev: net_device pointer passed unmodified to notifier function
1638 * @info: notifier information data
1639 *
1640 * Call all network notifier blocks. Parameters and return value
1641 * are as for raw_notifier_call_chain().
1642 */
1643
stephen hemminger1d143d92013-12-29 14:01:29 -08001644static int call_netdevice_notifiers_info(unsigned long val,
1645 struct net_device *dev,
1646 struct netdev_notifier_info *info)
Jiri Pirko351638e2013-05-28 01:30:21 +00001647{
1648 ASSERT_RTNL();
1649 netdev_notifier_info_init(info, dev);
1650 return raw_notifier_call_chain(&netdev_chain, val, info);
1651}
Jiri Pirko351638e2013-05-28 01:30:21 +00001652
1653/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001654 * call_netdevice_notifiers - call all network notifier blocks
1655 * @val: value passed unmodified to notifier function
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001656 * @dev: net_device pointer passed unmodified to notifier function
Linus Torvalds1da177e2005-04-16 15:20:36 -07001657 *
1658 * Call all network notifier blocks. Parameters and return value
Alan Sternf07d5b92006-05-09 15:23:03 -07001659 * are as for raw_notifier_call_chain().
Linus Torvalds1da177e2005-04-16 15:20:36 -07001660 */
1661
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001662int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001663{
Jiri Pirko351638e2013-05-28 01:30:21 +00001664 struct netdev_notifier_info info;
1665
1666 return call_netdevice_notifiers_info(val, dev, &info);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001667}
stephen hemmingeredf947f2011-03-24 13:24:01 +00001668EXPORT_SYMBOL(call_netdevice_notifiers);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001669
Pablo Neira1cf519002015-05-13 18:19:37 +02001670#ifdef CONFIG_NET_INGRESS
Daniel Borkmann45771392015-04-10 23:07:54 +02001671static struct static_key ingress_needed __read_mostly;
1672
1673void net_inc_ingress_queue(void)
1674{
1675 static_key_slow_inc(&ingress_needed);
1676}
1677EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
1678
1679void net_dec_ingress_queue(void)
1680{
1681 static_key_slow_dec(&ingress_needed);
1682}
1683EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
1684#endif
1685
Daniel Borkmann1f211a12016-01-07 22:29:47 +01001686#ifdef CONFIG_NET_EGRESS
1687static struct static_key egress_needed __read_mostly;
1688
1689void net_inc_egress_queue(void)
1690{
1691 static_key_slow_inc(&egress_needed);
1692}
1693EXPORT_SYMBOL_GPL(net_inc_egress_queue);
1694
1695void net_dec_egress_queue(void)
1696{
1697 static_key_slow_dec(&egress_needed);
1698}
1699EXPORT_SYMBOL_GPL(net_dec_egress_queue);
1700#endif
1701
Ingo Molnarc5905af2012-02-24 08:31:31 +01001702static struct static_key netstamp_needed __read_mostly;
Eric Dumazetb90e5792011-11-28 11:16:50 +00001703#ifdef HAVE_JUMP_LABEL
Eric Dumazetb90e5792011-11-28 11:16:50 +00001704static atomic_t netstamp_needed_deferred;
Eric Dumazet13baa002017-03-01 14:28:39 -08001705static atomic_t netstamp_wanted;
Eric Dumazet5fa8bbd2017-02-02 10:31:35 -08001706static void netstamp_clear(struct work_struct *work)
1707{
1708 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
Eric Dumazet13baa002017-03-01 14:28:39 -08001709 int wanted;
Eric Dumazet5fa8bbd2017-02-02 10:31:35 -08001710
Eric Dumazet13baa002017-03-01 14:28:39 -08001711 wanted = atomic_add_return(deferred, &netstamp_wanted);
1712 if (wanted > 0)
1713 static_key_enable(&netstamp_needed);
1714 else
1715 static_key_disable(&netstamp_needed);
Eric Dumazet5fa8bbd2017-02-02 10:31:35 -08001716}
1717static DECLARE_WORK(netstamp_work, netstamp_clear);
Eric Dumazetb90e5792011-11-28 11:16:50 +00001718#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001719
1720void net_enable_timestamp(void)
1721{
Eric Dumazet13baa002017-03-01 14:28:39 -08001722#ifdef HAVE_JUMP_LABEL
1723 int wanted;
1724
1725 while (1) {
1726 wanted = atomic_read(&netstamp_wanted);
1727 if (wanted <= 0)
1728 break;
1729 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted)
1730 return;
1731 }
1732 atomic_inc(&netstamp_needed_deferred);
1733 schedule_work(&netstamp_work);
1734#else
Ingo Molnarc5905af2012-02-24 08:31:31 +01001735 static_key_slow_inc(&netstamp_needed);
Eric Dumazet13baa002017-03-01 14:28:39 -08001736#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001737}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001738EXPORT_SYMBOL(net_enable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001739
1740void net_disable_timestamp(void)
1741{
Eric Dumazetb90e5792011-11-28 11:16:50 +00001742#ifdef HAVE_JUMP_LABEL
Eric Dumazet13baa002017-03-01 14:28:39 -08001743 int wanted;
1744
1745 while (1) {
1746 wanted = atomic_read(&netstamp_wanted);
1747 if (wanted <= 1)
1748 break;
1749 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted)
1750 return;
1751 }
1752 atomic_dec(&netstamp_needed_deferred);
Eric Dumazet5fa8bbd2017-02-02 10:31:35 -08001753 schedule_work(&netstamp_work);
1754#else
Ingo Molnarc5905af2012-02-24 08:31:31 +01001755 static_key_slow_dec(&netstamp_needed);
Eric Dumazet5fa8bbd2017-02-02 10:31:35 -08001756#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001757}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001758EXPORT_SYMBOL(net_disable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001759
Eric Dumazet3b098e22010-05-15 23:57:10 -07001760static inline void net_timestamp_set(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001761{
Thomas Gleixner2456e852016-12-25 11:38:40 +01001762 skb->tstamp = 0;
Ingo Molnarc5905af2012-02-24 08:31:31 +01001763 if (static_key_false(&netstamp_needed))
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001764 __net_timestamp(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001765}
1766
Eric Dumazet588f0332011-11-15 04:12:55 +00001767#define net_timestamp_check(COND, SKB) \
Ingo Molnarc5905af2012-02-24 08:31:31 +01001768 if (static_key_false(&netstamp_needed)) { \
Thomas Gleixner2456e852016-12-25 11:38:40 +01001769 if ((COND) && !(SKB)->tstamp) \
Eric Dumazet588f0332011-11-15 04:12:55 +00001770 __net_timestamp(SKB); \
1771 } \
Eric Dumazet3b098e22010-05-15 23:57:10 -07001772
Nikolay Aleksandrovf4b05d22016-04-28 17:59:28 +02001773bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001774{
1775 unsigned int len;
1776
1777 if (!(dev->flags & IFF_UP))
1778 return false;
1779
1780 len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1781 if (skb->len <= len)
1782 return true;
1783
1784 /* if TSO is enabled, we don't care about the length as the packet
1785 * could be forwarded without being segmented before
1786 */
1787 if (skb_is_gso(skb))
1788 return true;
1789
1790 return false;
1791}
Vlad Yasevich1ee481f2014-03-27 17:32:29 -04001792EXPORT_SYMBOL_GPL(is_skb_forwardable);
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001793
Herbert Xua0265d22014-04-17 13:45:03 +08001794int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1795{
Martin KaFai Lau4e3264d2016-11-09 15:36:33 -08001796 int ret = ____dev_forward_skb(dev, skb);
1797
1798 if (likely(!ret)) {
1799 skb->protocol = eth_type_trans(skb, dev);
1800 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
Herbert Xua0265d22014-04-17 13:45:03 +08001801 }
1802
Martin KaFai Lau4e3264d2016-11-09 15:36:33 -08001803 return ret;
Herbert Xua0265d22014-04-17 13:45:03 +08001804}
1805EXPORT_SYMBOL_GPL(__dev_forward_skb);
1806
Arnd Bergmann44540962009-11-26 06:07:08 +00001807/**
1808 * dev_forward_skb - loopback an skb to another netif
1809 *
1810 * @dev: destination network device
1811 * @skb: buffer to forward
1812 *
1813 * return values:
1814 * NET_RX_SUCCESS (no congestion)
Eric Dumazet6ec82562010-05-06 00:53:53 -07001815 * NET_RX_DROP (packet was dropped, but freed)
Arnd Bergmann44540962009-11-26 06:07:08 +00001816 *
1817 * dev_forward_skb can be used for injecting an skb from the
1818 * start_xmit function of one device into the receive queue
1819 * of another device.
1820 *
1821 * The receiving device may be in another namespace, so
1822 * we have to clear all information in the skb that could
1823 * impact namespace isolation.
1824 */
1825int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1826{
Herbert Xua0265d22014-04-17 13:45:03 +08001827 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001828}
1829EXPORT_SYMBOL_GPL(dev_forward_skb);
1830
Changli Gao71d9dec2010-12-15 19:57:25 +00001831static inline int deliver_skb(struct sk_buff *skb,
1832 struct packet_type *pt_prev,
1833 struct net_device *orig_dev)
1834{
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00001835 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
1836 return -ENOMEM;
Changli Gao71d9dec2010-12-15 19:57:25 +00001837 atomic_inc(&skb->users);
1838 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1839}
1840
Salam Noureddine7866a622015-01-27 11:35:48 -08001841static inline void deliver_ptype_list_skb(struct sk_buff *skb,
1842 struct packet_type **pt,
Jiri Pirkofbcb2172015-03-30 16:56:01 +02001843 struct net_device *orig_dev,
1844 __be16 type,
Salam Noureddine7866a622015-01-27 11:35:48 -08001845 struct list_head *ptype_list)
1846{
1847 struct packet_type *ptype, *pt_prev = *pt;
1848
1849 list_for_each_entry_rcu(ptype, ptype_list, list) {
1850 if (ptype->type != type)
1851 continue;
1852 if (pt_prev)
Jiri Pirkofbcb2172015-03-30 16:56:01 +02001853 deliver_skb(skb, pt_prev, orig_dev);
Salam Noureddine7866a622015-01-27 11:35:48 -08001854 pt_prev = ptype;
1855 }
1856 *pt = pt_prev;
1857}
1858
Eric Leblondc0de08d2012-08-16 22:02:58 +00001859static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1860{
Eric Leblonda3d744e2012-11-06 02:10:10 +00001861 if (!ptype->af_packet_priv || !skb->sk)
Eric Leblondc0de08d2012-08-16 22:02:58 +00001862 return false;
1863
1864 if (ptype->id_match)
1865 return ptype->id_match(ptype, skb->sk);
1866 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1867 return true;
1868
1869 return false;
1870}
1871
Linus Torvalds1da177e2005-04-16 15:20:36 -07001872/*
1873 * Support routine. Sends outgoing frames to any network
1874 * taps currently in use.
1875 */
1876
David Ahern74b20582016-05-10 11:19:50 -07001877void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001878{
1879 struct packet_type *ptype;
Changli Gao71d9dec2010-12-15 19:57:25 +00001880 struct sk_buff *skb2 = NULL;
1881 struct packet_type *pt_prev = NULL;
Salam Noureddine7866a622015-01-27 11:35:48 -08001882 struct list_head *ptype_list = &ptype_all;
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001883
Linus Torvalds1da177e2005-04-16 15:20:36 -07001884 rcu_read_lock();
Salam Noureddine7866a622015-01-27 11:35:48 -08001885again:
1886 list_for_each_entry_rcu(ptype, ptype_list, list) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001887 /* Never send packets back to the socket
1888 * they originated from - MvS (miquels@drinkel.ow.org)
1889 */
Salam Noureddine7866a622015-01-27 11:35:48 -08001890 if (skb_loop_sk(ptype, skb))
1891 continue;
Changli Gao71d9dec2010-12-15 19:57:25 +00001892
Salam Noureddine7866a622015-01-27 11:35:48 -08001893 if (pt_prev) {
1894 deliver_skb(skb2, pt_prev, skb->dev);
Changli Gao71d9dec2010-12-15 19:57:25 +00001895 pt_prev = ptype;
Salam Noureddine7866a622015-01-27 11:35:48 -08001896 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001897 }
Salam Noureddine7866a622015-01-27 11:35:48 -08001898
1899 /* need to clone skb, done only once */
1900 skb2 = skb_clone(skb, GFP_ATOMIC);
1901 if (!skb2)
1902 goto out_unlock;
1903
1904 net_timestamp_set(skb2);
1905
1906 /* skb->nh should be correctly
1907 * set by sender, so that the second statement is
1908 * just protection against buggy protocols.
1909 */
1910 skb_reset_mac_header(skb2);
1911
1912 if (skb_network_header(skb2) < skb2->data ||
1913 skb_network_header(skb2) > skb_tail_pointer(skb2)) {
1914 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1915 ntohs(skb2->protocol),
1916 dev->name);
1917 skb_reset_network_header(skb2);
1918 }
1919
1920 skb2->transport_header = skb2->network_header;
1921 skb2->pkt_type = PACKET_OUTGOING;
1922 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001923 }
Salam Noureddine7866a622015-01-27 11:35:48 -08001924
1925 if (ptype_list == &ptype_all) {
1926 ptype_list = &dev->ptype_all;
1927 goto again;
1928 }
1929out_unlock:
Changli Gao71d9dec2010-12-15 19:57:25 +00001930 if (pt_prev)
1931 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001932 rcu_read_unlock();
1933}
David Ahern74b20582016-05-10 11:19:50 -07001934EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001935
Ben Hutchings2c530402012-07-10 10:55:09 +00001936/**
1937 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
John Fastabend4f57c082011-01-17 08:06:04 +00001938 * @dev: Network device
1939 * @txq: number of queues available
1940 *
1941 * If real_num_tx_queues is changed the tc mappings may no longer be
1942 * valid. To resolve this verify the tc mapping remains valid and if
1943 * not NULL the mapping. With no priorities mapping to this
1944 * offset/count pair it will no longer be used. In the worst case TC0
1945 * is invalid nothing can be done so disable priority mappings. If is
1946 * expected that drivers will fix this mapping if they can before
1947 * calling netif_set_real_num_tx_queues.
1948 */
Eric Dumazetbb134d22011-01-20 19:18:08 +00001949static void netif_setup_tc(struct net_device *dev, unsigned int txq)
John Fastabend4f57c082011-01-17 08:06:04 +00001950{
1951 int i;
1952 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1953
1954 /* If TC0 is invalidated disable TC mapping */
1955 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001956 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
John Fastabend4f57c082011-01-17 08:06:04 +00001957 dev->num_tc = 0;
1958 return;
1959 }
1960
1961 /* Invalidated prio to tc mappings set to TC0 */
1962 for (i = 1; i < TC_BITMASK + 1; i++) {
1963 int q = netdev_get_prio_tc_map(dev, i);
1964
1965 tc = &dev->tc_to_txq[q];
1966 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001967 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1968 i, q);
John Fastabend4f57c082011-01-17 08:06:04 +00001969 netdev_set_prio_tc_map(dev, i, 0);
1970 }
1971 }
1972}
1973
Alexander Duyck8d059b02016-10-28 11:43:49 -04001974int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
1975{
1976 if (dev->num_tc) {
1977 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1978 int i;
1979
1980 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
1981 if ((txq - tc->offset) < tc->count)
1982 return i;
1983 }
1984
1985 return -1;
1986 }
1987
1988 return 0;
1989}
1990
Alexander Duyck537c00d2013-01-10 08:57:02 +00001991#ifdef CONFIG_XPS
1992static DEFINE_MUTEX(xps_map_mutex);
1993#define xmap_dereference(P) \
1994 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
1995
Alexander Duyck6234f872016-10-28 11:46:49 -04001996static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
1997 int tci, u16 index)
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001998{
1999 struct xps_map *map = NULL;
2000 int pos;
2001
2002 if (dev_maps)
Alexander Duyck6234f872016-10-28 11:46:49 -04002003 map = xmap_dereference(dev_maps->cpu_map[tci]);
2004 if (!map)
2005 return false;
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00002006
Alexander Duyck6234f872016-10-28 11:46:49 -04002007 for (pos = map->len; pos--;) {
2008 if (map->queues[pos] != index)
2009 continue;
2010
2011 if (map->len > 1) {
2012 map->queues[pos] = map->queues[--map->len];
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00002013 break;
2014 }
Alexander Duyck6234f872016-10-28 11:46:49 -04002015
2016 RCU_INIT_POINTER(dev_maps->cpu_map[tci], NULL);
2017 kfree_rcu(map, rcu);
2018 return false;
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00002019 }
2020
Alexander Duyck6234f872016-10-28 11:46:49 -04002021 return true;
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00002022}
2023
Alexander Duyck6234f872016-10-28 11:46:49 -04002024static bool remove_xps_queue_cpu(struct net_device *dev,
2025 struct xps_dev_maps *dev_maps,
2026 int cpu, u16 offset, u16 count)
2027{
Alexander Duyck184c4492016-10-28 11:50:13 -04002028 int num_tc = dev->num_tc ? : 1;
2029 bool active = false;
2030 int tci;
Alexander Duyck6234f872016-10-28 11:46:49 -04002031
Alexander Duyck184c4492016-10-28 11:50:13 -04002032 for (tci = cpu * num_tc; num_tc--; tci++) {
2033 int i, j;
2034
2035 for (i = count, j = offset; i--; j++) {
2036 if (!remove_xps_queue(dev_maps, cpu, j))
2037 break;
2038 }
2039
2040 active |= i < 0;
Alexander Duyck6234f872016-10-28 11:46:49 -04002041 }
2042
Alexander Duyck184c4492016-10-28 11:50:13 -04002043 return active;
Alexander Duyck6234f872016-10-28 11:46:49 -04002044}
2045
2046static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2047 u16 count)
Alexander Duyck537c00d2013-01-10 08:57:02 +00002048{
2049 struct xps_dev_maps *dev_maps;
Alexander Duyck024e9672013-01-10 08:57:46 +00002050 int cpu, i;
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00002051 bool active = false;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002052
2053 mutex_lock(&xps_map_mutex);
2054 dev_maps = xmap_dereference(dev->xps_maps);
2055
2056 if (!dev_maps)
2057 goto out_no_maps;
2058
Alexander Duyck6234f872016-10-28 11:46:49 -04002059 for_each_possible_cpu(cpu)
2060 active |= remove_xps_queue_cpu(dev, dev_maps, cpu,
2061 offset, count);
Alexander Duyck537c00d2013-01-10 08:57:02 +00002062
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00002063 if (!active) {
Alexander Duyck537c00d2013-01-10 08:57:02 +00002064 RCU_INIT_POINTER(dev->xps_maps, NULL);
2065 kfree_rcu(dev_maps, rcu);
2066 }
2067
Alexander Duyck6234f872016-10-28 11:46:49 -04002068 for (i = offset + (count - 1); count--; i--)
Alexander Duyck024e9672013-01-10 08:57:46 +00002069 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
2070 NUMA_NO_NODE);
2071
Alexander Duyck537c00d2013-01-10 08:57:02 +00002072out_no_maps:
2073 mutex_unlock(&xps_map_mutex);
2074}
2075
Alexander Duyck6234f872016-10-28 11:46:49 -04002076static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2077{
2078 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2079}
2080
Alexander Duyck01c5f862013-01-10 08:57:35 +00002081static struct xps_map *expand_xps_map(struct xps_map *map,
2082 int cpu, u16 index)
2083{
2084 struct xps_map *new_map;
2085 int alloc_len = XPS_MIN_MAP_ALLOC;
2086 int i, pos;
2087
2088 for (pos = 0; map && pos < map->len; pos++) {
2089 if (map->queues[pos] != index)
2090 continue;
2091 return map;
2092 }
2093
2094 /* Need to add queue to this CPU's existing map */
2095 if (map) {
2096 if (pos < map->alloc_len)
2097 return map;
2098
2099 alloc_len = map->alloc_len * 2;
2100 }
2101
2102 /* Need to allocate new map to store queue on this CPU's map */
2103 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2104 cpu_to_node(cpu));
2105 if (!new_map)
2106 return NULL;
2107
2108 for (i = 0; i < pos; i++)
2109 new_map->queues[i] = map->queues[i];
2110 new_map->alloc_len = alloc_len;
2111 new_map->len = pos;
2112
2113 return new_map;
2114}
2115
Michael S. Tsirkin35735402013-10-02 09:14:06 +03002116int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2117 u16 index)
Alexander Duyck537c00d2013-01-10 08:57:02 +00002118{
Alexander Duyck01c5f862013-01-10 08:57:35 +00002119 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
Alexander Duyck184c4492016-10-28 11:50:13 -04002120 int i, cpu, tci, numa_node_id = -2;
2121 int maps_sz, num_tc = 1, tc = 0;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002122 struct xps_map *map, *new_map;
Alexander Duyck01c5f862013-01-10 08:57:35 +00002123 bool active = false;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002124
Alexander Duyck184c4492016-10-28 11:50:13 -04002125 if (dev->num_tc) {
2126 num_tc = dev->num_tc;
2127 tc = netdev_txq_to_tc(dev, index);
2128 if (tc < 0)
2129 return -EINVAL;
2130 }
2131
2132 maps_sz = XPS_DEV_MAPS_SIZE(num_tc);
2133 if (maps_sz < L1_CACHE_BYTES)
2134 maps_sz = L1_CACHE_BYTES;
2135
Alexander Duyck537c00d2013-01-10 08:57:02 +00002136 mutex_lock(&xps_map_mutex);
2137
2138 dev_maps = xmap_dereference(dev->xps_maps);
2139
Alexander Duyck01c5f862013-01-10 08:57:35 +00002140 /* allocate memory for queue storage */
Alexander Duyck184c4492016-10-28 11:50:13 -04002141 for_each_cpu_and(cpu, cpu_online_mask, mask) {
Alexander Duyck01c5f862013-01-10 08:57:35 +00002142 if (!new_dev_maps)
2143 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
Alexander Duyck2bb60cb2013-02-22 06:38:44 +00002144 if (!new_dev_maps) {
2145 mutex_unlock(&xps_map_mutex);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002146 return -ENOMEM;
Alexander Duyck2bb60cb2013-02-22 06:38:44 +00002147 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002148
Alexander Duyck184c4492016-10-28 11:50:13 -04002149 tci = cpu * num_tc + tc;
2150 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[tci]) :
Alexander Duyck01c5f862013-01-10 08:57:35 +00002151 NULL;
2152
2153 map = expand_xps_map(map, cpu, index);
2154 if (!map)
2155 goto error;
2156
Alexander Duyck184c4492016-10-28 11:50:13 -04002157 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002158 }
2159
2160 if (!new_dev_maps)
2161 goto out_no_new_maps;
2162
2163 for_each_possible_cpu(cpu) {
Alexander Duyck184c4492016-10-28 11:50:13 -04002164 /* copy maps belonging to foreign traffic classes */
2165 for (i = tc, tci = cpu * num_tc; dev_maps && i--; tci++) {
2166 /* fill in the new device map from the old device map */
2167 map = xmap_dereference(dev_maps->cpu_map[tci]);
2168 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2169 }
2170
2171 /* We need to explicitly update tci as prevous loop
2172 * could break out early if dev_maps is NULL.
2173 */
2174 tci = cpu * num_tc + tc;
2175
Alexander Duyck01c5f862013-01-10 08:57:35 +00002176 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
2177 /* add queue to CPU maps */
2178 int pos = 0;
2179
Alexander Duyck184c4492016-10-28 11:50:13 -04002180 map = xmap_dereference(new_dev_maps->cpu_map[tci]);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002181 while ((pos < map->len) && (map->queues[pos] != index))
2182 pos++;
2183
2184 if (pos == map->len)
2185 map->queues[map->len++] = index;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002186#ifdef CONFIG_NUMA
Alexander Duyck537c00d2013-01-10 08:57:02 +00002187 if (numa_node_id == -2)
2188 numa_node_id = cpu_to_node(cpu);
2189 else if (numa_node_id != cpu_to_node(cpu))
2190 numa_node_id = -1;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002191#endif
Alexander Duyck01c5f862013-01-10 08:57:35 +00002192 } else if (dev_maps) {
2193 /* fill in the new device map from the old device map */
Alexander Duyck184c4492016-10-28 11:50:13 -04002194 map = xmap_dereference(dev_maps->cpu_map[tci]);
2195 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
Alexander Duyck537c00d2013-01-10 08:57:02 +00002196 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002197
Alexander Duyck184c4492016-10-28 11:50:13 -04002198 /* copy maps belonging to foreign traffic classes */
2199 for (i = num_tc - tc, tci++; dev_maps && --i; tci++) {
2200 /* fill in the new device map from the old device map */
2201 map = xmap_dereference(dev_maps->cpu_map[tci]);
2202 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2203 }
Alexander Duyck537c00d2013-01-10 08:57:02 +00002204 }
2205
Alexander Duyck01c5f862013-01-10 08:57:35 +00002206 rcu_assign_pointer(dev->xps_maps, new_dev_maps);
2207
Alexander Duyck537c00d2013-01-10 08:57:02 +00002208 /* Cleanup old maps */
Alexander Duyck184c4492016-10-28 11:50:13 -04002209 if (!dev_maps)
2210 goto out_no_old_maps;
2211
2212 for_each_possible_cpu(cpu) {
2213 for (i = num_tc, tci = cpu * num_tc; i--; tci++) {
2214 new_map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2215 map = xmap_dereference(dev_maps->cpu_map[tci]);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002216 if (map && map != new_map)
2217 kfree_rcu(map, rcu);
2218 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002219 }
Alexander Duyck537c00d2013-01-10 08:57:02 +00002220
Alexander Duyck184c4492016-10-28 11:50:13 -04002221 kfree_rcu(dev_maps, rcu);
2222
2223out_no_old_maps:
Alexander Duyck01c5f862013-01-10 08:57:35 +00002224 dev_maps = new_dev_maps;
2225 active = true;
2226
2227out_no_new_maps:
2228 /* update Tx queue numa node */
Alexander Duyck537c00d2013-01-10 08:57:02 +00002229 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2230 (numa_node_id >= 0) ? numa_node_id :
2231 NUMA_NO_NODE);
2232
Alexander Duyck01c5f862013-01-10 08:57:35 +00002233 if (!dev_maps)
2234 goto out_no_maps;
2235
2236 /* removes queue from unused CPUs */
2237 for_each_possible_cpu(cpu) {
Alexander Duyck184c4492016-10-28 11:50:13 -04002238 for (i = tc, tci = cpu * num_tc; i--; tci++)
2239 active |= remove_xps_queue(dev_maps, tci, index);
2240 if (!cpumask_test_cpu(cpu, mask) || !cpu_online(cpu))
2241 active |= remove_xps_queue(dev_maps, tci, index);
2242 for (i = num_tc - tc, tci++; --i; tci++)
2243 active |= remove_xps_queue(dev_maps, tci, index);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002244 }
2245
2246 /* free map if not active */
2247 if (!active) {
2248 RCU_INIT_POINTER(dev->xps_maps, NULL);
2249 kfree_rcu(dev_maps, rcu);
2250 }
2251
2252out_no_maps:
Alexander Duyck537c00d2013-01-10 08:57:02 +00002253 mutex_unlock(&xps_map_mutex);
2254
2255 return 0;
2256error:
Alexander Duyck01c5f862013-01-10 08:57:35 +00002257 /* remove any maps that we added */
2258 for_each_possible_cpu(cpu) {
Alexander Duyck184c4492016-10-28 11:50:13 -04002259 for (i = num_tc, tci = cpu * num_tc; i--; tci++) {
2260 new_map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2261 map = dev_maps ?
2262 xmap_dereference(dev_maps->cpu_map[tci]) :
2263 NULL;
2264 if (new_map && new_map != map)
2265 kfree(new_map);
2266 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002267 }
2268
Alexander Duyck537c00d2013-01-10 08:57:02 +00002269 mutex_unlock(&xps_map_mutex);
2270
Alexander Duyck537c00d2013-01-10 08:57:02 +00002271 kfree(new_dev_maps);
2272 return -ENOMEM;
2273}
2274EXPORT_SYMBOL(netif_set_xps_queue);
2275
2276#endif
Alexander Duyck9cf1f6a2016-10-28 11:43:20 -04002277void netdev_reset_tc(struct net_device *dev)
2278{
Alexander Duyck6234f872016-10-28 11:46:49 -04002279#ifdef CONFIG_XPS
2280 netif_reset_xps_queues_gt(dev, 0);
2281#endif
Alexander Duyck9cf1f6a2016-10-28 11:43:20 -04002282 dev->num_tc = 0;
2283 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2284 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2285}
2286EXPORT_SYMBOL(netdev_reset_tc);
2287
2288int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2289{
2290 if (tc >= dev->num_tc)
2291 return -EINVAL;
2292
Alexander Duyck6234f872016-10-28 11:46:49 -04002293#ifdef CONFIG_XPS
2294 netif_reset_xps_queues(dev, offset, count);
2295#endif
Alexander Duyck9cf1f6a2016-10-28 11:43:20 -04002296 dev->tc_to_txq[tc].count = count;
2297 dev->tc_to_txq[tc].offset = offset;
2298 return 0;
2299}
2300EXPORT_SYMBOL(netdev_set_tc_queue);
2301
2302int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2303{
2304 if (num_tc > TC_MAX_QUEUE)
2305 return -EINVAL;
2306
Alexander Duyck6234f872016-10-28 11:46:49 -04002307#ifdef CONFIG_XPS
2308 netif_reset_xps_queues_gt(dev, 0);
2309#endif
Alexander Duyck9cf1f6a2016-10-28 11:43:20 -04002310 dev->num_tc = num_tc;
2311 return 0;
2312}
2313EXPORT_SYMBOL(netdev_set_num_tc);
2314
John Fastabendf0796d52010-07-01 13:21:57 +00002315/*
2316 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2317 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
2318 */
Tom Herberte6484932010-10-18 18:04:39 +00002319int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
John Fastabendf0796d52010-07-01 13:21:57 +00002320{
Tom Herbert1d24eb42010-11-21 13:17:27 +00002321 int rc;
2322
Tom Herberte6484932010-10-18 18:04:39 +00002323 if (txq < 1 || txq > dev->num_tx_queues)
2324 return -EINVAL;
John Fastabendf0796d52010-07-01 13:21:57 +00002325
Ben Hutchings5c565802011-02-15 19:39:21 +00002326 if (dev->reg_state == NETREG_REGISTERED ||
2327 dev->reg_state == NETREG_UNREGISTERING) {
Tom Herberte6484932010-10-18 18:04:39 +00002328 ASSERT_RTNL();
2329
Tom Herbert1d24eb42010-11-21 13:17:27 +00002330 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2331 txq);
Tom Herbertbf264142010-11-26 08:36:09 +00002332 if (rc)
2333 return rc;
2334
John Fastabend4f57c082011-01-17 08:06:04 +00002335 if (dev->num_tc)
2336 netif_setup_tc(dev, txq);
2337
Alexander Duyck024e9672013-01-10 08:57:46 +00002338 if (txq < dev->real_num_tx_queues) {
Tom Herberte6484932010-10-18 18:04:39 +00002339 qdisc_reset_all_tx_gt(dev, txq);
Alexander Duyck024e9672013-01-10 08:57:46 +00002340#ifdef CONFIG_XPS
2341 netif_reset_xps_queues_gt(dev, txq);
2342#endif
2343 }
John Fastabendf0796d52010-07-01 13:21:57 +00002344 }
Tom Herberte6484932010-10-18 18:04:39 +00002345
2346 dev->real_num_tx_queues = txq;
2347 return 0;
John Fastabendf0796d52010-07-01 13:21:57 +00002348}
2349EXPORT_SYMBOL(netif_set_real_num_tx_queues);
Denis Vlasenko56079432006-03-29 15:57:29 -08002350
Michael Daltona953be52014-01-16 22:23:28 -08002351#ifdef CONFIG_SYSFS
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002352/**
2353 * netif_set_real_num_rx_queues - set actual number of RX queues used
2354 * @dev: Network device
2355 * @rxq: Actual number of RX queues
2356 *
2357 * This must be called either with the rtnl_lock held or before
2358 * registration of the net device. Returns 0 on success, or a
Ben Hutchings4e7f7952010-10-08 10:33:39 -07002359 * negative error code. If called before registration, it always
2360 * succeeds.
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002361 */
2362int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2363{
2364 int rc;
2365
Tom Herbertbd25fa72010-10-18 18:00:16 +00002366 if (rxq < 1 || rxq > dev->num_rx_queues)
2367 return -EINVAL;
2368
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002369 if (dev->reg_state == NETREG_REGISTERED) {
2370 ASSERT_RTNL();
2371
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002372 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2373 rxq);
2374 if (rc)
2375 return rc;
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002376 }
2377
2378 dev->real_num_rx_queues = rxq;
2379 return 0;
2380}
2381EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2382#endif
2383
Ben Hutchings2c530402012-07-10 10:55:09 +00002384/**
2385 * netif_get_num_default_rss_queues - default number of RSS queues
Yuval Mintz16917b82012-07-01 03:18:50 +00002386 *
2387 * This routine should set an upper limit on the number of RSS queues
2388 * used by default by multiqueue devices.
2389 */
Ben Hutchingsa55b1382012-07-10 10:54:38 +00002390int netif_get_num_default_rss_queues(void)
Yuval Mintz16917b82012-07-01 03:18:50 +00002391{
Hariprasad Shenai40e4e712016-06-08 18:09:08 +05302392 return is_kdump_kernel() ?
2393 1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
Yuval Mintz16917b82012-07-01 03:18:50 +00002394}
2395EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2396
Eric Dumazet3bcb8462016-06-04 20:02:28 -07002397static void __netif_reschedule(struct Qdisc *q)
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002398{
2399 struct softnet_data *sd;
2400 unsigned long flags;
2401
2402 local_irq_save(flags);
Christoph Lameter903ceff2014-08-17 12:30:35 -05002403 sd = this_cpu_ptr(&softnet_data);
Changli Gaoa9cbd582010-04-26 23:06:24 +00002404 q->next_sched = NULL;
2405 *sd->output_queue_tailp = q;
2406 sd->output_queue_tailp = &q->next_sched;
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002407 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2408 local_irq_restore(flags);
2409}
2410
David S. Miller37437bb2008-07-16 02:15:04 -07002411void __netif_schedule(struct Qdisc *q)
Denis Vlasenko56079432006-03-29 15:57:29 -08002412{
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002413 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2414 __netif_reschedule(q);
Denis Vlasenko56079432006-03-29 15:57:29 -08002415}
2416EXPORT_SYMBOL(__netif_schedule);
2417
Eric Dumazete6247022013-12-05 04:45:08 -08002418struct dev_kfree_skb_cb {
2419 enum skb_free_reason reason;
2420};
2421
2422static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
Denis Vlasenko56079432006-03-29 15:57:29 -08002423{
Eric Dumazete6247022013-12-05 04:45:08 -08002424 return (struct dev_kfree_skb_cb *)skb->cb;
Denis Vlasenko56079432006-03-29 15:57:29 -08002425}
Denis Vlasenko56079432006-03-29 15:57:29 -08002426
John Fastabend46e5da40a2014-09-12 20:04:52 -07002427void netif_schedule_queue(struct netdev_queue *txq)
2428{
2429 rcu_read_lock();
2430 if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
2431 struct Qdisc *q = rcu_dereference(txq->qdisc);
2432
2433 __netif_schedule(q);
2434 }
2435 rcu_read_unlock();
2436}
2437EXPORT_SYMBOL(netif_schedule_queue);
2438
John Fastabend46e5da40a2014-09-12 20:04:52 -07002439void netif_tx_wake_queue(struct netdev_queue *dev_queue)
2440{
2441 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
2442 struct Qdisc *q;
2443
2444 rcu_read_lock();
2445 q = rcu_dereference(dev_queue->qdisc);
2446 __netif_schedule(q);
2447 rcu_read_unlock();
2448 }
2449}
2450EXPORT_SYMBOL(netif_tx_wake_queue);
2451
Eric Dumazete6247022013-12-05 04:45:08 -08002452void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
2453{
2454 unsigned long flags;
2455
Myungho Jung98998862017-04-25 11:58:15 -07002456 if (unlikely(!skb))
2457 return;
2458
Eric Dumazete6247022013-12-05 04:45:08 -08002459 if (likely(atomic_read(&skb->users) == 1)) {
2460 smp_rmb();
2461 atomic_set(&skb->users, 0);
2462 } else if (likely(!atomic_dec_and_test(&skb->users))) {
2463 return;
2464 }
2465 get_kfree_skb_cb(skb)->reason = reason;
2466 local_irq_save(flags);
2467 skb->next = __this_cpu_read(softnet_data.completion_queue);
2468 __this_cpu_write(softnet_data.completion_queue, skb);
2469 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2470 local_irq_restore(flags);
2471}
2472EXPORT_SYMBOL(__dev_kfree_skb_irq);
2473
2474void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
Denis Vlasenko56079432006-03-29 15:57:29 -08002475{
2476 if (in_irq() || irqs_disabled())
Eric Dumazete6247022013-12-05 04:45:08 -08002477 __dev_kfree_skb_irq(skb, reason);
Denis Vlasenko56079432006-03-29 15:57:29 -08002478 else
2479 dev_kfree_skb(skb);
2480}
Eric Dumazete6247022013-12-05 04:45:08 -08002481EXPORT_SYMBOL(__dev_kfree_skb_any);
Denis Vlasenko56079432006-03-29 15:57:29 -08002482
2483
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002484/**
2485 * netif_device_detach - mark device as removed
2486 * @dev: network device
2487 *
2488 * Mark device as removed from system and therefore no longer available.
2489 */
Denis Vlasenko56079432006-03-29 15:57:29 -08002490void netif_device_detach(struct net_device *dev)
2491{
2492 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2493 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00002494 netif_tx_stop_all_queues(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08002495 }
2496}
2497EXPORT_SYMBOL(netif_device_detach);
2498
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002499/**
2500 * netif_device_attach - mark device as attached
2501 * @dev: network device
2502 *
2503 * Mark device as attached from system and restart if needed.
2504 */
Denis Vlasenko56079432006-03-29 15:57:29 -08002505void netif_device_attach(struct net_device *dev)
2506{
2507 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2508 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00002509 netif_tx_wake_all_queues(dev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002510 __netdev_watchdog_up(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08002511 }
2512}
2513EXPORT_SYMBOL(netif_device_attach);
2514
Jiri Pirko5605c762015-05-12 14:56:12 +02002515/*
2516 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2517 * to be used as a distribution range.
2518 */
2519u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb,
2520 unsigned int num_tx_queues)
2521{
2522 u32 hash;
2523 u16 qoffset = 0;
2524 u16 qcount = num_tx_queues;
2525
2526 if (skb_rx_queue_recorded(skb)) {
2527 hash = skb_get_rx_queue(skb);
2528 while (unlikely(hash >= num_tx_queues))
2529 hash -= num_tx_queues;
2530 return hash;
2531 }
2532
2533 if (dev->num_tc) {
2534 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
tchardingf4563a72017-02-09 17:56:07 +11002535
Jiri Pirko5605c762015-05-12 14:56:12 +02002536 qoffset = dev->tc_to_txq[tc].offset;
2537 qcount = dev->tc_to_txq[tc].count;
2538 }
2539
2540 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
2541}
2542EXPORT_SYMBOL(__skb_tx_hash);
2543
Ben Hutchings36c92472012-01-17 07:57:56 +00002544static void skb_warn_bad_offload(const struct sk_buff *skb)
2545{
Wei Tang84d15ae2016-06-16 21:17:49 +08002546 static const netdev_features_t null_features;
Ben Hutchings36c92472012-01-17 07:57:56 +00002547 struct net_device *dev = skb->dev;
Bjørn Mork88ad4172015-11-16 19:16:40 +01002548 const char *name = "";
Ben Hutchings36c92472012-01-17 07:57:56 +00002549
Ben Greearc846ad92013-04-19 10:45:52 +00002550 if (!net_ratelimit())
2551 return;
2552
Bjørn Mork88ad4172015-11-16 19:16:40 +01002553 if (dev) {
2554 if (dev->dev.parent)
2555 name = dev_driver_string(dev->dev.parent);
2556 else
2557 name = netdev_name(dev);
2558 }
Ben Hutchings36c92472012-01-17 07:57:56 +00002559 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2560 "gso_type=%d ip_summed=%d\n",
Bjørn Mork88ad4172015-11-16 19:16:40 +01002561 name, dev ? &dev->features : &null_features,
Michał Mirosław65e9d2f2012-01-17 10:00:40 +00002562 skb->sk ? &skb->sk->sk_route_caps : &null_features,
Ben Hutchings36c92472012-01-17 07:57:56 +00002563 skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2564 skb_shinfo(skb)->gso_type, skb->ip_summed);
2565}
2566
Linus Torvalds1da177e2005-04-16 15:20:36 -07002567/*
2568 * Invalidate hardware checksum when packet is to be mangled, and
2569 * complete checksum manually on outgoing path.
2570 */
Patrick McHardy84fa7932006-08-29 16:44:56 -07002571int skb_checksum_help(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002572{
Al Virod3bc23e2006-11-14 21:24:49 -08002573 __wsum csum;
Herbert Xu663ead32007-04-09 11:59:07 -07002574 int ret = 0, offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002575
Patrick McHardy84fa7932006-08-29 16:44:56 -07002576 if (skb->ip_summed == CHECKSUM_COMPLETE)
Herbert Xua430a432006-07-08 13:34:56 -07002577 goto out_set_summed;
2578
2579 if (unlikely(skb_shinfo(skb)->gso_size)) {
Ben Hutchings36c92472012-01-17 07:57:56 +00002580 skb_warn_bad_offload(skb);
2581 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002582 }
2583
Eric Dumazetcef401d2013-01-25 20:34:37 +00002584 /* Before computing a checksum, we should make sure no frag could
2585 * be modified by an external entity : checksum could be wrong.
2586 */
2587 if (skb_has_shared_frag(skb)) {
2588 ret = __skb_linearize(skb);
2589 if (ret)
2590 goto out;
2591 }
2592
Michał Mirosław55508d62010-12-14 15:24:08 +00002593 offset = skb_checksum_start_offset(skb);
Herbert Xua0308472007-10-15 01:47:15 -07002594 BUG_ON(offset >= skb_headlen(skb));
2595 csum = skb_checksum(skb, offset, skb->len - offset, 0);
2596
2597 offset += skb->csum_offset;
2598 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2599
2600 if (skb_cloned(skb) &&
2601 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002602 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2603 if (ret)
2604 goto out;
2605 }
2606
Eric Dumazet4f2e4ad2016-10-29 11:02:36 -07002607 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
Herbert Xua430a432006-07-08 13:34:56 -07002608out_set_summed:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002609 skb->ip_summed = CHECKSUM_NONE;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002610out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002611 return ret;
2612}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002613EXPORT_SYMBOL(skb_checksum_help);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002614
Vlad Yasevich53d64712014-03-27 17:26:18 -04002615__be16 skb_network_protocol(struct sk_buff *skb, int *depth)
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002616{
2617 __be16 type = skb->protocol;
2618
Pravin B Shelar19acc322013-05-07 20:41:07 +00002619 /* Tunnel gso handlers can set protocol to ethernet. */
2620 if (type == htons(ETH_P_TEB)) {
2621 struct ethhdr *eth;
2622
2623 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2624 return 0;
2625
2626 eth = (struct ethhdr *)skb_mac_header(skb);
2627 type = eth->h_proto;
2628 }
2629
Toshiaki Makitad4bcef32015-01-29 20:37:07 +09002630 return __vlan_get_protocol(skb, type, depth);
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002631}
2632
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002633/**
2634 * skb_mac_gso_segment - mac layer segmentation handler.
2635 * @skb: buffer to segment
2636 * @features: features for the output path (see dev->features)
2637 */
2638struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2639 netdev_features_t features)
2640{
2641 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2642 struct packet_offload *ptype;
Vlad Yasevich53d64712014-03-27 17:26:18 -04002643 int vlan_depth = skb->mac_len;
2644 __be16 type = skb_network_protocol(skb, &vlan_depth);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002645
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002646 if (unlikely(!type))
2647 return ERR_PTR(-EINVAL);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002648
Vlad Yasevich53d64712014-03-27 17:26:18 -04002649 __skb_pull(skb, vlan_depth);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002650
2651 rcu_read_lock();
2652 list_for_each_entry_rcu(ptype, &offload_base, list) {
2653 if (ptype->type == type && ptype->callbacks.gso_segment) {
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002654 segs = ptype->callbacks.gso_segment(skb, features);
2655 break;
2656 }
2657 }
2658 rcu_read_unlock();
2659
2660 __skb_push(skb, skb->data - skb_mac_header(skb));
2661
2662 return segs;
2663}
2664EXPORT_SYMBOL(skb_mac_gso_segment);
2665
2666
Cong Wang12b00042013-02-05 16:36:38 +00002667/* openvswitch calls this on rx path, so we need a different check.
2668 */
2669static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2670{
2671 if (tx_path)
Eric Dumazet6e7bc472017-02-03 14:29:42 -08002672 return skb->ip_summed != CHECKSUM_PARTIAL &&
2673 skb->ip_summed != CHECKSUM_NONE;
2674
2675 return skb->ip_summed == CHECKSUM_NONE;
Cong Wang12b00042013-02-05 16:36:38 +00002676}
2677
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002678/**
Cong Wang12b00042013-02-05 16:36:38 +00002679 * __skb_gso_segment - Perform segmentation on skb.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002680 * @skb: buffer to segment
Herbert Xu576a30e2006-06-27 13:22:38 -07002681 * @features: features for the output path (see dev->features)
Cong Wang12b00042013-02-05 16:36:38 +00002682 * @tx_path: whether it is called in TX path
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002683 *
2684 * This function segments the given skb and returns a list of segments.
Herbert Xu576a30e2006-06-27 13:22:38 -07002685 *
2686 * It may return NULL if the skb requires no segmentation. This is
2687 * only possible when GSO is used for verifying header integrity.
Konstantin Khlebnikov9207f9d2016-01-08 15:21:46 +03002688 *
2689 * Segmentation preserves SKB_SGO_CB_OFFSET bytes of previous skb cb.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002690 */
Cong Wang12b00042013-02-05 16:36:38 +00002691struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2692 netdev_features_t features, bool tx_path)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002693{
Eric Dumazetb2504a52017-01-31 10:20:32 -08002694 struct sk_buff *segs;
2695
Cong Wang12b00042013-02-05 16:36:38 +00002696 if (unlikely(skb_needs_check(skb, tx_path))) {
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002697 int err;
2698
Eric Dumazetb2504a52017-01-31 10:20:32 -08002699 /* We're going to init ->check field in TCP or UDP header */
françois romieua40e0a62014-07-15 23:55:35 +02002700 err = skb_cow_head(skb, 0);
2701 if (err < 0)
Herbert Xua430a432006-07-08 13:34:56 -07002702 return ERR_PTR(err);
2703 }
2704
Alexander Duyck802ab552016-04-10 21:45:03 -04002705 /* Only report GSO partial support if it will enable us to
2706 * support segmentation on this frame without needing additional
2707 * work.
2708 */
2709 if (features & NETIF_F_GSO_PARTIAL) {
2710 netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
2711 struct net_device *dev = skb->dev;
2712
2713 partial_features |= dev->features & dev->gso_partial_features;
2714 if (!skb_gso_ok(skb, features | partial_features))
2715 features &= ~NETIF_F_GSO_PARTIAL;
2716 }
2717
Konstantin Khlebnikov9207f9d2016-01-08 15:21:46 +03002718 BUILD_BUG_ON(SKB_SGO_CB_OFFSET +
2719 sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
2720
Pravin B Shelar68c33162013-02-14 14:02:41 +00002721 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
Eric Dumazet3347c962013-10-19 11:42:56 -07002722 SKB_GSO_CB(skb)->encap_level = 0;
2723
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002724 skb_reset_mac_header(skb);
2725 skb_reset_mac_len(skb);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002726
Eric Dumazetb2504a52017-01-31 10:20:32 -08002727 segs = skb_mac_gso_segment(skb, features);
2728
2729 if (unlikely(skb_needs_check(skb, tx_path)))
2730 skb_warn_bad_offload(skb);
2731
2732 return segs;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002733}
Cong Wang12b00042013-02-05 16:36:38 +00002734EXPORT_SYMBOL(__skb_gso_segment);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002735
Herbert Xufb286bb2005-11-10 13:01:24 -08002736/* Take action when hardware reception checksum errors are detected. */
2737#ifdef CONFIG_BUG
2738void netdev_rx_csum_fault(struct net_device *dev)
2739{
2740 if (net_ratelimit()) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00002741 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
Herbert Xufb286bb2005-11-10 13:01:24 -08002742 dump_stack();
2743 }
2744}
2745EXPORT_SYMBOL(netdev_rx_csum_fault);
2746#endif
2747
Linus Torvalds1da177e2005-04-16 15:20:36 -07002748/* Actually, we should eliminate this check as soon as we know, that:
2749 * 1. IOMMU is present and allows to map all the memory.
2750 * 2. No high memory really exists on this machine.
2751 */
2752
Florian Westphalc1e756b2014-05-05 15:00:44 +02002753static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002754{
Herbert Xu3d3a8532006-06-27 13:33:10 -07002755#ifdef CONFIG_HIGHMEM
Linus Torvalds1da177e2005-04-16 15:20:36 -07002756 int i;
tchardingf4563a72017-02-09 17:56:07 +11002757
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002758 if (!(dev->features & NETIF_F_HIGHDMA)) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002759 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2760 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
tchardingf4563a72017-02-09 17:56:07 +11002761
Ian Campbellea2ab692011-08-22 23:44:58 +00002762 if (PageHighMem(skb_frag_page(frag)))
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002763 return 1;
Ian Campbellea2ab692011-08-22 23:44:58 +00002764 }
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002765 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002766
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002767 if (PCI_DMA_BUS_IS_PHYS) {
2768 struct device *pdev = dev->dev.parent;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002769
Eric Dumazet9092c652010-04-02 13:34:49 -07002770 if (!pdev)
2771 return 0;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002772 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002773 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2774 dma_addr_t addr = page_to_phys(skb_frag_page(frag));
tchardingf4563a72017-02-09 17:56:07 +11002775
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002776 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2777 return 1;
2778 }
2779 }
Herbert Xu3d3a8532006-06-27 13:33:10 -07002780#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002781 return 0;
2782}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002783
Simon Horman3b392dd2014-06-04 08:53:17 +09002784/* If MPLS offload request, verify we are testing hardware MPLS features
2785 * instead of standard features for the netdev.
2786 */
Pravin B Shelard0edc7b2014-12-23 16:20:11 -08002787#if IS_ENABLED(CONFIG_NET_MPLS_GSO)
Simon Horman3b392dd2014-06-04 08:53:17 +09002788static netdev_features_t net_mpls_features(struct sk_buff *skb,
2789 netdev_features_t features,
2790 __be16 type)
2791{
Simon Horman25cd9ba2014-10-06 05:05:13 -07002792 if (eth_p_mpls(type))
Simon Horman3b392dd2014-06-04 08:53:17 +09002793 features &= skb->dev->mpls_features;
2794
2795 return features;
2796}
2797#else
2798static netdev_features_t net_mpls_features(struct sk_buff *skb,
2799 netdev_features_t features,
2800 __be16 type)
2801{
2802 return features;
2803}
2804#endif
2805
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002806static netdev_features_t harmonize_features(struct sk_buff *skb,
Florian Westphalc1e756b2014-05-05 15:00:44 +02002807 netdev_features_t features)
Jesse Grossf01a5232011-01-09 06:23:31 +00002808{
Vlad Yasevich53d64712014-03-27 17:26:18 -04002809 int tmp;
Simon Horman3b392dd2014-06-04 08:53:17 +09002810 __be16 type;
2811
2812 type = skb_network_protocol(skb, &tmp);
2813 features = net_mpls_features(skb, features, type);
Vlad Yasevich53d64712014-03-27 17:26:18 -04002814
Ed Cashinc0d680e2012-09-19 15:49:00 +00002815 if (skb->ip_summed != CHECKSUM_NONE &&
Simon Horman3b392dd2014-06-04 08:53:17 +09002816 !can_checksum_protocol(features, type)) {
Alexander Duyck996e8022016-05-02 09:25:10 -07002817 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
Jesse Grossf01a5232011-01-09 06:23:31 +00002818 }
Eric Dumazet7be2c822017-01-18 12:12:17 -08002819 if (illegal_highdma(skb->dev, skb))
2820 features &= ~NETIF_F_SG;
Jesse Grossf01a5232011-01-09 06:23:31 +00002821
2822 return features;
2823}
2824
Toshiaki Makitae38f3022015-03-27 14:31:13 +09002825netdev_features_t passthru_features_check(struct sk_buff *skb,
2826 struct net_device *dev,
2827 netdev_features_t features)
2828{
2829 return features;
2830}
2831EXPORT_SYMBOL(passthru_features_check);
2832
Toshiaki Makita8cb65d02015-03-27 14:31:12 +09002833static netdev_features_t dflt_features_check(const struct sk_buff *skb,
2834 struct net_device *dev,
2835 netdev_features_t features)
2836{
2837 return vlan_features_check(skb, features);
2838}
2839
Alexander Duyckcbc53e02016-04-10 21:44:51 -04002840static netdev_features_t gso_features_check(const struct sk_buff *skb,
2841 struct net_device *dev,
2842 netdev_features_t features)
2843{
2844 u16 gso_segs = skb_shinfo(skb)->gso_segs;
2845
2846 if (gso_segs > dev->gso_max_segs)
2847 return features & ~NETIF_F_GSO_MASK;
2848
Alexander Duyck802ab552016-04-10 21:45:03 -04002849 /* Support for GSO partial features requires software
2850 * intervention before we can actually process the packets
2851 * so we need to strip support for any partial features now
2852 * and we can pull them back in after we have partially
2853 * segmented the frame.
2854 */
2855 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
2856 features &= ~dev->gso_partial_features;
2857
2858 /* Make sure to clear the IPv4 ID mangling feature if the
2859 * IPv4 header has the potential to be fragmented.
Alexander Duyckcbc53e02016-04-10 21:44:51 -04002860 */
2861 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
2862 struct iphdr *iph = skb->encapsulation ?
2863 inner_ip_hdr(skb) : ip_hdr(skb);
2864
2865 if (!(iph->frag_off & htons(IP_DF)))
2866 features &= ~NETIF_F_TSO_MANGLEID;
2867 }
2868
2869 return features;
2870}
2871
Florian Westphalc1e756b2014-05-05 15:00:44 +02002872netdev_features_t netif_skb_features(struct sk_buff *skb)
Jesse Gross58e998c2010-10-29 12:14:55 +00002873{
Jesse Gross5f352272014-12-23 22:37:26 -08002874 struct net_device *dev = skb->dev;
Eric Dumazetfcbeb972014-10-05 10:11:27 -07002875 netdev_features_t features = dev->features;
Jesse Gross58e998c2010-10-29 12:14:55 +00002876
Alexander Duyckcbc53e02016-04-10 21:44:51 -04002877 if (skb_is_gso(skb))
2878 features = gso_features_check(skb, dev, features);
Ben Hutchings30b678d2012-07-30 15:57:00 +00002879
Jesse Gross5f352272014-12-23 22:37:26 -08002880 /* If encapsulation offload request, verify we are testing
2881 * hardware encapsulation features instead of standard
2882 * features for the netdev
2883 */
2884 if (skb->encapsulation)
2885 features &= dev->hw_enc_features;
2886
Toshiaki Makitaf5a7fb82015-03-27 14:31:11 +09002887 if (skb_vlan_tagged(skb))
2888 features = netdev_intersect_features(features,
2889 dev->vlan_features |
2890 NETIF_F_HW_VLAN_CTAG_TX |
2891 NETIF_F_HW_VLAN_STAG_TX);
Jesse Gross58e998c2010-10-29 12:14:55 +00002892
Jesse Gross5f352272014-12-23 22:37:26 -08002893 if (dev->netdev_ops->ndo_features_check)
2894 features &= dev->netdev_ops->ndo_features_check(skb, dev,
2895 features);
Toshiaki Makita8cb65d02015-03-27 14:31:12 +09002896 else
2897 features &= dflt_features_check(skb, dev, features);
Jesse Gross5f352272014-12-23 22:37:26 -08002898
Florian Westphalc1e756b2014-05-05 15:00:44 +02002899 return harmonize_features(skb, features);
Jesse Gross58e998c2010-10-29 12:14:55 +00002900}
Florian Westphalc1e756b2014-05-05 15:00:44 +02002901EXPORT_SYMBOL(netif_skb_features);
Jesse Gross58e998c2010-10-29 12:14:55 +00002902
David S. Miller2ea25512014-08-29 21:10:01 -07002903static int xmit_one(struct sk_buff *skb, struct net_device *dev,
David S. Miller95f6b3d2014-08-29 21:57:30 -07002904 struct netdev_queue *txq, bool more)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002905{
David S. Miller2ea25512014-08-29 21:10:01 -07002906 unsigned int len;
2907 int rc;
Stephen Hemminger00829822008-11-20 20:14:53 -08002908
Salam Noureddine7866a622015-01-27 11:35:48 -08002909 if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all))
David S. Miller2ea25512014-08-29 21:10:01 -07002910 dev_queue_xmit_nit(skb, dev);
Jesse Grossfc741212011-01-09 06:23:32 +00002911
David S. Miller2ea25512014-08-29 21:10:01 -07002912 len = skb->len;
2913 trace_net_dev_start_xmit(skb, dev);
David S. Miller95f6b3d2014-08-29 21:57:30 -07002914 rc = netdev_start_xmit(skb, dev, txq, more);
David S. Miller2ea25512014-08-29 21:10:01 -07002915 trace_net_dev_xmit(skb, rc, dev, len);
Eric Dumazetadf30902009-06-02 05:19:30 +00002916
Patrick McHardy572a9d72009-11-10 06:14:14 +00002917 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002918}
David S. Miller2ea25512014-08-29 21:10:01 -07002919
David S. Miller8dcda222014-09-01 15:06:40 -07002920struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
2921 struct netdev_queue *txq, int *ret)
David S. Miller7f2e8702014-08-29 21:19:14 -07002922{
2923 struct sk_buff *skb = first;
2924 int rc = NETDEV_TX_OK;
2925
2926 while (skb) {
2927 struct sk_buff *next = skb->next;
2928
2929 skb->next = NULL;
David S. Miller95f6b3d2014-08-29 21:57:30 -07002930 rc = xmit_one(skb, dev, txq, next != NULL);
David S. Miller7f2e8702014-08-29 21:19:14 -07002931 if (unlikely(!dev_xmit_complete(rc))) {
2932 skb->next = next;
2933 goto out;
2934 }
2935
2936 skb = next;
2937 if (netif_xmit_stopped(txq) && skb) {
2938 rc = NETDEV_TX_BUSY;
2939 break;
2940 }
2941 }
2942
2943out:
2944 *ret = rc;
2945 return skb;
2946}
2947
Eric Dumazet1ff0dc92014-10-06 11:26:27 -07002948static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
2949 netdev_features_t features)
David S. Millereae3f882014-08-30 15:17:13 -07002950{
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01002951 if (skb_vlan_tag_present(skb) &&
Jiri Pirko59682502014-11-19 14:04:59 +01002952 !vlan_hw_offload_capable(features, skb->vlan_proto))
2953 skb = __vlan_hwaccel_push_inside(skb);
David S. Millereae3f882014-08-30 15:17:13 -07002954 return skb;
2955}
2956
Eric Dumazet55a93b32014-10-03 15:31:07 -07002957static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
David S. Millereae3f882014-08-30 15:17:13 -07002958{
2959 netdev_features_t features;
2960
David S. Millereae3f882014-08-30 15:17:13 -07002961 features = netif_skb_features(skb);
2962 skb = validate_xmit_vlan(skb, features);
2963 if (unlikely(!skb))
2964 goto out_null;
2965
Johannes Berg8b86a612015-04-17 15:45:04 +02002966 if (netif_needs_gso(skb, features)) {
David S. Millerce937182014-08-30 19:22:20 -07002967 struct sk_buff *segs;
2968
2969 segs = skb_gso_segment(skb, features);
Jason Wangcecda692014-09-19 16:04:38 +08002970 if (IS_ERR(segs)) {
Jason Wangaf6dabc2014-12-19 11:09:13 +08002971 goto out_kfree_skb;
Jason Wangcecda692014-09-19 16:04:38 +08002972 } else if (segs) {
2973 consume_skb(skb);
2974 skb = segs;
2975 }
David S. Millereae3f882014-08-30 15:17:13 -07002976 } else {
2977 if (skb_needs_linearize(skb, features) &&
2978 __skb_linearize(skb))
2979 goto out_kfree_skb;
2980
Steffen Klassertf6e27112017-04-14 10:07:28 +02002981 if (validate_xmit_xfrm(skb, features))
2982 goto out_kfree_skb;
2983
David S. Millereae3f882014-08-30 15:17:13 -07002984 /* If packet is not checksummed and device does not
2985 * support checksumming for this protocol, complete
2986 * checksumming here.
2987 */
2988 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2989 if (skb->encapsulation)
2990 skb_set_inner_transport_header(skb,
2991 skb_checksum_start_offset(skb));
2992 else
2993 skb_set_transport_header(skb,
2994 skb_checksum_start_offset(skb));
Tom Herberta1882222015-12-14 11:19:43 -08002995 if (!(features & NETIF_F_CSUM_MASK) &&
David S. Millereae3f882014-08-30 15:17:13 -07002996 skb_checksum_help(skb))
2997 goto out_kfree_skb;
2998 }
2999 }
3000
3001 return skb;
3002
3003out_kfree_skb:
3004 kfree_skb(skb);
3005out_null:
Eric Dumazetd21fd632016-04-12 21:50:07 -07003006 atomic_long_inc(&dev->tx_dropped);
David S. Millereae3f882014-08-30 15:17:13 -07003007 return NULL;
3008}
3009
Eric Dumazet55a93b32014-10-03 15:31:07 -07003010struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev)
3011{
3012 struct sk_buff *next, *head = NULL, *tail;
3013
Eric Dumazetbec3cfd2014-10-03 20:59:19 -07003014 for (; skb != NULL; skb = next) {
Eric Dumazet55a93b32014-10-03 15:31:07 -07003015 next = skb->next;
3016 skb->next = NULL;
Eric Dumazet55a93b32014-10-03 15:31:07 -07003017
Eric Dumazetbec3cfd2014-10-03 20:59:19 -07003018 /* in case skb wont be segmented, point to itself */
3019 skb->prev = skb;
3020
3021 skb = validate_xmit_skb(skb, dev);
3022 if (!skb)
3023 continue;
3024
3025 if (!head)
3026 head = skb;
3027 else
3028 tail->next = skb;
3029 /* If skb was segmented, skb->prev points to
3030 * the last segment. If not, it still contains skb.
3031 */
3032 tail = skb->prev;
Eric Dumazet55a93b32014-10-03 15:31:07 -07003033 }
3034 return head;
3035}
Willem de Bruijn104ba782016-10-26 11:23:07 -04003036EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07003037
Eric Dumazet1def9232013-01-10 12:36:42 +00003038static void qdisc_pkt_len_init(struct sk_buff *skb)
3039{
3040 const struct skb_shared_info *shinfo = skb_shinfo(skb);
3041
3042 qdisc_skb_cb(skb)->pkt_len = skb->len;
3043
3044 /* To get more precise estimation of bytes sent on wire,
3045 * we add to pkt_len the headers size of all segments
3046 */
3047 if (shinfo->gso_size) {
Eric Dumazet757b8b12013-01-15 21:14:21 -08003048 unsigned int hdr_len;
Jason Wang15e5a032013-03-25 20:19:59 +00003049 u16 gso_segs = shinfo->gso_segs;
Eric Dumazet1def9232013-01-10 12:36:42 +00003050
Eric Dumazet757b8b12013-01-15 21:14:21 -08003051 /* mac layer + network layer */
3052 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3053
3054 /* + transport layer */
Eric Dumazet1def9232013-01-10 12:36:42 +00003055 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
3056 hdr_len += tcp_hdrlen(skb);
3057 else
3058 hdr_len += sizeof(struct udphdr);
Jason Wang15e5a032013-03-25 20:19:59 +00003059
3060 if (shinfo->gso_type & SKB_GSO_DODGY)
3061 gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3062 shinfo->gso_size);
3063
3064 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
Eric Dumazet1def9232013-01-10 12:36:42 +00003065 }
3066}
3067
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003068static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3069 struct net_device *dev,
3070 struct netdev_queue *txq)
3071{
3072 spinlock_t *root_lock = qdisc_lock(q);
Eric Dumazet520ac302016-06-21 23:16:49 -07003073 struct sk_buff *to_free = NULL;
Eric Dumazeta2da5702011-01-20 03:48:19 +00003074 bool contended;
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003075 int rc;
3076
Eric Dumazeta2da5702011-01-20 03:48:19 +00003077 qdisc_calculate_pkt_len(skb, q);
Eric Dumazet79640a42010-06-02 05:09:29 -07003078 /*
3079 * Heuristic to force contended enqueues to serialize on a
3080 * separate lock before trying to get qdisc main lock.
Eric Dumazetf9eb8ae2016-06-06 09:37:15 -07003081 * This permits qdisc->running owner to get the lock more
Ying Xue9bf2b8c2014-06-26 15:56:31 +08003082 * often and dequeue packets faster.
Eric Dumazet79640a42010-06-02 05:09:29 -07003083 */
Eric Dumazeta2da5702011-01-20 03:48:19 +00003084 contended = qdisc_is_running(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07003085 if (unlikely(contended))
3086 spin_lock(&q->busylock);
3087
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003088 spin_lock(root_lock);
3089 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
Eric Dumazet520ac302016-06-21 23:16:49 -07003090 __qdisc_drop(skb, &to_free);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003091 rc = NET_XMIT_DROP;
3092 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
Eric Dumazetbc135b22010-06-02 03:23:51 -07003093 qdisc_run_begin(q)) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003094 /*
3095 * This is a work-conserving queue; there are no old skbs
3096 * waiting to be sent out; and the qdisc is not running -
3097 * xmit the skb directly.
3098 */
Eric Dumazetbfe0d022011-01-09 08:30:54 +00003099
Eric Dumazetbfe0d022011-01-09 08:30:54 +00003100 qdisc_bstats_update(q, skb);
3101
Eric Dumazet55a93b32014-10-03 15:31:07 -07003102 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
Eric Dumazet79640a42010-06-02 05:09:29 -07003103 if (unlikely(contended)) {
3104 spin_unlock(&q->busylock);
3105 contended = false;
3106 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003107 __qdisc_run(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07003108 } else
Eric Dumazetbc135b22010-06-02 03:23:51 -07003109 qdisc_run_end(q);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003110
3111 rc = NET_XMIT_SUCCESS;
3112 } else {
Eric Dumazet520ac302016-06-21 23:16:49 -07003113 rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
Eric Dumazet79640a42010-06-02 05:09:29 -07003114 if (qdisc_run_begin(q)) {
3115 if (unlikely(contended)) {
3116 spin_unlock(&q->busylock);
3117 contended = false;
3118 }
3119 __qdisc_run(q);
3120 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003121 }
3122 spin_unlock(root_lock);
Eric Dumazet520ac302016-06-21 23:16:49 -07003123 if (unlikely(to_free))
3124 kfree_skb_list(to_free);
Eric Dumazet79640a42010-06-02 05:09:29 -07003125 if (unlikely(contended))
3126 spin_unlock(&q->busylock);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003127 return rc;
3128}
3129
Daniel Borkmann86f85152013-12-29 17:27:11 +01003130#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
Neil Horman5bc14212011-11-22 05:10:51 +00003131static void skb_update_prio(struct sk_buff *skb)
3132{
Igor Maravic6977a792011-11-25 07:44:54 +00003133 struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
Neil Horman5bc14212011-11-22 05:10:51 +00003134
Eric Dumazet91c68ce2012-07-08 21:45:10 +00003135 if (!skb->priority && skb->sk && map) {
Tejun Heo2a56a1f2015-12-07 17:38:52 -05003136 unsigned int prioidx =
3137 sock_cgroup_prioidx(&skb->sk->sk_cgrp_data);
Eric Dumazet91c68ce2012-07-08 21:45:10 +00003138
3139 if (prioidx < map->priomap_len)
3140 skb->priority = map->priomap[prioidx];
3141 }
Neil Horman5bc14212011-11-22 05:10:51 +00003142}
3143#else
3144#define skb_update_prio(skb)
3145#endif
3146
hannes@stressinduktion.orgf60e5992015-04-01 17:07:44 +02003147DEFINE_PER_CPU(int, xmit_recursion);
3148EXPORT_SYMBOL(xmit_recursion);
3149
Dave Jonesd29f7492008-07-22 14:09:06 -07003150/**
Michel Machado95603e22012-06-12 10:16:35 +00003151 * dev_loopback_xmit - loop back @skb
Eric W. Biederman0c4b51f2015-09-15 20:04:18 -05003152 * @net: network namespace this loopback is happening in
3153 * @sk: sk needed to be a netfilter okfn
Michel Machado95603e22012-06-12 10:16:35 +00003154 * @skb: buffer to transmit
3155 */
Eric W. Biederman0c4b51f2015-09-15 20:04:18 -05003156int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
Michel Machado95603e22012-06-12 10:16:35 +00003157{
3158 skb_reset_mac_header(skb);
3159 __skb_pull(skb, skb_network_offset(skb));
3160 skb->pkt_type = PACKET_LOOPBACK;
3161 skb->ip_summed = CHECKSUM_UNNECESSARY;
3162 WARN_ON(!skb_dst(skb));
3163 skb_dst_force(skb);
3164 netif_rx_ni(skb);
3165 return 0;
3166}
3167EXPORT_SYMBOL(dev_loopback_xmit);
3168
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003169#ifdef CONFIG_NET_EGRESS
3170static struct sk_buff *
3171sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
3172{
3173 struct tcf_proto *cl = rcu_dereference_bh(dev->egress_cl_list);
3174 struct tcf_result cl_res;
3175
3176 if (!cl)
3177 return skb;
3178
Willem de Bruijn8dc07fd2017-01-07 17:06:37 -05003179 /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003180 qdisc_bstats_cpu_update(cl->q, skb);
3181
3182 switch (tc_classify(skb, cl, &cl_res, false)) {
3183 case TC_ACT_OK:
3184 case TC_ACT_RECLASSIFY:
3185 skb->tc_index = TC_H_MIN(cl_res.classid);
3186 break;
3187 case TC_ACT_SHOT:
3188 qdisc_qstats_cpu_drop(cl->q);
3189 *ret = NET_XMIT_DROP;
Daniel Borkmann7e2c3ae2016-05-15 23:28:29 +02003190 kfree_skb(skb);
3191 return NULL;
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003192 case TC_ACT_STOLEN:
3193 case TC_ACT_QUEUED:
3194 *ret = NET_XMIT_SUCCESS;
Daniel Borkmann7e2c3ae2016-05-15 23:28:29 +02003195 consume_skb(skb);
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003196 return NULL;
3197 case TC_ACT_REDIRECT:
3198 /* No need to push/pop skb's mac_header here on egress! */
3199 skb_do_redirect(skb);
3200 *ret = NET_XMIT_SUCCESS;
3201 return NULL;
3202 default:
3203 break;
3204 }
3205
3206 return skb;
3207}
3208#endif /* CONFIG_NET_EGRESS */
3209
Jiri Pirko638b2a62015-05-12 14:56:13 +02003210static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
3211{
3212#ifdef CONFIG_XPS
3213 struct xps_dev_maps *dev_maps;
3214 struct xps_map *map;
3215 int queue_index = -1;
3216
3217 rcu_read_lock();
3218 dev_maps = rcu_dereference(dev->xps_maps);
3219 if (dev_maps) {
Alexander Duyck184c4492016-10-28 11:50:13 -04003220 unsigned int tci = skb->sender_cpu - 1;
3221
3222 if (dev->num_tc) {
3223 tci *= dev->num_tc;
3224 tci += netdev_get_prio_tc_map(dev, skb->priority);
3225 }
3226
3227 map = rcu_dereference(dev_maps->cpu_map[tci]);
Jiri Pirko638b2a62015-05-12 14:56:13 +02003228 if (map) {
3229 if (map->len == 1)
3230 queue_index = map->queues[0];
3231 else
3232 queue_index = map->queues[reciprocal_scale(skb_get_hash(skb),
3233 map->len)];
3234 if (unlikely(queue_index >= dev->real_num_tx_queues))
3235 queue_index = -1;
3236 }
3237 }
3238 rcu_read_unlock();
3239
3240 return queue_index;
3241#else
3242 return -1;
3243#endif
3244}
3245
3246static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
3247{
3248 struct sock *sk = skb->sk;
3249 int queue_index = sk_tx_queue_get(sk);
3250
3251 if (queue_index < 0 || skb->ooo_okay ||
3252 queue_index >= dev->real_num_tx_queues) {
3253 int new_index = get_xps_queue(dev, skb);
tchardingf4563a72017-02-09 17:56:07 +11003254
Jiri Pirko638b2a62015-05-12 14:56:13 +02003255 if (new_index < 0)
3256 new_index = skb_tx_hash(dev, skb);
3257
3258 if (queue_index != new_index && sk &&
Eric Dumazet004a5d02015-10-04 21:08:10 -07003259 sk_fullsock(sk) &&
Jiri Pirko638b2a62015-05-12 14:56:13 +02003260 rcu_access_pointer(sk->sk_dst_cache))
3261 sk_tx_queue_set(sk, new_index);
3262
3263 queue_index = new_index;
3264 }
3265
3266 return queue_index;
3267}
3268
3269struct netdev_queue *netdev_pick_tx(struct net_device *dev,
3270 struct sk_buff *skb,
3271 void *accel_priv)
3272{
3273 int queue_index = 0;
3274
3275#ifdef CONFIG_XPS
Eric Dumazet52bd2d62015-11-18 06:30:50 -08003276 u32 sender_cpu = skb->sender_cpu - 1;
3277
3278 if (sender_cpu >= (u32)NR_CPUS)
Jiri Pirko638b2a62015-05-12 14:56:13 +02003279 skb->sender_cpu = raw_smp_processor_id() + 1;
3280#endif
3281
3282 if (dev->real_num_tx_queues != 1) {
3283 const struct net_device_ops *ops = dev->netdev_ops;
tchardingf4563a72017-02-09 17:56:07 +11003284
Jiri Pirko638b2a62015-05-12 14:56:13 +02003285 if (ops->ndo_select_queue)
3286 queue_index = ops->ndo_select_queue(dev, skb, accel_priv,
3287 __netdev_pick_tx);
3288 else
3289 queue_index = __netdev_pick_tx(dev, skb);
3290
3291 if (!accel_priv)
3292 queue_index = netdev_cap_txqueue(dev, queue_index);
3293 }
3294
3295 skb_set_queue_mapping(skb, queue_index);
3296 return netdev_get_tx_queue(dev, queue_index);
3297}
3298
Michel Machado95603e22012-06-12 10:16:35 +00003299/**
Jason Wang9d08dd32014-01-20 11:25:13 +08003300 * __dev_queue_xmit - transmit a buffer
Dave Jonesd29f7492008-07-22 14:09:06 -07003301 * @skb: buffer to transmit
Jason Wang9d08dd32014-01-20 11:25:13 +08003302 * @accel_priv: private data used for L2 forwarding offload
Dave Jonesd29f7492008-07-22 14:09:06 -07003303 *
3304 * Queue a buffer for transmission to a network device. The caller must
3305 * have set the device and priority and built the buffer before calling
3306 * this function. The function can be called from an interrupt.
3307 *
3308 * A negative errno code is returned on a failure. A success does not
3309 * guarantee the frame will be transmitted as it may be dropped due
3310 * to congestion or traffic shaping.
3311 *
3312 * -----------------------------------------------------------------------------------
3313 * I notice this method can also return errors from the queue disciplines,
3314 * including NET_XMIT_DROP, which is a positive value. So, errors can also
3315 * be positive.
3316 *
3317 * Regardless of the return value, the skb is consumed, so it is currently
3318 * difficult to retry a send to this method. (You can bump the ref count
3319 * before sending to hold a reference for retry if you are careful.)
3320 *
3321 * When calling this method, interrupts MUST be enabled. This is because
3322 * the BH enable code must have IRQs enabled so that it will not deadlock.
3323 * --BLG
3324 */
Rashika Kheria0a59f3a2014-02-09 20:26:25 +05303325static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003326{
3327 struct net_device *dev = skb->dev;
David S. Millerdc2b4842008-07-08 17:18:23 -07003328 struct netdev_queue *txq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003329 struct Qdisc *q;
3330 int rc = -ENOMEM;
3331
Eric Dumazet6d1ccff2013-02-05 20:22:20 +00003332 skb_reset_mac_header(skb);
3333
Willem de Bruijne7fd2882014-08-04 22:11:48 -04003334 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
3335 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
3336
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003337 /* Disable soft irqs for various locks below. Also
3338 * stops preemption for RCU.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003339 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003340 rcu_read_lock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003341
Neil Horman5bc14212011-11-22 05:10:51 +00003342 skb_update_prio(skb);
3343
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003344 qdisc_pkt_len_init(skb);
3345#ifdef CONFIG_NET_CLS_ACT
Willem de Bruijn8dc07fd2017-01-07 17:06:37 -05003346 skb->tc_at_ingress = 0;
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003347# ifdef CONFIG_NET_EGRESS
3348 if (static_key_false(&egress_needed)) {
3349 skb = sch_handle_egress(skb, &rc, dev);
3350 if (!skb)
3351 goto out;
3352 }
3353# endif
3354#endif
Eric Dumazet02875872014-10-05 18:38:35 -07003355 /* If device/qdisc don't need skb->dst, release it right now while
3356 * its hot in this cpu cache.
3357 */
3358 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
3359 skb_dst_drop(skb);
3360 else
3361 skb_dst_force(skb);
3362
Jason Wangf663dd92014-01-10 16:18:26 +08003363 txq = netdev_pick_tx(dev, skb, accel_priv);
Paul E. McKenneya898def2010-02-22 17:04:49 -08003364 q = rcu_dereference_bh(txq->qdisc);
David S. Miller37437bb2008-07-16 02:15:04 -07003365
Koki Sanagicf66ba52010-08-23 18:45:02 +09003366 trace_net_dev_queue(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003367 if (q->enqueue) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003368 rc = __dev_xmit_skb(skb, q, dev, txq);
David S. Miller37437bb2008-07-16 02:15:04 -07003369 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003370 }
3371
3372 /* The device has no queue. Common case for software devices:
tchardingeb13da12017-02-09 17:56:06 +11003373 * loopback, all the sorts of tunnels...
Linus Torvalds1da177e2005-04-16 15:20:36 -07003374
tchardingeb13da12017-02-09 17:56:06 +11003375 * Really, it is unlikely that netif_tx_lock protection is necessary
3376 * here. (f.e. loopback and IP tunnels are clean ignoring statistics
3377 * counters.)
3378 * However, it is possible, that they rely on protection
3379 * made by us here.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003380
tchardingeb13da12017-02-09 17:56:06 +11003381 * Check this and shot the lock. It is not prone from deadlocks.
3382 *Either shot noqueue qdisc, it is even simpler 8)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003383 */
3384 if (dev->flags & IFF_UP) {
3385 int cpu = smp_processor_id(); /* ok because BHs are off */
3386
David S. Millerc773e842008-07-08 23:13:53 -07003387 if (txq->xmit_lock_owner != cpu) {
Daniel Borkmanna70b5062016-06-10 21:19:06 +02003388 if (unlikely(__this_cpu_read(xmit_recursion) >
3389 XMIT_RECURSION_LIMIT))
Eric Dumazet745e20f2010-09-29 13:23:09 -07003390 goto recursion_alert;
3391
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02003392 skb = validate_xmit_skb(skb, dev);
3393 if (!skb)
Eric Dumazetd21fd632016-04-12 21:50:07 -07003394 goto out;
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02003395
David S. Millerc773e842008-07-08 23:13:53 -07003396 HARD_TX_LOCK(dev, txq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003397
Tom Herbert734664982011-11-28 16:32:44 +00003398 if (!netif_xmit_stopped(txq)) {
Eric Dumazet745e20f2010-09-29 13:23:09 -07003399 __this_cpu_inc(xmit_recursion);
David S. Millerce937182014-08-30 19:22:20 -07003400 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
Eric Dumazet745e20f2010-09-29 13:23:09 -07003401 __this_cpu_dec(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00003402 if (dev_xmit_complete(rc)) {
David S. Millerc773e842008-07-08 23:13:53 -07003403 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003404 goto out;
3405 }
3406 }
David S. Millerc773e842008-07-08 23:13:53 -07003407 HARD_TX_UNLOCK(dev, txq);
Joe Perchese87cc472012-05-13 21:56:26 +00003408 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
3409 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003410 } else {
3411 /* Recursion is detected! It is possible,
Eric Dumazet745e20f2010-09-29 13:23:09 -07003412 * unfortunately
3413 */
3414recursion_alert:
Joe Perchese87cc472012-05-13 21:56:26 +00003415 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
3416 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003417 }
3418 }
3419
3420 rc = -ENETDOWN;
Herbert Xud4828d82006-06-22 02:28:18 -07003421 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003422
Eric Dumazet015f0682014-03-27 08:45:56 -07003423 atomic_long_inc(&dev->tx_dropped);
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02003424 kfree_skb_list(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003425 return rc;
3426out:
Herbert Xud4828d82006-06-22 02:28:18 -07003427 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003428 return rc;
3429}
Jason Wangf663dd92014-01-10 16:18:26 +08003430
Eric W. Biederman2b4aa3c2015-09-15 20:04:07 -05003431int dev_queue_xmit(struct sk_buff *skb)
Jason Wangf663dd92014-01-10 16:18:26 +08003432{
3433 return __dev_queue_xmit(skb, NULL);
3434}
Eric W. Biederman2b4aa3c2015-09-15 20:04:07 -05003435EXPORT_SYMBOL(dev_queue_xmit);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003436
Jason Wangf663dd92014-01-10 16:18:26 +08003437int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv)
3438{
3439 return __dev_queue_xmit(skb, accel_priv);
3440}
3441EXPORT_SYMBOL(dev_queue_xmit_accel);
3442
Linus Torvalds1da177e2005-04-16 15:20:36 -07003443
tchardingeb13da12017-02-09 17:56:06 +11003444/*************************************************************************
3445 * Receiver routines
3446 *************************************************************************/
Linus Torvalds1da177e2005-04-16 15:20:36 -07003447
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07003448int netdev_max_backlog __read_mostly = 1000;
Eric Dumazetc9e6bc62012-09-27 19:29:05 +00003449EXPORT_SYMBOL(netdev_max_backlog);
3450
Eric Dumazet3b098e22010-05-15 23:57:10 -07003451int netdev_tstamp_prequeue __read_mostly = 1;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07003452int netdev_budget __read_mostly = 300;
Matthew Whitehead7acf8a12017-04-19 12:37:10 -04003453unsigned int __read_mostly netdev_budget_usecs = 2000;
Matthias Tafelmeier3d48b532016-12-29 21:37:21 +01003454int weight_p __read_mostly = 64; /* old backlog weight */
3455int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */
3456int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */
3457int dev_rx_weight __read_mostly = 64;
3458int dev_tx_weight __read_mostly = 64;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003459
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003460/* Called with irq disabled */
3461static inline void ____napi_schedule(struct softnet_data *sd,
3462 struct napi_struct *napi)
3463{
3464 list_add_tail(&napi->poll_list, &sd->poll_list);
3465 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3466}
3467
Eric Dumazetdf334542010-03-24 19:13:54 +00003468#ifdef CONFIG_RPS
Tom Herbertfec5e652010-04-16 16:01:27 -07003469
3470/* One global table that all flow-based protocols share. */
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00003471struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
Tom Herbertfec5e652010-04-16 16:01:27 -07003472EXPORT_SYMBOL(rps_sock_flow_table);
Eric Dumazet567e4b72015-02-06 12:59:01 -08003473u32 rps_cpu_mask __read_mostly;
3474EXPORT_SYMBOL(rps_cpu_mask);
Tom Herbertfec5e652010-04-16 16:01:27 -07003475
Ingo Molnarc5905af2012-02-24 08:31:31 +01003476struct static_key rps_needed __read_mostly;
Jason Wang3df97ba2016-04-25 23:13:42 -04003477EXPORT_SYMBOL(rps_needed);
Eric Dumazet13bfff22016-12-07 08:29:10 -08003478struct static_key rfs_needed __read_mostly;
3479EXPORT_SYMBOL(rfs_needed);
Eric Dumazetadc93002011-11-17 03:13:26 +00003480
Ben Hutchingsc4454772011-01-19 11:03:53 +00003481static struct rps_dev_flow *
3482set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3483 struct rps_dev_flow *rflow, u16 next_cpu)
3484{
Eric Dumazeta31196b2015-04-25 09:35:24 -07003485 if (next_cpu < nr_cpu_ids) {
Ben Hutchingsc4454772011-01-19 11:03:53 +00003486#ifdef CONFIG_RFS_ACCEL
3487 struct netdev_rx_queue *rxqueue;
3488 struct rps_dev_flow_table *flow_table;
3489 struct rps_dev_flow *old_rflow;
3490 u32 flow_id;
3491 u16 rxq_index;
3492 int rc;
3493
3494 /* Should we steer this flow to a different hardware queue? */
Ben Hutchings69a19ee2011-02-15 20:32:04 +00003495 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
3496 !(dev->features & NETIF_F_NTUPLE))
Ben Hutchingsc4454772011-01-19 11:03:53 +00003497 goto out;
3498 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
3499 if (rxq_index == skb_get_rx_queue(skb))
3500 goto out;
3501
3502 rxqueue = dev->_rx + rxq_index;
3503 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3504 if (!flow_table)
3505 goto out;
Tom Herbert61b905d2014-03-24 15:34:47 -07003506 flow_id = skb_get_hash(skb) & flow_table->mask;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003507 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
3508 rxq_index, flow_id);
3509 if (rc < 0)
3510 goto out;
3511 old_rflow = rflow;
3512 rflow = &flow_table->flows[flow_id];
Ben Hutchingsc4454772011-01-19 11:03:53 +00003513 rflow->filter = rc;
3514 if (old_rflow->filter == rflow->filter)
3515 old_rflow->filter = RPS_NO_FILTER;
3516 out:
3517#endif
3518 rflow->last_qtail =
Ben Hutchings09994d12011-10-03 04:42:46 +00003519 per_cpu(softnet_data, next_cpu).input_queue_head;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003520 }
3521
Ben Hutchings09994d12011-10-03 04:42:46 +00003522 rflow->cpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003523 return rflow;
3524}
3525
Tom Herbert0a9627f2010-03-16 08:03:29 +00003526/*
3527 * get_rps_cpu is called from netif_receive_skb and returns the target
3528 * CPU from the RPS map of the receiving queue for a given skb.
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003529 * rcu_read_lock must be held on entry.
Tom Herbert0a9627f2010-03-16 08:03:29 +00003530 */
Tom Herbertfec5e652010-04-16 16:01:27 -07003531static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3532 struct rps_dev_flow **rflowp)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003533{
Eric Dumazet567e4b72015-02-06 12:59:01 -08003534 const struct rps_sock_flow_table *sock_flow_table;
3535 struct netdev_rx_queue *rxqueue = dev->_rx;
Tom Herbertfec5e652010-04-16 16:01:27 -07003536 struct rps_dev_flow_table *flow_table;
Eric Dumazet567e4b72015-02-06 12:59:01 -08003537 struct rps_map *map;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003538 int cpu = -1;
Eric Dumazet567e4b72015-02-06 12:59:01 -08003539 u32 tcpu;
Tom Herbert61b905d2014-03-24 15:34:47 -07003540 u32 hash;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003541
Tom Herbert0a9627f2010-03-16 08:03:29 +00003542 if (skb_rx_queue_recorded(skb)) {
3543 u16 index = skb_get_rx_queue(skb);
Eric Dumazet567e4b72015-02-06 12:59:01 -08003544
Ben Hutchings62fe0b42010-09-27 08:24:33 +00003545 if (unlikely(index >= dev->real_num_rx_queues)) {
3546 WARN_ONCE(dev->real_num_rx_queues > 1,
3547 "%s received packet on queue %u, but number "
3548 "of RX queues is %u\n",
3549 dev->name, index, dev->real_num_rx_queues);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003550 goto done;
3551 }
Eric Dumazet567e4b72015-02-06 12:59:01 -08003552 rxqueue += index;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003553 }
3554
Eric Dumazet567e4b72015-02-06 12:59:01 -08003555 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
3556
3557 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3558 map = rcu_dereference(rxqueue->rps_map);
3559 if (!flow_table && !map)
3560 goto done;
3561
Changli Gao2d47b452010-08-17 19:00:56 +00003562 skb_reset_network_header(skb);
Tom Herbert61b905d2014-03-24 15:34:47 -07003563 hash = skb_get_hash(skb);
3564 if (!hash)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003565 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003566
Tom Herbertfec5e652010-04-16 16:01:27 -07003567 sock_flow_table = rcu_dereference(rps_sock_flow_table);
3568 if (flow_table && sock_flow_table) {
Tom Herbertfec5e652010-04-16 16:01:27 -07003569 struct rps_dev_flow *rflow;
Eric Dumazet567e4b72015-02-06 12:59:01 -08003570 u32 next_cpu;
3571 u32 ident;
Tom Herbertfec5e652010-04-16 16:01:27 -07003572
Eric Dumazet567e4b72015-02-06 12:59:01 -08003573 /* First check into global flow table if there is a match */
3574 ident = sock_flow_table->ents[hash & sock_flow_table->mask];
3575 if ((ident ^ hash) & ~rps_cpu_mask)
3576 goto try_rps;
3577
3578 next_cpu = ident & rps_cpu_mask;
3579
3580 /* OK, now we know there is a match,
3581 * we can look at the local (per receive queue) flow table
3582 */
Tom Herbert61b905d2014-03-24 15:34:47 -07003583 rflow = &flow_table->flows[hash & flow_table->mask];
Tom Herbertfec5e652010-04-16 16:01:27 -07003584 tcpu = rflow->cpu;
3585
Tom Herbertfec5e652010-04-16 16:01:27 -07003586 /*
3587 * If the desired CPU (where last recvmsg was done) is
3588 * different from current CPU (one in the rx-queue flow
3589 * table entry), switch if one of the following holds:
Eric Dumazeta31196b2015-04-25 09:35:24 -07003590 * - Current CPU is unset (>= nr_cpu_ids).
Tom Herbertfec5e652010-04-16 16:01:27 -07003591 * - Current CPU is offline.
3592 * - The current CPU's queue tail has advanced beyond the
3593 * last packet that was enqueued using this table entry.
3594 * This guarantees that all previous packets for the flow
3595 * have been dequeued, thus preserving in order delivery.
3596 */
3597 if (unlikely(tcpu != next_cpu) &&
Eric Dumazeta31196b2015-04-25 09:35:24 -07003598 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
Tom Herbertfec5e652010-04-16 16:01:27 -07003599 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
Tom Herbertbaefa312012-11-16 09:04:15 +00003600 rflow->last_qtail)) >= 0)) {
3601 tcpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003602 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
Tom Herbertbaefa312012-11-16 09:04:15 +00003603 }
Ben Hutchingsc4454772011-01-19 11:03:53 +00003604
Eric Dumazeta31196b2015-04-25 09:35:24 -07003605 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
Tom Herbertfec5e652010-04-16 16:01:27 -07003606 *rflowp = rflow;
3607 cpu = tcpu;
3608 goto done;
3609 }
3610 }
3611
Eric Dumazet567e4b72015-02-06 12:59:01 -08003612try_rps:
3613
Tom Herbert0a9627f2010-03-16 08:03:29 +00003614 if (map) {
Daniel Borkmann8fc54f62014-08-23 20:58:54 +02003615 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
Tom Herbert0a9627f2010-03-16 08:03:29 +00003616 if (cpu_online(tcpu)) {
3617 cpu = tcpu;
3618 goto done;
3619 }
3620 }
3621
3622done:
Tom Herbert0a9627f2010-03-16 08:03:29 +00003623 return cpu;
3624}
3625
Ben Hutchingsc4454772011-01-19 11:03:53 +00003626#ifdef CONFIG_RFS_ACCEL
3627
3628/**
3629 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
3630 * @dev: Device on which the filter was set
3631 * @rxq_index: RX queue index
3632 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
3633 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
3634 *
3635 * Drivers that implement ndo_rx_flow_steer() should periodically call
3636 * this function for each installed filter and remove the filters for
3637 * which it returns %true.
3638 */
3639bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
3640 u32 flow_id, u16 filter_id)
3641{
3642 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
3643 struct rps_dev_flow_table *flow_table;
3644 struct rps_dev_flow *rflow;
3645 bool expire = true;
Eric Dumazeta31196b2015-04-25 09:35:24 -07003646 unsigned int cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003647
3648 rcu_read_lock();
3649 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3650 if (flow_table && flow_id <= flow_table->mask) {
3651 rflow = &flow_table->flows[flow_id];
3652 cpu = ACCESS_ONCE(rflow->cpu);
Eric Dumazeta31196b2015-04-25 09:35:24 -07003653 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
Ben Hutchingsc4454772011-01-19 11:03:53 +00003654 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
3655 rflow->last_qtail) <
3656 (int)(10 * flow_table->mask)))
3657 expire = false;
3658 }
3659 rcu_read_unlock();
3660 return expire;
3661}
3662EXPORT_SYMBOL(rps_may_expire_flow);
3663
3664#endif /* CONFIG_RFS_ACCEL */
3665
Tom Herbert0a9627f2010-03-16 08:03:29 +00003666/* Called from hardirq (IPI) context */
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003667static void rps_trigger_softirq(void *data)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003668{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003669 struct softnet_data *sd = data;
3670
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003671 ____napi_schedule(sd, &sd->backlog);
Changli Gaodee42872010-05-02 05:42:16 +00003672 sd->received_rps++;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003673}
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003674
Tom Herbertfec5e652010-04-16 16:01:27 -07003675#endif /* CONFIG_RPS */
Tom Herbert0a9627f2010-03-16 08:03:29 +00003676
3677/*
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003678 * Check if this softnet_data structure is another cpu one
3679 * If yes, queue it to our IPI list and return 1
3680 * If no, return 0
3681 */
3682static int rps_ipi_queued(struct softnet_data *sd)
3683{
3684#ifdef CONFIG_RPS
Christoph Lameter903ceff2014-08-17 12:30:35 -05003685 struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003686
3687 if (sd != mysd) {
3688 sd->rps_ipi_next = mysd->rps_ipi_list;
3689 mysd->rps_ipi_list = sd;
3690
3691 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3692 return 1;
3693 }
3694#endif /* CONFIG_RPS */
3695 return 0;
3696}
3697
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003698#ifdef CONFIG_NET_FLOW_LIMIT
3699int netdev_flow_limit_table_len __read_mostly = (1 << 12);
3700#endif
3701
3702static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
3703{
3704#ifdef CONFIG_NET_FLOW_LIMIT
3705 struct sd_flow_limit *fl;
3706 struct softnet_data *sd;
3707 unsigned int old_flow, new_flow;
3708
3709 if (qlen < (netdev_max_backlog >> 1))
3710 return false;
3711
Christoph Lameter903ceff2014-08-17 12:30:35 -05003712 sd = this_cpu_ptr(&softnet_data);
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003713
3714 rcu_read_lock();
3715 fl = rcu_dereference(sd->flow_limit);
3716 if (fl) {
Tom Herbert3958afa1b2013-12-15 22:12:06 -08003717 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003718 old_flow = fl->history[fl->history_head];
3719 fl->history[fl->history_head] = new_flow;
3720
3721 fl->history_head++;
3722 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
3723
3724 if (likely(fl->buckets[old_flow]))
3725 fl->buckets[old_flow]--;
3726
3727 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
3728 fl->count++;
3729 rcu_read_unlock();
3730 return true;
3731 }
3732 }
3733 rcu_read_unlock();
3734#endif
3735 return false;
3736}
3737
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003738/*
Tom Herbert0a9627f2010-03-16 08:03:29 +00003739 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3740 * queue (may be a remote CPU queue).
3741 */
Tom Herbertfec5e652010-04-16 16:01:27 -07003742static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3743 unsigned int *qtail)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003744{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003745 struct softnet_data *sd;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003746 unsigned long flags;
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003747 unsigned int qlen;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003748
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003749 sd = &per_cpu(softnet_data, cpu);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003750
3751 local_irq_save(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003752
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003753 rps_lock(sd);
Julian Anastasove9e4dd32015-07-09 09:59:09 +03003754 if (!netif_running(skb->dev))
3755 goto drop;
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003756 qlen = skb_queue_len(&sd->input_pkt_queue);
3757 if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
Li RongQinge008f3f2014-12-08 09:42:55 +08003758 if (qlen) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00003759enqueue:
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003760 __skb_queue_tail(&sd->input_pkt_queue, skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003761 input_queue_tail_incr_save(sd, qtail);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003762 rps_unlock(sd);
Changli Gao152102c2010-03-30 20:16:22 +00003763 local_irq_restore(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003764 return NET_RX_SUCCESS;
3765 }
3766
Eric Dumazetebda37c22010-05-06 23:51:21 +00003767 /* Schedule NAPI for backlog device
3768 * We can use non atomic operation since we own the queue lock
3769 */
3770 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003771 if (!rps_ipi_queued(sd))
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003772 ____napi_schedule(sd, &sd->backlog);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003773 }
3774 goto enqueue;
3775 }
3776
Julian Anastasove9e4dd32015-07-09 09:59:09 +03003777drop:
Changli Gaodee42872010-05-02 05:42:16 +00003778 sd->dropped++;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003779 rps_unlock(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003780
Tom Herbert0a9627f2010-03-16 08:03:29 +00003781 local_irq_restore(flags);
3782
Eric Dumazetcaf586e2010-09-30 21:06:55 +00003783 atomic_long_inc(&skb->dev->rx_dropped);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003784 kfree_skb(skb);
3785 return NET_RX_DROP;
3786}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003787
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003788static int netif_rx_internal(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003789{
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003790 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003791
Eric Dumazet588f0332011-11-15 04:12:55 +00003792 net_timestamp_check(netdev_tstamp_prequeue, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003793
Koki Sanagicf66ba52010-08-23 18:45:02 +09003794 trace_netif_rx(skb);
Eric Dumazetdf334542010-03-24 19:13:54 +00003795#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01003796 if (static_key_false(&rps_needed)) {
Tom Herbertfec5e652010-04-16 16:01:27 -07003797 struct rps_dev_flow voidflow, *rflow = &voidflow;
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003798 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003799
Changli Gaocece1942010-08-07 20:35:43 -07003800 preempt_disable();
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003801 rcu_read_lock();
Tom Herbertfec5e652010-04-16 16:01:27 -07003802
3803 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003804 if (cpu < 0)
3805 cpu = smp_processor_id();
Tom Herbertfec5e652010-04-16 16:01:27 -07003806
3807 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3808
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003809 rcu_read_unlock();
Changli Gaocece1942010-08-07 20:35:43 -07003810 preempt_enable();
Eric Dumazetadc93002011-11-17 03:13:26 +00003811 } else
3812#endif
Tom Herbertfec5e652010-04-16 16:01:27 -07003813 {
3814 unsigned int qtail;
tchardingf4563a72017-02-09 17:56:07 +11003815
Tom Herbertfec5e652010-04-16 16:01:27 -07003816 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
3817 put_cpu();
3818 }
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003819 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003820}
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003821
3822/**
3823 * netif_rx - post buffer to the network code
3824 * @skb: buffer to post
3825 *
3826 * This function receives a packet from a device driver and queues it for
3827 * the upper (protocol) levels to process. It always succeeds. The buffer
3828 * may be dropped during processing for congestion control or by the
3829 * protocol layers.
3830 *
3831 * return values:
3832 * NET_RX_SUCCESS (no congestion)
3833 * NET_RX_DROP (packet was dropped)
3834 *
3835 */
3836
3837int netif_rx(struct sk_buff *skb)
3838{
3839 trace_netif_rx_entry(skb);
3840
3841 return netif_rx_internal(skb);
3842}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003843EXPORT_SYMBOL(netif_rx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003844
3845int netif_rx_ni(struct sk_buff *skb)
3846{
3847 int err;
3848
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003849 trace_netif_rx_ni_entry(skb);
3850
Linus Torvalds1da177e2005-04-16 15:20:36 -07003851 preempt_disable();
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003852 err = netif_rx_internal(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003853 if (local_softirq_pending())
3854 do_softirq();
3855 preempt_enable();
3856
3857 return err;
3858}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003859EXPORT_SYMBOL(netif_rx_ni);
3860
Emese Revfy0766f782016-06-20 20:42:34 +02003861static __latent_entropy void net_tx_action(struct softirq_action *h)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003862{
Christoph Lameter903ceff2014-08-17 12:30:35 -05003863 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003864
3865 if (sd->completion_queue) {
3866 struct sk_buff *clist;
3867
3868 local_irq_disable();
3869 clist = sd->completion_queue;
3870 sd->completion_queue = NULL;
3871 local_irq_enable();
3872
3873 while (clist) {
3874 struct sk_buff *skb = clist;
tchardingf4563a72017-02-09 17:56:07 +11003875
Linus Torvalds1da177e2005-04-16 15:20:36 -07003876 clist = clist->next;
3877
Ilpo Järvinen547b7922008-07-25 21:43:18 -07003878 WARN_ON(atomic_read(&skb->users));
Eric Dumazete6247022013-12-05 04:45:08 -08003879 if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
3880 trace_consume_skb(skb);
3881 else
3882 trace_kfree_skb(skb, net_tx_action);
Jesper Dangaard Brouer15fad712016-02-08 13:15:04 +01003883
3884 if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
3885 __kfree_skb(skb);
3886 else
3887 __kfree_skb_defer(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003888 }
Jesper Dangaard Brouer15fad712016-02-08 13:15:04 +01003889
3890 __kfree_skb_flush();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003891 }
3892
3893 if (sd->output_queue) {
David S. Miller37437bb2008-07-16 02:15:04 -07003894 struct Qdisc *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003895
3896 local_irq_disable();
3897 head = sd->output_queue;
3898 sd->output_queue = NULL;
Changli Gaoa9cbd582010-04-26 23:06:24 +00003899 sd->output_queue_tailp = &sd->output_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003900 local_irq_enable();
3901
3902 while (head) {
David S. Miller37437bb2008-07-16 02:15:04 -07003903 struct Qdisc *q = head;
3904 spinlock_t *root_lock;
3905
Linus Torvalds1da177e2005-04-16 15:20:36 -07003906 head = head->next_sched;
3907
David S. Miller5fb66222008-08-02 20:02:43 -07003908 root_lock = qdisc_lock(q);
Eric Dumazet3bcb8462016-06-04 20:02:28 -07003909 spin_lock(root_lock);
3910 /* We need to make sure head->next_sched is read
3911 * before clearing __QDISC_STATE_SCHED
3912 */
3913 smp_mb__before_atomic();
3914 clear_bit(__QDISC_STATE_SCHED, &q->state);
3915 qdisc_run(q);
3916 spin_unlock(root_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003917 }
3918 }
3919}
3920
Javier Martinez Canillas181402a2016-09-09 08:43:15 -04003921#if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
Michał Mirosławda678292009-06-05 05:35:28 +00003922/* This hook is defined here for ATM LANE */
3923int (*br_fdb_test_addr_hook)(struct net_device *dev,
3924 unsigned char *addr) __read_mostly;
Stephen Hemminger4fb019a2009-09-11 11:50:08 -07003925EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
Michał Mirosławda678292009-06-05 05:35:28 +00003926#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07003927
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003928static inline struct sk_buff *
3929sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
3930 struct net_device *orig_dev)
Herbert Xuf697c3e2007-10-14 00:38:47 -07003931{
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02003932#ifdef CONFIG_NET_CLS_ACT
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003933 struct tcf_proto *cl = rcu_dereference_bh(skb->dev->ingress_cl_list);
3934 struct tcf_result cl_res;
Eric Dumazet24824a02010-10-02 06:11:55 +00003935
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02003936 /* If there's at least one ingress present somewhere (so
3937 * we get here via enabled static key), remaining devices
3938 * that are not configured with an ingress qdisc will bail
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003939 * out here.
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02003940 */
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003941 if (!cl)
Daniel Borkmann45771392015-04-10 23:07:54 +02003942 return skb;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003943 if (*pt_prev) {
3944 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3945 *pt_prev = NULL;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003946 }
3947
Florian Westphal33654952015-05-14 00:36:28 +02003948 qdisc_skb_cb(skb)->pkt_len = skb->len;
Willem de Bruijn8dc07fd2017-01-07 17:06:37 -05003949 skb->tc_at_ingress = 1;
Eric Dumazet24ea5912015-07-06 05:18:03 -07003950 qdisc_bstats_cpu_update(cl->q, skb);
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02003951
Daniel Borkmann3b3ae882015-08-26 23:00:06 +02003952 switch (tc_classify(skb, cl, &cl_res, false)) {
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003953 case TC_ACT_OK:
3954 case TC_ACT_RECLASSIFY:
3955 skb->tc_index = TC_H_MIN(cl_res.classid);
3956 break;
3957 case TC_ACT_SHOT:
Eric Dumazet24ea5912015-07-06 05:18:03 -07003958 qdisc_qstats_cpu_drop(cl->q);
Eric Dumazet8a3a4c62016-05-06 15:55:50 -07003959 kfree_skb(skb);
3960 return NULL;
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003961 case TC_ACT_STOLEN:
3962 case TC_ACT_QUEUED:
Eric Dumazet8a3a4c62016-05-06 15:55:50 -07003963 consume_skb(skb);
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003964 return NULL;
Alexei Starovoitov27b29f62015-09-15 23:05:43 -07003965 case TC_ACT_REDIRECT:
3966 /* skb_mac_header check was done by cls/act_bpf, so
3967 * we can safely push the L2 header back before
3968 * redirecting to another netdev
3969 */
3970 __skb_push(skb, skb->mac_len);
3971 skb_do_redirect(skb);
3972 return NULL;
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003973 default:
3974 break;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003975 }
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02003976#endif /* CONFIG_NET_CLS_ACT */
Herbert Xuf697c3e2007-10-14 00:38:47 -07003977 return skb;
3978}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003979
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003980/**
Mahesh Bandewar24b27fc2016-09-01 22:18:34 -07003981 * netdev_is_rx_handler_busy - check if receive handler is registered
3982 * @dev: device to check
3983 *
3984 * Check if a receive handler is already registered for a given device.
3985 * Return true if there one.
3986 *
3987 * The caller must hold the rtnl_mutex.
3988 */
3989bool netdev_is_rx_handler_busy(struct net_device *dev)
3990{
3991 ASSERT_RTNL();
3992 return dev && rtnl_dereference(dev->rx_handler);
3993}
3994EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
3995
3996/**
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003997 * netdev_rx_handler_register - register receive handler
3998 * @dev: device to register a handler for
3999 * @rx_handler: receive handler to register
Jiri Pirko93e2c322010-06-10 03:34:59 +00004000 * @rx_handler_data: data pointer that is used by rx handler
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004001 *
Masanari Iidae2278672014-02-18 22:54:36 +09004002 * Register a receive handler for a device. This handler will then be
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004003 * called from __netif_receive_skb. A negative errno code is returned
4004 * on a failure.
4005 *
4006 * The caller must hold the rtnl_mutex.
Jiri Pirko8a4eb572011-03-12 03:14:39 +00004007 *
4008 * For a general description of rx_handler, see enum rx_handler_result.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004009 */
4010int netdev_rx_handler_register(struct net_device *dev,
Jiri Pirko93e2c322010-06-10 03:34:59 +00004011 rx_handler_func_t *rx_handler,
4012 void *rx_handler_data)
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004013{
Mahesh Bandewar1b7cd002017-01-18 15:02:49 -08004014 if (netdev_is_rx_handler_busy(dev))
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004015 return -EBUSY;
4016
Eric Dumazet00cfec32013-03-29 03:01:22 +00004017 /* Note: rx_handler_data must be set before rx_handler */
Jiri Pirko93e2c322010-06-10 03:34:59 +00004018 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004019 rcu_assign_pointer(dev->rx_handler, rx_handler);
4020
4021 return 0;
4022}
4023EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
4024
4025/**
4026 * netdev_rx_handler_unregister - unregister receive handler
4027 * @dev: device to unregister a handler from
4028 *
Kusanagi Kouichi166ec362013-03-18 02:59:52 +00004029 * Unregister a receive handler from a device.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004030 *
4031 * The caller must hold the rtnl_mutex.
4032 */
4033void netdev_rx_handler_unregister(struct net_device *dev)
4034{
4035
4036 ASSERT_RTNL();
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00004037 RCU_INIT_POINTER(dev->rx_handler, NULL);
Eric Dumazet00cfec32013-03-29 03:01:22 +00004038 /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
4039 * section has a guarantee to see a non NULL rx_handler_data
4040 * as well.
4041 */
4042 synchronize_net();
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00004043 RCU_INIT_POINTER(dev->rx_handler_data, NULL);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004044}
4045EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
4046
Mel Gormanb4b9e352012-07-31 16:44:26 -07004047/*
4048 * Limit the use of PFMEMALLOC reserves to those protocols that implement
4049 * the special handling of PFMEMALLOC skbs.
4050 */
4051static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
4052{
4053 switch (skb->protocol) {
Joe Perches2b8837a2014-03-12 10:04:17 -07004054 case htons(ETH_P_ARP):
4055 case htons(ETH_P_IP):
4056 case htons(ETH_P_IPV6):
4057 case htons(ETH_P_8021Q):
4058 case htons(ETH_P_8021AD):
Mel Gormanb4b9e352012-07-31 16:44:26 -07004059 return true;
4060 default:
4061 return false;
4062 }
4063}
4064
Pablo Neirae687ad62015-05-13 18:19:38 +02004065static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
4066 int *ret, struct net_device *orig_dev)
4067{
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02004068#ifdef CONFIG_NETFILTER_INGRESS
Pablo Neirae687ad62015-05-13 18:19:38 +02004069 if (nf_hook_ingress_active(skb)) {
Aaron Conole2c1e2702016-09-21 11:35:03 -04004070 int ingress_retval;
4071
Pablo Neirae687ad62015-05-13 18:19:38 +02004072 if (*pt_prev) {
4073 *ret = deliver_skb(skb, *pt_prev, orig_dev);
4074 *pt_prev = NULL;
4075 }
4076
Aaron Conole2c1e2702016-09-21 11:35:03 -04004077 rcu_read_lock();
4078 ingress_retval = nf_hook_ingress(skb);
4079 rcu_read_unlock();
4080 return ingress_retval;
Pablo Neirae687ad62015-05-13 18:19:38 +02004081 }
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02004082#endif /* CONFIG_NETFILTER_INGRESS */
Pablo Neirae687ad62015-05-13 18:19:38 +02004083 return 0;
4084}
Pablo Neirae687ad62015-05-13 18:19:38 +02004085
David S. Miller9754e292013-02-14 15:57:38 -05004086static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004087{
4088 struct packet_type *ptype, *pt_prev;
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004089 rx_handler_func_t *rx_handler;
David S. Millerf2ccd8f2005-08-09 19:34:12 -07004090 struct net_device *orig_dev;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00004091 bool deliver_exact = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004092 int ret = NET_RX_DROP;
Al Viro252e33462006-11-14 20:48:11 -08004093 __be16 type;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004094
Eric Dumazet588f0332011-11-15 04:12:55 +00004095 net_timestamp_check(!netdev_tstamp_prequeue, skb);
Eric Dumazet81bbb3d2009-09-30 16:42:42 -07004096
Koki Sanagicf66ba52010-08-23 18:45:02 +09004097 trace_netif_receive_skb(skb);
Patrick McHardy9b22ea52008-11-04 14:49:57 -08004098
Joe Eykholtcc9bd5c2008-07-02 18:22:00 -07004099 orig_dev = skb->dev;
Jiri Pirko1765a572011-02-12 06:48:36 +00004100
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07004101 skb_reset_network_header(skb);
Eric Dumazetfda55ec2013-01-07 09:28:21 +00004102 if (!skb_transport_header_was_set(skb))
4103 skb_reset_transport_header(skb);
Jiri Pirko0b5c9db2011-06-10 06:56:58 +00004104 skb_reset_mac_len(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004105
4106 pt_prev = NULL;
4107
David S. Miller63d8ea72011-02-28 10:48:59 -08004108another_round:
David S. Millerb6858172012-07-23 16:27:54 -07004109 skb->skb_iif = skb->dev->ifindex;
David S. Miller63d8ea72011-02-28 10:48:59 -08004110
4111 __this_cpu_inc(softnet_data.processed);
4112
Patrick McHardy8ad227f2013-04-19 02:04:31 +00004113 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
4114 skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
Vlad Yasevich0d5501c2014-08-08 14:42:13 -04004115 skb = skb_vlan_untag(skb);
Jiri Pirkobcc6d472011-04-07 19:48:33 +00004116 if (unlikely(!skb))
Julian Anastasov2c17d272015-07-09 09:59:10 +03004117 goto out;
Jiri Pirkobcc6d472011-04-07 19:48:33 +00004118 }
4119
Willem de Bruijne7246e12017-01-07 17:06:35 -05004120 if (skb_skip_tc_classify(skb))
4121 goto skip_classify;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004122
David S. Miller9754e292013-02-14 15:57:38 -05004123 if (pfmemalloc)
Mel Gormanb4b9e352012-07-31 16:44:26 -07004124 goto skip_taps;
4125
Linus Torvalds1da177e2005-04-16 15:20:36 -07004126 list_for_each_entry_rcu(ptype, &ptype_all, list) {
Salam Noureddine7866a622015-01-27 11:35:48 -08004127 if (pt_prev)
4128 ret = deliver_skb(skb, pt_prev, orig_dev);
4129 pt_prev = ptype;
4130 }
4131
4132 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
4133 if (pt_prev)
4134 ret = deliver_skb(skb, pt_prev, orig_dev);
4135 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004136 }
4137
Mel Gormanb4b9e352012-07-31 16:44:26 -07004138skip_taps:
Pablo Neira1cf519002015-05-13 18:19:37 +02004139#ifdef CONFIG_NET_INGRESS
Daniel Borkmann45771392015-04-10 23:07:54 +02004140 if (static_key_false(&ingress_needed)) {
Daniel Borkmann1f211a12016-01-07 22:29:47 +01004141 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev);
Daniel Borkmann45771392015-04-10 23:07:54 +02004142 if (!skb)
Julian Anastasov2c17d272015-07-09 09:59:10 +03004143 goto out;
Pablo Neirae687ad62015-05-13 18:19:38 +02004144
4145 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
Julian Anastasov2c17d272015-07-09 09:59:10 +03004146 goto out;
Daniel Borkmann45771392015-04-10 23:07:54 +02004147 }
Pablo Neira1cf519002015-05-13 18:19:37 +02004148#endif
Willem de Bruijna5135bc2017-01-07 17:06:36 -05004149 skb_reset_tc(skb);
Willem de Bruijne7246e12017-01-07 17:06:35 -05004150skip_classify:
David S. Miller9754e292013-02-14 15:57:38 -05004151 if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07004152 goto drop;
4153
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01004154 if (skb_vlan_tag_present(skb)) {
John Fastabend24257172011-10-10 09:16:41 +00004155 if (pt_prev) {
4156 ret = deliver_skb(skb, pt_prev, orig_dev);
4157 pt_prev = NULL;
4158 }
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00004159 if (vlan_do_receive(&skb))
John Fastabend24257172011-10-10 09:16:41 +00004160 goto another_round;
4161 else if (unlikely(!skb))
Julian Anastasov2c17d272015-07-09 09:59:10 +03004162 goto out;
John Fastabend24257172011-10-10 09:16:41 +00004163 }
4164
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00004165 rx_handler = rcu_dereference(skb->dev->rx_handler);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004166 if (rx_handler) {
4167 if (pt_prev) {
4168 ret = deliver_skb(skb, pt_prev, orig_dev);
4169 pt_prev = NULL;
4170 }
Jiri Pirko8a4eb572011-03-12 03:14:39 +00004171 switch (rx_handler(&skb)) {
4172 case RX_HANDLER_CONSUMED:
Cristian Bercaru3bc1b1a2013-03-08 07:03:38 +00004173 ret = NET_RX_SUCCESS;
Julian Anastasov2c17d272015-07-09 09:59:10 +03004174 goto out;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00004175 case RX_HANDLER_ANOTHER:
David S. Miller63d8ea72011-02-28 10:48:59 -08004176 goto another_round;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00004177 case RX_HANDLER_EXACT:
4178 deliver_exact = true;
4179 case RX_HANDLER_PASS:
4180 break;
4181 default:
4182 BUG();
4183 }
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004184 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004185
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01004186 if (unlikely(skb_vlan_tag_present(skb))) {
4187 if (skb_vlan_tag_get_id(skb))
Eric Dumazetd4b812d2013-07-18 07:19:26 -07004188 skb->pkt_type = PACKET_OTHERHOST;
4189 /* Note: we might in the future use prio bits
4190 * and set skb->priority like in vlan_do_receive()
4191 * For the time being, just ignore Priority Code Point
4192 */
4193 skb->vlan_tci = 0;
4194 }
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00004195
Linus Torvalds1da177e2005-04-16 15:20:36 -07004196 type = skb->protocol;
Salam Noureddine7866a622015-01-27 11:35:48 -08004197
4198 /* deliver only exact match when indicated */
4199 if (likely(!deliver_exact)) {
4200 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4201 &ptype_base[ntohs(type) &
4202 PTYPE_HASH_MASK]);
4203 }
4204
4205 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4206 &orig_dev->ptype_specific);
4207
4208 if (unlikely(skb->dev != orig_dev)) {
4209 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4210 &skb->dev->ptype_specific);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004211 }
4212
4213 if (pt_prev) {
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00004214 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
Michael S. Tsirkin0e698bf2012-09-15 22:44:16 +00004215 goto drop;
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00004216 else
4217 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004218 } else {
Mel Gormanb4b9e352012-07-31 16:44:26 -07004219drop:
Jarod Wilson6e7333d2016-02-01 18:51:05 -05004220 if (!deliver_exact)
4221 atomic_long_inc(&skb->dev->rx_dropped);
4222 else
4223 atomic_long_inc(&skb->dev->rx_nohandler);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004224 kfree_skb(skb);
4225 /* Jamal, now you will not able to escape explaining
4226 * me how you were going to use this. :-)
4227 */
4228 ret = NET_RX_DROP;
4229 }
4230
Julian Anastasov2c17d272015-07-09 09:59:10 +03004231out:
David S. Miller9754e292013-02-14 15:57:38 -05004232 return ret;
4233}
4234
4235static int __netif_receive_skb(struct sk_buff *skb)
4236{
4237 int ret;
4238
4239 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
Vlastimil Babkaf1083042017-05-08 15:59:53 -07004240 unsigned int noreclaim_flag;
David S. Miller9754e292013-02-14 15:57:38 -05004241
4242 /*
4243 * PFMEMALLOC skbs are special, they should
4244 * - be delivered to SOCK_MEMALLOC sockets only
4245 * - stay away from userspace
4246 * - have bounded memory usage
4247 *
4248 * Use PF_MEMALLOC as this saves us from propagating the allocation
4249 * context down to all allocation sites.
4250 */
Vlastimil Babkaf1083042017-05-08 15:59:53 -07004251 noreclaim_flag = memalloc_noreclaim_save();
David S. Miller9754e292013-02-14 15:57:38 -05004252 ret = __netif_receive_skb_core(skb, true);
Vlastimil Babkaf1083042017-05-08 15:59:53 -07004253 memalloc_noreclaim_restore(noreclaim_flag);
David S. Miller9754e292013-02-14 15:57:38 -05004254 } else
4255 ret = __netif_receive_skb_core(skb, false);
4256
Linus Torvalds1da177e2005-04-16 15:20:36 -07004257 return ret;
4258}
Tom Herbert0a9627f2010-03-16 08:03:29 +00004259
David S. Millerb5cdae32017-04-18 15:36:58 -04004260static struct static_key generic_xdp_needed __read_mostly;
4261
4262static int generic_xdp_install(struct net_device *dev, struct netdev_xdp *xdp)
4263{
4264 struct bpf_prog *new = xdp->prog;
4265 int ret = 0;
4266
4267 switch (xdp->command) {
4268 case XDP_SETUP_PROG: {
4269 struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
4270
4271 rcu_assign_pointer(dev->xdp_prog, new);
4272 if (old)
4273 bpf_prog_put(old);
4274
4275 if (old && !new) {
4276 static_key_slow_dec(&generic_xdp_needed);
4277 } else if (new && !old) {
4278 static_key_slow_inc(&generic_xdp_needed);
4279 dev_disable_lro(dev);
4280 }
4281 break;
4282 }
4283
4284 case XDP_QUERY_PROG:
4285 xdp->prog_attached = !!rcu_access_pointer(dev->xdp_prog);
4286 break;
4287
4288 default:
4289 ret = -EINVAL;
4290 break;
4291 }
4292
4293 return ret;
4294}
4295
4296static u32 netif_receive_generic_xdp(struct sk_buff *skb,
4297 struct bpf_prog *xdp_prog)
4298{
4299 struct xdp_buff xdp;
4300 u32 act = XDP_DROP;
4301 void *orig_data;
4302 int hlen, off;
4303 u32 mac_len;
4304
4305 /* Reinjected packets coming from act_mirred or similar should
4306 * not get XDP generic processing.
4307 */
4308 if (skb_cloned(skb))
4309 return XDP_PASS;
4310
4311 if (skb_linearize(skb))
4312 goto do_drop;
4313
4314 /* The XDP program wants to see the packet starting at the MAC
4315 * header.
4316 */
4317 mac_len = skb->data - skb_mac_header(skb);
4318 hlen = skb_headlen(skb) + mac_len;
4319 xdp.data = skb->data - mac_len;
4320 xdp.data_end = xdp.data + hlen;
4321 xdp.data_hard_start = skb->data - skb_headroom(skb);
4322 orig_data = xdp.data;
4323
4324 act = bpf_prog_run_xdp(xdp_prog, &xdp);
4325
4326 off = xdp.data - orig_data;
4327 if (off > 0)
4328 __skb_pull(skb, off);
4329 else if (off < 0)
4330 __skb_push(skb, -off);
4331
4332 switch (act) {
4333 case XDP_TX:
4334 __skb_push(skb, mac_len);
4335 /* fall through */
4336 case XDP_PASS:
4337 break;
4338
4339 default:
4340 bpf_warn_invalid_xdp_action(act);
4341 /* fall through */
4342 case XDP_ABORTED:
4343 trace_xdp_exception(skb->dev, xdp_prog, act);
4344 /* fall through */
4345 case XDP_DROP:
4346 do_drop:
4347 kfree_skb(skb);
4348 break;
4349 }
4350
4351 return act;
4352}
4353
4354/* When doing generic XDP we have to bypass the qdisc layer and the
4355 * network taps in order to match in-driver-XDP behavior.
4356 */
4357static void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)
4358{
4359 struct net_device *dev = skb->dev;
4360 struct netdev_queue *txq;
4361 bool free_skb = true;
4362 int cpu, rc;
4363
4364 txq = netdev_pick_tx(dev, skb, NULL);
4365 cpu = smp_processor_id();
4366 HARD_TX_LOCK(dev, txq, cpu);
4367 if (!netif_xmit_stopped(txq)) {
4368 rc = netdev_start_xmit(skb, dev, txq, 0);
4369 if (dev_xmit_complete(rc))
4370 free_skb = false;
4371 }
4372 HARD_TX_UNLOCK(dev, txq);
4373 if (free_skb) {
4374 trace_xdp_exception(dev, xdp_prog, XDP_TX);
4375 kfree_skb(skb);
4376 }
4377}
4378
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004379static int netif_receive_skb_internal(struct sk_buff *skb)
Tom Herbert0a9627f2010-03-16 08:03:29 +00004380{
Julian Anastasov2c17d272015-07-09 09:59:10 +03004381 int ret;
4382
Eric Dumazet588f0332011-11-15 04:12:55 +00004383 net_timestamp_check(netdev_tstamp_prequeue, skb);
Eric Dumazet3b098e22010-05-15 23:57:10 -07004384
Richard Cochranc1f19b52010-07-17 08:49:36 +00004385 if (skb_defer_rx_timestamp(skb))
4386 return NET_RX_SUCCESS;
4387
Julian Anastasov2c17d272015-07-09 09:59:10 +03004388 rcu_read_lock();
4389
David S. Millerb5cdae32017-04-18 15:36:58 -04004390 if (static_key_false(&generic_xdp_needed)) {
4391 struct bpf_prog *xdp_prog = rcu_dereference(skb->dev->xdp_prog);
4392
4393 if (xdp_prog) {
4394 u32 act = netif_receive_generic_xdp(skb, xdp_prog);
4395
4396 if (act != XDP_PASS) {
4397 rcu_read_unlock();
4398 if (act == XDP_TX)
4399 generic_xdp_tx(skb, xdp_prog);
4400 return NET_RX_DROP;
4401 }
4402 }
4403 }
4404
Eric Dumazetdf334542010-03-24 19:13:54 +00004405#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01004406 if (static_key_false(&rps_needed)) {
Eric Dumazet3b098e22010-05-15 23:57:10 -07004407 struct rps_dev_flow voidflow, *rflow = &voidflow;
Julian Anastasov2c17d272015-07-09 09:59:10 +03004408 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
Tom Herbertfec5e652010-04-16 16:01:27 -07004409
Eric Dumazet3b098e22010-05-15 23:57:10 -07004410 if (cpu >= 0) {
4411 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4412 rcu_read_unlock();
Eric Dumazetadc93002011-11-17 03:13:26 +00004413 return ret;
Eric Dumazet3b098e22010-05-15 23:57:10 -07004414 }
Tom Herbertfec5e652010-04-16 16:01:27 -07004415 }
Tom Herbert1e94d722010-03-18 17:45:44 -07004416#endif
Julian Anastasov2c17d272015-07-09 09:59:10 +03004417 ret = __netif_receive_skb(skb);
4418 rcu_read_unlock();
4419 return ret;
Tom Herbert0a9627f2010-03-16 08:03:29 +00004420}
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004421
4422/**
4423 * netif_receive_skb - process receive buffer from network
4424 * @skb: buffer to process
4425 *
4426 * netif_receive_skb() is the main receive data processing function.
4427 * It always succeeds. The buffer may be dropped during processing
4428 * for congestion control or by the protocol layers.
4429 *
4430 * This function may only be called from softirq context and interrupts
4431 * should be enabled.
4432 *
4433 * Return values (usually ignored):
4434 * NET_RX_SUCCESS: no congestion
4435 * NET_RX_DROP: packet was dropped
4436 */
Eric W. Biederman04eb4482015-09-15 20:04:15 -05004437int netif_receive_skb(struct sk_buff *skb)
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004438{
4439 trace_netif_receive_skb_entry(skb);
4440
4441 return netif_receive_skb_internal(skb);
4442}
Eric W. Biederman04eb4482015-09-15 20:04:15 -05004443EXPORT_SYMBOL(netif_receive_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004444
Eric Dumazet41852492016-08-26 12:50:39 -07004445DEFINE_PER_CPU(struct work_struct, flush_works);
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004446
4447/* Network device is going away, flush any packets still pending */
4448static void flush_backlog(struct work_struct *work)
4449{
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004450 struct sk_buff *skb, *tmp;
4451 struct softnet_data *sd;
4452
4453 local_bh_disable();
4454 sd = this_cpu_ptr(&softnet_data);
4455
4456 local_irq_disable();
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004457 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07004458 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
Eric Dumazet41852492016-08-26 12:50:39 -07004459 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004460 __skb_unlink(skb, &sd->input_pkt_queue);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07004461 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00004462 input_queue_head_incr(sd);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07004463 }
Changli Gao6e7676c2010-04-27 15:07:33 -07004464 }
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004465 rps_unlock(sd);
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004466 local_irq_enable();
Changli Gao6e7676c2010-04-27 15:07:33 -07004467
4468 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
Eric Dumazet41852492016-08-26 12:50:39 -07004469 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
Changli Gao6e7676c2010-04-27 15:07:33 -07004470 __skb_unlink(skb, &sd->process_queue);
4471 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00004472 input_queue_head_incr(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07004473 }
4474 }
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004475 local_bh_enable();
4476}
4477
Eric Dumazet41852492016-08-26 12:50:39 -07004478static void flush_all_backlogs(void)
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004479{
4480 unsigned int cpu;
4481
4482 get_online_cpus();
4483
Eric Dumazet41852492016-08-26 12:50:39 -07004484 for_each_online_cpu(cpu)
4485 queue_work_on(cpu, system_highpri_wq,
4486 per_cpu_ptr(&flush_works, cpu));
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004487
4488 for_each_online_cpu(cpu)
Eric Dumazet41852492016-08-26 12:50:39 -07004489 flush_work(per_cpu_ptr(&flush_works, cpu));
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004490
4491 put_online_cpus();
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07004492}
4493
Herbert Xud565b0a2008-12-15 23:38:52 -08004494static int napi_gro_complete(struct sk_buff *skb)
4495{
Vlad Yasevich22061d82012-11-15 08:49:11 +00004496 struct packet_offload *ptype;
Herbert Xud565b0a2008-12-15 23:38:52 -08004497 __be16 type = skb->protocol;
Vlad Yasevich22061d82012-11-15 08:49:11 +00004498 struct list_head *head = &offload_base;
Herbert Xud565b0a2008-12-15 23:38:52 -08004499 int err = -ENOENT;
4500
Eric Dumazetc3c7c252012-12-06 13:54:59 +00004501 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
4502
Herbert Xufc59f9a2009-04-14 15:11:06 -07004503 if (NAPI_GRO_CB(skb)->count == 1) {
4504 skb_shinfo(skb)->gso_size = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004505 goto out;
Herbert Xufc59f9a2009-04-14 15:11:06 -07004506 }
Herbert Xud565b0a2008-12-15 23:38:52 -08004507
4508 rcu_read_lock();
4509 list_for_each_entry_rcu(ptype, head, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00004510 if (ptype->type != type || !ptype->callbacks.gro_complete)
Herbert Xud565b0a2008-12-15 23:38:52 -08004511 continue;
4512
Jerry Chu299603e82013-12-11 20:53:45 -08004513 err = ptype->callbacks.gro_complete(skb, 0);
Herbert Xud565b0a2008-12-15 23:38:52 -08004514 break;
4515 }
4516 rcu_read_unlock();
4517
4518 if (err) {
4519 WARN_ON(&ptype->list == head);
4520 kfree_skb(skb);
4521 return NET_RX_SUCCESS;
4522 }
4523
4524out:
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004525 return netif_receive_skb_internal(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004526}
4527
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004528/* napi->gro_list contains packets ordered by age.
4529 * youngest packets at the head of it.
4530 * Complete skbs in reverse order to reduce latencies.
4531 */
4532void napi_gro_flush(struct napi_struct *napi, bool flush_old)
Herbert Xud565b0a2008-12-15 23:38:52 -08004533{
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004534 struct sk_buff *skb, *prev = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08004535
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004536 /* scan list and build reverse chain */
4537 for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
4538 skb->prev = prev;
4539 prev = skb;
Herbert Xud565b0a2008-12-15 23:38:52 -08004540 }
4541
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004542 for (skb = prev; skb; skb = prev) {
4543 skb->next = NULL;
4544
4545 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
4546 return;
4547
4548 prev = skb->prev;
4549 napi_gro_complete(skb);
4550 napi->gro_count--;
4551 }
4552
Herbert Xud565b0a2008-12-15 23:38:52 -08004553 napi->gro_list = NULL;
4554}
Eric Dumazet86cac582010-08-31 18:25:32 +00004555EXPORT_SYMBOL(napi_gro_flush);
Herbert Xud565b0a2008-12-15 23:38:52 -08004556
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004557static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
4558{
4559 struct sk_buff *p;
4560 unsigned int maclen = skb->dev->hard_header_len;
Tom Herbert0b4cec82014-01-15 08:58:06 -08004561 u32 hash = skb_get_hash_raw(skb);
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004562
4563 for (p = napi->gro_list; p; p = p->next) {
4564 unsigned long diffs;
4565
Tom Herbert0b4cec82014-01-15 08:58:06 -08004566 NAPI_GRO_CB(p)->flush = 0;
4567
4568 if (hash != skb_get_hash_raw(p)) {
4569 NAPI_GRO_CB(p)->same_flow = 0;
4570 continue;
4571 }
4572
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004573 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
4574 diffs |= p->vlan_tci ^ skb->vlan_tci;
Jesse Grossce87fc62016-01-20 17:59:49 -08004575 diffs |= skb_metadata_dst_cmp(p, skb);
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004576 if (maclen == ETH_HLEN)
4577 diffs |= compare_ether_header(skb_mac_header(p),
Eric Dumazeta50e2332014-03-29 21:28:21 -07004578 skb_mac_header(skb));
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004579 else if (!diffs)
4580 diffs = memcmp(skb_mac_header(p),
Eric Dumazeta50e2332014-03-29 21:28:21 -07004581 skb_mac_header(skb),
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004582 maclen);
4583 NAPI_GRO_CB(p)->same_flow = !diffs;
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004584 }
4585}
4586
Jerry Chu299603e82013-12-11 20:53:45 -08004587static void skb_gro_reset_offset(struct sk_buff *skb)
4588{
4589 const struct skb_shared_info *pinfo = skb_shinfo(skb);
4590 const skb_frag_t *frag0 = &pinfo->frags[0];
4591
4592 NAPI_GRO_CB(skb)->data_offset = 0;
4593 NAPI_GRO_CB(skb)->frag0 = NULL;
4594 NAPI_GRO_CB(skb)->frag0_len = 0;
4595
4596 if (skb_mac_header(skb) == skb_tail_pointer(skb) &&
4597 pinfo->nr_frags &&
4598 !PageHighMem(skb_frag_page(frag0))) {
4599 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
Eric Dumazet7cfd5fd2017-01-10 19:52:43 -08004600 NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,
4601 skb_frag_size(frag0),
4602 skb->end - skb->tail);
Herbert Xud565b0a2008-12-15 23:38:52 -08004603 }
4604}
4605
Eric Dumazeta50e2332014-03-29 21:28:21 -07004606static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
4607{
4608 struct skb_shared_info *pinfo = skb_shinfo(skb);
4609
4610 BUG_ON(skb->end - skb->tail < grow);
4611
4612 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
4613
4614 skb->data_len -= grow;
4615 skb->tail += grow;
4616
4617 pinfo->frags[0].page_offset += grow;
4618 skb_frag_size_sub(&pinfo->frags[0], grow);
4619
4620 if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
4621 skb_frag_unref(skb, 0);
4622 memmove(pinfo->frags, pinfo->frags + 1,
4623 --pinfo->nr_frags * sizeof(pinfo->frags[0]));
4624 }
4625}
4626
Rami Rosenbb728822012-11-28 21:55:25 +00004627static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xud565b0a2008-12-15 23:38:52 -08004628{
4629 struct sk_buff **pp = NULL;
Vlad Yasevich22061d82012-11-15 08:49:11 +00004630 struct packet_offload *ptype;
Herbert Xud565b0a2008-12-15 23:38:52 -08004631 __be16 type = skb->protocol;
Vlad Yasevich22061d82012-11-15 08:49:11 +00004632 struct list_head *head = &offload_base;
Herbert Xu0da2afd52008-12-26 14:57:42 -08004633 int same_flow;
Ben Hutchings5b252f02009-10-29 07:17:09 +00004634 enum gro_result ret;
Eric Dumazeta50e2332014-03-29 21:28:21 -07004635 int grow;
Herbert Xud565b0a2008-12-15 23:38:52 -08004636
David S. Millerb5cdae32017-04-18 15:36:58 -04004637 if (netif_elide_gro(skb->dev))
Herbert Xud565b0a2008-12-15 23:38:52 -08004638 goto normal;
4639
Eric Dumazetd61d0722016-11-07 11:12:27 -08004640 if (skb->csum_bad)
Herbert Xuf17f5c92009-01-14 14:36:12 -08004641 goto normal;
4642
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004643 gro_list_prepare(napi, skb);
4644
Herbert Xud565b0a2008-12-15 23:38:52 -08004645 rcu_read_lock();
4646 list_for_each_entry_rcu(ptype, head, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00004647 if (ptype->type != type || !ptype->callbacks.gro_receive)
Herbert Xud565b0a2008-12-15 23:38:52 -08004648 continue;
4649
Herbert Xu86911732009-01-29 14:19:50 +00004650 skb_set_network_header(skb, skb_gro_offset(skb));
Eric Dumazetefd94502013-02-14 17:31:48 +00004651 skb_reset_mac_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004652 NAPI_GRO_CB(skb)->same_flow = 0;
Eric Dumazetd61d0722016-11-07 11:12:27 -08004653 NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb);
Herbert Xu5d38a072009-01-04 16:13:40 -08004654 NAPI_GRO_CB(skb)->free = 0;
Jesse Grossfac8e0f2016-03-19 09:32:01 -07004655 NAPI_GRO_CB(skb)->encap_mark = 0;
Sabrina Dubrocafcd91dd2016-10-20 15:58:02 +02004656 NAPI_GRO_CB(skb)->recursion_counter = 0;
Alexander Duycka0ca1532016-04-05 09:13:39 -07004657 NAPI_GRO_CB(skb)->is_fou = 0;
Alexander Duyck15305452016-04-10 21:44:57 -04004658 NAPI_GRO_CB(skb)->is_atomic = 1;
Tom Herbert15e23962015-02-10 16:30:31 -08004659 NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004660
Tom Herbert662880f2014-08-27 21:26:56 -07004661 /* Setup for GRO checksum validation */
4662 switch (skb->ip_summed) {
4663 case CHECKSUM_COMPLETE:
4664 NAPI_GRO_CB(skb)->csum = skb->csum;
4665 NAPI_GRO_CB(skb)->csum_valid = 1;
4666 NAPI_GRO_CB(skb)->csum_cnt = 0;
4667 break;
4668 case CHECKSUM_UNNECESSARY:
4669 NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
4670 NAPI_GRO_CB(skb)->csum_valid = 0;
4671 break;
4672 default:
4673 NAPI_GRO_CB(skb)->csum_cnt = 0;
4674 NAPI_GRO_CB(skb)->csum_valid = 0;
4675 }
Herbert Xud565b0a2008-12-15 23:38:52 -08004676
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00004677 pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004678 break;
4679 }
4680 rcu_read_unlock();
4681
4682 if (&ptype->list == head)
4683 goto normal;
4684
Steffen Klassert25393d32017-02-15 09:39:44 +01004685 if (IS_ERR(pp) && PTR_ERR(pp) == -EINPROGRESS) {
4686 ret = GRO_CONSUMED;
4687 goto ok;
4688 }
4689
Herbert Xu0da2afd52008-12-26 14:57:42 -08004690 same_flow = NAPI_GRO_CB(skb)->same_flow;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004691 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
Herbert Xu0da2afd52008-12-26 14:57:42 -08004692
Herbert Xud565b0a2008-12-15 23:38:52 -08004693 if (pp) {
4694 struct sk_buff *nskb = *pp;
4695
4696 *pp = nskb->next;
4697 nskb->next = NULL;
4698 napi_gro_complete(nskb);
Herbert Xu4ae55442009-02-08 18:00:36 +00004699 napi->gro_count--;
Herbert Xud565b0a2008-12-15 23:38:52 -08004700 }
4701
Herbert Xu0da2afd52008-12-26 14:57:42 -08004702 if (same_flow)
Herbert Xud565b0a2008-12-15 23:38:52 -08004703 goto ok;
4704
Eric Dumazet600adc12014-01-09 14:12:19 -08004705 if (NAPI_GRO_CB(skb)->flush)
Herbert Xud565b0a2008-12-15 23:38:52 -08004706 goto normal;
Herbert Xud565b0a2008-12-15 23:38:52 -08004707
Eric Dumazet600adc12014-01-09 14:12:19 -08004708 if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) {
4709 struct sk_buff *nskb = napi->gro_list;
4710
4711 /* locate the end of the list to select the 'oldest' flow */
4712 while (nskb->next) {
4713 pp = &nskb->next;
4714 nskb = *pp;
4715 }
4716 *pp = NULL;
4717 nskb->next = NULL;
4718 napi_gro_complete(nskb);
4719 } else {
4720 napi->gro_count++;
4721 }
Herbert Xud565b0a2008-12-15 23:38:52 -08004722 NAPI_GRO_CB(skb)->count = 1;
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004723 NAPI_GRO_CB(skb)->age = jiffies;
Eric Dumazet29e98242014-05-16 11:34:37 -07004724 NAPI_GRO_CB(skb)->last = skb;
Herbert Xu86911732009-01-29 14:19:50 +00004725 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004726 skb->next = napi->gro_list;
4727 napi->gro_list = skb;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004728 ret = GRO_HELD;
Herbert Xud565b0a2008-12-15 23:38:52 -08004729
Herbert Xuad0f9902009-02-01 01:24:55 -08004730pull:
Eric Dumazeta50e2332014-03-29 21:28:21 -07004731 grow = skb_gro_offset(skb) - skb_headlen(skb);
4732 if (grow > 0)
4733 gro_pull_from_frag0(skb, grow);
Herbert Xud565b0a2008-12-15 23:38:52 -08004734ok:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004735 return ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08004736
4737normal:
Herbert Xuad0f9902009-02-01 01:24:55 -08004738 ret = GRO_NORMAL;
4739 goto pull;
Herbert Xu5d38a072009-01-04 16:13:40 -08004740}
Herbert Xu96e93ea2009-01-06 10:49:34 -08004741
Jerry Chubf5a7552014-01-07 10:23:19 -08004742struct packet_offload *gro_find_receive_by_type(__be16 type)
4743{
4744 struct list_head *offload_head = &offload_base;
4745 struct packet_offload *ptype;
4746
4747 list_for_each_entry_rcu(ptype, offload_head, list) {
4748 if (ptype->type != type || !ptype->callbacks.gro_receive)
4749 continue;
4750 return ptype;
4751 }
4752 return NULL;
4753}
Or Gerlitze27a2f82014-01-20 13:59:20 +02004754EXPORT_SYMBOL(gro_find_receive_by_type);
Jerry Chubf5a7552014-01-07 10:23:19 -08004755
4756struct packet_offload *gro_find_complete_by_type(__be16 type)
4757{
4758 struct list_head *offload_head = &offload_base;
4759 struct packet_offload *ptype;
4760
4761 list_for_each_entry_rcu(ptype, offload_head, list) {
4762 if (ptype->type != type || !ptype->callbacks.gro_complete)
4763 continue;
4764 return ptype;
4765 }
4766 return NULL;
4767}
Or Gerlitze27a2f82014-01-20 13:59:20 +02004768EXPORT_SYMBOL(gro_find_complete_by_type);
Herbert Xu96e93ea2009-01-06 10:49:34 -08004769
Rami Rosenbb728822012-11-28 21:55:25 +00004770static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
Herbert Xu5d38a072009-01-04 16:13:40 -08004771{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004772 switch (ret) {
4773 case GRO_NORMAL:
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004774 if (netif_receive_skb_internal(skb))
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004775 ret = GRO_DROP;
4776 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08004777
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004778 case GRO_DROP:
Herbert Xu5d38a072009-01-04 16:13:40 -08004779 kfree_skb(skb);
4780 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00004781
Eric Dumazetdaa86542012-04-19 07:07:40 +00004782 case GRO_MERGED_FREE:
Jesse Grossce87fc62016-01-20 17:59:49 -08004783 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) {
4784 skb_dst_drop(skb);
Steffen Klassertf991bb92017-01-30 06:45:38 +01004785 secpath_reset(skb);
Eric Dumazetd7e88832012-04-30 08:10:34 +00004786 kmem_cache_free(skbuff_head_cache, skb);
Jesse Grossce87fc62016-01-20 17:59:49 -08004787 } else {
Eric Dumazetd7e88832012-04-30 08:10:34 +00004788 __kfree_skb(skb);
Jesse Grossce87fc62016-01-20 17:59:49 -08004789 }
Eric Dumazetdaa86542012-04-19 07:07:40 +00004790 break;
4791
Ben Hutchings5b252f02009-10-29 07:17:09 +00004792 case GRO_HELD:
4793 case GRO_MERGED:
Steffen Klassert25393d32017-02-15 09:39:44 +01004794 case GRO_CONSUMED:
Ben Hutchings5b252f02009-10-29 07:17:09 +00004795 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08004796 }
4797
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004798 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004799}
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004800
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004801gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004802{
Eric Dumazet93f93a42015-11-18 06:30:59 -08004803 skb_mark_napi_id(skb, napi);
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004804 trace_napi_gro_receive_entry(skb);
Herbert Xu86911732009-01-29 14:19:50 +00004805
Eric Dumazeta50e2332014-03-29 21:28:21 -07004806 skb_gro_reset_offset(skb);
4807
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004808 return napi_skb_finish(dev_gro_receive(napi, skb), skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004809}
4810EXPORT_SYMBOL(napi_gro_receive);
4811
stephen hemmingerd0c2b0d2010-10-19 07:12:10 +00004812static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08004813{
Eric Dumazet93a35f52014-10-23 06:30:30 -07004814 if (unlikely(skb->pfmemalloc)) {
4815 consume_skb(skb);
4816 return;
4817 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08004818 __skb_pull(skb, skb_headlen(skb));
Eric Dumazet2a2a4592012-03-21 06:58:03 +00004819 /* restore the reserve we had after netdev_alloc_skb_ip_align() */
4820 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
Jesse Gross3701e512010-10-20 13:56:06 +00004821 skb->vlan_tci = 0;
Herbert Xu66c46d72011-01-29 20:44:54 -08004822 skb->dev = napi->dev;
Andy Gospodarek6d152e22011-02-02 14:53:25 -08004823 skb->skb_iif = 0;
Jerry Chuc3caf112014-07-14 15:54:46 -07004824 skb->encapsulation = 0;
4825 skb_shinfo(skb)->gso_type = 0;
Eric Dumazete33d0ba2014-04-03 09:28:10 -07004826 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
Steffen Klassertf991bb92017-01-30 06:45:38 +01004827 secpath_reset(skb);
Herbert Xu96e93ea2009-01-06 10:49:34 -08004828
4829 napi->skb = skb;
4830}
Herbert Xu96e93ea2009-01-06 10:49:34 -08004831
Herbert Xu76620aa2009-04-16 02:02:07 -07004832struct sk_buff *napi_get_frags(struct napi_struct *napi)
Herbert Xu5d38a072009-01-04 16:13:40 -08004833{
Herbert Xu5d38a072009-01-04 16:13:40 -08004834 struct sk_buff *skb = napi->skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08004835
4836 if (!skb) {
Alexander Duyckfd11a832014-12-09 19:40:49 -08004837 skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
Eric Dumazete2f9dc32015-11-19 12:11:23 -08004838 if (skb) {
4839 napi->skb = skb;
4840 skb_mark_napi_id(skb, napi);
4841 }
Herbert Xu5d38a072009-01-04 16:13:40 -08004842 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08004843 return skb;
4844}
Herbert Xu76620aa2009-04-16 02:02:07 -07004845EXPORT_SYMBOL(napi_get_frags);
Herbert Xu96e93ea2009-01-06 10:49:34 -08004846
Eric Dumazeta50e2332014-03-29 21:28:21 -07004847static gro_result_t napi_frags_finish(struct napi_struct *napi,
4848 struct sk_buff *skb,
4849 gro_result_t ret)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004850{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004851 switch (ret) {
4852 case GRO_NORMAL:
Eric Dumazeta50e2332014-03-29 21:28:21 -07004853 case GRO_HELD:
4854 __skb_push(skb, ETH_HLEN);
4855 skb->protocol = eth_type_trans(skb, skb->dev);
4856 if (ret == GRO_NORMAL && netif_receive_skb_internal(skb))
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004857 ret = GRO_DROP;
Herbert Xu86911732009-01-29 14:19:50 +00004858 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004859
4860 case GRO_DROP:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004861 case GRO_MERGED_FREE:
4862 napi_reuse_skb(napi, skb);
4863 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00004864
4865 case GRO_MERGED:
Steffen Klassert25393d32017-02-15 09:39:44 +01004866 case GRO_CONSUMED:
Ben Hutchings5b252f02009-10-29 07:17:09 +00004867 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004868 }
4869
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004870 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004871}
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004872
Eric Dumazeta50e2332014-03-29 21:28:21 -07004873/* Upper GRO stack assumes network header starts at gro_offset=0
4874 * Drivers could call both napi_gro_frags() and napi_gro_receive()
4875 * We copy ethernet header into skb->data to have a common layout.
4876 */
Eric Dumazet4adb9c42012-05-18 20:49:06 +00004877static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
Herbert Xu96e93ea2009-01-06 10:49:34 -08004878{
Herbert Xu76620aa2009-04-16 02:02:07 -07004879 struct sk_buff *skb = napi->skb;
Eric Dumazeta50e2332014-03-29 21:28:21 -07004880 const struct ethhdr *eth;
4881 unsigned int hlen = sizeof(*eth);
Herbert Xu76620aa2009-04-16 02:02:07 -07004882
4883 napi->skb = NULL;
4884
Eric Dumazeta50e2332014-03-29 21:28:21 -07004885 skb_reset_mac_header(skb);
4886 skb_gro_reset_offset(skb);
4887
4888 eth = skb_gro_header_fast(skb, 0);
4889 if (unlikely(skb_gro_header_hard(skb, hlen))) {
4890 eth = skb_gro_header_slow(skb, hlen, 0);
4891 if (unlikely(!eth)) {
Aaron Conole4da46ce2016-04-02 15:26:43 -04004892 net_warn_ratelimited("%s: dropping impossible skb from %s\n",
4893 __func__, napi->dev->name);
Eric Dumazeta50e2332014-03-29 21:28:21 -07004894 napi_reuse_skb(napi, skb);
4895 return NULL;
4896 }
4897 } else {
4898 gro_pull_from_frag0(skb, hlen);
4899 NAPI_GRO_CB(skb)->frag0 += hlen;
4900 NAPI_GRO_CB(skb)->frag0_len -= hlen;
Herbert Xu76620aa2009-04-16 02:02:07 -07004901 }
Eric Dumazeta50e2332014-03-29 21:28:21 -07004902 __skb_pull(skb, hlen);
4903
4904 /*
4905 * This works because the only protocols we care about don't require
4906 * special handling.
4907 * We'll fix it up properly in napi_frags_finish()
4908 */
4909 skb->protocol = eth->h_proto;
Herbert Xu76620aa2009-04-16 02:02:07 -07004910
Herbert Xu76620aa2009-04-16 02:02:07 -07004911 return skb;
4912}
Herbert Xu76620aa2009-04-16 02:02:07 -07004913
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004914gro_result_t napi_gro_frags(struct napi_struct *napi)
Herbert Xu76620aa2009-04-16 02:02:07 -07004915{
4916 struct sk_buff *skb = napi_frags_skb(napi);
Herbert Xu96e93ea2009-01-06 10:49:34 -08004917
4918 if (!skb)
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004919 return GRO_DROP;
Herbert Xu96e93ea2009-01-06 10:49:34 -08004920
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004921 trace_napi_gro_frags_entry(skb);
4922
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004923 return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
Herbert Xu5d38a072009-01-04 16:13:40 -08004924}
4925EXPORT_SYMBOL(napi_gro_frags);
4926
Tom Herbert573e8fc2014-08-22 13:33:47 -07004927/* Compute the checksum from gro_offset and return the folded value
4928 * after adding in any pseudo checksum.
4929 */
4930__sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
4931{
4932 __wsum wsum;
4933 __sum16 sum;
4934
4935 wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
4936
4937 /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
4938 sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
4939 if (likely(!sum)) {
4940 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
4941 !skb->csum_complete_sw)
4942 netdev_rx_csum_fault(skb->dev);
4943 }
4944
4945 NAPI_GRO_CB(skb)->csum = wsum;
4946 NAPI_GRO_CB(skb)->csum_valid = 1;
4947
4948 return sum;
4949}
4950EXPORT_SYMBOL(__skb_gro_checksum_complete);
4951
ashwanth@codeaurora.org773fc8f2017-06-09 14:24:58 +05304952static void net_rps_send_ipi(struct softnet_data *remsd)
4953{
4954#ifdef CONFIG_RPS
4955 while (remsd) {
4956 struct softnet_data *next = remsd->rps_ipi_next;
4957
4958 if (cpu_online(remsd->cpu))
4959 smp_call_function_single_async(remsd->cpu, &remsd->csd);
4960 remsd = next;
4961 }
4962#endif
4963}
4964
Eric Dumazete326bed2010-04-22 00:22:45 -07004965/*
Zhi Yong Wu855abcf2014-01-01 04:34:50 +08004966 * net_rps_action_and_irq_enable sends any pending IPI's for rps.
Eric Dumazete326bed2010-04-22 00:22:45 -07004967 * Note: called with local irq disabled, but exits with local irq enabled.
4968 */
4969static void net_rps_action_and_irq_enable(struct softnet_data *sd)
4970{
4971#ifdef CONFIG_RPS
4972 struct softnet_data *remsd = sd->rps_ipi_list;
4973
4974 if (remsd) {
4975 sd->rps_ipi_list = NULL;
4976
4977 local_irq_enable();
4978
4979 /* Send pending IPI's to kick RPS processing on remote cpus. */
ashwanth@codeaurora.org773fc8f2017-06-09 14:24:58 +05304980 net_rps_send_ipi(remsd);
Eric Dumazete326bed2010-04-22 00:22:45 -07004981 } else
4982#endif
4983 local_irq_enable();
4984}
4985
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004986static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
4987{
4988#ifdef CONFIG_RPS
4989 return sd->rps_ipi_list != NULL;
4990#else
4991 return false;
4992#endif
4993}
4994
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004995static int process_backlog(struct napi_struct *napi, int quota)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004996{
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004997 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004998 bool again = true;
4999 int work = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005000
Eric Dumazete326bed2010-04-22 00:22:45 -07005001 /* Check if we have pending ipi, its better to send them now,
5002 * not waiting net_rx_action() end.
5003 */
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005004 if (sd_has_rps_ipi_waiting(sd)) {
Eric Dumazete326bed2010-04-22 00:22:45 -07005005 local_irq_disable();
5006 net_rps_action_and_irq_enable(sd);
5007 }
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005008
Matthias Tafelmeier3d48b532016-12-29 21:37:21 +01005009 napi->weight = dev_rx_weight;
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005010 while (again) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005011 struct sk_buff *skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005012
Changli Gao6e7676c2010-04-27 15:07:33 -07005013 while ((skb = __skb_dequeue(&sd->process_queue))) {
Julian Anastasov2c17d272015-07-09 09:59:10 +03005014 rcu_read_lock();
Changli Gao6e7676c2010-04-27 15:07:33 -07005015 __netif_receive_skb(skb);
Julian Anastasov2c17d272015-07-09 09:59:10 +03005016 rcu_read_unlock();
Tom Herbert76cc8b12010-05-20 18:37:59 +00005017 input_queue_head_incr(sd);
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005018 if (++work >= quota)
Tom Herbert76cc8b12010-05-20 18:37:59 +00005019 return work;
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005020
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005021 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005022
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005023 local_irq_disable();
Changli Gao6e7676c2010-04-27 15:07:33 -07005024 rps_lock(sd);
Tom Herbert11ef7a82014-06-30 09:50:40 -07005025 if (skb_queue_empty(&sd->input_pkt_queue)) {
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07005026 /*
5027 * Inline a custom version of __napi_complete().
5028 * only current cpu owns and manipulates this napi,
Tom Herbert11ef7a82014-06-30 09:50:40 -07005029 * and NAPI_STATE_SCHED is the only possible flag set
5030 * on backlog.
5031 * We can use a plain write instead of clear_bit(),
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07005032 * and we dont need an smp_mb() memory barrier.
5033 */
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07005034 napi->state = 0;
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005035 again = false;
5036 } else {
5037 skb_queue_splice_tail_init(&sd->input_pkt_queue,
5038 &sd->process_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07005039 }
5040 rps_unlock(sd);
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005041 local_irq_enable();
Changli Gao6e7676c2010-04-27 15:07:33 -07005042 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005043
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005044 return work;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005045}
5046
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005047/**
5048 * __napi_schedule - schedule for receive
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07005049 * @n: entry to schedule
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005050 *
Eric Dumazetbc9ad162014-10-28 18:05:13 -07005051 * The entry's receive function will be scheduled to run.
5052 * Consider using __napi_schedule_irqoff() if hard irqs are masked.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005053 */
Harvey Harrisonb5606c22008-02-13 15:03:16 -08005054void __napi_schedule(struct napi_struct *n)
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005055{
5056 unsigned long flags;
5057
5058 local_irq_save(flags);
Christoph Lameter903ceff2014-08-17 12:30:35 -05005059 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005060 local_irq_restore(flags);
5061}
5062EXPORT_SYMBOL(__napi_schedule);
5063
Eric Dumazetbc9ad162014-10-28 18:05:13 -07005064/**
Eric Dumazet39e6c822017-02-28 10:34:50 -08005065 * napi_schedule_prep - check if napi can be scheduled
5066 * @n: napi context
5067 *
5068 * Test if NAPI routine is already running, and if not mark
5069 * it as running. This is used as a condition variable
5070 * insure only one NAPI poll instance runs. We also make
5071 * sure there is no pending NAPI disable.
5072 */
5073bool napi_schedule_prep(struct napi_struct *n)
5074{
5075 unsigned long val, new;
5076
5077 do {
5078 val = READ_ONCE(n->state);
5079 if (unlikely(val & NAPIF_STATE_DISABLE))
5080 return false;
5081 new = val | NAPIF_STATE_SCHED;
5082
5083 /* Sets STATE_MISSED bit if STATE_SCHED was already set
5084 * This was suggested by Alexander Duyck, as compiler
5085 * emits better code than :
5086 * if (val & NAPIF_STATE_SCHED)
5087 * new |= NAPIF_STATE_MISSED;
5088 */
5089 new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED *
5090 NAPIF_STATE_MISSED;
5091 } while (cmpxchg(&n->state, val, new) != val);
5092
5093 return !(val & NAPIF_STATE_SCHED);
5094}
5095EXPORT_SYMBOL(napi_schedule_prep);
5096
5097/**
Eric Dumazetbc9ad162014-10-28 18:05:13 -07005098 * __napi_schedule_irqoff - schedule for receive
5099 * @n: entry to schedule
5100 *
5101 * Variant of __napi_schedule() assuming hard irqs are masked
5102 */
5103void __napi_schedule_irqoff(struct napi_struct *n)
5104{
5105 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
5106}
5107EXPORT_SYMBOL(__napi_schedule_irqoff);
5108
Eric Dumazet364b6052016-11-15 10:15:13 -08005109bool napi_complete_done(struct napi_struct *n, int work_done)
Herbert Xud565b0a2008-12-15 23:38:52 -08005110{
Eric Dumazet39e6c822017-02-28 10:34:50 -08005111 unsigned long flags, val, new;
Herbert Xud565b0a2008-12-15 23:38:52 -08005112
5113 /*
Eric Dumazet217f6972016-11-15 10:15:11 -08005114 * 1) Don't let napi dequeue from the cpu poll list
5115 * just in case its running on a different cpu.
5116 * 2) If we are busy polling, do nothing here, we have
5117 * the guarantee we will be called later.
Herbert Xud565b0a2008-12-15 23:38:52 -08005118 */
Eric Dumazet217f6972016-11-15 10:15:11 -08005119 if (unlikely(n->state & (NAPIF_STATE_NPSVC |
5120 NAPIF_STATE_IN_BUSY_POLL)))
Eric Dumazet364b6052016-11-15 10:15:13 -08005121 return false;
Herbert Xud565b0a2008-12-15 23:38:52 -08005122
Eric Dumazet3b47d302014-11-06 21:09:44 -08005123 if (n->gro_list) {
5124 unsigned long timeout = 0;
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005125
Eric Dumazet3b47d302014-11-06 21:09:44 -08005126 if (work_done)
5127 timeout = n->dev->gro_flush_timeout;
5128
5129 if (timeout)
5130 hrtimer_start(&n->timer, ns_to_ktime(timeout),
5131 HRTIMER_MODE_REL_PINNED);
5132 else
5133 napi_gro_flush(n, false);
5134 }
Eric Dumazet02c16022017-02-04 15:25:02 -08005135 if (unlikely(!list_empty(&n->poll_list))) {
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005136 /* If n->poll_list is not empty, we need to mask irqs */
5137 local_irq_save(flags);
Eric Dumazet02c16022017-02-04 15:25:02 -08005138 list_del_init(&n->poll_list);
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005139 local_irq_restore(flags);
5140 }
Eric Dumazet39e6c822017-02-28 10:34:50 -08005141
5142 do {
5143 val = READ_ONCE(n->state);
5144
5145 WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED));
5146
5147 new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED);
5148
5149 /* If STATE_MISSED was set, leave STATE_SCHED set,
5150 * because we will call napi->poll() one more time.
5151 * This C code was suggested by Alexander Duyck to help gcc.
5152 */
5153 new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED *
5154 NAPIF_STATE_SCHED;
5155 } while (cmpxchg(&n->state, val, new) != val);
5156
5157 if (unlikely(val & NAPIF_STATE_MISSED)) {
5158 __napi_schedule(n);
5159 return false;
5160 }
5161
Eric Dumazet364b6052016-11-15 10:15:13 -08005162 return true;
Herbert Xud565b0a2008-12-15 23:38:52 -08005163}
Eric Dumazet3b47d302014-11-06 21:09:44 -08005164EXPORT_SYMBOL(napi_complete_done);
Herbert Xud565b0a2008-12-15 23:38:52 -08005165
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005166/* must be called under rcu_read_lock(), as we dont take a reference */
Eric Dumazet02d62e82015-11-18 06:30:52 -08005167static struct napi_struct *napi_by_id(unsigned int napi_id)
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005168{
5169 unsigned int hash = napi_id % HASH_SIZE(napi_hash);
5170 struct napi_struct *napi;
5171
5172 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
5173 if (napi->napi_id == napi_id)
5174 return napi;
5175
5176 return NULL;
5177}
Eric Dumazet02d62e82015-11-18 06:30:52 -08005178
5179#if defined(CONFIG_NET_RX_BUSY_POLL)
Eric Dumazet217f6972016-11-15 10:15:11 -08005180
Eric Dumazetce6aea92015-11-18 06:30:54 -08005181#define BUSY_POLL_BUDGET 8
Eric Dumazet217f6972016-11-15 10:15:11 -08005182
5183static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock)
5184{
5185 int rc;
5186
Eric Dumazet39e6c822017-02-28 10:34:50 -08005187 /* Busy polling means there is a high chance device driver hard irq
5188 * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was
5189 * set in napi_schedule_prep().
5190 * Since we are about to call napi->poll() once more, we can safely
5191 * clear NAPI_STATE_MISSED.
5192 *
5193 * Note: x86 could use a single "lock and ..." instruction
5194 * to perform these two clear_bit()
5195 */
5196 clear_bit(NAPI_STATE_MISSED, &napi->state);
Eric Dumazet217f6972016-11-15 10:15:11 -08005197 clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
5198
5199 local_bh_disable();
5200
5201 /* All we really want here is to re-enable device interrupts.
5202 * Ideally, a new ndo_busy_poll_stop() could avoid another round.
5203 */
5204 rc = napi->poll(napi, BUSY_POLL_BUDGET);
5205 netpoll_poll_unlock(have_poll_lock);
5206 if (rc == BUSY_POLL_BUDGET)
5207 __napi_schedule(napi);
5208 local_bh_enable();
5209 if (local_softirq_pending())
5210 do_softirq();
5211}
5212
Sridhar Samudrala7db6b042017-03-24 10:08:24 -07005213void napi_busy_loop(unsigned int napi_id,
5214 bool (*loop_end)(void *, unsigned long),
5215 void *loop_end_arg)
Eric Dumazet02d62e82015-11-18 06:30:52 -08005216{
Sridhar Samudrala7db6b042017-03-24 10:08:24 -07005217 unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
Eric Dumazet217f6972016-11-15 10:15:11 -08005218 int (*napi_poll)(struct napi_struct *napi, int budget);
Eric Dumazet217f6972016-11-15 10:15:11 -08005219 void *have_poll_lock = NULL;
Eric Dumazet02d62e82015-11-18 06:30:52 -08005220 struct napi_struct *napi;
Eric Dumazet217f6972016-11-15 10:15:11 -08005221
5222restart:
Eric Dumazet217f6972016-11-15 10:15:11 -08005223 napi_poll = NULL;
Eric Dumazet02d62e82015-11-18 06:30:52 -08005224
Eric Dumazet2a028ec2015-11-18 06:30:53 -08005225 rcu_read_lock();
Eric Dumazet02d62e82015-11-18 06:30:52 -08005226
Alexander Duyck545cd5e2017-03-24 10:07:53 -07005227 napi = napi_by_id(napi_id);
Eric Dumazet02d62e82015-11-18 06:30:52 -08005228 if (!napi)
5229 goto out;
5230
Eric Dumazet217f6972016-11-15 10:15:11 -08005231 preempt_disable();
5232 for (;;) {
Alexander Duyck2b5cd0d2017-03-24 10:08:12 -07005233 int work = 0;
5234
Eric Dumazet2a028ec2015-11-18 06:30:53 -08005235 local_bh_disable();
Eric Dumazet217f6972016-11-15 10:15:11 -08005236 if (!napi_poll) {
5237 unsigned long val = READ_ONCE(napi->state);
5238
5239 /* If multiple threads are competing for this napi,
5240 * we avoid dirtying napi->state as much as we can.
5241 */
5242 if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
5243 NAPIF_STATE_IN_BUSY_POLL))
5244 goto count;
5245 if (cmpxchg(&napi->state, val,
5246 val | NAPIF_STATE_IN_BUSY_POLL |
5247 NAPIF_STATE_SCHED) != val)
5248 goto count;
5249 have_poll_lock = netpoll_poll_lock(napi);
5250 napi_poll = napi->poll;
5251 }
Alexander Duyck2b5cd0d2017-03-24 10:08:12 -07005252 work = napi_poll(napi, BUSY_POLL_BUDGET);
5253 trace_napi_poll(napi, work, BUSY_POLL_BUDGET);
Eric Dumazet217f6972016-11-15 10:15:11 -08005254count:
Alexander Duyck2b5cd0d2017-03-24 10:08:12 -07005255 if (work > 0)
Sridhar Samudrala7db6b042017-03-24 10:08:24 -07005256 __NET_ADD_STATS(dev_net(napi->dev),
Alexander Duyck2b5cd0d2017-03-24 10:08:12 -07005257 LINUX_MIB_BUSYPOLLRXPACKETS, work);
Eric Dumazet2a028ec2015-11-18 06:30:53 -08005258 local_bh_enable();
Eric Dumazet02d62e82015-11-18 06:30:52 -08005259
Sridhar Samudrala7db6b042017-03-24 10:08:24 -07005260 if (!loop_end || loop_end(loop_end_arg, start_time))
Eric Dumazet217f6972016-11-15 10:15:11 -08005261 break;
Eric Dumazet02d62e82015-11-18 06:30:52 -08005262
Eric Dumazet217f6972016-11-15 10:15:11 -08005263 if (unlikely(need_resched())) {
5264 if (napi_poll)
5265 busy_poll_stop(napi, have_poll_lock);
5266 preempt_enable();
5267 rcu_read_unlock();
5268 cond_resched();
Sridhar Samudrala7db6b042017-03-24 10:08:24 -07005269 if (loop_end(loop_end_arg, start_time))
Alexander Duyck2b5cd0d2017-03-24 10:08:12 -07005270 return;
Eric Dumazet217f6972016-11-15 10:15:11 -08005271 goto restart;
5272 }
Linus Torvalds6cdf89b2016-12-12 10:48:02 -08005273 cpu_relax();
Eric Dumazet217f6972016-11-15 10:15:11 -08005274 }
5275 if (napi_poll)
5276 busy_poll_stop(napi, have_poll_lock);
5277 preempt_enable();
Eric Dumazet02d62e82015-11-18 06:30:52 -08005278out:
Eric Dumazet2a028ec2015-11-18 06:30:53 -08005279 rcu_read_unlock();
Eric Dumazet02d62e82015-11-18 06:30:52 -08005280}
Sridhar Samudrala7db6b042017-03-24 10:08:24 -07005281EXPORT_SYMBOL(napi_busy_loop);
Eric Dumazet02d62e82015-11-18 06:30:52 -08005282
5283#endif /* CONFIG_NET_RX_BUSY_POLL */
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005284
Eric Dumazet149d6ad2016-11-08 11:07:28 -08005285static void napi_hash_add(struct napi_struct *napi)
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005286{
Eric Dumazetd64b5e82015-11-18 06:31:00 -08005287 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) ||
5288 test_and_set_bit(NAPI_STATE_HASHED, &napi->state))
Eric Dumazet52bd2d62015-11-18 06:30:50 -08005289 return;
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005290
Eric Dumazet52bd2d62015-11-18 06:30:50 -08005291 spin_lock(&napi_hash_lock);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005292
Alexander Duyck545cd5e2017-03-24 10:07:53 -07005293 /* 0..NR_CPUS range is reserved for sender_cpu use */
Eric Dumazet52bd2d62015-11-18 06:30:50 -08005294 do {
Alexander Duyck545cd5e2017-03-24 10:07:53 -07005295 if (unlikely(++napi_gen_id < MIN_NAPI_ID))
5296 napi_gen_id = MIN_NAPI_ID;
Eric Dumazet52bd2d62015-11-18 06:30:50 -08005297 } while (napi_by_id(napi_gen_id));
5298 napi->napi_id = napi_gen_id;
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005299
Eric Dumazet52bd2d62015-11-18 06:30:50 -08005300 hlist_add_head_rcu(&napi->napi_hash_node,
5301 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005302
Eric Dumazet52bd2d62015-11-18 06:30:50 -08005303 spin_unlock(&napi_hash_lock);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005304}
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005305
5306/* Warning : caller is responsible to make sure rcu grace period
5307 * is respected before freeing memory containing @napi
5308 */
Eric Dumazet34cbe272015-11-18 06:31:02 -08005309bool napi_hash_del(struct napi_struct *napi)
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005310{
Eric Dumazet34cbe272015-11-18 06:31:02 -08005311 bool rcu_sync_needed = false;
5312
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005313 spin_lock(&napi_hash_lock);
5314
Eric Dumazet34cbe272015-11-18 06:31:02 -08005315 if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) {
5316 rcu_sync_needed = true;
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005317 hlist_del_rcu(&napi->napi_hash_node);
Eric Dumazet34cbe272015-11-18 06:31:02 -08005318 }
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005319 spin_unlock(&napi_hash_lock);
Eric Dumazet34cbe272015-11-18 06:31:02 -08005320 return rcu_sync_needed;
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005321}
5322EXPORT_SYMBOL_GPL(napi_hash_del);
5323
Eric Dumazet3b47d302014-11-06 21:09:44 -08005324static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
5325{
5326 struct napi_struct *napi;
5327
5328 napi = container_of(timer, struct napi_struct, timer);
Eric Dumazet39e6c822017-02-28 10:34:50 -08005329
5330 /* Note : we use a relaxed variant of napi_schedule_prep() not setting
5331 * NAPI_STATE_MISSED, since we do not react to a device IRQ.
5332 */
5333 if (napi->gro_list && !napi_disable_pending(napi) &&
5334 !test_and_set_bit(NAPI_STATE_SCHED, &napi->state))
5335 __napi_schedule_irqoff(napi);
Eric Dumazet3b47d302014-11-06 21:09:44 -08005336
5337 return HRTIMER_NORESTART;
5338}
5339
Herbert Xud565b0a2008-12-15 23:38:52 -08005340void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
5341 int (*poll)(struct napi_struct *, int), int weight)
5342{
5343 INIT_LIST_HEAD(&napi->poll_list);
Eric Dumazet3b47d302014-11-06 21:09:44 -08005344 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
5345 napi->timer.function = napi_watchdog;
Herbert Xu4ae55442009-02-08 18:00:36 +00005346 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08005347 napi->gro_list = NULL;
Herbert Xu5d38a072009-01-04 16:13:40 -08005348 napi->skb = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08005349 napi->poll = poll;
Eric Dumazet82dc3c63c2013-03-05 15:57:22 +00005350 if (weight > NAPI_POLL_WEIGHT)
5351 pr_err_once("netif_napi_add() called with weight %d on device %s\n",
5352 weight, dev->name);
Herbert Xud565b0a2008-12-15 23:38:52 -08005353 napi->weight = weight;
5354 list_add(&napi->dev_list, &dev->napi_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08005355 napi->dev = dev;
Herbert Xu5d38a072009-01-04 16:13:40 -08005356#ifdef CONFIG_NETPOLL
Herbert Xud565b0a2008-12-15 23:38:52 -08005357 napi->poll_owner = -1;
5358#endif
5359 set_bit(NAPI_STATE_SCHED, &napi->state);
Eric Dumazet93d05d42015-11-18 06:31:03 -08005360 napi_hash_add(napi);
Herbert Xud565b0a2008-12-15 23:38:52 -08005361}
5362EXPORT_SYMBOL(netif_napi_add);
5363
Eric Dumazet3b47d302014-11-06 21:09:44 -08005364void napi_disable(struct napi_struct *n)
5365{
5366 might_sleep();
5367 set_bit(NAPI_STATE_DISABLE, &n->state);
5368
5369 while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
5370 msleep(1);
Neil Horman2d8bff1262015-09-23 14:57:58 -04005371 while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state))
5372 msleep(1);
Eric Dumazet3b47d302014-11-06 21:09:44 -08005373
5374 hrtimer_cancel(&n->timer);
5375
5376 clear_bit(NAPI_STATE_DISABLE, &n->state);
5377}
5378EXPORT_SYMBOL(napi_disable);
5379
Eric Dumazet93d05d42015-11-18 06:31:03 -08005380/* Must be called in process context */
Herbert Xud565b0a2008-12-15 23:38:52 -08005381void netif_napi_del(struct napi_struct *napi)
5382{
Eric Dumazet93d05d42015-11-18 06:31:03 -08005383 might_sleep();
5384 if (napi_hash_del(napi))
5385 synchronize_net();
Peter P Waskiewicz Jrd7b06632008-12-26 01:35:35 -08005386 list_del_init(&napi->dev_list);
Herbert Xu76620aa2009-04-16 02:02:07 -07005387 napi_free_frags(napi);
Herbert Xud565b0a2008-12-15 23:38:52 -08005388
Eric Dumazet289dccb2013-12-20 14:29:08 -08005389 kfree_skb_list(napi->gro_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08005390 napi->gro_list = NULL;
Herbert Xu4ae55442009-02-08 18:00:36 +00005391 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08005392}
5393EXPORT_SYMBOL(netif_napi_del);
5394
Herbert Xu726ce702014-12-21 07:16:21 +11005395static int napi_poll(struct napi_struct *n, struct list_head *repoll)
5396{
5397 void *have;
5398 int work, weight;
5399
5400 list_del_init(&n->poll_list);
5401
5402 have = netpoll_poll_lock(n);
5403
5404 weight = n->weight;
5405
5406 /* This NAPI_STATE_SCHED test is for avoiding a race
5407 * with netpoll's poll_napi(). Only the entity which
5408 * obtains the lock and sees NAPI_STATE_SCHED set will
5409 * actually make the ->poll() call. Therefore we avoid
5410 * accidentally calling ->poll() when NAPI is not scheduled.
5411 */
5412 work = 0;
5413 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
5414 work = n->poll(n, weight);
Jesper Dangaard Brouer1db19db2016-07-07 18:01:32 +02005415 trace_napi_poll(n, work, weight);
Herbert Xu726ce702014-12-21 07:16:21 +11005416 }
5417
5418 WARN_ON_ONCE(work > weight);
5419
5420 if (likely(work < weight))
5421 goto out_unlock;
5422
5423 /* Drivers must not modify the NAPI state if they
5424 * consume the entire weight. In such cases this code
5425 * still "owns" the NAPI instance and therefore can
5426 * move the instance around on the list at-will.
5427 */
5428 if (unlikely(napi_disable_pending(n))) {
5429 napi_complete(n);
5430 goto out_unlock;
5431 }
5432
5433 if (n->gro_list) {
5434 /* flush too old packets
5435 * If HZ < 1000, flush all packets.
5436 */
5437 napi_gro_flush(n, HZ >= 1000);
5438 }
5439
Herbert Xu001ce542014-12-21 07:16:22 +11005440 /* Some drivers may have called napi_schedule
5441 * prior to exhausting their budget.
5442 */
5443 if (unlikely(!list_empty(&n->poll_list))) {
5444 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
5445 n->dev ? n->dev->name : "backlog");
5446 goto out_unlock;
5447 }
5448
Herbert Xu726ce702014-12-21 07:16:21 +11005449 list_add_tail(&n->poll_list, repoll);
5450
5451out_unlock:
5452 netpoll_poll_unlock(have);
5453
5454 return work;
5455}
5456
Emese Revfy0766f782016-06-20 20:42:34 +02005457static __latent_entropy void net_rx_action(struct softirq_action *h)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005458{
Christoph Lameter903ceff2014-08-17 12:30:35 -05005459 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
Matthew Whitehead7acf8a12017-04-19 12:37:10 -04005460 unsigned long time_limit = jiffies +
5461 usecs_to_jiffies(netdev_budget_usecs);
Stephen Hemminger51b0bde2005-06-23 20:14:40 -07005462 int budget = netdev_budget;
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005463 LIST_HEAD(list);
5464 LIST_HEAD(repoll);
Matt Mackall53fb95d2005-08-11 19:27:43 -07005465
Linus Torvalds1da177e2005-04-16 15:20:36 -07005466 local_irq_disable();
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005467 list_splice_init(&sd->poll_list, &list);
5468 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005469
Herbert Xuceb8d5b2014-12-21 07:16:25 +11005470 for (;;) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005471 struct napi_struct *n;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005472
Herbert Xuceb8d5b2014-12-21 07:16:25 +11005473 if (list_empty(&list)) {
5474 if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
Eric Dumazetf52dffe2016-11-23 08:44:56 -08005475 goto out;
Herbert Xuceb8d5b2014-12-21 07:16:25 +11005476 break;
5477 }
5478
Herbert Xu6bd373e2014-12-21 07:16:24 +11005479 n = list_first_entry(&list, struct napi_struct, poll_list);
5480 budget -= napi_poll(n, &repoll);
5481
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005482 /* If softirq window is exhausted then punt.
Stephen Hemminger24f8b232008-11-03 17:14:38 -08005483 * Allow this to run for 2 jiffies since which will allow
5484 * an average latency of 1.5/HZ.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005485 */
Herbert Xuceb8d5b2014-12-21 07:16:25 +11005486 if (unlikely(budget <= 0 ||
5487 time_after_eq(jiffies, time_limit))) {
5488 sd->time_squeeze++;
5489 break;
5490 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005491 }
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005492
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005493 local_irq_disable();
5494
5495 list_splice_tail_init(&sd->poll_list, &list);
5496 list_splice_tail(&repoll, &list);
5497 list_splice(&list, &sd->poll_list);
5498 if (!list_empty(&sd->poll_list))
5499 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
5500
Eric Dumazete326bed2010-04-22 00:22:45 -07005501 net_rps_action_and_irq_enable(sd);
Eric Dumazetf52dffe2016-11-23 08:44:56 -08005502out:
5503 __kfree_skb_flush();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005504}
5505
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02005506struct netdev_adjacent {
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005507 struct net_device *dev;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005508
5509 /* upper master flag, there can only be one master device per list */
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005510 bool master;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005511
Veaceslav Falico5d261912013-08-28 23:25:05 +02005512 /* counter for the number of times this device was added to us */
5513 u16 ref_nr;
5514
Veaceslav Falico402dae92013-09-25 09:20:09 +02005515 /* private field for the users */
5516 void *private;
5517
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005518 struct list_head list;
5519 struct rcu_head rcu;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005520};
5521
Michal Kubeček6ea29da2015-09-24 10:59:05 +02005522static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005523 struct list_head *adj_list)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005524{
Veaceslav Falico5d261912013-08-28 23:25:05 +02005525 struct netdev_adjacent *adj;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005526
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005527 list_for_each_entry(adj, adj_list, list) {
Veaceslav Falico5d261912013-08-28 23:25:05 +02005528 if (adj->dev == adj_dev)
5529 return adj;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005530 }
5531 return NULL;
5532}
5533
David Ahernf1170fd2016-10-17 19:15:51 -07005534static int __netdev_has_upper_dev(struct net_device *upper_dev, void *data)
5535{
5536 struct net_device *dev = data;
5537
5538 return upper_dev == dev;
5539}
5540
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005541/**
5542 * netdev_has_upper_dev - Check if device is linked to an upper device
5543 * @dev: device
5544 * @upper_dev: upper device to check
5545 *
5546 * Find out if a device is linked to specified upper device and return true
5547 * in case it is. Note that this checks only immediate upper device,
5548 * not through a complete stack of devices. The caller must hold the RTNL lock.
5549 */
5550bool netdev_has_upper_dev(struct net_device *dev,
5551 struct net_device *upper_dev)
5552{
5553 ASSERT_RTNL();
5554
David Ahernf1170fd2016-10-17 19:15:51 -07005555 return netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev,
5556 upper_dev);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005557}
5558EXPORT_SYMBOL(netdev_has_upper_dev);
5559
5560/**
David Ahern1a3f0602016-10-17 19:15:44 -07005561 * netdev_has_upper_dev_all - Check if device is linked to an upper device
5562 * @dev: device
5563 * @upper_dev: upper device to check
5564 *
5565 * Find out if a device is linked to specified upper device and return true
5566 * in case it is. Note that this checks the entire upper device chain.
5567 * The caller must hold rcu lock.
5568 */
5569
David Ahern1a3f0602016-10-17 19:15:44 -07005570bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
5571 struct net_device *upper_dev)
5572{
5573 return !!netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev,
5574 upper_dev);
5575}
5576EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu);
5577
5578/**
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005579 * netdev_has_any_upper_dev - Check if device is linked to some device
5580 * @dev: device
5581 *
5582 * Find out if a device is linked to an upper device and return true in case
5583 * it is. The caller must hold the RTNL lock.
5584 */
stephen hemminger1d143d92013-12-29 14:01:29 -08005585static bool netdev_has_any_upper_dev(struct net_device *dev)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005586{
5587 ASSERT_RTNL();
5588
David Ahernf1170fd2016-10-17 19:15:51 -07005589 return !list_empty(&dev->adj_list.upper);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005590}
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005591
5592/**
5593 * netdev_master_upper_dev_get - Get master upper device
5594 * @dev: device
5595 *
5596 * Find a master upper device and return pointer to it or NULL in case
5597 * it's not there. The caller must hold the RTNL lock.
5598 */
5599struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
5600{
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02005601 struct netdev_adjacent *upper;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005602
5603 ASSERT_RTNL();
5604
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005605 if (list_empty(&dev->adj_list.upper))
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005606 return NULL;
5607
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005608 upper = list_first_entry(&dev->adj_list.upper,
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02005609 struct netdev_adjacent, list);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005610 if (likely(upper->master))
5611 return upper->dev;
5612 return NULL;
5613}
5614EXPORT_SYMBOL(netdev_master_upper_dev_get);
5615
David Ahern0f524a82016-10-17 19:15:52 -07005616/**
5617 * netdev_has_any_lower_dev - Check if device is linked to some device
5618 * @dev: device
5619 *
5620 * Find out if a device is linked to a lower device and return true in case
5621 * it is. The caller must hold the RTNL lock.
5622 */
5623static bool netdev_has_any_lower_dev(struct net_device *dev)
5624{
5625 ASSERT_RTNL();
5626
5627 return !list_empty(&dev->adj_list.lower);
5628}
5629
Veaceslav Falicob6ccba42013-09-25 09:20:23 +02005630void *netdev_adjacent_get_private(struct list_head *adj_list)
5631{
5632 struct netdev_adjacent *adj;
5633
5634 adj = list_entry(adj_list, struct netdev_adjacent, list);
5635
5636 return adj->private;
5637}
5638EXPORT_SYMBOL(netdev_adjacent_get_private);
5639
Veaceslav Falico31088a12013-09-25 09:20:12 +02005640/**
Vlad Yasevich44a40852014-05-16 17:20:38 -04005641 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
5642 * @dev: device
5643 * @iter: list_head ** of the current position
5644 *
5645 * Gets the next device from the dev's upper list, starting from iter
5646 * position. The caller must hold RCU read lock.
5647 */
5648struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
5649 struct list_head **iter)
5650{
5651 struct netdev_adjacent *upper;
5652
5653 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5654
5655 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5656
5657 if (&upper->list == &dev->adj_list.upper)
5658 return NULL;
5659
5660 *iter = &upper->list;
5661
5662 return upper->dev;
5663}
5664EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
5665
David Ahern1a3f0602016-10-17 19:15:44 -07005666static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev,
5667 struct list_head **iter)
5668{
5669 struct netdev_adjacent *upper;
5670
5671 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5672
5673 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5674
5675 if (&upper->list == &dev->adj_list.upper)
5676 return NULL;
5677
5678 *iter = &upper->list;
5679
5680 return upper->dev;
5681}
5682
5683int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
5684 int (*fn)(struct net_device *dev,
5685 void *data),
5686 void *data)
5687{
5688 struct net_device *udev;
5689 struct list_head *iter;
5690 int ret;
5691
5692 for (iter = &dev->adj_list.upper,
5693 udev = netdev_next_upper_dev_rcu(dev, &iter);
5694 udev;
5695 udev = netdev_next_upper_dev_rcu(dev, &iter)) {
5696 /* first is the upper device itself */
5697 ret = fn(udev, data);
5698 if (ret)
5699 return ret;
5700
5701 /* then look at all of its upper devices */
5702 ret = netdev_walk_all_upper_dev_rcu(udev, fn, data);
5703 if (ret)
5704 return ret;
5705 }
5706
5707 return 0;
5708}
5709EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu);
5710
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005711/**
Veaceslav Falico31088a12013-09-25 09:20:12 +02005712 * netdev_lower_get_next_private - Get the next ->private from the
5713 * lower neighbour list
5714 * @dev: device
5715 * @iter: list_head ** of the current position
5716 *
5717 * Gets the next netdev_adjacent->private from the dev's lower neighbour
5718 * list, starting from iter position. The caller must hold either hold the
5719 * RTNL lock or its own locking that guarantees that the neighbour lower
subashab@codeaurora.orgb4691392015-07-24 03:03:29 +00005720 * list will remain unchanged.
Veaceslav Falico31088a12013-09-25 09:20:12 +02005721 */
5722void *netdev_lower_get_next_private(struct net_device *dev,
5723 struct list_head **iter)
5724{
5725 struct netdev_adjacent *lower;
5726
5727 lower = list_entry(*iter, struct netdev_adjacent, list);
5728
5729 if (&lower->list == &dev->adj_list.lower)
5730 return NULL;
5731
Veaceslav Falico6859e7d2014-04-07 11:25:12 +02005732 *iter = lower->list.next;
Veaceslav Falico31088a12013-09-25 09:20:12 +02005733
5734 return lower->private;
5735}
5736EXPORT_SYMBOL(netdev_lower_get_next_private);
5737
5738/**
5739 * netdev_lower_get_next_private_rcu - Get the next ->private from the
5740 * lower neighbour list, RCU
5741 * variant
5742 * @dev: device
5743 * @iter: list_head ** of the current position
5744 *
5745 * Gets the next netdev_adjacent->private from the dev's lower neighbour
5746 * list, starting from iter position. The caller must hold RCU read lock.
5747 */
5748void *netdev_lower_get_next_private_rcu(struct net_device *dev,
5749 struct list_head **iter)
5750{
5751 struct netdev_adjacent *lower;
5752
5753 WARN_ON_ONCE(!rcu_read_lock_held());
5754
5755 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5756
5757 if (&lower->list == &dev->adj_list.lower)
5758 return NULL;
5759
Veaceslav Falico6859e7d2014-04-07 11:25:12 +02005760 *iter = &lower->list;
Veaceslav Falico31088a12013-09-25 09:20:12 +02005761
5762 return lower->private;
5763}
5764EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
5765
5766/**
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04005767 * netdev_lower_get_next - Get the next device from the lower neighbour
5768 * list
5769 * @dev: device
5770 * @iter: list_head ** of the current position
5771 *
5772 * Gets the next netdev_adjacent from the dev's lower neighbour
5773 * list, starting from iter position. The caller must hold RTNL lock or
5774 * its own locking that guarantees that the neighbour lower
subashab@codeaurora.orgb4691392015-07-24 03:03:29 +00005775 * list will remain unchanged.
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04005776 */
5777void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
5778{
5779 struct netdev_adjacent *lower;
5780
Nikolay Aleksandrovcfdd28b2016-02-17 18:00:31 +01005781 lower = list_entry(*iter, struct netdev_adjacent, list);
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04005782
5783 if (&lower->list == &dev->adj_list.lower)
5784 return NULL;
5785
Nikolay Aleksandrovcfdd28b2016-02-17 18:00:31 +01005786 *iter = lower->list.next;
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04005787
5788 return lower->dev;
5789}
5790EXPORT_SYMBOL(netdev_lower_get_next);
5791
David Ahern1a3f0602016-10-17 19:15:44 -07005792static struct net_device *netdev_next_lower_dev(struct net_device *dev,
5793 struct list_head **iter)
5794{
5795 struct netdev_adjacent *lower;
5796
David Ahern46b5ab12016-10-26 13:21:33 -07005797 lower = list_entry((*iter)->next, struct netdev_adjacent, list);
David Ahern1a3f0602016-10-17 19:15:44 -07005798
5799 if (&lower->list == &dev->adj_list.lower)
5800 return NULL;
5801
David Ahern46b5ab12016-10-26 13:21:33 -07005802 *iter = &lower->list;
David Ahern1a3f0602016-10-17 19:15:44 -07005803
5804 return lower->dev;
5805}
5806
5807int netdev_walk_all_lower_dev(struct net_device *dev,
5808 int (*fn)(struct net_device *dev,
5809 void *data),
5810 void *data)
5811{
5812 struct net_device *ldev;
5813 struct list_head *iter;
5814 int ret;
5815
5816 for (iter = &dev->adj_list.lower,
5817 ldev = netdev_next_lower_dev(dev, &iter);
5818 ldev;
5819 ldev = netdev_next_lower_dev(dev, &iter)) {
5820 /* first is the lower device itself */
5821 ret = fn(ldev, data);
5822 if (ret)
5823 return ret;
5824
5825 /* then look at all of its lower devices */
5826 ret = netdev_walk_all_lower_dev(ldev, fn, data);
5827 if (ret)
5828 return ret;
5829 }
5830
5831 return 0;
5832}
5833EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev);
5834
David Ahern1a3f0602016-10-17 19:15:44 -07005835static struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
5836 struct list_head **iter)
5837{
5838 struct netdev_adjacent *lower;
5839
5840 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5841 if (&lower->list == &dev->adj_list.lower)
5842 return NULL;
5843
5844 *iter = &lower->list;
5845
5846 return lower->dev;
5847}
5848
5849int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
5850 int (*fn)(struct net_device *dev,
5851 void *data),
5852 void *data)
5853{
5854 struct net_device *ldev;
5855 struct list_head *iter;
5856 int ret;
5857
5858 for (iter = &dev->adj_list.lower,
5859 ldev = netdev_next_lower_dev_rcu(dev, &iter);
5860 ldev;
5861 ldev = netdev_next_lower_dev_rcu(dev, &iter)) {
5862 /* first is the lower device itself */
5863 ret = fn(ldev, data);
5864 if (ret)
5865 return ret;
5866
5867 /* then look at all of its lower devices */
5868 ret = netdev_walk_all_lower_dev_rcu(ldev, fn, data);
5869 if (ret)
5870 return ret;
5871 }
5872
5873 return 0;
5874}
5875EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu);
5876
Jiri Pirko7ce856a2016-07-04 08:23:12 +02005877/**
dingtianhonge001bfa2013-12-13 10:19:55 +08005878 * netdev_lower_get_first_private_rcu - Get the first ->private from the
5879 * lower neighbour list, RCU
5880 * variant
5881 * @dev: device
5882 *
5883 * Gets the first netdev_adjacent->private from the dev's lower neighbour
5884 * list. The caller must hold RCU read lock.
5885 */
5886void *netdev_lower_get_first_private_rcu(struct net_device *dev)
5887{
5888 struct netdev_adjacent *lower;
5889
5890 lower = list_first_or_null_rcu(&dev->adj_list.lower,
5891 struct netdev_adjacent, list);
5892 if (lower)
5893 return lower->private;
5894 return NULL;
5895}
5896EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
5897
5898/**
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005899 * netdev_master_upper_dev_get_rcu - Get master upper device
5900 * @dev: device
5901 *
5902 * Find a master upper device and return pointer to it or NULL in case
5903 * it's not there. The caller must hold the RCU read lock.
5904 */
5905struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
5906{
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02005907 struct netdev_adjacent *upper;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005908
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005909 upper = list_first_or_null_rcu(&dev->adj_list.upper,
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02005910 struct netdev_adjacent, list);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005911 if (upper && likely(upper->master))
5912 return upper->dev;
5913 return NULL;
5914}
5915EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
5916
Rashika Kheria0a59f3a2014-02-09 20:26:25 +05305917static int netdev_adjacent_sysfs_add(struct net_device *dev,
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005918 struct net_device *adj_dev,
5919 struct list_head *dev_list)
5920{
5921 char linkname[IFNAMSIZ+7];
tchardingf4563a72017-02-09 17:56:07 +11005922
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005923 sprintf(linkname, dev_list == &dev->adj_list.upper ?
5924 "upper_%s" : "lower_%s", adj_dev->name);
5925 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
5926 linkname);
5927}
Rashika Kheria0a59f3a2014-02-09 20:26:25 +05305928static void netdev_adjacent_sysfs_del(struct net_device *dev,
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005929 char *name,
5930 struct list_head *dev_list)
5931{
5932 char linkname[IFNAMSIZ+7];
tchardingf4563a72017-02-09 17:56:07 +11005933
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005934 sprintf(linkname, dev_list == &dev->adj_list.upper ?
5935 "upper_%s" : "lower_%s", name);
5936 sysfs_remove_link(&(dev->dev.kobj), linkname);
5937}
5938
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04005939static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
5940 struct net_device *adj_dev,
5941 struct list_head *dev_list)
5942{
5943 return (dev_list == &dev->adj_list.upper ||
5944 dev_list == &dev->adj_list.lower) &&
5945 net_eq(dev_net(dev), dev_net(adj_dev));
5946}
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005947
Veaceslav Falico5d261912013-08-28 23:25:05 +02005948static int __netdev_adjacent_dev_insert(struct net_device *dev,
5949 struct net_device *adj_dev,
Veaceslav Falico7863c052013-09-25 09:20:06 +02005950 struct list_head *dev_list,
Veaceslav Falico402dae92013-09-25 09:20:09 +02005951 void *private, bool master)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005952{
5953 struct netdev_adjacent *adj;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005954 int ret;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005955
Michal Kubeček6ea29da2015-09-24 10:59:05 +02005956 adj = __netdev_find_adj(adj_dev, dev_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005957
5958 if (adj) {
David Ahern790510d2016-10-17 19:15:43 -07005959 adj->ref_nr += 1;
David Ahern67b62f92016-10-17 19:15:53 -07005960 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n",
5961 dev->name, adj_dev->name, adj->ref_nr);
5962
Veaceslav Falico5d261912013-08-28 23:25:05 +02005963 return 0;
5964 }
5965
5966 adj = kmalloc(sizeof(*adj), GFP_KERNEL);
5967 if (!adj)
5968 return -ENOMEM;
5969
5970 adj->dev = adj_dev;
5971 adj->master = master;
David Ahern790510d2016-10-17 19:15:43 -07005972 adj->ref_nr = 1;
Veaceslav Falico402dae92013-09-25 09:20:09 +02005973 adj->private = private;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005974 dev_hold(adj_dev);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005975
David Ahern67b62f92016-10-17 19:15:53 -07005976 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n",
5977 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005978
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04005979 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005980 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
Veaceslav Falico5831d662013-09-25 09:20:32 +02005981 if (ret)
5982 goto free_adj;
5983 }
5984
Veaceslav Falico7863c052013-09-25 09:20:06 +02005985 /* Ensure that master link is always the first item in list. */
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005986 if (master) {
5987 ret = sysfs_create_link(&(dev->dev.kobj),
5988 &(adj_dev->dev.kobj), "master");
5989 if (ret)
Veaceslav Falico5831d662013-09-25 09:20:32 +02005990 goto remove_symlinks;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005991
Veaceslav Falico7863c052013-09-25 09:20:06 +02005992 list_add_rcu(&adj->list, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005993 } else {
Veaceslav Falico7863c052013-09-25 09:20:06 +02005994 list_add_tail_rcu(&adj->list, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005995 }
Veaceslav Falico5d261912013-08-28 23:25:05 +02005996
5997 return 0;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005998
Veaceslav Falico5831d662013-09-25 09:20:32 +02005999remove_symlinks:
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04006000 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
Veaceslav Falico3ee32702014-01-14 21:58:50 +01006001 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02006002free_adj:
6003 kfree(adj);
Nikolay Aleksandrov974daef2013-10-23 15:28:56 +02006004 dev_put(adj_dev);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02006005
6006 return ret;
Veaceslav Falico5d261912013-08-28 23:25:05 +02006007}
6008
stephen hemminger1d143d92013-12-29 14:01:29 -08006009static void __netdev_adjacent_dev_remove(struct net_device *dev,
6010 struct net_device *adj_dev,
Andrew Collins93409032016-10-03 13:43:02 -06006011 u16 ref_nr,
stephen hemminger1d143d92013-12-29 14:01:29 -08006012 struct list_head *dev_list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02006013{
6014 struct netdev_adjacent *adj;
6015
David Ahern67b62f92016-10-17 19:15:53 -07006016 pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n",
6017 dev->name, adj_dev->name, ref_nr);
6018
Michal Kubeček6ea29da2015-09-24 10:59:05 +02006019 adj = __netdev_find_adj(adj_dev, dev_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02006020
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006021 if (!adj) {
David Ahern67b62f92016-10-17 19:15:53 -07006022 pr_err("Adjacency does not exist for device %s from %s\n",
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006023 dev->name, adj_dev->name);
David Ahern67b62f92016-10-17 19:15:53 -07006024 WARN_ON(1);
6025 return;
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006026 }
Veaceslav Falico5d261912013-08-28 23:25:05 +02006027
Andrew Collins93409032016-10-03 13:43:02 -06006028 if (adj->ref_nr > ref_nr) {
David Ahern67b62f92016-10-17 19:15:53 -07006029 pr_debug("adjacency: %s to %s ref_nr - %d = %d\n",
6030 dev->name, adj_dev->name, ref_nr,
6031 adj->ref_nr - ref_nr);
Andrew Collins93409032016-10-03 13:43:02 -06006032 adj->ref_nr -= ref_nr;
Veaceslav Falico5d261912013-08-28 23:25:05 +02006033 return;
6034 }
6035
Veaceslav Falico842d67a2013-09-25 09:20:31 +02006036 if (adj->master)
6037 sysfs_remove_link(&(dev->dev.kobj), "master");
6038
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04006039 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
Veaceslav Falico3ee32702014-01-14 21:58:50 +01006040 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
Veaceslav Falico5831d662013-09-25 09:20:32 +02006041
Veaceslav Falico5d261912013-08-28 23:25:05 +02006042 list_del_rcu(&adj->list);
David Ahern67b62f92016-10-17 19:15:53 -07006043 pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n",
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006044 adj_dev->name, dev->name, adj_dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02006045 dev_put(adj_dev);
6046 kfree_rcu(adj, rcu);
6047}
6048
stephen hemminger1d143d92013-12-29 14:01:29 -08006049static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
6050 struct net_device *upper_dev,
6051 struct list_head *up_list,
6052 struct list_head *down_list,
6053 void *private, bool master)
Veaceslav Falico5d261912013-08-28 23:25:05 +02006054{
6055 int ret;
6056
David Ahern790510d2016-10-17 19:15:43 -07006057 ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list,
Andrew Collins93409032016-10-03 13:43:02 -06006058 private, master);
Veaceslav Falico5d261912013-08-28 23:25:05 +02006059 if (ret)
6060 return ret;
6061
David Ahern790510d2016-10-17 19:15:43 -07006062 ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list,
Andrew Collins93409032016-10-03 13:43:02 -06006063 private, false);
Veaceslav Falico5d261912013-08-28 23:25:05 +02006064 if (ret) {
David Ahern790510d2016-10-17 19:15:43 -07006065 __netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02006066 return ret;
6067 }
6068
6069 return 0;
6070}
6071
stephen hemminger1d143d92013-12-29 14:01:29 -08006072static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
6073 struct net_device *upper_dev,
Andrew Collins93409032016-10-03 13:43:02 -06006074 u16 ref_nr,
stephen hemminger1d143d92013-12-29 14:01:29 -08006075 struct list_head *up_list,
6076 struct list_head *down_list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02006077{
Andrew Collins93409032016-10-03 13:43:02 -06006078 __netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list);
6079 __netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02006080}
6081
stephen hemminger1d143d92013-12-29 14:01:29 -08006082static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
6083 struct net_device *upper_dev,
6084 void *private, bool master)
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006085{
David Ahernf1170fd2016-10-17 19:15:51 -07006086 return __netdev_adjacent_dev_link_lists(dev, upper_dev,
6087 &dev->adj_list.upper,
6088 &upper_dev->adj_list.lower,
6089 private, master);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006090}
6091
stephen hemminger1d143d92013-12-29 14:01:29 -08006092static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
6093 struct net_device *upper_dev)
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006094{
Andrew Collins93409032016-10-03 13:43:02 -06006095 __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1,
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006096 &dev->adj_list.upper,
6097 &upper_dev->adj_list.lower);
6098}
Veaceslav Falico5d261912013-08-28 23:25:05 +02006099
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006100static int __netdev_upper_dev_link(struct net_device *dev,
Veaceslav Falico402dae92013-09-25 09:20:09 +02006101 struct net_device *upper_dev, bool master,
Jiri Pirko29bf24a2015-12-03 12:12:11 +01006102 void *upper_priv, void *upper_info)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006103{
Jiri Pirko0e4ead92015-08-27 09:31:18 +02006104 struct netdev_notifier_changeupper_info changeupper_info;
Veaceslav Falico5d261912013-08-28 23:25:05 +02006105 int ret = 0;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006106
6107 ASSERT_RTNL();
6108
6109 if (dev == upper_dev)
6110 return -EBUSY;
6111
6112 /* To prevent loops, check if dev is not upper device to upper_dev. */
David Ahernf1170fd2016-10-17 19:15:51 -07006113 if (netdev_has_upper_dev(upper_dev, dev))
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006114 return -EBUSY;
6115
David Ahernf1170fd2016-10-17 19:15:51 -07006116 if (netdev_has_upper_dev(dev, upper_dev))
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006117 return -EEXIST;
6118
6119 if (master && netdev_master_upper_dev_get(dev))
6120 return -EBUSY;
6121
Jiri Pirko0e4ead92015-08-27 09:31:18 +02006122 changeupper_info.upper_dev = upper_dev;
6123 changeupper_info.master = master;
6124 changeupper_info.linking = true;
Jiri Pirko29bf24a2015-12-03 12:12:11 +01006125 changeupper_info.upper_info = upper_info;
Jiri Pirko0e4ead92015-08-27 09:31:18 +02006126
Jiri Pirko573c7ba2015-10-16 14:01:22 +02006127 ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev,
6128 &changeupper_info.info);
6129 ret = notifier_to_errno(ret);
6130 if (ret)
6131 return ret;
6132
Jiri Pirko6dffb042015-12-03 12:12:10 +01006133 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
Veaceslav Falico402dae92013-09-25 09:20:09 +02006134 master);
Veaceslav Falico5d261912013-08-28 23:25:05 +02006135 if (ret)
6136 return ret;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006137
Ido Schimmelb03804e2015-12-03 12:12:03 +01006138 ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev,
6139 &changeupper_info.info);
6140 ret = notifier_to_errno(ret);
6141 if (ret)
David Ahernf1170fd2016-10-17 19:15:51 -07006142 goto rollback;
Ido Schimmelb03804e2015-12-03 12:12:03 +01006143
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006144 return 0;
Veaceslav Falico5d261912013-08-28 23:25:05 +02006145
David Ahernf1170fd2016-10-17 19:15:51 -07006146rollback:
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006147 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
Veaceslav Falico5d261912013-08-28 23:25:05 +02006148
6149 return ret;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006150}
6151
6152/**
6153 * netdev_upper_dev_link - Add a link to the upper device
6154 * @dev: device
6155 * @upper_dev: new upper device
6156 *
6157 * Adds a link to device which is upper to this one. The caller must hold
6158 * the RTNL lock. On a failure a negative errno code is returned.
6159 * On success the reference counts are adjusted and the function
6160 * returns zero.
6161 */
6162int netdev_upper_dev_link(struct net_device *dev,
6163 struct net_device *upper_dev)
6164{
Jiri Pirko29bf24a2015-12-03 12:12:11 +01006165 return __netdev_upper_dev_link(dev, upper_dev, false, NULL, NULL);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006166}
6167EXPORT_SYMBOL(netdev_upper_dev_link);
6168
6169/**
6170 * netdev_master_upper_dev_link - Add a master link to the upper device
6171 * @dev: device
6172 * @upper_dev: new upper device
Jiri Pirko6dffb042015-12-03 12:12:10 +01006173 * @upper_priv: upper device private
Jiri Pirko29bf24a2015-12-03 12:12:11 +01006174 * @upper_info: upper info to be passed down via notifier
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006175 *
6176 * Adds a link to device which is upper to this one. In this case, only
6177 * one master upper device can be linked, although other non-master devices
6178 * might be linked as well. The caller must hold the RTNL lock.
6179 * On a failure a negative errno code is returned. On success the reference
6180 * counts are adjusted and the function returns zero.
6181 */
6182int netdev_master_upper_dev_link(struct net_device *dev,
Jiri Pirko6dffb042015-12-03 12:12:10 +01006183 struct net_device *upper_dev,
Jiri Pirko29bf24a2015-12-03 12:12:11 +01006184 void *upper_priv, void *upper_info)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006185{
Jiri Pirko29bf24a2015-12-03 12:12:11 +01006186 return __netdev_upper_dev_link(dev, upper_dev, true,
6187 upper_priv, upper_info);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006188}
6189EXPORT_SYMBOL(netdev_master_upper_dev_link);
6190
6191/**
6192 * netdev_upper_dev_unlink - Removes a link to upper device
6193 * @dev: device
6194 * @upper_dev: new upper device
6195 *
6196 * Removes a link to device which is upper to this one. The caller must hold
6197 * the RTNL lock.
6198 */
6199void netdev_upper_dev_unlink(struct net_device *dev,
6200 struct net_device *upper_dev)
6201{
Jiri Pirko0e4ead92015-08-27 09:31:18 +02006202 struct netdev_notifier_changeupper_info changeupper_info;
tchardingf4563a72017-02-09 17:56:07 +11006203
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006204 ASSERT_RTNL();
6205
Jiri Pirko0e4ead92015-08-27 09:31:18 +02006206 changeupper_info.upper_dev = upper_dev;
6207 changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
6208 changeupper_info.linking = false;
6209
Jiri Pirko573c7ba2015-10-16 14:01:22 +02006210 call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev,
6211 &changeupper_info.info);
6212
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006213 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
Veaceslav Falico5d261912013-08-28 23:25:05 +02006214
Jiri Pirko0e4ead92015-08-27 09:31:18 +02006215 call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev,
6216 &changeupper_info.info);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006217}
6218EXPORT_SYMBOL(netdev_upper_dev_unlink);
6219
Moni Shoua61bd3852015-02-03 16:48:29 +02006220/**
6221 * netdev_bonding_info_change - Dispatch event about slave change
6222 * @dev: device
Masanari Iida4a26e4532015-02-14 22:26:34 +09006223 * @bonding_info: info to dispatch
Moni Shoua61bd3852015-02-03 16:48:29 +02006224 *
6225 * Send NETDEV_BONDING_INFO to netdev notifiers with info.
6226 * The caller must hold the RTNL lock.
6227 */
6228void netdev_bonding_info_change(struct net_device *dev,
6229 struct netdev_bonding_info *bonding_info)
6230{
6231 struct netdev_notifier_bonding_info info;
6232
6233 memcpy(&info.bonding_info, bonding_info,
6234 sizeof(struct netdev_bonding_info));
6235 call_netdevice_notifiers_info(NETDEV_BONDING_INFO, dev,
6236 &info.info);
6237}
6238EXPORT_SYMBOL(netdev_bonding_info_change);
6239
Eric Dumazet2ce1ee12015-02-04 13:37:44 -08006240static void netdev_adjacent_add_links(struct net_device *dev)
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006241{
6242 struct netdev_adjacent *iter;
6243
6244 struct net *net = dev_net(dev);
6245
6246 list_for_each_entry(iter, &dev->adj_list.upper, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08006247 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006248 continue;
6249 netdev_adjacent_sysfs_add(iter->dev, dev,
6250 &iter->dev->adj_list.lower);
6251 netdev_adjacent_sysfs_add(dev, iter->dev,
6252 &dev->adj_list.upper);
6253 }
6254
6255 list_for_each_entry(iter, &dev->adj_list.lower, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08006256 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006257 continue;
6258 netdev_adjacent_sysfs_add(iter->dev, dev,
6259 &iter->dev->adj_list.upper);
6260 netdev_adjacent_sysfs_add(dev, iter->dev,
6261 &dev->adj_list.lower);
6262 }
6263}
6264
Eric Dumazet2ce1ee12015-02-04 13:37:44 -08006265static void netdev_adjacent_del_links(struct net_device *dev)
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006266{
6267 struct netdev_adjacent *iter;
6268
6269 struct net *net = dev_net(dev);
6270
6271 list_for_each_entry(iter, &dev->adj_list.upper, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08006272 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006273 continue;
6274 netdev_adjacent_sysfs_del(iter->dev, dev->name,
6275 &iter->dev->adj_list.lower);
6276 netdev_adjacent_sysfs_del(dev, iter->dev->name,
6277 &dev->adj_list.upper);
6278 }
6279
6280 list_for_each_entry(iter, &dev->adj_list.lower, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08006281 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006282 continue;
6283 netdev_adjacent_sysfs_del(iter->dev, dev->name,
6284 &iter->dev->adj_list.upper);
6285 netdev_adjacent_sysfs_del(dev, iter->dev->name,
6286 &dev->adj_list.lower);
6287 }
6288}
6289
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01006290void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
Veaceslav Falico402dae92013-09-25 09:20:09 +02006291{
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01006292 struct netdev_adjacent *iter;
Veaceslav Falico402dae92013-09-25 09:20:09 +02006293
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006294 struct net *net = dev_net(dev);
6295
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01006296 list_for_each_entry(iter, &dev->adj_list.upper, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08006297 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006298 continue;
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01006299 netdev_adjacent_sysfs_del(iter->dev, oldname,
6300 &iter->dev->adj_list.lower);
6301 netdev_adjacent_sysfs_add(iter->dev, dev,
6302 &iter->dev->adj_list.lower);
6303 }
Veaceslav Falico402dae92013-09-25 09:20:09 +02006304
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01006305 list_for_each_entry(iter, &dev->adj_list.lower, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08006306 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006307 continue;
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01006308 netdev_adjacent_sysfs_del(iter->dev, oldname,
6309 &iter->dev->adj_list.upper);
6310 netdev_adjacent_sysfs_add(iter->dev, dev,
6311 &iter->dev->adj_list.upper);
6312 }
Veaceslav Falico402dae92013-09-25 09:20:09 +02006313}
Veaceslav Falico402dae92013-09-25 09:20:09 +02006314
6315void *netdev_lower_dev_get_private(struct net_device *dev,
6316 struct net_device *lower_dev)
6317{
6318 struct netdev_adjacent *lower;
6319
6320 if (!lower_dev)
6321 return NULL;
Michal Kubeček6ea29da2015-09-24 10:59:05 +02006322 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
Veaceslav Falico402dae92013-09-25 09:20:09 +02006323 if (!lower)
6324 return NULL;
6325
6326 return lower->private;
6327}
6328EXPORT_SYMBOL(netdev_lower_dev_get_private);
6329
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04006330
Sabrina Dubroca952fcfd2016-08-12 16:10:33 +02006331int dev_get_nest_level(struct net_device *dev)
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04006332{
6333 struct net_device *lower = NULL;
6334 struct list_head *iter;
6335 int max_nest = -1;
6336 int nest;
6337
6338 ASSERT_RTNL();
6339
6340 netdev_for_each_lower_dev(dev, lower, iter) {
Sabrina Dubroca952fcfd2016-08-12 16:10:33 +02006341 nest = dev_get_nest_level(lower);
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04006342 if (max_nest < nest)
6343 max_nest = nest;
6344 }
6345
Sabrina Dubroca952fcfd2016-08-12 16:10:33 +02006346 return max_nest + 1;
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04006347}
6348EXPORT_SYMBOL(dev_get_nest_level);
6349
Jiri Pirko04d48262015-12-03 12:12:15 +01006350/**
6351 * netdev_lower_change - Dispatch event about lower device state change
6352 * @lower_dev: device
6353 * @lower_state_info: state to dispatch
6354 *
6355 * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info.
6356 * The caller must hold the RTNL lock.
6357 */
6358void netdev_lower_state_changed(struct net_device *lower_dev,
6359 void *lower_state_info)
6360{
6361 struct netdev_notifier_changelowerstate_info changelowerstate_info;
6362
6363 ASSERT_RTNL();
6364 changelowerstate_info.lower_state_info = lower_state_info;
6365 call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE, lower_dev,
6366 &changelowerstate_info.info);
6367}
6368EXPORT_SYMBOL(netdev_lower_state_changed);
6369
Patrick McHardyb6c40d62008-10-07 15:26:48 -07006370static void dev_change_rx_flags(struct net_device *dev, int flags)
6371{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006372 const struct net_device_ops *ops = dev->netdev_ops;
6373
Vlad Yasevichd2615bf2013-11-19 20:47:15 -05006374 if (ops->ndo_change_rx_flags)
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006375 ops->ndo_change_rx_flags(dev, flags);
Patrick McHardyb6c40d62008-10-07 15:26:48 -07006376}
6377
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006378static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
Patrick McHardy4417da62007-06-27 01:28:10 -07006379{
Eric Dumazetb536db92011-11-30 21:42:26 +00006380 unsigned int old_flags = dev->flags;
Eric W. Biedermand04a48b2012-05-23 17:01:57 -06006381 kuid_t uid;
6382 kgid_t gid;
Patrick McHardy4417da62007-06-27 01:28:10 -07006383
Patrick McHardy24023452007-07-14 18:51:31 -07006384 ASSERT_RTNL();
6385
Wang Chendad9b332008-06-18 01:48:28 -07006386 dev->flags |= IFF_PROMISC;
6387 dev->promiscuity += inc;
6388 if (dev->promiscuity == 0) {
6389 /*
6390 * Avoid overflow.
6391 * If inc causes overflow, untouch promisc and return error.
6392 */
6393 if (inc < 0)
6394 dev->flags &= ~IFF_PROMISC;
6395 else {
6396 dev->promiscuity -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006397 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
6398 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07006399 return -EOVERFLOW;
6400 }
6401 }
Patrick McHardy4417da62007-06-27 01:28:10 -07006402 if (dev->flags != old_flags) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006403 pr_info("device %s %s promiscuous mode\n",
6404 dev->name,
6405 dev->flags & IFF_PROMISC ? "entered" : "left");
David Howells8192b0c2008-11-14 10:39:10 +11006406 if (audit_enabled) {
6407 current_uid_gid(&uid, &gid);
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05006408 audit_log(current->audit_context, GFP_ATOMIC,
6409 AUDIT_ANOM_PROMISCUOUS,
6410 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
6411 dev->name, (dev->flags & IFF_PROMISC),
6412 (old_flags & IFF_PROMISC),
Eric W. Biedermane1760bd2012-09-10 22:39:43 -07006413 from_kuid(&init_user_ns, audit_get_loginuid(current)),
Eric W. Biedermand04a48b2012-05-23 17:01:57 -06006414 from_kuid(&init_user_ns, uid),
6415 from_kgid(&init_user_ns, gid),
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05006416 audit_get_sessionid(current));
David Howells8192b0c2008-11-14 10:39:10 +11006417 }
Patrick McHardy24023452007-07-14 18:51:31 -07006418
Patrick McHardyb6c40d62008-10-07 15:26:48 -07006419 dev_change_rx_flags(dev, IFF_PROMISC);
Patrick McHardy4417da62007-06-27 01:28:10 -07006420 }
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006421 if (notify)
6422 __dev_notify_flags(dev, old_flags, IFF_PROMISC);
Wang Chendad9b332008-06-18 01:48:28 -07006423 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07006424}
6425
Linus Torvalds1da177e2005-04-16 15:20:36 -07006426/**
6427 * dev_set_promiscuity - update promiscuity count on a device
6428 * @dev: device
6429 * @inc: modifier
6430 *
Stephen Hemminger3041a062006-05-26 13:25:24 -07006431 * Add or remove promiscuity from a device. While the count in the device
Linus Torvalds1da177e2005-04-16 15:20:36 -07006432 * remains above zero the interface remains promiscuous. Once it hits zero
6433 * the device reverts back to normal filtering operation. A negative inc
6434 * value is used to drop promiscuity on the device.
Wang Chendad9b332008-06-18 01:48:28 -07006435 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006436 */
Wang Chendad9b332008-06-18 01:48:28 -07006437int dev_set_promiscuity(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006438{
Eric Dumazetb536db92011-11-30 21:42:26 +00006439 unsigned int old_flags = dev->flags;
Wang Chendad9b332008-06-18 01:48:28 -07006440 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006441
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006442 err = __dev_set_promiscuity(dev, inc, true);
Patrick McHardy4b5a6982008-07-06 15:49:08 -07006443 if (err < 0)
Wang Chendad9b332008-06-18 01:48:28 -07006444 return err;
Patrick McHardy4417da62007-06-27 01:28:10 -07006445 if (dev->flags != old_flags)
6446 dev_set_rx_mode(dev);
Wang Chendad9b332008-06-18 01:48:28 -07006447 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006448}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006449EXPORT_SYMBOL(dev_set_promiscuity);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006450
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006451static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006452{
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006453 unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006454
Patrick McHardy24023452007-07-14 18:51:31 -07006455 ASSERT_RTNL();
6456
Linus Torvalds1da177e2005-04-16 15:20:36 -07006457 dev->flags |= IFF_ALLMULTI;
Wang Chendad9b332008-06-18 01:48:28 -07006458 dev->allmulti += inc;
6459 if (dev->allmulti == 0) {
6460 /*
6461 * Avoid overflow.
6462 * If inc causes overflow, untouch allmulti and return error.
6463 */
6464 if (inc < 0)
6465 dev->flags &= ~IFF_ALLMULTI;
6466 else {
6467 dev->allmulti -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006468 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
6469 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07006470 return -EOVERFLOW;
6471 }
6472 }
Patrick McHardy24023452007-07-14 18:51:31 -07006473 if (dev->flags ^ old_flags) {
Patrick McHardyb6c40d62008-10-07 15:26:48 -07006474 dev_change_rx_flags(dev, IFF_ALLMULTI);
Patrick McHardy4417da62007-06-27 01:28:10 -07006475 dev_set_rx_mode(dev);
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006476 if (notify)
6477 __dev_notify_flags(dev, old_flags,
6478 dev->gflags ^ old_gflags);
Patrick McHardy24023452007-07-14 18:51:31 -07006479 }
Wang Chendad9b332008-06-18 01:48:28 -07006480 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07006481}
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006482
6483/**
6484 * dev_set_allmulti - update allmulti count on a device
6485 * @dev: device
6486 * @inc: modifier
6487 *
6488 * Add or remove reception of all multicast frames to a device. While the
6489 * count in the device remains above zero the interface remains listening
6490 * to all interfaces. Once it hits zero the device reverts back to normal
6491 * filtering operation. A negative @inc value is used to drop the counter
6492 * when releasing a resource needing all multicasts.
6493 * Return 0 if successful or a negative errno code on error.
6494 */
6495
6496int dev_set_allmulti(struct net_device *dev, int inc)
6497{
6498 return __dev_set_allmulti(dev, inc, true);
6499}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006500EXPORT_SYMBOL(dev_set_allmulti);
Patrick McHardy4417da62007-06-27 01:28:10 -07006501
6502/*
6503 * Upload unicast and multicast address lists to device and
6504 * configure RX filtering. When the device doesn't support unicast
Joe Perches53ccaae2007-12-20 14:02:06 -08006505 * filtering it is put in promiscuous mode while unicast addresses
Patrick McHardy4417da62007-06-27 01:28:10 -07006506 * are present.
6507 */
6508void __dev_set_rx_mode(struct net_device *dev)
6509{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006510 const struct net_device_ops *ops = dev->netdev_ops;
6511
Patrick McHardy4417da62007-06-27 01:28:10 -07006512 /* dev_open will call this function so the list will stay sane. */
6513 if (!(dev->flags&IFF_UP))
6514 return;
6515
6516 if (!netif_device_present(dev))
YOSHIFUJI Hideaki40b77c92007-07-19 10:43:23 +09006517 return;
Patrick McHardy4417da62007-06-27 01:28:10 -07006518
Jiri Pirko01789342011-08-16 06:29:00 +00006519 if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
Patrick McHardy4417da62007-06-27 01:28:10 -07006520 /* Unicast addresses changes may only happen under the rtnl,
6521 * therefore calling __dev_set_promiscuity here is safe.
6522 */
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08006523 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006524 __dev_set_promiscuity(dev, 1, false);
Joe Perches2d348d12011-07-25 16:17:35 -07006525 dev->uc_promisc = true;
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08006526 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006527 __dev_set_promiscuity(dev, -1, false);
Joe Perches2d348d12011-07-25 16:17:35 -07006528 dev->uc_promisc = false;
Patrick McHardy4417da62007-06-27 01:28:10 -07006529 }
Patrick McHardy4417da62007-06-27 01:28:10 -07006530 }
Jiri Pirko01789342011-08-16 06:29:00 +00006531
6532 if (ops->ndo_set_rx_mode)
6533 ops->ndo_set_rx_mode(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07006534}
6535
6536void dev_set_rx_mode(struct net_device *dev)
6537{
David S. Millerb9e40852008-07-15 00:15:08 -07006538 netif_addr_lock_bh(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07006539 __dev_set_rx_mode(dev);
David S. Millerb9e40852008-07-15 00:15:08 -07006540 netif_addr_unlock_bh(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006541}
6542
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006543/**
6544 * dev_get_flags - get flags reported to userspace
6545 * @dev: device
6546 *
6547 * Get the combination of flag bits exported through APIs to userspace.
6548 */
Eric Dumazet95c96172012-04-15 05:58:06 +00006549unsigned int dev_get_flags(const struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006550{
Eric Dumazet95c96172012-04-15 05:58:06 +00006551 unsigned int flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006552
6553 flags = (dev->flags & ~(IFF_PROMISC |
6554 IFF_ALLMULTI |
Stefan Rompfb00055a2006-03-20 17:09:11 -08006555 IFF_RUNNING |
6556 IFF_LOWER_UP |
6557 IFF_DORMANT)) |
Linus Torvalds1da177e2005-04-16 15:20:36 -07006558 (dev->gflags & (IFF_PROMISC |
6559 IFF_ALLMULTI));
6560
Stefan Rompfb00055a2006-03-20 17:09:11 -08006561 if (netif_running(dev)) {
6562 if (netif_oper_up(dev))
6563 flags |= IFF_RUNNING;
6564 if (netif_carrier_ok(dev))
6565 flags |= IFF_LOWER_UP;
6566 if (netif_dormant(dev))
6567 flags |= IFF_DORMANT;
6568 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006569
6570 return flags;
6571}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006572EXPORT_SYMBOL(dev_get_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006573
Patrick McHardybd380812010-02-26 06:34:53 +00006574int __dev_change_flags(struct net_device *dev, unsigned int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006575{
Eric Dumazetb536db92011-11-30 21:42:26 +00006576 unsigned int old_flags = dev->flags;
Patrick McHardybd380812010-02-26 06:34:53 +00006577 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006578
Patrick McHardy24023452007-07-14 18:51:31 -07006579 ASSERT_RTNL();
6580
Linus Torvalds1da177e2005-04-16 15:20:36 -07006581 /*
6582 * Set the flags on our device.
6583 */
6584
6585 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
6586 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
6587 IFF_AUTOMEDIA)) |
6588 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
6589 IFF_ALLMULTI));
6590
6591 /*
6592 * Load in the correct multicast list now the flags have changed.
6593 */
6594
Patrick McHardyb6c40d62008-10-07 15:26:48 -07006595 if ((old_flags ^ flags) & IFF_MULTICAST)
6596 dev_change_rx_flags(dev, IFF_MULTICAST);
Patrick McHardy24023452007-07-14 18:51:31 -07006597
Patrick McHardy4417da62007-06-27 01:28:10 -07006598 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006599
6600 /*
6601 * Have we downed the interface. We handle IFF_UP ourselves
6602 * according to user attempts to set it, rather than blindly
6603 * setting it.
6604 */
6605
6606 ret = 0;
Peter Pan(潘卫平)d215d102014-06-16 21:57:22 +08006607 if ((old_flags ^ flags) & IFF_UP)
Patrick McHardybd380812010-02-26 06:34:53 +00006608 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006609
Linus Torvalds1da177e2005-04-16 15:20:36 -07006610 if ((flags ^ dev->gflags) & IFF_PROMISC) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006611 int inc = (flags & IFF_PROMISC) ? 1 : -1;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006612 unsigned int old_flags = dev->flags;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006613
Linus Torvalds1da177e2005-04-16 15:20:36 -07006614 dev->gflags ^= IFF_PROMISC;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006615
6616 if (__dev_set_promiscuity(dev, inc, false) >= 0)
6617 if (dev->flags != old_flags)
6618 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006619 }
6620
6621 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
tchardingeb13da12017-02-09 17:56:06 +11006622 * is important. Some (broken) drivers set IFF_PROMISC, when
6623 * IFF_ALLMULTI is requested not asking us and not reporting.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006624 */
6625 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006626 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
6627
Linus Torvalds1da177e2005-04-16 15:20:36 -07006628 dev->gflags ^= IFF_ALLMULTI;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006629 __dev_set_allmulti(dev, inc, false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006630 }
6631
Patrick McHardybd380812010-02-26 06:34:53 +00006632 return ret;
6633}
6634
Nicolas Dichtela528c212013-09-25 12:02:44 +02006635void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
6636 unsigned int gchanges)
Patrick McHardybd380812010-02-26 06:34:53 +00006637{
6638 unsigned int changes = dev->flags ^ old_flags;
6639
Nicolas Dichtela528c212013-09-25 12:02:44 +02006640 if (gchanges)
Alexei Starovoitov7f294052013-10-23 16:02:42 -07006641 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC);
Nicolas Dichtela528c212013-09-25 12:02:44 +02006642
Patrick McHardybd380812010-02-26 06:34:53 +00006643 if (changes & IFF_UP) {
6644 if (dev->flags & IFF_UP)
6645 call_netdevice_notifiers(NETDEV_UP, dev);
6646 else
6647 call_netdevice_notifiers(NETDEV_DOWN, dev);
6648 }
6649
6650 if (dev->flags & IFF_UP &&
Jiri Pirkobe9efd32013-05-28 01:30:22 +00006651 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
6652 struct netdev_notifier_change_info change_info;
6653
6654 change_info.flags_changed = changes;
6655 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
6656 &change_info.info);
6657 }
Patrick McHardybd380812010-02-26 06:34:53 +00006658}
6659
6660/**
6661 * dev_change_flags - change device settings
6662 * @dev: device
6663 * @flags: device state flags
6664 *
6665 * Change settings on device based state flags. The flags are
6666 * in the userspace exported format.
6667 */
Eric Dumazetb536db92011-11-30 21:42:26 +00006668int dev_change_flags(struct net_device *dev, unsigned int flags)
Patrick McHardybd380812010-02-26 06:34:53 +00006669{
Eric Dumazetb536db92011-11-30 21:42:26 +00006670 int ret;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006671 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
Patrick McHardybd380812010-02-26 06:34:53 +00006672
6673 ret = __dev_change_flags(dev, flags);
6674 if (ret < 0)
6675 return ret;
6676
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006677 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
Nicolas Dichtela528c212013-09-25 12:02:44 +02006678 __dev_notify_flags(dev, old_flags, changes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006679 return ret;
6680}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006681EXPORT_SYMBOL(dev_change_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006682
Veaceslav Falico2315dc92014-01-10 16:56:25 +01006683static int __dev_set_mtu(struct net_device *dev, int new_mtu)
6684{
6685 const struct net_device_ops *ops = dev->netdev_ops;
6686
6687 if (ops->ndo_change_mtu)
6688 return ops->ndo_change_mtu(dev, new_mtu);
6689
6690 dev->mtu = new_mtu;
6691 return 0;
6692}
6693
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006694/**
6695 * dev_set_mtu - Change maximum transfer unit
6696 * @dev: device
6697 * @new_mtu: new transfer unit
6698 *
6699 * Change the maximum transfer size of the network device.
6700 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07006701int dev_set_mtu(struct net_device *dev, int new_mtu)
6702{
Veaceslav Falico2315dc92014-01-10 16:56:25 +01006703 int err, orig_mtu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006704
6705 if (new_mtu == dev->mtu)
6706 return 0;
6707
Jarod Wilson61e84622016-10-07 22:04:33 -04006708 /* MTU must be positive, and in range */
6709 if (new_mtu < 0 || new_mtu < dev->min_mtu) {
6710 net_err_ratelimited("%s: Invalid MTU %d requested, hw min %d\n",
6711 dev->name, new_mtu, dev->min_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006712 return -EINVAL;
Jarod Wilson61e84622016-10-07 22:04:33 -04006713 }
6714
6715 if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) {
6716 net_err_ratelimited("%s: Invalid MTU %d requested, hw max %d\n",
Jakub Kicinskia0e65de2016-10-17 18:02:22 +01006717 dev->name, new_mtu, dev->max_mtu);
Jarod Wilson61e84622016-10-07 22:04:33 -04006718 return -EINVAL;
6719 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006720
6721 if (!netif_device_present(dev))
6722 return -ENODEV;
6723
Veaceslav Falico1d486bf2014-01-16 00:02:18 +01006724 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
6725 err = notifier_to_errno(err);
6726 if (err)
6727 return err;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006728
Veaceslav Falico2315dc92014-01-10 16:56:25 +01006729 orig_mtu = dev->mtu;
6730 err = __dev_set_mtu(dev, new_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006731
Veaceslav Falico2315dc92014-01-10 16:56:25 +01006732 if (!err) {
6733 err = call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
6734 err = notifier_to_errno(err);
6735 if (err) {
6736 /* setting mtu back and notifying everyone again,
6737 * so that they have a chance to revert changes.
6738 */
6739 __dev_set_mtu(dev, orig_mtu);
6740 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
6741 }
6742 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006743 return err;
6744}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006745EXPORT_SYMBOL(dev_set_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006746
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006747/**
Vlad Dogarucbda10f2011-01-13 23:38:30 +00006748 * dev_set_group - Change group this device belongs to
6749 * @dev: device
6750 * @new_group: group this device should belong to
6751 */
6752void dev_set_group(struct net_device *dev, int new_group)
6753{
6754 dev->group = new_group;
6755}
6756EXPORT_SYMBOL(dev_set_group);
6757
6758/**
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006759 * dev_set_mac_address - Change Media Access Control Address
6760 * @dev: device
6761 * @sa: new address
6762 *
6763 * Change the hardware (MAC) address of the device
6764 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07006765int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
6766{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006767 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006768 int err;
6769
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006770 if (!ops->ndo_set_mac_address)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006771 return -EOPNOTSUPP;
6772 if (sa->sa_family != dev->type)
6773 return -EINVAL;
6774 if (!netif_device_present(dev))
6775 return -ENODEV;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006776 err = ops->ndo_set_mac_address(dev, sa);
Jiri Pirkof6521512013-01-01 03:30:14 +00006777 if (err)
6778 return err;
Jiri Pirkofbdeca22013-01-01 03:30:16 +00006779 dev->addr_assign_type = NET_ADDR_SET;
Jiri Pirkof6521512013-01-01 03:30:14 +00006780 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04006781 add_device_randomness(dev->dev_addr, dev->addr_len);
Jiri Pirkof6521512013-01-01 03:30:14 +00006782 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006783}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006784EXPORT_SYMBOL(dev_set_mac_address);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006785
Jiri Pirko4bf84c32012-12-27 23:49:37 +00006786/**
6787 * dev_change_carrier - Change device carrier
6788 * @dev: device
Randy Dunlap691b3b72013-03-04 12:32:43 +00006789 * @new_carrier: new value
Jiri Pirko4bf84c32012-12-27 23:49:37 +00006790 *
6791 * Change device carrier
6792 */
6793int dev_change_carrier(struct net_device *dev, bool new_carrier)
6794{
6795 const struct net_device_ops *ops = dev->netdev_ops;
6796
6797 if (!ops->ndo_change_carrier)
6798 return -EOPNOTSUPP;
6799 if (!netif_device_present(dev))
6800 return -ENODEV;
6801 return ops->ndo_change_carrier(dev, new_carrier);
6802}
6803EXPORT_SYMBOL(dev_change_carrier);
6804
Linus Torvalds1da177e2005-04-16 15:20:36 -07006805/**
Jiri Pirko66b52b02013-07-29 18:16:49 +02006806 * dev_get_phys_port_id - Get device physical port ID
6807 * @dev: device
6808 * @ppid: port ID
6809 *
6810 * Get device physical port ID
6811 */
6812int dev_get_phys_port_id(struct net_device *dev,
Jiri Pirko02637fc2014-11-28 14:34:16 +01006813 struct netdev_phys_item_id *ppid)
Jiri Pirko66b52b02013-07-29 18:16:49 +02006814{
6815 const struct net_device_ops *ops = dev->netdev_ops;
6816
6817 if (!ops->ndo_get_phys_port_id)
6818 return -EOPNOTSUPP;
6819 return ops->ndo_get_phys_port_id(dev, ppid);
6820}
6821EXPORT_SYMBOL(dev_get_phys_port_id);
6822
6823/**
David Aherndb24a902015-03-17 20:23:15 -06006824 * dev_get_phys_port_name - Get device physical port name
6825 * @dev: device
6826 * @name: port name
Luis de Bethencourted49e652016-03-21 16:31:14 +00006827 * @len: limit of bytes to copy to name
David Aherndb24a902015-03-17 20:23:15 -06006828 *
6829 * Get device physical port name
6830 */
6831int dev_get_phys_port_name(struct net_device *dev,
6832 char *name, size_t len)
6833{
6834 const struct net_device_ops *ops = dev->netdev_ops;
6835
6836 if (!ops->ndo_get_phys_port_name)
6837 return -EOPNOTSUPP;
6838 return ops->ndo_get_phys_port_name(dev, name, len);
6839}
6840EXPORT_SYMBOL(dev_get_phys_port_name);
6841
6842/**
Anuradha Karuppiahd746d702015-07-14 13:43:19 -07006843 * dev_change_proto_down - update protocol port state information
6844 * @dev: device
6845 * @proto_down: new value
6846 *
6847 * This info can be used by switch drivers to set the phys state of the
6848 * port.
6849 */
6850int dev_change_proto_down(struct net_device *dev, bool proto_down)
6851{
6852 const struct net_device_ops *ops = dev->netdev_ops;
6853
6854 if (!ops->ndo_change_proto_down)
6855 return -EOPNOTSUPP;
6856 if (!netif_device_present(dev))
6857 return -ENODEV;
6858 return ops->ndo_change_proto_down(dev, proto_down);
6859}
6860EXPORT_SYMBOL(dev_change_proto_down);
6861
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02006862bool __dev_xdp_attached(struct net_device *dev, xdp_op_t xdp_op)
6863{
6864 struct netdev_xdp xdp;
6865
6866 memset(&xdp, 0, sizeof(xdp));
6867 xdp.command = XDP_QUERY_PROG;
6868
6869 /* Query must always succeed. */
6870 WARN_ON(xdp_op(dev, &xdp) < 0);
6871 return xdp.prog_attached;
6872}
6873
6874static int dev_xdp_install(struct net_device *dev, xdp_op_t xdp_op,
6875 struct netlink_ext_ack *extack,
6876 struct bpf_prog *prog)
6877{
6878 struct netdev_xdp xdp;
6879
6880 memset(&xdp, 0, sizeof(xdp));
6881 xdp.command = XDP_SETUP_PROG;
6882 xdp.extack = extack;
6883 xdp.prog = prog;
6884
6885 return xdp_op(dev, &xdp);
6886}
6887
Anuradha Karuppiahd746d702015-07-14 13:43:19 -07006888/**
Brenden Blancoa7862b42016-07-19 12:16:48 -07006889 * dev_change_xdp_fd - set or clear a bpf program for a device rx path
6890 * @dev: device
Jakub Kicinskib5d60982017-05-01 15:53:43 -07006891 * @extack: netlink extended ack
Brenden Blancoa7862b42016-07-19 12:16:48 -07006892 * @fd: new program fd or negative value to clear
Daniel Borkmann85de8572016-11-28 23:16:54 +01006893 * @flags: xdp-related flags
Brenden Blancoa7862b42016-07-19 12:16:48 -07006894 *
6895 * Set or clear a bpf program for a device
6896 */
Jakub Kicinskiddf9f972017-04-30 21:46:46 -07006897int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
6898 int fd, u32 flags)
Brenden Blancoa7862b42016-07-19 12:16:48 -07006899{
6900 const struct net_device_ops *ops = dev->netdev_ops;
6901 struct bpf_prog *prog = NULL;
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02006902 xdp_op_t xdp_op, xdp_chk;
Brenden Blancoa7862b42016-07-19 12:16:48 -07006903 int err;
6904
Daniel Borkmann85de8572016-11-28 23:16:54 +01006905 ASSERT_RTNL();
6906
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02006907 xdp_op = xdp_chk = ops->ndo_xdp;
Daniel Borkmann0489df92017-05-12 01:04:45 +02006908 if (!xdp_op && (flags & XDP_FLAGS_DRV_MODE))
6909 return -EOPNOTSUPP;
David S. Millerb5cdae32017-04-18 15:36:58 -04006910 if (!xdp_op || (flags & XDP_FLAGS_SKB_MODE))
6911 xdp_op = generic_xdp_install;
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02006912 if (xdp_op == xdp_chk)
6913 xdp_chk = generic_xdp_install;
David S. Millerb5cdae32017-04-18 15:36:58 -04006914
Brenden Blancoa7862b42016-07-19 12:16:48 -07006915 if (fd >= 0) {
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02006916 if (xdp_chk && __dev_xdp_attached(dev, xdp_chk))
6917 return -EEXIST;
6918 if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) &&
6919 __dev_xdp_attached(dev, xdp_op))
6920 return -EBUSY;
Daniel Borkmann85de8572016-11-28 23:16:54 +01006921
Brenden Blancoa7862b42016-07-19 12:16:48 -07006922 prog = bpf_prog_get_type(fd, BPF_PROG_TYPE_XDP);
6923 if (IS_ERR(prog))
6924 return PTR_ERR(prog);
6925 }
6926
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02006927 err = dev_xdp_install(dev, xdp_op, extack, prog);
Brenden Blancoa7862b42016-07-19 12:16:48 -07006928 if (err < 0 && prog)
6929 bpf_prog_put(prog);
6930
6931 return err;
6932}
Brenden Blancoa7862b42016-07-19 12:16:48 -07006933
6934/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07006935 * dev_new_index - allocate an ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07006936 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07006937 *
6938 * Returns a suitable unique value for a new device interface
6939 * number. The caller must hold the rtnl semaphore or the
6940 * dev_base_lock to be sure it remains unique.
6941 */
Eric W. Biederman881d9662007-09-17 11:56:21 -07006942static int dev_new_index(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006943{
Pavel Emelyanovaa79e662012-08-08 21:53:19 +00006944 int ifindex = net->ifindex;
tchardingf4563a72017-02-09 17:56:07 +11006945
Linus Torvalds1da177e2005-04-16 15:20:36 -07006946 for (;;) {
6947 if (++ifindex <= 0)
6948 ifindex = 1;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006949 if (!__dev_get_by_index(net, ifindex))
Pavel Emelyanovaa79e662012-08-08 21:53:19 +00006950 return net->ifindex = ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006951 }
6952}
6953
Linus Torvalds1da177e2005-04-16 15:20:36 -07006954/* Delayed registration/unregisteration */
Denis Cheng3b5b34f2007-12-07 00:49:17 -08006955static LIST_HEAD(net_todo_list);
Cong Wang200b9162014-05-12 15:11:20 -07006956DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006957
Stephen Hemminger6f05f622007-03-08 20:46:03 -08006958static void net_set_todo(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006959{
Linus Torvalds1da177e2005-04-16 15:20:36 -07006960 list_add_tail(&dev->todo_list, &net_todo_list);
Eric W. Biederman50624c92013-09-23 21:19:49 -07006961 dev_net(dev)->dev_unreg_count++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006962}
6963
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006964static void rollback_registered_many(struct list_head *head)
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006965{
Krishna Kumare93737b2009-12-08 22:26:02 +00006966 struct net_device *dev, *tmp;
Eric W. Biederman5cde2822013-10-05 19:26:05 -07006967 LIST_HEAD(close_head);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006968
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006969 BUG_ON(dev_boot_phase);
6970 ASSERT_RTNL();
6971
Krishna Kumare93737b2009-12-08 22:26:02 +00006972 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006973 /* Some devices call without registering
Krishna Kumare93737b2009-12-08 22:26:02 +00006974 * for initialization unwind. Remove those
6975 * devices and proceed with the remaining.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006976 */
6977 if (dev->reg_state == NETREG_UNINITIALIZED) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006978 pr_debug("unregister_netdevice: device %s/%p never was registered\n",
6979 dev->name, dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006980
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006981 WARN_ON(1);
Krishna Kumare93737b2009-12-08 22:26:02 +00006982 list_del(&dev->unreg_list);
6983 continue;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006984 }
Eric Dumazet449f4542011-05-19 12:24:16 +00006985 dev->dismantle = true;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006986 BUG_ON(dev->reg_state != NETREG_REGISTERED);
Octavian Purdila44345722010-12-13 12:44:07 +00006987 }
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006988
Octavian Purdila44345722010-12-13 12:44:07 +00006989 /* If device is running, close it first. */
Eric W. Biederman5cde2822013-10-05 19:26:05 -07006990 list_for_each_entry(dev, head, unreg_list)
6991 list_add_tail(&dev->close_list, &close_head);
David S. Miller99c4a262015-03-18 22:52:33 -04006992 dev_close_many(&close_head, true);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006993
Octavian Purdila44345722010-12-13 12:44:07 +00006994 list_for_each_entry(dev, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006995 /* And unlink it from device chain. */
6996 unlist_netdevice(dev);
6997
6998 dev->reg_state = NETREG_UNREGISTERING;
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006999 }
Eric Dumazet41852492016-08-26 12:50:39 -07007000 flush_all_backlogs();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07007001
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007002 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07007003
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007004 list_for_each_entry(dev, head, unreg_list) {
Mahesh Bandewar395eea62014-12-03 13:46:24 -08007005 struct sk_buff *skb = NULL;
7006
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007007 /* Shutdown queueing discipline. */
7008 dev_shutdown(dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07007009
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07007010
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007011 /* Notify protocols, that we are about to destroy
tchardingeb13da12017-02-09 17:56:06 +11007012 * this device. They should clean all the things.
7013 */
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007014 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
7015
Mahesh Bandewar395eea62014-12-03 13:46:24 -08007016 if (!dev->rtnl_link_ops ||
7017 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
7018 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U,
7019 GFP_KERNEL);
7020
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007021 /*
7022 * Flush the unicast and multicast chains
7023 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00007024 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00007025 dev_mc_flush(dev);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007026
7027 if (dev->netdev_ops->ndo_uninit)
7028 dev->netdev_ops->ndo_uninit(dev);
7029
Mahesh Bandewar395eea62014-12-03 13:46:24 -08007030 if (skb)
7031 rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
Roopa Prabhu56bfa7e2014-05-01 11:40:30 -07007032
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007033 /* Notifier chain MUST detach us all upper devices. */
7034 WARN_ON(netdev_has_any_upper_dev(dev));
David Ahern0f524a82016-10-17 19:15:52 -07007035 WARN_ON(netdev_has_any_lower_dev(dev));
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007036
7037 /* Remove entries from kobject tree */
7038 netdev_unregister_kobject(dev);
Alexander Duyck024e9672013-01-10 08:57:46 +00007039#ifdef CONFIG_XPS
7040 /* Remove XPS queueing entries */
7041 netif_reset_xps_queues_gt(dev, 0);
7042#endif
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007043 }
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07007044
Eric W. Biederman850a5452011-10-13 22:25:23 +00007045 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07007046
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00007047 list_for_each_entry(dev, head, unreg_list)
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007048 dev_put(dev);
7049}
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07007050
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007051static void rollback_registered(struct net_device *dev)
7052{
7053 LIST_HEAD(single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07007054
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007055 list_add(&dev->unreg_list, &single);
7056 rollback_registered_many(&single);
Eric Dumazetceaaec92011-02-17 22:59:19 +00007057 list_del(&single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07007058}
7059
Jarod Wilsonfd867d52015-11-02 21:55:59 -05007060static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
7061 struct net_device *upper, netdev_features_t features)
7062{
7063 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
7064 netdev_features_t feature;
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05007065 int feature_bit;
Jarod Wilsonfd867d52015-11-02 21:55:59 -05007066
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05007067 for_each_netdev_feature(&upper_disables, feature_bit) {
7068 feature = __NETIF_F_BIT(feature_bit);
Jarod Wilsonfd867d52015-11-02 21:55:59 -05007069 if (!(upper->wanted_features & feature)
7070 && (features & feature)) {
7071 netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
7072 &feature, upper->name);
7073 features &= ~feature;
7074 }
7075 }
7076
7077 return features;
7078}
7079
7080static void netdev_sync_lower_features(struct net_device *upper,
7081 struct net_device *lower, netdev_features_t features)
7082{
7083 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
7084 netdev_features_t feature;
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05007085 int feature_bit;
Jarod Wilsonfd867d52015-11-02 21:55:59 -05007086
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05007087 for_each_netdev_feature(&upper_disables, feature_bit) {
7088 feature = __NETIF_F_BIT(feature_bit);
Jarod Wilsonfd867d52015-11-02 21:55:59 -05007089 if (!(features & feature) && (lower->features & feature)) {
7090 netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
7091 &feature, lower->name);
7092 lower->wanted_features &= ~feature;
7093 netdev_update_features(lower);
7094
7095 if (unlikely(lower->features & feature))
7096 netdev_WARN(upper, "failed to disable %pNF on %s!\n",
7097 &feature, lower->name);
7098 }
7099 }
7100}
7101
Michał Mirosławc8f44af2011-11-15 15:29:55 +00007102static netdev_features_t netdev_fix_features(struct net_device *dev,
7103 netdev_features_t features)
Herbert Xub63365a2008-10-23 01:11:29 -07007104{
Michał Mirosław57422dc2011-01-22 12:14:12 +00007105 /* Fix illegal checksum combinations */
7106 if ((features & NETIF_F_HW_CSUM) &&
7107 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04007108 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
Michał Mirosław57422dc2011-01-22 12:14:12 +00007109 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
7110 }
7111
Herbert Xub63365a2008-10-23 01:11:29 -07007112 /* TSO requires that SG is present as well. */
Ben Hutchingsea2d3682011-04-12 14:38:37 +00007113 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04007114 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
Ben Hutchingsea2d3682011-04-12 14:38:37 +00007115 features &= ~NETIF_F_ALL_TSO;
Herbert Xub63365a2008-10-23 01:11:29 -07007116 }
7117
Pravin B Shelarec5f0612013-03-07 09:28:01 +00007118 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
7119 !(features & NETIF_F_IP_CSUM)) {
7120 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
7121 features &= ~NETIF_F_TSO;
7122 features &= ~NETIF_F_TSO_ECN;
7123 }
7124
7125 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
7126 !(features & NETIF_F_IPV6_CSUM)) {
7127 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
7128 features &= ~NETIF_F_TSO6;
7129 }
7130
Alexander Duyckb1dc4972016-05-02 09:38:24 -07007131 /* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */
7132 if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO))
7133 features &= ~NETIF_F_TSO_MANGLEID;
7134
Ben Hutchings31d8b9e2011-04-12 14:47:15 +00007135 /* TSO ECN requires that TSO is present as well. */
7136 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
7137 features &= ~NETIF_F_TSO_ECN;
7138
Michał Mirosław212b5732011-02-15 16:59:16 +00007139 /* Software GSO depends on SG. */
7140 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04007141 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
Michał Mirosław212b5732011-02-15 16:59:16 +00007142 features &= ~NETIF_F_GSO;
7143 }
7144
Michał Mirosławacd11302011-01-24 15:45:15 -08007145 /* UFO needs SG and checksumming */
Herbert Xub63365a2008-10-23 01:11:29 -07007146 if (features & NETIF_F_UFO) {
Michał Mirosław79032642010-11-30 06:38:00 +00007147 /* maybe split UFO into V4 and V6? */
Tom Herbertc8cd0982015-12-14 11:19:44 -08007148 if (!(features & NETIF_F_HW_CSUM) &&
7149 ((features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) !=
7150 (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04007151 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08007152 "Dropping NETIF_F_UFO since no checksum offload features.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07007153 features &= ~NETIF_F_UFO;
7154 }
7155
7156 if (!(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04007157 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08007158 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07007159 features &= ~NETIF_F_UFO;
7160 }
7161 }
7162
Alexander Duyck802ab552016-04-10 21:45:03 -04007163 /* GSO partial features require GSO partial be set */
7164 if ((features & dev->gso_partial_features) &&
7165 !(features & NETIF_F_GSO_PARTIAL)) {
7166 netdev_dbg(dev,
7167 "Dropping partially supported GSO features since no GSO partial.\n");
7168 features &= ~dev->gso_partial_features;
7169 }
7170
Herbert Xub63365a2008-10-23 01:11:29 -07007171 return features;
7172}
Herbert Xub63365a2008-10-23 01:11:29 -07007173
Michał Mirosław6cb6a272011-04-02 22:48:47 -07007174int __netdev_update_features(struct net_device *dev)
Michał Mirosław5455c692011-02-15 16:59:17 +00007175{
Jarod Wilsonfd867d52015-11-02 21:55:59 -05007176 struct net_device *upper, *lower;
Michał Mirosławc8f44af2011-11-15 15:29:55 +00007177 netdev_features_t features;
Jarod Wilsonfd867d52015-11-02 21:55:59 -05007178 struct list_head *iter;
Jarod Wilsone7868a82015-11-03 23:09:32 -05007179 int err = -1;
Michał Mirosław5455c692011-02-15 16:59:17 +00007180
Michał Mirosław87267482011-04-12 09:56:38 +00007181 ASSERT_RTNL();
7182
Michał Mirosław5455c692011-02-15 16:59:17 +00007183 features = netdev_get_wanted_features(dev);
7184
7185 if (dev->netdev_ops->ndo_fix_features)
7186 features = dev->netdev_ops->ndo_fix_features(dev, features);
7187
7188 /* driver might be less strict about feature dependencies */
7189 features = netdev_fix_features(dev, features);
7190
Jarod Wilsonfd867d52015-11-02 21:55:59 -05007191 /* some features can't be enabled if they're off an an upper device */
7192 netdev_for_each_upper_dev_rcu(dev, upper, iter)
7193 features = netdev_sync_upper_features(dev, upper, features);
7194
Michał Mirosław5455c692011-02-15 16:59:17 +00007195 if (dev->features == features)
Jarod Wilsone7868a82015-11-03 23:09:32 -05007196 goto sync_lower;
Michał Mirosław5455c692011-02-15 16:59:17 +00007197
Michał Mirosławc8f44af2011-11-15 15:29:55 +00007198 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
7199 &dev->features, &features);
Michał Mirosław5455c692011-02-15 16:59:17 +00007200
7201 if (dev->netdev_ops->ndo_set_features)
7202 err = dev->netdev_ops->ndo_set_features(dev, features);
Nikolay Aleksandrov5f8dc332015-11-13 14:54:01 +01007203 else
7204 err = 0;
Michał Mirosław5455c692011-02-15 16:59:17 +00007205
Michał Mirosław6cb6a272011-04-02 22:48:47 -07007206 if (unlikely(err < 0)) {
Michał Mirosław5455c692011-02-15 16:59:17 +00007207 netdev_err(dev,
Michał Mirosławc8f44af2011-11-15 15:29:55 +00007208 "set_features() failed (%d); wanted %pNF, left %pNF\n",
7209 err, &features, &dev->features);
Nikolay Aleksandrov17b85d22015-11-17 15:49:06 +01007210 /* return non-0 since some features might have changed and
7211 * it's better to fire a spurious notification than miss it
7212 */
7213 return -1;
Michał Mirosław6cb6a272011-04-02 22:48:47 -07007214 }
7215
Jarod Wilsone7868a82015-11-03 23:09:32 -05007216sync_lower:
Jarod Wilsonfd867d52015-11-02 21:55:59 -05007217 /* some features must be disabled on lower devices when disabled
7218 * on an upper device (think: bonding master or bridge)
7219 */
7220 netdev_for_each_lower_dev(dev, lower, iter)
7221 netdev_sync_lower_features(dev, lower, features);
7222
Michał Mirosław6cb6a272011-04-02 22:48:47 -07007223 if (!err)
7224 dev->features = features;
7225
Jarod Wilsone7868a82015-11-03 23:09:32 -05007226 return err < 0 ? 0 : 1;
Michał Mirosław6cb6a272011-04-02 22:48:47 -07007227}
7228
Michał Mirosławafe12cc2011-05-07 03:22:17 +00007229/**
7230 * netdev_update_features - recalculate device features
7231 * @dev: the device to check
7232 *
7233 * Recalculate dev->features set and send notifications if it
7234 * has changed. Should be called after driver or hardware dependent
7235 * conditions might have changed that influence the features.
7236 */
Michał Mirosław6cb6a272011-04-02 22:48:47 -07007237void netdev_update_features(struct net_device *dev)
7238{
7239 if (__netdev_update_features(dev))
7240 netdev_features_change(dev);
Michał Mirosław5455c692011-02-15 16:59:17 +00007241}
7242EXPORT_SYMBOL(netdev_update_features);
7243
Linus Torvalds1da177e2005-04-16 15:20:36 -07007244/**
Michał Mirosławafe12cc2011-05-07 03:22:17 +00007245 * netdev_change_features - recalculate device features
7246 * @dev: the device to check
7247 *
7248 * Recalculate dev->features set and send notifications even
7249 * if they have not changed. Should be called instead of
7250 * netdev_update_features() if also dev->vlan_features might
7251 * have changed to allow the changes to be propagated to stacked
7252 * VLAN devices.
7253 */
7254void netdev_change_features(struct net_device *dev)
7255{
7256 __netdev_update_features(dev);
7257 netdev_features_change(dev);
7258}
7259EXPORT_SYMBOL(netdev_change_features);
7260
7261/**
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08007262 * netif_stacked_transfer_operstate - transfer operstate
7263 * @rootdev: the root or lower level device to transfer state from
7264 * @dev: the device to transfer operstate to
7265 *
7266 * Transfer operational state from root to device. This is normally
7267 * called when a stacking relationship exists between the root
7268 * device and the device(a leaf device).
7269 */
7270void netif_stacked_transfer_operstate(const struct net_device *rootdev,
7271 struct net_device *dev)
7272{
7273 if (rootdev->operstate == IF_OPER_DORMANT)
7274 netif_dormant_on(dev);
7275 else
7276 netif_dormant_off(dev);
7277
Zhang Shengju0575c862017-04-26 17:49:38 +08007278 if (netif_carrier_ok(rootdev))
7279 netif_carrier_on(dev);
7280 else
7281 netif_carrier_off(dev);
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08007282}
7283EXPORT_SYMBOL(netif_stacked_transfer_operstate);
7284
Michael Daltona953be52014-01-16 22:23:28 -08007285#ifdef CONFIG_SYSFS
Eric Dumazet1b4bf462010-09-23 17:26:35 +00007286static int netif_alloc_rx_queues(struct net_device *dev)
7287{
Eric Dumazet1b4bf462010-09-23 17:26:35 +00007288 unsigned int i, count = dev->num_rx_queues;
Tom Herbertbd25fa72010-10-18 18:00:16 +00007289 struct netdev_rx_queue *rx;
Pankaj Gupta10595902015-01-12 11:41:28 +05307290 size_t sz = count * sizeof(*rx);
Eric Dumazet1b4bf462010-09-23 17:26:35 +00007291
Tom Herbertbd25fa72010-10-18 18:00:16 +00007292 BUG_ON(count < 1);
Eric Dumazet1b4bf462010-09-23 17:26:35 +00007293
Michal Hockoda6bc572017-05-08 15:57:31 -07007294 rx = kvzalloc(sz, GFP_KERNEL | __GFP_REPEAT);
7295 if (!rx)
7296 return -ENOMEM;
7297
Tom Herbertbd25fa72010-10-18 18:00:16 +00007298 dev->_rx = rx;
7299
Tom Herbertbd25fa72010-10-18 18:00:16 +00007300 for (i = 0; i < count; i++)
Tom Herbertfe822242010-11-09 10:47:38 +00007301 rx[i].dev = dev;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00007302 return 0;
7303}
Tom Herbertbf264142010-11-26 08:36:09 +00007304#endif
Eric Dumazet1b4bf462010-09-23 17:26:35 +00007305
Changli Gaoaa942102010-12-04 02:31:41 +00007306static void netdev_init_one_queue(struct net_device *dev,
7307 struct netdev_queue *queue, void *_unused)
7308{
7309 /* Initialize queue lock */
7310 spin_lock_init(&queue->_xmit_lock);
7311 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
7312 queue->xmit_lock_owner = -1;
Changli Gaob236da62010-12-14 03:09:15 +00007313 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
Changli Gaoaa942102010-12-04 02:31:41 +00007314 queue->dev = dev;
Tom Herbert114cf582011-11-28 16:33:09 +00007315#ifdef CONFIG_BQL
7316 dql_init(&queue->dql, HZ);
7317#endif
Changli Gaoaa942102010-12-04 02:31:41 +00007318}
7319
Eric Dumazet60877a32013-06-20 01:15:51 -07007320static void netif_free_tx_queues(struct net_device *dev)
7321{
WANG Cong4cb28972014-06-02 15:55:22 -07007322 kvfree(dev->_tx);
Eric Dumazet60877a32013-06-20 01:15:51 -07007323}
7324
Tom Herberte6484932010-10-18 18:04:39 +00007325static int netif_alloc_netdev_queues(struct net_device *dev)
7326{
7327 unsigned int count = dev->num_tx_queues;
7328 struct netdev_queue *tx;
Eric Dumazet60877a32013-06-20 01:15:51 -07007329 size_t sz = count * sizeof(*tx);
Tom Herberte6484932010-10-18 18:04:39 +00007330
Eric Dumazetd3397272015-07-06 17:13:26 +02007331 if (count < 1 || count > 0xffff)
7332 return -EINVAL;
Tom Herberte6484932010-10-18 18:04:39 +00007333
Michal Hockoda6bc572017-05-08 15:57:31 -07007334 tx = kvzalloc(sz, GFP_KERNEL | __GFP_REPEAT);
7335 if (!tx)
7336 return -ENOMEM;
7337
Tom Herberte6484932010-10-18 18:04:39 +00007338 dev->_tx = tx;
Tom Herbert1d24eb42010-11-21 13:17:27 +00007339
Tom Herberte6484932010-10-18 18:04:39 +00007340 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
7341 spin_lock_init(&dev->tx_global_lock);
Changli Gaoaa942102010-12-04 02:31:41 +00007342
7343 return 0;
Tom Herberte6484932010-10-18 18:04:39 +00007344}
7345
Denys Vlasenkoa2029242015-05-11 21:17:53 +02007346void netif_tx_stop_all_queues(struct net_device *dev)
7347{
7348 unsigned int i;
7349
7350 for (i = 0; i < dev->num_tx_queues; i++) {
7351 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
tchardingf4563a72017-02-09 17:56:07 +11007352
Denys Vlasenkoa2029242015-05-11 21:17:53 +02007353 netif_tx_stop_queue(txq);
7354 }
7355}
7356EXPORT_SYMBOL(netif_tx_stop_all_queues);
7357
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08007358/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07007359 * register_netdevice - register a network device
7360 * @dev: device to register
7361 *
7362 * Take a completed network device structure and add it to the kernel
7363 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
7364 * chain. 0 is returned on success. A negative errno code is returned
7365 * on a failure to set up the device, or if the name is a duplicate.
7366 *
7367 * Callers must hold the rtnl semaphore. You may want
7368 * register_netdev() instead of this.
7369 *
7370 * BUGS:
7371 * The locking appears insufficient to guarantee two parallel registers
7372 * will not get the same name.
7373 */
7374
7375int register_netdevice(struct net_device *dev)
7376{
Linus Torvalds1da177e2005-04-16 15:20:36 -07007377 int ret;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08007378 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007379
7380 BUG_ON(dev_boot_phase);
7381 ASSERT_RTNL();
7382
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007383 might_sleep();
7384
Linus Torvalds1da177e2005-04-16 15:20:36 -07007385 /* When net_device's are persistent, this will be fatal. */
7386 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08007387 BUG_ON(!net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007388
David S. Millerf1f28aa2008-07-15 00:08:33 -07007389 spin_lock_init(&dev->addr_list_lock);
David S. Millercf508b12008-07-22 14:16:42 -07007390 netdev_set_addr_lockdep_class(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007391
Gao feng828de4f2012-09-13 20:58:27 +00007392 ret = dev_get_valid_name(net, dev, dev->name);
Peter Pan(潘卫平)0696c3a2011-05-12 15:46:56 +00007393 if (ret < 0)
7394 goto out;
7395
Linus Torvalds1da177e2005-04-16 15:20:36 -07007396 /* Init, if this function is available */
Stephen Hemmingerd3147742008-11-19 21:32:24 -08007397 if (dev->netdev_ops->ndo_init) {
7398 ret = dev->netdev_ops->ndo_init(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007399 if (ret) {
7400 if (ret > 0)
7401 ret = -EIO;
Adrian Bunk90833aa2006-11-13 16:02:22 -08007402 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007403 }
7404 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09007405
Patrick McHardyf6469682013-04-19 02:04:27 +00007406 if (((dev->hw_features | dev->features) &
7407 NETIF_F_HW_VLAN_CTAG_FILTER) &&
Michał Mirosławd2ed2732013-01-29 15:14:16 +00007408 (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
7409 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
7410 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
7411 ret = -EINVAL;
7412 goto err_uninit;
7413 }
7414
Pavel Emelyanov9c7dafb2012-08-08 21:52:46 +00007415 ret = -EBUSY;
7416 if (!dev->ifindex)
7417 dev->ifindex = dev_new_index(net);
7418 else if (__dev_get_by_index(net, dev->ifindex))
7419 goto err_uninit;
7420
Michał Mirosław5455c692011-02-15 16:59:17 +00007421 /* Transfer changeable features to wanted_features and enable
7422 * software offloads (GSO and GRO).
7423 */
7424 dev->hw_features |= NETIF_F_SOFT_FEATURES;
Michał Mirosław14d12322011-02-22 16:52:28 +00007425 dev->features |= NETIF_F_SOFT_FEATURES;
7426 dev->wanted_features = dev->features & dev->hw_features;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007427
Alexander Duyckcbc53e02016-04-10 21:44:51 -04007428 if (!(dev->flags & IFF_LOOPBACK))
Michał Mirosław34324dc2011-11-15 15:29:55 +00007429 dev->hw_features |= NETIF_F_NOCACHE_COPY;
Alexander Duyckcbc53e02016-04-10 21:44:51 -04007430
Alexander Duyck7f348a62016-04-20 16:51:00 -04007431 /* If IPv4 TCP segmentation offload is supported we should also
7432 * allow the device to enable segmenting the frame with the option
7433 * of ignoring a static IP ID value. This doesn't enable the
7434 * feature itself but allows the user to enable it later.
7435 */
Alexander Duyckcbc53e02016-04-10 21:44:51 -04007436 if (dev->hw_features & NETIF_F_TSO)
7437 dev->hw_features |= NETIF_F_TSO_MANGLEID;
Alexander Duyck7f348a62016-04-20 16:51:00 -04007438 if (dev->vlan_features & NETIF_F_TSO)
7439 dev->vlan_features |= NETIF_F_TSO_MANGLEID;
7440 if (dev->mpls_features & NETIF_F_TSO)
7441 dev->mpls_features |= NETIF_F_TSO_MANGLEID;
7442 if (dev->hw_enc_features & NETIF_F_TSO)
7443 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
Tom Herbertc6e1a0d2011-04-04 22:30:30 -07007444
Michał Mirosław1180e7d2011-07-14 14:41:11 -07007445 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
Brandon Philips16c3ea72010-09-15 09:24:24 +00007446 */
Michał Mirosław1180e7d2011-07-14 14:41:11 -07007447 dev->vlan_features |= NETIF_F_HIGHDMA;
Brandon Philips16c3ea72010-09-15 09:24:24 +00007448
Pravin B Shelaree579672013-03-07 09:28:08 +00007449 /* Make NETIF_F_SG inheritable to tunnel devices.
7450 */
Alexander Duyck802ab552016-04-10 21:45:03 -04007451 dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL;
Pravin B Shelaree579672013-03-07 09:28:08 +00007452
Simon Horman0d89d202013-05-23 21:02:52 +00007453 /* Make NETIF_F_SG inheritable to MPLS.
7454 */
7455 dev->mpls_features |= NETIF_F_SG;
7456
Johannes Berg7ffbe3f2009-10-02 05:15:27 +00007457 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
7458 ret = notifier_to_errno(ret);
7459 if (ret)
7460 goto err_uninit;
7461
Eric W. Biederman8b41d182007-09-26 22:02:53 -07007462 ret = netdev_register_kobject(dev);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007463 if (ret)
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07007464 goto err_uninit;
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007465 dev->reg_state = NETREG_REGISTERED;
7466
Michał Mirosław6cb6a272011-04-02 22:48:47 -07007467 __netdev_update_features(dev);
Michał Mirosław8e9b59b2011-02-22 16:52:28 +00007468
Linus Torvalds1da177e2005-04-16 15:20:36 -07007469 /*
7470 * Default initial state at registry is that the
7471 * device is present.
7472 */
7473
7474 set_bit(__LINK_STATE_PRESENT, &dev->state);
7475
Ben Hutchings8f4cccb2012-08-20 22:16:51 +01007476 linkwatch_init_dev(dev);
7477
Linus Torvalds1da177e2005-04-16 15:20:36 -07007478 dev_init_scheduler(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007479 dev_hold(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007480 list_netdevice(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04007481 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007482
Jiri Pirko948b3372013-01-08 01:38:25 +00007483 /* If the device has permanent device address, driver should
7484 * set dev_addr and also addr_assign_type should be set to
7485 * NET_ADDR_PERM (default value).
7486 */
7487 if (dev->addr_assign_type == NET_ADDR_PERM)
7488 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
7489
Linus Torvalds1da177e2005-04-16 15:20:36 -07007490 /* Notify protocols, that a new device appeared. */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07007491 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07007492 ret = notifier_to_errno(ret);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07007493 if (ret) {
7494 rollback_registered(dev);
7495 dev->reg_state = NETREG_UNREGISTERED;
7496 }
Eric W. Biedermand90a9092009-12-12 22:11:15 +00007497 /*
7498 * Prevent userspace races by waiting until the network
7499 * device is fully setup before sending notifications.
7500 */
Patrick McHardya2835762010-02-26 06:34:51 +00007501 if (!dev->rtnl_link_ops ||
7502 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
Alexei Starovoitov7f294052013-10-23 16:02:42 -07007503 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007504
7505out:
7506 return ret;
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07007507
7508err_uninit:
Stephen Hemmingerd3147742008-11-19 21:32:24 -08007509 if (dev->netdev_ops->ndo_uninit)
7510 dev->netdev_ops->ndo_uninit(dev);
David S. Millercf124db2017-05-08 12:52:56 -04007511 if (dev->priv_destructor)
7512 dev->priv_destructor(dev);
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07007513 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007514}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07007515EXPORT_SYMBOL(register_netdevice);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007516
7517/**
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08007518 * init_dummy_netdev - init a dummy network device for NAPI
7519 * @dev: device to init
7520 *
7521 * This takes a network device structure and initialize the minimum
7522 * amount of fields so it can be used to schedule NAPI polls without
7523 * registering a full blown interface. This is to be used by drivers
7524 * that need to tie several hardware interfaces to a single NAPI
7525 * poll scheduler due to HW limitations.
7526 */
7527int init_dummy_netdev(struct net_device *dev)
7528{
7529 /* Clear everything. Note we don't initialize spinlocks
7530 * are they aren't supposed to be taken by any of the
7531 * NAPI code and this dummy netdev is supposed to be
7532 * only ever used for NAPI polls
7533 */
7534 memset(dev, 0, sizeof(struct net_device));
7535
7536 /* make sure we BUG if trying to hit standard
7537 * register/unregister code path
7538 */
7539 dev->reg_state = NETREG_DUMMY;
7540
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08007541 /* NAPI wants this */
7542 INIT_LIST_HEAD(&dev->napi_list);
7543
7544 /* a dummy interface is started by default */
7545 set_bit(__LINK_STATE_PRESENT, &dev->state);
7546 set_bit(__LINK_STATE_START, &dev->state);
7547
Eric Dumazet29b44332010-10-11 10:22:12 +00007548 /* Note : We dont allocate pcpu_refcnt for dummy devices,
7549 * because users of this 'device' dont need to change
7550 * its refcount.
7551 */
7552
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08007553 return 0;
7554}
7555EXPORT_SYMBOL_GPL(init_dummy_netdev);
7556
7557
7558/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07007559 * register_netdev - register a network device
7560 * @dev: device to register
7561 *
7562 * Take a completed network device structure and add it to the kernel
7563 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
7564 * chain. 0 is returned on success. A negative errno code is returned
7565 * on a failure to set up the device, or if the name is a duplicate.
7566 *
Borislav Petkov38b4da32007-04-20 22:14:10 -07007567 * This is a wrapper around register_netdevice that takes the rtnl semaphore
Linus Torvalds1da177e2005-04-16 15:20:36 -07007568 * and expands the device name if you passed a format string to
7569 * alloc_netdev.
7570 */
7571int register_netdev(struct net_device *dev)
7572{
7573 int err;
7574
7575 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007576 err = register_netdevice(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007577 rtnl_unlock();
7578 return err;
7579}
7580EXPORT_SYMBOL(register_netdev);
7581
Eric Dumazet29b44332010-10-11 10:22:12 +00007582int netdev_refcnt_read(const struct net_device *dev)
7583{
7584 int i, refcnt = 0;
7585
7586 for_each_possible_cpu(i)
7587 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
7588 return refcnt;
7589}
7590EXPORT_SYMBOL(netdev_refcnt_read);
7591
Ben Hutchings2c530402012-07-10 10:55:09 +00007592/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07007593 * netdev_wait_allrefs - wait until all references are gone.
Randy Dunlap3de7a372012-08-18 14:36:44 +00007594 * @dev: target net_device
Linus Torvalds1da177e2005-04-16 15:20:36 -07007595 *
7596 * This is called when unregistering network devices.
7597 *
7598 * Any protocol or device that holds a reference should register
7599 * for netdevice notification, and cleanup and put back the
7600 * reference if they receive an UNREGISTER event.
7601 * We can get stuck here if buggy protocols don't correctly
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09007602 * call dev_put.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007603 */
7604static void netdev_wait_allrefs(struct net_device *dev)
7605{
7606 unsigned long rebroadcast_time, warning_time;
Eric Dumazet29b44332010-10-11 10:22:12 +00007607 int refcnt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007608
Eric Dumazete014deb2009-11-17 05:59:21 +00007609 linkwatch_forget_dev(dev);
7610
Linus Torvalds1da177e2005-04-16 15:20:36 -07007611 rebroadcast_time = warning_time = jiffies;
Eric Dumazet29b44332010-10-11 10:22:12 +00007612 refcnt = netdev_refcnt_read(dev);
7613
7614 while (refcnt != 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07007615 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08007616 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007617
7618 /* Rebroadcast unregister notification */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07007619 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007620
Eric Dumazet748e2d92012-08-22 21:50:59 +00007621 __rtnl_unlock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00007622 rcu_barrier();
Eric Dumazet748e2d92012-08-22 21:50:59 +00007623 rtnl_lock();
7624
Eric Dumazet0115e8e2012-08-22 17:19:46 +00007625 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007626 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
7627 &dev->state)) {
7628 /* We must not have linkwatch events
7629 * pending on unregister. If this
7630 * happens, we simply run the queue
7631 * unscheduled, resulting in a noop
7632 * for this device.
7633 */
7634 linkwatch_run_queue();
7635 }
7636
Stephen Hemminger6756ae42006-03-20 22:23:58 -08007637 __rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007638
7639 rebroadcast_time = jiffies;
7640 }
7641
7642 msleep(250);
7643
Eric Dumazet29b44332010-10-11 10:22:12 +00007644 refcnt = netdev_refcnt_read(dev);
7645
Linus Torvalds1da177e2005-04-16 15:20:36 -07007646 if (time_after(jiffies, warning_time + 10 * HZ)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00007647 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
7648 dev->name, refcnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007649 warning_time = jiffies;
7650 }
7651 }
7652}
7653
7654/* The sequence is:
7655 *
7656 * rtnl_lock();
7657 * ...
7658 * register_netdevice(x1);
7659 * register_netdevice(x2);
7660 * ...
7661 * unregister_netdevice(y1);
7662 * unregister_netdevice(y2);
7663 * ...
7664 * rtnl_unlock();
7665 * free_netdev(y1);
7666 * free_netdev(y2);
7667 *
Herbert Xu58ec3b42008-10-07 15:50:03 -07007668 * We are invoked by rtnl_unlock().
Linus Torvalds1da177e2005-04-16 15:20:36 -07007669 * This allows us to deal with problems:
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007670 * 1) We can delete sysfs objects which invoke hotplug
Linus Torvalds1da177e2005-04-16 15:20:36 -07007671 * without deadlocking with linkwatch via keventd.
7672 * 2) Since we run with the RTNL semaphore not held, we can sleep
7673 * safely in order to wait for the netdev refcnt to drop to zero.
Herbert Xu58ec3b42008-10-07 15:50:03 -07007674 *
7675 * We must not return until all unregister events added during
7676 * the interval the lock was held have been completed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007677 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07007678void netdev_run_todo(void)
7679{
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07007680 struct list_head list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007681
Linus Torvalds1da177e2005-04-16 15:20:36 -07007682 /* Snapshot list, allow later requests */
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07007683 list_replace_init(&net_todo_list, &list);
Herbert Xu58ec3b42008-10-07 15:50:03 -07007684
7685 __rtnl_unlock();
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07007686
Eric Dumazet0115e8e2012-08-22 17:19:46 +00007687
7688 /* Wait for rcu callbacks to finish before next phase */
Eric W. Biederman850a5452011-10-13 22:25:23 +00007689 if (!list_empty(&list))
7690 rcu_barrier();
7691
Linus Torvalds1da177e2005-04-16 15:20:36 -07007692 while (!list_empty(&list)) {
7693 struct net_device *dev
stephen hemmingere5e26d72010-02-24 14:01:38 +00007694 = list_first_entry(&list, struct net_device, todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007695 list_del(&dev->todo_list);
7696
Eric Dumazet748e2d92012-08-22 21:50:59 +00007697 rtnl_lock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00007698 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Eric Dumazet748e2d92012-08-22 21:50:59 +00007699 __rtnl_unlock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00007700
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007701 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00007702 pr_err("network todo '%s' but state %d\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07007703 dev->name, dev->reg_state);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007704 dump_stack();
7705 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007706 }
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007707
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007708 dev->reg_state = NETREG_UNREGISTERED;
7709
7710 netdev_wait_allrefs(dev);
7711
7712 /* paranoia */
Eric Dumazet29b44332010-10-11 10:22:12 +00007713 BUG_ON(netdev_refcnt_read(dev));
Salam Noureddine7866a622015-01-27 11:35:48 -08007714 BUG_ON(!list_empty(&dev->ptype_all));
7715 BUG_ON(!list_empty(&dev->ptype_specific));
Eric Dumazet33d480c2011-08-11 19:30:52 +00007716 WARN_ON(rcu_access_pointer(dev->ip_ptr));
7717 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
Ilpo Järvinen547b7922008-07-25 21:43:18 -07007718 WARN_ON(dev->dn_ptr);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007719
David S. Millercf124db2017-05-08 12:52:56 -04007720 if (dev->priv_destructor)
7721 dev->priv_destructor(dev);
7722 if (dev->needs_free_netdev)
7723 free_netdev(dev);
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07007724
Eric W. Biederman50624c92013-09-23 21:19:49 -07007725 /* Report a network device has been unregistered */
7726 rtnl_lock();
7727 dev_net(dev)->dev_unreg_count--;
7728 __rtnl_unlock();
7729 wake_up(&netdev_unregistering_wq);
7730
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07007731 /* Free network device */
7732 kobject_put(&dev->dev.kobj);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007733 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007734}
7735
Jarod Wilson92566452016-02-01 18:51:04 -05007736/* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has
7737 * all the same fields in the same order as net_device_stats, with only
7738 * the type differing, but rtnl_link_stats64 may have additional fields
7739 * at the end for newer counters.
Ben Hutchings3cfde792010-07-09 09:11:52 +00007740 */
Eric Dumazet77a1abf2012-03-05 04:50:09 +00007741void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
7742 const struct net_device_stats *netdev_stats)
Ben Hutchings3cfde792010-07-09 09:11:52 +00007743{
7744#if BITS_PER_LONG == 64
Jarod Wilson92566452016-02-01 18:51:04 -05007745 BUILD_BUG_ON(sizeof(*stats64) < sizeof(*netdev_stats));
Eric Dumazet77a1abf2012-03-05 04:50:09 +00007746 memcpy(stats64, netdev_stats, sizeof(*stats64));
Jarod Wilson92566452016-02-01 18:51:04 -05007747 /* zero out counters that only exist in rtnl_link_stats64 */
7748 memset((char *)stats64 + sizeof(*netdev_stats), 0,
7749 sizeof(*stats64) - sizeof(*netdev_stats));
Ben Hutchings3cfde792010-07-09 09:11:52 +00007750#else
Jarod Wilson92566452016-02-01 18:51:04 -05007751 size_t i, n = sizeof(*netdev_stats) / sizeof(unsigned long);
Ben Hutchings3cfde792010-07-09 09:11:52 +00007752 const unsigned long *src = (const unsigned long *)netdev_stats;
7753 u64 *dst = (u64 *)stats64;
7754
Jarod Wilson92566452016-02-01 18:51:04 -05007755 BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64));
Ben Hutchings3cfde792010-07-09 09:11:52 +00007756 for (i = 0; i < n; i++)
7757 dst[i] = src[i];
Jarod Wilson92566452016-02-01 18:51:04 -05007758 /* zero out counters that only exist in rtnl_link_stats64 */
7759 memset((char *)stats64 + n * sizeof(u64), 0,
7760 sizeof(*stats64) - n * sizeof(u64));
Ben Hutchings3cfde792010-07-09 09:11:52 +00007761#endif
7762}
Eric Dumazet77a1abf2012-03-05 04:50:09 +00007763EXPORT_SYMBOL(netdev_stats_to_stats64);
Ben Hutchings3cfde792010-07-09 09:11:52 +00007764
Eric Dumazetd83345a2009-11-16 03:36:51 +00007765/**
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08007766 * dev_get_stats - get network device statistics
7767 * @dev: device to get statistics from
Eric Dumazet28172732010-07-07 14:58:56 -07007768 * @storage: place to store stats
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08007769 *
Ben Hutchingsd7753512010-07-09 09:12:41 +00007770 * Get network statistics from device. Return @storage.
7771 * The device driver may provide its own method by setting
7772 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
7773 * otherwise the internal statistics structure is used.
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08007774 */
Ben Hutchingsd7753512010-07-09 09:12:41 +00007775struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
7776 struct rtnl_link_stats64 *storage)
Eric Dumazet7004bf22009-05-18 00:34:33 +00007777{
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08007778 const struct net_device_ops *ops = dev->netdev_ops;
7779
Eric Dumazet28172732010-07-07 14:58:56 -07007780 if (ops->ndo_get_stats64) {
7781 memset(storage, 0, sizeof(*storage));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00007782 ops->ndo_get_stats64(dev, storage);
7783 } else if (ops->ndo_get_stats) {
Ben Hutchings3cfde792010-07-09 09:11:52 +00007784 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00007785 } else {
7786 netdev_stats_to_stats64(storage, &dev->stats);
Eric Dumazet28172732010-07-07 14:58:56 -07007787 }
Eric Dumazetcaf586e2010-09-30 21:06:55 +00007788 storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
Eric Dumazet015f0682014-03-27 08:45:56 -07007789 storage->tx_dropped += atomic_long_read(&dev->tx_dropped);
Jarod Wilson6e7333d2016-02-01 18:51:05 -05007790 storage->rx_nohandler += atomic_long_read(&dev->rx_nohandler);
Eric Dumazet28172732010-07-07 14:58:56 -07007791 return storage;
Rusty Russellc45d2862007-03-28 14:29:08 -07007792}
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08007793EXPORT_SYMBOL(dev_get_stats);
Rusty Russellc45d2862007-03-28 14:29:08 -07007794
Eric Dumazet24824a02010-10-02 06:11:55 +00007795struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
David S. Millerdc2b4842008-07-08 17:18:23 -07007796{
Eric Dumazet24824a02010-10-02 06:11:55 +00007797 struct netdev_queue *queue = dev_ingress_queue(dev);
David S. Millerdc2b4842008-07-08 17:18:23 -07007798
Eric Dumazet24824a02010-10-02 06:11:55 +00007799#ifdef CONFIG_NET_CLS_ACT
7800 if (queue)
7801 return queue;
7802 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
7803 if (!queue)
7804 return NULL;
7805 netdev_init_one_queue(dev, queue, NULL);
Eric Dumazet2ce1ee12015-02-04 13:37:44 -08007806 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
Eric Dumazet24824a02010-10-02 06:11:55 +00007807 queue->qdisc_sleeping = &noop_qdisc;
7808 rcu_assign_pointer(dev->ingress_queue, queue);
7809#endif
7810 return queue;
David S. Millerbb949fb2008-07-08 16:55:56 -07007811}
7812
Eric Dumazet2c60db02012-09-16 09:17:26 +00007813static const struct ethtool_ops default_ethtool_ops;
7814
Stanislaw Gruszkad07d7502013-01-10 23:19:10 +00007815void netdev_set_default_ethtool_ops(struct net_device *dev,
7816 const struct ethtool_ops *ops)
7817{
7818 if (dev->ethtool_ops == &default_ethtool_ops)
7819 dev->ethtool_ops = ops;
7820}
7821EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
7822
Eric Dumazet74d332c2013-10-30 13:10:44 -07007823void netdev_freemem(struct net_device *dev)
7824{
7825 char *addr = (char *)dev - dev->padded;
7826
WANG Cong4cb28972014-06-02 15:55:22 -07007827 kvfree(addr);
Eric Dumazet74d332c2013-10-30 13:10:44 -07007828}
7829
Linus Torvalds1da177e2005-04-16 15:20:36 -07007830/**
tcharding722c9a02017-02-09 17:56:04 +11007831 * alloc_netdev_mqs - allocate network device
7832 * @sizeof_priv: size of private data to allocate space for
7833 * @name: device name format string
7834 * @name_assign_type: origin of device name
7835 * @setup: callback to initialize device
7836 * @txqs: the number of TX subqueues to allocate
7837 * @rxqs: the number of RX subqueues to allocate
Linus Torvalds1da177e2005-04-16 15:20:36 -07007838 *
tcharding722c9a02017-02-09 17:56:04 +11007839 * Allocates a struct net_device with private data area for driver use
7840 * and performs basic initialization. Also allocates subqueue structs
7841 * for each queue on the device.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007842 */
Tom Herbert36909ea2011-01-09 19:36:31 +00007843struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
Tom Gundersenc835a672014-07-14 16:37:24 +02007844 unsigned char name_assign_type,
Tom Herbert36909ea2011-01-09 19:36:31 +00007845 void (*setup)(struct net_device *),
7846 unsigned int txqs, unsigned int rxqs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007847{
Linus Torvalds1da177e2005-04-16 15:20:36 -07007848 struct net_device *dev;
Stephen Hemminger79439862008-07-21 13:28:44 -07007849 size_t alloc_size;
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00007850 struct net_device *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007851
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07007852 BUG_ON(strlen(name) >= sizeof(dev->name));
7853
Tom Herbert36909ea2011-01-09 19:36:31 +00007854 if (txqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00007855 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
Tom Herbert55513fb2010-10-18 17:55:58 +00007856 return NULL;
7857 }
7858
Michael Daltona953be52014-01-16 22:23:28 -08007859#ifdef CONFIG_SYSFS
Tom Herbert36909ea2011-01-09 19:36:31 +00007860 if (rxqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00007861 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
Tom Herbert36909ea2011-01-09 19:36:31 +00007862 return NULL;
7863 }
7864#endif
7865
David S. Millerfd2ea0a2008-07-17 01:56:23 -07007866 alloc_size = sizeof(struct net_device);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07007867 if (sizeof_priv) {
7868 /* ensure 32-byte alignment of private area */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00007869 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07007870 alloc_size += sizeof_priv;
7871 }
7872 /* ensure 32-byte alignment of whole construct */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00007873 alloc_size += NETDEV_ALIGN - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007874
Michal Hockoda6bc572017-05-08 15:57:31 -07007875 p = kvzalloc(alloc_size, GFP_KERNEL | __GFP_REPEAT);
Joe Perches62b59422013-02-04 16:48:16 +00007876 if (!p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007877 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007878
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00007879 dev = PTR_ALIGN(p, NETDEV_ALIGN);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007880 dev->padded = (char *)dev - (char *)p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00007881
Eric Dumazet29b44332010-10-11 10:22:12 +00007882 dev->pcpu_refcnt = alloc_percpu(int);
7883 if (!dev->pcpu_refcnt)
Eric Dumazet74d332c2013-10-30 13:10:44 -07007884 goto free_dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00007885
Linus Torvalds1da177e2005-04-16 15:20:36 -07007886 if (dev_addr_init(dev))
Eric Dumazet29b44332010-10-11 10:22:12 +00007887 goto free_pcpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007888
Jiri Pirko22bedad32010-04-01 21:22:57 +00007889 dev_mc_init(dev);
Jiri Pirkoa748ee22010-04-01 21:22:09 +00007890 dev_uc_init(dev);
Jiri Pirkoccffad252009-05-22 23:22:17 +00007891
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09007892 dev_net_set(dev, &init_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007893
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07007894 dev->gso_max_size = GSO_MAX_SIZE;
Ben Hutchings30b678d2012-07-30 15:57:00 +00007895 dev->gso_max_segs = GSO_MAX_SEGS;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007896
Herbert Xud565b0a2008-12-15 23:38:52 -08007897 INIT_LIST_HEAD(&dev->napi_list);
Eric W. Biederman9fdce092009-10-30 14:51:13 +00007898 INIT_LIST_HEAD(&dev->unreg_list);
Eric W. Biederman5cde2822013-10-05 19:26:05 -07007899 INIT_LIST_HEAD(&dev->close_list);
Eric Dumazete014deb2009-11-17 05:59:21 +00007900 INIT_LIST_HEAD(&dev->link_watch_list);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02007901 INIT_LIST_HEAD(&dev->adj_list.upper);
7902 INIT_LIST_HEAD(&dev->adj_list.lower);
Salam Noureddine7866a622015-01-27 11:35:48 -08007903 INIT_LIST_HEAD(&dev->ptype_all);
7904 INIT_LIST_HEAD(&dev->ptype_specific);
Jiri Kosina59cc1f62016-08-10 11:05:15 +02007905#ifdef CONFIG_NET_SCHED
7906 hash_init(dev->qdisc_hash);
7907#endif
Eric Dumazet02875872014-10-05 18:38:35 -07007908 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007909 setup(dev);
David S. Miller8d3bdbd2011-02-08 15:02:50 -08007910
Phil Suttera8131042016-02-17 15:37:43 +01007911 if (!dev->tx_queue_len) {
Phil Sutterf84bb1e2015-08-27 21:21:36 +02007912 dev->priv_flags |= IFF_NO_QUEUE;
Jesper Dangaard Brouer11597082016-11-03 14:56:06 +01007913 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
Phil Suttera8131042016-02-17 15:37:43 +01007914 }
Phil Sutter906470c2015-08-18 10:30:48 +02007915
David S. Miller8d3bdbd2011-02-08 15:02:50 -08007916 dev->num_tx_queues = txqs;
7917 dev->real_num_tx_queues = txqs;
7918 if (netif_alloc_netdev_queues(dev))
7919 goto free_all;
7920
Michael Daltona953be52014-01-16 22:23:28 -08007921#ifdef CONFIG_SYSFS
David S. Miller8d3bdbd2011-02-08 15:02:50 -08007922 dev->num_rx_queues = rxqs;
7923 dev->real_num_rx_queues = rxqs;
7924 if (netif_alloc_rx_queues(dev))
7925 goto free_all;
7926#endif
7927
Linus Torvalds1da177e2005-04-16 15:20:36 -07007928 strcpy(dev->name, name);
Tom Gundersenc835a672014-07-14 16:37:24 +02007929 dev->name_assign_type = name_assign_type;
Vlad Dogarucbda10f2011-01-13 23:38:30 +00007930 dev->group = INIT_NETDEV_GROUP;
Eric Dumazet2c60db02012-09-16 09:17:26 +00007931 if (!dev->ethtool_ops)
7932 dev->ethtool_ops = &default_ethtool_ops;
Pablo Neirae687ad62015-05-13 18:19:38 +02007933
7934 nf_hook_ingress_init(dev);
7935
Linus Torvalds1da177e2005-04-16 15:20:36 -07007936 return dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00007937
David S. Miller8d3bdbd2011-02-08 15:02:50 -08007938free_all:
7939 free_netdev(dev);
7940 return NULL;
7941
Eric Dumazet29b44332010-10-11 10:22:12 +00007942free_pcpu:
7943 free_percpu(dev->pcpu_refcnt);
Eric Dumazet74d332c2013-10-30 13:10:44 -07007944free_dev:
7945 netdev_freemem(dev);
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00007946 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007947}
Tom Herbert36909ea2011-01-09 19:36:31 +00007948EXPORT_SYMBOL(alloc_netdev_mqs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007949
7950/**
tcharding722c9a02017-02-09 17:56:04 +11007951 * free_netdev - free network device
7952 * @dev: device
Linus Torvalds1da177e2005-04-16 15:20:36 -07007953 *
tcharding722c9a02017-02-09 17:56:04 +11007954 * This function does the last stage of destroying an allocated device
7955 * interface. The reference to the device object is released. If this
7956 * is the last reference then it will be freed.Must be called in process
7957 * context.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007958 */
7959void free_netdev(struct net_device *dev)
7960{
Herbert Xud565b0a2008-12-15 23:38:52 -08007961 struct napi_struct *p, *n;
David S. Millerb5cdae32017-04-18 15:36:58 -04007962 struct bpf_prog *prog;
Herbert Xud565b0a2008-12-15 23:38:52 -08007963
Eric Dumazet93d05d42015-11-18 06:31:03 -08007964 might_sleep();
Eric Dumazet60877a32013-06-20 01:15:51 -07007965 netif_free_tx_queues(dev);
Michael Daltona953be52014-01-16 22:23:28 -08007966#ifdef CONFIG_SYSFS
Pankaj Gupta10595902015-01-12 11:41:28 +05307967 kvfree(dev->_rx);
Tom Herbertfe822242010-11-09 10:47:38 +00007968#endif
David S. Millere8a04642008-07-17 00:34:19 -07007969
Eric Dumazet33d480c2011-08-11 19:30:52 +00007970 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
Eric Dumazet24824a02010-10-02 06:11:55 +00007971
Jiri Pirkof001fde2009-05-05 02:48:28 +00007972 /* Flush device addresses */
7973 dev_addr_flush(dev);
7974
Herbert Xud565b0a2008-12-15 23:38:52 -08007975 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
7976 netif_napi_del(p);
7977
Eric Dumazet29b44332010-10-11 10:22:12 +00007978 free_percpu(dev->pcpu_refcnt);
7979 dev->pcpu_refcnt = NULL;
7980
David S. Millerb5cdae32017-04-18 15:36:58 -04007981 prog = rcu_dereference_protected(dev->xdp_prog, 1);
7982 if (prog) {
7983 bpf_prog_put(prog);
7984 static_key_slow_dec(&generic_xdp_needed);
7985 }
7986
Stephen Hemminger3041a062006-05-26 13:25:24 -07007987 /* Compatibility with error handling in drivers */
Linus Torvalds1da177e2005-04-16 15:20:36 -07007988 if (dev->reg_state == NETREG_UNINITIALIZED) {
Eric Dumazet74d332c2013-10-30 13:10:44 -07007989 netdev_freemem(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007990 return;
7991 }
7992
7993 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
7994 dev->reg_state = NETREG_RELEASED;
7995
Greg Kroah-Hartman43cb76d2002-04-09 12:14:34 -07007996 /* will free via device release */
7997 put_device(&dev->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007998}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07007999EXPORT_SYMBOL(free_netdev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09008000
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07008001/**
8002 * synchronize_net - Synchronize with packet receive processing
8003 *
8004 * Wait for packets currently being received to be done.
8005 * Does not block later packets from starting.
8006 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09008007void synchronize_net(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008008{
8009 might_sleep();
Eric Dumazetbe3fc412011-05-23 23:07:32 +00008010 if (rtnl_is_locked())
8011 synchronize_rcu_expedited();
8012 else
8013 synchronize_rcu();
Linus Torvalds1da177e2005-04-16 15:20:36 -07008014}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07008015EXPORT_SYMBOL(synchronize_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008016
8017/**
Eric Dumazet44a08732009-10-27 07:03:04 +00008018 * unregister_netdevice_queue - remove device from the kernel
Linus Torvalds1da177e2005-04-16 15:20:36 -07008019 * @dev: device
Eric Dumazet44a08732009-10-27 07:03:04 +00008020 * @head: list
Jaswinder Singh Rajput6ebfbc02009-11-22 20:43:13 -08008021 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07008022 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08008023 * from the kernel tables.
Eric Dumazet44a08732009-10-27 07:03:04 +00008024 * If head not NULL, device is queued to be unregistered later.
Linus Torvalds1da177e2005-04-16 15:20:36 -07008025 *
8026 * Callers must hold the rtnl semaphore. You may want
8027 * unregister_netdev() instead of this.
8028 */
8029
Eric Dumazet44a08732009-10-27 07:03:04 +00008030void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008031{
Herbert Xua6620712007-12-12 19:21:56 -08008032 ASSERT_RTNL();
8033
Eric Dumazet44a08732009-10-27 07:03:04 +00008034 if (head) {
Eric W. Biederman9fdce092009-10-30 14:51:13 +00008035 list_move_tail(&dev->unreg_list, head);
Eric Dumazet44a08732009-10-27 07:03:04 +00008036 } else {
8037 rollback_registered(dev);
8038 /* Finish processing unregister after unlock */
8039 net_set_todo(dev);
8040 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07008041}
Eric Dumazet44a08732009-10-27 07:03:04 +00008042EXPORT_SYMBOL(unregister_netdevice_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008043
8044/**
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008045 * unregister_netdevice_many - unregister many devices
8046 * @head: list of devices
Eric Dumazet87757a92014-06-06 06:44:03 -07008047 *
8048 * Note: As most callers use a stack allocated list_head,
8049 * we force a list_del() to make sure stack wont be corrupted later.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008050 */
8051void unregister_netdevice_many(struct list_head *head)
8052{
8053 struct net_device *dev;
8054
8055 if (!list_empty(head)) {
8056 rollback_registered_many(head);
8057 list_for_each_entry(dev, head, unreg_list)
8058 net_set_todo(dev);
Eric Dumazet87757a92014-06-06 06:44:03 -07008059 list_del(head);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008060 }
8061}
Eric Dumazet63c80992009-10-27 07:06:49 +00008062EXPORT_SYMBOL(unregister_netdevice_many);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008063
8064/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07008065 * unregister_netdev - remove device from the kernel
8066 * @dev: device
8067 *
8068 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08008069 * from the kernel tables.
Linus Torvalds1da177e2005-04-16 15:20:36 -07008070 *
8071 * This is just a wrapper for unregister_netdevice that takes
8072 * the rtnl semaphore. In general you want to use this and not
8073 * unregister_netdevice.
8074 */
8075void unregister_netdev(struct net_device *dev)
8076{
8077 rtnl_lock();
8078 unregister_netdevice(dev);
8079 rtnl_unlock();
8080}
Linus Torvalds1da177e2005-04-16 15:20:36 -07008081EXPORT_SYMBOL(unregister_netdev);
8082
Eric W. Biedermance286d32007-09-12 13:53:49 +02008083/**
8084 * dev_change_net_namespace - move device to different nethost namespace
8085 * @dev: device
8086 * @net: network namespace
8087 * @pat: If not NULL name pattern to try if the current device name
8088 * is already taken in the destination network namespace.
8089 *
8090 * This function shuts down a device interface and moves it
8091 * to a new network namespace. On success 0 is returned, on
8092 * a failure a netagive errno code is returned.
8093 *
8094 * Callers must hold the rtnl semaphore.
8095 */
8096
8097int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
8098{
Eric W. Biedermance286d32007-09-12 13:53:49 +02008099 int err;
8100
8101 ASSERT_RTNL();
8102
8103 /* Don't allow namespace local devices to be moved. */
8104 err = -EINVAL;
8105 if (dev->features & NETIF_F_NETNS_LOCAL)
8106 goto out;
8107
8108 /* Ensure the device has been registrered */
Eric W. Biedermance286d32007-09-12 13:53:49 +02008109 if (dev->reg_state != NETREG_REGISTERED)
8110 goto out;
8111
8112 /* Get out if there is nothing todo */
8113 err = 0;
YOSHIFUJI Hideaki878628f2008-03-26 03:57:35 +09008114 if (net_eq(dev_net(dev), net))
Eric W. Biedermance286d32007-09-12 13:53:49 +02008115 goto out;
8116
8117 /* Pick the destination device name, and ensure
8118 * we can use it in the destination network namespace.
8119 */
8120 err = -EEXIST;
Octavian Purdilad9031022009-11-18 02:36:59 +00008121 if (__dev_get_by_name(net, dev->name)) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02008122 /* We get here if we can't use the current device name */
8123 if (!pat)
8124 goto out;
Gao feng828de4f2012-09-13 20:58:27 +00008125 if (dev_get_valid_name(net, dev, pat) < 0)
Eric W. Biedermance286d32007-09-12 13:53:49 +02008126 goto out;
8127 }
8128
8129 /*
8130 * And now a mini version of register_netdevice unregister_netdevice.
8131 */
8132
8133 /* If device is running close it first. */
Pavel Emelyanov9b772652007-10-10 02:49:09 -07008134 dev_close(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02008135
8136 /* And unlink it from device chain */
8137 err = -ENODEV;
8138 unlist_netdevice(dev);
8139
8140 synchronize_net();
8141
8142 /* Shutdown queueing discipline. */
8143 dev_shutdown(dev);
8144
8145 /* Notify protocols, that we are about to destroy
tchardingeb13da12017-02-09 17:56:06 +11008146 * this device. They should clean all the things.
8147 *
8148 * Note that dev->reg_state stays at NETREG_REGISTERED.
8149 * This is wanted because this way 8021q and macvlan know
8150 * the device is just moving and can keep their slaves up.
8151 */
Eric W. Biedermance286d32007-09-12 13:53:49 +02008152 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Gao feng6549dd42012-08-23 15:36:55 +00008153 rcu_barrier();
8154 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Alexei Starovoitov7f294052013-10-23 16:02:42 -07008155 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U, GFP_KERNEL);
Eric W. Biedermance286d32007-09-12 13:53:49 +02008156
8157 /*
8158 * Flush the unicast and multicast chains
8159 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00008160 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00008161 dev_mc_flush(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02008162
Serge Hallyn4e66ae22012-12-03 16:17:12 +00008163 /* Send a netdev-removed uevent to the old namespace */
8164 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04008165 netdev_adjacent_del_links(dev);
Serge Hallyn4e66ae22012-12-03 16:17:12 +00008166
Eric W. Biedermance286d32007-09-12 13:53:49 +02008167 /* Actually switch the network namespace */
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09008168 dev_net_set(dev, net);
Eric W. Biedermance286d32007-09-12 13:53:49 +02008169
Eric W. Biedermance286d32007-09-12 13:53:49 +02008170 /* If there is an ifindex conflict assign a new one */
Nicolas Dichtel7a66bbc2015-04-02 17:07:09 +02008171 if (__dev_get_by_index(net, dev->ifindex))
Eric W. Biedermance286d32007-09-12 13:53:49 +02008172 dev->ifindex = dev_new_index(net);
Eric W. Biedermance286d32007-09-12 13:53:49 +02008173
Serge Hallyn4e66ae22012-12-03 16:17:12 +00008174 /* Send a netdev-add uevent to the new namespace */
8175 kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04008176 netdev_adjacent_add_links(dev);
Serge Hallyn4e66ae22012-12-03 16:17:12 +00008177
Eric W. Biederman8b41d182007-09-26 22:02:53 -07008178 /* Fixup kobjects */
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07008179 err = device_rename(&dev->dev, dev->name);
Eric W. Biederman8b41d182007-09-26 22:02:53 -07008180 WARN_ON(err);
Eric W. Biedermance286d32007-09-12 13:53:49 +02008181
8182 /* Add the device back in the hashes */
8183 list_netdevice(dev);
8184
8185 /* Notify protocols, that a new device appeared. */
8186 call_netdevice_notifiers(NETDEV_REGISTER, dev);
8187
Eric W. Biedermand90a9092009-12-12 22:11:15 +00008188 /*
8189 * Prevent userspace races by waiting until the network
8190 * device is fully setup before sending notifications.
8191 */
Alexei Starovoitov7f294052013-10-23 16:02:42 -07008192 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
Eric W. Biedermand90a9092009-12-12 22:11:15 +00008193
Eric W. Biedermance286d32007-09-12 13:53:49 +02008194 synchronize_net();
8195 err = 0;
8196out:
8197 return err;
8198}
Johannes Berg463d0182009-07-14 00:33:35 +02008199EXPORT_SYMBOL_GPL(dev_change_net_namespace);
Eric W. Biedermance286d32007-09-12 13:53:49 +02008200
Sebastian Andrzej Siewiorf0bf90d2016-11-03 15:50:04 +01008201static int dev_cpu_dead(unsigned int oldcpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008202{
8203 struct sk_buff **list_skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008204 struct sk_buff *skb;
Sebastian Andrzej Siewiorf0bf90d2016-11-03 15:50:04 +01008205 unsigned int cpu;
Ashwanth Goli97d8b6e2017-06-13 16:54:55 +05308206 struct softnet_data *sd, *oldsd, *remsd = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008207
Linus Torvalds1da177e2005-04-16 15:20:36 -07008208 local_irq_disable();
8209 cpu = smp_processor_id();
8210 sd = &per_cpu(softnet_data, cpu);
8211 oldsd = &per_cpu(softnet_data, oldcpu);
8212
8213 /* Find end of our completion_queue. */
8214 list_skb = &sd->completion_queue;
8215 while (*list_skb)
8216 list_skb = &(*list_skb)->next;
8217 /* Append completion queue from offline CPU. */
8218 *list_skb = oldsd->completion_queue;
8219 oldsd->completion_queue = NULL;
8220
Linus Torvalds1da177e2005-04-16 15:20:36 -07008221 /* Append output queue from offline CPU. */
Changli Gaoa9cbd582010-04-26 23:06:24 +00008222 if (oldsd->output_queue) {
8223 *sd->output_queue_tailp = oldsd->output_queue;
8224 sd->output_queue_tailp = oldsd->output_queue_tailp;
8225 oldsd->output_queue = NULL;
8226 oldsd->output_queue_tailp = &oldsd->output_queue;
8227 }
Eric Dumazetac64da02015-01-15 17:04:22 -08008228 /* Append NAPI poll list from offline CPU, with one exception :
8229 * process_backlog() must be called by cpu owning percpu backlog.
8230 * We properly handle process_queue & input_pkt_queue later.
8231 */
8232 while (!list_empty(&oldsd->poll_list)) {
8233 struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
8234 struct napi_struct,
8235 poll_list);
8236
8237 list_del_init(&napi->poll_list);
8238 if (napi->poll == process_backlog)
8239 napi->state = 0;
8240 else
8241 ____napi_schedule(sd, napi);
Heiko Carstens264524d2011-06-06 20:50:03 +00008242 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07008243
8244 raise_softirq_irqoff(NET_TX_SOFTIRQ);
8245 local_irq_enable();
8246
ashwanth@codeaurora.org773fc8f2017-06-09 14:24:58 +05308247#ifdef CONFIG_RPS
8248 remsd = oldsd->rps_ipi_list;
8249 oldsd->rps_ipi_list = NULL;
8250#endif
8251 /* send out pending IPI's on offline CPU */
8252 net_rps_send_ipi(remsd);
8253
Linus Torvalds1da177e2005-04-16 15:20:36 -07008254 /* Process offline CPU's input_pkt_queue */
Tom Herbert76cc8b12010-05-20 18:37:59 +00008255 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
Eric Dumazet91e83132015-02-05 14:58:14 -08008256 netif_rx_ni(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00008257 input_queue_head_incr(oldsd);
8258 }
Eric Dumazetac64da02015-01-15 17:04:22 -08008259 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
Eric Dumazet91e83132015-02-05 14:58:14 -08008260 netif_rx_ni(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00008261 input_queue_head_incr(oldsd);
Tom Herbertfec5e652010-04-16 16:01:27 -07008262 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07008263
Sebastian Andrzej Siewiorf0bf90d2016-11-03 15:50:04 +01008264 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008265}
Linus Torvalds1da177e2005-04-16 15:20:36 -07008266
Herbert Xu7f353bf2007-08-10 15:47:58 -07008267/**
Herbert Xub63365a2008-10-23 01:11:29 -07008268 * netdev_increment_features - increment feature set by one
8269 * @all: current feature set
8270 * @one: new feature set
8271 * @mask: mask feature set
Herbert Xu7f353bf2007-08-10 15:47:58 -07008272 *
8273 * Computes a new feature set after adding a device with feature set
Herbert Xub63365a2008-10-23 01:11:29 -07008274 * @one to the master device with current feature set @all. Will not
8275 * enable anything that is off in @mask. Returns the new feature set.
Herbert Xu7f353bf2007-08-10 15:47:58 -07008276 */
Michał Mirosławc8f44af2011-11-15 15:29:55 +00008277netdev_features_t netdev_increment_features(netdev_features_t all,
8278 netdev_features_t one, netdev_features_t mask)
Herbert Xu7f353bf2007-08-10 15:47:58 -07008279{
Tom Herbertc8cd0982015-12-14 11:19:44 -08008280 if (mask & NETIF_F_HW_CSUM)
Tom Herberta1882222015-12-14 11:19:43 -08008281 mask |= NETIF_F_CSUM_MASK;
Michał Mirosław1742f182011-04-22 06:31:16 +00008282 mask |= NETIF_F_VLAN_CHALLENGED;
8283
Tom Herberta1882222015-12-14 11:19:43 -08008284 all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask;
Michał Mirosław1742f182011-04-22 06:31:16 +00008285 all &= one | ~NETIF_F_ALL_FOR_ALL;
8286
Michał Mirosław1742f182011-04-22 06:31:16 +00008287 /* If one device supports hw checksumming, set for all. */
Tom Herbertc8cd0982015-12-14 11:19:44 -08008288 if (all & NETIF_F_HW_CSUM)
8289 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM);
Herbert Xu7f353bf2007-08-10 15:47:58 -07008290
8291 return all;
8292}
Herbert Xub63365a2008-10-23 01:11:29 -07008293EXPORT_SYMBOL(netdev_increment_features);
Herbert Xu7f353bf2007-08-10 15:47:58 -07008294
Baruch Siach430f03c2013-06-02 20:43:55 +00008295static struct hlist_head * __net_init netdev_create_hash(void)
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07008296{
8297 int i;
8298 struct hlist_head *hash;
8299
8300 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
8301 if (hash != NULL)
8302 for (i = 0; i < NETDEV_HASHENTRIES; i++)
8303 INIT_HLIST_HEAD(&hash[i]);
8304
8305 return hash;
8306}
8307
Eric W. Biederman881d9662007-09-17 11:56:21 -07008308/* Initialize per network namespace state */
Pavel Emelyanov46650792007-10-08 20:38:39 -07008309static int __net_init netdev_init(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07008310{
Rustad, Mark D734b6542012-07-18 09:06:07 +00008311 if (net != &init_net)
8312 INIT_LIST_HEAD(&net->dev_base_head);
Eric W. Biederman881d9662007-09-17 11:56:21 -07008313
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07008314 net->dev_name_head = netdev_create_hash();
8315 if (net->dev_name_head == NULL)
8316 goto err_name;
Eric W. Biederman881d9662007-09-17 11:56:21 -07008317
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07008318 net->dev_index_head = netdev_create_hash();
8319 if (net->dev_index_head == NULL)
8320 goto err_idx;
Eric W. Biederman881d9662007-09-17 11:56:21 -07008321
8322 return 0;
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07008323
8324err_idx:
8325 kfree(net->dev_name_head);
8326err_name:
8327 return -ENOMEM;
Eric W. Biederman881d9662007-09-17 11:56:21 -07008328}
8329
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07008330/**
8331 * netdev_drivername - network driver for the device
8332 * @dev: network device
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07008333 *
8334 * Determine network driver for device.
8335 */
David S. Miller3019de12011-06-06 16:41:33 -07008336const char *netdev_drivername(const struct net_device *dev)
Arjan van de Ven6579e572008-07-21 13:31:48 -07008337{
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07008338 const struct device_driver *driver;
8339 const struct device *parent;
David S. Miller3019de12011-06-06 16:41:33 -07008340 const char *empty = "";
Arjan van de Ven6579e572008-07-21 13:31:48 -07008341
8342 parent = dev->dev.parent;
Arjan van de Ven6579e572008-07-21 13:31:48 -07008343 if (!parent)
David S. Miller3019de12011-06-06 16:41:33 -07008344 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07008345
8346 driver = parent->driver;
8347 if (driver && driver->name)
David S. Miller3019de12011-06-06 16:41:33 -07008348 return driver->name;
8349 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07008350}
8351
Joe Perches6ea754e2014-09-22 11:10:50 -07008352static void __netdev_printk(const char *level, const struct net_device *dev,
8353 struct va_format *vaf)
Joe Perches256df2f2010-06-27 01:02:35 +00008354{
Joe Perchesb004ff42012-09-12 20:12:19 -07008355 if (dev && dev->dev.parent) {
Joe Perches6ea754e2014-09-22 11:10:50 -07008356 dev_printk_emit(level[1] - '0',
8357 dev->dev.parent,
8358 "%s %s %s%s: %pV",
8359 dev_driver_string(dev->dev.parent),
8360 dev_name(dev->dev.parent),
8361 netdev_name(dev), netdev_reg_state(dev),
8362 vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07008363 } else if (dev) {
Joe Perches6ea754e2014-09-22 11:10:50 -07008364 printk("%s%s%s: %pV",
8365 level, netdev_name(dev), netdev_reg_state(dev), vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07008366 } else {
Joe Perches6ea754e2014-09-22 11:10:50 -07008367 printk("%s(NULL net_device): %pV", level, vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07008368 }
Joe Perches256df2f2010-06-27 01:02:35 +00008369}
8370
Joe Perches6ea754e2014-09-22 11:10:50 -07008371void netdev_printk(const char *level, const struct net_device *dev,
8372 const char *format, ...)
Joe Perches256df2f2010-06-27 01:02:35 +00008373{
8374 struct va_format vaf;
8375 va_list args;
Joe Perches256df2f2010-06-27 01:02:35 +00008376
8377 va_start(args, format);
8378
8379 vaf.fmt = format;
8380 vaf.va = &args;
8381
Joe Perches6ea754e2014-09-22 11:10:50 -07008382 __netdev_printk(level, dev, &vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07008383
Joe Perches256df2f2010-06-27 01:02:35 +00008384 va_end(args);
Joe Perches256df2f2010-06-27 01:02:35 +00008385}
8386EXPORT_SYMBOL(netdev_printk);
8387
8388#define define_netdev_printk_level(func, level) \
Joe Perches6ea754e2014-09-22 11:10:50 -07008389void func(const struct net_device *dev, const char *fmt, ...) \
Joe Perches256df2f2010-06-27 01:02:35 +00008390{ \
Joe Perches256df2f2010-06-27 01:02:35 +00008391 struct va_format vaf; \
8392 va_list args; \
8393 \
8394 va_start(args, fmt); \
8395 \
8396 vaf.fmt = fmt; \
8397 vaf.va = &args; \
8398 \
Joe Perches6ea754e2014-09-22 11:10:50 -07008399 __netdev_printk(level, dev, &vaf); \
Joe Perchesb004ff42012-09-12 20:12:19 -07008400 \
Joe Perches256df2f2010-06-27 01:02:35 +00008401 va_end(args); \
Joe Perches256df2f2010-06-27 01:02:35 +00008402} \
8403EXPORT_SYMBOL(func);
8404
8405define_netdev_printk_level(netdev_emerg, KERN_EMERG);
8406define_netdev_printk_level(netdev_alert, KERN_ALERT);
8407define_netdev_printk_level(netdev_crit, KERN_CRIT);
8408define_netdev_printk_level(netdev_err, KERN_ERR);
8409define_netdev_printk_level(netdev_warn, KERN_WARNING);
8410define_netdev_printk_level(netdev_notice, KERN_NOTICE);
8411define_netdev_printk_level(netdev_info, KERN_INFO);
8412
Pavel Emelyanov46650792007-10-08 20:38:39 -07008413static void __net_exit netdev_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07008414{
8415 kfree(net->dev_name_head);
8416 kfree(net->dev_index_head);
8417}
8418
Denis V. Lunev022cbae2007-11-13 03:23:50 -08008419static struct pernet_operations __net_initdata netdev_net_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07008420 .init = netdev_init,
8421 .exit = netdev_exit,
8422};
8423
Pavel Emelyanov46650792007-10-08 20:38:39 -07008424static void __net_exit default_device_exit(struct net *net)
Eric W. Biedermance286d32007-09-12 13:53:49 +02008425{
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00008426 struct net_device *dev, *aux;
Eric W. Biedermance286d32007-09-12 13:53:49 +02008427 /*
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00008428 * Push all migratable network devices back to the
Eric W. Biedermance286d32007-09-12 13:53:49 +02008429 * initial network namespace
8430 */
8431 rtnl_lock();
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00008432 for_each_netdev_safe(net, dev, aux) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02008433 int err;
Pavel Emelyanovaca51392008-05-08 01:24:25 -07008434 char fb_name[IFNAMSIZ];
Eric W. Biedermance286d32007-09-12 13:53:49 +02008435
8436 /* Ignore unmoveable devices (i.e. loopback) */
8437 if (dev->features & NETIF_F_NETNS_LOCAL)
8438 continue;
8439
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00008440 /* Leave virtual devices for the generic cleanup */
8441 if (dev->rtnl_link_ops)
8442 continue;
Eric W. Biedermand0c082c2008-11-05 15:59:38 -08008443
Lucas De Marchi25985ed2011-03-30 22:57:33 -03008444 /* Push remaining network devices to init_net */
Pavel Emelyanovaca51392008-05-08 01:24:25 -07008445 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
8446 err = dev_change_net_namespace(dev, &init_net, fb_name);
Eric W. Biedermance286d32007-09-12 13:53:49 +02008447 if (err) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00008448 pr_emerg("%s: failed to move %s to init_net: %d\n",
8449 __func__, dev->name, err);
Pavel Emelyanovaca51392008-05-08 01:24:25 -07008450 BUG();
Eric W. Biedermance286d32007-09-12 13:53:49 +02008451 }
8452 }
8453 rtnl_unlock();
8454}
8455
Eric W. Biederman50624c92013-09-23 21:19:49 -07008456static void __net_exit rtnl_lock_unregistering(struct list_head *net_list)
8457{
8458 /* Return with the rtnl_lock held when there are no network
8459 * devices unregistering in any network namespace in net_list.
8460 */
8461 struct net *net;
8462 bool unregistering;
Peter Zijlstraff960a72014-10-29 17:04:56 +01008463 DEFINE_WAIT_FUNC(wait, woken_wake_function);
Eric W. Biederman50624c92013-09-23 21:19:49 -07008464
Peter Zijlstraff960a72014-10-29 17:04:56 +01008465 add_wait_queue(&netdev_unregistering_wq, &wait);
Eric W. Biederman50624c92013-09-23 21:19:49 -07008466 for (;;) {
Eric W. Biederman50624c92013-09-23 21:19:49 -07008467 unregistering = false;
8468 rtnl_lock();
8469 list_for_each_entry(net, net_list, exit_list) {
8470 if (net->dev_unreg_count > 0) {
8471 unregistering = true;
8472 break;
8473 }
8474 }
8475 if (!unregistering)
8476 break;
8477 __rtnl_unlock();
Peter Zijlstraff960a72014-10-29 17:04:56 +01008478
8479 wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
Eric W. Biederman50624c92013-09-23 21:19:49 -07008480 }
Peter Zijlstraff960a72014-10-29 17:04:56 +01008481 remove_wait_queue(&netdev_unregistering_wq, &wait);
Eric W. Biederman50624c92013-09-23 21:19:49 -07008482}
8483
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00008484static void __net_exit default_device_exit_batch(struct list_head *net_list)
8485{
8486 /* At exit all network devices most be removed from a network
Uwe Kleine-Königb5950762010-11-01 15:38:34 -04008487 * namespace. Do this in the reverse order of registration.
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00008488 * Do this across as many network namespaces as possible to
8489 * improve batching efficiency.
8490 */
8491 struct net_device *dev;
8492 struct net *net;
8493 LIST_HEAD(dev_kill_list);
8494
Eric W. Biederman50624c92013-09-23 21:19:49 -07008495 /* To prevent network device cleanup code from dereferencing
8496 * loopback devices or network devices that have been freed
8497 * wait here for all pending unregistrations to complete,
8498 * before unregistring the loopback device and allowing the
8499 * network namespace be freed.
8500 *
8501 * The netdev todo list containing all network devices
8502 * unregistrations that happen in default_device_exit_batch
8503 * will run in the rtnl_unlock() at the end of
8504 * default_device_exit_batch.
8505 */
8506 rtnl_lock_unregistering(net_list);
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00008507 list_for_each_entry(net, net_list, exit_list) {
8508 for_each_netdev_reverse(net, dev) {
Jiri Pirkob0ab2fa2014-06-26 09:58:25 +02008509 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00008510 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
8511 else
8512 unregister_netdevice_queue(dev, &dev_kill_list);
8513 }
8514 }
8515 unregister_netdevice_many(&dev_kill_list);
8516 rtnl_unlock();
8517}
8518
Denis V. Lunev022cbae2007-11-13 03:23:50 -08008519static struct pernet_operations __net_initdata default_device_ops = {
Eric W. Biedermance286d32007-09-12 13:53:49 +02008520 .exit = default_device_exit,
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00008521 .exit_batch = default_device_exit_batch,
Eric W. Biedermance286d32007-09-12 13:53:49 +02008522};
8523
Linus Torvalds1da177e2005-04-16 15:20:36 -07008524/*
8525 * Initialize the DEV module. At boot time this walks the device list and
8526 * unhooks any devices that fail to initialise (normally hardware not
8527 * present) and leaves us with a valid list of present and active devices.
8528 *
8529 */
8530
8531/*
8532 * This is called single threaded during boot, so no need
8533 * to take the rtnl semaphore.
8534 */
8535static int __init net_dev_init(void)
8536{
8537 int i, rc = -ENOMEM;
8538
8539 BUG_ON(!dev_boot_phase);
8540
Linus Torvalds1da177e2005-04-16 15:20:36 -07008541 if (dev_proc_init())
8542 goto out;
8543
Eric W. Biederman8b41d182007-09-26 22:02:53 -07008544 if (netdev_kobject_init())
Linus Torvalds1da177e2005-04-16 15:20:36 -07008545 goto out;
8546
8547 INIT_LIST_HEAD(&ptype_all);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08008548 for (i = 0; i < PTYPE_HASH_SIZE; i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008549 INIT_LIST_HEAD(&ptype_base[i]);
8550
Vlad Yasevich62532da2012-11-15 08:49:10 +00008551 INIT_LIST_HEAD(&offload_base);
8552
Eric W. Biederman881d9662007-09-17 11:56:21 -07008553 if (register_pernet_subsys(&netdev_net_ops))
8554 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008555
8556 /*
8557 * Initialise the packet receive queues.
8558 */
8559
KAMEZAWA Hiroyuki6f912042006-04-10 22:52:50 -07008560 for_each_possible_cpu(i) {
Eric Dumazet41852492016-08-26 12:50:39 -07008561 struct work_struct *flush = per_cpu_ptr(&flush_works, i);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00008562 struct softnet_data *sd = &per_cpu(softnet_data, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008563
Eric Dumazet41852492016-08-26 12:50:39 -07008564 INIT_WORK(flush, flush_backlog);
8565
Eric Dumazete36fa2f2010-04-19 21:17:14 +00008566 skb_queue_head_init(&sd->input_pkt_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07008567 skb_queue_head_init(&sd->process_queue);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00008568 INIT_LIST_HEAD(&sd->poll_list);
Changli Gaoa9cbd582010-04-26 23:06:24 +00008569 sd->output_queue_tailp = &sd->output_queue;
Eric Dumazetdf334542010-03-24 19:13:54 +00008570#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +00008571 sd->csd.func = rps_trigger_softirq;
8572 sd->csd.info = sd;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00008573 sd->cpu = i;
Tom Herbert1e94d722010-03-18 17:45:44 -07008574#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00008575
Eric Dumazete36fa2f2010-04-19 21:17:14 +00008576 sd->backlog.poll = process_backlog;
8577 sd->backlog.weight = weight_p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008578 }
8579
Linus Torvalds1da177e2005-04-16 15:20:36 -07008580 dev_boot_phase = 0;
8581
Eric W. Biederman505d4f72008-11-07 22:54:20 -08008582 /* The loopback device is special if any other network devices
8583 * is present in a network namespace the loopback device must
8584 * be present. Since we now dynamically allocate and free the
8585 * loopback device ensure this invariant is maintained by
8586 * keeping the loopback device as the first device on the
8587 * list of network devices. Ensuring the loopback devices
8588 * is the first device that appears and the last network device
8589 * that disappears.
8590 */
8591 if (register_pernet_device(&loopback_net_ops))
8592 goto out;
8593
8594 if (register_pernet_device(&default_device_ops))
8595 goto out;
8596
Carlos R. Mafra962cf362008-05-15 11:15:37 -03008597 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
8598 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008599
Sebastian Andrzej Siewiorf0bf90d2016-11-03 15:50:04 +01008600 rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead",
8601 NULL, dev_cpu_dead);
8602 WARN_ON(rc < 0);
Thomas Graff38a9eb2015-07-21 10:43:56 +02008603 dst_subsys_init();
Linus Torvalds1da177e2005-04-16 15:20:36 -07008604 rc = 0;
8605out:
8606 return rc;
8607}
8608
8609subsys_initcall(net_dev_init);