blob: 7882e3aa6448222af4def4abd985fe4a86d8237c [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001/* SPDX-License-Identifier: GPL-2.0 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002#ifndef __NET_SCHED_GENERIC_H
3#define __NET_SCHED_GENERIC_H
4
Linus Torvalds1da177e2005-04-16 15:20:36 -07005#include <linux/netdevice.h>
6#include <linux/types.h>
7#include <linux/rcupdate.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07008#include <linux/pkt_sched.h>
9#include <linux/pkt_cls.h>
John Fastabend22e0f8b2014-09-28 11:52:56 -070010#include <linux/percpu.h>
Jesper Dangaard Brouer5772e9a2014-10-01 22:35:59 +020011#include <linux/dynamic_queue_limits.h>
Jiri Pirko5bc17012017-05-17 11:08:01 +020012#include <linux/list.h>
Reshetova, Elena7b936402017-07-04 15:53:07 +030013#include <linux/refcount.h>
Cong Wang7aa00452017-10-26 18:24:28 -070014#include <linux/workqueue.h>
Vlad Buslovc266f642019-02-11 10:55:32 +020015#include <linux/mutex.h>
Vlad Buslov4f8116c2019-08-26 16:44:57 +030016#include <linux/rwsem.h>
Vlad Buslov97394be2019-08-26 16:44:58 +030017#include <linux/atomic.h>
John Hurley59eb87c2019-11-02 14:17:47 +000018#include <linux/hashtable.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <net/gen_stats.h>
Thomas Grafbe577dd2007-03-22 11:55:50 -070020#include <net/rtnetlink.h>
Pablo Neira Ayusoa7323312019-07-19 18:20:15 +020021#include <net/flow_offload.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022
23struct Qdisc_ops;
24struct qdisc_walker;
25struct tcf_walker;
26struct module;
Petar Penkovd58e4682018-09-14 07:46:18 -070027struct bpf_flow_keys;
Linus Torvalds1da177e2005-04-16 15:20:36 -070028
Eric Dumazetfd2c3ef2009-11-03 03:26:03 +000029struct qdisc_rate_table {
Linus Torvalds1da177e2005-04-16 15:20:36 -070030 struct tc_ratespec rate;
31 u32 data[256];
32 struct qdisc_rate_table *next;
33 int refcnt;
34};
35
Eric Dumazetfd2c3ef2009-11-03 03:26:03 +000036enum qdisc_state_t {
David S. Miller37437bb2008-07-16 02:15:04 -070037 __QDISC_STATE_SCHED,
David S. Millera9312ae2008-08-17 21:51:03 -070038 __QDISC_STATE_DEACTIVATED,
Yunsheng Lina90c57f2021-05-14 11:16:59 +080039 __QDISC_STATE_MISSED,
Yunsheng Linc4fef012021-06-22 14:49:56 +080040 __QDISC_STATE_DRAINING,
David S. Millere2627c82008-07-16 00:56:32 -070041};
42
Yunsheng Linc4fef012021-06-22 14:49:56 +080043#define QDISC_STATE_MISSED BIT(__QDISC_STATE_MISSED)
44#define QDISC_STATE_DRAINING BIT(__QDISC_STATE_DRAINING)
45
46#define QDISC_STATE_NON_EMPTY (QDISC_STATE_MISSED | \
47 QDISC_STATE_DRAINING)
48
Jussi Kivilinna175f9c12008-07-20 00:08:47 -070049struct qdisc_size_table {
Eric Dumazeta2da5702011-01-20 03:48:19 +000050 struct rcu_head rcu;
Jussi Kivilinna175f9c12008-07-20 00:08:47 -070051 struct list_head list;
52 struct tc_sizespec szopts;
53 int refcnt;
54 u16 data[];
55};
56
Florian Westphal48da34b2016-09-18 00:57:34 +020057/* similar to sk_buff_head, but skb->prev pointer is undefined. */
58struct qdisc_skb_head {
59 struct sk_buff *head;
60 struct sk_buff *tail;
Paolo Abeni73eb6282019-04-10 14:32:41 +020061 __u32 qlen;
Florian Westphal48da34b2016-09-18 00:57:34 +020062 spinlock_t lock;
63};
64
Eric Dumazetfd2c3ef2009-11-03 03:26:03 +000065struct Qdisc {
Eric Dumazet520ac302016-06-21 23:16:49 -070066 int (*enqueue)(struct sk_buff *skb,
67 struct Qdisc *sch,
68 struct sk_buff **to_free);
69 struct sk_buff * (*dequeue)(struct Qdisc *sch);
Eric Dumazet05bdd2f2011-10-20 17:45:43 -040070 unsigned int flags;
Jarek Poplawskib00355d2009-02-01 01:12:42 -080071#define TCQ_F_BUILTIN 1
Eric Dumazetfd245a42011-01-20 05:27:16 +000072#define TCQ_F_INGRESS 2
73#define TCQ_F_CAN_BYPASS 4
74#define TCQ_F_MQROOT 8
Eric Dumazet1abbe132012-12-11 15:54:33 +000075#define TCQ_F_ONETXQUEUE 0x10 /* dequeue_skb() can assume all skbs are for
76 * q->dev_queue : It can test
77 * netif_xmit_frozen_or_stopped() before
78 * dequeueing next packet.
79 * Its true for MQ/MQPRIO slaves, or non
80 * multiqueue device.
81 */
Jarek Poplawskib00355d2009-02-01 01:12:42 -080082#define TCQ_F_WARN_NONWC (1 << 16)
John Fastabend22e0f8b2014-09-28 11:52:56 -070083#define TCQ_F_CPUSTATS 0x20 /* run using percpu statistics */
Eric Dumazet4eaf3b82015-12-01 20:08:51 -080084#define TCQ_F_NOPARENT 0x40 /* root of its hierarchy :
85 * qdisc_tree_decrease_qlen() should stop.
86 */
Jiri Kosina49b49972017-03-08 16:03:32 +010087#define TCQ_F_INVISIBLE 0x80 /* invisible by default in dump */
John Fastabend6b3ba912017-12-07 09:54:25 -080088#define TCQ_F_NOLOCK 0x100 /* qdisc does not require locking */
Yuval Mintz7a4fa292017-12-14 15:54:29 +020089#define TCQ_F_OFFLOADED 0x200 /* qdisc is offloaded to HW */
Eric Dumazet45203a32013-06-06 08:43:22 -070090 u32 limit;
Eric Dumazet05bdd2f2011-10-20 17:45:43 -040091 const struct Qdisc_ops *ops;
Eric Dumazeta2da5702011-01-20 03:48:19 +000092 struct qdisc_size_table __rcu *stab;
Jiri Kosina59cc1f62016-08-10 11:05:15 +020093 struct hlist_node hash;
Linus Torvalds1da177e2005-04-16 15:20:36 -070094 u32 handle;
95 u32 parent;
David S. Miller72b25a92008-07-18 20:54:17 -070096
Eric Dumazet5e140df2009-03-20 01:33:32 -070097 struct netdev_queue *dev_queue;
Eric Dumazet5e140df2009-03-20 01:33:32 -070098
Eric Dumazet1c0d32f2016-12-04 09:48:16 -080099 struct net_rate_estimator __rcu *rate_est;
Ahmed S. Darwish50dc9a82021-10-16 10:49:09 +0200100 struct gnet_stats_basic_sync __percpu *cpu_bstats;
Eric Dumazet0d32ef82015-01-29 17:30:12 -0800101 struct gnet_stats_queue __percpu *cpu_qstats;
Eric Dumazet846e4632020-10-07 09:51:11 -0700102 int pad;
Paolo Abenie9be0e92018-05-25 16:28:44 +0200103 refcount_t refcnt;
Eric Dumazet0d32ef82015-01-29 17:30:12 -0800104
Eric Dumazet5e140df2009-03-20 01:33:32 -0700105 /*
106 * For performance sake on SMP, we put highly modified fields at the end
107 */
John Fastabenda53851e2017-12-07 09:55:45 -0800108 struct sk_buff_head gso_skb ____cacheline_aligned_in_smp;
Florian Westphal48da34b2016-09-18 00:57:34 +0200109 struct qdisc_skb_head q;
Ahmed S. Darwish50dc9a82021-10-16 10:49:09 +0200110 struct gnet_stats_basic_sync bstats;
Eric Dumazetf9eb8ae2016-06-06 09:37:15 -0700111 seqcount_t running;
Eric Dumazet0d32ef82015-01-29 17:30:12 -0800112 struct gnet_stats_queue qstats;
Eric Dumazet4d202a02016-06-21 23:16:52 -0700113 unsigned long state;
114 struct Qdisc *next_sched;
John Fastabend70e57d52017-12-07 09:56:23 -0800115 struct sk_buff_head skb_bad_txq;
Eric Dumazet45203a32013-06-06 08:43:22 -0700116
117 spinlock_t busylock ____cacheline_aligned_in_smp;
Paolo Abeni96009c72018-05-15 16:24:36 +0200118 spinlock_t seqlock;
Paolo Abeni28cff532019-03-22 16:01:55 +0100119
Vlad Buslov3a7d0d02018-09-24 19:22:51 +0300120 struct rcu_head rcu;
Eric Dumazet846e4632020-10-07 09:51:11 -0700121
122 /* private data */
123 long privdata[] ____cacheline_aligned;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700124};
125
Eric Dumazet551143d2017-08-24 21:12:28 -0700126static inline void qdisc_refcount_inc(struct Qdisc *qdisc)
127{
128 if (qdisc->flags & TCQ_F_BUILTIN)
129 return;
130 refcount_inc(&qdisc->refcnt);
131}
132
Vlad Buslov9d7e82c2018-09-24 19:22:52 +0300133/* Intended to be used by unlocked users, when concurrent qdisc release is
134 * possible.
135 */
136
137static inline struct Qdisc *qdisc_refcount_inc_nz(struct Qdisc *qdisc)
138{
139 if (qdisc->flags & TCQ_F_BUILTIN)
140 return qdisc;
141 if (refcount_inc_not_zero(&qdisc->refcnt))
142 return qdisc;
143 return NULL;
144}
145
Paolo Abeni96009c72018-05-15 16:24:36 +0200146static inline bool qdisc_is_running(struct Qdisc *qdisc)
Eric Dumazetbc135b22010-06-02 03:23:51 -0700147{
Paolo Abeni32f7b442018-05-15 10:50:31 +0200148 if (qdisc->flags & TCQ_F_NOLOCK)
Paolo Abeni96009c72018-05-15 16:24:36 +0200149 return spin_is_locked(&qdisc->seqlock);
Eric Dumazetf9eb8ae2016-06-06 09:37:15 -0700150 return (raw_read_seqcount(&qdisc->running) & 1) ? true : false;
Eric Dumazetbc135b22010-06-02 03:23:51 -0700151}
152
Yunsheng Linc4fef012021-06-22 14:49:56 +0800153static inline bool nolock_qdisc_is_empty(const struct Qdisc *qdisc)
154{
155 return !(READ_ONCE(qdisc->state) & QDISC_STATE_NON_EMPTY);
156}
157
Paolo Abeni9c01c9f2019-04-10 14:32:39 +0200158static inline bool qdisc_is_percpu_stats(const struct Qdisc *q)
159{
160 return q->flags & TCQ_F_CPUSTATS;
161}
162
Paolo Abeni28cff532019-03-22 16:01:55 +0100163static inline bool qdisc_is_empty(const struct Qdisc *qdisc)
164{
Paolo Abeni9c01c9f2019-04-10 14:32:39 +0200165 if (qdisc_is_percpu_stats(qdisc))
Yunsheng Lind3e0f572021-06-22 14:49:57 +0800166 return nolock_qdisc_is_empty(qdisc);
Eric Dumazet90b2be22019-11-08 08:45:23 -0800167 return !READ_ONCE(qdisc->q.qlen);
Paolo Abeni28cff532019-03-22 16:01:55 +0100168}
169
Eric Dumazetbc135b22010-06-02 03:23:51 -0700170static inline bool qdisc_run_begin(struct Qdisc *qdisc)
171{
Paolo Abeni32f7b442018-05-15 10:50:31 +0200172 if (qdisc->flags & TCQ_F_NOLOCK) {
Yunsheng Lina90c57f2021-05-14 11:16:59 +0800173 if (spin_trylock(&qdisc->seqlock))
Yunsheng Lind3e0f572021-06-22 14:49:57 +0800174 return true;
Yunsheng Lina90c57f2021-05-14 11:16:59 +0800175
Yunsheng Lin89837eb2021-06-17 09:04:14 +0800176 /* Paired with smp_mb__after_atomic() to make sure
177 * STATE_MISSED checking is synchronized with clearing
178 * in pfifo_fast_dequeue().
179 */
180 smp_mb__before_atomic();
181
Yunsheng Lina90c57f2021-05-14 11:16:59 +0800182 /* If the MISSED flag is set, it means other thread has
183 * set the MISSED flag before second spin_trylock(), so
184 * we can return false here to avoid multi cpus doing
185 * the set_bit() and second spin_trylock() concurrently.
186 */
187 if (test_bit(__QDISC_STATE_MISSED, &qdisc->state))
188 return false;
189
190 /* Set the MISSED flag before the second spin_trylock(),
191 * if the second spin_trylock() return false, it means
192 * other cpu holding the lock will do dequeuing for us
193 * or it will see the MISSED flag set after releasing
194 * lock and reschedule the net_tx_action() to do the
195 * dequeuing.
196 */
197 set_bit(__QDISC_STATE_MISSED, &qdisc->state);
198
Yunsheng Lin89837eb2021-06-17 09:04:14 +0800199 /* spin_trylock() only has load-acquire semantic, so use
200 * smp_mb__after_atomic() to ensure STATE_MISSED is set
201 * before doing the second spin_trylock().
202 */
203 smp_mb__after_atomic();
204
Yunsheng Lina90c57f2021-05-14 11:16:59 +0800205 /* Retry again in case other CPU may not see the new flag
206 * after it releases the lock at the end of qdisc_run_end().
207 */
Yunsheng Lind3e0f572021-06-22 14:49:57 +0800208 return spin_trylock(&qdisc->seqlock);
Paolo Abeni32f7b442018-05-15 10:50:31 +0200209 } else if (qdisc_is_running(qdisc)) {
Eric Dumazetfd245a42011-01-20 05:27:16 +0000210 return false;
Paolo Abeni32f7b442018-05-15 10:50:31 +0200211 }
Eric Dumazet52fbb292016-06-09 07:45:11 -0700212 /* Variant of write_seqcount_begin() telling lockdep a trylock
213 * was attempted.
214 */
215 raw_write_seqcount_begin(&qdisc->running);
216 seqcount_acquire(&qdisc->running.dep_map, 0, 1, _RET_IP_);
Eric Dumazetfd245a42011-01-20 05:27:16 +0000217 return true;
Eric Dumazetbc135b22010-06-02 03:23:51 -0700218}
219
220static inline void qdisc_run_end(struct Qdisc *qdisc)
221{
Yunsheng Lina90c57f2021-05-14 11:16:59 +0800222 if (qdisc->flags & TCQ_F_NOLOCK) {
Paolo Abeni96009c72018-05-15 16:24:36 +0200223 spin_unlock(&qdisc->seqlock);
Yunsheng Lina90c57f2021-05-14 11:16:59 +0800224
225 if (unlikely(test_bit(__QDISC_STATE_MISSED,
Yunsheng Linc4fef012021-06-22 14:49:56 +0800226 &qdisc->state)))
Yunsheng Lina90c57f2021-05-14 11:16:59 +0800227 __netif_schedule(qdisc);
Yunsheng Lindd252962021-06-22 14:49:55 +0800228 } else {
229 write_seqcount_end(&qdisc->running);
Yunsheng Lina90c57f2021-05-14 11:16:59 +0800230 }
Eric Dumazetfd245a42011-01-20 05:27:16 +0000231}
232
Jesper Dangaard Brouer5772e9a2014-10-01 22:35:59 +0200233static inline bool qdisc_may_bulk(const struct Qdisc *qdisc)
234{
235 return qdisc->flags & TCQ_F_ONETXQUEUE;
236}
237
238static inline int qdisc_avail_bulklimit(const struct netdev_queue *txq)
239{
240#ifdef CONFIG_BQL
241 /* Non-BQL migrated drivers will return 0, too. */
242 return dql_avail(&txq->dql);
243#else
244 return 0;
245#endif
246}
247
Eric Dumazetfd2c3ef2009-11-03 03:26:03 +0000248struct Qdisc_class_ops {
Vlad Buslovdfcd2a22019-02-11 10:55:46 +0200249 unsigned int flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250 /* Child qdisc manipulation */
Jarek Poplawski926e61b2009-09-15 02:53:07 -0700251 struct netdev_queue * (*select_queue)(struct Qdisc *, struct tcmsg *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700252 int (*graft)(struct Qdisc *, unsigned long cl,
Alexander Aring653d6fd2017-12-20 12:35:17 -0500253 struct Qdisc *, struct Qdisc **,
254 struct netlink_ext_ack *extack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255 struct Qdisc * (*leaf)(struct Qdisc *, unsigned long cl);
Patrick McHardy43effa12006-11-29 17:35:48 -0800256 void (*qlen_notify)(struct Qdisc *, unsigned long);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700257
258 /* Class manipulation routines */
WANG Cong143976c2017-08-24 16:51:29 -0700259 unsigned long (*find)(struct Qdisc *, u32 classid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700260 int (*change)(struct Qdisc *, u32, u32,
Alexander Aring793d81d2017-12-20 12:35:15 -0500261 struct nlattr **, unsigned long *,
262 struct netlink_ext_ack *);
Maxim Mikityanskiy4dd78a72021-01-19 14:08:12 +0200263 int (*delete)(struct Qdisc *, unsigned long,
264 struct netlink_ext_ack *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700265 void (*walk)(struct Qdisc *, struct qdisc_walker * arg);
266
267 /* Filter manipulation */
Alexander Aring0ac4bd62017-12-04 18:39:59 -0500268 struct tcf_block * (*tcf_block)(struct Qdisc *sch,
Alexander Aringcbaacc42017-12-20 12:35:16 -0500269 unsigned long arg,
270 struct netlink_ext_ack *extack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271 unsigned long (*bind_tcf)(struct Qdisc *, unsigned long,
272 u32 classid);
273 void (*unbind_tcf)(struct Qdisc *, unsigned long);
274
275 /* rtnetlink specific */
276 int (*dump)(struct Qdisc *, unsigned long,
277 struct sk_buff *skb, struct tcmsg*);
278 int (*dump_stats)(struct Qdisc *, unsigned long,
279 struct gnet_dump *);
280};
281
Vlad Buslovdfcd2a22019-02-11 10:55:46 +0200282/* Qdisc_class_ops flag values */
283
284/* Implements API that doesn't require rtnl lock */
285enum qdisc_class_ops_flags {
286 QDISC_CLASS_OPS_DOIT_UNLOCKED = 1,
287};
288
Eric Dumazetfd2c3ef2009-11-03 03:26:03 +0000289struct Qdisc_ops {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700290 struct Qdisc_ops *next;
Eric Dumazet20fea082007-11-14 01:44:41 -0800291 const struct Qdisc_class_ops *cl_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292 char id[IFNAMSIZ];
293 int priv_size;
John Fastabendd59f5ff2017-12-07 09:55:26 -0800294 unsigned int static_flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700295
Eric Dumazet520ac302016-06-21 23:16:49 -0700296 int (*enqueue)(struct sk_buff *skb,
297 struct Qdisc *sch,
298 struct sk_buff **to_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299 struct sk_buff * (*dequeue)(struct Qdisc *);
Jarek Poplawski90d841fd2008-10-31 00:43:45 -0700300 struct sk_buff * (*peek)(struct Qdisc *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301
Alexander Aringe63d7df2017-12-20 12:35:13 -0500302 int (*init)(struct Qdisc *sch, struct nlattr *arg,
303 struct netlink_ext_ack *extack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700304 void (*reset)(struct Qdisc *);
305 void (*destroy)(struct Qdisc *);
Alexander Aring0ac4bd62017-12-04 18:39:59 -0500306 int (*change)(struct Qdisc *sch,
Alexander Aring20307212017-12-20 12:35:14 -0500307 struct nlattr *arg,
308 struct netlink_ext_ack *extack);
Alexander Aring0ac4bd62017-12-04 18:39:59 -0500309 void (*attach)(struct Qdisc *sch);
Cong Wang48bfd552018-01-25 18:26:23 -0800310 int (*change_tx_queue_len)(struct Qdisc *, unsigned int);
Jakub Kicinski1e080f12021-09-13 15:53:30 -0700311 void (*change_real_num_tx)(struct Qdisc *sch,
312 unsigned int new_real_tx);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700313
314 int (*dump)(struct Qdisc *, struct sk_buff *);
315 int (*dump_stats)(struct Qdisc *, struct gnet_dump *);
316
Jiri Pirkod47a6b02018-01-17 11:46:52 +0100317 void (*ingress_block_set)(struct Qdisc *sch,
318 u32 block_index);
319 void (*egress_block_set)(struct Qdisc *sch,
320 u32 block_index);
321 u32 (*ingress_block_get)(struct Qdisc *sch);
322 u32 (*egress_block_get)(struct Qdisc *sch);
323
Linus Torvalds1da177e2005-04-16 15:20:36 -0700324 struct module *owner;
325};
326
327
Eric Dumazetfd2c3ef2009-11-03 03:26:03 +0000328struct tcf_result {
Jiri Pirkodb505142017-05-17 11:08:03 +0200329 union {
330 struct {
331 unsigned long class;
332 u32 classid;
333 };
334 const struct tcf_proto *goto_tp;
Paolo Abenicd11b1642018-07-30 14:30:44 +0200335
John Hurley720f22f2019-06-24 23:13:35 +0100336 /* used in the skb_tc_reinsert function */
Paolo Abenicd11b1642018-07-30 14:30:44 +0200337 struct {
338 bool ingress;
339 struct gnet_stats_queue *qstats;
340 };
Jiri Pirkodb505142017-05-17 11:08:03 +0200341 };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700342};
343
Jiri Pirko9f407f12018-07-23 09:23:07 +0200344struct tcf_chain;
345
Eric Dumazetfd2c3ef2009-11-03 03:26:03 +0000346struct tcf_proto_ops {
WANG Cong36272872013-12-15 20:15:11 -0800347 struct list_head head;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348 char kind[IFNAMSIZ];
349
Eric Dumazetdc7f9f62011-07-05 23:25:42 +0000350 int (*classify)(struct sk_buff *,
351 const struct tcf_proto *,
352 struct tcf_result *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353 int (*init)(struct tcf_proto*);
Vlad Buslov12db03b2019-02-11 10:55:45 +0200354 void (*destroy)(struct tcf_proto *tp, bool rtnl_held,
Jakub Kicinski715df5e2018-01-24 12:54:13 -0800355 struct netlink_ext_ack *extack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700356
WANG Cong8113c092017-08-04 21:31:43 -0700357 void* (*get)(struct tcf_proto*, u32 handle);
Vlad Buslov7d5509f2019-02-11 10:55:44 +0200358 void (*put)(struct tcf_proto *tp, void *f);
Benjamin LaHaisec1b52732013-01-14 05:15:39 +0000359 int (*change)(struct net *net, struct sk_buff *,
Eric W. Biedermanaf4c6642012-05-25 13:42:45 -0600360 struct tcf_proto*, unsigned long,
Patrick McHardyadd93b62008-01-22 22:11:33 -0800361 u32 handle, struct nlattr **,
Cong Wang695176b2021-07-29 16:12:14 -0700362 void **, u32,
Alexander Aring7306db32018-01-18 11:20:51 -0500363 struct netlink_ext_ack *);
Alexander Aring8865fdd2018-01-18 11:20:49 -0500364 int (*delete)(struct tcf_proto *tp, void *arg,
Vlad Buslov12db03b2019-02-11 10:55:45 +0200365 bool *last, bool rtnl_held,
Alexander Aring571acf22018-01-18 11:20:53 -0500366 struct netlink_ext_ack *);
Davide Carattia5b72a02019-12-28 16:36:58 +0100367 bool (*delete_empty)(struct tcf_proto *tp);
Vlad Buslov12db03b2019-02-11 10:55:45 +0200368 void (*walk)(struct tcf_proto *tp,
369 struct tcf_walker *arg, bool rtnl_held);
John Hurleye56185c2018-06-25 14:30:05 -0700370 int (*reoffload)(struct tcf_proto *tp, bool add,
Pablo Neira Ayusoa7323312019-07-19 18:20:15 +0200371 flow_setup_cb_t *cb, void *cb_priv,
John Hurleye56185c2018-06-25 14:30:05 -0700372 struct netlink_ext_ack *extack);
Vlad Buslova449a3e2019-08-26 16:45:00 +0300373 void (*hw_add)(struct tcf_proto *tp,
374 void *type_data);
375 void (*hw_del)(struct tcf_proto *tp,
376 void *type_data);
Cong Wang2e24cd72020-01-23 16:26:18 -0800377 void (*bind_class)(void *, u32, unsigned long,
378 void *, unsigned long);
Jiri Pirko9f407f12018-07-23 09:23:07 +0200379 void * (*tmplt_create)(struct net *net,
380 struct tcf_chain *chain,
381 struct nlattr **tca,
382 struct netlink_ext_ack *extack);
383 void (*tmplt_destroy)(void *tmplt_priv);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384
385 /* rtnetlink specific */
WANG Cong8113c092017-08-04 21:31:43 -0700386 int (*dump)(struct net*, struct tcf_proto*, void *,
Vlad Buslov12db03b2019-02-11 10:55:45 +0200387 struct sk_buff *skb, struct tcmsg*,
388 bool);
Vlad Buslovf8ab1802020-05-15 14:40:11 +0300389 int (*terse_dump)(struct net *net,
390 struct tcf_proto *tp, void *fh,
391 struct sk_buff *skb,
392 struct tcmsg *t, bool rtnl_held);
Jiri Pirko9f407f12018-07-23 09:23:07 +0200393 int (*tmplt_dump)(struct sk_buff *skb,
394 struct net *net,
395 void *tmplt_priv);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396
397 struct module *owner;
Vlad Buslov12db03b2019-02-11 10:55:45 +0200398 int flags;
399};
400
Davide Carattia5b72a02019-12-28 16:36:58 +0100401/* Classifiers setting TCF_PROTO_OPS_DOIT_UNLOCKED in tcf_proto_ops->flags
402 * are expected to implement tcf_proto_ops->delete_empty(), otherwise race
403 * conditions can occur when filters are inserted/deleted simultaneously.
404 */
Vlad Buslov12db03b2019-02-11 10:55:45 +0200405enum tcf_proto_ops_flags {
406 TCF_PROTO_OPS_DOIT_UNLOCKED = 1,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407};
408
Eric Dumazetfd2c3ef2009-11-03 03:26:03 +0000409struct tcf_proto {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700410 /* Fast access part */
John Fastabend25d8c0d2014-09-12 20:05:27 -0700411 struct tcf_proto __rcu *next;
412 void __rcu *root;
Paolo Abeni7fd4b282018-07-30 14:30:43 +0200413
414 /* called under RCU BH lock*/
Eric Dumazetdc7f9f62011-07-05 23:25:42 +0000415 int (*classify)(struct sk_buff *,
416 const struct tcf_proto *,
417 struct tcf_result *);
Al Viro66c6f522006-11-20 18:07:51 -0800418 __be16 protocol;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700419
420 /* All the rest */
421 u32 prio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422 void *data;
Eric Dumazetdc7f9f62011-07-05 23:25:42 +0000423 const struct tcf_proto_ops *ops;
Jiri Pirko5bc17012017-05-17 11:08:01 +0200424 struct tcf_chain *chain;
Vlad Buslov8b646782019-02-11 10:55:41 +0200425 /* Lock protects tcf_proto shared state and can be used by unlocked
426 * classifiers to protect their private data.
427 */
428 spinlock_t lock;
429 bool deleting;
Vlad Buslov4dbfa762019-02-11 10:55:39 +0200430 refcount_t refcnt;
John Fastabend25d8c0d2014-09-12 20:05:27 -0700431 struct rcu_head rcu;
John Hurley59eb87c2019-11-02 14:17:47 +0000432 struct hlist_node destroy_ht_node;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700433};
434
Jussi Kivilinna175f9c12008-07-20 00:08:47 -0700435struct qdisc_skb_cb {
Stanislav Fomichev089b19a2019-04-22 08:55:44 -0700436 struct {
437 unsigned int pkt_len;
438 u16 slave_dev_queue_mapping;
439 u16 tc_classid;
Petar Penkovd58e4682018-09-14 07:46:18 -0700440 };
Eric Dumazet25711782014-09-18 08:02:05 -0700441#define QDISC_CB_PRIV_LEN 20
442 unsigned char data[QDISC_CB_PRIV_LEN];
wenxu038ebb12020-07-31 10:45:01 +0800443 u16 mru;
wenxu7baf2422021-01-19 16:31:50 +0800444 bool post_ct;
Jussi Kivilinna175f9c12008-07-20 00:08:47 -0700445};
446
Jiri Pirkoc7eb7d72017-11-03 11:46:24 +0100447typedef void tcf_chain_head_change_t(struct tcf_proto *tp_head, void *priv);
448
Jiri Pirko2190d1d2017-05-17 11:07:59 +0200449struct tcf_chain {
Vlad Busloved76f5e2019-02-11 10:55:38 +0200450 /* Protects filter_chain. */
451 struct mutex filter_chain_lock;
Jiri Pirko2190d1d2017-05-17 11:07:59 +0200452 struct tcf_proto __rcu *filter_chain;
Jiri Pirko5bc17012017-05-17 11:08:01 +0200453 struct list_head list;
454 struct tcf_block *block;
455 u32 index; /* chain index */
456 unsigned int refcnt;
Jiri Pirko1f3ed382018-07-27 09:45:05 +0200457 unsigned int action_refcnt;
Jiri Pirko32a4f5e2018-07-23 09:23:06 +0200458 bool explicitly_created;
Vlad Buslov726d06122019-02-11 10:55:42 +0200459 bool flushing;
Jiri Pirko9f407f12018-07-23 09:23:07 +0200460 const struct tcf_proto_ops *tmplt_ops;
461 void *tmplt_priv;
Davide Carattiee3bbfe2019-03-20 15:00:16 +0100462 struct rcu_head rcu;
Jiri Pirko6529eab2017-05-17 11:07:55 +0200463};
464
Jiri Pirko2190d1d2017-05-17 11:07:59 +0200465struct tcf_block {
Vlad Buslovc266f642019-02-11 10:55:32 +0200466 /* Lock protects tcf_block and lifetime-management data of chains
467 * attached to the block (refcnt, action_refcnt, explicitly_created).
468 */
469 struct mutex lock;
Jiri Pirko5bc17012017-05-17 11:08:01 +0200470 struct list_head chain_list;
Jiri Pirko48617382018-01-17 11:46:46 +0100471 u32 index; /* block index for shared blocks */
Cong Wanga7df4872020-04-30 20:53:49 -0700472 u32 classid; /* which class this block belongs to */
Vlad Buslovcfebd7e2018-09-24 19:22:54 +0300473 refcount_t refcnt;
Jiri Pirko855319b2017-10-13 14:00:58 +0200474 struct net *net;
Jiri Pirko69d78ef2017-10-13 14:00:57 +0200475 struct Qdisc *q;
Vlad Buslov4f8116c2019-08-26 16:44:57 +0300476 struct rw_semaphore cb_lock; /* protects cb_list and offload counters */
Pablo Neira Ayuso14bfb132019-07-19 18:20:16 +0200477 struct flow_block flow_block;
Jiri Pirkof36fe1c2018-01-17 11:46:48 +0100478 struct list_head owner_list;
479 bool keep_dst;
Vlad Buslov97394be2019-08-26 16:44:58 +0300480 atomic_t offloadcnt; /* Number of oddloaded filters */
Jiri Pirkocaa72602018-01-17 11:46:50 +0100481 unsigned int nooffloaddevcnt; /* Number of devs unable to do offload */
Vlad Buslovc9f14472019-08-26 16:45:01 +0300482 unsigned int lockeddevcnt; /* Number of devs that require rtnl lock. */
Jiri Pirkof71e0ca42018-07-23 09:23:05 +0200483 struct {
484 struct tcf_chain *chain;
485 struct list_head filter_chain_list;
486 } chain0;
Vlad Buslov0607e432018-09-24 19:22:57 +0300487 struct rcu_head rcu;
John Hurley59eb87c2019-11-02 14:17:47 +0000488 DECLARE_HASHTABLE(proto_destroy_ht, 7);
489 struct mutex proto_destroy_lock; /* Lock for proto_destroy hashtable. */
Jiri Pirko2190d1d2017-05-17 11:07:59 +0200490};
491
Vlad Busloved76f5e2019-02-11 10:55:38 +0200492static inline bool lockdep_tcf_chain_is_locked(struct tcf_chain *chain)
493{
494 return lockdep_is_held(&chain->filter_chain_lock);
495}
Vlad Buslov8b646782019-02-11 10:55:41 +0200496
497static inline bool lockdep_tcf_proto_is_locked(struct tcf_proto *tp)
498{
499 return lockdep_is_held(&tp->lock);
500}
Vlad Busloved76f5e2019-02-11 10:55:38 +0200501
502#define tcf_chain_dereference(p, chain) \
503 rcu_dereference_protected(p, lockdep_tcf_chain_is_locked(chain))
504
Vlad Buslov8b646782019-02-11 10:55:41 +0200505#define tcf_proto_dereference(p, tp) \
506 rcu_dereference_protected(p, lockdep_tcf_proto_is_locked(tp))
507
David S. Miller16bda132012-02-06 15:14:37 -0500508static inline void qdisc_cb_private_validate(const struct sk_buff *skb, int sz)
509{
510 struct qdisc_skb_cb *qcb;
Eric Dumazet5ee31c682012-06-12 06:03:51 +0000511
wenxu038ebb12020-07-31 10:45:01 +0800512 BUILD_BUG_ON(sizeof(skb->cb) < sizeof(*qcb));
David S. Miller16bda132012-02-06 15:14:37 -0500513 BUILD_BUG_ON(sizeof(qcb->data) < sz);
514}
515
Paolo Abeni73eb6282019-04-10 14:32:41 +0200516static inline int qdisc_qlen_cpu(const struct Qdisc *q)
517{
518 return this_cpu_ptr(q->cpu_qstats)->qlen;
519}
520
Eric Dumazet05bdd2f2011-10-20 17:45:43 -0400521static inline int qdisc_qlen(const struct Qdisc *q)
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +0000522{
523 return q->q.qlen;
524}
525
Paolo Abeni73eb6282019-04-10 14:32:41 +0200526static inline int qdisc_qlen_sum(const struct Qdisc *q)
John Fastabend7e660162017-12-07 09:57:00 -0800527{
Paolo Abeni73eb6282019-04-10 14:32:41 +0200528 __u32 qlen = q->qstats.qlen;
529 int i;
John Fastabend7e660162017-12-07 09:57:00 -0800530
Paolo Abeni73eb6282019-04-10 14:32:41 +0200531 if (qdisc_is_percpu_stats(q)) {
532 for_each_possible_cpu(i)
533 qlen += per_cpu_ptr(q->cpu_qstats, i)->qlen;
534 } else {
Jakub Kicinski6172abc2018-05-25 21:53:30 -0700535 qlen += q->q.qlen;
Paolo Abeni73eb6282019-04-10 14:32:41 +0200536 }
John Fastabend7e660162017-12-07 09:57:00 -0800537
538 return qlen;
539}
540
Eric Dumazetbfe0d022011-01-09 08:30:54 +0000541static inline struct qdisc_skb_cb *qdisc_skb_cb(const struct sk_buff *skb)
Jussi Kivilinna175f9c12008-07-20 00:08:47 -0700542{
543 return (struct qdisc_skb_cb *)skb->cb;
544}
545
David S. Miller838740002008-07-17 00:53:03 -0700546static inline spinlock_t *qdisc_lock(struct Qdisc *qdisc)
547{
548 return &qdisc->q.lock;
549}
550
Eric Dumazet05bdd2f2011-10-20 17:45:43 -0400551static inline struct Qdisc *qdisc_root(const struct Qdisc *qdisc)
David S. Miller7698b4f2008-07-16 01:42:40 -0700552{
John Fastabend46e5da40a2014-09-12 20:04:52 -0700553 struct Qdisc *q = rcu_dereference_rtnl(qdisc->dev_queue->qdisc);
554
555 return q;
David S. Miller7698b4f2008-07-16 01:42:40 -0700556}
557
Eric Dumazet159d2c72019-09-24 13:11:26 -0700558static inline struct Qdisc *qdisc_root_bh(const struct Qdisc *qdisc)
559{
560 return rcu_dereference_bh(qdisc->dev_queue->qdisc);
561}
562
Eric Dumazet05bdd2f2011-10-20 17:45:43 -0400563static inline struct Qdisc *qdisc_root_sleeping(const struct Qdisc *qdisc)
Jarek Poplawski2540e052008-08-21 05:11:14 -0700564{
565 return qdisc->dev_queue->qdisc_sleeping;
566}
567
David S. Miller7e43f112008-08-02 23:27:37 -0700568/* The qdisc root lock is a mechanism by which to top level
569 * of a qdisc tree can be locked from any qdisc node in the
570 * forest. This allows changing the configuration of some
571 * aspect of the qdisc tree while blocking out asynchronous
572 * qdisc access in the packet processing paths.
573 *
574 * It is only legal to do this when the root will not change
575 * on us. Otherwise we'll potentially lock the wrong qdisc
576 * root. This is enforced by holding the RTNL semaphore, which
577 * all users of this lock accessor must do.
578 */
Eric Dumazet05bdd2f2011-10-20 17:45:43 -0400579static inline spinlock_t *qdisc_root_lock(const struct Qdisc *qdisc)
David S. Miller7698b4f2008-07-16 01:42:40 -0700580{
581 struct Qdisc *root = qdisc_root(qdisc);
582
David S. Miller7e43f112008-08-02 23:27:37 -0700583 ASSERT_RTNL();
David S. Miller838740002008-07-17 00:53:03 -0700584 return qdisc_lock(root);
David S. Miller7698b4f2008-07-16 01:42:40 -0700585}
586
Eric Dumazet05bdd2f2011-10-20 17:45:43 -0400587static inline spinlock_t *qdisc_root_sleeping_lock(const struct Qdisc *qdisc)
Jarek Poplawskif6f9b932008-08-27 02:25:17 -0700588{
589 struct Qdisc *root = qdisc_root_sleeping(qdisc);
590
591 ASSERT_RTNL();
592 return qdisc_lock(root);
593}
594
Eric Dumazetedb09eb2016-06-06 09:37:16 -0700595static inline seqcount_t *qdisc_root_sleeping_running(const struct Qdisc *qdisc)
596{
597 struct Qdisc *root = qdisc_root_sleeping(qdisc);
598
599 ASSERT_RTNL();
600 return &root->running;
601}
602
Eric Dumazet05bdd2f2011-10-20 17:45:43 -0400603static inline struct net_device *qdisc_dev(const struct Qdisc *qdisc)
David S. Miller5ce2d482008-07-08 17:06:30 -0700604{
605 return qdisc->dev_queue->dev;
606}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700607
Maxim Mikityanskiyca1e4ab2021-01-19 14:08:11 +0200608static inline void sch_tree_lock(struct Qdisc *q)
David S. Miller78a5b302008-07-16 03:12:24 -0700609{
Maxim Mikityanskiyca1e4ab2021-01-19 14:08:11 +0200610 if (q->flags & TCQ_F_MQROOT)
611 spin_lock_bh(qdisc_lock(q));
612 else
613 spin_lock_bh(qdisc_root_sleeping_lock(q));
David S. Miller78a5b302008-07-16 03:12:24 -0700614}
615
Maxim Mikityanskiyca1e4ab2021-01-19 14:08:11 +0200616static inline void sch_tree_unlock(struct Qdisc *q)
David S. Miller78a5b302008-07-16 03:12:24 -0700617{
Maxim Mikityanskiyca1e4ab2021-01-19 14:08:11 +0200618 if (q->flags & TCQ_F_MQROOT)
619 spin_unlock_bh(qdisc_lock(q));
620 else
621 spin_unlock_bh(qdisc_root_sleeping_lock(q));
David S. Miller78a5b302008-07-16 03:12:24 -0700622}
623
Thomas Grafe41a33e2005-07-05 14:14:30 -0700624extern struct Qdisc noop_qdisc;
625extern struct Qdisc_ops noop_qdisc_ops;
David S. Miller6ec1c692009-09-06 01:58:51 -0700626extern struct Qdisc_ops pfifo_fast_ops;
627extern struct Qdisc_ops mq_qdisc_ops;
Phil Sutterd66d6c32015-08-27 21:21:38 +0200628extern struct Qdisc_ops noqueue_qdisc_ops;
stephen hemminger6da7c8f2013-08-27 16:19:08 -0700629extern const struct Qdisc_ops *default_qdisc_ops;
Eric Dumazet1f27cde2016-03-02 08:21:43 -0800630static inline const struct Qdisc_ops *
631get_default_qdisc_ops(const struct net_device *dev, int ntx)
632{
633 return ntx < dev->real_num_tx_queues ?
634 default_qdisc_ops : &pfifo_fast_ops;
635}
Thomas Grafe41a33e2005-07-05 14:14:30 -0700636
Eric Dumazetfd2c3ef2009-11-03 03:26:03 +0000637struct Qdisc_class_common {
Patrick McHardy6fe1c7a2008-07-05 23:21:31 -0700638 u32 classid;
639 struct hlist_node hnode;
640};
641
Eric Dumazetfd2c3ef2009-11-03 03:26:03 +0000642struct Qdisc_class_hash {
Patrick McHardy6fe1c7a2008-07-05 23:21:31 -0700643 struct hlist_head *hash;
644 unsigned int hashsize;
645 unsigned int hashmask;
646 unsigned int hashelems;
647};
648
649static inline unsigned int qdisc_class_hash(u32 id, u32 mask)
650{
651 id ^= id >> 8;
652 id ^= id >> 4;
653 return id & mask;
654}
655
656static inline struct Qdisc_class_common *
Eric Dumazet05bdd2f2011-10-20 17:45:43 -0400657qdisc_class_find(const struct Qdisc_class_hash *hash, u32 id)
Patrick McHardy6fe1c7a2008-07-05 23:21:31 -0700658{
659 struct Qdisc_class_common *cl;
Patrick McHardy6fe1c7a2008-07-05 23:21:31 -0700660 unsigned int h;
661
Gao Feng7d3f0cd2017-08-18 15:23:24 +0800662 if (!id)
663 return NULL;
664
Patrick McHardy6fe1c7a2008-07-05 23:21:31 -0700665 h = qdisc_class_hash(id, hash->hashmask);
Sasha Levinb67bfe02013-02-27 17:06:00 -0800666 hlist_for_each_entry(cl, &hash->hash[h], hnode) {
Patrick McHardy6fe1c7a2008-07-05 23:21:31 -0700667 if (cl->classid == id)
668 return cl;
669 }
670 return NULL;
671}
672
Amritha Nambiar384c1812017-10-27 02:35:34 -0700673static inline int tc_classid_to_hwtc(struct net_device *dev, u32 classid)
674{
675 u32 hwtc = TC_H_MIN(classid) - TC_H_MIN_PRIORITY;
676
677 return (hwtc < netdev_get_num_tc(dev)) ? hwtc : -EINVAL;
678}
679
Joe Perches5c152572013-07-30 22:47:13 -0700680int qdisc_class_hash_init(struct Qdisc_class_hash *);
681void qdisc_class_hash_insert(struct Qdisc_class_hash *,
682 struct Qdisc_class_common *);
683void qdisc_class_hash_remove(struct Qdisc_class_hash *,
684 struct Qdisc_class_common *);
685void qdisc_class_hash_grow(struct Qdisc *, struct Qdisc_class_hash *);
686void qdisc_class_hash_destroy(struct Qdisc_class_hash *);
Patrick McHardy6fe1c7a2008-07-05 23:21:31 -0700687
Cong Wang48bfd552018-01-25 18:26:23 -0800688int dev_qdisc_change_tx_queue_len(struct net_device *dev);
Jakub Kicinski1e080f12021-09-13 15:53:30 -0700689void dev_qdisc_change_real_num_tx(struct net_device *dev,
690 unsigned int new_real_tx);
Joe Perches5c152572013-07-30 22:47:13 -0700691void dev_init_scheduler(struct net_device *dev);
692void dev_shutdown(struct net_device *dev);
693void dev_activate(struct net_device *dev);
694void dev_deactivate(struct net_device *dev);
695void dev_deactivate_many(struct list_head *head);
696struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue,
697 struct Qdisc *qdisc);
698void qdisc_reset(struct Qdisc *qdisc);
Vlad Buslov86bd4462018-09-24 19:22:50 +0300699void qdisc_put(struct Qdisc *qdisc);
Vlad Buslov3a7d0d02018-09-24 19:22:51 +0300700void qdisc_put_unlocked(struct Qdisc *qdisc);
Toke Høiland-Jørgensen5f2939d2019-01-09 17:10:57 +0100701void qdisc_tree_reduce_backlog(struct Qdisc *qdisc, int n, int len);
Jakub Kicinskib5928432018-11-07 17:33:34 -0800702#ifdef CONFIG_NET_SCHED
703int qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type,
704 void *type_data);
Jakub Kicinskibfaee912018-11-07 17:33:37 -0800705void qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch,
706 struct Qdisc *new, struct Qdisc *old,
707 enum tc_setup_type type, void *type_data,
708 struct netlink_ext_ack *extack);
Jakub Kicinskib5928432018-11-07 17:33:34 -0800709#else
710static inline int
711qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type,
712 void *type_data)
713{
714 q->flags &= ~TCQ_F_OFFLOADED;
715 return 0;
716}
Jakub Kicinskibfaee912018-11-07 17:33:37 -0800717
718static inline void
719qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch,
720 struct Qdisc *new, struct Qdisc *old,
721 enum tc_setup_type type, void *type_data,
722 struct netlink_ext_ack *extack)
723{
724}
Jakub Kicinskib5928432018-11-07 17:33:34 -0800725#endif
Joe Perches5c152572013-07-30 22:47:13 -0700726struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
Alexander Aringd0bd6842017-12-20 12:35:20 -0500727 const struct Qdisc_ops *ops,
728 struct netlink_ext_ack *extack);
Daniel Borkmann81d947e2018-01-15 23:12:09 +0100729void qdisc_free(struct Qdisc *qdisc);
Joe Perches5c152572013-07-30 22:47:13 -0700730struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue,
Alexander Aringa38a98822017-12-20 12:35:21 -0500731 const struct Qdisc_ops *ops, u32 parentid,
732 struct netlink_ext_ack *extack);
Joe Perches5c152572013-07-30 22:47:13 -0700733void __qdisc_calculate_pkt_len(struct sk_buff *skb,
734 const struct qdisc_size_table *stab);
Alexei Starovoitov27b29f62015-09-15 23:05:43 -0700735int skb_do_redirect(struct sk_buff *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736
Daniel Borkmannfdc54322016-01-07 15:50:22 +0100737static inline bool skb_at_tc_ingress(const struct sk_buff *skb)
738{
739#ifdef CONFIG_NET_CLS_ACT
Willem de Bruijn8dc07fd2017-01-07 17:06:37 -0500740 return skb->tc_at_ingress;
Daniel Borkmannfdc54322016-01-07 15:50:22 +0100741#else
742 return false;
743#endif
744}
745
Willem de Bruijne7246e12017-01-07 17:06:35 -0500746static inline bool skb_skip_tc_classify(struct sk_buff *skb)
747{
748#ifdef CONFIG_NET_CLS_ACT
749 if (skb->tc_skip_classify) {
750 skb->tc_skip_classify = 0;
751 return true;
752 }
753#endif
754 return false;
755}
756
Gal Pressman3a053b12018-02-28 15:59:15 +0200757/* Reset all TX qdiscs greater than index of a device. */
John Fastabendf0796d52010-07-01 13:21:57 +0000758static inline void qdisc_reset_all_tx_gt(struct net_device *dev, unsigned int i)
David S. Miller5aa70992008-07-08 22:59:10 -0700759{
John Fastabend4ef6acf2010-07-01 13:21:35 +0000760 struct Qdisc *qdisc;
761
John Fastabendf0796d52010-07-01 13:21:57 +0000762 for (; i < dev->num_tx_queues; i++) {
John Fastabend46e5da40a2014-09-12 20:04:52 -0700763 qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc);
John Fastabend4ef6acf2010-07-01 13:21:35 +0000764 if (qdisc) {
765 spin_lock_bh(qdisc_lock(qdisc));
766 qdisc_reset(qdisc);
767 spin_unlock_bh(qdisc_lock(qdisc));
768 }
769 }
David S. Miller5aa70992008-07-08 22:59:10 -0700770}
771
David S. Miller3e745dd2008-07-08 23:00:25 -0700772/* Are all TX queues of the device empty? */
773static inline bool qdisc_all_tx_empty(const struct net_device *dev)
774{
David S. Millere8a04642008-07-17 00:34:19 -0700775 unsigned int i;
John Fastabend46e5da40a2014-09-12 20:04:52 -0700776
777 rcu_read_lock();
David S. Millere8a04642008-07-17 00:34:19 -0700778 for (i = 0; i < dev->num_tx_queues; i++) {
779 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
John Fastabend46e5da40a2014-09-12 20:04:52 -0700780 const struct Qdisc *q = rcu_dereference(txq->qdisc);
David S. Miller3e745dd2008-07-08 23:00:25 -0700781
Paolo Abeni1f5e6fd2019-04-10 14:32:38 +0200782 if (!qdisc_is_empty(q)) {
John Fastabend46e5da40a2014-09-12 20:04:52 -0700783 rcu_read_unlock();
David S. Millere8a04642008-07-17 00:34:19 -0700784 return false;
John Fastabend46e5da40a2014-09-12 20:04:52 -0700785 }
David S. Millere8a04642008-07-17 00:34:19 -0700786 }
John Fastabend46e5da40a2014-09-12 20:04:52 -0700787 rcu_read_unlock();
David S. Millere8a04642008-07-17 00:34:19 -0700788 return true;
David S. Miller3e745dd2008-07-08 23:00:25 -0700789}
790
David S. Miller6fa98642008-07-08 23:01:06 -0700791/* Are any of the TX qdiscs changing? */
Eric Dumazet05bdd2f2011-10-20 17:45:43 -0400792static inline bool qdisc_tx_changing(const struct net_device *dev)
David S. Miller6fa98642008-07-08 23:01:06 -0700793{
David S. Millere8a04642008-07-17 00:34:19 -0700794 unsigned int i;
John Fastabend46e5da40a2014-09-12 20:04:52 -0700795
David S. Millere8a04642008-07-17 00:34:19 -0700796 for (i = 0; i < dev->num_tx_queues; i++) {
797 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
John Fastabend46e5da40a2014-09-12 20:04:52 -0700798 if (rcu_access_pointer(txq->qdisc) != txq->qdisc_sleeping)
David S. Millere8a04642008-07-17 00:34:19 -0700799 return true;
800 }
801 return false;
David S. Miller6fa98642008-07-08 23:01:06 -0700802}
803
David S. Millere8a04642008-07-17 00:34:19 -0700804/* Is the device using the noop qdisc on all queues? */
David S. Miller05297942008-07-08 23:01:27 -0700805static inline bool qdisc_tx_is_noop(const struct net_device *dev)
806{
David S. Millere8a04642008-07-17 00:34:19 -0700807 unsigned int i;
John Fastabend46e5da40a2014-09-12 20:04:52 -0700808
David S. Millere8a04642008-07-17 00:34:19 -0700809 for (i = 0; i < dev->num_tx_queues; i++) {
810 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
John Fastabend46e5da40a2014-09-12 20:04:52 -0700811 if (rcu_access_pointer(txq->qdisc) != &noop_qdisc)
David S. Millere8a04642008-07-17 00:34:19 -0700812 return false;
813 }
814 return true;
David S. Miller05297942008-07-08 23:01:27 -0700815}
816
Eric Dumazetbfe0d022011-01-09 08:30:54 +0000817static inline unsigned int qdisc_pkt_len(const struct sk_buff *skb)
Jussi Kivilinna0abf77e2008-07-20 00:08:27 -0700818{
Jussi Kivilinna175f9c12008-07-20 00:08:47 -0700819 return qdisc_skb_cb(skb)->pkt_len;
Jussi Kivilinna0abf77e2008-07-20 00:08:27 -0700820}
821
Jarek Poplawskic27f3392008-08-04 22:39:11 -0700822/* additional qdisc xmit flags (NET_XMIT_MASK in linux/netdevice.h) */
Jarek Poplawski378a2f02008-08-04 22:31:03 -0700823enum net_xmit_qdisc_t {
824 __NET_XMIT_STOLEN = 0x00010000,
Jarek Poplawskic27f3392008-08-04 22:39:11 -0700825 __NET_XMIT_BYPASS = 0x00020000,
Jarek Poplawski378a2f02008-08-04 22:31:03 -0700826};
827
Jarek Poplawskic27f3392008-08-04 22:39:11 -0700828#ifdef CONFIG_NET_CLS_ACT
Jarek Poplawski378a2f02008-08-04 22:31:03 -0700829#define net_xmit_drop_count(e) ((e) & __NET_XMIT_STOLEN ? 0 : 1)
Jarek Poplawski378a2f02008-08-04 22:31:03 -0700830#else
831#define net_xmit_drop_count(e) (1)
832#endif
833
Eric Dumazeta2da5702011-01-20 03:48:19 +0000834static inline void qdisc_calculate_pkt_len(struct sk_buff *skb,
835 const struct Qdisc *sch)
Jussi Kivilinna5f861732008-07-20 00:08:04 -0700836{
David S. Miller3a682fb2008-07-20 18:13:01 -0700837#ifdef CONFIG_NET_SCHED
Eric Dumazeta2da5702011-01-20 03:48:19 +0000838 struct qdisc_size_table *stab = rcu_dereference_bh(sch->stab);
839
840 if (stab)
841 __qdisc_calculate_pkt_len(skb, stab);
David S. Miller3a682fb2008-07-20 18:13:01 -0700842#endif
Eric Dumazeta2da5702011-01-20 03:48:19 +0000843}
844
Petr Machataac5c66f2020-07-14 20:03:08 +0300845static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch,
Eric Dumazet520ac302016-06-21 23:16:49 -0700846 struct sk_buff **to_free)
Eric Dumazeta2da5702011-01-20 03:48:19 +0000847{
848 qdisc_calculate_pkt_len(skb, sch);
Petr Machataac5c66f2020-07-14 20:03:08 +0300849 return sch->enqueue(skb, sch, to_free);
Jussi Kivilinna5f861732008-07-20 00:08:04 -0700850}
851
Ahmed S. Darwish50dc9a82021-10-16 10:49:09 +0200852static inline void _bstats_update(struct gnet_stats_basic_sync *bstats,
Amir Vadai38040702016-05-13 12:55:35 +0000853 __u64 bytes, __u32 packets)
854{
Ahmed S. Darwish67c9e62702021-10-16 10:49:07 +0200855 u64_stats_update_begin(&bstats->syncp);
Ahmed S. Darwish50dc9a82021-10-16 10:49:09 +0200856 u64_stats_add(&bstats->bytes, bytes);
857 u64_stats_add(&bstats->packets, packets);
Ahmed S. Darwish67c9e62702021-10-16 10:49:07 +0200858 u64_stats_update_end(&bstats->syncp);
Amir Vadai38040702016-05-13 12:55:35 +0000859}
860
Ahmed S. Darwish50dc9a82021-10-16 10:49:09 +0200861static inline void bstats_update(struct gnet_stats_basic_sync *bstats,
Eric Dumazetbfe0d022011-01-09 08:30:54 +0000862 const struct sk_buff *skb)
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +0000863{
Amir Vadai38040702016-05-13 12:55:35 +0000864 _bstats_update(bstats,
865 qdisc_pkt_len(skb),
866 skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1);
867}
868
Eric Dumazet24ea5912015-07-06 05:18:03 -0700869static inline void qdisc_bstats_cpu_update(struct Qdisc *sch,
870 const struct sk_buff *skb)
871{
Ahmed S. Darwish50dc9a82021-10-16 10:49:09 +0200872 bstats_update(this_cpu_ptr(sch->cpu_bstats), skb);
Eric Dumazet24ea5912015-07-06 05:18:03 -0700873}
874
Eric Dumazetbfe0d022011-01-09 08:30:54 +0000875static inline void qdisc_bstats_update(struct Qdisc *sch,
876 const struct sk_buff *skb)
877{
878 bstats_update(&sch->bstats, skb);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +0000879}
880
John Fastabend25331d62014-09-28 11:53:29 -0700881static inline void qdisc_qstats_backlog_dec(struct Qdisc *sch,
882 const struct sk_buff *skb)
883{
884 sch->qstats.backlog -= qdisc_pkt_len(skb);
885}
886
John Fastabend40bd0362017-12-07 09:55:07 -0800887static inline void qdisc_qstats_cpu_backlog_dec(struct Qdisc *sch,
888 const struct sk_buff *skb)
889{
890 this_cpu_sub(sch->cpu_qstats->backlog, qdisc_pkt_len(skb));
891}
892
John Fastabend25331d62014-09-28 11:53:29 -0700893static inline void qdisc_qstats_backlog_inc(struct Qdisc *sch,
894 const struct sk_buff *skb)
895{
896 sch->qstats.backlog += qdisc_pkt_len(skb);
897}
898
John Fastabend40bd0362017-12-07 09:55:07 -0800899static inline void qdisc_qstats_cpu_backlog_inc(struct Qdisc *sch,
900 const struct sk_buff *skb)
901{
902 this_cpu_add(sch->cpu_qstats->backlog, qdisc_pkt_len(skb));
903}
904
Paolo Abeni73eb6282019-04-10 14:32:41 +0200905static inline void qdisc_qstats_cpu_qlen_inc(struct Qdisc *sch)
John Fastabend40bd0362017-12-07 09:55:07 -0800906{
Paolo Abeni73eb6282019-04-10 14:32:41 +0200907 this_cpu_inc(sch->cpu_qstats->qlen);
John Fastabend40bd0362017-12-07 09:55:07 -0800908}
909
Paolo Abeni73eb6282019-04-10 14:32:41 +0200910static inline void qdisc_qstats_cpu_qlen_dec(struct Qdisc *sch)
John Fastabend40bd0362017-12-07 09:55:07 -0800911{
Paolo Abeni73eb6282019-04-10 14:32:41 +0200912 this_cpu_dec(sch->cpu_qstats->qlen);
John Fastabend40bd0362017-12-07 09:55:07 -0800913}
914
915static inline void qdisc_qstats_cpu_requeues_inc(struct Qdisc *sch)
916{
917 this_cpu_inc(sch->cpu_qstats->requeues);
918}
919
John Fastabend25331d62014-09-28 11:53:29 -0700920static inline void __qdisc_qstats_drop(struct Qdisc *sch, int count)
921{
922 sch->qstats.drops += count;
923}
924
Eric Dumazet24ea5912015-07-06 05:18:03 -0700925static inline void qstats_drop_inc(struct gnet_stats_queue *qstats)
John Fastabend25331d62014-09-28 11:53:29 -0700926{
Eric Dumazet24ea5912015-07-06 05:18:03 -0700927 qstats->drops++;
John Fastabend25331d62014-09-28 11:53:29 -0700928}
929
Eric Dumazet24ea5912015-07-06 05:18:03 -0700930static inline void qstats_overlimit_inc(struct gnet_stats_queue *qstats)
John Fastabendb0ab6f92014-09-28 11:54:24 -0700931{
Eric Dumazet24ea5912015-07-06 05:18:03 -0700932 qstats->overlimits++;
933}
John Fastabendb0ab6f92014-09-28 11:54:24 -0700934
Eric Dumazet24ea5912015-07-06 05:18:03 -0700935static inline void qdisc_qstats_drop(struct Qdisc *sch)
936{
937 qstats_drop_inc(&sch->qstats);
938}
939
940static inline void qdisc_qstats_cpu_drop(struct Qdisc *sch)
941{
Eric Dumazeteb60a8d2016-08-24 10:23:34 -0700942 this_cpu_inc(sch->cpu_qstats->drops);
John Fastabendb0ab6f92014-09-28 11:54:24 -0700943}
944
John Fastabend25331d62014-09-28 11:53:29 -0700945static inline void qdisc_qstats_overlimit(struct Qdisc *sch)
946{
947 sch->qstats.overlimits++;
948}
949
Paolo Abeni5dd431b2019-03-28 16:53:12 +0100950static inline int qdisc_qstats_copy(struct gnet_dump *d, struct Qdisc *sch)
951{
952 __u32 qlen = qdisc_qlen_sum(sch);
953
954 return gnet_stats_copy_queue(d, sch->cpu_qstats, &sch->qstats, qlen);
955}
956
957static inline void qdisc_qstats_qlen_backlog(struct Qdisc *sch, __u32 *qlen,
958 __u32 *backlog)
959{
960 struct gnet_stats_queue qstats = { 0 };
Paolo Abeni5dd431b2019-03-28 16:53:12 +0100961
Sebastian Andrzej Siewior10940eb2021-10-16 10:49:05 +0200962 gnet_stats_add_queue(&qstats, sch->cpu_qstats, &sch->qstats);
963 *qlen = qstats.qlen + qdisc_qlen(sch);
Paolo Abeni5dd431b2019-03-28 16:53:12 +0100964 *backlog = qstats.backlog;
965}
966
Paolo Abenie5f0e8f2019-03-28 16:53:13 +0100967static inline void qdisc_tree_flush_backlog(struct Qdisc *sch)
968{
969 __u32 qlen, backlog;
970
971 qdisc_qstats_qlen_backlog(sch, &qlen, &backlog);
972 qdisc_tree_reduce_backlog(sch, qlen, backlog);
973}
974
975static inline void qdisc_purge_queue(struct Qdisc *sch)
976{
977 __u32 qlen, backlog;
978
979 qdisc_qstats_qlen_backlog(sch, &qlen, &backlog);
980 qdisc_reset(sch);
981 qdisc_tree_reduce_backlog(sch, qlen, backlog);
982}
983
Florian Westphal48da34b2016-09-18 00:57:34 +0200984static inline void qdisc_skb_head_init(struct qdisc_skb_head *qh)
Thomas Graf9972b252005-06-18 22:57:26 -0700985{
Florian Westphal48da34b2016-09-18 00:57:34 +0200986 qh->head = NULL;
987 qh->tail = NULL;
988 qh->qlen = 0;
989}
990
David S. Milleraea890b2018-07-29 16:22:13 -0700991static inline void __qdisc_enqueue_tail(struct sk_buff *skb,
992 struct qdisc_skb_head *qh)
Florian Westphal48da34b2016-09-18 00:57:34 +0200993{
994 struct sk_buff *last = qh->tail;
995
996 if (last) {
997 skb->next = NULL;
998 last->next = skb;
999 qh->tail = skb;
1000 } else {
1001 qh->tail = skb;
1002 qh->head = skb;
1003 }
1004 qh->qlen++;
Thomas Graf9972b252005-06-18 22:57:26 -07001005}
1006
1007static inline int qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch)
1008{
David S. Milleraea890b2018-07-29 16:22:13 -07001009 __qdisc_enqueue_tail(skb, &sch->q);
1010 qdisc_qstats_backlog_inc(sch, skb);
1011 return NET_XMIT_SUCCESS;
Thomas Graf9972b252005-06-18 22:57:26 -07001012}
1013
David S. Miller59697732018-07-29 16:33:28 -07001014static inline void __qdisc_enqueue_head(struct sk_buff *skb,
1015 struct qdisc_skb_head *qh)
1016{
1017 skb->next = qh->head;
1018
1019 if (!qh->head)
1020 qh->tail = skb;
1021 qh->head = skb;
1022 qh->qlen++;
1023}
1024
Florian Westphal48da34b2016-09-18 00:57:34 +02001025static inline struct sk_buff *__qdisc_dequeue_head(struct qdisc_skb_head *qh)
Thomas Graf9972b252005-06-18 22:57:26 -07001026{
Florian Westphal48da34b2016-09-18 00:57:34 +02001027 struct sk_buff *skb = qh->head;
1028
1029 if (likely(skb != NULL)) {
1030 qh->head = skb->next;
1031 qh->qlen--;
1032 if (qh->head == NULL)
1033 qh->tail = NULL;
1034 skb->next = NULL;
1035 }
Thomas Graf9972b252005-06-18 22:57:26 -07001036
Florian Westphalec323362016-09-18 00:57:32 +02001037 return skb;
1038}
1039
1040static inline struct sk_buff *qdisc_dequeue_head(struct Qdisc *sch)
1041{
1042 struct sk_buff *skb = __qdisc_dequeue_head(&sch->q);
1043
Eric Dumazet9190b3b2011-01-20 23:31:33 -08001044 if (likely(skb != NULL)) {
John Fastabend25331d62014-09-28 11:53:29 -07001045 qdisc_qstats_backlog_dec(sch, skb);
Eric Dumazet9190b3b2011-01-20 23:31:33 -08001046 qdisc_bstats_update(sch, skb);
1047 }
Thomas Graf9972b252005-06-18 22:57:26 -07001048
1049 return skb;
1050}
1051
Eric Dumazet520ac302016-06-21 23:16:49 -07001052/* Instead of calling kfree_skb() while root qdisc lock is held,
1053 * queue the skb for future freeing at end of __dev_xmit_skb()
1054 */
1055static inline void __qdisc_drop(struct sk_buff *skb, struct sk_buff **to_free)
1056{
1057 skb->next = *to_free;
1058 *to_free = skb;
1059}
1060
Alexey Kodanev35d889d2018-03-05 20:52:54 +03001061static inline void __qdisc_drop_all(struct sk_buff *skb,
1062 struct sk_buff **to_free)
1063{
1064 if (skb->prev)
1065 skb->prev->next = *to_free;
1066 else
1067 skb->next = *to_free;
1068 *to_free = skb;
1069}
1070
Hagen Paul Pfeifer57dbb2d2010-01-24 12:30:59 +00001071static inline unsigned int __qdisc_queue_drop_head(struct Qdisc *sch,
Florian Westphal48da34b2016-09-18 00:57:34 +02001072 struct qdisc_skb_head *qh,
Eric Dumazet520ac302016-06-21 23:16:49 -07001073 struct sk_buff **to_free)
Hagen Paul Pfeifer57dbb2d2010-01-24 12:30:59 +00001074{
Florian Westphal48da34b2016-09-18 00:57:34 +02001075 struct sk_buff *skb = __qdisc_dequeue_head(qh);
Hagen Paul Pfeifer57dbb2d2010-01-24 12:30:59 +00001076
1077 if (likely(skb != NULL)) {
1078 unsigned int len = qdisc_pkt_len(skb);
Eric Dumazet520ac302016-06-21 23:16:49 -07001079
John Fastabend25331d62014-09-28 11:53:29 -07001080 qdisc_qstats_backlog_dec(sch, skb);
Eric Dumazet520ac302016-06-21 23:16:49 -07001081 __qdisc_drop(skb, to_free);
Hagen Paul Pfeifer57dbb2d2010-01-24 12:30:59 +00001082 return len;
1083 }
1084
1085 return 0;
1086}
1087
Patrick McHardy48a8f512008-10-31 00:44:18 -07001088static inline struct sk_buff *qdisc_peek_head(struct Qdisc *sch)
1089{
Florian Westphal48da34b2016-09-18 00:57:34 +02001090 const struct qdisc_skb_head *qh = &sch->q;
1091
1092 return qh->head;
Patrick McHardy48a8f512008-10-31 00:44:18 -07001093}
1094
Jarek Poplawski77be1552008-10-31 00:47:01 -07001095/* generic pseudo peek method for non-work-conserving qdisc */
1096static inline struct sk_buff *qdisc_peek_dequeued(struct Qdisc *sch)
1097{
John Fastabenda53851e2017-12-07 09:55:45 -08001098 struct sk_buff *skb = skb_peek(&sch->gso_skb);
1099
Jarek Poplawski77be1552008-10-31 00:47:01 -07001100 /* we can reuse ->gso_skb because peek isn't called for root qdiscs */
John Fastabenda53851e2017-12-07 09:55:45 -08001101 if (!skb) {
1102 skb = sch->dequeue(sch);
1103
1104 if (skb) {
1105 __skb_queue_head(&sch->gso_skb, skb);
Jarek Poplawski61c9eaf2008-11-05 16:02:34 -08001106 /* it's still part of the queue */
John Fastabenda53851e2017-12-07 09:55:45 -08001107 qdisc_qstats_backlog_inc(sch, skb);
Jarek Poplawski61c9eaf2008-11-05 16:02:34 -08001108 sch->q.qlen++;
WANG Conga27758f2016-06-03 15:05:57 -07001109 }
Jarek Poplawski61c9eaf2008-11-05 16:02:34 -08001110 }
Jarek Poplawski77be1552008-10-31 00:47:01 -07001111
John Fastabenda53851e2017-12-07 09:55:45 -08001112 return skb;
Jarek Poplawski77be1552008-10-31 00:47:01 -07001113}
1114
Paolo Abeni8a53e612019-04-10 14:32:40 +02001115static inline void qdisc_update_stats_at_dequeue(struct Qdisc *sch,
1116 struct sk_buff *skb)
1117{
1118 if (qdisc_is_percpu_stats(sch)) {
1119 qdisc_qstats_cpu_backlog_dec(sch, skb);
1120 qdisc_bstats_cpu_update(sch, skb);
Paolo Abeni73eb6282019-04-10 14:32:41 +02001121 qdisc_qstats_cpu_qlen_dec(sch);
Paolo Abeni8a53e612019-04-10 14:32:40 +02001122 } else {
1123 qdisc_qstats_backlog_dec(sch, skb);
1124 qdisc_bstats_update(sch, skb);
1125 sch->q.qlen--;
1126 }
1127}
1128
1129static inline void qdisc_update_stats_at_enqueue(struct Qdisc *sch,
1130 unsigned int pkt_len)
1131{
1132 if (qdisc_is_percpu_stats(sch)) {
Paolo Abeni73eb6282019-04-10 14:32:41 +02001133 qdisc_qstats_cpu_qlen_inc(sch);
Paolo Abeni8a53e612019-04-10 14:32:40 +02001134 this_cpu_add(sch->cpu_qstats->backlog, pkt_len);
1135 } else {
1136 sch->qstats.backlog += pkt_len;
1137 sch->q.qlen++;
1138 }
1139}
1140
Jarek Poplawski77be1552008-10-31 00:47:01 -07001141/* use instead of qdisc->dequeue() for all qdiscs queried with ->peek() */
1142static inline struct sk_buff *qdisc_dequeue_peeked(struct Qdisc *sch)
1143{
John Fastabenda53851e2017-12-07 09:55:45 -08001144 struct sk_buff *skb = skb_peek(&sch->gso_skb);
Jarek Poplawski77be1552008-10-31 00:47:01 -07001145
Jarek Poplawski61c9eaf2008-11-05 16:02:34 -08001146 if (skb) {
John Fastabenda53851e2017-12-07 09:55:45 -08001147 skb = __skb_dequeue(&sch->gso_skb);
Paolo Abeni9c01c9f2019-04-10 14:32:39 +02001148 if (qdisc_is_percpu_stats(sch)) {
1149 qdisc_qstats_cpu_backlog_dec(sch, skb);
Paolo Abeni73eb6282019-04-10 14:32:41 +02001150 qdisc_qstats_cpu_qlen_dec(sch);
Paolo Abeni9c01c9f2019-04-10 14:32:39 +02001151 } else {
1152 qdisc_qstats_backlog_dec(sch, skb);
1153 sch->q.qlen--;
1154 }
Jarek Poplawski61c9eaf2008-11-05 16:02:34 -08001155 } else {
Jarek Poplawski77be1552008-10-31 00:47:01 -07001156 skb = sch->dequeue(sch);
Jarek Poplawski61c9eaf2008-11-05 16:02:34 -08001157 }
Jarek Poplawski77be1552008-10-31 00:47:01 -07001158
1159 return skb;
1160}
1161
Florian Westphal48da34b2016-09-18 00:57:34 +02001162static inline void __qdisc_reset_queue(struct qdisc_skb_head *qh)
Thomas Graf9972b252005-06-18 22:57:26 -07001163{
1164 /*
1165 * We do not know the backlog in bytes of this list, it
1166 * is up to the caller to correct it
1167 */
Florian Westphal48da34b2016-09-18 00:57:34 +02001168 ASSERT_RTNL();
1169 if (qh->qlen) {
1170 rtnl_kfree_skbs(qh->head, qh->tail);
1171
1172 qh->head = NULL;
1173 qh->tail = NULL;
1174 qh->qlen = 0;
Eric Dumazet1b5c5492016-06-13 20:21:50 -07001175 }
Thomas Graf9972b252005-06-18 22:57:26 -07001176}
1177
1178static inline void qdisc_reset_queue(struct Qdisc *sch)
1179{
Eric Dumazet1b5c5492016-06-13 20:21:50 -07001180 __qdisc_reset_queue(&sch->q);
Thomas Graf9972b252005-06-18 22:57:26 -07001181 sch->qstats.backlog = 0;
1182}
1183
WANG Cong86a79962016-02-25 14:55:00 -08001184static inline struct Qdisc *qdisc_replace(struct Qdisc *sch, struct Qdisc *new,
1185 struct Qdisc **pold)
1186{
1187 struct Qdisc *old;
1188
1189 sch_tree_lock(sch);
1190 old = *pold;
1191 *pold = new;
Paolo Abenie5f0e8f2019-03-28 16:53:13 +01001192 if (old != NULL)
Alexander Ovechkin938e0fc2021-02-01 23:00:49 +03001193 qdisc_purge_queue(old);
WANG Cong86a79962016-02-25 14:55:00 -08001194 sch_tree_unlock(sch);
1195
1196 return old;
1197}
1198
Eric Dumazet1b5c5492016-06-13 20:21:50 -07001199static inline void rtnl_qdisc_drop(struct sk_buff *skb, struct Qdisc *sch)
1200{
1201 rtnl_kfree_skbs(skb, skb);
1202 qdisc_qstats_drop(sch);
1203}
1204
John Fastabend40bd0362017-12-07 09:55:07 -08001205static inline int qdisc_drop_cpu(struct sk_buff *skb, struct Qdisc *sch,
1206 struct sk_buff **to_free)
1207{
1208 __qdisc_drop(skb, to_free);
1209 qdisc_qstats_cpu_drop(sch);
1210
1211 return NET_XMIT_DROP;
1212}
Eric Dumazet520ac302016-06-21 23:16:49 -07001213
1214static inline int qdisc_drop(struct sk_buff *skb, struct Qdisc *sch,
1215 struct sk_buff **to_free)
Thomas Graf9972b252005-06-18 22:57:26 -07001216{
Eric Dumazet520ac302016-06-21 23:16:49 -07001217 __qdisc_drop(skb, to_free);
John Fastabend25331d62014-09-28 11:53:29 -07001218 qdisc_qstats_drop(sch);
Thomas Graf9972b252005-06-18 22:57:26 -07001219
1220 return NET_XMIT_DROP;
1221}
1222
Alexey Kodanev35d889d2018-03-05 20:52:54 +03001223static inline int qdisc_drop_all(struct sk_buff *skb, struct Qdisc *sch,
1224 struct sk_buff **to_free)
1225{
1226 __qdisc_drop_all(skb, to_free);
1227 qdisc_qstats_drop(sch);
1228
1229 return NET_XMIT_DROP;
1230}
1231
Jesper Dangaard Brouere9bef552007-09-12 16:35:24 +02001232/* Length to Time (L2T) lookup in a qdisc_rate_table, to determine how
1233 long it will take to send a packet given its size.
1234 */
1235static inline u32 qdisc_l2t(struct qdisc_rate_table* rtab, unsigned int pktlen)
1236{
Jesper Dangaard Brouere08b0992007-09-12 16:36:28 +02001237 int slot = pktlen + rtab->rate.cell_align + rtab->rate.overhead;
1238 if (slot < 0)
1239 slot = 0;
Jesper Dangaard Brouere9bef552007-09-12 16:35:24 +02001240 slot >>= rtab->rate.cell_log;
1241 if (slot > 255)
Eric Dumazeta02cec22010-09-22 20:43:57 +00001242 return rtab->data[255]*(slot >> 8) + rtab->data[slot & 0xFF];
Jesper Dangaard Brouere9bef552007-09-12 16:35:24 +02001243 return rtab->data[slot];
1244}
1245
Jiri Pirko292f1c72013-02-12 00:12:03 +00001246struct psched_ratecfg {
Eric Dumazet130d3d62013-06-06 13:56:19 -07001247 u64 rate_bytes_ps; /* bytes per second */
Eric Dumazet01cb71d2013-06-02 13:55:05 +00001248 u32 mult;
1249 u16 overhead;
Jesper Dangaard Brouer8a8e3d82013-08-14 23:47:11 +02001250 u8 linklayer;
Eric Dumazet01cb71d2013-06-02 13:55:05 +00001251 u8 shift;
Jiri Pirko292f1c72013-02-12 00:12:03 +00001252};
1253
1254static inline u64 psched_l2t_ns(const struct psched_ratecfg *r,
1255 unsigned int len)
1256{
Jesper Dangaard Brouer8a8e3d82013-08-14 23:47:11 +02001257 len += r->overhead;
1258
1259 if (unlikely(r->linklayer == TC_LINKLAYER_ATM))
1260 return ((u64)(DIV_ROUND_UP(len,48)*53) * r->mult) >> r->shift;
1261
1262 return ((u64)len * r->mult) >> r->shift;
Jiri Pirko292f1c72013-02-12 00:12:03 +00001263}
1264
Joe Perches5c152572013-07-30 22:47:13 -07001265void psched_ratecfg_precompute(struct psched_ratecfg *r,
Eric Dumazet3e1e3aa2013-09-19 09:10:03 -07001266 const struct tc_ratespec *conf,
1267 u64 rate64);
Jiri Pirko292f1c72013-02-12 00:12:03 +00001268
Eric Dumazet01cb71d2013-06-02 13:55:05 +00001269static inline void psched_ratecfg_getrate(struct tc_ratespec *res,
1270 const struct psched_ratecfg *r)
Jiri Pirko292f1c72013-02-12 00:12:03 +00001271{
Eric Dumazet01cb71d2013-06-02 13:55:05 +00001272 memset(res, 0, sizeof(*res));
Eric Dumazet3e1e3aa2013-09-19 09:10:03 -07001273
1274 /* legacy struct tc_ratespec has a 32bit @rate field
1275 * Qdisc using 64bit rate should add new attributes
1276 * in order to maintain compatibility.
1277 */
1278 res->rate = min_t(u64, r->rate_bytes_ps, ~0U);
1279
Eric Dumazet01cb71d2013-06-02 13:55:05 +00001280 res->overhead = r->overhead;
Jesper Dangaard Brouer8a8e3d82013-08-14 23:47:11 +02001281 res->linklayer = (r->linklayer & TC_LINKLAYER_MASK);
Jiri Pirko292f1c72013-02-12 00:12:03 +00001282}
1283
Baowen Zheng2ffe0392021-03-12 15:08:31 +01001284struct psched_pktrate {
1285 u64 rate_pkts_ps; /* packets per second */
1286 u32 mult;
1287 u8 shift;
1288};
1289
1290static inline u64 psched_pkt2t_ns(const struct psched_pktrate *r,
1291 unsigned int pkt_num)
1292{
1293 return ((u64)pkt_num * r->mult) >> r->shift;
1294}
1295
1296void psched_ppscfg_precompute(struct psched_pktrate *r, u64 pktrate64);
1297
Jiri Pirko46209402017-11-03 11:46:25 +01001298/* Mini Qdisc serves for specific needs of ingress/clsact Qdisc.
1299 * The fast path only needs to access filter list and to update stats
1300 */
1301struct mini_Qdisc {
1302 struct tcf_proto *filter_list;
Paul Blakey7d17c542020-02-16 12:01:22 +02001303 struct tcf_block *block;
Ahmed S. Darwish50dc9a82021-10-16 10:49:09 +02001304 struct gnet_stats_basic_sync __percpu *cpu_bstats;
Jiri Pirko46209402017-11-03 11:46:25 +01001305 struct gnet_stats_queue __percpu *cpu_qstats;
1306 struct rcu_head rcu;
1307};
1308
1309static inline void mini_qdisc_bstats_cpu_update(struct mini_Qdisc *miniq,
1310 const struct sk_buff *skb)
1311{
Ahmed S. Darwish50dc9a82021-10-16 10:49:09 +02001312 bstats_update(this_cpu_ptr(miniq->cpu_bstats), skb);
Jiri Pirko46209402017-11-03 11:46:25 +01001313}
1314
1315static inline void mini_qdisc_qstats_cpu_drop(struct mini_Qdisc *miniq)
1316{
1317 this_cpu_inc(miniq->cpu_qstats->drops);
1318}
1319
1320struct mini_Qdisc_pair {
1321 struct mini_Qdisc miniq1;
1322 struct mini_Qdisc miniq2;
1323 struct mini_Qdisc __rcu **p_miniq;
1324};
1325
1326void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp,
1327 struct tcf_proto *tp_head);
1328void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc,
1329 struct mini_Qdisc __rcu **p_miniq);
Paul Blakey7d17c542020-02-16 12:01:22 +02001330void mini_qdisc_pair_block_init(struct mini_Qdisc_pair *miniqp,
1331 struct tcf_block *block);
Jiri Pirko46209402017-11-03 11:46:25 +01001332
Jakub Kicinskif7116fb2021-09-17 06:55:06 -07001333void mq_change_real_num_tx(struct Qdisc *sch, unsigned int new_real_tx);
1334
wenxuc1294122020-11-25 12:01:23 +08001335int sch_frag_xmit_hook(struct sk_buff *skb, int (*xmit)(struct sk_buff *skb));
Paolo Abenicd11b1642018-07-30 14:30:44 +02001336
Linus Torvalds1da177e2005-04-16 15:20:36 -07001337#endif