blob: f8028d73edf1a78fff6cbdb313afb6e9ba87c502 [file] [log] [blame]
Thomas Gleixner2874c5f2019-05-27 08:55:01 +02001// SPDX-License-Identifier: GPL-2.0-or-later
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
3 * net/sched/cls_api.c Packet classifier API.
4 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07005 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
6 *
7 * Changes:
8 *
9 * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
Linus Torvalds1da177e2005-04-16 15:20:36 -070010 */
11
Linus Torvalds1da177e2005-04-16 15:20:36 -070012#include <linux/module.h>
13#include <linux/types.h>
14#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/string.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <linux/errno.h>
Jiri Pirko33a48922017-02-09 14:38:57 +010017#include <linux/err.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/skbuff.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <linux/init.h>
20#include <linux/kmod.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090021#include <linux/slab.h>
Jiri Pirko48617382018-01-17 11:46:46 +010022#include <linux/idr.h>
John Hurley7f76fa32018-11-09 21:21:26 -080023#include <linux/rhashtable.h>
John Hurley59eb87c2019-11-02 14:17:47 +000024#include <linux/jhash.h>
Paul Blakey43719292020-02-16 12:01:23 +020025#include <linux/rculist.h>
Denis V. Lunevb8542722007-12-01 00:21:31 +110026#include <net/net_namespace.h>
27#include <net/sock.h>
Arnaldo Carvalho de Melodc5fc572007-03-25 23:06:12 -070028#include <net/netlink.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#include <net/pkt_sched.h>
30#include <net/pkt_cls.h>
Pablo Neira Ayusoe3ab7862019-02-02 12:50:45 +010031#include <net/tc_act/tc_pedit.h>
Pablo Neira Ayuso3a7b6862019-02-02 12:50:46 +010032#include <net/tc_act/tc_mirred.h>
33#include <net/tc_act/tc_vlan.h>
34#include <net/tc_act/tc_tunnel_key.h>
35#include <net/tc_act/tc_csum.h>
36#include <net/tc_act/tc_gact.h>
Pieter Jansen van Vuuren8c8cfc62019-05-04 04:46:22 -070037#include <net/tc_act/tc_police.h>
Pieter Jansen van Vuurena7a7be62019-05-04 04:46:16 -070038#include <net/tc_act/tc_sample.h>
Pablo Neira Ayuso3a7b6862019-02-02 12:50:46 +010039#include <net/tc_act/tc_skbedit.h>
Paul Blakeyb57dc7c2019-07-09 10:30:48 +030040#include <net/tc_act/tc_ct.h>
John Hurley6749d5902019-07-23 15:33:59 +010041#include <net/tc_act/tc_mpls.h>
Po Liud29bdd62020-05-01 08:53:16 +080042#include <net/tc_act/tc_gate.h>
wenxu4e481902019-08-07 09:13:52 +080043#include <net/flow_offload.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070044
Davide Carattie3314732018-10-10 22:00:58 +020045extern const struct nla_policy rtm_tca_policy[TCA_MAX + 1];
46
Linus Torvalds1da177e2005-04-16 15:20:36 -070047/* The list of all installed classifier types */
WANG Cong36272872013-12-15 20:15:11 -080048static LIST_HEAD(tcf_proto_base);
Linus Torvalds1da177e2005-04-16 15:20:36 -070049
50/* Protects list of registered TC modules. It is pure SMP lock. */
51static DEFINE_RWLOCK(cls_mod_lock);
52
John Hurley59eb87c2019-11-02 14:17:47 +000053static u32 destroy_obj_hashfn(const struct tcf_proto *tp)
54{
55 return jhash_3words(tp->chain->index, tp->prio,
56 (__force __u32)tp->protocol, 0);
57}
58
59static void tcf_proto_signal_destroying(struct tcf_chain *chain,
60 struct tcf_proto *tp)
61{
62 struct tcf_block *block = chain->block;
63
64 mutex_lock(&block->proto_destroy_lock);
65 hash_add_rcu(block->proto_destroy_ht, &tp->destroy_ht_node,
66 destroy_obj_hashfn(tp));
67 mutex_unlock(&block->proto_destroy_lock);
68}
69
70static bool tcf_proto_cmp(const struct tcf_proto *tp1,
71 const struct tcf_proto *tp2)
72{
73 return tp1->chain->index == tp2->chain->index &&
74 tp1->prio == tp2->prio &&
75 tp1->protocol == tp2->protocol;
76}
77
78static bool tcf_proto_exists_destroying(struct tcf_chain *chain,
79 struct tcf_proto *tp)
80{
81 u32 hash = destroy_obj_hashfn(tp);
82 struct tcf_proto *iter;
83 bool found = false;
84
85 rcu_read_lock();
86 hash_for_each_possible_rcu(chain->block->proto_destroy_ht, iter,
87 destroy_ht_node, hash) {
88 if (tcf_proto_cmp(tp, iter)) {
89 found = true;
90 break;
91 }
92 }
93 rcu_read_unlock();
94
95 return found;
96}
97
98static void
99tcf_proto_signal_destroyed(struct tcf_chain *chain, struct tcf_proto *tp)
100{
101 struct tcf_block *block = chain->block;
102
103 mutex_lock(&block->proto_destroy_lock);
104 if (hash_hashed(&tp->destroy_ht_node))
105 hash_del_rcu(&tp->destroy_ht_node);
106 mutex_unlock(&block->proto_destroy_lock);
107}
108
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109/* Find classifier type by string name */
110
Jiri Pirkof34e8bf2018-07-23 09:23:04 +0200111static const struct tcf_proto_ops *__tcf_proto_lookup_ops(const char *kind)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112{
Eric Dumazetdcd76082013-12-20 10:04:18 -0800113 const struct tcf_proto_ops *t, *res = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114
115 if (kind) {
116 read_lock(&cls_mod_lock);
WANG Cong36272872013-12-15 20:15:11 -0800117 list_for_each_entry(t, &tcf_proto_base, head) {
Jiri Pirko33a48922017-02-09 14:38:57 +0100118 if (strcmp(kind, t->kind) == 0) {
Eric Dumazetdcd76082013-12-20 10:04:18 -0800119 if (try_module_get(t->owner))
120 res = t;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700121 break;
122 }
123 }
124 read_unlock(&cls_mod_lock);
125 }
Eric Dumazetdcd76082013-12-20 10:04:18 -0800126 return res;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700127}
128
Jiri Pirkof34e8bf2018-07-23 09:23:04 +0200129static const struct tcf_proto_ops *
Vlad Buslov12db03b2019-02-11 10:55:45 +0200130tcf_proto_lookup_ops(const char *kind, bool rtnl_held,
131 struct netlink_ext_ack *extack)
Jiri Pirkof34e8bf2018-07-23 09:23:04 +0200132{
133 const struct tcf_proto_ops *ops;
134
135 ops = __tcf_proto_lookup_ops(kind);
136 if (ops)
137 return ops;
138#ifdef CONFIG_MODULES
Vlad Buslov12db03b2019-02-11 10:55:45 +0200139 if (rtnl_held)
140 rtnl_unlock();
Jiri Pirkof34e8bf2018-07-23 09:23:04 +0200141 request_module("cls_%s", kind);
Vlad Buslov12db03b2019-02-11 10:55:45 +0200142 if (rtnl_held)
143 rtnl_lock();
Jiri Pirkof34e8bf2018-07-23 09:23:04 +0200144 ops = __tcf_proto_lookup_ops(kind);
145 /* We dropped the RTNL semaphore in order to perform
146 * the module load. So, even if we succeeded in loading
147 * the module we have to replay the request. We indicate
148 * this using -EAGAIN.
149 */
150 if (ops) {
151 module_put(ops->owner);
152 return ERR_PTR(-EAGAIN);
153 }
154#endif
155 NL_SET_ERR_MSG(extack, "TC classifier not found");
156 return ERR_PTR(-ENOENT);
157}
158
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159/* Register(unregister) new classifier type */
160
161int register_tcf_proto_ops(struct tcf_proto_ops *ops)
162{
WANG Cong36272872013-12-15 20:15:11 -0800163 struct tcf_proto_ops *t;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164 int rc = -EEXIST;
165
166 write_lock(&cls_mod_lock);
WANG Cong36272872013-12-15 20:15:11 -0800167 list_for_each_entry(t, &tcf_proto_base, head)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168 if (!strcmp(ops->kind, t->kind))
169 goto out;
170
WANG Cong36272872013-12-15 20:15:11 -0800171 list_add_tail(&ops->head, &tcf_proto_base);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172 rc = 0;
173out:
174 write_unlock(&cls_mod_lock);
175 return rc;
176}
Stephen Hemmingeraa767bf2008-01-21 02:26:41 -0800177EXPORT_SYMBOL(register_tcf_proto_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178
Cong Wang7aa00452017-10-26 18:24:28 -0700179static struct workqueue_struct *tc_filter_wq;
180
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181int unregister_tcf_proto_ops(struct tcf_proto_ops *ops)
182{
WANG Cong36272872013-12-15 20:15:11 -0800183 struct tcf_proto_ops *t;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184 int rc = -ENOENT;
185
Daniel Borkmannc78e1742015-05-20 17:13:33 +0200186 /* Wait for outstanding call_rcu()s, if any, from a
187 * tcf_proto_ops's destroy() handler.
188 */
189 rcu_barrier();
Cong Wang7aa00452017-10-26 18:24:28 -0700190 flush_workqueue(tc_filter_wq);
Daniel Borkmannc78e1742015-05-20 17:13:33 +0200191
Linus Torvalds1da177e2005-04-16 15:20:36 -0700192 write_lock(&cls_mod_lock);
Eric Dumazetdcd76082013-12-20 10:04:18 -0800193 list_for_each_entry(t, &tcf_proto_base, head) {
194 if (t == ops) {
195 list_del(&t->head);
196 rc = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700197 break;
Eric Dumazetdcd76082013-12-20 10:04:18 -0800198 }
199 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200 write_unlock(&cls_mod_lock);
201 return rc;
202}
Stephen Hemmingeraa767bf2008-01-21 02:26:41 -0800203EXPORT_SYMBOL(unregister_tcf_proto_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700204
Cong Wangaaa908f2018-05-23 15:26:53 -0700205bool tcf_queue_work(struct rcu_work *rwork, work_func_t func)
Cong Wang7aa00452017-10-26 18:24:28 -0700206{
Cong Wangaaa908f2018-05-23 15:26:53 -0700207 INIT_RCU_WORK(rwork, func);
208 return queue_rcu_work(tc_filter_wq, rwork);
Cong Wang7aa00452017-10-26 18:24:28 -0700209}
210EXPORT_SYMBOL(tcf_queue_work);
211
Linus Torvalds1da177e2005-04-16 15:20:36 -0700212/* Select new prio value from the range, managed by kernel. */
213
Stephen Hemmingeraa767bf2008-01-21 02:26:41 -0800214static inline u32 tcf_auto_prio(struct tcf_proto *tp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700215{
Stephen Hemmingeraa767bf2008-01-21 02:26:41 -0800216 u32 first = TC_H_MAKE(0xC0000000U, 0U);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217
218 if (tp)
Eric Dumazetcc7ec452011-01-19 19:26:56 +0000219 first = tp->prio - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220
Jiri Pirko79619732017-05-17 11:07:58 +0200221 return TC_H_MAJ(first);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700222}
223
Cong Wang6f96c3c2019-10-07 13:26:28 -0700224static bool tcf_proto_check_kind(struct nlattr *kind, char *name)
225{
226 if (kind)
227 return nla_strlcpy(name, kind, IFNAMSIZ) >= IFNAMSIZ;
228 memset(name, 0, IFNAMSIZ);
229 return false;
230}
231
Vlad Buslov470502d2019-02-11 10:55:48 +0200232static bool tcf_proto_is_unlocked(const char *kind)
233{
234 const struct tcf_proto_ops *ops;
235 bool ret;
236
Cong Wang6f96c3c2019-10-07 13:26:28 -0700237 if (strlen(kind) == 0)
238 return false;
239
Vlad Buslov470502d2019-02-11 10:55:48 +0200240 ops = tcf_proto_lookup_ops(kind, false, NULL);
241 /* On error return false to take rtnl lock. Proto lookup/create
242 * functions will perform lookup again and properly handle errors.
243 */
244 if (IS_ERR(ops))
245 return false;
246
247 ret = !!(ops->flags & TCF_PROTO_OPS_DOIT_UNLOCKED);
248 module_put(ops->owner);
249 return ret;
250}
251
Jiri Pirko33a48922017-02-09 14:38:57 +0100252static struct tcf_proto *tcf_proto_create(const char *kind, u32 protocol,
Alexander Aringc35a4ac2018-01-18 11:20:50 -0500253 u32 prio, struct tcf_chain *chain,
Vlad Buslov12db03b2019-02-11 10:55:45 +0200254 bool rtnl_held,
Alexander Aringc35a4ac2018-01-18 11:20:50 -0500255 struct netlink_ext_ack *extack)
Jiri Pirko33a48922017-02-09 14:38:57 +0100256{
257 struct tcf_proto *tp;
258 int err;
259
260 tp = kzalloc(sizeof(*tp), GFP_KERNEL);
261 if (!tp)
262 return ERR_PTR(-ENOBUFS);
263
Vlad Buslov12db03b2019-02-11 10:55:45 +0200264 tp->ops = tcf_proto_lookup_ops(kind, rtnl_held, extack);
Jiri Pirkof34e8bf2018-07-23 09:23:04 +0200265 if (IS_ERR(tp->ops)) {
266 err = PTR_ERR(tp->ops);
Jiri Pirkod68d75f2018-05-11 17:45:32 +0200267 goto errout;
Jiri Pirko33a48922017-02-09 14:38:57 +0100268 }
269 tp->classify = tp->ops->classify;
270 tp->protocol = protocol;
271 tp->prio = prio;
Jiri Pirko5bc17012017-05-17 11:08:01 +0200272 tp->chain = chain;
Vlad Buslov8b646782019-02-11 10:55:41 +0200273 spin_lock_init(&tp->lock);
Vlad Buslov4dbfa762019-02-11 10:55:39 +0200274 refcount_set(&tp->refcnt, 1);
Jiri Pirko33a48922017-02-09 14:38:57 +0100275
276 err = tp->ops->init(tp);
277 if (err) {
278 module_put(tp->ops->owner);
279 goto errout;
280 }
281 return tp;
282
283errout:
284 kfree(tp);
285 return ERR_PTR(err);
286}
287
Vlad Buslov4dbfa762019-02-11 10:55:39 +0200288static void tcf_proto_get(struct tcf_proto *tp)
289{
290 refcount_inc(&tp->refcnt);
291}
292
293static void tcf_chain_put(struct tcf_chain *chain);
294
Vlad Buslov12db03b2019-02-11 10:55:45 +0200295static void tcf_proto_destroy(struct tcf_proto *tp, bool rtnl_held,
John Hurley59eb87c2019-11-02 14:17:47 +0000296 bool sig_destroy, struct netlink_ext_ack *extack)
Jiri Pirkocf1facd2017-02-09 14:38:56 +0100297{
Vlad Buslov12db03b2019-02-11 10:55:45 +0200298 tp->ops->destroy(tp, rtnl_held, extack);
John Hurley59eb87c2019-11-02 14:17:47 +0000299 if (sig_destroy)
300 tcf_proto_signal_destroyed(tp->chain, tp);
Vlad Buslov4dbfa762019-02-11 10:55:39 +0200301 tcf_chain_put(tp->chain);
WANG Cong763dbf62017-04-19 14:21:21 -0700302 module_put(tp->ops->owner);
303 kfree_rcu(tp, rcu);
Jiri Pirkocf1facd2017-02-09 14:38:56 +0100304}
305
Vlad Buslov12db03b2019-02-11 10:55:45 +0200306static void tcf_proto_put(struct tcf_proto *tp, bool rtnl_held,
Vlad Buslov4dbfa762019-02-11 10:55:39 +0200307 struct netlink_ext_ack *extack)
308{
309 if (refcount_dec_and_test(&tp->refcnt))
John Hurley59eb87c2019-11-02 14:17:47 +0000310 tcf_proto_destroy(tp, rtnl_held, true, extack);
Vlad Buslov4dbfa762019-02-11 10:55:39 +0200311}
312
Davide Carattia5b72a02019-12-28 16:36:58 +0100313static bool tcf_proto_check_delete(struct tcf_proto *tp)
Vlad Buslov8b646782019-02-11 10:55:41 +0200314{
Davide Carattia5b72a02019-12-28 16:36:58 +0100315 if (tp->ops->delete_empty)
316 return tp->ops->delete_empty(tp);
Vlad Buslov8b646782019-02-11 10:55:41 +0200317
Davide Carattia5b72a02019-12-28 16:36:58 +0100318 tp->deleting = true;
Vlad Buslov8b646782019-02-11 10:55:41 +0200319 return tp->deleting;
320}
321
322static void tcf_proto_mark_delete(struct tcf_proto *tp)
323{
324 spin_lock(&tp->lock);
325 tp->deleting = true;
326 spin_unlock(&tp->lock);
327}
328
329static bool tcf_proto_is_deleting(struct tcf_proto *tp)
330{
331 bool deleting;
332
333 spin_lock(&tp->lock);
334 deleting = tp->deleting;
335 spin_unlock(&tp->lock);
336
337 return deleting;
338}
339
Vlad Buslovc266f642019-02-11 10:55:32 +0200340#define ASSERT_BLOCK_LOCKED(block) \
341 lockdep_assert_held(&(block)->lock)
342
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100343struct tcf_filter_chain_list_item {
344 struct list_head list;
345 tcf_chain_head_change_t *chain_head_change;
346 void *chain_head_change_priv;
347};
348
Jiri Pirko5bc17012017-05-17 11:08:01 +0200349static struct tcf_chain *tcf_chain_create(struct tcf_block *block,
350 u32 chain_index)
Jiri Pirko2190d1d2017-05-17 11:07:59 +0200351{
Jiri Pirko5bc17012017-05-17 11:08:01 +0200352 struct tcf_chain *chain;
353
Vlad Buslovc266f642019-02-11 10:55:32 +0200354 ASSERT_BLOCK_LOCKED(block);
355
Jiri Pirko5bc17012017-05-17 11:08:01 +0200356 chain = kzalloc(sizeof(*chain), GFP_KERNEL);
357 if (!chain)
358 return NULL;
Paul Blakey43719292020-02-16 12:01:23 +0200359 list_add_tail_rcu(&chain->list, &block->chain_list);
Vlad Busloved76f5e2019-02-11 10:55:38 +0200360 mutex_init(&chain->filter_chain_lock);
Jiri Pirko5bc17012017-05-17 11:08:01 +0200361 chain->block = block;
362 chain->index = chain_index;
Cong Wange2ef7542017-09-11 16:33:31 -0700363 chain->refcnt = 1;
Jiri Pirkof71e0ca42018-07-23 09:23:05 +0200364 if (!chain->index)
365 block->chain0.chain = chain;
Jiri Pirko5bc17012017-05-17 11:08:01 +0200366 return chain;
Jiri Pirko2190d1d2017-05-17 11:07:59 +0200367}
368
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100369static void tcf_chain_head_change_item(struct tcf_filter_chain_list_item *item,
370 struct tcf_proto *tp_head)
371{
372 if (item->chain_head_change)
373 item->chain_head_change(tp_head, item->chain_head_change_priv);
374}
Jiri Pirkof71e0ca42018-07-23 09:23:05 +0200375
376static void tcf_chain0_head_change(struct tcf_chain *chain,
377 struct tcf_proto *tp_head)
Jiri Pirkoc7eb7d72017-11-03 11:46:24 +0100378{
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100379 struct tcf_filter_chain_list_item *item;
Jiri Pirkof71e0ca42018-07-23 09:23:05 +0200380 struct tcf_block *block = chain->block;
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100381
Jiri Pirkof71e0ca42018-07-23 09:23:05 +0200382 if (chain->index)
383 return;
Vlad Buslov165f0132019-02-11 10:55:35 +0200384
385 mutex_lock(&block->lock);
Jiri Pirkof71e0ca42018-07-23 09:23:05 +0200386 list_for_each_entry(item, &block->chain0.filter_chain_list, list)
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100387 tcf_chain_head_change_item(item, tp_head);
Vlad Buslov165f0132019-02-11 10:55:35 +0200388 mutex_unlock(&block->lock);
Jiri Pirkoc7eb7d72017-11-03 11:46:24 +0100389}
390
Vlad Buslovc266f642019-02-11 10:55:32 +0200391/* Returns true if block can be safely freed. */
392
393static bool tcf_chain_detach(struct tcf_chain *chain)
Jiri Pirkof93e1cd2017-05-20 15:01:32 +0200394{
Cong Wangefbf7892017-12-04 10:48:18 -0800395 struct tcf_block *block = chain->block;
396
Vlad Buslovc266f642019-02-11 10:55:32 +0200397 ASSERT_BLOCK_LOCKED(block);
398
Paul Blakey43719292020-02-16 12:01:23 +0200399 list_del_rcu(&chain->list);
Jiri Pirkof71e0ca42018-07-23 09:23:05 +0200400 if (!chain->index)
401 block->chain0.chain = NULL;
Vlad Buslovc266f642019-02-11 10:55:32 +0200402
403 if (list_empty(&block->chain_list) &&
404 refcount_read(&block->refcnt) == 0)
405 return true;
406
407 return false;
408}
409
410static void tcf_block_destroy(struct tcf_block *block)
411{
412 mutex_destroy(&block->lock);
John Hurley59eb87c2019-11-02 14:17:47 +0000413 mutex_destroy(&block->proto_destroy_lock);
Vlad Buslovc266f642019-02-11 10:55:32 +0200414 kfree_rcu(block, rcu);
415}
416
417static void tcf_chain_destroy(struct tcf_chain *chain, bool free_block)
418{
419 struct tcf_block *block = chain->block;
420
Vlad Busloved76f5e2019-02-11 10:55:38 +0200421 mutex_destroy(&chain->filter_chain_lock);
Davide Carattiee3bbfe2019-03-20 15:00:16 +0100422 kfree_rcu(chain, rcu);
Vlad Buslovc266f642019-02-11 10:55:32 +0200423 if (free_block)
424 tcf_block_destroy(block);
Cong Wange2ef7542017-09-11 16:33:31 -0700425}
Jiri Pirko744a4cf2017-08-22 22:46:49 +0200426
Cong Wange2ef7542017-09-11 16:33:31 -0700427static void tcf_chain_hold(struct tcf_chain *chain)
428{
Vlad Buslovc266f642019-02-11 10:55:32 +0200429 ASSERT_BLOCK_LOCKED(chain->block);
430
Cong Wange2ef7542017-09-11 16:33:31 -0700431 ++chain->refcnt;
Jiri Pirko2190d1d2017-05-17 11:07:59 +0200432}
433
Jiri Pirko3d32f4c2018-08-01 12:36:55 +0200434static bool tcf_chain_held_by_acts_only(struct tcf_chain *chain)
Jiri Pirko1f3ed382018-07-27 09:45:05 +0200435{
Vlad Buslovc266f642019-02-11 10:55:32 +0200436 ASSERT_BLOCK_LOCKED(chain->block);
437
Jiri Pirko1f3ed382018-07-27 09:45:05 +0200438 /* In case all the references are action references, this
Jiri Pirko3d32f4c2018-08-01 12:36:55 +0200439 * chain should not be shown to the user.
Jiri Pirko1f3ed382018-07-27 09:45:05 +0200440 */
441 return chain->refcnt == chain->action_refcnt;
442}
443
Jiri Pirko32a4f5e2018-07-23 09:23:06 +0200444static struct tcf_chain *tcf_chain_lookup(struct tcf_block *block,
445 u32 chain_index)
Jiri Pirko5bc17012017-05-17 11:08:01 +0200446{
447 struct tcf_chain *chain;
448
Vlad Buslovc266f642019-02-11 10:55:32 +0200449 ASSERT_BLOCK_LOCKED(block);
450
Jiri Pirko5bc17012017-05-17 11:08:01 +0200451 list_for_each_entry(chain, &block->chain_list, list) {
Jiri Pirko32a4f5e2018-07-23 09:23:06 +0200452 if (chain->index == chain_index)
Cong Wange2ef7542017-09-11 16:33:31 -0700453 return chain;
Jiri Pirko32a4f5e2018-07-23 09:23:06 +0200454 }
455 return NULL;
456}
457
Paul Blakeyaf699622020-02-16 12:01:24 +0200458#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
459static struct tcf_chain *tcf_chain_lookup_rcu(const struct tcf_block *block,
460 u32 chain_index)
461{
462 struct tcf_chain *chain;
463
464 list_for_each_entry_rcu(chain, &block->chain_list, list) {
465 if (chain->index == chain_index)
466 return chain;
467 }
468 return NULL;
469}
470#endif
471
Jiri Pirko32a4f5e2018-07-23 09:23:06 +0200472static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
473 u32 seq, u16 flags, int event, bool unicast);
474
Jiri Pirko53681402018-08-01 12:36:56 +0200475static struct tcf_chain *__tcf_chain_get(struct tcf_block *block,
476 u32 chain_index, bool create,
477 bool by_act)
Jiri Pirko32a4f5e2018-07-23 09:23:06 +0200478{
Vlad Buslovc266f642019-02-11 10:55:32 +0200479 struct tcf_chain *chain = NULL;
480 bool is_first_reference;
Jiri Pirko32a4f5e2018-07-23 09:23:06 +0200481
Vlad Buslovc266f642019-02-11 10:55:32 +0200482 mutex_lock(&block->lock);
483 chain = tcf_chain_lookup(block, chain_index);
Jiri Pirko32a4f5e2018-07-23 09:23:06 +0200484 if (chain) {
485 tcf_chain_hold(chain);
Jiri Pirko53681402018-08-01 12:36:56 +0200486 } else {
487 if (!create)
Vlad Buslovc266f642019-02-11 10:55:32 +0200488 goto errout;
Jiri Pirko53681402018-08-01 12:36:56 +0200489 chain = tcf_chain_create(block, chain_index);
490 if (!chain)
Vlad Buslovc266f642019-02-11 10:55:32 +0200491 goto errout;
Jiri Pirko5bc17012017-05-17 11:08:01 +0200492 }
Jiri Pirko80532382017-09-06 13:14:19 +0200493
Jiri Pirko53681402018-08-01 12:36:56 +0200494 if (by_act)
495 ++chain->action_refcnt;
Vlad Buslovc266f642019-02-11 10:55:32 +0200496 is_first_reference = chain->refcnt - chain->action_refcnt == 1;
497 mutex_unlock(&block->lock);
Jiri Pirko53681402018-08-01 12:36:56 +0200498
499 /* Send notification only in case we got the first
500 * non-action reference. Until then, the chain acts only as
501 * a placeholder for actions pointing to it and user ought
502 * not know about them.
503 */
Vlad Buslovc266f642019-02-11 10:55:32 +0200504 if (is_first_reference && !by_act)
Jiri Pirko53681402018-08-01 12:36:56 +0200505 tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
506 RTM_NEWCHAIN, false);
507
Jiri Pirko32a4f5e2018-07-23 09:23:06 +0200508 return chain;
Vlad Buslovc266f642019-02-11 10:55:32 +0200509
510errout:
511 mutex_unlock(&block->lock);
512 return chain;
Jiri Pirko5bc17012017-05-17 11:08:01 +0200513}
Jiri Pirko53681402018-08-01 12:36:56 +0200514
Jiri Pirko290b1c82018-08-01 12:36:57 +0200515static struct tcf_chain *tcf_chain_get(struct tcf_block *block, u32 chain_index,
516 bool create)
Jiri Pirko53681402018-08-01 12:36:56 +0200517{
518 return __tcf_chain_get(block, chain_index, create, false);
519}
Jiri Pirko5bc17012017-05-17 11:08:01 +0200520
Jiri Pirko1f3ed382018-07-27 09:45:05 +0200521struct tcf_chain *tcf_chain_get_by_act(struct tcf_block *block, u32 chain_index)
522{
Jiri Pirko53681402018-08-01 12:36:56 +0200523 return __tcf_chain_get(block, chain_index, true, true);
Jiri Pirko1f3ed382018-07-27 09:45:05 +0200524}
525EXPORT_SYMBOL(tcf_chain_get_by_act);
526
Vlad Buslova5654822019-02-11 10:55:37 +0200527static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops,
528 void *tmplt_priv);
529static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops,
530 void *tmplt_priv, u32 chain_index,
531 struct tcf_block *block, struct sk_buff *oskb,
532 u32 seq, u16 flags, bool unicast);
Jiri Pirko9f407f12018-07-23 09:23:07 +0200533
Vlad Buslov91052fa2019-02-11 10:55:33 +0200534static void __tcf_chain_put(struct tcf_chain *chain, bool by_act,
535 bool explicitly_created)
Jiri Pirko5bc17012017-05-17 11:08:01 +0200536{
Vlad Buslovc266f642019-02-11 10:55:32 +0200537 struct tcf_block *block = chain->block;
Vlad Buslova5654822019-02-11 10:55:37 +0200538 const struct tcf_proto_ops *tmplt_ops;
Vlad Buslovb62989f2019-03-06 17:50:43 +0200539 bool free_block = false;
Vlad Buslovc266f642019-02-11 10:55:32 +0200540 unsigned int refcnt;
Vlad Buslova5654822019-02-11 10:55:37 +0200541 void *tmplt_priv;
Vlad Buslovc266f642019-02-11 10:55:32 +0200542
543 mutex_lock(&block->lock);
Vlad Buslov91052fa2019-02-11 10:55:33 +0200544 if (explicitly_created) {
545 if (!chain->explicitly_created) {
546 mutex_unlock(&block->lock);
547 return;
548 }
549 chain->explicitly_created = false;
550 }
551
Jiri Pirko53681402018-08-01 12:36:56 +0200552 if (by_act)
553 chain->action_refcnt--;
Vlad Buslovc266f642019-02-11 10:55:32 +0200554
555 /* tc_chain_notify_delete can't be called while holding block lock.
556 * However, when block is unlocked chain can be changed concurrently, so
557 * save these to temporary variables.
558 */
559 refcnt = --chain->refcnt;
Vlad Buslova5654822019-02-11 10:55:37 +0200560 tmplt_ops = chain->tmplt_ops;
561 tmplt_priv = chain->tmplt_priv;
Jiri Pirko53681402018-08-01 12:36:56 +0200562
563 /* The last dropped non-action reference will trigger notification. */
Vlad Buslovb62989f2019-03-06 17:50:43 +0200564 if (refcnt - chain->action_refcnt == 0 && !by_act) {
565 tc_chain_notify_delete(tmplt_ops, tmplt_priv, chain->index,
Vlad Buslova5654822019-02-11 10:55:37 +0200566 block, NULL, 0, 0, false);
Vlad Buslov726d06122019-02-11 10:55:42 +0200567 /* Last reference to chain, no need to lock. */
568 chain->flushing = false;
569 }
Jiri Pirko53681402018-08-01 12:36:56 +0200570
Vlad Buslovb62989f2019-03-06 17:50:43 +0200571 if (refcnt == 0)
572 free_block = tcf_chain_detach(chain);
573 mutex_unlock(&block->lock);
574
Vlad Buslovc266f642019-02-11 10:55:32 +0200575 if (refcnt == 0) {
Vlad Buslova5654822019-02-11 10:55:37 +0200576 tc_chain_tmplt_del(tmplt_ops, tmplt_priv);
Vlad Buslovc266f642019-02-11 10:55:32 +0200577 tcf_chain_destroy(chain, free_block);
Jiri Pirko32a4f5e2018-07-23 09:23:06 +0200578 }
Jiri Pirko5bc17012017-05-17 11:08:01 +0200579}
Jiri Pirko53681402018-08-01 12:36:56 +0200580
Jiri Pirko290b1c82018-08-01 12:36:57 +0200581static void tcf_chain_put(struct tcf_chain *chain)
Jiri Pirko53681402018-08-01 12:36:56 +0200582{
Vlad Buslov91052fa2019-02-11 10:55:33 +0200583 __tcf_chain_put(chain, false, false);
Jiri Pirko53681402018-08-01 12:36:56 +0200584}
Jiri Pirko5bc17012017-05-17 11:08:01 +0200585
Jiri Pirko1f3ed382018-07-27 09:45:05 +0200586void tcf_chain_put_by_act(struct tcf_chain *chain)
587{
Vlad Buslov91052fa2019-02-11 10:55:33 +0200588 __tcf_chain_put(chain, true, false);
Jiri Pirko1f3ed382018-07-27 09:45:05 +0200589}
590EXPORT_SYMBOL(tcf_chain_put_by_act);
591
Jiri Pirko32a4f5e2018-07-23 09:23:06 +0200592static void tcf_chain_put_explicitly_created(struct tcf_chain *chain)
593{
Vlad Buslov91052fa2019-02-11 10:55:33 +0200594 __tcf_chain_put(chain, false, true);
Jiri Pirko32a4f5e2018-07-23 09:23:06 +0200595}
596
Vlad Buslov12db03b2019-02-11 10:55:45 +0200597static void tcf_chain_flush(struct tcf_chain *chain, bool rtnl_held)
Jiri Pirko290b1c82018-08-01 12:36:57 +0200598{
Vlad Buslov4dbfa762019-02-11 10:55:39 +0200599 struct tcf_proto *tp, *tp_next;
Jiri Pirko290b1c82018-08-01 12:36:57 +0200600
Vlad Busloved76f5e2019-02-11 10:55:38 +0200601 mutex_lock(&chain->filter_chain_lock);
602 tp = tcf_chain_dereference(chain->filter_chain, chain);
John Hurley59eb87c2019-11-02 14:17:47 +0000603 while (tp) {
604 tp_next = rcu_dereference_protected(tp->next, 1);
605 tcf_proto_signal_destroying(chain, tp);
606 tp = tp_next;
607 }
608 tp = tcf_chain_dereference(chain->filter_chain, chain);
Vlad Buslov4dbfa762019-02-11 10:55:39 +0200609 RCU_INIT_POINTER(chain->filter_chain, NULL);
Jiri Pirko290b1c82018-08-01 12:36:57 +0200610 tcf_chain0_head_change(chain, NULL);
Vlad Buslov726d06122019-02-11 10:55:42 +0200611 chain->flushing = true;
Vlad Busloved76f5e2019-02-11 10:55:38 +0200612 mutex_unlock(&chain->filter_chain_lock);
613
Jiri Pirko290b1c82018-08-01 12:36:57 +0200614 while (tp) {
Vlad Buslov4dbfa762019-02-11 10:55:39 +0200615 tp_next = rcu_dereference_protected(tp->next, 1);
Vlad Buslov12db03b2019-02-11 10:55:45 +0200616 tcf_proto_put(tp, rtnl_held, NULL);
Vlad Buslov4dbfa762019-02-11 10:55:39 +0200617 tp = tp_next;
Jiri Pirko290b1c82018-08-01 12:36:57 +0200618 }
619}
620
wenxu4e481902019-08-07 09:13:52 +0800621static int tcf_block_setup(struct tcf_block *block,
622 struct flow_block_offload *bo);
623
Pablo Neira Ayuso324a8232020-05-29 02:25:36 +0200624static void tcf_block_offload_init(struct flow_block_offload *bo,
625 struct net_device *dev,
626 enum flow_block_command command,
627 enum flow_block_binder_type binder_type,
628 struct flow_block *flow_block,
629 bool shared, struct netlink_ext_ack *extack)
630{
631 bo->net = dev_net(dev);
632 bo->command = command;
633 bo->binder_type = binder_type;
634 bo->block = flow_block;
635 bo->block_shared = shared;
636 bo->extack = extack;
637 INIT_LIST_HEAD(&bo->cb_list);
638}
639
Pablo Neira Ayuso0fdcf782020-05-29 02:25:37 +0200640static void tcf_block_unbind(struct tcf_block *block,
641 struct flow_block_offload *bo);
John Hurley7f76fa32018-11-09 21:21:26 -0800642
Pablo Neira Ayuso0fdcf782020-05-29 02:25:37 +0200643static void tc_block_indr_cleanup(struct flow_block_cb *block_cb)
644{
645 struct tcf_block *block = block_cb->indr.data;
646 struct net_device *dev = block_cb->indr.dev;
647 struct netlink_ext_ack extack = {};
648 struct flow_block_offload bo;
649
650 tcf_block_offload_init(&bo, dev, FLOW_BLOCK_UNBIND,
651 block_cb->indr.binder_type,
652 &block->flow_block, tcf_block_shared(block),
653 &extack);
654 down_write(&block->cb_lock);
wenxua1db2172020-06-18 20:49:10 +0800655 list_del(&block_cb->driver_list);
Pablo Neira Ayuso0fdcf782020-05-29 02:25:37 +0200656 list_move(&block_cb->list, &bo.cb_list);
657 up_write(&block->cb_lock);
658 rtnl_lock();
659 tcf_block_unbind(block, &bo);
660 rtnl_unlock();
John Hurley7f76fa32018-11-09 21:21:26 -0800661}
662
Jiri Pirkocaa72602018-01-17 11:46:50 +0100663static bool tcf_block_offload_in_use(struct tcf_block *block)
Jiri Pirko8c4083b2017-10-19 15:50:29 +0200664{
Vlad Buslov97394be2019-08-26 16:44:58 +0300665 return atomic_read(&block->offloadcnt);
Jiri Pirkocaa72602018-01-17 11:46:50 +0100666}
667
668static int tcf_block_offload_cmd(struct tcf_block *block,
669 struct net_device *dev,
670 struct tcf_block_ext_info *ei,
Pablo Neira Ayuso9c0e1892019-07-09 22:55:40 +0200671 enum flow_block_command command,
John Hurley60513bd2018-06-25 14:30:04 -0700672 struct netlink_ext_ack *extack)
Jiri Pirkocaa72602018-01-17 11:46:50 +0100673{
Pablo Neira Ayuso955bcb62019-07-09 22:55:46 +0200674 struct flow_block_offload bo = {};
Pablo Neira Ayuso59094b12019-07-09 22:55:45 +0200675 int err;
Jiri Pirko8c4083b2017-10-19 15:50:29 +0200676
Pablo Neira Ayuso324a8232020-05-29 02:25:36 +0200677 tcf_block_offload_init(&bo, dev, command, ei->binder_type,
678 &block->flow_block, tcf_block_shared(block),
679 extack);
Pablo Neira Ayuso59094b12019-07-09 22:55:45 +0200680
Pablo Neira Ayuso0fdcf782020-05-29 02:25:37 +0200681 if (dev->netdev_ops->ndo_setup_tc)
682 err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
683 else
684 err = flow_indr_dev_setup_offload(dev, TC_SETUP_BLOCK, block,
685 &bo, tc_block_indr_cleanup);
686
Jesper Dangaard Brouerb70ba692020-04-23 16:57:45 +0200687 if (err < 0) {
688 if (err != -EOPNOTSUPP)
689 NL_SET_ERR_MSG(extack, "Driver ndo_setup_tc failed");
Pablo Neira Ayuso59094b12019-07-09 22:55:45 +0200690 return err;
Jesper Dangaard Brouerb70ba692020-04-23 16:57:45 +0200691 }
Pablo Neira Ayuso59094b12019-07-09 22:55:45 +0200692
693 return tcf_block_setup(block, &bo);
Jiri Pirko8c4083b2017-10-19 15:50:29 +0200694}
695
Jiri Pirkocaa72602018-01-17 11:46:50 +0100696static int tcf_block_offload_bind(struct tcf_block *block, struct Qdisc *q,
John Hurley60513bd2018-06-25 14:30:04 -0700697 struct tcf_block_ext_info *ei,
698 struct netlink_ext_ack *extack)
Jiri Pirko8c4083b2017-10-19 15:50:29 +0200699{
Jiri Pirkocaa72602018-01-17 11:46:50 +0100700 struct net_device *dev = q->dev_queue->dev;
701 int err;
702
Vlad Buslov4f8116c2019-08-26 16:44:57 +0300703 down_write(&block->cb_lock);
Jiri Pirkocaa72602018-01-17 11:46:50 +0100704
705 /* If tc offload feature is disabled and the block we try to bind
706 * to already has some offloaded filters, forbid to bind.
707 */
Pablo Neira Ayuso0fdcf782020-05-29 02:25:37 +0200708 if (dev->netdev_ops->ndo_setup_tc &&
709 !tc_can_offload(dev) &&
710 tcf_block_offload_in_use(block)) {
John Hurley60513bd2018-06-25 14:30:04 -0700711 NL_SET_ERR_MSG(extack, "Bind to offloaded block failed as dev has offload disabled");
Vlad Buslov4f8116c2019-08-26 16:44:57 +0300712 err = -EOPNOTSUPP;
713 goto err_unlock;
John Hurley60513bd2018-06-25 14:30:04 -0700714 }
Jiri Pirkocaa72602018-01-17 11:46:50 +0100715
Pablo Neira Ayuso9c0e1892019-07-09 22:55:40 +0200716 err = tcf_block_offload_cmd(block, dev, ei, FLOW_BLOCK_BIND, extack);
Jiri Pirkocaa72602018-01-17 11:46:50 +0100717 if (err == -EOPNOTSUPP)
718 goto no_offload_dev_inc;
John Hurley7f76fa32018-11-09 21:21:26 -0800719 if (err)
Vlad Buslov4f8116c2019-08-26 16:44:57 +0300720 goto err_unlock;
John Hurley7f76fa32018-11-09 21:21:26 -0800721
Vlad Buslov4f8116c2019-08-26 16:44:57 +0300722 up_write(&block->cb_lock);
John Hurley7f76fa32018-11-09 21:21:26 -0800723 return 0;
Jiri Pirkocaa72602018-01-17 11:46:50 +0100724
725no_offload_dev_inc:
Pablo Neira Ayuso0fdcf782020-05-29 02:25:37 +0200726 if (tcf_block_offload_in_use(block))
Vlad Buslov4f8116c2019-08-26 16:44:57 +0300727 goto err_unlock;
Pablo Neira Ayuso0fdcf782020-05-29 02:25:37 +0200728
Vlad Buslov4f8116c2019-08-26 16:44:57 +0300729 err = 0;
Jiri Pirkocaa72602018-01-17 11:46:50 +0100730 block->nooffloaddevcnt++;
Vlad Buslov4f8116c2019-08-26 16:44:57 +0300731err_unlock:
732 up_write(&block->cb_lock);
733 return err;
Jiri Pirko8c4083b2017-10-19 15:50:29 +0200734}
735
736static void tcf_block_offload_unbind(struct tcf_block *block, struct Qdisc *q,
737 struct tcf_block_ext_info *ei)
738{
Jiri Pirkocaa72602018-01-17 11:46:50 +0100739 struct net_device *dev = q->dev_queue->dev;
740 int err;
741
Vlad Buslov4f8116c2019-08-26 16:44:57 +0300742 down_write(&block->cb_lock);
Pablo Neira Ayuso9c0e1892019-07-09 22:55:40 +0200743 err = tcf_block_offload_cmd(block, dev, ei, FLOW_BLOCK_UNBIND, NULL);
Jiri Pirkocaa72602018-01-17 11:46:50 +0100744 if (err == -EOPNOTSUPP)
745 goto no_offload_dev_dec;
Vlad Buslov4f8116c2019-08-26 16:44:57 +0300746 up_write(&block->cb_lock);
Jiri Pirkocaa72602018-01-17 11:46:50 +0100747 return;
748
749no_offload_dev_dec:
750 WARN_ON(block->nooffloaddevcnt-- == 0);
Vlad Buslov4f8116c2019-08-26 16:44:57 +0300751 up_write(&block->cb_lock);
Jiri Pirko8c4083b2017-10-19 15:50:29 +0200752}
753
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100754static int
Jiri Pirkof71e0ca42018-07-23 09:23:05 +0200755tcf_chain0_head_change_cb_add(struct tcf_block *block,
756 struct tcf_block_ext_info *ei,
757 struct netlink_ext_ack *extack)
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100758{
759 struct tcf_filter_chain_list_item *item;
Vlad Buslov165f0132019-02-11 10:55:35 +0200760 struct tcf_chain *chain0;
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100761
762 item = kmalloc(sizeof(*item), GFP_KERNEL);
763 if (!item) {
764 NL_SET_ERR_MSG(extack, "Memory allocation for head change callback item failed");
765 return -ENOMEM;
766 }
767 item->chain_head_change = ei->chain_head_change;
768 item->chain_head_change_priv = ei->chain_head_change_priv;
Vlad Buslov165f0132019-02-11 10:55:35 +0200769
770 mutex_lock(&block->lock);
771 chain0 = block->chain0.chain;
Vlad Busloved76f5e2019-02-11 10:55:38 +0200772 if (chain0)
773 tcf_chain_hold(chain0);
774 else
775 list_add(&item->list, &block->chain0.filter_chain_list);
Vlad Buslov165f0132019-02-11 10:55:35 +0200776 mutex_unlock(&block->lock);
777
Vlad Busloved76f5e2019-02-11 10:55:38 +0200778 if (chain0) {
779 struct tcf_proto *tp_head;
780
781 mutex_lock(&chain0->filter_chain_lock);
782
783 tp_head = tcf_chain_dereference(chain0->filter_chain, chain0);
784 if (tp_head)
785 tcf_chain_head_change_item(item, tp_head);
786
787 mutex_lock(&block->lock);
788 list_add(&item->list, &block->chain0.filter_chain_list);
789 mutex_unlock(&block->lock);
790
791 mutex_unlock(&chain0->filter_chain_lock);
792 tcf_chain_put(chain0);
793 }
794
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100795 return 0;
796}
797
798static void
Jiri Pirkof71e0ca42018-07-23 09:23:05 +0200799tcf_chain0_head_change_cb_del(struct tcf_block *block,
800 struct tcf_block_ext_info *ei)
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100801{
802 struct tcf_filter_chain_list_item *item;
803
Vlad Buslov165f0132019-02-11 10:55:35 +0200804 mutex_lock(&block->lock);
Jiri Pirkof71e0ca42018-07-23 09:23:05 +0200805 list_for_each_entry(item, &block->chain0.filter_chain_list, list) {
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100806 if ((!ei->chain_head_change && !ei->chain_head_change_priv) ||
807 (item->chain_head_change == ei->chain_head_change &&
808 item->chain_head_change_priv == ei->chain_head_change_priv)) {
Vlad Buslov165f0132019-02-11 10:55:35 +0200809 if (block->chain0.chain)
Jiri Pirkof71e0ca42018-07-23 09:23:05 +0200810 tcf_chain_head_change_item(item, NULL);
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100811 list_del(&item->list);
Vlad Buslov165f0132019-02-11 10:55:35 +0200812 mutex_unlock(&block->lock);
813
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100814 kfree(item);
815 return;
816 }
817 }
Vlad Buslov165f0132019-02-11 10:55:35 +0200818 mutex_unlock(&block->lock);
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100819 WARN_ON(1);
820}
821
Jiri Pirko48617382018-01-17 11:46:46 +0100822struct tcf_net {
Vlad Buslovab281622018-09-24 19:22:56 +0300823 spinlock_t idr_lock; /* Protects idr */
Jiri Pirko48617382018-01-17 11:46:46 +0100824 struct idr idr;
825};
826
827static unsigned int tcf_net_id;
828
829static int tcf_block_insert(struct tcf_block *block, struct net *net,
Jiri Pirkobb047dd2018-02-13 12:00:16 +0100830 struct netlink_ext_ack *extack)
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100831{
Jiri Pirko48617382018-01-17 11:46:46 +0100832 struct tcf_net *tn = net_generic(net, tcf_net_id);
Vlad Buslovab281622018-09-24 19:22:56 +0300833 int err;
Jiri Pirko48617382018-01-17 11:46:46 +0100834
Vlad Buslovab281622018-09-24 19:22:56 +0300835 idr_preload(GFP_KERNEL);
836 spin_lock(&tn->idr_lock);
837 err = idr_alloc_u32(&tn->idr, block, &block->index, block->index,
838 GFP_NOWAIT);
839 spin_unlock(&tn->idr_lock);
840 idr_preload_end();
841
842 return err;
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100843}
844
Jiri Pirko48617382018-01-17 11:46:46 +0100845static void tcf_block_remove(struct tcf_block *block, struct net *net)
Jiri Pirko6529eab2017-05-17 11:07:55 +0200846{
Jiri Pirko48617382018-01-17 11:46:46 +0100847 struct tcf_net *tn = net_generic(net, tcf_net_id);
848
Vlad Buslovab281622018-09-24 19:22:56 +0300849 spin_lock(&tn->idr_lock);
Matthew Wilcox9c160942017-11-28 09:48:43 -0500850 idr_remove(&tn->idr, block->index);
Vlad Buslovab281622018-09-24 19:22:56 +0300851 spin_unlock(&tn->idr_lock);
Jiri Pirko48617382018-01-17 11:46:46 +0100852}
853
854static struct tcf_block *tcf_block_create(struct net *net, struct Qdisc *q,
Jiri Pirkobb047dd2018-02-13 12:00:16 +0100855 u32 block_index,
Jiri Pirko48617382018-01-17 11:46:46 +0100856 struct netlink_ext_ack *extack)
857{
858 struct tcf_block *block;
Jiri Pirko6529eab2017-05-17 11:07:55 +0200859
Jiri Pirko48617382018-01-17 11:46:46 +0100860 block = kzalloc(sizeof(*block), GFP_KERNEL);
Alexander Aring8d1a77f2017-12-20 12:35:19 -0500861 if (!block) {
862 NL_SET_ERR_MSG(extack, "Memory allocation for block failed");
Jiri Pirko48617382018-01-17 11:46:46 +0100863 return ERR_PTR(-ENOMEM);
Alexander Aring8d1a77f2017-12-20 12:35:19 -0500864 }
Vlad Buslovc266f642019-02-11 10:55:32 +0200865 mutex_init(&block->lock);
John Hurley59eb87c2019-11-02 14:17:47 +0000866 mutex_init(&block->proto_destroy_lock);
Vlad Buslov4f8116c2019-08-26 16:44:57 +0300867 init_rwsem(&block->cb_lock);
Pablo Neira Ayuso14bfb132019-07-19 18:20:16 +0200868 flow_block_init(&block->flow_block);
Jiri Pirko5bc17012017-05-17 11:08:01 +0200869 INIT_LIST_HEAD(&block->chain_list);
Jiri Pirkof36fe1c2018-01-17 11:46:48 +0100870 INIT_LIST_HEAD(&block->owner_list);
Jiri Pirkof71e0ca42018-07-23 09:23:05 +0200871 INIT_LIST_HEAD(&block->chain0.filter_chain_list);
Jiri Pirkoacb67442017-10-19 15:50:31 +0200872
Vlad Buslovcfebd7e2018-09-24 19:22:54 +0300873 refcount_set(&block->refcnt, 1);
Jiri Pirko48617382018-01-17 11:46:46 +0100874 block->net = net;
Jiri Pirkobb047dd2018-02-13 12:00:16 +0100875 block->index = block_index;
876
877 /* Don't store q pointer for blocks which are shared */
878 if (!tcf_block_shared(block))
879 block->q = q;
Jiri Pirko48617382018-01-17 11:46:46 +0100880 return block;
Jiri Pirko48617382018-01-17 11:46:46 +0100881}
882
883static struct tcf_block *tcf_block_lookup(struct net *net, u32 block_index)
884{
885 struct tcf_net *tn = net_generic(net, tcf_net_id);
886
Matthew Wilcox322d8842017-11-28 10:01:24 -0500887 return idr_find(&tn->idr, block_index);
Jiri Pirko48617382018-01-17 11:46:46 +0100888}
889
Vlad Buslov0607e432018-09-24 19:22:57 +0300890static struct tcf_block *tcf_block_refcnt_get(struct net *net, u32 block_index)
891{
892 struct tcf_block *block;
893
894 rcu_read_lock();
895 block = tcf_block_lookup(net, block_index);
896 if (block && !refcount_inc_not_zero(&block->refcnt))
897 block = NULL;
898 rcu_read_unlock();
899
900 return block;
901}
902
Vlad Buslovbbf73832019-02-11 10:55:36 +0200903static struct tcf_chain *
904__tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain)
905{
906 mutex_lock(&block->lock);
907 if (chain)
908 chain = list_is_last(&chain->list, &block->chain_list) ?
909 NULL : list_next_entry(chain, list);
910 else
911 chain = list_first_entry_or_null(&block->chain_list,
912 struct tcf_chain, list);
913
914 /* skip all action-only chains */
915 while (chain && tcf_chain_held_by_acts_only(chain))
916 chain = list_is_last(&chain->list, &block->chain_list) ?
917 NULL : list_next_entry(chain, list);
918
919 if (chain)
920 tcf_chain_hold(chain);
921 mutex_unlock(&block->lock);
922
923 return chain;
924}
925
926/* Function to be used by all clients that want to iterate over all chains on
927 * block. It properly obtains block->lock and takes reference to chain before
928 * returning it. Users of this function must be tolerant to concurrent chain
929 * insertion/deletion or ensure that no concurrent chain modification is
930 * possible. Note that all netlink dump callbacks cannot guarantee to provide
931 * consistent dump because rtnl lock is released each time skb is filled with
932 * data and sent to user-space.
933 */
934
935struct tcf_chain *
936tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain)
937{
938 struct tcf_chain *chain_next = __tcf_get_next_chain(block, chain);
939
940 if (chain)
941 tcf_chain_put(chain);
942
943 return chain_next;
944}
945EXPORT_SYMBOL(tcf_get_next_chain);
946
Vlad Buslovfe2923a2019-02-11 10:55:40 +0200947static struct tcf_proto *
948__tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp)
949{
Vlad Buslov8b646782019-02-11 10:55:41 +0200950 u32 prio = 0;
951
Vlad Buslovfe2923a2019-02-11 10:55:40 +0200952 ASSERT_RTNL();
953 mutex_lock(&chain->filter_chain_lock);
954
Vlad Buslov8b646782019-02-11 10:55:41 +0200955 if (!tp) {
Vlad Buslovfe2923a2019-02-11 10:55:40 +0200956 tp = tcf_chain_dereference(chain->filter_chain, chain);
Vlad Buslov8b646782019-02-11 10:55:41 +0200957 } else if (tcf_proto_is_deleting(tp)) {
958 /* 'deleting' flag is set and chain->filter_chain_lock was
959 * unlocked, which means next pointer could be invalid. Restart
960 * search.
961 */
962 prio = tp->prio + 1;
963 tp = tcf_chain_dereference(chain->filter_chain, chain);
964
965 for (; tp; tp = tcf_chain_dereference(tp->next, chain))
966 if (!tp->deleting && tp->prio >= prio)
967 break;
968 } else {
Vlad Buslovfe2923a2019-02-11 10:55:40 +0200969 tp = tcf_chain_dereference(tp->next, chain);
Vlad Buslov8b646782019-02-11 10:55:41 +0200970 }
Vlad Buslovfe2923a2019-02-11 10:55:40 +0200971
972 if (tp)
973 tcf_proto_get(tp);
974
975 mutex_unlock(&chain->filter_chain_lock);
976
977 return tp;
978}
979
980/* Function to be used by all clients that want to iterate over all tp's on
981 * chain. Users of this function must be tolerant to concurrent tp
982 * insertion/deletion or ensure that no concurrent chain modification is
983 * possible. Note that all netlink dump callbacks cannot guarantee to provide
984 * consistent dump because rtnl lock is released each time skb is filled with
985 * data and sent to user-space.
986 */
987
988struct tcf_proto *
Vlad Buslov12db03b2019-02-11 10:55:45 +0200989tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp,
990 bool rtnl_held)
Vlad Buslovfe2923a2019-02-11 10:55:40 +0200991{
992 struct tcf_proto *tp_next = __tcf_get_next_proto(chain, tp);
993
994 if (tp)
Vlad Buslov12db03b2019-02-11 10:55:45 +0200995 tcf_proto_put(tp, rtnl_held, NULL);
Vlad Buslovfe2923a2019-02-11 10:55:40 +0200996
997 return tp_next;
998}
999EXPORT_SYMBOL(tcf_get_next_proto);
1000
Vlad Buslov12db03b2019-02-11 10:55:45 +02001001static void tcf_block_flush_all_chains(struct tcf_block *block, bool rtnl_held)
Vlad Buslovf0023432018-09-24 19:22:55 +03001002{
1003 struct tcf_chain *chain;
1004
Vlad Buslovbbf73832019-02-11 10:55:36 +02001005 /* Last reference to block. At this point chains cannot be added or
1006 * removed concurrently.
Vlad Buslovf0023432018-09-24 19:22:55 +03001007 */
Vlad Buslovbbf73832019-02-11 10:55:36 +02001008 for (chain = tcf_get_next_chain(block, NULL);
1009 chain;
1010 chain = tcf_get_next_chain(block, chain)) {
Vlad Buslovf0023432018-09-24 19:22:55 +03001011 tcf_chain_put_explicitly_created(chain);
Vlad Buslov12db03b2019-02-11 10:55:45 +02001012 tcf_chain_flush(chain, rtnl_held);
Vlad Buslovf0023432018-09-24 19:22:55 +03001013 }
1014}
1015
Vlad Buslov18d3eef2019-02-11 10:55:47 +02001016/* Lookup Qdisc and increments its reference counter.
1017 * Set parent, if necessary.
1018 */
1019
1020static int __tcf_qdisc_find(struct net *net, struct Qdisc **q,
1021 u32 *parent, int ifindex, bool rtnl_held,
1022 struct netlink_ext_ack *extack)
1023{
1024 const struct Qdisc_class_ops *cops;
1025 struct net_device *dev;
1026 int err = 0;
1027
1028 if (ifindex == TCM_IFINDEX_MAGIC_BLOCK)
1029 return 0;
1030
1031 rcu_read_lock();
1032
1033 /* Find link */
1034 dev = dev_get_by_index_rcu(net, ifindex);
1035 if (!dev) {
1036 rcu_read_unlock();
1037 return -ENODEV;
1038 }
1039
1040 /* Find qdisc */
1041 if (!*parent) {
1042 *q = dev->qdisc;
1043 *parent = (*q)->handle;
1044 } else {
1045 *q = qdisc_lookup_rcu(dev, TC_H_MAJ(*parent));
1046 if (!*q) {
1047 NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
1048 err = -EINVAL;
1049 goto errout_rcu;
1050 }
1051 }
1052
1053 *q = qdisc_refcount_inc_nz(*q);
1054 if (!*q) {
1055 NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
1056 err = -EINVAL;
1057 goto errout_rcu;
1058 }
1059
1060 /* Is it classful? */
1061 cops = (*q)->ops->cl_ops;
1062 if (!cops) {
1063 NL_SET_ERR_MSG(extack, "Qdisc not classful");
1064 err = -EINVAL;
1065 goto errout_qdisc;
1066 }
1067
1068 if (!cops->tcf_block) {
1069 NL_SET_ERR_MSG(extack, "Class doesn't support blocks");
1070 err = -EOPNOTSUPP;
1071 goto errout_qdisc;
1072 }
1073
1074errout_rcu:
1075 /* At this point we know that qdisc is not noop_qdisc,
1076 * which means that qdisc holds a reference to net_device
1077 * and we hold a reference to qdisc, so it is safe to release
1078 * rcu read lock.
1079 */
1080 rcu_read_unlock();
1081 return err;
1082
1083errout_qdisc:
1084 rcu_read_unlock();
1085
1086 if (rtnl_held)
1087 qdisc_put(*q);
1088 else
1089 qdisc_put_unlocked(*q);
1090 *q = NULL;
1091
1092 return err;
1093}
1094
1095static int __tcf_qdisc_cl_find(struct Qdisc *q, u32 parent, unsigned long *cl,
1096 int ifindex, struct netlink_ext_ack *extack)
1097{
1098 if (ifindex == TCM_IFINDEX_MAGIC_BLOCK)
1099 return 0;
1100
1101 /* Do we search for filter, attached to class? */
1102 if (TC_H_MIN(parent)) {
1103 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1104
1105 *cl = cops->find(q, parent);
1106 if (*cl == 0) {
1107 NL_SET_ERR_MSG(extack, "Specified class doesn't exist");
1108 return -ENOENT;
1109 }
1110 }
1111
1112 return 0;
1113}
1114
1115static struct tcf_block *__tcf_block_find(struct net *net, struct Qdisc *q,
1116 unsigned long cl, int ifindex,
1117 u32 block_index,
1118 struct netlink_ext_ack *extack)
1119{
1120 struct tcf_block *block;
1121
1122 if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
1123 block = tcf_block_refcnt_get(net, block_index);
1124 if (!block) {
1125 NL_SET_ERR_MSG(extack, "Block of given index was not found");
1126 return ERR_PTR(-EINVAL);
1127 }
1128 } else {
1129 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1130
1131 block = cops->tcf_block(q, cl, extack);
1132 if (!block)
1133 return ERR_PTR(-EINVAL);
1134
1135 if (tcf_block_shared(block)) {
1136 NL_SET_ERR_MSG(extack, "This filter block is shared. Please use the block index to manipulate the filters");
1137 return ERR_PTR(-EOPNOTSUPP);
1138 }
1139
1140 /* Always take reference to block in order to support execution
1141 * of rules update path of cls API without rtnl lock. Caller
1142 * must release block when it is finished using it. 'if' block
1143 * of this conditional obtain reference to block by calling
1144 * tcf_block_refcnt_get().
1145 */
1146 refcount_inc(&block->refcnt);
1147 }
1148
1149 return block;
1150}
1151
Vlad Buslov0607e432018-09-24 19:22:57 +03001152static void __tcf_block_put(struct tcf_block *block, struct Qdisc *q,
Vlad Buslov12db03b2019-02-11 10:55:45 +02001153 struct tcf_block_ext_info *ei, bool rtnl_held)
Vlad Buslov0607e432018-09-24 19:22:57 +03001154{
Vlad Buslovc266f642019-02-11 10:55:32 +02001155 if (refcount_dec_and_mutex_lock(&block->refcnt, &block->lock)) {
Vlad Buslov0607e432018-09-24 19:22:57 +03001156 /* Flushing/putting all chains will cause the block to be
1157 * deallocated when last chain is freed. However, if chain_list
1158 * is empty, block has to be manually deallocated. After block
1159 * reference counter reached 0, it is no longer possible to
1160 * increment it or add new chains to block.
1161 */
1162 bool free_block = list_empty(&block->chain_list);
1163
Vlad Buslovc266f642019-02-11 10:55:32 +02001164 mutex_unlock(&block->lock);
Vlad Buslov0607e432018-09-24 19:22:57 +03001165 if (tcf_block_shared(block))
1166 tcf_block_remove(block, block->net);
Vlad Buslov0607e432018-09-24 19:22:57 +03001167
1168 if (q)
1169 tcf_block_offload_unbind(block, q, ei);
1170
1171 if (free_block)
Vlad Buslovc266f642019-02-11 10:55:32 +02001172 tcf_block_destroy(block);
Vlad Buslov0607e432018-09-24 19:22:57 +03001173 else
Vlad Buslov12db03b2019-02-11 10:55:45 +02001174 tcf_block_flush_all_chains(block, rtnl_held);
Vlad Buslov0607e432018-09-24 19:22:57 +03001175 } else if (q) {
1176 tcf_block_offload_unbind(block, q, ei);
1177 }
1178}
1179
Vlad Buslov12db03b2019-02-11 10:55:45 +02001180static void tcf_block_refcnt_put(struct tcf_block *block, bool rtnl_held)
Vlad Buslov0607e432018-09-24 19:22:57 +03001181{
Vlad Buslov12db03b2019-02-11 10:55:45 +02001182 __tcf_block_put(block, NULL, NULL, rtnl_held);
Vlad Buslov0607e432018-09-24 19:22:57 +03001183}
1184
Vlad Buslovc431f892018-05-31 09:52:53 +03001185/* Find tcf block.
1186 * Set q, parent, cl when appropriate.
1187 */
1188
1189static struct tcf_block *tcf_block_find(struct net *net, struct Qdisc **q,
1190 u32 *parent, unsigned long *cl,
1191 int ifindex, u32 block_index,
1192 struct netlink_ext_ack *extack)
1193{
1194 struct tcf_block *block;
Vlad Buslove368fdb2018-09-24 19:22:53 +03001195 int err = 0;
Vlad Buslovc431f892018-05-31 09:52:53 +03001196
Vlad Buslov18d3eef2019-02-11 10:55:47 +02001197 ASSERT_RTNL();
Vlad Buslovc431f892018-05-31 09:52:53 +03001198
Vlad Buslov18d3eef2019-02-11 10:55:47 +02001199 err = __tcf_qdisc_find(net, q, parent, ifindex, true, extack);
1200 if (err)
1201 goto errout;
Vlad Buslove368fdb2018-09-24 19:22:53 +03001202
Vlad Buslov18d3eef2019-02-11 10:55:47 +02001203 err = __tcf_qdisc_cl_find(*q, *parent, cl, ifindex, extack);
1204 if (err)
1205 goto errout_qdisc;
Vlad Buslovc431f892018-05-31 09:52:53 +03001206
Vlad Buslov18d3eef2019-02-11 10:55:47 +02001207 block = __tcf_block_find(net, *q, *cl, ifindex, block_index, extack);
Dan Carpenteraf736bf2019-02-18 12:26:32 +03001208 if (IS_ERR(block)) {
1209 err = PTR_ERR(block);
Vlad Buslov18d3eef2019-02-11 10:55:47 +02001210 goto errout_qdisc;
Dan Carpenteraf736bf2019-02-18 12:26:32 +03001211 }
Vlad Buslovc431f892018-05-31 09:52:53 +03001212
1213 return block;
Vlad Buslove368fdb2018-09-24 19:22:53 +03001214
Vlad Buslove368fdb2018-09-24 19:22:53 +03001215errout_qdisc:
Vlad Buslov18d3eef2019-02-11 10:55:47 +02001216 if (*q)
Vlad Buslove368fdb2018-09-24 19:22:53 +03001217 qdisc_put(*q);
Vlad Buslov18d3eef2019-02-11 10:55:47 +02001218errout:
1219 *q = NULL;
Vlad Buslove368fdb2018-09-24 19:22:53 +03001220 return ERR_PTR(err);
1221}
1222
Vlad Buslov12db03b2019-02-11 10:55:45 +02001223static void tcf_block_release(struct Qdisc *q, struct tcf_block *block,
1224 bool rtnl_held)
Vlad Buslove368fdb2018-09-24 19:22:53 +03001225{
Vlad Buslov787ce6d2018-09-24 19:22:58 +03001226 if (!IS_ERR_OR_NULL(block))
Vlad Buslov12db03b2019-02-11 10:55:45 +02001227 tcf_block_refcnt_put(block, rtnl_held);
Vlad Buslov787ce6d2018-09-24 19:22:58 +03001228
Vlad Buslov470502d2019-02-11 10:55:48 +02001229 if (q) {
1230 if (rtnl_held)
1231 qdisc_put(q);
1232 else
1233 qdisc_put_unlocked(q);
1234 }
Vlad Buslovc431f892018-05-31 09:52:53 +03001235}
1236
Jiri Pirkof36fe1c2018-01-17 11:46:48 +01001237struct tcf_block_owner_item {
1238 struct list_head list;
1239 struct Qdisc *q;
Pablo Neira Ayuso32f8c402019-07-09 22:55:41 +02001240 enum flow_block_binder_type binder_type;
Jiri Pirkof36fe1c2018-01-17 11:46:48 +01001241};
1242
1243static void
1244tcf_block_owner_netif_keep_dst(struct tcf_block *block,
1245 struct Qdisc *q,
Pablo Neira Ayuso32f8c402019-07-09 22:55:41 +02001246 enum flow_block_binder_type binder_type)
Jiri Pirkof36fe1c2018-01-17 11:46:48 +01001247{
1248 if (block->keep_dst &&
Pablo Neira Ayuso32f8c402019-07-09 22:55:41 +02001249 binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS &&
1250 binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS)
Jiri Pirkof36fe1c2018-01-17 11:46:48 +01001251 netif_keep_dst(qdisc_dev(q));
1252}
1253
1254void tcf_block_netif_keep_dst(struct tcf_block *block)
1255{
1256 struct tcf_block_owner_item *item;
1257
1258 block->keep_dst = true;
1259 list_for_each_entry(item, &block->owner_list, list)
1260 tcf_block_owner_netif_keep_dst(block, item->q,
1261 item->binder_type);
1262}
1263EXPORT_SYMBOL(tcf_block_netif_keep_dst);
1264
1265static int tcf_block_owner_add(struct tcf_block *block,
1266 struct Qdisc *q,
Pablo Neira Ayuso32f8c402019-07-09 22:55:41 +02001267 enum flow_block_binder_type binder_type)
Jiri Pirkof36fe1c2018-01-17 11:46:48 +01001268{
1269 struct tcf_block_owner_item *item;
1270
1271 item = kmalloc(sizeof(*item), GFP_KERNEL);
1272 if (!item)
1273 return -ENOMEM;
1274 item->q = q;
1275 item->binder_type = binder_type;
1276 list_add(&item->list, &block->owner_list);
1277 return 0;
1278}
1279
1280static void tcf_block_owner_del(struct tcf_block *block,
1281 struct Qdisc *q,
Pablo Neira Ayuso32f8c402019-07-09 22:55:41 +02001282 enum flow_block_binder_type binder_type)
Jiri Pirkof36fe1c2018-01-17 11:46:48 +01001283{
1284 struct tcf_block_owner_item *item;
1285
1286 list_for_each_entry(item, &block->owner_list, list) {
1287 if (item->q == q && item->binder_type == binder_type) {
1288 list_del(&item->list);
1289 kfree(item);
1290 return;
1291 }
1292 }
1293 WARN_ON(1);
1294}
1295
Jiri Pirko48617382018-01-17 11:46:46 +01001296int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,
1297 struct tcf_block_ext_info *ei,
1298 struct netlink_ext_ack *extack)
1299{
1300 struct net *net = qdisc_net(q);
1301 struct tcf_block *block = NULL;
Jiri Pirko48617382018-01-17 11:46:46 +01001302 int err;
1303
Vlad Buslov787ce6d2018-09-24 19:22:58 +03001304 if (ei->block_index)
Jiri Pirko48617382018-01-17 11:46:46 +01001305 /* block_index not 0 means the shared block is requested */
Vlad Buslov787ce6d2018-09-24 19:22:58 +03001306 block = tcf_block_refcnt_get(net, ei->block_index);
Jiri Pirko48617382018-01-17 11:46:46 +01001307
1308 if (!block) {
Jiri Pirkobb047dd2018-02-13 12:00:16 +01001309 block = tcf_block_create(net, q, ei->block_index, extack);
Jiri Pirko48617382018-01-17 11:46:46 +01001310 if (IS_ERR(block))
1311 return PTR_ERR(block);
Jiri Pirkobb047dd2018-02-13 12:00:16 +01001312 if (tcf_block_shared(block)) {
1313 err = tcf_block_insert(block, net, extack);
Jiri Pirko48617382018-01-17 11:46:46 +01001314 if (err)
1315 goto err_block_insert;
1316 }
1317 }
1318
Jiri Pirkof36fe1c2018-01-17 11:46:48 +01001319 err = tcf_block_owner_add(block, q, ei->binder_type);
1320 if (err)
1321 goto err_block_owner_add;
1322
1323 tcf_block_owner_netif_keep_dst(block, q, ei->binder_type);
1324
Jiri Pirkof71e0ca42018-07-23 09:23:05 +02001325 err = tcf_chain0_head_change_cb_add(block, ei, extack);
Jiri Pirkoa9b19442018-01-17 11:46:45 +01001326 if (err)
Jiri Pirkof71e0ca42018-07-23 09:23:05 +02001327 goto err_chain0_head_change_cb_add;
Jiri Pirkocaa72602018-01-17 11:46:50 +01001328
John Hurley60513bd2018-06-25 14:30:04 -07001329 err = tcf_block_offload_bind(block, q, ei, extack);
Jiri Pirkocaa72602018-01-17 11:46:50 +01001330 if (err)
1331 goto err_block_offload_bind;
1332
Jiri Pirko6529eab2017-05-17 11:07:55 +02001333 *p_block = block;
1334 return 0;
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001335
Jiri Pirkocaa72602018-01-17 11:46:50 +01001336err_block_offload_bind:
Jiri Pirkof71e0ca42018-07-23 09:23:05 +02001337 tcf_chain0_head_change_cb_del(block, ei);
1338err_chain0_head_change_cb_add:
Jiri Pirkof36fe1c2018-01-17 11:46:48 +01001339 tcf_block_owner_del(block, q, ei->binder_type);
1340err_block_owner_add:
Jiri Pirko48617382018-01-17 11:46:46 +01001341err_block_insert:
Vlad Buslov12db03b2019-02-11 10:55:45 +02001342 tcf_block_refcnt_put(block, true);
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001343 return err;
Jiri Pirko6529eab2017-05-17 11:07:55 +02001344}
Jiri Pirko8c4083b2017-10-19 15:50:29 +02001345EXPORT_SYMBOL(tcf_block_get_ext);
1346
Jiri Pirkoc7eb7d72017-11-03 11:46:24 +01001347static void tcf_chain_head_change_dflt(struct tcf_proto *tp_head, void *priv)
1348{
1349 struct tcf_proto __rcu **p_filter_chain = priv;
1350
1351 rcu_assign_pointer(*p_filter_chain, tp_head);
1352}
1353
Jiri Pirko8c4083b2017-10-19 15:50:29 +02001354int tcf_block_get(struct tcf_block **p_block,
Alexander Aring8d1a77f2017-12-20 12:35:19 -05001355 struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,
1356 struct netlink_ext_ack *extack)
Jiri Pirko8c4083b2017-10-19 15:50:29 +02001357{
Jiri Pirkoc7eb7d72017-11-03 11:46:24 +01001358 struct tcf_block_ext_info ei = {
1359 .chain_head_change = tcf_chain_head_change_dflt,
1360 .chain_head_change_priv = p_filter_chain,
1361 };
Jiri Pirko8c4083b2017-10-19 15:50:29 +02001362
Jiri Pirkoc7eb7d72017-11-03 11:46:24 +01001363 WARN_ON(!p_filter_chain);
Alexander Aring8d1a77f2017-12-20 12:35:19 -05001364 return tcf_block_get_ext(p_block, q, &ei, extack);
Jiri Pirko8c4083b2017-10-19 15:50:29 +02001365}
Jiri Pirko6529eab2017-05-17 11:07:55 +02001366EXPORT_SYMBOL(tcf_block_get);
1367
Cong Wang7aa00452017-10-26 18:24:28 -07001368/* XXX: Standalone actions are not allowed to jump to any chain, and bound
Roman Kapla60b3f52017-11-24 12:27:58 +01001369 * actions should be all removed after flushing.
Cong Wang7aa00452017-10-26 18:24:28 -07001370 */
Jiri Pirkoc7eb7d72017-11-03 11:46:24 +01001371void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q,
David S. Millere1ea2f92017-10-30 14:10:01 +09001372 struct tcf_block_ext_info *ei)
Cong Wang7aa00452017-10-26 18:24:28 -07001373{
David S. Millerc30abd52017-12-16 22:11:55 -05001374 if (!block)
1375 return;
Jiri Pirkof71e0ca42018-07-23 09:23:05 +02001376 tcf_chain0_head_change_cb_del(block, ei);
Jiri Pirkof36fe1c2018-01-17 11:46:48 +01001377 tcf_block_owner_del(block, q, ei->binder_type);
Roman Kapla60b3f52017-11-24 12:27:58 +01001378
Vlad Buslov12db03b2019-02-11 10:55:45 +02001379 __tcf_block_put(block, q, ei, true);
Jiri Pirko6529eab2017-05-17 11:07:55 +02001380}
Jiri Pirko8c4083b2017-10-19 15:50:29 +02001381EXPORT_SYMBOL(tcf_block_put_ext);
1382
1383void tcf_block_put(struct tcf_block *block)
1384{
1385 struct tcf_block_ext_info ei = {0, };
1386
Jiri Pirko4853f122017-12-21 13:13:59 +01001387 if (!block)
1388 return;
Jiri Pirkoc7eb7d72017-11-03 11:46:24 +01001389 tcf_block_put_ext(block, block->q, &ei);
Jiri Pirko8c4083b2017-10-19 15:50:29 +02001390}
David S. Millere1ea2f92017-10-30 14:10:01 +09001391
Jiri Pirko6529eab2017-05-17 11:07:55 +02001392EXPORT_SYMBOL(tcf_block_put);
Jiri Pirkocf1facd2017-02-09 14:38:56 +01001393
John Hurley32636742018-06-25 14:30:10 -07001394static int
Pablo Neira Ayusoa7323312019-07-19 18:20:15 +02001395tcf_block_playback_offloads(struct tcf_block *block, flow_setup_cb_t *cb,
John Hurley32636742018-06-25 14:30:10 -07001396 void *cb_priv, bool add, bool offload_in_use,
1397 struct netlink_ext_ack *extack)
1398{
Vlad Buslovbbf73832019-02-11 10:55:36 +02001399 struct tcf_chain *chain, *chain_prev;
Vlad Buslovfe2923a2019-02-11 10:55:40 +02001400 struct tcf_proto *tp, *tp_prev;
John Hurley32636742018-06-25 14:30:10 -07001401 int err;
1402
Vlad Buslov4f8116c2019-08-26 16:44:57 +03001403 lockdep_assert_held(&block->cb_lock);
1404
Vlad Buslovbbf73832019-02-11 10:55:36 +02001405 for (chain = __tcf_get_next_chain(block, NULL);
1406 chain;
1407 chain_prev = chain,
1408 chain = __tcf_get_next_chain(block, chain),
1409 tcf_chain_put(chain_prev)) {
Vlad Buslovfe2923a2019-02-11 10:55:40 +02001410 for (tp = __tcf_get_next_proto(chain, NULL); tp;
1411 tp_prev = tp,
1412 tp = __tcf_get_next_proto(chain, tp),
Vlad Buslov12db03b2019-02-11 10:55:45 +02001413 tcf_proto_put(tp_prev, true, NULL)) {
John Hurley32636742018-06-25 14:30:10 -07001414 if (tp->ops->reoffload) {
1415 err = tp->ops->reoffload(tp, add, cb, cb_priv,
1416 extack);
1417 if (err && add)
1418 goto err_playback_remove;
1419 } else if (add && offload_in_use) {
1420 err = -EOPNOTSUPP;
1421 NL_SET_ERR_MSG(extack, "Filter HW offload failed - classifier without re-offloading support");
1422 goto err_playback_remove;
1423 }
1424 }
1425 }
1426
1427 return 0;
1428
1429err_playback_remove:
Vlad Buslov12db03b2019-02-11 10:55:45 +02001430 tcf_proto_put(tp, true, NULL);
Vlad Buslovbbf73832019-02-11 10:55:36 +02001431 tcf_chain_put(chain);
John Hurley32636742018-06-25 14:30:10 -07001432 tcf_block_playback_offloads(block, cb, cb_priv, false, offload_in_use,
1433 extack);
1434 return err;
1435}
1436
Pablo Neira Ayuso59094b12019-07-09 22:55:45 +02001437static int tcf_block_bind(struct tcf_block *block,
1438 struct flow_block_offload *bo)
1439{
1440 struct flow_block_cb *block_cb, *next;
1441 int err, i = 0;
1442
Vlad Buslov4f8116c2019-08-26 16:44:57 +03001443 lockdep_assert_held(&block->cb_lock);
1444
Pablo Neira Ayuso59094b12019-07-09 22:55:45 +02001445 list_for_each_entry(block_cb, &bo->cb_list, list) {
1446 err = tcf_block_playback_offloads(block, block_cb->cb,
1447 block_cb->cb_priv, true,
1448 tcf_block_offload_in_use(block),
1449 bo->extack);
1450 if (err)
1451 goto err_unroll;
Vlad Buslovc9f14472019-08-26 16:45:01 +03001452 if (!bo->unlocked_driver_cb)
1453 block->lockeddevcnt++;
Pablo Neira Ayuso59094b12019-07-09 22:55:45 +02001454
1455 i++;
1456 }
Pablo Neira Ayuso14bfb132019-07-19 18:20:16 +02001457 list_splice(&bo->cb_list, &block->flow_block.cb_list);
Pablo Neira Ayuso59094b12019-07-09 22:55:45 +02001458
1459 return 0;
1460
1461err_unroll:
1462 list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
1463 if (i-- > 0) {
1464 list_del(&block_cb->list);
1465 tcf_block_playback_offloads(block, block_cb->cb,
1466 block_cb->cb_priv, false,
1467 tcf_block_offload_in_use(block),
1468 NULL);
Vlad Buslovc9f14472019-08-26 16:45:01 +03001469 if (!bo->unlocked_driver_cb)
1470 block->lockeddevcnt--;
Pablo Neira Ayuso59094b12019-07-09 22:55:45 +02001471 }
1472 flow_block_cb_free(block_cb);
1473 }
1474
1475 return err;
1476}
1477
1478static void tcf_block_unbind(struct tcf_block *block,
1479 struct flow_block_offload *bo)
1480{
1481 struct flow_block_cb *block_cb, *next;
1482
Vlad Buslov4f8116c2019-08-26 16:44:57 +03001483 lockdep_assert_held(&block->cb_lock);
1484
Pablo Neira Ayuso59094b12019-07-09 22:55:45 +02001485 list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
1486 tcf_block_playback_offloads(block, block_cb->cb,
1487 block_cb->cb_priv, false,
1488 tcf_block_offload_in_use(block),
1489 NULL);
1490 list_del(&block_cb->list);
1491 flow_block_cb_free(block_cb);
Vlad Buslovc9f14472019-08-26 16:45:01 +03001492 if (!bo->unlocked_driver_cb)
1493 block->lockeddevcnt--;
Pablo Neira Ayuso59094b12019-07-09 22:55:45 +02001494 }
1495}
1496
1497static int tcf_block_setup(struct tcf_block *block,
1498 struct flow_block_offload *bo)
1499{
1500 int err;
1501
1502 switch (bo->command) {
1503 case FLOW_BLOCK_BIND:
1504 err = tcf_block_bind(block, bo);
1505 break;
1506 case FLOW_BLOCK_UNBIND:
1507 err = 0;
1508 tcf_block_unbind(block, bo);
1509 break;
1510 default:
1511 WARN_ON_ONCE(1);
1512 err = -EOPNOTSUPP;
1513 }
1514
1515 return err;
1516}
1517
Jiri Pirko87d83092017-05-17 11:07:54 +02001518/* Main classifier routine: scans classifier chain attached
1519 * to this qdisc, (optionally) tests for protocol and asks
1520 * specific classifiers.
1521 */
Paul Blakey9410c942020-02-16 12:01:21 +02001522static inline int __tcf_classify(struct sk_buff *skb,
1523 const struct tcf_proto *tp,
Paul Blakeyaf699622020-02-16 12:01:24 +02001524 const struct tcf_proto *orig_tp,
Paul Blakey9410c942020-02-16 12:01:21 +02001525 struct tcf_result *res,
1526 bool compat_mode,
1527 u32 *last_executed_chain)
Jiri Pirko87d83092017-05-17 11:07:54 +02001528{
Jiri Pirko87d83092017-05-17 11:07:54 +02001529#ifdef CONFIG_NET_CLS_ACT
1530 const int max_reclassify_loop = 4;
Jiri Pirkoee538dc2017-05-23 09:11:59 +02001531 const struct tcf_proto *first_tp;
Jiri Pirko87d83092017-05-17 11:07:54 +02001532 int limit = 0;
1533
1534reclassify:
1535#endif
1536 for (; tp; tp = rcu_dereference_bh(tp->next)) {
Cong Wangcd0c4e72019-01-11 18:55:42 -08001537 __be16 protocol = tc_skb_protocol(skb);
Jiri Pirko87d83092017-05-17 11:07:54 +02001538 int err;
1539
1540 if (tp->protocol != protocol &&
1541 tp->protocol != htons(ETH_P_ALL))
1542 continue;
1543
1544 err = tp->classify(skb, tp, res);
1545#ifdef CONFIG_NET_CLS_ACT
Jiri Pirkodb505142017-05-17 11:08:03 +02001546 if (unlikely(err == TC_ACT_RECLASSIFY && !compat_mode)) {
Jiri Pirkoee538dc2017-05-23 09:11:59 +02001547 first_tp = orig_tp;
Paul Blakey9410c942020-02-16 12:01:21 +02001548 *last_executed_chain = first_tp->chain->index;
Jiri Pirko87d83092017-05-17 11:07:54 +02001549 goto reset;
Jiri Pirkodb505142017-05-17 11:08:03 +02001550 } else if (unlikely(TC_ACT_EXT_CMP(err, TC_ACT_GOTO_CHAIN))) {
Jiri Pirkoee538dc2017-05-23 09:11:59 +02001551 first_tp = res->goto_tp;
Paul Blakey9410c942020-02-16 12:01:21 +02001552 *last_executed_chain = err & TC_ACT_EXT_VAL_MASK;
Jiri Pirkodb505142017-05-17 11:08:03 +02001553 goto reset;
1554 }
Jiri Pirko87d83092017-05-17 11:07:54 +02001555#endif
1556 if (err >= 0)
1557 return err;
1558 }
1559
1560 return TC_ACT_UNSPEC; /* signal: continue lookup */
1561#ifdef CONFIG_NET_CLS_ACT
1562reset:
1563 if (unlikely(limit++ >= max_reclassify_loop)) {
Jiri Pirko9d3aaff2018-01-17 11:46:47 +01001564 net_notice_ratelimited("%u: reclassify loop, rule prio %u, protocol %02x\n",
1565 tp->chain->block->index,
1566 tp->prio & 0xffff,
Jiri Pirko87d83092017-05-17 11:07:54 +02001567 ntohs(tp->protocol));
1568 return TC_ACT_SHOT;
1569 }
1570
Jiri Pirkoee538dc2017-05-23 09:11:59 +02001571 tp = first_tp;
Jiri Pirko87d83092017-05-17 11:07:54 +02001572 goto reclassify;
1573#endif
1574}
Paul Blakey9410c942020-02-16 12:01:21 +02001575
1576int tcf_classify(struct sk_buff *skb, const struct tcf_proto *tp,
1577 struct tcf_result *res, bool compat_mode)
1578{
1579 u32 last_executed_chain = 0;
1580
Paul Blakeyaf699622020-02-16 12:01:24 +02001581 return __tcf_classify(skb, tp, tp, res, compat_mode,
Paul Blakey9410c942020-02-16 12:01:21 +02001582 &last_executed_chain);
1583}
Jiri Pirko87d83092017-05-17 11:07:54 +02001584EXPORT_SYMBOL(tcf_classify);
1585
Paul Blakey7d17c542020-02-16 12:01:22 +02001586int tcf_classify_ingress(struct sk_buff *skb,
1587 const struct tcf_block *ingress_block,
1588 const struct tcf_proto *tp,
Paul Blakey9410c942020-02-16 12:01:21 +02001589 struct tcf_result *res, bool compat_mode)
1590{
1591#if !IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
1592 u32 last_executed_chain = 0;
1593
Paul Blakeyaf699622020-02-16 12:01:24 +02001594 return __tcf_classify(skb, tp, tp, res, compat_mode,
Paul Blakey9410c942020-02-16 12:01:21 +02001595 &last_executed_chain);
1596#else
1597 u32 last_executed_chain = tp ? tp->chain->index : 0;
Paul Blakeyaf699622020-02-16 12:01:24 +02001598 const struct tcf_proto *orig_tp = tp;
Paul Blakey9410c942020-02-16 12:01:21 +02001599 struct tc_skb_ext *ext;
1600 int ret;
1601
Paul Blakeyaf699622020-02-16 12:01:24 +02001602 ext = skb_ext_find(skb, TC_SKB_EXT);
1603
1604 if (ext && ext->chain) {
1605 struct tcf_chain *fchain;
1606
1607 fchain = tcf_chain_lookup_rcu(ingress_block, ext->chain);
1608 if (!fchain)
1609 return TC_ACT_SHOT;
1610
1611 /* Consume, so cloned/redirect skbs won't inherit ext */
1612 skb_ext_del(skb, TC_SKB_EXT);
1613
1614 tp = rcu_dereference_bh(fchain->filter_chain);
Paul Blakeya080da62020-04-06 18:36:56 +03001615 last_executed_chain = fchain->index;
Paul Blakeyaf699622020-02-16 12:01:24 +02001616 }
1617
1618 ret = __tcf_classify(skb, tp, orig_tp, res, compat_mode,
1619 &last_executed_chain);
Paul Blakey9410c942020-02-16 12:01:21 +02001620
1621 /* If we missed on some chain */
1622 if (ret == TC_ACT_UNSPEC && last_executed_chain) {
1623 ext = skb_ext_add(skb, TC_SKB_EXT);
1624 if (WARN_ON_ONCE(!ext))
1625 return TC_ACT_SHOT;
1626 ext->chain = last_executed_chain;
1627 }
1628
1629 return ret;
1630#endif
1631}
1632EXPORT_SYMBOL(tcf_classify_ingress);
1633
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001634struct tcf_chain_info {
1635 struct tcf_proto __rcu **pprev;
1636 struct tcf_proto __rcu *next;
1637};
1638
Vlad Busloved76f5e2019-02-11 10:55:38 +02001639static struct tcf_proto *tcf_chain_tp_prev(struct tcf_chain *chain,
1640 struct tcf_chain_info *chain_info)
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001641{
Vlad Busloved76f5e2019-02-11 10:55:38 +02001642 return tcf_chain_dereference(*chain_info->pprev, chain);
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001643}
1644
Vlad Buslov726d06122019-02-11 10:55:42 +02001645static int tcf_chain_tp_insert(struct tcf_chain *chain,
1646 struct tcf_chain_info *chain_info,
1647 struct tcf_proto *tp)
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001648{
Vlad Buslov726d06122019-02-11 10:55:42 +02001649 if (chain->flushing)
1650 return -EAGAIN;
1651
Jiri Pirkoc7eb7d72017-11-03 11:46:24 +01001652 if (*chain_info->pprev == chain->filter_chain)
Jiri Pirkof71e0ca42018-07-23 09:23:05 +02001653 tcf_chain0_head_change(chain, tp);
Vlad Buslov4dbfa762019-02-11 10:55:39 +02001654 tcf_proto_get(tp);
Vlad Busloved76f5e2019-02-11 10:55:38 +02001655 RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain, chain_info));
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001656 rcu_assign_pointer(*chain_info->pprev, tp);
Vlad Buslov726d06122019-02-11 10:55:42 +02001657
1658 return 0;
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001659}
1660
1661static void tcf_chain_tp_remove(struct tcf_chain *chain,
1662 struct tcf_chain_info *chain_info,
1663 struct tcf_proto *tp)
1664{
Vlad Busloved76f5e2019-02-11 10:55:38 +02001665 struct tcf_proto *next = tcf_chain_dereference(chain_info->next, chain);
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001666
Vlad Buslov8b646782019-02-11 10:55:41 +02001667 tcf_proto_mark_delete(tp);
Jiri Pirkoc7eb7d72017-11-03 11:46:24 +01001668 if (tp == chain->filter_chain)
Jiri Pirkof71e0ca42018-07-23 09:23:05 +02001669 tcf_chain0_head_change(chain, next);
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001670 RCU_INIT_POINTER(*chain_info->pprev, next);
1671}
1672
1673static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
1674 struct tcf_chain_info *chain_info,
1675 u32 protocol, u32 prio,
Vlad Buslov8b646782019-02-11 10:55:41 +02001676 bool prio_allocate);
1677
1678/* Try to insert new proto.
1679 * If proto with specified priority already exists, free new proto
1680 * and return existing one.
1681 */
1682
1683static struct tcf_proto *tcf_chain_tp_insert_unique(struct tcf_chain *chain,
1684 struct tcf_proto *tp_new,
Vlad Buslov12db03b2019-02-11 10:55:45 +02001685 u32 protocol, u32 prio,
1686 bool rtnl_held)
Vlad Buslov8b646782019-02-11 10:55:41 +02001687{
1688 struct tcf_chain_info chain_info;
1689 struct tcf_proto *tp;
Vlad Buslov726d06122019-02-11 10:55:42 +02001690 int err = 0;
Vlad Buslov8b646782019-02-11 10:55:41 +02001691
1692 mutex_lock(&chain->filter_chain_lock);
1693
John Hurley59eb87c2019-11-02 14:17:47 +00001694 if (tcf_proto_exists_destroying(chain, tp_new)) {
1695 mutex_unlock(&chain->filter_chain_lock);
1696 tcf_proto_destroy(tp_new, rtnl_held, false, NULL);
1697 return ERR_PTR(-EAGAIN);
1698 }
1699
Vlad Buslov8b646782019-02-11 10:55:41 +02001700 tp = tcf_chain_tp_find(chain, &chain_info,
1701 protocol, prio, false);
1702 if (!tp)
Vlad Buslov726d06122019-02-11 10:55:42 +02001703 err = tcf_chain_tp_insert(chain, &chain_info, tp_new);
Vlad Buslov8b646782019-02-11 10:55:41 +02001704 mutex_unlock(&chain->filter_chain_lock);
1705
1706 if (tp) {
John Hurley59eb87c2019-11-02 14:17:47 +00001707 tcf_proto_destroy(tp_new, rtnl_held, false, NULL);
Vlad Buslov8b646782019-02-11 10:55:41 +02001708 tp_new = tp;
Vlad Buslov726d06122019-02-11 10:55:42 +02001709 } else if (err) {
John Hurley59eb87c2019-11-02 14:17:47 +00001710 tcf_proto_destroy(tp_new, rtnl_held, false, NULL);
Vlad Buslov726d06122019-02-11 10:55:42 +02001711 tp_new = ERR_PTR(err);
Vlad Buslov8b646782019-02-11 10:55:41 +02001712 }
1713
1714 return tp_new;
1715}
1716
1717static void tcf_chain_tp_delete_empty(struct tcf_chain *chain,
Vlad Buslov12db03b2019-02-11 10:55:45 +02001718 struct tcf_proto *tp, bool rtnl_held,
Vlad Buslov8b646782019-02-11 10:55:41 +02001719 struct netlink_ext_ack *extack)
1720{
1721 struct tcf_chain_info chain_info;
1722 struct tcf_proto *tp_iter;
1723 struct tcf_proto **pprev;
1724 struct tcf_proto *next;
1725
1726 mutex_lock(&chain->filter_chain_lock);
1727
1728 /* Atomically find and remove tp from chain. */
1729 for (pprev = &chain->filter_chain;
1730 (tp_iter = tcf_chain_dereference(*pprev, chain));
1731 pprev = &tp_iter->next) {
1732 if (tp_iter == tp) {
1733 chain_info.pprev = pprev;
1734 chain_info.next = tp_iter->next;
1735 WARN_ON(tp_iter->deleting);
1736 break;
1737 }
1738 }
1739 /* Verify that tp still exists and no new filters were inserted
1740 * concurrently.
1741 * Mark tp for deletion if it is empty.
1742 */
Davide Carattia5b72a02019-12-28 16:36:58 +01001743 if (!tp_iter || !tcf_proto_check_delete(tp)) {
Vlad Buslov8b646782019-02-11 10:55:41 +02001744 mutex_unlock(&chain->filter_chain_lock);
1745 return;
1746 }
1747
John Hurley59eb87c2019-11-02 14:17:47 +00001748 tcf_proto_signal_destroying(chain, tp);
Vlad Buslov8b646782019-02-11 10:55:41 +02001749 next = tcf_chain_dereference(chain_info.next, chain);
1750 if (tp == chain->filter_chain)
1751 tcf_chain0_head_change(chain, next);
1752 RCU_INIT_POINTER(*chain_info.pprev, next);
1753 mutex_unlock(&chain->filter_chain_lock);
1754
Vlad Buslov12db03b2019-02-11 10:55:45 +02001755 tcf_proto_put(tp, rtnl_held, extack);
Vlad Buslov8b646782019-02-11 10:55:41 +02001756}
1757
1758static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
1759 struct tcf_chain_info *chain_info,
1760 u32 protocol, u32 prio,
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001761 bool prio_allocate)
1762{
1763 struct tcf_proto **pprev;
1764 struct tcf_proto *tp;
1765
1766 /* Check the chain for existence of proto-tcf with this priority */
1767 for (pprev = &chain->filter_chain;
Vlad Busloved76f5e2019-02-11 10:55:38 +02001768 (tp = tcf_chain_dereference(*pprev, chain));
1769 pprev = &tp->next) {
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001770 if (tp->prio >= prio) {
1771 if (tp->prio == prio) {
1772 if (prio_allocate ||
1773 (tp->protocol != protocol && protocol))
1774 return ERR_PTR(-EINVAL);
1775 } else {
1776 tp = NULL;
1777 }
1778 break;
1779 }
1780 }
1781 chain_info->pprev = pprev;
Vlad Buslov4dbfa762019-02-11 10:55:39 +02001782 if (tp) {
1783 chain_info->next = tp->next;
1784 tcf_proto_get(tp);
1785 } else {
1786 chain_info->next = NULL;
1787 }
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001788 return tp;
1789}
1790
WANG Cong71203712017-08-07 15:26:50 -07001791static int tcf_fill_node(struct net *net, struct sk_buff *skb,
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001792 struct tcf_proto *tp, struct tcf_block *block,
1793 struct Qdisc *q, u32 parent, void *fh,
Vlad Buslov12db03b2019-02-11 10:55:45 +02001794 u32 portid, u32 seq, u16 flags, int event,
Vlad Buslovf8ab1802020-05-15 14:40:11 +03001795 bool terse_dump, bool rtnl_held)
WANG Cong71203712017-08-07 15:26:50 -07001796{
1797 struct tcmsg *tcm;
1798 struct nlmsghdr *nlh;
1799 unsigned char *b = skb_tail_pointer(skb);
1800
1801 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
1802 if (!nlh)
1803 goto out_nlmsg_trim;
1804 tcm = nlmsg_data(nlh);
1805 tcm->tcm_family = AF_UNSPEC;
1806 tcm->tcm__pad1 = 0;
1807 tcm->tcm__pad2 = 0;
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001808 if (q) {
1809 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
1810 tcm->tcm_parent = parent;
1811 } else {
1812 tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK;
1813 tcm->tcm_block_index = block->index;
1814 }
WANG Cong71203712017-08-07 15:26:50 -07001815 tcm->tcm_info = TC_H_MAKE(tp->prio, tp->protocol);
1816 if (nla_put_string(skb, TCA_KIND, tp->ops->kind))
1817 goto nla_put_failure;
1818 if (nla_put_u32(skb, TCA_CHAIN, tp->chain->index))
1819 goto nla_put_failure;
1820 if (!fh) {
1821 tcm->tcm_handle = 0;
Vlad Buslovf8ab1802020-05-15 14:40:11 +03001822 } else if (terse_dump) {
1823 if (tp->ops->terse_dump) {
1824 if (tp->ops->terse_dump(net, tp, fh, skb, tcm,
1825 rtnl_held) < 0)
1826 goto nla_put_failure;
1827 } else {
1828 goto cls_op_not_supp;
1829 }
WANG Cong71203712017-08-07 15:26:50 -07001830 } else {
Vlad Buslov12db03b2019-02-11 10:55:45 +02001831 if (tp->ops->dump &&
1832 tp->ops->dump(net, tp, fh, skb, tcm, rtnl_held) < 0)
WANG Cong71203712017-08-07 15:26:50 -07001833 goto nla_put_failure;
1834 }
1835 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
1836 return skb->len;
1837
1838out_nlmsg_trim:
1839nla_put_failure:
Vlad Buslovf8ab1802020-05-15 14:40:11 +03001840cls_op_not_supp:
WANG Cong71203712017-08-07 15:26:50 -07001841 nlmsg_trim(skb, b);
1842 return -1;
1843}
1844
1845static int tfilter_notify(struct net *net, struct sk_buff *oskb,
1846 struct nlmsghdr *n, struct tcf_proto *tp,
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001847 struct tcf_block *block, struct Qdisc *q,
Vlad Buslov12db03b2019-02-11 10:55:45 +02001848 u32 parent, void *fh, int event, bool unicast,
1849 bool rtnl_held)
WANG Cong71203712017-08-07 15:26:50 -07001850{
1851 struct sk_buff *skb;
1852 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
Zhike Wang5b5f99b2019-03-11 03:15:54 -07001853 int err = 0;
WANG Cong71203712017-08-07 15:26:50 -07001854
1855 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1856 if (!skb)
1857 return -ENOBUFS;
1858
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001859 if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
Vlad Buslov12db03b2019-02-11 10:55:45 +02001860 n->nlmsg_seq, n->nlmsg_flags, event,
Vlad Buslovf8ab1802020-05-15 14:40:11 +03001861 false, rtnl_held) <= 0) {
WANG Cong71203712017-08-07 15:26:50 -07001862 kfree_skb(skb);
1863 return -EINVAL;
1864 }
1865
1866 if (unicast)
Zhike Wang5b5f99b2019-03-11 03:15:54 -07001867 err = netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
1868 else
1869 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1870 n->nlmsg_flags & NLM_F_ECHO);
WANG Cong71203712017-08-07 15:26:50 -07001871
Zhike Wang5b5f99b2019-03-11 03:15:54 -07001872 if (err > 0)
1873 err = 0;
1874 return err;
WANG Cong71203712017-08-07 15:26:50 -07001875}
1876
1877static int tfilter_del_notify(struct net *net, struct sk_buff *oskb,
1878 struct nlmsghdr *n, struct tcf_proto *tp,
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001879 struct tcf_block *block, struct Qdisc *q,
Alexander Aringc35a4ac2018-01-18 11:20:50 -05001880 u32 parent, void *fh, bool unicast, bool *last,
Vlad Buslov12db03b2019-02-11 10:55:45 +02001881 bool rtnl_held, struct netlink_ext_ack *extack)
WANG Cong71203712017-08-07 15:26:50 -07001882{
1883 struct sk_buff *skb;
1884 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1885 int err;
1886
1887 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1888 if (!skb)
1889 return -ENOBUFS;
1890
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001891 if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
Vlad Buslov12db03b2019-02-11 10:55:45 +02001892 n->nlmsg_seq, n->nlmsg_flags, RTM_DELTFILTER,
Vlad Buslovf8ab1802020-05-15 14:40:11 +03001893 false, rtnl_held) <= 0) {
Alexander Aringc35a4ac2018-01-18 11:20:50 -05001894 NL_SET_ERR_MSG(extack, "Failed to build del event notification");
WANG Cong71203712017-08-07 15:26:50 -07001895 kfree_skb(skb);
1896 return -EINVAL;
1897 }
1898
Vlad Buslov12db03b2019-02-11 10:55:45 +02001899 err = tp->ops->delete(tp, fh, last, rtnl_held, extack);
WANG Cong71203712017-08-07 15:26:50 -07001900 if (err) {
1901 kfree_skb(skb);
1902 return err;
1903 }
1904
1905 if (unicast)
Zhike Wang5b5f99b2019-03-11 03:15:54 -07001906 err = netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
1907 else
1908 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1909 n->nlmsg_flags & NLM_F_ECHO);
Alexander Aringc35a4ac2018-01-18 11:20:50 -05001910 if (err < 0)
1911 NL_SET_ERR_MSG(extack, "Failed to send filter delete notification");
Zhike Wang5b5f99b2019-03-11 03:15:54 -07001912
1913 if (err > 0)
1914 err = 0;
Alexander Aringc35a4ac2018-01-18 11:20:50 -05001915 return err;
WANG Cong71203712017-08-07 15:26:50 -07001916}
1917
1918static void tfilter_notify_chain(struct net *net, struct sk_buff *oskb,
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001919 struct tcf_block *block, struct Qdisc *q,
1920 u32 parent, struct nlmsghdr *n,
Vlad Buslov12db03b2019-02-11 10:55:45 +02001921 struct tcf_chain *chain, int event,
1922 bool rtnl_held)
WANG Cong71203712017-08-07 15:26:50 -07001923{
1924 struct tcf_proto *tp;
1925
Vlad Buslov12db03b2019-02-11 10:55:45 +02001926 for (tp = tcf_get_next_proto(chain, NULL, rtnl_held);
1927 tp; tp = tcf_get_next_proto(chain, tp, rtnl_held))
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001928 tfilter_notify(net, oskb, n, tp, block,
Vlad Buslov12db03b2019-02-11 10:55:45 +02001929 q, parent, NULL, event, false, rtnl_held);
WANG Cong71203712017-08-07 15:26:50 -07001930}
1931
Vlad Buslov7d5509f2019-02-11 10:55:44 +02001932static void tfilter_put(struct tcf_proto *tp, void *fh)
1933{
1934 if (tp->ops->put && fh)
1935 tp->ops->put(tp, fh);
1936}
1937
Vlad Buslovc431f892018-05-31 09:52:53 +03001938static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
David Ahernc21ef3e2017-04-16 09:48:24 -07001939 struct netlink_ext_ack *extack)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001940{
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09001941 struct net *net = sock_net(skb->sk);
Patrick McHardyadd93b62008-01-22 22:11:33 -08001942 struct nlattr *tca[TCA_MAX + 1];
Cong Wang6f96c3c2019-10-07 13:26:28 -07001943 char name[IFNAMSIZ];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001944 struct tcmsg *t;
1945 u32 protocol;
1946 u32 prio;
Jiri Pirko9d36d9e2017-05-17 11:07:57 +02001947 bool prio_allocate;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001948 u32 parent;
Jiri Pirko5bc17012017-05-17 11:08:01 +02001949 u32 chain_index;
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001950 struct Qdisc *q = NULL;
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001951 struct tcf_chain_info chain_info;
Jiri Pirko5bc17012017-05-17 11:08:01 +02001952 struct tcf_chain *chain = NULL;
Jiri Pirko6529eab2017-05-17 11:07:55 +02001953 struct tcf_block *block;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001954 struct tcf_proto *tp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001955 unsigned long cl;
WANG Cong8113c092017-08-04 21:31:43 -07001956 void *fh;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001957 int err;
Daniel Borkmann628185c2016-12-21 18:04:11 +01001958 int tp_created;
Vlad Buslov470502d2019-02-11 10:55:48 +02001959 bool rtnl_held = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001960
Vlad Buslovc431f892018-05-31 09:52:53 +03001961 if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
Eric W. Biedermandfc47ef2012-11-16 03:03:00 +00001962 return -EPERM;
Hong zhi guode179c82013-03-25 17:36:33 +00001963
Linus Torvalds1da177e2005-04-16 15:20:36 -07001964replay:
Daniel Borkmann628185c2016-12-21 18:04:11 +01001965 tp_created = 0;
1966
Johannes Berg8cb08172019-04-26 14:07:28 +02001967 err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
1968 rtm_tca_policy, extack);
Hong zhi guode179c82013-03-25 17:36:33 +00001969 if (err < 0)
1970 return err;
1971
David S. Miller942b8162012-06-26 21:48:50 -07001972 t = nlmsg_data(n);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001973 protocol = TC_H_MIN(t->tcm_info);
1974 prio = TC_H_MAJ(t->tcm_info);
Jiri Pirko9d36d9e2017-05-17 11:07:57 +02001975 prio_allocate = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001976 parent = t->tcm_parent;
Vlad Buslov4dbfa762019-02-11 10:55:39 +02001977 tp = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001978 cl = 0;
Vlad Buslov470502d2019-02-11 10:55:48 +02001979 block = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001980
1981 if (prio == 0) {
Vlad Buslovc431f892018-05-31 09:52:53 +03001982 /* If no priority is provided by the user,
1983 * we allocate one.
1984 */
1985 if (n->nlmsg_flags & NLM_F_CREATE) {
1986 prio = TC_H_MAKE(0x80000000U, 0U);
1987 prio_allocate = true;
1988 } else {
Alexander Aringc35a4ac2018-01-18 11:20:50 -05001989 NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001990 return -ENOENT;
Daniel Borkmannea7f8272016-06-10 23:10:22 +02001991 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001992 }
1993
1994 /* Find head of filter chain. */
1995
Vlad Buslov470502d2019-02-11 10:55:48 +02001996 err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
1997 if (err)
1998 return err;
1999
Cong Wang6f96c3c2019-10-07 13:26:28 -07002000 if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
2001 NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
2002 err = -EINVAL;
2003 goto errout;
2004 }
2005
Vlad Buslov470502d2019-02-11 10:55:48 +02002006 /* Take rtnl mutex if rtnl_held was set to true on previous iteration,
2007 * block is shared (no qdisc found), qdisc is not unlocked, classifier
2008 * type is not specified, classifier is not unlocked.
2009 */
2010 if (rtnl_held ||
2011 (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
Cong Wang6f96c3c2019-10-07 13:26:28 -07002012 !tcf_proto_is_unlocked(name)) {
Vlad Buslov470502d2019-02-11 10:55:48 +02002013 rtnl_held = true;
2014 rtnl_lock();
2015 }
2016
2017 err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
2018 if (err)
2019 goto errout;
2020
2021 block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
2022 extack);
Vlad Buslovc431f892018-05-31 09:52:53 +03002023 if (IS_ERR(block)) {
2024 err = PTR_ERR(block);
2025 goto errout;
Jiri Pirko6bb16e72017-02-09 14:38:58 +01002026 }
Cong Wanga7df4872020-04-30 20:53:49 -07002027 block->classid = parent;
Jiri Pirko5bc17012017-05-17 11:08:01 +02002028
2029 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2030 if (chain_index > TC_ACT_EXT_VAL_MASK) {
Alexander Aringc35a4ac2018-01-18 11:20:50 -05002031 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
Jiri Pirko5bc17012017-05-17 11:08:01 +02002032 err = -EINVAL;
2033 goto errout;
2034 }
Vlad Buslovc431f892018-05-31 09:52:53 +03002035 chain = tcf_chain_get(block, chain_index, true);
Jiri Pirko5bc17012017-05-17 11:08:01 +02002036 if (!chain) {
Jiri Pirkod5ed72a2018-08-27 20:58:43 +02002037 NL_SET_ERR_MSG(extack, "Cannot create specified filter chain");
Vlad Buslovc431f892018-05-31 09:52:53 +03002038 err = -ENOMEM;
Daniel Borkmannea7f8272016-06-10 23:10:22 +02002039 goto errout;
2040 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002041
Vlad Busloved76f5e2019-02-11 10:55:38 +02002042 mutex_lock(&chain->filter_chain_lock);
Jiri Pirko2190d1d2017-05-17 11:07:59 +02002043 tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2044 prio, prio_allocate);
2045 if (IS_ERR(tp)) {
Alexander Aringc35a4ac2018-01-18 11:20:50 -05002046 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
Jiri Pirko2190d1d2017-05-17 11:07:59 +02002047 err = PTR_ERR(tp);
Vlad Busloved76f5e2019-02-11 10:55:38 +02002048 goto errout_locked;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002049 }
2050
2051 if (tp == NULL) {
Vlad Buslov8b646782019-02-11 10:55:41 +02002052 struct tcf_proto *tp_new = NULL;
2053
Vlad Buslov726d06122019-02-11 10:55:42 +02002054 if (chain->flushing) {
2055 err = -EAGAIN;
2056 goto errout_locked;
2057 }
2058
Linus Torvalds1da177e2005-04-16 15:20:36 -07002059 /* Proto-tcf does not exist, create new one */
2060
Jiri Pirko6bb16e72017-02-09 14:38:58 +01002061 if (tca[TCA_KIND] == NULL || !protocol) {
Alexander Aringc35a4ac2018-01-18 11:20:50 -05002062 NL_SET_ERR_MSG(extack, "Filter kind and protocol must be specified");
Jiri Pirko6bb16e72017-02-09 14:38:58 +01002063 err = -EINVAL;
Vlad Busloved76f5e2019-02-11 10:55:38 +02002064 goto errout_locked;
Jiri Pirko6bb16e72017-02-09 14:38:58 +01002065 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002066
Vlad Buslovc431f892018-05-31 09:52:53 +03002067 if (!(n->nlmsg_flags & NLM_F_CREATE)) {
Alexander Aringc35a4ac2018-01-18 11:20:50 -05002068 NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
Jiri Pirko6bb16e72017-02-09 14:38:58 +01002069 err = -ENOENT;
Vlad Busloved76f5e2019-02-11 10:55:38 +02002070 goto errout_locked;
Jiri Pirko6bb16e72017-02-09 14:38:58 +01002071 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002072
Jiri Pirko9d36d9e2017-05-17 11:07:57 +02002073 if (prio_allocate)
Vlad Busloved76f5e2019-02-11 10:55:38 +02002074 prio = tcf_auto_prio(tcf_chain_tp_prev(chain,
2075 &chain_info));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002076
Vlad Busloved76f5e2019-02-11 10:55:38 +02002077 mutex_unlock(&chain->filter_chain_lock);
Eric Dumazet36d79af2020-01-21 11:02:20 -08002078 tp_new = tcf_proto_create(name, protocol, prio, chain,
2079 rtnl_held, extack);
Vlad Buslov8b646782019-02-11 10:55:41 +02002080 if (IS_ERR(tp_new)) {
2081 err = PTR_ERR(tp_new);
Vlad Buslov726d06122019-02-11 10:55:42 +02002082 goto errout_tp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002083 }
Vlad Busloved76f5e2019-02-11 10:55:38 +02002084
Minoru Usui12186be2009-06-02 02:17:34 -07002085 tp_created = 1;
Vlad Buslov12db03b2019-02-11 10:55:45 +02002086 tp = tcf_chain_tp_insert_unique(chain, tp_new, protocol, prio,
2087 rtnl_held);
Vlad Buslov726d06122019-02-11 10:55:42 +02002088 if (IS_ERR(tp)) {
2089 err = PTR_ERR(tp);
2090 goto errout_tp;
2091 }
Vlad Busloved76f5e2019-02-11 10:55:38 +02002092 } else {
2093 mutex_unlock(&chain->filter_chain_lock);
Jiri Pirko6bb16e72017-02-09 14:38:58 +01002094 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002095
Vlad Buslov8b646782019-02-11 10:55:41 +02002096 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
2097 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
2098 err = -EINVAL;
2099 goto errout;
2100 }
2101
Linus Torvalds1da177e2005-04-16 15:20:36 -07002102 fh = tp->ops->get(tp, t->tcm_handle);
2103
WANG Cong8113c092017-08-04 21:31:43 -07002104 if (!fh) {
Vlad Buslovc431f892018-05-31 09:52:53 +03002105 if (!(n->nlmsg_flags & NLM_F_CREATE)) {
Alexander Aringc35a4ac2018-01-18 11:20:50 -05002106 NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
Jiri Pirko6bb16e72017-02-09 14:38:58 +01002107 err = -ENOENT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002108 goto errout;
Jiri Pirko6bb16e72017-02-09 14:38:58 +01002109 }
Vlad Buslovc431f892018-05-31 09:52:53 +03002110 } else if (n->nlmsg_flags & NLM_F_EXCL) {
Vlad Buslov7d5509f2019-02-11 10:55:44 +02002111 tfilter_put(tp, fh);
Vlad Buslovc431f892018-05-31 09:52:53 +03002112 NL_SET_ERR_MSG(extack, "Filter already exists");
2113 err = -EEXIST;
2114 goto errout;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002115 }
2116
Jiri Pirko9f407f12018-07-23 09:23:07 +02002117 if (chain->tmplt_ops && chain->tmplt_ops != tp->ops) {
2118 NL_SET_ERR_MSG(extack, "Chain template is set to a different filter kind");
2119 err = -EINVAL;
2120 goto errout;
2121 }
2122
Cong Wang2f7ef2f2014-04-25 13:54:06 -07002123 err = tp->ops->change(net, skb, tp, cl, t->tcm_handle, tca, &fh,
Alexander Aring7306db32018-01-18 11:20:51 -05002124 n->nlmsg_flags & NLM_F_CREATE ? TCA_ACT_NOREPLACE : TCA_ACT_REPLACE,
Vlad Buslov12db03b2019-02-11 10:55:45 +02002125 rtnl_held, extack);
Vlad Buslov7d5509f2019-02-11 10:55:44 +02002126 if (err == 0) {
Jiri Pirko7960d1d2018-01-17 11:46:51 +01002127 tfilter_notify(net, skb, n, tp, block, q, parent, fh,
Vlad Buslov12db03b2019-02-11 10:55:45 +02002128 RTM_NEWTFILTER, false, rtnl_held);
Vlad Buslov7d5509f2019-02-11 10:55:44 +02002129 tfilter_put(tp, fh);
Vlad Buslov503d81d2019-07-21 17:44:12 +03002130 /* q pointer is NULL for shared blocks */
2131 if (q)
2132 q->flags &= ~TCQ_F_CAN_BYPASS;
Vlad Buslov7d5509f2019-02-11 10:55:44 +02002133 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002134
2135errout:
Vlad Buslov8b646782019-02-11 10:55:41 +02002136 if (err && tp_created)
Vlad Buslov12db03b2019-02-11 10:55:45 +02002137 tcf_chain_tp_delete_empty(chain, tp, rtnl_held, NULL);
Vlad Buslov726d06122019-02-11 10:55:42 +02002138errout_tp:
Vlad Buslov4dbfa762019-02-11 10:55:39 +02002139 if (chain) {
2140 if (tp && !IS_ERR(tp))
Vlad Buslov12db03b2019-02-11 10:55:45 +02002141 tcf_proto_put(tp, rtnl_held, NULL);
Vlad Buslov4dbfa762019-02-11 10:55:39 +02002142 if (!tp_created)
2143 tcf_chain_put(chain);
2144 }
Vlad Buslov12db03b2019-02-11 10:55:45 +02002145 tcf_block_release(q, block, rtnl_held);
Vlad Buslov470502d2019-02-11 10:55:48 +02002146
2147 if (rtnl_held)
2148 rtnl_unlock();
2149
2150 if (err == -EAGAIN) {
2151 /* Take rtnl lock in case EAGAIN is caused by concurrent flush
2152 * of target chain.
2153 */
2154 rtnl_held = true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002155 /* Replay the request. */
2156 goto replay;
Vlad Buslov470502d2019-02-11 10:55:48 +02002157 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002158 return err;
Vlad Busloved76f5e2019-02-11 10:55:38 +02002159
2160errout_locked:
2161 mutex_unlock(&chain->filter_chain_lock);
2162 goto errout;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002163}
2164
Vlad Buslovc431f892018-05-31 09:52:53 +03002165static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
2166 struct netlink_ext_ack *extack)
2167{
2168 struct net *net = sock_net(skb->sk);
2169 struct nlattr *tca[TCA_MAX + 1];
Cong Wang6f96c3c2019-10-07 13:26:28 -07002170 char name[IFNAMSIZ];
Vlad Buslovc431f892018-05-31 09:52:53 +03002171 struct tcmsg *t;
2172 u32 protocol;
2173 u32 prio;
2174 u32 parent;
2175 u32 chain_index;
2176 struct Qdisc *q = NULL;
2177 struct tcf_chain_info chain_info;
2178 struct tcf_chain *chain = NULL;
Vlad Buslov470502d2019-02-11 10:55:48 +02002179 struct tcf_block *block = NULL;
Vlad Buslovc431f892018-05-31 09:52:53 +03002180 struct tcf_proto *tp = NULL;
2181 unsigned long cl = 0;
2182 void *fh = NULL;
2183 int err;
Vlad Buslov470502d2019-02-11 10:55:48 +02002184 bool rtnl_held = false;
Vlad Buslovc431f892018-05-31 09:52:53 +03002185
2186 if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
2187 return -EPERM;
2188
Johannes Berg8cb08172019-04-26 14:07:28 +02002189 err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
2190 rtm_tca_policy, extack);
Vlad Buslovc431f892018-05-31 09:52:53 +03002191 if (err < 0)
2192 return err;
2193
2194 t = nlmsg_data(n);
2195 protocol = TC_H_MIN(t->tcm_info);
2196 prio = TC_H_MAJ(t->tcm_info);
2197 parent = t->tcm_parent;
2198
2199 if (prio == 0 && (protocol || t->tcm_handle || tca[TCA_KIND])) {
2200 NL_SET_ERR_MSG(extack, "Cannot flush filters with protocol, handle or kind set");
2201 return -ENOENT;
2202 }
2203
2204 /* Find head of filter chain. */
2205
Vlad Buslov470502d2019-02-11 10:55:48 +02002206 err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
2207 if (err)
2208 return err;
2209
Cong Wang6f96c3c2019-10-07 13:26:28 -07002210 if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
2211 NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
2212 err = -EINVAL;
2213 goto errout;
2214 }
Vlad Buslov470502d2019-02-11 10:55:48 +02002215 /* Take rtnl mutex if flushing whole chain, block is shared (no qdisc
2216 * found), qdisc is not unlocked, classifier type is not specified,
2217 * classifier is not unlocked.
2218 */
2219 if (!prio ||
2220 (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
Cong Wang6f96c3c2019-10-07 13:26:28 -07002221 !tcf_proto_is_unlocked(name)) {
Vlad Buslov470502d2019-02-11 10:55:48 +02002222 rtnl_held = true;
2223 rtnl_lock();
2224 }
2225
2226 err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
2227 if (err)
2228 goto errout;
2229
2230 block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
2231 extack);
Vlad Buslovc431f892018-05-31 09:52:53 +03002232 if (IS_ERR(block)) {
2233 err = PTR_ERR(block);
2234 goto errout;
2235 }
2236
2237 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2238 if (chain_index > TC_ACT_EXT_VAL_MASK) {
2239 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2240 err = -EINVAL;
2241 goto errout;
2242 }
2243 chain = tcf_chain_get(block, chain_index, false);
2244 if (!chain) {
Jiri Pirko5ca8a252018-08-03 11:08:47 +02002245 /* User requested flush on non-existent chain. Nothing to do,
2246 * so just return success.
2247 */
2248 if (prio == 0) {
2249 err = 0;
2250 goto errout;
2251 }
Vlad Buslovc431f892018-05-31 09:52:53 +03002252 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
Jiri Pirkob7b42472018-08-27 20:58:44 +02002253 err = -ENOENT;
Vlad Buslovc431f892018-05-31 09:52:53 +03002254 goto errout;
2255 }
2256
2257 if (prio == 0) {
2258 tfilter_notify_chain(net, skb, block, q, parent, n,
Vlad Buslov12db03b2019-02-11 10:55:45 +02002259 chain, RTM_DELTFILTER, rtnl_held);
2260 tcf_chain_flush(chain, rtnl_held);
Vlad Buslovc431f892018-05-31 09:52:53 +03002261 err = 0;
2262 goto errout;
2263 }
2264
Vlad Busloved76f5e2019-02-11 10:55:38 +02002265 mutex_lock(&chain->filter_chain_lock);
Vlad Buslovc431f892018-05-31 09:52:53 +03002266 tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2267 prio, false);
2268 if (!tp || IS_ERR(tp)) {
2269 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
Vlad Buslov0e399032018-06-04 18:32:23 +03002270 err = tp ? PTR_ERR(tp) : -ENOENT;
Vlad Busloved76f5e2019-02-11 10:55:38 +02002271 goto errout_locked;
Vlad Buslovc431f892018-05-31 09:52:53 +03002272 } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
2273 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
2274 err = -EINVAL;
Vlad Busloved76f5e2019-02-11 10:55:38 +02002275 goto errout_locked;
2276 } else if (t->tcm_handle == 0) {
John Hurley59eb87c2019-11-02 14:17:47 +00002277 tcf_proto_signal_destroying(chain, tp);
Vlad Busloved76f5e2019-02-11 10:55:38 +02002278 tcf_chain_tp_remove(chain, &chain_info, tp);
2279 mutex_unlock(&chain->filter_chain_lock);
2280
Vlad Buslov12db03b2019-02-11 10:55:45 +02002281 tcf_proto_put(tp, rtnl_held, NULL);
Vlad Busloved76f5e2019-02-11 10:55:38 +02002282 tfilter_notify(net, skb, n, tp, block, q, parent, fh,
Vlad Buslov12db03b2019-02-11 10:55:45 +02002283 RTM_DELTFILTER, false, rtnl_held);
Vlad Busloved76f5e2019-02-11 10:55:38 +02002284 err = 0;
Vlad Buslovc431f892018-05-31 09:52:53 +03002285 goto errout;
2286 }
Vlad Busloved76f5e2019-02-11 10:55:38 +02002287 mutex_unlock(&chain->filter_chain_lock);
Vlad Buslovc431f892018-05-31 09:52:53 +03002288
2289 fh = tp->ops->get(tp, t->tcm_handle);
2290
2291 if (!fh) {
Vlad Busloved76f5e2019-02-11 10:55:38 +02002292 NL_SET_ERR_MSG(extack, "Specified filter handle not found");
2293 err = -ENOENT;
Vlad Buslovc431f892018-05-31 09:52:53 +03002294 } else {
2295 bool last;
2296
2297 err = tfilter_del_notify(net, skb, n, tp, block,
2298 q, parent, fh, false, &last,
Vlad Buslov12db03b2019-02-11 10:55:45 +02002299 rtnl_held, extack);
2300
Vlad Buslovc431f892018-05-31 09:52:53 +03002301 if (err)
2302 goto errout;
Vlad Buslov8b646782019-02-11 10:55:41 +02002303 if (last)
Vlad Buslov12db03b2019-02-11 10:55:45 +02002304 tcf_chain_tp_delete_empty(chain, tp, rtnl_held, extack);
Vlad Buslovc431f892018-05-31 09:52:53 +03002305 }
2306
2307errout:
Vlad Buslov4dbfa762019-02-11 10:55:39 +02002308 if (chain) {
2309 if (tp && !IS_ERR(tp))
Vlad Buslov12db03b2019-02-11 10:55:45 +02002310 tcf_proto_put(tp, rtnl_held, NULL);
Vlad Buslovc431f892018-05-31 09:52:53 +03002311 tcf_chain_put(chain);
Vlad Buslov4dbfa762019-02-11 10:55:39 +02002312 }
Vlad Buslov12db03b2019-02-11 10:55:45 +02002313 tcf_block_release(q, block, rtnl_held);
Vlad Buslov470502d2019-02-11 10:55:48 +02002314
2315 if (rtnl_held)
2316 rtnl_unlock();
2317
Vlad Buslovc431f892018-05-31 09:52:53 +03002318 return err;
Vlad Busloved76f5e2019-02-11 10:55:38 +02002319
2320errout_locked:
2321 mutex_unlock(&chain->filter_chain_lock);
2322 goto errout;
Vlad Buslovc431f892018-05-31 09:52:53 +03002323}
2324
2325static int tc_get_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
2326 struct netlink_ext_ack *extack)
2327{
2328 struct net *net = sock_net(skb->sk);
2329 struct nlattr *tca[TCA_MAX + 1];
Cong Wang6f96c3c2019-10-07 13:26:28 -07002330 char name[IFNAMSIZ];
Vlad Buslovc431f892018-05-31 09:52:53 +03002331 struct tcmsg *t;
2332 u32 protocol;
2333 u32 prio;
2334 u32 parent;
2335 u32 chain_index;
2336 struct Qdisc *q = NULL;
2337 struct tcf_chain_info chain_info;
2338 struct tcf_chain *chain = NULL;
Vlad Buslov470502d2019-02-11 10:55:48 +02002339 struct tcf_block *block = NULL;
Vlad Buslovc431f892018-05-31 09:52:53 +03002340 struct tcf_proto *tp = NULL;
2341 unsigned long cl = 0;
2342 void *fh = NULL;
2343 int err;
Vlad Buslov470502d2019-02-11 10:55:48 +02002344 bool rtnl_held = false;
Vlad Buslovc431f892018-05-31 09:52:53 +03002345
Johannes Berg8cb08172019-04-26 14:07:28 +02002346 err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
2347 rtm_tca_policy, extack);
Vlad Buslovc431f892018-05-31 09:52:53 +03002348 if (err < 0)
2349 return err;
2350
2351 t = nlmsg_data(n);
2352 protocol = TC_H_MIN(t->tcm_info);
2353 prio = TC_H_MAJ(t->tcm_info);
2354 parent = t->tcm_parent;
2355
2356 if (prio == 0) {
2357 NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero");
2358 return -ENOENT;
2359 }
2360
2361 /* Find head of filter chain. */
2362
Vlad Buslov470502d2019-02-11 10:55:48 +02002363 err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
2364 if (err)
2365 return err;
2366
Cong Wang6f96c3c2019-10-07 13:26:28 -07002367 if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
2368 NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
2369 err = -EINVAL;
2370 goto errout;
2371 }
Vlad Buslov470502d2019-02-11 10:55:48 +02002372 /* Take rtnl mutex if block is shared (no qdisc found), qdisc is not
2373 * unlocked, classifier type is not specified, classifier is not
2374 * unlocked.
2375 */
2376 if ((q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
Cong Wang6f96c3c2019-10-07 13:26:28 -07002377 !tcf_proto_is_unlocked(name)) {
Vlad Buslov470502d2019-02-11 10:55:48 +02002378 rtnl_held = true;
2379 rtnl_lock();
2380 }
2381
2382 err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
2383 if (err)
2384 goto errout;
2385
2386 block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
2387 extack);
Vlad Buslovc431f892018-05-31 09:52:53 +03002388 if (IS_ERR(block)) {
2389 err = PTR_ERR(block);
2390 goto errout;
2391 }
2392
2393 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2394 if (chain_index > TC_ACT_EXT_VAL_MASK) {
2395 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2396 err = -EINVAL;
2397 goto errout;
2398 }
2399 chain = tcf_chain_get(block, chain_index, false);
2400 if (!chain) {
2401 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
2402 err = -EINVAL;
2403 goto errout;
2404 }
2405
Vlad Busloved76f5e2019-02-11 10:55:38 +02002406 mutex_lock(&chain->filter_chain_lock);
Vlad Buslovc431f892018-05-31 09:52:53 +03002407 tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2408 prio, false);
Vlad Busloved76f5e2019-02-11 10:55:38 +02002409 mutex_unlock(&chain->filter_chain_lock);
Vlad Buslovc431f892018-05-31 09:52:53 +03002410 if (!tp || IS_ERR(tp)) {
2411 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
Vlad Buslov0e399032018-06-04 18:32:23 +03002412 err = tp ? PTR_ERR(tp) : -ENOENT;
Vlad Buslovc431f892018-05-31 09:52:53 +03002413 goto errout;
2414 } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
2415 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
2416 err = -EINVAL;
2417 goto errout;
2418 }
2419
2420 fh = tp->ops->get(tp, t->tcm_handle);
2421
2422 if (!fh) {
2423 NL_SET_ERR_MSG(extack, "Specified filter handle not found");
2424 err = -ENOENT;
2425 } else {
2426 err = tfilter_notify(net, skb, n, tp, block, q, parent,
Vlad Buslov12db03b2019-02-11 10:55:45 +02002427 fh, RTM_NEWTFILTER, true, rtnl_held);
Vlad Buslovc431f892018-05-31 09:52:53 +03002428 if (err < 0)
2429 NL_SET_ERR_MSG(extack, "Failed to send filter notify message");
2430 }
2431
Vlad Buslov7d5509f2019-02-11 10:55:44 +02002432 tfilter_put(tp, fh);
Vlad Buslovc431f892018-05-31 09:52:53 +03002433errout:
Vlad Buslov4dbfa762019-02-11 10:55:39 +02002434 if (chain) {
2435 if (tp && !IS_ERR(tp))
Vlad Buslov12db03b2019-02-11 10:55:45 +02002436 tcf_proto_put(tp, rtnl_held, NULL);
Vlad Buslovc431f892018-05-31 09:52:53 +03002437 tcf_chain_put(chain);
Vlad Buslov4dbfa762019-02-11 10:55:39 +02002438 }
Vlad Buslov12db03b2019-02-11 10:55:45 +02002439 tcf_block_release(q, block, rtnl_held);
Vlad Buslov470502d2019-02-11 10:55:48 +02002440
2441 if (rtnl_held)
2442 rtnl_unlock();
2443
Vlad Buslovc431f892018-05-31 09:52:53 +03002444 return err;
2445}
2446
Stephen Hemmingeraa767bf2008-01-21 02:26:41 -08002447struct tcf_dump_args {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002448 struct tcf_walker w;
2449 struct sk_buff *skb;
2450 struct netlink_callback *cb;
Jiri Pirko7960d1d2018-01-17 11:46:51 +01002451 struct tcf_block *block;
Jiri Pirkoa10fa202017-10-13 14:01:05 +02002452 struct Qdisc *q;
2453 u32 parent;
Vlad Buslovf8ab1802020-05-15 14:40:11 +03002454 bool terse_dump;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002455};
2456
WANG Cong8113c092017-08-04 21:31:43 -07002457static int tcf_node_dump(struct tcf_proto *tp, void *n, struct tcf_walker *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002458{
Stephen Hemmingeraa767bf2008-01-21 02:26:41 -08002459 struct tcf_dump_args *a = (void *)arg;
WANG Cong832d1d52014-01-09 16:14:01 -08002460 struct net *net = sock_net(a->skb->sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002461
Jiri Pirko7960d1d2018-01-17 11:46:51 +01002462 return tcf_fill_node(net, a->skb, tp, a->block, a->q, a->parent,
Jiri Pirkoa10fa202017-10-13 14:01:05 +02002463 n, NETLINK_CB(a->cb->skb).portid,
Jamal Hadi Salim5a7a5552016-09-18 08:45:33 -04002464 a->cb->nlh->nlmsg_seq, NLM_F_MULTI,
Vlad Buslovf8ab1802020-05-15 14:40:11 +03002465 RTM_NEWTFILTER, a->terse_dump, true);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002466}
2467
Jiri Pirkoa10fa202017-10-13 14:01:05 +02002468static bool tcf_chain_dump(struct tcf_chain *chain, struct Qdisc *q, u32 parent,
2469 struct sk_buff *skb, struct netlink_callback *cb,
Vlad Buslovf8ab1802020-05-15 14:40:11 +03002470 long index_start, long *p_index, bool terse)
Jiri Pirkoacb31fa2017-05-17 11:08:00 +02002471{
2472 struct net *net = sock_net(skb->sk);
Jiri Pirko7960d1d2018-01-17 11:46:51 +01002473 struct tcf_block *block = chain->block;
Jiri Pirkoacb31fa2017-05-17 11:08:00 +02002474 struct tcmsg *tcm = nlmsg_data(cb->nlh);
Vlad Buslovfe2923a2019-02-11 10:55:40 +02002475 struct tcf_proto *tp, *tp_prev;
Jiri Pirkoacb31fa2017-05-17 11:08:00 +02002476 struct tcf_dump_args arg;
Jiri Pirkoacb31fa2017-05-17 11:08:00 +02002477
Vlad Buslovfe2923a2019-02-11 10:55:40 +02002478 for (tp = __tcf_get_next_proto(chain, NULL);
2479 tp;
2480 tp_prev = tp,
2481 tp = __tcf_get_next_proto(chain, tp),
Vlad Buslov12db03b2019-02-11 10:55:45 +02002482 tcf_proto_put(tp_prev, true, NULL),
Vlad Buslovfe2923a2019-02-11 10:55:40 +02002483 (*p_index)++) {
Jiri Pirkoacb31fa2017-05-17 11:08:00 +02002484 if (*p_index < index_start)
2485 continue;
2486 if (TC_H_MAJ(tcm->tcm_info) &&
2487 TC_H_MAJ(tcm->tcm_info) != tp->prio)
2488 continue;
2489 if (TC_H_MIN(tcm->tcm_info) &&
2490 TC_H_MIN(tcm->tcm_info) != tp->protocol)
2491 continue;
2492 if (*p_index > index_start)
2493 memset(&cb->args[1], 0,
2494 sizeof(cb->args) - sizeof(cb->args[0]));
2495 if (cb->args[1] == 0) {
YueHaibing53189182018-07-17 20:58:14 +08002496 if (tcf_fill_node(net, skb, tp, block, q, parent, NULL,
Jiri Pirkoacb31fa2017-05-17 11:08:00 +02002497 NETLINK_CB(cb->skb).portid,
2498 cb->nlh->nlmsg_seq, NLM_F_MULTI,
Vlad Buslovf8ab1802020-05-15 14:40:11 +03002499 RTM_NEWTFILTER, false, true) <= 0)
Vlad Buslovfe2923a2019-02-11 10:55:40 +02002500 goto errout;
Jiri Pirkoacb31fa2017-05-17 11:08:00 +02002501 cb->args[1] = 1;
2502 }
2503 if (!tp->ops->walk)
2504 continue;
2505 arg.w.fn = tcf_node_dump;
2506 arg.skb = skb;
2507 arg.cb = cb;
Jiri Pirko7960d1d2018-01-17 11:46:51 +01002508 arg.block = block;
Jiri Pirkoa10fa202017-10-13 14:01:05 +02002509 arg.q = q;
2510 arg.parent = parent;
Jiri Pirkoacb31fa2017-05-17 11:08:00 +02002511 arg.w.stop = 0;
2512 arg.w.skip = cb->args[1] - 1;
2513 arg.w.count = 0;
Vlad Buslov01683a12018-07-09 13:29:11 +03002514 arg.w.cookie = cb->args[2];
Vlad Buslovf8ab1802020-05-15 14:40:11 +03002515 arg.terse_dump = terse;
Vlad Buslov12db03b2019-02-11 10:55:45 +02002516 tp->ops->walk(tp, &arg.w, true);
Vlad Buslov01683a12018-07-09 13:29:11 +03002517 cb->args[2] = arg.w.cookie;
Jiri Pirkoacb31fa2017-05-17 11:08:00 +02002518 cb->args[1] = arg.w.count + 1;
2519 if (arg.w.stop)
Vlad Buslovfe2923a2019-02-11 10:55:40 +02002520 goto errout;
Jiri Pirkoacb31fa2017-05-17 11:08:00 +02002521 }
Jiri Pirko5bc17012017-05-17 11:08:01 +02002522 return true;
Vlad Buslovfe2923a2019-02-11 10:55:40 +02002523
2524errout:
Vlad Buslov12db03b2019-02-11 10:55:45 +02002525 tcf_proto_put(tp, true, NULL);
Vlad Buslovfe2923a2019-02-11 10:55:40 +02002526 return false;
Jiri Pirkoacb31fa2017-05-17 11:08:00 +02002527}
2528
Vlad Buslovf8ab1802020-05-15 14:40:11 +03002529static const struct nla_policy tcf_tfilter_dump_policy[TCA_MAX + 1] = {
2530 [TCA_DUMP_FLAGS] = NLA_POLICY_BITFIELD32(TCA_DUMP_FLAGS_TERSE),
2531};
2532
Eric Dumazetbd27a872009-11-05 20:57:26 -08002533/* called with RTNL */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002534static int tc_dump_tfilter(struct sk_buff *skb, struct netlink_callback *cb)
2535{
Vlad Buslovbbf73832019-02-11 10:55:36 +02002536 struct tcf_chain *chain, *chain_prev;
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09002537 struct net *net = sock_net(skb->sk);
Jiri Pirko5bc17012017-05-17 11:08:01 +02002538 struct nlattr *tca[TCA_MAX + 1];
Jiri Pirko7960d1d2018-01-17 11:46:51 +01002539 struct Qdisc *q = NULL;
Jiri Pirko6529eab2017-05-17 11:07:55 +02002540 struct tcf_block *block;
David S. Miller942b8162012-06-26 21:48:50 -07002541 struct tcmsg *tcm = nlmsg_data(cb->nlh);
Vlad Buslovf8ab1802020-05-15 14:40:11 +03002542 bool terse_dump = false;
Jiri Pirkoacb31fa2017-05-17 11:08:00 +02002543 long index_start;
2544 long index;
Jiri Pirkoa10fa202017-10-13 14:01:05 +02002545 u32 parent;
Jiri Pirko5bc17012017-05-17 11:08:01 +02002546 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002547
Hong zhi guo573ce262013-03-27 06:47:04 +00002548 if (nlmsg_len(cb->nlh) < sizeof(*tcm))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002549 return skb->len;
Jiri Pirko5bc17012017-05-17 11:08:01 +02002550
Johannes Berg8cb08172019-04-26 14:07:28 +02002551 err = nlmsg_parse_deprecated(cb->nlh, sizeof(*tcm), tca, TCA_MAX,
Vlad Buslovf8ab1802020-05-15 14:40:11 +03002552 tcf_tfilter_dump_policy, cb->extack);
Jiri Pirko5bc17012017-05-17 11:08:01 +02002553 if (err)
2554 return err;
2555
Vlad Buslovf8ab1802020-05-15 14:40:11 +03002556 if (tca[TCA_DUMP_FLAGS]) {
2557 struct nla_bitfield32 flags =
2558 nla_get_bitfield32(tca[TCA_DUMP_FLAGS]);
2559
2560 terse_dump = flags.value & TCA_DUMP_FLAGS_TERSE;
2561 }
2562
Jiri Pirko7960d1d2018-01-17 11:46:51 +01002563 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
Vlad Buslov787ce6d2018-09-24 19:22:58 +03002564 block = tcf_block_refcnt_get(net, tcm->tcm_block_index);
Jiri Pirko7960d1d2018-01-17 11:46:51 +01002565 if (!block)
WANG Cong143976c2017-08-24 16:51:29 -07002566 goto out;
Jiri Pirkod680b352018-01-18 16:14:49 +01002567 /* If we work with block index, q is NULL and parent value
2568 * will never be used in the following code. The check
2569 * in tcf_fill_node prevents it. However, compiler does not
2570 * see that far, so set parent to zero to silence the warning
2571 * about parent being uninitialized.
2572 */
2573 parent = 0;
Jiri Pirko7960d1d2018-01-17 11:46:51 +01002574 } else {
2575 const struct Qdisc_class_ops *cops;
2576 struct net_device *dev;
2577 unsigned long cl = 0;
2578
2579 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
2580 if (!dev)
2581 return skb->len;
2582
2583 parent = tcm->tcm_parent;
Cong Wanga7df4872020-04-30 20:53:49 -07002584 if (!parent)
Jiri Pirko7960d1d2018-01-17 11:46:51 +01002585 q = dev->qdisc;
Cong Wanga7df4872020-04-30 20:53:49 -07002586 else
Jiri Pirko7960d1d2018-01-17 11:46:51 +01002587 q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
Jiri Pirko7960d1d2018-01-17 11:46:51 +01002588 if (!q)
2589 goto out;
2590 cops = q->ops->cl_ops;
2591 if (!cops)
2592 goto out;
2593 if (!cops->tcf_block)
2594 goto out;
2595 if (TC_H_MIN(tcm->tcm_parent)) {
2596 cl = cops->find(q, tcm->tcm_parent);
2597 if (cl == 0)
2598 goto out;
2599 }
2600 block = cops->tcf_block(q, cl, NULL);
2601 if (!block)
2602 goto out;
Cong Wanga7df4872020-04-30 20:53:49 -07002603 parent = block->classid;
Jiri Pirko7960d1d2018-01-17 11:46:51 +01002604 if (tcf_block_shared(block))
2605 q = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002606 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002607
Jiri Pirkoacb31fa2017-05-17 11:08:00 +02002608 index_start = cb->args[0];
2609 index = 0;
Jiri Pirko5bc17012017-05-17 11:08:01 +02002610
Vlad Buslovbbf73832019-02-11 10:55:36 +02002611 for (chain = __tcf_get_next_chain(block, NULL);
2612 chain;
2613 chain_prev = chain,
2614 chain = __tcf_get_next_chain(block, chain),
2615 tcf_chain_put(chain_prev)) {
Jiri Pirko5bc17012017-05-17 11:08:01 +02002616 if (tca[TCA_CHAIN] &&
2617 nla_get_u32(tca[TCA_CHAIN]) != chain->index)
2618 continue;
Jiri Pirkoa10fa202017-10-13 14:01:05 +02002619 if (!tcf_chain_dump(chain, q, parent, skb, cb,
Vlad Buslovf8ab1802020-05-15 14:40:11 +03002620 index_start, &index, terse_dump)) {
Vlad Buslovbbf73832019-02-11 10:55:36 +02002621 tcf_chain_put(chain);
Roman Kapl5ae437a2018-02-19 21:32:51 +01002622 err = -EMSGSIZE;
Jiri Pirko5bc17012017-05-17 11:08:01 +02002623 break;
Roman Kapl5ae437a2018-02-19 21:32:51 +01002624 }
Jiri Pirko5bc17012017-05-17 11:08:01 +02002625 }
2626
Vlad Buslov787ce6d2018-09-24 19:22:58 +03002627 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
Vlad Buslov12db03b2019-02-11 10:55:45 +02002628 tcf_block_refcnt_put(block, true);
Jiri Pirkoacb31fa2017-05-17 11:08:00 +02002629 cb->args[0] = index;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002630
Linus Torvalds1da177e2005-04-16 15:20:36 -07002631out:
Roman Kapl5ae437a2018-02-19 21:32:51 +01002632 /* If we did no progress, the error (EMSGSIZE) is real */
2633 if (skb->len == 0 && err)
2634 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002635 return skb->len;
2636}
2637
Vlad Buslova5654822019-02-11 10:55:37 +02002638static int tc_chain_fill_node(const struct tcf_proto_ops *tmplt_ops,
2639 void *tmplt_priv, u32 chain_index,
2640 struct net *net, struct sk_buff *skb,
2641 struct tcf_block *block,
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002642 u32 portid, u32 seq, u16 flags, int event)
2643{
2644 unsigned char *b = skb_tail_pointer(skb);
Jiri Pirko9f407f12018-07-23 09:23:07 +02002645 const struct tcf_proto_ops *ops;
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002646 struct nlmsghdr *nlh;
2647 struct tcmsg *tcm;
Jiri Pirko9f407f12018-07-23 09:23:07 +02002648 void *priv;
2649
Vlad Buslova5654822019-02-11 10:55:37 +02002650 ops = tmplt_ops;
2651 priv = tmplt_priv;
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002652
2653 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
2654 if (!nlh)
2655 goto out_nlmsg_trim;
2656 tcm = nlmsg_data(nlh);
2657 tcm->tcm_family = AF_UNSPEC;
2658 tcm->tcm__pad1 = 0;
2659 tcm->tcm__pad2 = 0;
2660 tcm->tcm_handle = 0;
2661 if (block->q) {
2662 tcm->tcm_ifindex = qdisc_dev(block->q)->ifindex;
2663 tcm->tcm_parent = block->q->handle;
2664 } else {
2665 tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK;
2666 tcm->tcm_block_index = block->index;
2667 }
2668
Vlad Buslova5654822019-02-11 10:55:37 +02002669 if (nla_put_u32(skb, TCA_CHAIN, chain_index))
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002670 goto nla_put_failure;
2671
Jiri Pirko9f407f12018-07-23 09:23:07 +02002672 if (ops) {
2673 if (nla_put_string(skb, TCA_KIND, ops->kind))
2674 goto nla_put_failure;
2675 if (ops->tmplt_dump(skb, net, priv) < 0)
2676 goto nla_put_failure;
2677 }
2678
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002679 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
2680 return skb->len;
2681
2682out_nlmsg_trim:
2683nla_put_failure:
2684 nlmsg_trim(skb, b);
2685 return -EMSGSIZE;
2686}
2687
2688static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
2689 u32 seq, u16 flags, int event, bool unicast)
2690{
2691 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
2692 struct tcf_block *block = chain->block;
2693 struct net *net = block->net;
2694 struct sk_buff *skb;
Zhike Wang5b5f99b2019-03-11 03:15:54 -07002695 int err = 0;
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002696
2697 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2698 if (!skb)
2699 return -ENOBUFS;
2700
Vlad Buslova5654822019-02-11 10:55:37 +02002701 if (tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv,
2702 chain->index, net, skb, block, portid,
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002703 seq, flags, event) <= 0) {
2704 kfree_skb(skb);
2705 return -EINVAL;
2706 }
2707
2708 if (unicast)
Zhike Wang5b5f99b2019-03-11 03:15:54 -07002709 err = netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
2710 else
2711 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
2712 flags & NLM_F_ECHO);
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002713
Zhike Wang5b5f99b2019-03-11 03:15:54 -07002714 if (err > 0)
2715 err = 0;
2716 return err;
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002717}
2718
Vlad Buslova5654822019-02-11 10:55:37 +02002719static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops,
2720 void *tmplt_priv, u32 chain_index,
2721 struct tcf_block *block, struct sk_buff *oskb,
2722 u32 seq, u16 flags, bool unicast)
2723{
2724 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
2725 struct net *net = block->net;
2726 struct sk_buff *skb;
2727
2728 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2729 if (!skb)
2730 return -ENOBUFS;
2731
2732 if (tc_chain_fill_node(tmplt_ops, tmplt_priv, chain_index, net, skb,
2733 block, portid, seq, flags, RTM_DELCHAIN) <= 0) {
2734 kfree_skb(skb);
2735 return -EINVAL;
2736 }
2737
2738 if (unicast)
2739 return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
2740
2741 return rtnetlink_send(skb, net, portid, RTNLGRP_TC, flags & NLM_F_ECHO);
2742}
2743
Jiri Pirko9f407f12018-07-23 09:23:07 +02002744static int tc_chain_tmplt_add(struct tcf_chain *chain, struct net *net,
2745 struct nlattr **tca,
2746 struct netlink_ext_ack *extack)
2747{
2748 const struct tcf_proto_ops *ops;
Eric Dumazet2dd56162019-12-07 11:34:45 -08002749 char name[IFNAMSIZ];
Jiri Pirko9f407f12018-07-23 09:23:07 +02002750 void *tmplt_priv;
2751
2752 /* If kind is not set, user did not specify template. */
2753 if (!tca[TCA_KIND])
2754 return 0;
2755
Eric Dumazet2dd56162019-12-07 11:34:45 -08002756 if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
2757 NL_SET_ERR_MSG(extack, "Specified TC chain template name too long");
2758 return -EINVAL;
2759 }
2760
2761 ops = tcf_proto_lookup_ops(name, true, extack);
Jiri Pirko9f407f12018-07-23 09:23:07 +02002762 if (IS_ERR(ops))
2763 return PTR_ERR(ops);
2764 if (!ops->tmplt_create || !ops->tmplt_destroy || !ops->tmplt_dump) {
2765 NL_SET_ERR_MSG(extack, "Chain templates are not supported with specified classifier");
2766 return -EOPNOTSUPP;
2767 }
2768
2769 tmplt_priv = ops->tmplt_create(net, chain, tca, extack);
2770 if (IS_ERR(tmplt_priv)) {
2771 module_put(ops->owner);
2772 return PTR_ERR(tmplt_priv);
2773 }
2774 chain->tmplt_ops = ops;
2775 chain->tmplt_priv = tmplt_priv;
2776 return 0;
2777}
2778
Vlad Buslova5654822019-02-11 10:55:37 +02002779static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops,
2780 void *tmplt_priv)
Jiri Pirko9f407f12018-07-23 09:23:07 +02002781{
Jiri Pirko9f407f12018-07-23 09:23:07 +02002782 /* If template ops are set, no work to do for us. */
Vlad Buslova5654822019-02-11 10:55:37 +02002783 if (!tmplt_ops)
Jiri Pirko9f407f12018-07-23 09:23:07 +02002784 return;
2785
Vlad Buslova5654822019-02-11 10:55:37 +02002786 tmplt_ops->tmplt_destroy(tmplt_priv);
2787 module_put(tmplt_ops->owner);
Jiri Pirko9f407f12018-07-23 09:23:07 +02002788}
2789
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002790/* Add/delete/get a chain */
2791
2792static int tc_ctl_chain(struct sk_buff *skb, struct nlmsghdr *n,
2793 struct netlink_ext_ack *extack)
2794{
2795 struct net *net = sock_net(skb->sk);
2796 struct nlattr *tca[TCA_MAX + 1];
2797 struct tcmsg *t;
2798 u32 parent;
2799 u32 chain_index;
2800 struct Qdisc *q = NULL;
2801 struct tcf_chain *chain = NULL;
2802 struct tcf_block *block;
2803 unsigned long cl;
2804 int err;
2805
2806 if (n->nlmsg_type != RTM_GETCHAIN &&
2807 !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
2808 return -EPERM;
2809
2810replay:
Johannes Berg8cb08172019-04-26 14:07:28 +02002811 err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
2812 rtm_tca_policy, extack);
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002813 if (err < 0)
2814 return err;
2815
2816 t = nlmsg_data(n);
2817 parent = t->tcm_parent;
2818 cl = 0;
2819
2820 block = tcf_block_find(net, &q, &parent, &cl,
2821 t->tcm_ifindex, t->tcm_block_index, extack);
2822 if (IS_ERR(block))
2823 return PTR_ERR(block);
2824
2825 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2826 if (chain_index > TC_ACT_EXT_VAL_MASK) {
2827 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
Vlad Buslove368fdb2018-09-24 19:22:53 +03002828 err = -EINVAL;
2829 goto errout_block;
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002830 }
Vlad Buslov2cbfab02019-02-11 10:55:34 +02002831
2832 mutex_lock(&block->lock);
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002833 chain = tcf_chain_lookup(block, chain_index);
2834 if (n->nlmsg_type == RTM_NEWCHAIN) {
2835 if (chain) {
Jiri Pirko3d32f4c2018-08-01 12:36:55 +02002836 if (tcf_chain_held_by_acts_only(chain)) {
Jiri Pirko1f3ed382018-07-27 09:45:05 +02002837 /* The chain exists only because there is
Jiri Pirko3d32f4c2018-08-01 12:36:55 +02002838 * some action referencing it.
Jiri Pirko1f3ed382018-07-27 09:45:05 +02002839 */
2840 tcf_chain_hold(chain);
2841 } else {
2842 NL_SET_ERR_MSG(extack, "Filter chain already exists");
Vlad Buslove368fdb2018-09-24 19:22:53 +03002843 err = -EEXIST;
Vlad Buslov2cbfab02019-02-11 10:55:34 +02002844 goto errout_block_locked;
Jiri Pirko1f3ed382018-07-27 09:45:05 +02002845 }
2846 } else {
2847 if (!(n->nlmsg_flags & NLM_F_CREATE)) {
2848 NL_SET_ERR_MSG(extack, "Need both RTM_NEWCHAIN and NLM_F_CREATE to create a new chain");
Vlad Buslove368fdb2018-09-24 19:22:53 +03002849 err = -ENOENT;
Vlad Buslov2cbfab02019-02-11 10:55:34 +02002850 goto errout_block_locked;
Jiri Pirko1f3ed382018-07-27 09:45:05 +02002851 }
2852 chain = tcf_chain_create(block, chain_index);
2853 if (!chain) {
2854 NL_SET_ERR_MSG(extack, "Failed to create filter chain");
Vlad Buslove368fdb2018-09-24 19:22:53 +03002855 err = -ENOMEM;
Vlad Buslov2cbfab02019-02-11 10:55:34 +02002856 goto errout_block_locked;
Jiri Pirko1f3ed382018-07-27 09:45:05 +02002857 }
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002858 }
2859 } else {
Jiri Pirko3d32f4c2018-08-01 12:36:55 +02002860 if (!chain || tcf_chain_held_by_acts_only(chain)) {
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002861 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
Vlad Buslove368fdb2018-09-24 19:22:53 +03002862 err = -EINVAL;
Vlad Buslov2cbfab02019-02-11 10:55:34 +02002863 goto errout_block_locked;
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002864 }
2865 tcf_chain_hold(chain);
2866 }
2867
Vlad Buslov2cbfab02019-02-11 10:55:34 +02002868 if (n->nlmsg_type == RTM_NEWCHAIN) {
2869 /* Modifying chain requires holding parent block lock. In case
2870 * the chain was successfully added, take a reference to the
2871 * chain. This ensures that an empty chain does not disappear at
2872 * the end of this function.
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002873 */
2874 tcf_chain_hold(chain);
2875 chain->explicitly_created = true;
Vlad Buslov2cbfab02019-02-11 10:55:34 +02002876 }
2877 mutex_unlock(&block->lock);
2878
2879 switch (n->nlmsg_type) {
2880 case RTM_NEWCHAIN:
2881 err = tc_chain_tmplt_add(chain, net, tca, extack);
2882 if (err) {
2883 tcf_chain_put_explicitly_created(chain);
2884 goto errout;
2885 }
2886
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002887 tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
2888 RTM_NEWCHAIN, false);
2889 break;
2890 case RTM_DELCHAIN:
Cong Wangf5b9bac2018-09-11 14:22:23 -07002891 tfilter_notify_chain(net, skb, block, q, parent, n,
Vlad Buslov12db03b2019-02-11 10:55:45 +02002892 chain, RTM_DELTFILTER, true);
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002893 /* Flush the chain first as the user requested chain removal. */
Vlad Buslov12db03b2019-02-11 10:55:45 +02002894 tcf_chain_flush(chain, true);
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002895 /* In case the chain was successfully deleted, put a reference
2896 * to the chain previously taken during addition.
2897 */
2898 tcf_chain_put_explicitly_created(chain);
2899 break;
2900 case RTM_GETCHAIN:
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002901 err = tc_chain_notify(chain, skb, n->nlmsg_seq,
2902 n->nlmsg_seq, n->nlmsg_type, true);
2903 if (err < 0)
2904 NL_SET_ERR_MSG(extack, "Failed to send chain notify message");
2905 break;
2906 default:
2907 err = -EOPNOTSUPP;
2908 NL_SET_ERR_MSG(extack, "Unsupported message type");
2909 goto errout;
2910 }
2911
2912errout:
2913 tcf_chain_put(chain);
Vlad Buslove368fdb2018-09-24 19:22:53 +03002914errout_block:
Vlad Buslov12db03b2019-02-11 10:55:45 +02002915 tcf_block_release(q, block, true);
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002916 if (err == -EAGAIN)
2917 /* Replay the request. */
2918 goto replay;
2919 return err;
Vlad Buslov2cbfab02019-02-11 10:55:34 +02002920
2921errout_block_locked:
2922 mutex_unlock(&block->lock);
2923 goto errout_block;
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002924}
2925
2926/* called with RTNL */
2927static int tc_dump_chain(struct sk_buff *skb, struct netlink_callback *cb)
2928{
2929 struct net *net = sock_net(skb->sk);
2930 struct nlattr *tca[TCA_MAX + 1];
2931 struct Qdisc *q = NULL;
2932 struct tcf_block *block;
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002933 struct tcmsg *tcm = nlmsg_data(cb->nlh);
Vlad Buslovace4a262019-02-25 17:45:44 +02002934 struct tcf_chain *chain;
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002935 long index_start;
2936 long index;
2937 u32 parent;
2938 int err;
2939
2940 if (nlmsg_len(cb->nlh) < sizeof(*tcm))
2941 return skb->len;
2942
Johannes Berg8cb08172019-04-26 14:07:28 +02002943 err = nlmsg_parse_deprecated(cb->nlh, sizeof(*tcm), tca, TCA_MAX,
2944 rtm_tca_policy, cb->extack);
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002945 if (err)
2946 return err;
2947
2948 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
Vlad Buslov787ce6d2018-09-24 19:22:58 +03002949 block = tcf_block_refcnt_get(net, tcm->tcm_block_index);
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002950 if (!block)
2951 goto out;
2952 /* If we work with block index, q is NULL and parent value
2953 * will never be used in the following code. The check
2954 * in tcf_fill_node prevents it. However, compiler does not
2955 * see that far, so set parent to zero to silence the warning
2956 * about parent being uninitialized.
2957 */
2958 parent = 0;
2959 } else {
2960 const struct Qdisc_class_ops *cops;
2961 struct net_device *dev;
2962 unsigned long cl = 0;
2963
2964 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
2965 if (!dev)
2966 return skb->len;
2967
2968 parent = tcm->tcm_parent;
2969 if (!parent) {
2970 q = dev->qdisc;
2971 parent = q->handle;
2972 } else {
2973 q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
2974 }
2975 if (!q)
2976 goto out;
2977 cops = q->ops->cl_ops;
2978 if (!cops)
2979 goto out;
2980 if (!cops->tcf_block)
2981 goto out;
2982 if (TC_H_MIN(tcm->tcm_parent)) {
2983 cl = cops->find(q, tcm->tcm_parent);
2984 if (cl == 0)
2985 goto out;
2986 }
2987 block = cops->tcf_block(q, cl, NULL);
2988 if (!block)
2989 goto out;
2990 if (tcf_block_shared(block))
2991 q = NULL;
2992 }
2993
2994 index_start = cb->args[0];
2995 index = 0;
2996
Vlad Buslovace4a262019-02-25 17:45:44 +02002997 mutex_lock(&block->lock);
2998 list_for_each_entry(chain, &block->chain_list, list) {
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002999 if ((tca[TCA_CHAIN] &&
3000 nla_get_u32(tca[TCA_CHAIN]) != chain->index))
3001 continue;
3002 if (index < index_start) {
3003 index++;
3004 continue;
3005 }
Vlad Buslovace4a262019-02-25 17:45:44 +02003006 if (tcf_chain_held_by_acts_only(chain))
3007 continue;
Vlad Buslova5654822019-02-11 10:55:37 +02003008 err = tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv,
3009 chain->index, net, skb, block,
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02003010 NETLINK_CB(cb->skb).portid,
3011 cb->nlh->nlmsg_seq, NLM_F_MULTI,
3012 RTM_NEWCHAIN);
Vlad Buslovace4a262019-02-25 17:45:44 +02003013 if (err <= 0)
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02003014 break;
3015 index++;
3016 }
Vlad Buslovace4a262019-02-25 17:45:44 +02003017 mutex_unlock(&block->lock);
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02003018
Vlad Buslov787ce6d2018-09-24 19:22:58 +03003019 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
Vlad Buslov12db03b2019-02-11 10:55:45 +02003020 tcf_block_refcnt_put(block, true);
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02003021 cb->args[0] = index;
3022
3023out:
3024 /* If we did no progress, the error (EMSGSIZE) is real */
3025 if (skb->len == 0 && err)
3026 return err;
3027 return skb->len;
3028}
3029
WANG Cong18d02642014-09-25 10:26:37 -07003030void tcf_exts_destroy(struct tcf_exts *exts)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003031{
3032#ifdef CONFIG_NET_CLS_ACT
Eric Dumazet3d66b892019-09-18 12:57:04 -07003033 if (exts->actions) {
3034 tcf_action_destroy(exts->actions, TCA_ACT_UNBIND);
3035 kfree(exts->actions);
3036 }
WANG Cong22dc13c2016-08-13 22:35:00 -07003037 exts->nr_actions = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003038#endif
3039}
Stephen Hemmingeraa767bf2008-01-21 02:26:41 -08003040EXPORT_SYMBOL(tcf_exts_destroy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003041
Benjamin LaHaisec1b52732013-01-14 05:15:39 +00003042int tcf_exts_validate(struct net *net, struct tcf_proto *tp, struct nlattr **tb,
Alexander Aring50a56192018-01-18 11:20:52 -05003043 struct nlattr *rate_tlv, struct tcf_exts *exts, bool ovr,
Vlad Buslovec6743a2019-02-11 10:55:43 +02003044 bool rtnl_held, struct netlink_ext_ack *extack)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003045{
Linus Torvalds1da177e2005-04-16 15:20:36 -07003046#ifdef CONFIG_NET_CLS_ACT
3047 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003048 struct tc_action *act;
Roman Mashakd04e6992018-03-08 16:59:17 -05003049 size_t attr_size = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003050
WANG Cong5da57f42013-12-15 20:15:07 -08003051 if (exts->police && tb[exts->police]) {
Jiri Pirko9fb9f252017-05-17 11:08:02 +02003052 act = tcf_action_init_1(net, tp, tb[exts->police],
3053 rate_tlv, "police", ovr,
Vlad Buslovec6743a2019-02-11 10:55:43 +02003054 TCA_ACT_BIND, rtnl_held,
3055 extack);
Patrick McHardyab27cfb2008-01-23 20:33:13 -08003056 if (IS_ERR(act))
3057 return PTR_ERR(act);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003058
WANG Cong33be6272013-12-15 20:15:05 -08003059 act->type = exts->type = TCA_OLD_COMPAT;
WANG Cong22dc13c2016-08-13 22:35:00 -07003060 exts->actions[0] = act;
3061 exts->nr_actions = 1;
WANG Cong5da57f42013-12-15 20:15:07 -08003062 } else if (exts->action && tb[exts->action]) {
Vlad Buslov90b73b72018-07-05 17:24:33 +03003063 int err;
WANG Cong22dc13c2016-08-13 22:35:00 -07003064
Jiri Pirko9fb9f252017-05-17 11:08:02 +02003065 err = tcf_action_init(net, tp, tb[exts->action],
3066 rate_tlv, NULL, ovr, TCA_ACT_BIND,
Vlad Buslovec6743a2019-02-11 10:55:43 +02003067 exts->actions, &attr_size,
3068 rtnl_held, extack);
Vlad Buslov90b73b72018-07-05 17:24:33 +03003069 if (err < 0)
WANG Cong33be6272013-12-15 20:15:05 -08003070 return err;
Vlad Buslov90b73b72018-07-05 17:24:33 +03003071 exts->nr_actions = err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003072 }
3073 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003074#else
WANG Cong5da57f42013-12-15 20:15:07 -08003075 if ((exts->action && tb[exts->action]) ||
Alexander Aring50a56192018-01-18 11:20:52 -05003076 (exts->police && tb[exts->police])) {
3077 NL_SET_ERR_MSG(extack, "Classifier actions are not supported per compile options (CONFIG_NET_CLS_ACT)");
Linus Torvalds1da177e2005-04-16 15:20:36 -07003078 return -EOPNOTSUPP;
Alexander Aring50a56192018-01-18 11:20:52 -05003079 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003080#endif
3081
3082 return 0;
3083}
Stephen Hemmingeraa767bf2008-01-21 02:26:41 -08003084EXPORT_SYMBOL(tcf_exts_validate);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003085
Jiri Pirko9b0d4442017-08-04 14:29:15 +02003086void tcf_exts_change(struct tcf_exts *dst, struct tcf_exts *src)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003087{
3088#ifdef CONFIG_NET_CLS_ACT
WANG Cong22dc13c2016-08-13 22:35:00 -07003089 struct tcf_exts old = *dst;
3090
Jiri Pirko9b0d4442017-08-04 14:29:15 +02003091 *dst = *src;
WANG Cong22dc13c2016-08-13 22:35:00 -07003092 tcf_exts_destroy(&old);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003093#endif
3094}
Stephen Hemmingeraa767bf2008-01-21 02:26:41 -08003095EXPORT_SYMBOL(tcf_exts_change);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003096
WANG Cong22dc13c2016-08-13 22:35:00 -07003097#ifdef CONFIG_NET_CLS_ACT
3098static struct tc_action *tcf_exts_first_act(struct tcf_exts *exts)
3099{
3100 if (exts->nr_actions == 0)
3101 return NULL;
3102 else
3103 return exts->actions[0];
3104}
3105#endif
WANG Cong33be6272013-12-15 20:15:05 -08003106
WANG Cong5da57f42013-12-15 20:15:07 -08003107int tcf_exts_dump(struct sk_buff *skb, struct tcf_exts *exts)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003108{
3109#ifdef CONFIG_NET_CLS_ACT
Cong Wang9cc63db2014-07-16 14:25:30 -07003110 struct nlattr *nest;
3111
Jiri Pirko978dfd82017-08-04 14:29:03 +02003112 if (exts->action && tcf_exts_has_actions(exts)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003113 /*
3114 * again for backward compatible mode - we want
3115 * to work with both old and new modes of entering
3116 * tc data even if iproute2 was newer - jhs
3117 */
WANG Cong33be6272013-12-15 20:15:05 -08003118 if (exts->type != TCA_OLD_COMPAT) {
Michal Kubecekae0be8d2019-04-26 11:13:06 +02003119 nest = nla_nest_start_noflag(skb, exts->action);
Patrick McHardy4b3550ef2008-01-23 20:34:11 -08003120 if (nest == NULL)
3121 goto nla_put_failure;
WANG Cong22dc13c2016-08-13 22:35:00 -07003122
Vlad Buslovca44b732020-05-15 14:40:12 +03003123 if (tcf_action_dump(skb, exts->actions, 0, 0, false)
3124 < 0)
Patrick McHardyadd93b62008-01-22 22:11:33 -08003125 goto nla_put_failure;
Patrick McHardy4b3550ef2008-01-23 20:34:11 -08003126 nla_nest_end(skb, nest);
WANG Cong5da57f42013-12-15 20:15:07 -08003127 } else if (exts->police) {
WANG Cong33be6272013-12-15 20:15:05 -08003128 struct tc_action *act = tcf_exts_first_act(exts);
Michal Kubecekae0be8d2019-04-26 11:13:06 +02003129 nest = nla_nest_start_noflag(skb, exts->police);
Jamal Hadi Salim63acd682013-12-23 08:02:12 -05003130 if (nest == NULL || !act)
Patrick McHardy4b3550ef2008-01-23 20:34:11 -08003131 goto nla_put_failure;
WANG Cong33be6272013-12-15 20:15:05 -08003132 if (tcf_action_dump_old(skb, act, 0, 0) < 0)
Patrick McHardyadd93b62008-01-22 22:11:33 -08003133 goto nla_put_failure;
Patrick McHardy4b3550ef2008-01-23 20:34:11 -08003134 nla_nest_end(skb, nest);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003135 }
3136 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003137 return 0;
Cong Wang9cc63db2014-07-16 14:25:30 -07003138
3139nla_put_failure:
3140 nla_nest_cancel(skb, nest);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003141 return -1;
Cong Wang9cc63db2014-07-16 14:25:30 -07003142#else
3143 return 0;
3144#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07003145}
Stephen Hemmingeraa767bf2008-01-21 02:26:41 -08003146EXPORT_SYMBOL(tcf_exts_dump);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003147
Vlad Buslovca44b732020-05-15 14:40:12 +03003148int tcf_exts_terse_dump(struct sk_buff *skb, struct tcf_exts *exts)
3149{
3150#ifdef CONFIG_NET_CLS_ACT
3151 struct nlattr *nest;
3152
3153 if (!exts->action || !tcf_exts_has_actions(exts))
3154 return 0;
3155
3156 nest = nla_nest_start_noflag(skb, exts->action);
3157 if (!nest)
3158 goto nla_put_failure;
3159
3160 if (tcf_action_dump(skb, exts->actions, 0, 0, true) < 0)
3161 goto nla_put_failure;
3162 nla_nest_end(skb, nest);
3163 return 0;
3164
3165nla_put_failure:
3166 nla_nest_cancel(skb, nest);
3167 return -1;
3168#else
3169 return 0;
3170#endif
3171}
3172EXPORT_SYMBOL(tcf_exts_terse_dump);
Stephen Hemmingeraa767bf2008-01-21 02:26:41 -08003173
WANG Cong5da57f42013-12-15 20:15:07 -08003174int tcf_exts_dump_stats(struct sk_buff *skb, struct tcf_exts *exts)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003175{
3176#ifdef CONFIG_NET_CLS_ACT
WANG Cong33be6272013-12-15 20:15:05 -08003177 struct tc_action *a = tcf_exts_first_act(exts);
Ignacy Gawędzkib057df22015-02-03 19:05:18 +01003178 if (a != NULL && tcf_action_copy_stats(skb, a, 1) < 0)
WANG Cong33be6272013-12-15 20:15:05 -08003179 return -1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003180#endif
3181 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003182}
Stephen Hemmingeraa767bf2008-01-21 02:26:41 -08003183EXPORT_SYMBOL(tcf_exts_dump_stats);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003184
Vlad Buslov40119212019-08-26 16:44:59 +03003185static void tcf_block_offload_inc(struct tcf_block *block, u32 *flags)
3186{
3187 if (*flags & TCA_CLS_FLAGS_IN_HW)
3188 return;
3189 *flags |= TCA_CLS_FLAGS_IN_HW;
3190 atomic_inc(&block->offloadcnt);
3191}
3192
3193static void tcf_block_offload_dec(struct tcf_block *block, u32 *flags)
3194{
3195 if (!(*flags & TCA_CLS_FLAGS_IN_HW))
3196 return;
3197 *flags &= ~TCA_CLS_FLAGS_IN_HW;
3198 atomic_dec(&block->offloadcnt);
3199}
3200
3201static void tc_cls_offload_cnt_update(struct tcf_block *block,
3202 struct tcf_proto *tp, u32 *cnt,
3203 u32 *flags, u32 diff, bool add)
3204{
3205 lockdep_assert_held(&block->cb_lock);
3206
3207 spin_lock(&tp->lock);
3208 if (add) {
3209 if (!*cnt)
3210 tcf_block_offload_inc(block, flags);
3211 *cnt += diff;
3212 } else {
3213 *cnt -= diff;
3214 if (!*cnt)
3215 tcf_block_offload_dec(block, flags);
3216 }
3217 spin_unlock(&tp->lock);
3218}
3219
3220static void
3221tc_cls_offload_cnt_reset(struct tcf_block *block, struct tcf_proto *tp,
3222 u32 *cnt, u32 *flags)
3223{
3224 lockdep_assert_held(&block->cb_lock);
3225
3226 spin_lock(&tp->lock);
3227 tcf_block_offload_dec(block, flags);
3228 *cnt = 0;
3229 spin_unlock(&tp->lock);
3230}
3231
3232static int
3233__tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
3234 void *type_data, bool err_stop)
Jiri Pirko717503b2017-10-11 09:41:09 +02003235{
Pablo Neira Ayuso955bcb62019-07-09 22:55:46 +02003236 struct flow_block_cb *block_cb;
Cong Wangaeb3fec2018-12-11 11:15:46 -08003237 int ok_count = 0;
3238 int err;
3239
Vlad Buslov40119212019-08-26 16:44:59 +03003240 list_for_each_entry(block_cb, &block->flow_block.cb_list, list) {
3241 err = block_cb->cb(type, type_data, block_cb->cb_priv);
3242 if (err) {
3243 if (err_stop)
3244 return err;
3245 } else {
3246 ok_count++;
3247 }
3248 }
3249 return ok_count;
3250}
3251
3252int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
3253 void *type_data, bool err_stop, bool rtnl_held)
3254{
Vlad Buslov11bd6342019-08-26 16:45:02 +03003255 bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
Vlad Buslov40119212019-08-26 16:44:59 +03003256 int ok_count;
3257
Vlad Buslov11bd6342019-08-26 16:45:02 +03003258retry:
3259 if (take_rtnl)
3260 rtnl_lock();
Vlad Buslov40119212019-08-26 16:44:59 +03003261 down_read(&block->cb_lock);
Vlad Buslov11bd6342019-08-26 16:45:02 +03003262 /* Need to obtain rtnl lock if block is bound to devs that require it.
3263 * In block bind code cb_lock is obtained while holding rtnl, so we must
3264 * obtain the locks in same order here.
3265 */
3266 if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3267 up_read(&block->cb_lock);
3268 take_rtnl = true;
3269 goto retry;
3270 }
3271
Vlad Buslov40119212019-08-26 16:44:59 +03003272 ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
Vlad Buslov11bd6342019-08-26 16:45:02 +03003273
Vlad Buslov40119212019-08-26 16:44:59 +03003274 up_read(&block->cb_lock);
Vlad Buslov11bd6342019-08-26 16:45:02 +03003275 if (take_rtnl)
3276 rtnl_unlock();
Vlad Buslov40119212019-08-26 16:44:59 +03003277 return ok_count;
3278}
3279EXPORT_SYMBOL(tc_setup_cb_call);
3280
3281/* Non-destructive filter add. If filter that wasn't already in hardware is
3282 * successfully offloaded, increment block offloads counter. On failure,
3283 * previously offloaded filter is considered to be intact and offloads counter
3284 * is not decremented.
3285 */
3286
3287int tc_setup_cb_add(struct tcf_block *block, struct tcf_proto *tp,
3288 enum tc_setup_type type, void *type_data, bool err_stop,
3289 u32 *flags, unsigned int *in_hw_count, bool rtnl_held)
3290{
Vlad Buslov11bd6342019-08-26 16:45:02 +03003291 bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
Vlad Buslov40119212019-08-26 16:44:59 +03003292 int ok_count;
3293
Vlad Buslov11bd6342019-08-26 16:45:02 +03003294retry:
3295 if (take_rtnl)
3296 rtnl_lock();
Vlad Buslov4f8116c2019-08-26 16:44:57 +03003297 down_read(&block->cb_lock);
Vlad Buslov11bd6342019-08-26 16:45:02 +03003298 /* Need to obtain rtnl lock if block is bound to devs that require it.
3299 * In block bind code cb_lock is obtained while holding rtnl, so we must
3300 * obtain the locks in same order here.
3301 */
3302 if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3303 up_read(&block->cb_lock);
3304 take_rtnl = true;
3305 goto retry;
3306 }
3307
Cong Wangaeb3fec2018-12-11 11:15:46 -08003308 /* Make sure all netdevs sharing this block are offload-capable. */
Vlad Buslov4f8116c2019-08-26 16:44:57 +03003309 if (block->nooffloaddevcnt && err_stop) {
3310 ok_count = -EOPNOTSUPP;
3311 goto err_unlock;
3312 }
Cong Wangaeb3fec2018-12-11 11:15:46 -08003313
Vlad Buslov40119212019-08-26 16:44:59 +03003314 ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
Vlad Buslova449a3e2019-08-26 16:45:00 +03003315 if (ok_count < 0)
3316 goto err_unlock;
3317
3318 if (tp->ops->hw_add)
3319 tp->ops->hw_add(tp, type_data);
Vlad Buslov40119212019-08-26 16:44:59 +03003320 if (ok_count > 0)
3321 tc_cls_offload_cnt_update(block, tp, in_hw_count, flags,
3322 ok_count, true);
Vlad Buslov4f8116c2019-08-26 16:44:57 +03003323err_unlock:
3324 up_read(&block->cb_lock);
Vlad Buslov11bd6342019-08-26 16:45:02 +03003325 if (take_rtnl)
3326 rtnl_unlock();
Vlad Buslov40119212019-08-26 16:44:59 +03003327 return ok_count < 0 ? ok_count : 0;
Jiri Pirko717503b2017-10-11 09:41:09 +02003328}
Vlad Buslov40119212019-08-26 16:44:59 +03003329EXPORT_SYMBOL(tc_setup_cb_add);
3330
3331/* Destructive filter replace. If filter that wasn't already in hardware is
3332 * successfully offloaded, increment block offload counter. On failure,
3333 * previously offloaded filter is considered to be destroyed and offload counter
3334 * is decremented.
3335 */
3336
3337int tc_setup_cb_replace(struct tcf_block *block, struct tcf_proto *tp,
3338 enum tc_setup_type type, void *type_data, bool err_stop,
3339 u32 *old_flags, unsigned int *old_in_hw_count,
3340 u32 *new_flags, unsigned int *new_in_hw_count,
3341 bool rtnl_held)
3342{
Vlad Buslov11bd6342019-08-26 16:45:02 +03003343 bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
Vlad Buslov40119212019-08-26 16:44:59 +03003344 int ok_count;
3345
Vlad Buslov11bd6342019-08-26 16:45:02 +03003346retry:
3347 if (take_rtnl)
3348 rtnl_lock();
Vlad Buslov40119212019-08-26 16:44:59 +03003349 down_read(&block->cb_lock);
Vlad Buslov11bd6342019-08-26 16:45:02 +03003350 /* Need to obtain rtnl lock if block is bound to devs that require it.
3351 * In block bind code cb_lock is obtained while holding rtnl, so we must
3352 * obtain the locks in same order here.
3353 */
3354 if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3355 up_read(&block->cb_lock);
3356 take_rtnl = true;
3357 goto retry;
3358 }
3359
Vlad Buslov40119212019-08-26 16:44:59 +03003360 /* Make sure all netdevs sharing this block are offload-capable. */
3361 if (block->nooffloaddevcnt && err_stop) {
3362 ok_count = -EOPNOTSUPP;
3363 goto err_unlock;
3364 }
3365
3366 tc_cls_offload_cnt_reset(block, tp, old_in_hw_count, old_flags);
Vlad Buslova449a3e2019-08-26 16:45:00 +03003367 if (tp->ops->hw_del)
3368 tp->ops->hw_del(tp, type_data);
Vlad Buslov40119212019-08-26 16:44:59 +03003369
3370 ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
Vlad Buslova449a3e2019-08-26 16:45:00 +03003371 if (ok_count < 0)
3372 goto err_unlock;
3373
3374 if (tp->ops->hw_add)
3375 tp->ops->hw_add(tp, type_data);
Vlad Buslov40119212019-08-26 16:44:59 +03003376 if (ok_count > 0)
Vlad Buslova449a3e2019-08-26 16:45:00 +03003377 tc_cls_offload_cnt_update(block, tp, new_in_hw_count,
3378 new_flags, ok_count, true);
Vlad Buslov40119212019-08-26 16:44:59 +03003379err_unlock:
3380 up_read(&block->cb_lock);
Vlad Buslov11bd6342019-08-26 16:45:02 +03003381 if (take_rtnl)
3382 rtnl_unlock();
Vlad Buslov40119212019-08-26 16:44:59 +03003383 return ok_count < 0 ? ok_count : 0;
3384}
3385EXPORT_SYMBOL(tc_setup_cb_replace);
3386
3387/* Destroy filter and decrement block offload counter, if filter was previously
3388 * offloaded.
3389 */
3390
3391int tc_setup_cb_destroy(struct tcf_block *block, struct tcf_proto *tp,
3392 enum tc_setup_type type, void *type_data, bool err_stop,
3393 u32 *flags, unsigned int *in_hw_count, bool rtnl_held)
3394{
Vlad Buslov11bd6342019-08-26 16:45:02 +03003395 bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
Vlad Buslov40119212019-08-26 16:44:59 +03003396 int ok_count;
3397
Vlad Buslov11bd6342019-08-26 16:45:02 +03003398retry:
3399 if (take_rtnl)
3400 rtnl_lock();
Vlad Buslov40119212019-08-26 16:44:59 +03003401 down_read(&block->cb_lock);
Vlad Buslov11bd6342019-08-26 16:45:02 +03003402 /* Need to obtain rtnl lock if block is bound to devs that require it.
3403 * In block bind code cb_lock is obtained while holding rtnl, so we must
3404 * obtain the locks in same order here.
3405 */
3406 if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3407 up_read(&block->cb_lock);
3408 take_rtnl = true;
3409 goto retry;
3410 }
3411
Vlad Buslov40119212019-08-26 16:44:59 +03003412 ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
3413
3414 tc_cls_offload_cnt_reset(block, tp, in_hw_count, flags);
Vlad Buslova449a3e2019-08-26 16:45:00 +03003415 if (tp->ops->hw_del)
3416 tp->ops->hw_del(tp, type_data);
3417
Vlad Buslov40119212019-08-26 16:44:59 +03003418 up_read(&block->cb_lock);
Vlad Buslov11bd6342019-08-26 16:45:02 +03003419 if (take_rtnl)
3420 rtnl_unlock();
Vlad Buslov40119212019-08-26 16:44:59 +03003421 return ok_count < 0 ? ok_count : 0;
3422}
3423EXPORT_SYMBOL(tc_setup_cb_destroy);
3424
3425int tc_setup_cb_reoffload(struct tcf_block *block, struct tcf_proto *tp,
3426 bool add, flow_setup_cb_t *cb,
3427 enum tc_setup_type type, void *type_data,
3428 void *cb_priv, u32 *flags, unsigned int *in_hw_count)
3429{
3430 int err = cb(type, type_data, cb_priv);
3431
3432 if (err) {
3433 if (add && tc_skip_sw(*flags))
3434 return err;
3435 } else {
3436 tc_cls_offload_cnt_update(block, tp, in_hw_count, flags, 1,
3437 add);
3438 }
3439
3440 return 0;
3441}
3442EXPORT_SYMBOL(tc_setup_cb_reoffload);
Jiri Pirkob3f55bd2017-10-11 09:41:08 +02003443
Jiri Pirko20084952020-02-25 11:45:18 +01003444static int tcf_act_get_cookie(struct flow_action_entry *entry,
3445 const struct tc_action *act)
3446{
3447 struct tc_cookie *cookie;
3448 int err = 0;
3449
3450 rcu_read_lock();
3451 cookie = rcu_dereference(act->act_cookie);
3452 if (cookie) {
3453 entry->cookie = flow_action_cookie_create(cookie->data,
3454 cookie->len,
3455 GFP_ATOMIC);
3456 if (!entry->cookie)
3457 err = -ENOMEM;
3458 }
3459 rcu_read_unlock();
3460 return err;
3461}
3462
3463static void tcf_act_put_cookie(struct flow_action_entry *entry)
3464{
3465 flow_action_cookie_destroy(entry->cookie);
3466}
3467
Vlad Buslov5a6ff4b2019-08-26 16:45:04 +03003468void tc_cleanup_flow_action(struct flow_action *flow_action)
3469{
3470 struct flow_action_entry *entry;
3471 int i;
3472
Jiri Pirko20084952020-02-25 11:45:18 +01003473 flow_action_for_each(i, entry, flow_action) {
3474 tcf_act_put_cookie(entry);
Vlad Buslov11589582019-09-13 18:28:39 +03003475 if (entry->destructor)
3476 entry->destructor(entry->destructor_priv);
Jiri Pirko20084952020-02-25 11:45:18 +01003477 }
Vlad Buslov5a6ff4b2019-08-26 16:45:04 +03003478}
3479EXPORT_SYMBOL(tc_cleanup_flow_action);
3480
Vlad Buslov11589582019-09-13 18:28:39 +03003481static void tcf_mirred_get_dev(struct flow_action_entry *entry,
3482 const struct tc_action *act)
3483{
Vlad Buslov470d5062019-09-13 18:28:41 +03003484#ifdef CONFIG_NET_CLS_ACT
3485 entry->dev = act->ops->get_dev(act, &entry->destructor);
Vlad Buslov11589582019-09-13 18:28:39 +03003486 if (!entry->dev)
3487 return;
Vlad Buslov11589582019-09-13 18:28:39 +03003488 entry->destructor_priv = entry->dev;
Vlad Buslov470d5062019-09-13 18:28:41 +03003489#endif
Vlad Buslov11589582019-09-13 18:28:39 +03003490}
3491
3492static void tcf_tunnel_encap_put_tunnel(void *priv)
3493{
3494 struct ip_tunnel_info *tunnel = priv;
3495
3496 kfree(tunnel);
3497}
3498
3499static int tcf_tunnel_encap_get_tunnel(struct flow_action_entry *entry,
3500 const struct tc_action *act)
3501{
3502 entry->tunnel = tcf_tunnel_info_copy(act);
3503 if (!entry->tunnel)
3504 return -ENOMEM;
3505 entry->destructor = tcf_tunnel_encap_put_tunnel;
3506 entry->destructor_priv = entry->tunnel;
3507 return 0;
3508}
3509
Vlad Buslov4a5da472019-09-13 18:28:40 +03003510static void tcf_sample_get_group(struct flow_action_entry *entry,
3511 const struct tc_action *act)
3512{
3513#ifdef CONFIG_NET_CLS_ACT
3514 entry->sample.psample_group =
3515 act->ops->get_psample_group(act, &entry->destructor);
3516 entry->destructor_priv = entry->sample.psample_group;
3517#endif
3518}
3519
Po Liud29bdd62020-05-01 08:53:16 +08003520static void tcf_gate_entry_destructor(void *priv)
3521{
3522 struct action_gate_entry *oe = priv;
3523
3524 kfree(oe);
3525}
3526
3527static int tcf_gate_get_entries(struct flow_action_entry *entry,
3528 const struct tc_action *act)
3529{
3530 entry->gate.entries = tcf_gate_get_list(act);
3531
3532 if (!entry->gate.entries)
3533 return -EINVAL;
3534
3535 entry->destructor = tcf_gate_entry_destructor;
3536 entry->destructor_priv = entry->gate.entries;
3537
3538 return 0;
3539}
3540
Pablo Neira Ayuso16f80362020-05-06 20:34:50 +02003541static enum flow_action_hw_stats tc_act_hw_stats(u8 hw_stats)
3542{
3543 if (WARN_ON_ONCE(hw_stats > TCA_ACT_HW_STATS_ANY))
3544 return FLOW_ACTION_HW_STATS_DONT_CARE;
3545 else if (!hw_stats)
3546 return FLOW_ACTION_HW_STATS_DISABLED;
3547
3548 return hw_stats;
3549}
3550
Pablo Neira Ayuso3a7b6862019-02-02 12:50:46 +01003551int tc_setup_flow_action(struct flow_action *flow_action,
Vlad Buslovb15e7a62020-02-17 12:12:12 +02003552 const struct tcf_exts *exts)
Pablo Neira Ayuso3a7b6862019-02-02 12:50:46 +01003553{
Vlad Buslov7a472812020-02-17 12:12:09 +02003554 struct tc_action *act;
Vlad Buslov9838b202019-08-26 16:45:03 +03003555 int i, j, k, err = 0;
Pablo Neira Ayuso3a7b6862019-02-02 12:50:46 +01003556
Jakub Kicinski0dfb2d82020-03-19 16:26:23 -07003557 BUILD_BUG_ON(TCA_ACT_HW_STATS_ANY != FLOW_ACTION_HW_STATS_ANY);
3558 BUILD_BUG_ON(TCA_ACT_HW_STATS_IMMEDIATE != FLOW_ACTION_HW_STATS_IMMEDIATE);
3559 BUILD_BUG_ON(TCA_ACT_HW_STATS_DELAYED != FLOW_ACTION_HW_STATS_DELAYED);
Jiri Pirko44f86582020-03-07 12:40:20 +01003560
Pablo Neira Ayuso3a7b6862019-02-02 12:50:46 +01003561 if (!exts)
3562 return 0;
3563
3564 j = 0;
3565 tcf_exts_for_each_action(i, act, exts) {
3566 struct flow_action_entry *entry;
3567
3568 entry = &flow_action->entries[j];
Vlad Buslov7a472812020-02-17 12:12:09 +02003569 spin_lock_bh(&act->tcfa_lock);
Jiri Pirko20084952020-02-25 11:45:18 +01003570 err = tcf_act_get_cookie(entry, act);
3571 if (err)
3572 goto err_out_locked;
Jiri Pirko44f86582020-03-07 12:40:20 +01003573
Pablo Neira Ayuso16f80362020-05-06 20:34:50 +02003574 entry->hw_stats = tc_act_hw_stats(act->hw_stats);
Jiri Pirko44f86582020-03-07 12:40:20 +01003575
Pablo Neira Ayuso3a7b6862019-02-02 12:50:46 +01003576 if (is_tcf_gact_ok(act)) {
3577 entry->id = FLOW_ACTION_ACCEPT;
3578 } else if (is_tcf_gact_shot(act)) {
3579 entry->id = FLOW_ACTION_DROP;
3580 } else if (is_tcf_gact_trap(act)) {
3581 entry->id = FLOW_ACTION_TRAP;
3582 } else if (is_tcf_gact_goto_chain(act)) {
3583 entry->id = FLOW_ACTION_GOTO;
3584 entry->chain_index = tcf_gact_goto_chain_index(act);
3585 } else if (is_tcf_mirred_egress_redirect(act)) {
3586 entry->id = FLOW_ACTION_REDIRECT;
Vlad Buslov11589582019-09-13 18:28:39 +03003587 tcf_mirred_get_dev(entry, act);
Pablo Neira Ayuso3a7b6862019-02-02 12:50:46 +01003588 } else if (is_tcf_mirred_egress_mirror(act)) {
3589 entry->id = FLOW_ACTION_MIRRED;
Vlad Buslov11589582019-09-13 18:28:39 +03003590 tcf_mirred_get_dev(entry, act);
John Hurley48e584a2019-08-04 16:09:06 +01003591 } else if (is_tcf_mirred_ingress_redirect(act)) {
3592 entry->id = FLOW_ACTION_REDIRECT_INGRESS;
Vlad Buslov11589582019-09-13 18:28:39 +03003593 tcf_mirred_get_dev(entry, act);
John Hurley48e584a2019-08-04 16:09:06 +01003594 } else if (is_tcf_mirred_ingress_mirror(act)) {
3595 entry->id = FLOW_ACTION_MIRRED_INGRESS;
Vlad Buslov11589582019-09-13 18:28:39 +03003596 tcf_mirred_get_dev(entry, act);
Pablo Neira Ayuso3a7b6862019-02-02 12:50:46 +01003597 } else if (is_tcf_vlan(act)) {
3598 switch (tcf_vlan_action(act)) {
3599 case TCA_VLAN_ACT_PUSH:
3600 entry->id = FLOW_ACTION_VLAN_PUSH;
3601 entry->vlan.vid = tcf_vlan_push_vid(act);
3602 entry->vlan.proto = tcf_vlan_push_proto(act);
3603 entry->vlan.prio = tcf_vlan_push_prio(act);
3604 break;
3605 case TCA_VLAN_ACT_POP:
3606 entry->id = FLOW_ACTION_VLAN_POP;
3607 break;
3608 case TCA_VLAN_ACT_MODIFY:
3609 entry->id = FLOW_ACTION_VLAN_MANGLE;
3610 entry->vlan.vid = tcf_vlan_push_vid(act);
3611 entry->vlan.proto = tcf_vlan_push_proto(act);
3612 entry->vlan.prio = tcf_vlan_push_prio(act);
3613 break;
3614 default:
Vlad Buslov9838b202019-08-26 16:45:03 +03003615 err = -EOPNOTSUPP;
Vlad Buslov7a472812020-02-17 12:12:09 +02003616 goto err_out_locked;
Pablo Neira Ayuso3a7b6862019-02-02 12:50:46 +01003617 }
3618 } else if (is_tcf_tunnel_set(act)) {
3619 entry->id = FLOW_ACTION_TUNNEL_ENCAP;
Vlad Buslov11589582019-09-13 18:28:39 +03003620 err = tcf_tunnel_encap_get_tunnel(entry, act);
3621 if (err)
Vlad Buslov7a472812020-02-17 12:12:09 +02003622 goto err_out_locked;
Pablo Neira Ayuso3a7b6862019-02-02 12:50:46 +01003623 } else if (is_tcf_tunnel_release(act)) {
3624 entry->id = FLOW_ACTION_TUNNEL_DECAP;
Pablo Neira Ayuso3a7b6862019-02-02 12:50:46 +01003625 } else if (is_tcf_pedit(act)) {
3626 for (k = 0; k < tcf_pedit_nkeys(act); k++) {
3627 switch (tcf_pedit_cmd(act, k)) {
3628 case TCA_PEDIT_KEY_EX_CMD_SET:
3629 entry->id = FLOW_ACTION_MANGLE;
3630 break;
3631 case TCA_PEDIT_KEY_EX_CMD_ADD:
3632 entry->id = FLOW_ACTION_ADD;
3633 break;
3634 default:
Vlad Buslov9838b202019-08-26 16:45:03 +03003635 err = -EOPNOTSUPP;
Vlad Buslov7a472812020-02-17 12:12:09 +02003636 goto err_out_locked;
Pablo Neira Ayuso3a7b6862019-02-02 12:50:46 +01003637 }
3638 entry->mangle.htype = tcf_pedit_htype(act, k);
3639 entry->mangle.mask = tcf_pedit_mask(act, k);
3640 entry->mangle.val = tcf_pedit_val(act, k);
3641 entry->mangle.offset = tcf_pedit_offset(act, k);
Pablo Neira Ayuso16f80362020-05-06 20:34:50 +02003642 entry->hw_stats = tc_act_hw_stats(act->hw_stats);
Petr Machata2c4b58d2020-03-18 19:42:29 +02003643 entry = &flow_action->entries[++j];
Pablo Neira Ayuso3a7b6862019-02-02 12:50:46 +01003644 }
3645 } else if (is_tcf_csum(act)) {
3646 entry->id = FLOW_ACTION_CSUM;
3647 entry->csum_flags = tcf_csum_update_flags(act);
3648 } else if (is_tcf_skbedit_mark(act)) {
3649 entry->id = FLOW_ACTION_MARK;
3650 entry->mark = tcf_skbedit_mark(act);
Pieter Jansen van Vuurena7a7be62019-05-04 04:46:16 -07003651 } else if (is_tcf_sample(act)) {
3652 entry->id = FLOW_ACTION_SAMPLE;
Pieter Jansen van Vuurena7a7be62019-05-04 04:46:16 -07003653 entry->sample.trunc_size = tcf_sample_trunc_size(act);
3654 entry->sample.truncate = tcf_sample_truncate(act);
3655 entry->sample.rate = tcf_sample_rate(act);
Vlad Buslov4a5da472019-09-13 18:28:40 +03003656 tcf_sample_get_group(entry, act);
Pieter Jansen van Vuuren8c8cfc62019-05-04 04:46:22 -07003657 } else if (is_tcf_police(act)) {
3658 entry->id = FLOW_ACTION_POLICE;
3659 entry->police.burst = tcf_police_tcfp_burst(act);
3660 entry->police.rate_bytes_ps =
3661 tcf_police_rate_bytes_ps(act);
Paul Blakeyb57dc7c2019-07-09 10:30:48 +03003662 } else if (is_tcf_ct(act)) {
3663 entry->id = FLOW_ACTION_CT;
3664 entry->ct.action = tcf_ct_action(act);
3665 entry->ct.zone = tcf_ct_zone(act);
Paul Blakeyedd58612020-03-12 12:23:09 +02003666 entry->ct.flow_table = tcf_ct_ft(act);
John Hurley6749d5902019-07-23 15:33:59 +01003667 } else if (is_tcf_mpls(act)) {
3668 switch (tcf_mpls_action(act)) {
3669 case TCA_MPLS_ACT_PUSH:
3670 entry->id = FLOW_ACTION_MPLS_PUSH;
3671 entry->mpls_push.proto = tcf_mpls_proto(act);
3672 entry->mpls_push.label = tcf_mpls_label(act);
3673 entry->mpls_push.tc = tcf_mpls_tc(act);
3674 entry->mpls_push.bos = tcf_mpls_bos(act);
3675 entry->mpls_push.ttl = tcf_mpls_ttl(act);
3676 break;
3677 case TCA_MPLS_ACT_POP:
3678 entry->id = FLOW_ACTION_MPLS_POP;
3679 entry->mpls_pop.proto = tcf_mpls_proto(act);
3680 break;
3681 case TCA_MPLS_ACT_MODIFY:
3682 entry->id = FLOW_ACTION_MPLS_MANGLE;
3683 entry->mpls_mangle.label = tcf_mpls_label(act);
3684 entry->mpls_mangle.tc = tcf_mpls_tc(act);
3685 entry->mpls_mangle.bos = tcf_mpls_bos(act);
3686 entry->mpls_mangle.ttl = tcf_mpls_ttl(act);
3687 break;
3688 default:
Vlad Buslov7a472812020-02-17 12:12:09 +02003689 goto err_out_locked;
John Hurley6749d5902019-07-23 15:33:59 +01003690 }
John Hurleyfb1b7752019-08-04 16:09:04 +01003691 } else if (is_tcf_skbedit_ptype(act)) {
3692 entry->id = FLOW_ACTION_PTYPE;
3693 entry->ptype = tcf_skbedit_ptype(act);
Petr Machata2ce12412020-03-19 15:47:21 +02003694 } else if (is_tcf_skbedit_priority(act)) {
3695 entry->id = FLOW_ACTION_PRIORITY;
3696 entry->priority = tcf_skbedit_priority(act);
Po Liud29bdd62020-05-01 08:53:16 +08003697 } else if (is_tcf_gate(act)) {
3698 entry->id = FLOW_ACTION_GATE;
3699 entry->gate.index = tcf_gate_index(act);
3700 entry->gate.prio = tcf_gate_prio(act);
3701 entry->gate.basetime = tcf_gate_basetime(act);
3702 entry->gate.cycletime = tcf_gate_cycletime(act);
3703 entry->gate.cycletimeext = tcf_gate_cycletimeext(act);
3704 entry->gate.num_entries = tcf_gate_num_entries(act);
3705 err = tcf_gate_get_entries(entry, act);
3706 if (err)
3707 goto err_out;
Pablo Neira Ayuso3a7b6862019-02-02 12:50:46 +01003708 } else {
Vlad Buslov9838b202019-08-26 16:45:03 +03003709 err = -EOPNOTSUPP;
Vlad Buslov7a472812020-02-17 12:12:09 +02003710 goto err_out_locked;
Pablo Neira Ayuso3a7b6862019-02-02 12:50:46 +01003711 }
Vlad Buslov7a472812020-02-17 12:12:09 +02003712 spin_unlock_bh(&act->tcfa_lock);
Pablo Neira Ayuso3a7b6862019-02-02 12:50:46 +01003713
3714 if (!is_tcf_pedit(act))
3715 j++;
3716 }
Vlad Buslov9838b202019-08-26 16:45:03 +03003717
Pablo Neira Ayuso3a7b6862019-02-02 12:50:46 +01003718err_out:
Vlad Buslov5a6ff4b2019-08-26 16:45:04 +03003719 if (err)
3720 tc_cleanup_flow_action(flow_action);
3721
Vlad Buslov9838b202019-08-26 16:45:03 +03003722 return err;
Vlad Buslov7a472812020-02-17 12:12:09 +02003723err_out_locked:
3724 spin_unlock_bh(&act->tcfa_lock);
3725 goto err_out;
Pablo Neira Ayuso3a7b6862019-02-02 12:50:46 +01003726}
3727EXPORT_SYMBOL(tc_setup_flow_action);
3728
Pablo Neira Ayusoe3ab7862019-02-02 12:50:45 +01003729unsigned int tcf_exts_num_actions(struct tcf_exts *exts)
3730{
3731 unsigned int num_acts = 0;
3732 struct tc_action *act;
3733 int i;
3734
3735 tcf_exts_for_each_action(i, act, exts) {
3736 if (is_tcf_pedit(act))
3737 num_acts += tcf_pedit_nkeys(act);
3738 else
3739 num_acts++;
3740 }
3741 return num_acts;
3742}
3743EXPORT_SYMBOL(tcf_exts_num_actions);
3744
Jiri Pirko48617382018-01-17 11:46:46 +01003745static __net_init int tcf_net_init(struct net *net)
3746{
3747 struct tcf_net *tn = net_generic(net, tcf_net_id);
3748
Vlad Buslovab281622018-09-24 19:22:56 +03003749 spin_lock_init(&tn->idr_lock);
Jiri Pirko48617382018-01-17 11:46:46 +01003750 idr_init(&tn->idr);
3751 return 0;
3752}
3753
3754static void __net_exit tcf_net_exit(struct net *net)
3755{
3756 struct tcf_net *tn = net_generic(net, tcf_net_id);
3757
3758 idr_destroy(&tn->idr);
3759}
3760
3761static struct pernet_operations tcf_net_ops = {
3762 .init = tcf_net_init,
3763 .exit = tcf_net_exit,
3764 .id = &tcf_net_id,
3765 .size = sizeof(struct tcf_net),
3766};
3767
Linus Torvalds1da177e2005-04-16 15:20:36 -07003768static int __init tc_filter_init(void)
3769{
Jiri Pirko48617382018-01-17 11:46:46 +01003770 int err;
3771
Cong Wang7aa00452017-10-26 18:24:28 -07003772 tc_filter_wq = alloc_ordered_workqueue("tc_filter_workqueue", 0);
3773 if (!tc_filter_wq)
3774 return -ENOMEM;
3775
Jiri Pirko48617382018-01-17 11:46:46 +01003776 err = register_pernet_subsys(&tcf_net_ops);
3777 if (err)
3778 goto err_register_pernet_subsys;
3779
Vlad Buslov470502d2019-02-11 10:55:48 +02003780 rtnl_register(PF_UNSPEC, RTM_NEWTFILTER, tc_new_tfilter, NULL,
3781 RTNL_FLAG_DOIT_UNLOCKED);
3782 rtnl_register(PF_UNSPEC, RTM_DELTFILTER, tc_del_tfilter, NULL,
3783 RTNL_FLAG_DOIT_UNLOCKED);
Vlad Buslovc431f892018-05-31 09:52:53 +03003784 rtnl_register(PF_UNSPEC, RTM_GETTFILTER, tc_get_tfilter,
Vlad Buslov470502d2019-02-11 10:55:48 +02003785 tc_dump_tfilter, RTNL_FLAG_DOIT_UNLOCKED);
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02003786 rtnl_register(PF_UNSPEC, RTM_NEWCHAIN, tc_ctl_chain, NULL, 0);
3787 rtnl_register(PF_UNSPEC, RTM_DELCHAIN, tc_ctl_chain, NULL, 0);
3788 rtnl_register(PF_UNSPEC, RTM_GETCHAIN, tc_ctl_chain,
3789 tc_dump_chain, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003790
Linus Torvalds1da177e2005-04-16 15:20:36 -07003791 return 0;
Jiri Pirko48617382018-01-17 11:46:46 +01003792
3793err_register_pernet_subsys:
3794 destroy_workqueue(tc_filter_wq);
3795 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003796}
3797
3798subsys_initcall(tc_filter_init);