blob: e137698e8aef7bb246d6b8bd9c269881f2f19d5b [file] [log] [blame]
Tom Herbert43a0c672016-08-15 14:51:01 -07001/*
2 * Stream Parser
3 *
4 * Copyright (c) 2016 Tom Herbert <tom@herbertland.com>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2
8 * as published by the Free Software Foundation.
9 */
10
11#include <linux/bpf.h>
12#include <linux/errno.h>
13#include <linux/errqueue.h>
14#include <linux/file.h>
15#include <linux/in.h>
16#include <linux/kernel.h>
Paul Gortmaker15253b42019-04-20 23:29:48 -040017#include <linux/export.h>
18#include <linux/init.h>
Tom Herbert43a0c672016-08-15 14:51:01 -070019#include <linux/net.h>
20#include <linux/netdevice.h>
21#include <linux/poll.h>
22#include <linux/rculist.h>
23#include <linux/skbuff.h>
24#include <linux/socket.h>
25#include <linux/uaccess.h>
26#include <linux/workqueue.h>
27#include <net/strparser.h>
28#include <net/netns/generic.h>
29#include <net/sock.h>
Tom Herbert43a0c672016-08-15 14:51:01 -070030
31static struct workqueue_struct *strp_wq;
32
Tom Herbertbbb03022017-07-28 16:22:43 -070033struct _strp_msg {
34 /* Internal cb structure. struct strp_msg must be first for passing
Tom Herbert43a0c672016-08-15 14:51:01 -070035 * to upper layer.
36 */
Tom Herbertbbb03022017-07-28 16:22:43 -070037 struct strp_msg strp;
Tom Herbert43a0c672016-08-15 14:51:01 -070038 int accum_len;
Tom Herbert43a0c672016-08-15 14:51:01 -070039};
40
Tom Herbertbbb03022017-07-28 16:22:43 -070041static inline struct _strp_msg *_strp_msg(struct sk_buff *skb)
Tom Herbert43a0c672016-08-15 14:51:01 -070042{
Tom Herbertbbb03022017-07-28 16:22:43 -070043 return (struct _strp_msg *)((void *)skb->cb +
Tom Herbert43a0c672016-08-15 14:51:01 -070044 offsetof(struct qdisc_skb_cb, data));
45}
46
47/* Lower lock held */
Tom Herbertbbb03022017-07-28 16:22:43 -070048static void strp_abort_strp(struct strparser *strp, int err)
Tom Herbert43a0c672016-08-15 14:51:01 -070049{
Tom Herbert43a0c672016-08-15 14:51:01 -070050 /* Unrecoverable error in receive */
51
Tom Herbert829385f2017-10-20 16:40:43 -070052 cancel_delayed_work(&strp->msg_timer_work);
Tom Herbert43a0c672016-08-15 14:51:01 -070053
Tom Herbertbbb03022017-07-28 16:22:43 -070054 if (strp->stopped)
Tom Herbert43a0c672016-08-15 14:51:01 -070055 return;
56
Tom Herbertbbb03022017-07-28 16:22:43 -070057 strp->stopped = 1;
Tom Herbert43a0c672016-08-15 14:51:01 -070058
Tom Herbertbbb03022017-07-28 16:22:43 -070059 if (strp->sk) {
60 struct sock *sk = strp->sk;
61
62 /* Report an error on the lower socket */
Dave Watsoncd00edc2018-03-26 12:31:21 -070063 sk->sk_err = -err;
Tom Herbertbbb03022017-07-28 16:22:43 -070064 sk->sk_error_report(sk);
65 }
Tom Herbert43a0c672016-08-15 14:51:01 -070066}
67
Tom Herbertbbb03022017-07-28 16:22:43 -070068static void strp_start_timer(struct strparser *strp, long timeo)
Tom Herbert43a0c672016-08-15 14:51:01 -070069{
Doron Roberts-Kedes7c5aba22018-04-20 12:11:11 -070070 if (timeo && timeo != LONG_MAX)
Tom Herbert829385f2017-10-20 16:40:43 -070071 mod_delayed_work(strp_wq, &strp->msg_timer_work, timeo);
Tom Herbert43a0c672016-08-15 14:51:01 -070072}
73
74/* Lower lock held */
75static void strp_parser_err(struct strparser *strp, int err,
76 read_descriptor_t *desc)
77{
78 desc->error = err;
Tom Herbertbbb03022017-07-28 16:22:43 -070079 kfree_skb(strp->skb_head);
80 strp->skb_head = NULL;
Tom Herbert43a0c672016-08-15 14:51:01 -070081 strp->cb.abort_parser(strp, err);
82}
83
Tom Herbert96a59082016-08-28 14:43:19 -070084static inline int strp_peek_len(struct strparser *strp)
85{
Tom Herbertbbb03022017-07-28 16:22:43 -070086 if (strp->sk) {
87 struct socket *sock = strp->sk->sk_socket;
Tom Herbert96a59082016-08-28 14:43:19 -070088
Tom Herbertbbb03022017-07-28 16:22:43 -070089 return sock->ops->peek_len(sock);
90 }
91
92 /* If we don't have an associated socket there's nothing to peek.
93 * Return int max to avoid stopping the strparser.
94 */
95
96 return INT_MAX;
Tom Herbert96a59082016-08-28 14:43:19 -070097}
98
Tom Herbert43a0c672016-08-15 14:51:01 -070099/* Lower socket lock held */
Tom Herbertbbb03022017-07-28 16:22:43 -0700100static int __strp_recv(read_descriptor_t *desc, struct sk_buff *orig_skb,
101 unsigned int orig_offset, size_t orig_len,
102 size_t max_msg_size, long timeo)
Tom Herbert43a0c672016-08-15 14:51:01 -0700103{
104 struct strparser *strp = (struct strparser *)desc->arg.data;
Tom Herbertbbb03022017-07-28 16:22:43 -0700105 struct _strp_msg *stm;
Tom Herbert43a0c672016-08-15 14:51:01 -0700106 struct sk_buff *head, *skb;
107 size_t eaten = 0, cand_len;
108 ssize_t extra;
109 int err;
110 bool cloned_orig = false;
111
Tom Herbertbbb03022017-07-28 16:22:43 -0700112 if (strp->paused)
Tom Herbert43a0c672016-08-15 14:51:01 -0700113 return 0;
114
Tom Herbertbbb03022017-07-28 16:22:43 -0700115 head = strp->skb_head;
Tom Herbert43a0c672016-08-15 14:51:01 -0700116 if (head) {
117 /* Message already in progress */
Tom Herbert43a0c672016-08-15 14:51:01 -0700118 if (unlikely(orig_offset)) {
119 /* Getting data with a non-zero offset when a message is
120 * in progress is not expected. If it does happen, we
121 * need to clone and pull since we can't deal with
122 * offsets in the skbs for a message expect in the head.
123 */
124 orig_skb = skb_clone(orig_skb, GFP_ATOMIC);
125 if (!orig_skb) {
Tom Herbertbbb03022017-07-28 16:22:43 -0700126 STRP_STATS_INCR(strp->stats.mem_fail);
Tom Herbert43a0c672016-08-15 14:51:01 -0700127 desc->error = -ENOMEM;
128 return 0;
129 }
130 if (!pskb_pull(orig_skb, orig_offset)) {
Tom Herbertbbb03022017-07-28 16:22:43 -0700131 STRP_STATS_INCR(strp->stats.mem_fail);
Tom Herbert43a0c672016-08-15 14:51:01 -0700132 kfree_skb(orig_skb);
133 desc->error = -ENOMEM;
134 return 0;
135 }
136 cloned_orig = true;
137 orig_offset = 0;
138 }
139
Tom Herbertbbb03022017-07-28 16:22:43 -0700140 if (!strp->skb_nextp) {
Tom Herbert43a0c672016-08-15 14:51:01 -0700141 /* We are going to append to the frags_list of head.
142 * Need to unshare the frag_list.
143 */
Jakub Kicinski4a9c2e32019-04-10 11:04:32 -0700144 err = skb_unclone(head, GFP_ATOMIC);
145 if (err) {
146 STRP_STATS_INCR(strp->stats.mem_fail);
147 desc->error = err;
148 return 0;
Tom Herbert43a0c672016-08-15 14:51:01 -0700149 }
150
151 if (unlikely(skb_shinfo(head)->frag_list)) {
152 /* We can't append to an sk_buff that already
153 * has a frag_list. We create a new head, point
154 * the frag_list of that to the old head, and
155 * then are able to use the old head->next for
156 * appending to the message.
157 */
158 if (WARN_ON(head->next)) {
159 desc->error = -EINVAL;
160 return 0;
161 }
162
163 skb = alloc_skb(0, GFP_ATOMIC);
164 if (!skb) {
Tom Herbertbbb03022017-07-28 16:22:43 -0700165 STRP_STATS_INCR(strp->stats.mem_fail);
Tom Herbert43a0c672016-08-15 14:51:01 -0700166 desc->error = -ENOMEM;
167 return 0;
168 }
169 skb->len = head->len;
170 skb->data_len = head->len;
171 skb->truesize = head->truesize;
Tom Herbertbbb03022017-07-28 16:22:43 -0700172 *_strp_msg(skb) = *_strp_msg(head);
173 strp->skb_nextp = &head->next;
Tom Herbert43a0c672016-08-15 14:51:01 -0700174 skb_shinfo(skb)->frag_list = head;
Tom Herbertbbb03022017-07-28 16:22:43 -0700175 strp->skb_head = skb;
Tom Herbert43a0c672016-08-15 14:51:01 -0700176 head = skb;
177 } else {
Tom Herbertbbb03022017-07-28 16:22:43 -0700178 strp->skb_nextp =
Tom Herbert43a0c672016-08-15 14:51:01 -0700179 &skb_shinfo(head)->frag_list;
180 }
181 }
182 }
183
184 while (eaten < orig_len) {
185 /* Always clone since we will consume something */
186 skb = skb_clone(orig_skb, GFP_ATOMIC);
187 if (!skb) {
Tom Herbertbbb03022017-07-28 16:22:43 -0700188 STRP_STATS_INCR(strp->stats.mem_fail);
Tom Herbert43a0c672016-08-15 14:51:01 -0700189 desc->error = -ENOMEM;
190 break;
191 }
192
193 cand_len = orig_len - eaten;
194
Tom Herbertbbb03022017-07-28 16:22:43 -0700195 head = strp->skb_head;
Tom Herbert43a0c672016-08-15 14:51:01 -0700196 if (!head) {
197 head = skb;
Tom Herbertbbb03022017-07-28 16:22:43 -0700198 strp->skb_head = head;
199 /* Will set skb_nextp on next packet if needed */
200 strp->skb_nextp = NULL;
201 stm = _strp_msg(head);
202 memset(stm, 0, sizeof(*stm));
203 stm->strp.offset = orig_offset + eaten;
Tom Herbert43a0c672016-08-15 14:51:01 -0700204 } else {
Vakul Garg4e485d02018-06-30 00:45:55 +0530205 /* Unclone if we are appending to an skb that we
Tom Herbert43a0c672016-08-15 14:51:01 -0700206 * already share a frag_list with.
207 */
Vakul Garg4e485d02018-06-30 00:45:55 +0530208 if (skb_has_frag_list(skb)) {
209 err = skb_unclone(skb, GFP_ATOMIC);
210 if (err) {
211 STRP_STATS_INCR(strp->stats.mem_fail);
212 desc->error = err;
213 break;
214 }
Tom Herbert43a0c672016-08-15 14:51:01 -0700215 }
216
Tom Herbertbbb03022017-07-28 16:22:43 -0700217 stm = _strp_msg(head);
218 *strp->skb_nextp = skb;
219 strp->skb_nextp = &skb->next;
Tom Herbert43a0c672016-08-15 14:51:01 -0700220 head->data_len += skb->len;
221 head->len += skb->len;
222 head->truesize += skb->truesize;
223 }
224
Tom Herbertbbb03022017-07-28 16:22:43 -0700225 if (!stm->strp.full_len) {
Tom Herbert43a0c672016-08-15 14:51:01 -0700226 ssize_t len;
227
228 len = (*strp->cb.parse_msg)(strp, head);
229
230 if (!len) {
231 /* Need more header to determine length */
Tom Herbertbbb03022017-07-28 16:22:43 -0700232 if (!stm->accum_len) {
Tom Herbert43a0c672016-08-15 14:51:01 -0700233 /* Start RX timer for new message */
Tom Herbertbbb03022017-07-28 16:22:43 -0700234 strp_start_timer(strp, timeo);
Tom Herbert43a0c672016-08-15 14:51:01 -0700235 }
Tom Herbertbbb03022017-07-28 16:22:43 -0700236 stm->accum_len += cand_len;
Tom Herbert43a0c672016-08-15 14:51:01 -0700237 eaten += cand_len;
Tom Herbertbbb03022017-07-28 16:22:43 -0700238 STRP_STATS_INCR(strp->stats.need_more_hdr);
Tom Herbert43a0c672016-08-15 14:51:01 -0700239 WARN_ON(eaten != orig_len);
240 break;
241 } else if (len < 0) {
Tom Herbertbbb03022017-07-28 16:22:43 -0700242 if (len == -ESTRPIPE && stm->accum_len) {
Tom Herbert43a0c672016-08-15 14:51:01 -0700243 len = -ENODATA;
Tom Herbertbbb03022017-07-28 16:22:43 -0700244 strp->unrecov_intr = 1;
Tom Herbert43a0c672016-08-15 14:51:01 -0700245 } else {
Tom Herbertbbb03022017-07-28 16:22:43 -0700246 strp->interrupted = 1;
Tom Herbert43a0c672016-08-15 14:51:01 -0700247 }
Geert Uytterhoeven6d3a4c42016-10-06 15:41:49 +0200248 strp_parser_err(strp, len, desc);
Tom Herbert43a0c672016-08-15 14:51:01 -0700249 break;
Tom Herbertbbb03022017-07-28 16:22:43 -0700250 } else if (len > max_msg_size) {
Tom Herbert43a0c672016-08-15 14:51:01 -0700251 /* Message length exceeds maximum allowed */
Tom Herbertbbb03022017-07-28 16:22:43 -0700252 STRP_STATS_INCR(strp->stats.msg_too_big);
Tom Herbert43a0c672016-08-15 14:51:01 -0700253 strp_parser_err(strp, -EMSGSIZE, desc);
254 break;
255 } else if (len <= (ssize_t)head->len -
Tom Herbertbbb03022017-07-28 16:22:43 -0700256 skb->len - stm->strp.offset) {
Tom Herbert43a0c672016-08-15 14:51:01 -0700257 /* Length must be into new skb (and also
258 * greater than zero)
259 */
Tom Herbertbbb03022017-07-28 16:22:43 -0700260 STRP_STATS_INCR(strp->stats.bad_hdr_len);
Tom Herbert43a0c672016-08-15 14:51:01 -0700261 strp_parser_err(strp, -EPROTO, desc);
262 break;
263 }
264
Tom Herbertbbb03022017-07-28 16:22:43 -0700265 stm->strp.full_len = len;
Tom Herbert43a0c672016-08-15 14:51:01 -0700266 }
267
Tom Herbertbbb03022017-07-28 16:22:43 -0700268 extra = (ssize_t)(stm->accum_len + cand_len) -
269 stm->strp.full_len;
Tom Herbert43a0c672016-08-15 14:51:01 -0700270
271 if (extra < 0) {
272 /* Message not complete yet. */
Tom Herbertbbb03022017-07-28 16:22:43 -0700273 if (stm->strp.full_len - stm->accum_len >
Tom Herbert96a59082016-08-28 14:43:19 -0700274 strp_peek_len(strp)) {
Tom Herbertbbb03022017-07-28 16:22:43 -0700275 /* Don't have the whole message in the socket
276 * buffer. Set strp->need_bytes to wait for
Tom Herbert43a0c672016-08-15 14:51:01 -0700277 * the rest of the message. Also, set "early
278 * eaten" since we've already buffered the skb
Tom Herbert96a59082016-08-28 14:43:19 -0700279 * but don't consume yet per strp_read_sock.
Tom Herbert43a0c672016-08-15 14:51:01 -0700280 */
281
Tom Herbertbbb03022017-07-28 16:22:43 -0700282 if (!stm->accum_len) {
Tom Herbert43a0c672016-08-15 14:51:01 -0700283 /* Start RX timer for new message */
Tom Herbertbbb03022017-07-28 16:22:43 -0700284 strp_start_timer(strp, timeo);
Tom Herbert43a0c672016-08-15 14:51:01 -0700285 }
286
Doron Roberts-Kedes9d0c75b2018-04-11 15:05:16 -0700287 stm->accum_len += cand_len;
Doron Roberts-Kedes977c7112018-06-26 18:33:33 -0700288 eaten += cand_len;
Tom Herbertbbb03022017-07-28 16:22:43 -0700289 strp->need_bytes = stm->strp.full_len -
290 stm->accum_len;
Tom Herbertbbb03022017-07-28 16:22:43 -0700291 STRP_STATS_ADD(strp->stats.bytes, cand_len);
Tom Herbert43a0c672016-08-15 14:51:01 -0700292 desc->count = 0; /* Stop reading socket */
293 break;
294 }
Tom Herbertbbb03022017-07-28 16:22:43 -0700295 stm->accum_len += cand_len;
Tom Herbert43a0c672016-08-15 14:51:01 -0700296 eaten += cand_len;
297 WARN_ON(eaten != orig_len);
298 break;
299 }
300
Jakub Kicinski93e21252019-04-10 13:18:57 -0700301 /* Positive extra indicates more bytes than needed for the
Tom Herbert43a0c672016-08-15 14:51:01 -0700302 * message
303 */
304
305 WARN_ON(extra > cand_len);
306
307 eaten += (cand_len - extra);
308
309 /* Hurray, we have a new message! */
Tom Herbert829385f2017-10-20 16:40:43 -0700310 cancel_delayed_work(&strp->msg_timer_work);
Tom Herbertbbb03022017-07-28 16:22:43 -0700311 strp->skb_head = NULL;
Doron Roberts-Kedes9d0c75b2018-04-11 15:05:16 -0700312 strp->need_bytes = 0;
Tom Herbertbbb03022017-07-28 16:22:43 -0700313 STRP_STATS_INCR(strp->stats.msgs);
Tom Herbert43a0c672016-08-15 14:51:01 -0700314
315 /* Give skb to upper layer */
316 strp->cb.rcv_msg(strp, head);
317
Tom Herbertbbb03022017-07-28 16:22:43 -0700318 if (unlikely(strp->paused)) {
Tom Herbert43a0c672016-08-15 14:51:01 -0700319 /* Upper layer paused strp */
320 break;
321 }
322 }
323
324 if (cloned_orig)
325 kfree_skb(orig_skb);
326
Tom Herbertbbb03022017-07-28 16:22:43 -0700327 STRP_STATS_ADD(strp->stats.bytes, eaten);
Tom Herbert43a0c672016-08-15 14:51:01 -0700328
329 return eaten;
330}
331
Tom Herbertbbb03022017-07-28 16:22:43 -0700332int strp_process(struct strparser *strp, struct sk_buff *orig_skb,
333 unsigned int orig_offset, size_t orig_len,
334 size_t max_msg_size, long timeo)
335{
336 read_descriptor_t desc; /* Dummy arg to strp_recv */
337
338 desc.arg.data = strp;
339
340 return __strp_recv(&desc, orig_skb, orig_offset, orig_len,
341 max_msg_size, timeo);
342}
343EXPORT_SYMBOL_GPL(strp_process);
344
345static int strp_recv(read_descriptor_t *desc, struct sk_buff *orig_skb,
346 unsigned int orig_offset, size_t orig_len)
347{
348 struct strparser *strp = (struct strparser *)desc->arg.data;
349
350 return __strp_recv(desc, orig_skb, orig_offset, orig_len,
351 strp->sk->sk_rcvbuf, strp->sk->sk_rcvtimeo);
352}
353
Tom Herbert43a0c672016-08-15 14:51:01 -0700354static int default_read_sock_done(struct strparser *strp, int err)
355{
356 return err;
357}
358
359/* Called with lock held on lower socket */
Tom Herbert96a59082016-08-28 14:43:19 -0700360static int strp_read_sock(struct strparser *strp)
Tom Herbert43a0c672016-08-15 14:51:01 -0700361{
Tom Herbert96a59082016-08-28 14:43:19 -0700362 struct socket *sock = strp->sk->sk_socket;
Tom Herbert43a0c672016-08-15 14:51:01 -0700363 read_descriptor_t desc;
364
John Fastabendf26de112017-08-15 22:30:47 -0700365 if (unlikely(!sock || !sock->ops || !sock->ops->read_sock))
366 return -EBUSY;
367
Tom Herbert43a0c672016-08-15 14:51:01 -0700368 desc.arg.data = strp;
369 desc.error = 0;
370 desc.count = 1; /* give more than one skb per call */
371
Tom Herbert96a59082016-08-28 14:43:19 -0700372 /* sk should be locked here, so okay to do read_sock */
373 sock->ops->read_sock(strp->sk, &desc, strp_recv);
Tom Herbert43a0c672016-08-15 14:51:01 -0700374
375 desc.error = strp->cb.read_sock_done(strp, desc.error);
376
377 return desc.error;
378}
379
380/* Lower sock lock held */
Tom Herbert96a59082016-08-28 14:43:19 -0700381void strp_data_ready(struct strparser *strp)
Tom Herbert43a0c672016-08-15 14:51:01 -0700382{
Vakul Garg456488c2018-06-21 03:29:49 +0530383 if (unlikely(strp->stopped) || strp->paused)
Tom Herbert43a0c672016-08-15 14:51:01 -0700384 return;
385
Tom Herbertbbb03022017-07-28 16:22:43 -0700386 /* This check is needed to synchronize with do_strp_work.
387 * do_strp_work acquires a process lock (lock_sock) whereas
Tom Herbert43a0c672016-08-15 14:51:01 -0700388 * the lock held here is bh_lock_sock. The two locks can be
389 * held by different threads at the same time, but bh_lock_sock
390 * allows a thread in BH context to safely check if the process
391 * lock is held. In this case, if the lock is held, queue work.
392 */
Tom Herbertd66fa9e2017-12-28 11:00:44 -0800393 if (sock_owned_by_user_nocheck(strp->sk)) {
Tom Herbertbbb03022017-07-28 16:22:43 -0700394 queue_work(strp_wq, &strp->work);
Tom Herbert43a0c672016-08-15 14:51:01 -0700395 return;
396 }
397
Tom Herbertbbb03022017-07-28 16:22:43 -0700398 if (strp->need_bytes) {
Doron Roberts-Kedes9d0c75b2018-04-11 15:05:16 -0700399 if (strp_peek_len(strp) < strp->need_bytes)
Tom Herbert43a0c672016-08-15 14:51:01 -0700400 return;
401 }
402
Tom Herbert96a59082016-08-28 14:43:19 -0700403 if (strp_read_sock(strp) == -ENOMEM)
Tom Herbertbbb03022017-07-28 16:22:43 -0700404 queue_work(strp_wq, &strp->work);
Tom Herbert43a0c672016-08-15 14:51:01 -0700405}
Tom Herbert96a59082016-08-28 14:43:19 -0700406EXPORT_SYMBOL_GPL(strp_data_ready);
Tom Herbert43a0c672016-08-15 14:51:01 -0700407
Tom Herbertbbb03022017-07-28 16:22:43 -0700408static void do_strp_work(struct strparser *strp)
Tom Herbert43a0c672016-08-15 14:51:01 -0700409{
Tom Herbert96a59082016-08-28 14:43:19 -0700410 /* We need the read lock to synchronize with strp_data_ready. We
411 * need the socket lock for calling strp_read_sock.
Tom Herbert43a0c672016-08-15 14:51:01 -0700412 */
Tom Herbertbbb03022017-07-28 16:22:43 -0700413 strp->cb.lock(strp);
Tom Herbert43a0c672016-08-15 14:51:01 -0700414
Tom Herbertbbb03022017-07-28 16:22:43 -0700415 if (unlikely(strp->stopped))
Tom Herbert43a0c672016-08-15 14:51:01 -0700416 goto out;
417
Tom Herbertbbb03022017-07-28 16:22:43 -0700418 if (strp->paused)
Tom Herbert43a0c672016-08-15 14:51:01 -0700419 goto out;
420
Tom Herbert96a59082016-08-28 14:43:19 -0700421 if (strp_read_sock(strp) == -ENOMEM)
Tom Herbertbbb03022017-07-28 16:22:43 -0700422 queue_work(strp_wq, &strp->work);
Tom Herbert43a0c672016-08-15 14:51:01 -0700423
424out:
Tom Herbertbbb03022017-07-28 16:22:43 -0700425 strp->cb.unlock(strp);
Tom Herbert43a0c672016-08-15 14:51:01 -0700426}
427
Tom Herbertbbb03022017-07-28 16:22:43 -0700428static void strp_work(struct work_struct *w)
Tom Herbert43a0c672016-08-15 14:51:01 -0700429{
Tom Herbertbbb03022017-07-28 16:22:43 -0700430 do_strp_work(container_of(w, struct strparser, work));
Tom Herbert43a0c672016-08-15 14:51:01 -0700431}
432
Tom Herbert829385f2017-10-20 16:40:43 -0700433static void strp_msg_timeout(struct work_struct *w)
Tom Herbert43a0c672016-08-15 14:51:01 -0700434{
Tom Herbert829385f2017-10-20 16:40:43 -0700435 struct strparser *strp = container_of(w, struct strparser,
436 msg_timer_work.work);
Tom Herbert43a0c672016-08-15 14:51:01 -0700437
438 /* Message assembly timed out */
Tom Herbertbbb03022017-07-28 16:22:43 -0700439 STRP_STATS_INCR(strp->stats.msg_timeouts);
440 strp->cb.lock(strp);
Dave Watsoncd00edc2018-03-26 12:31:21 -0700441 strp->cb.abort_parser(strp, -ETIMEDOUT);
Tom Herbertbbb03022017-07-28 16:22:43 -0700442 strp->cb.unlock(strp);
443}
444
445static void strp_sock_lock(struct strparser *strp)
446{
447 lock_sock(strp->sk);
448}
449
450static void strp_sock_unlock(struct strparser *strp)
451{
Tom Herbert43a0c672016-08-15 14:51:01 -0700452 release_sock(strp->sk);
453}
454
Tom Herbertbbb03022017-07-28 16:22:43 -0700455int strp_init(struct strparser *strp, struct sock *sk,
Eric Biggers3fd87122017-08-24 14:38:51 -0700456 const struct strp_callbacks *cb)
Tom Herbert43a0c672016-08-15 14:51:01 -0700457{
Tom Herbert96a59082016-08-28 14:43:19 -0700458
Tom Herbert43a0c672016-08-15 14:51:01 -0700459 if (!cb || !cb->rcv_msg || !cb->parse_msg)
460 return -EINVAL;
461
Tom Herbertbbb03022017-07-28 16:22:43 -0700462 /* The sk (sock) arg determines the mode of the stream parser.
463 *
464 * If the sock is set then the strparser is in receive callback mode.
465 * The upper layer calls strp_data_ready to kick receive processing
466 * and strparser calls the read_sock function on the socket to
467 * get packets.
468 *
469 * If the sock is not set then the strparser is in general mode.
470 * The upper layer calls strp_process for each skb to be parsed.
471 */
472
John Fastabendf26de112017-08-15 22:30:47 -0700473 if (!sk) {
Tom Herbertbbb03022017-07-28 16:22:43 -0700474 if (!cb->lock || !cb->unlock)
475 return -EINVAL;
476 }
Tom Herbert96a59082016-08-28 14:43:19 -0700477
Tom Herbert43a0c672016-08-15 14:51:01 -0700478 memset(strp, 0, sizeof(*strp));
479
Tom Herbertbbb03022017-07-28 16:22:43 -0700480 strp->sk = sk;
Tom Herbert43a0c672016-08-15 14:51:01 -0700481
Tom Herbertbbb03022017-07-28 16:22:43 -0700482 strp->cb.lock = cb->lock ? : strp_sock_lock;
483 strp->cb.unlock = cb->unlock ? : strp_sock_unlock;
Tom Herbert43a0c672016-08-15 14:51:01 -0700484 strp->cb.rcv_msg = cb->rcv_msg;
485 strp->cb.parse_msg = cb->parse_msg;
486 strp->cb.read_sock_done = cb->read_sock_done ? : default_read_sock_done;
Tom Herbertbbb03022017-07-28 16:22:43 -0700487 strp->cb.abort_parser = cb->abort_parser ? : strp_abort_strp;
488
Tom Herbert829385f2017-10-20 16:40:43 -0700489 INIT_DELAYED_WORK(&strp->msg_timer_work, strp_msg_timeout);
Tom Herbertbbb03022017-07-28 16:22:43 -0700490 INIT_WORK(&strp->work, strp_work);
Tom Herbert43a0c672016-08-15 14:51:01 -0700491
492 return 0;
493}
494EXPORT_SYMBOL_GPL(strp_init);
495
Doron Roberts-Kedes7170e602018-06-06 09:33:28 -0700496/* Sock process lock held (lock_sock) */
497void __strp_unpause(struct strparser *strp)
498{
499 strp->paused = 0;
500
501 if (strp->need_bytes) {
502 if (strp_peek_len(strp) < strp->need_bytes)
503 return;
504 }
505 strp_read_sock(strp);
506}
507EXPORT_SYMBOL_GPL(__strp_unpause);
508
Tom Herbertcff6a332016-08-23 11:55:30 -0700509void strp_unpause(struct strparser *strp)
510{
Tom Herbertbbb03022017-07-28 16:22:43 -0700511 strp->paused = 0;
Tom Herbertcff6a332016-08-23 11:55:30 -0700512
Tom Herbertbbb03022017-07-28 16:22:43 -0700513 /* Sync setting paused with RX work */
Tom Herbertcff6a332016-08-23 11:55:30 -0700514 smp_mb();
515
Tom Herbertbbb03022017-07-28 16:22:43 -0700516 queue_work(strp_wq, &strp->work);
Tom Herbertcff6a332016-08-23 11:55:30 -0700517}
518EXPORT_SYMBOL_GPL(strp_unpause);
519
Tom Herbert96a59082016-08-28 14:43:19 -0700520/* strp must already be stopped so that strp_recv will no longer be called.
Tom Herbert43a0c672016-08-15 14:51:01 -0700521 * Note that strp_done is not called with the lower socket held.
522 */
523void strp_done(struct strparser *strp)
524{
Tom Herbertbbb03022017-07-28 16:22:43 -0700525 WARN_ON(!strp->stopped);
Tom Herbert43a0c672016-08-15 14:51:01 -0700526
Tom Herbert829385f2017-10-20 16:40:43 -0700527 cancel_delayed_work_sync(&strp->msg_timer_work);
Tom Herbertbbb03022017-07-28 16:22:43 -0700528 cancel_work_sync(&strp->work);
Tom Herbert43a0c672016-08-15 14:51:01 -0700529
Tom Herbertbbb03022017-07-28 16:22:43 -0700530 if (strp->skb_head) {
531 kfree_skb(strp->skb_head);
532 strp->skb_head = NULL;
Tom Herbert43a0c672016-08-15 14:51:01 -0700533 }
534}
535EXPORT_SYMBOL_GPL(strp_done);
536
537void strp_stop(struct strparser *strp)
538{
Tom Herbertbbb03022017-07-28 16:22:43 -0700539 strp->stopped = 1;
Tom Herbert43a0c672016-08-15 14:51:01 -0700540}
541EXPORT_SYMBOL_GPL(strp_stop);
542
543void strp_check_rcv(struct strparser *strp)
544{
Tom Herbertbbb03022017-07-28 16:22:43 -0700545 queue_work(strp_wq, &strp->work);
Tom Herbert43a0c672016-08-15 14:51:01 -0700546}
547EXPORT_SYMBOL_GPL(strp_check_rcv);
548
Paul Gortmaker15253b42019-04-20 23:29:48 -0400549static int __init strp_dev_init(void)
Tom Herbert43a0c672016-08-15 14:51:01 -0700550{
551 strp_wq = create_singlethread_workqueue("kstrp");
Kangjie Lu228cd2d2019-03-14 23:12:06 -0500552 if (unlikely(!strp_wq))
553 return -ENOMEM;
Tom Herbert43a0c672016-08-15 14:51:01 -0700554
555 return 0;
556}
Paul Gortmaker15253b42019-04-20 23:29:48 -0400557device_initcall(strp_dev_init);