blob: 9d0388bed0c1d2166214c95081f4778afe9f50ed [file] [log] [blame]
Thomas Gleixner2874c5f2019-05-27 08:55:01 +02001// SPDX-License-Identifier: GPL-2.0-or-later
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
3 * Routines having to do with the 'struct sk_buff' memory handlers.
4 *
Alan Cox113aa832008-10-13 19:01:08 -07005 * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk>
Linus Torvalds1da177e2005-04-16 15:20:36 -07006 * Florian La Roche <rzsfl@rz.uni-sb.de>
7 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07008 * Fixes:
9 * Alan Cox : Fixed the worst of the load
10 * balancer bugs.
11 * Dave Platt : Interrupt stacking fix.
12 * Richard Kooijman : Timestamp fixes.
13 * Alan Cox : Changed buffer format.
14 * Alan Cox : destructor hook for AF_UNIX etc.
15 * Linus Torvalds : Better skb_clone.
16 * Alan Cox : Added skb_copy.
17 * Alan Cox : Added all the changed routines Linus
18 * only put in the headers
19 * Ray VanTassle : Fixed --skb->lock in free
20 * Alan Cox : skb_copy copy arp field
21 * Andi Kleen : slabified it.
22 * Robert Olsson : Removed skb_head_pool
23 *
24 * NOTE:
25 * The __skb_ routines should be called with interrupts
26 * disabled, or you better be *real* sure that the operation is atomic
27 * with respect to whatever list is being frobbed (e.g. via lock_sock()
28 * or via disabling bottom half handlers, etc).
Linus Torvalds1da177e2005-04-16 15:20:36 -070029 */
30
31/*
32 * The functions in this file will not compile correctly with gcc 2.4.x
33 */
34
Joe Perchese005d192012-05-16 19:58:40 +000035#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36
Linus Torvalds1da177e2005-04-16 15:20:36 -070037#include <linux/module.h>
38#include <linux/types.h>
39#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070040#include <linux/mm.h>
41#include <linux/interrupt.h>
42#include <linux/in.h>
43#include <linux/inet.h>
44#include <linux/slab.h>
Florian Westphalde960aa2014-01-26 10:58:16 +010045#include <linux/tcp.h>
46#include <linux/udp.h>
Marcelo Ricardo Leitner90017ac2016-06-02 15:05:43 -030047#include <linux/sctp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070048#include <linux/netdevice.h>
49#ifdef CONFIG_NET_CLS_ACT
50#include <net/pkt_sched.h>
51#endif
52#include <linux/string.h>
53#include <linux/skbuff.h>
Jens Axboe9c55e012007-11-06 23:30:13 -080054#include <linux/splice.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070055#include <linux/cache.h>
56#include <linux/rtnetlink.h>
57#include <linux/init.h>
David Howells716ea3a2007-04-02 20:19:53 -070058#include <linux/scatterlist.h>
Patrick Ohlyac45f602009-02-12 05:03:37 +000059#include <linux/errqueue.h>
Linus Torvalds268bb0c2011-05-20 12:50:29 -070060#include <linux/prefetch.h>
Vlad Yasevich0d5501c2014-08-08 14:42:13 -040061#include <linux/if_vlan.h>
John Hurley2a2ea502019-07-07 15:01:57 +010062#include <linux/mpls.h>
Sebastian Andrzej Siewior183f47f2021-02-18 18:31:24 +010063#include <linux/kcov.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070064
65#include <net/protocol.h>
66#include <net/dst.h>
67#include <net/sock.h>
68#include <net/checksum.h>
Paul Durranted1f50c2014-01-09 10:02:46 +000069#include <net/ip6_checksum.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070070#include <net/xfrm.h>
John Hurley8822e272019-07-07 15:01:54 +010071#include <net/mpls.h>
Mat Martineau3ee17bc2020-01-09 07:59:19 -080072#include <net/mptcp.h>
Jeremy Kerr78476d32021-10-29 11:01:44 +080073#include <net/mctp.h>
Ilias Apalodimas6a5bcd82021-06-07 21:02:38 +020074#include <net/page_pool.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070075
Linus Torvalds7c0f6ba2016-12-24 11:46:01 -080076#include <linux/uaccess.h>
Steven Rostedtad8d75f2009-04-14 19:39:12 -040077#include <trace/events/skb.h>
Eric Dumazet51c56b02012-04-05 11:35:15 +020078#include <linux/highmem.h>
Willem de Bruijnb245be12015-01-30 13:29:32 -050079#include <linux/capability.h>
80#include <linux/user_namespace.h>
Matteo Croce2544af02019-05-29 17:13:48 +020081#include <linux/indirect_call_wrapper.h>
Al Viroa1f8e7f72006-10-19 16:08:53 -040082
Bart Van Assche7b7ed882019-03-25 09:17:23 -070083#include "datagram.h"
Vasily Averin7f678de2021-10-22 13:28:37 +030084#include "sock_destructor.h"
Bart Van Assche7b7ed882019-03-25 09:17:23 -070085
Alexey Dobriyan08009a72018-02-24 21:20:33 +030086struct kmem_cache *skbuff_head_cache __ro_after_init;
87static struct kmem_cache *skbuff_fclone_cache __ro_after_init;
Florian Westphaldf5042f2018-12-18 17:15:16 +010088#ifdef CONFIG_SKB_EXTENSIONS
89static struct kmem_cache *skbuff_ext_cache __ro_after_init;
90#endif
Hans Westgaard Ry5f74f82e2016-02-03 09:26:57 +010091int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS;
92EXPORT_SYMBOL(sysctl_max_skb_frags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070093
Linus Torvalds1da177e2005-04-16 15:20:36 -070094/**
Jean Sacrenf05de732013-02-11 13:30:38 +000095 * skb_panic - private function for out-of-line support
96 * @skb: buffer
97 * @sz: size
98 * @addr: address
James Hogan99d58512013-02-13 11:20:27 +000099 * @msg: skb_over_panic or skb_under_panic
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100 *
Jean Sacrenf05de732013-02-11 13:30:38 +0000101 * Out-of-line support for skb_put() and skb_push().
102 * Called via the wrapper skb_over_panic() or skb_under_panic().
103 * Keep out of line to prevent kernel bloat.
104 * __builtin_return_address is not used because it is not always reliable.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105 */
Jean Sacrenf05de732013-02-11 13:30:38 +0000106static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr,
James Hogan99d58512013-02-13 11:20:27 +0000107 const char msg[])
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108{
Jesper Dangaard Brouer41a46912020-04-27 18:37:43 +0200109 pr_emerg("%s: text:%px len:%d put:%d head:%px data:%px tail:%#lx end:%#lx dev:%s\n",
James Hogan99d58512013-02-13 11:20:27 +0000110 msg, addr, skb->len, sz, skb->head, skb->data,
Joe Perchese005d192012-05-16 19:58:40 +0000111 (unsigned long)skb->tail, (unsigned long)skb->end,
112 skb->dev ? skb->dev->name : "<NULL>");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113 BUG();
114}
115
Jean Sacrenf05de732013-02-11 13:30:38 +0000116static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117{
Jean Sacrenf05de732013-02-11 13:30:38 +0000118 skb_panic(skb, sz, addr, __func__);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119}
120
Jean Sacrenf05de732013-02-11 13:30:38 +0000121static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr)
122{
123 skb_panic(skb, sz, addr, __func__);
124}
Mel Gormanc93bdd02012-07-31 16:44:19 -0700125
Alexander Lobakin50fad4b52021-02-13 14:12:13 +0000126#define NAPI_SKB_CACHE_SIZE 64
Alexander Lobakinf450d532021-02-13 14:12:25 +0000127#define NAPI_SKB_CACHE_BULK 16
128#define NAPI_SKB_CACHE_HALF (NAPI_SKB_CACHE_SIZE / 2)
Alexander Lobakin50fad4b52021-02-13 14:12:13 +0000129
130struct napi_alloc_cache {
131 struct page_frag_cache page;
132 unsigned int skb_count;
133 void *skb_cache[NAPI_SKB_CACHE_SIZE];
134};
135
136static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache);
137static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache);
138
Yajun Deng32e35732021-09-14 11:49:35 +0800139void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask)
Alexander Lobakin50fad4b52021-02-13 14:12:13 +0000140{
141 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
142
Alexander Lobakin50fad4b52021-02-13 14:12:13 +0000143 fragsz = SKB_DATA_ALIGN(fragsz);
144
Yajun Deng32e35732021-09-14 11:49:35 +0800145 return page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC, align_mask);
Alexander Lobakin50fad4b52021-02-13 14:12:13 +0000146}
147EXPORT_SYMBOL(__napi_alloc_frag_align);
148
149void *__netdev_alloc_frag_align(unsigned int fragsz, unsigned int align_mask)
150{
Alexander Lobakin50fad4b52021-02-13 14:12:13 +0000151 void *data;
152
153 fragsz = SKB_DATA_ALIGN(fragsz);
Changbin Duafa79d02021-08-13 22:57:49 +0800154 if (in_hardirq() || irqs_disabled()) {
Yajun Deng32e35732021-09-14 11:49:35 +0800155 struct page_frag_cache *nc = this_cpu_ptr(&netdev_alloc_cache);
156
Alexander Lobakin50fad4b52021-02-13 14:12:13 +0000157 data = page_frag_alloc_align(nc, fragsz, GFP_ATOMIC, align_mask);
158 } else {
Yajun Deng32e35732021-09-14 11:49:35 +0800159 struct napi_alloc_cache *nc;
160
Alexander Lobakin50fad4b52021-02-13 14:12:13 +0000161 local_bh_disable();
Yajun Deng32e35732021-09-14 11:49:35 +0800162 nc = this_cpu_ptr(&napi_alloc_cache);
163 data = page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC, align_mask);
Alexander Lobakin50fad4b52021-02-13 14:12:13 +0000164 local_bh_enable();
165 }
166 return data;
167}
168EXPORT_SYMBOL(__netdev_alloc_frag_align);
169
Alexander Lobakinf450d532021-02-13 14:12:25 +0000170static struct sk_buff *napi_skb_cache_get(void)
171{
172 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
173 struct sk_buff *skb;
174
175 if (unlikely(!nc->skb_count))
176 nc->skb_count = kmem_cache_alloc_bulk(skbuff_head_cache,
177 GFP_ATOMIC,
178 NAPI_SKB_CACHE_BULK,
179 nc->skb_cache);
180 if (unlikely(!nc->skb_count))
181 return NULL;
182
183 skb = nc->skb_cache[--nc->skb_count];
184 kasan_unpoison_object_data(skbuff_head_cache, skb);
185
186 return skb;
187}
188
Jesper Dangaard Brouerba0509b2019-04-12 17:07:37 +0200189/* Caller must provide SKB that is memset cleared */
Alexander Lobakin483126b2021-02-13 14:11:26 +0000190static void __build_skb_around(struct sk_buff *skb, void *data,
191 unsigned int frag_size)
Jesper Dangaard Brouerba0509b2019-04-12 17:07:37 +0200192{
193 struct skb_shared_info *shinfo;
194 unsigned int size = frag_size ? : ksize(data);
195
196 size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
197
198 /* Assumes caller memset cleared SKB */
199 skb->truesize = SKB_TRUESIZE(size);
200 refcount_set(&skb->users, 1);
201 skb->head = data;
202 skb->data = data;
203 skb_reset_tail_pointer(skb);
204 skb->end = skb->tail + size;
205 skb->mac_header = (typeof(skb->mac_header))~0U;
206 skb->transport_header = (typeof(skb->transport_header))~0U;
207
208 /* make sure we initialize shinfo sequentially */
209 shinfo = skb_shinfo(skb);
210 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
211 atomic_set(&shinfo->dataref, 1);
212
Aleksandr Nogikh6370cc32020-10-29 17:36:19 +0000213 skb_set_kcov_handle(skb, kcov_common_handle());
Jesper Dangaard Brouerba0509b2019-04-12 17:07:37 +0200214}
215
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216/**
Eric Dumazet2ea2f622015-04-24 16:05:01 -0700217 * __build_skb - build a network buffer
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000218 * @data: data buffer provided by caller
Eric Dumazet2ea2f622015-04-24 16:05:01 -0700219 * @frag_size: size of data, or 0 if head was kmalloced
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000220 *
221 * Allocate a new &sk_buff. Caller provides space holding head and
Florian Fainellideceb4c2013-07-23 20:22:39 +0100222 * skb_shared_info. @data must have been allocated by kmalloc() only if
Eric Dumazet2ea2f622015-04-24 16:05:01 -0700223 * @frag_size is 0, otherwise data should come from the page allocator
224 * or vmalloc()
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000225 * The return is the new skb buffer.
226 * On a failure the return is %NULL, and @data is not freed.
227 * Notes :
228 * Before IO, driver allocates only data buffer where NIC put incoming frame
229 * Driver should add room at head (NET_SKB_PAD) and
230 * MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info))
231 * After IO, driver calls build_skb(), to allocate sk_buff and populate it
232 * before giving packet to stack.
233 * RX rings only contains data buffers, not full skbs.
234 */
Eric Dumazet2ea2f622015-04-24 16:05:01 -0700235struct sk_buff *__build_skb(void *data, unsigned int frag_size)
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000236{
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000237 struct sk_buff *skb;
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000238
239 skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC);
Jesper Dangaard Brouerba0509b2019-04-12 17:07:37 +0200240 if (unlikely(!skb))
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000241 return NULL;
242
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000243 memset(skb, 0, offsetof(struct sk_buff, tail));
Alexander Lobakin483126b2021-02-13 14:11:26 +0000244 __build_skb_around(skb, data, frag_size);
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000245
Alexander Lobakin483126b2021-02-13 14:11:26 +0000246 return skb;
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000247}
Eric Dumazet2ea2f622015-04-24 16:05:01 -0700248
249/* build_skb() is wrapper over __build_skb(), that specifically
250 * takes care of skb->head and skb->pfmemalloc
251 * This means that if @frag_size is not zero, then @data must be backed
252 * by a page fragment, not kmalloc() or vmalloc()
253 */
254struct sk_buff *build_skb(void *data, unsigned int frag_size)
255{
256 struct sk_buff *skb = __build_skb(data, frag_size);
257
258 if (skb && frag_size) {
259 skb->head_frag = 1;
Michal Hocko2f064f32015-08-21 14:11:51 -0700260 if (page_is_pfmemalloc(virt_to_head_page(data)))
Eric Dumazet2ea2f622015-04-24 16:05:01 -0700261 skb->pfmemalloc = 1;
262 }
263 return skb;
264}
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000265EXPORT_SYMBOL(build_skb);
266
Jesper Dangaard Brouerba0509b2019-04-12 17:07:37 +0200267/**
268 * build_skb_around - build a network buffer around provided skb
269 * @skb: sk_buff provide by caller, must be memset cleared
270 * @data: data buffer provided by caller
271 * @frag_size: size of data, or 0 if head was kmalloced
272 */
273struct sk_buff *build_skb_around(struct sk_buff *skb,
274 void *data, unsigned int frag_size)
275{
276 if (unlikely(!skb))
277 return NULL;
278
Alexander Lobakin483126b2021-02-13 14:11:26 +0000279 __build_skb_around(skb, data, frag_size);
Jesper Dangaard Brouerba0509b2019-04-12 17:07:37 +0200280
Alexander Lobakin483126b2021-02-13 14:11:26 +0000281 if (frag_size) {
Jesper Dangaard Brouerba0509b2019-04-12 17:07:37 +0200282 skb->head_frag = 1;
283 if (page_is_pfmemalloc(virt_to_head_page(data)))
284 skb->pfmemalloc = 1;
285 }
286 return skb;
287}
288EXPORT_SYMBOL(build_skb_around);
289
Alexander Lobakinf450d532021-02-13 14:12:25 +0000290/**
291 * __napi_build_skb - build a network buffer
292 * @data: data buffer provided by caller
293 * @frag_size: size of data, or 0 if head was kmalloced
294 *
295 * Version of __build_skb() that uses NAPI percpu caches to obtain
296 * skbuff_head instead of inplace allocation.
297 *
298 * Returns a new &sk_buff on success, %NULL on allocation failure.
299 */
300static struct sk_buff *__napi_build_skb(void *data, unsigned int frag_size)
301{
302 struct sk_buff *skb;
303
304 skb = napi_skb_cache_get();
305 if (unlikely(!skb))
306 return NULL;
307
308 memset(skb, 0, offsetof(struct sk_buff, tail));
309 __build_skb_around(skb, data, frag_size);
310
311 return skb;
312}
313
314/**
315 * napi_build_skb - build a network buffer
316 * @data: data buffer provided by caller
317 * @frag_size: size of data, or 0 if head was kmalloced
318 *
319 * Version of __napi_build_skb() that takes care of skb->head_frag
320 * and skb->pfmemalloc when the data is a page or page fragment.
321 *
322 * Returns a new &sk_buff on success, %NULL on allocation failure.
323 */
324struct sk_buff *napi_build_skb(void *data, unsigned int frag_size)
325{
326 struct sk_buff *skb = __napi_build_skb(data, frag_size);
327
328 if (likely(skb) && frag_size) {
329 skb->head_frag = 1;
330 skb_propagate_pfmemalloc(virt_to_head_page(data), skb);
331 }
332
333 return skb;
334}
335EXPORT_SYMBOL(napi_build_skb);
336
Alexander Lobakin5381b232021-02-13 14:11:00 +0000337/*
338 * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells
339 * the caller if emergency pfmemalloc reserves are being used. If it is and
340 * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves
341 * may be used. Otherwise, the packet data may be discarded until enough
342 * memory is free
343 */
Alexander Lobakinef280952021-02-13 14:11:11 +0000344static void *kmalloc_reserve(size_t size, gfp_t flags, int node,
345 bool *pfmemalloc)
Alexander Lobakin5381b232021-02-13 14:11:00 +0000346{
347 void *obj;
348 bool ret_pfmemalloc = false;
349
350 /*
351 * Try a regular allocation, when that fails and we're not entitled
352 * to the reserves, fail.
353 */
354 obj = kmalloc_node_track_caller(size,
355 flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
356 node);
357 if (obj || !(gfp_pfmemalloc_allowed(flags)))
358 goto out;
359
360 /* Try again but now we are using pfmemalloc reserves */
361 ret_pfmemalloc = true;
362 obj = kmalloc_node_track_caller(size, flags, node);
363
364out:
365 if (pfmemalloc)
366 *pfmemalloc = ret_pfmemalloc;
367
368 return obj;
369}
370
371/* Allocate a new skbuff. We do this ourselves so we can fill in a few
372 * 'private' fields and also do memory statistics to find all the
373 * [BEEP] leaks.
374 *
375 */
376
377/**
378 * __alloc_skb - allocate a network buffer
379 * @size: size to allocate
380 * @gfp_mask: allocation mask
381 * @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache
382 * instead of head cache and allocate a cloned (child) skb.
383 * If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for
384 * allocations in case the data is required for writeback
385 * @node: numa node to allocate memory on
386 *
387 * Allocate a new &sk_buff. The returned buffer has no headroom and a
388 * tail room of at least size bytes. The object has a reference count
389 * of one. The return is the buffer. On a failure the return is %NULL.
390 *
391 * Buffers may only be allocated from interrupts using a @gfp_mask of
392 * %GFP_ATOMIC.
393 */
394struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
395 int flags, int node)
396{
397 struct kmem_cache *cache;
Alexander Lobakin5381b232021-02-13 14:11:00 +0000398 struct sk_buff *skb;
Li RongQinga5df6332021-09-22 14:17:19 +0800399 unsigned int osize;
Alexander Lobakin5381b232021-02-13 14:11:00 +0000400 bool pfmemalloc;
Li RongQinga5df6332021-09-22 14:17:19 +0800401 u8 *data;
Alexander Lobakin5381b232021-02-13 14:11:00 +0000402
403 cache = (flags & SKB_ALLOC_FCLONE)
404 ? skbuff_fclone_cache : skbuff_head_cache;
405
406 if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))
407 gfp_mask |= __GFP_MEMALLOC;
408
409 /* Get the HEAD */
Alexander Lobakind13612b2021-02-13 14:12:38 +0000410 if ((flags & (SKB_ALLOC_FCLONE | SKB_ALLOC_NAPI)) == SKB_ALLOC_NAPI &&
411 likely(node == NUMA_NO_NODE || node == numa_mem_id()))
412 skb = napi_skb_cache_get();
413 else
414 skb = kmem_cache_alloc_node(cache, gfp_mask & ~GFP_DMA, node);
Alexander Lobakindf1ae022021-02-13 14:11:39 +0000415 if (unlikely(!skb))
416 return NULL;
Alexander Lobakin5381b232021-02-13 14:11:00 +0000417 prefetchw(skb);
418
419 /* We do our best to align skb_shared_info on a separate cache
420 * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives
421 * aligned memory blocks, unless SLUB/SLAB debug is enabled.
422 * Both skb->head and skb_shared_info are cache line aligned.
423 */
424 size = SKB_DATA_ALIGN(size);
425 size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
426 data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc);
Alexander Lobakindf1ae022021-02-13 14:11:39 +0000427 if (unlikely(!data))
Alexander Lobakin5381b232021-02-13 14:11:00 +0000428 goto nodata;
429 /* kmalloc(size) might give us more room than requested.
430 * Put skb_shared_info exactly at the end of allocated zone,
431 * to allow max possible filling before reallocation.
432 */
Li RongQinga5df6332021-09-22 14:17:19 +0800433 osize = ksize(data);
434 size = SKB_WITH_OVERHEAD(osize);
Alexander Lobakin5381b232021-02-13 14:11:00 +0000435 prefetchw(data + size);
436
437 /*
438 * Only clear those fields we need to clear, not those that we will
439 * actually initialise below. Hence, don't put any more fields after
440 * the tail pointer in struct sk_buff!
441 */
442 memset(skb, 0, offsetof(struct sk_buff, tail));
Li RongQinga5df6332021-09-22 14:17:19 +0800443 __build_skb_around(skb, data, osize);
Alexander Lobakin5381b232021-02-13 14:11:00 +0000444 skb->pfmemalloc = pfmemalloc;
Alexander Lobakin5381b232021-02-13 14:11:00 +0000445
446 if (flags & SKB_ALLOC_FCLONE) {
447 struct sk_buff_fclones *fclones;
448
449 fclones = container_of(skb, struct sk_buff_fclones, skb1);
450
451 skb->fclone = SKB_FCLONE_ORIG;
452 refcount_set(&fclones->fclone_ref, 1);
453
454 fclones->skb2.fclone = SKB_FCLONE_CLONE;
455 }
456
Alexander Lobakin5381b232021-02-13 14:11:00 +0000457 return skb;
Alexander Lobakindf1ae022021-02-13 14:11:39 +0000458
Alexander Lobakin5381b232021-02-13 14:11:00 +0000459nodata:
460 kmem_cache_free(cache, skb);
Alexander Lobakindf1ae022021-02-13 14:11:39 +0000461 return NULL;
Alexander Lobakin5381b232021-02-13 14:11:00 +0000462}
463EXPORT_SYMBOL(__alloc_skb);
464
Sebastian Andrzej Siewior7ba7aea2019-06-07 21:20:34 +0200465/**
Alexander Duyckfd11a832014-12-09 19:40:49 -0800466 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device
467 * @dev: network device to receive on
Masanari Iidad7499162015-08-24 22:56:54 +0900468 * @len: length to allocate
Alexander Duyckfd11a832014-12-09 19:40:49 -0800469 * @gfp_mask: get_free_pages mask, passed to alloc_skb
470 *
471 * Allocate a new &sk_buff and assign it a usage count of one. The
472 * buffer has NET_SKB_PAD headroom built in. Users should allocate
473 * the headroom they think they need without accounting for the
474 * built in space. The built in space is used for optimisations.
475 *
476 * %NULL is returned if there is no free memory.
477 */
Alexander Duyck94519802015-05-06 21:11:40 -0700478struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len,
479 gfp_t gfp_mask)
Alexander Duyckfd11a832014-12-09 19:40:49 -0800480{
Alexander Duyckb63ae8c2015-05-06 21:11:57 -0700481 struct page_frag_cache *nc;
Alexander Duyckfd11a832014-12-09 19:40:49 -0800482 struct sk_buff *skb;
Alexander Duyck94519802015-05-06 21:11:40 -0700483 bool pfmemalloc;
484 void *data;
Alexander Duyckfd11a832014-12-09 19:40:49 -0800485
Alexander Duyck94519802015-05-06 21:11:40 -0700486 len += NET_SKB_PAD;
Alexander Duyckfd11a832014-12-09 19:40:49 -0800487
Alexander Lobakin66c55602021-01-15 15:04:40 +0000488 /* If requested length is either too small or too big,
489 * we use kmalloc() for skb->head allocation.
490 */
491 if (len <= SKB_WITH_OVERHEAD(1024) ||
492 len > SKB_WITH_OVERHEAD(PAGE_SIZE) ||
Mel Gormand0164ad2015-11-06 16:28:21 -0800493 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
Alexander Duycka080e7b2015-05-13 13:34:13 -0700494 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
495 if (!skb)
496 goto skb_fail;
497 goto skb_success;
498 }
Alexander Duyck94519802015-05-06 21:11:40 -0700499
500 len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
501 len = SKB_DATA_ALIGN(len);
502
503 if (sk_memalloc_socks())
504 gfp_mask |= __GFP_MEMALLOC;
505
Changbin Duafa79d02021-08-13 22:57:49 +0800506 if (in_hardirq() || irqs_disabled()) {
Sebastian Andrzej Siewior92dcabd2019-06-07 21:20:35 +0200507 nc = this_cpu_ptr(&netdev_alloc_cache);
508 data = page_frag_alloc(nc, len, gfp_mask);
509 pfmemalloc = nc->pfmemalloc;
510 } else {
511 local_bh_disable();
512 nc = this_cpu_ptr(&napi_alloc_cache.page);
513 data = page_frag_alloc(nc, len, gfp_mask);
514 pfmemalloc = nc->pfmemalloc;
515 local_bh_enable();
516 }
Alexander Duyck94519802015-05-06 21:11:40 -0700517
518 if (unlikely(!data))
519 return NULL;
520
521 skb = __build_skb(data, len);
522 if (unlikely(!skb)) {
Alexander Duyck181edb22015-05-06 21:12:03 -0700523 skb_free_frag(data);
Alexander Duyck94519802015-05-06 21:11:40 -0700524 return NULL;
Christoph Hellwig7b2e4972006-08-07 16:09:04 -0700525 }
Alexander Duyckfd11a832014-12-09 19:40:49 -0800526
Alexander Duyck94519802015-05-06 21:11:40 -0700527 if (pfmemalloc)
528 skb->pfmemalloc = 1;
529 skb->head_frag = 1;
530
Alexander Duycka080e7b2015-05-13 13:34:13 -0700531skb_success:
Alexander Duyck94519802015-05-06 21:11:40 -0700532 skb_reserve(skb, NET_SKB_PAD);
533 skb->dev = dev;
534
Alexander Duycka080e7b2015-05-13 13:34:13 -0700535skb_fail:
Christoph Hellwig8af27452006-07-31 22:35:23 -0700536 return skb;
537}
David S. Millerb4ac530fc2009-02-10 02:09:24 -0800538EXPORT_SYMBOL(__netdev_alloc_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700539
Alexander Duyckfd11a832014-12-09 19:40:49 -0800540/**
541 * __napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance
542 * @napi: napi instance this buffer was allocated for
Masanari Iidad7499162015-08-24 22:56:54 +0900543 * @len: length to allocate
Alexander Duyckfd11a832014-12-09 19:40:49 -0800544 * @gfp_mask: get_free_pages mask, passed to alloc_skb and alloc_pages
545 *
546 * Allocate a new sk_buff for use in NAPI receive. This buffer will
547 * attempt to allocate the head from a special reserved region used
548 * only for NAPI Rx allocation. By doing this we can save several
549 * CPU cycles by avoiding having to disable and re-enable IRQs.
550 *
551 * %NULL is returned if there is no free memory.
552 */
Alexander Duyck94519802015-05-06 21:11:40 -0700553struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len,
554 gfp_t gfp_mask)
Alexander Duyckfd11a832014-12-09 19:40:49 -0800555{
Eric Dumazet3226b152021-01-13 08:18:19 -0800556 struct napi_alloc_cache *nc;
Alexander Duyckfd11a832014-12-09 19:40:49 -0800557 struct sk_buff *skb;
Alexander Duyck94519802015-05-06 21:11:40 -0700558 void *data;
Alexander Duyckfd11a832014-12-09 19:40:49 -0800559
Alexander Duyck94519802015-05-06 21:11:40 -0700560 len += NET_SKB_PAD + NET_IP_ALIGN;
Alexander Duyckfd11a832014-12-09 19:40:49 -0800561
Eric Dumazet3226b152021-01-13 08:18:19 -0800562 /* If requested length is either too small or too big,
563 * we use kmalloc() for skb->head allocation.
564 */
565 if (len <= SKB_WITH_OVERHEAD(1024) ||
566 len > SKB_WITH_OVERHEAD(PAGE_SIZE) ||
Mel Gormand0164ad2015-11-06 16:28:21 -0800567 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
Alexander Lobakincfb8ec62021-02-13 14:12:49 +0000568 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX | SKB_ALLOC_NAPI,
569 NUMA_NO_NODE);
Alexander Duycka080e7b2015-05-13 13:34:13 -0700570 if (!skb)
571 goto skb_fail;
572 goto skb_success;
573 }
Alexander Duyck94519802015-05-06 21:11:40 -0700574
Eric Dumazet3226b152021-01-13 08:18:19 -0800575 nc = this_cpu_ptr(&napi_alloc_cache);
Alexander Duyck94519802015-05-06 21:11:40 -0700576 len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
577 len = SKB_DATA_ALIGN(len);
578
579 if (sk_memalloc_socks())
580 gfp_mask |= __GFP_MEMALLOC;
581
Alexander Duyck8c2dd3e2017-01-10 16:58:06 -0800582 data = page_frag_alloc(&nc->page, len, gfp_mask);
Alexander Duyck94519802015-05-06 21:11:40 -0700583 if (unlikely(!data))
584 return NULL;
585
Alexander Lobakincfb8ec62021-02-13 14:12:49 +0000586 skb = __napi_build_skb(data, len);
Alexander Duyck94519802015-05-06 21:11:40 -0700587 if (unlikely(!skb)) {
Alexander Duyck181edb22015-05-06 21:12:03 -0700588 skb_free_frag(data);
Alexander Duyck94519802015-05-06 21:11:40 -0700589 return NULL;
Alexander Duyckfd11a832014-12-09 19:40:49 -0800590 }
591
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100592 if (nc->page.pfmemalloc)
Alexander Duyck94519802015-05-06 21:11:40 -0700593 skb->pfmemalloc = 1;
594 skb->head_frag = 1;
595
Alexander Duycka080e7b2015-05-13 13:34:13 -0700596skb_success:
Alexander Duyck94519802015-05-06 21:11:40 -0700597 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
598 skb->dev = napi->dev;
599
Alexander Duycka080e7b2015-05-13 13:34:13 -0700600skb_fail:
Alexander Duyckfd11a832014-12-09 19:40:49 -0800601 return skb;
602}
603EXPORT_SYMBOL(__napi_alloc_skb);
604
Peter Zijlstra654bed12008-10-07 14:22:33 -0700605void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
Eric Dumazet50269e12012-03-23 23:59:33 +0000606 int size, unsigned int truesize)
Peter Zijlstra654bed12008-10-07 14:22:33 -0700607{
608 skb_fill_page_desc(skb, i, page, off, size);
609 skb->len += size;
610 skb->data_len += size;
Eric Dumazet50269e12012-03-23 23:59:33 +0000611 skb->truesize += truesize;
Peter Zijlstra654bed12008-10-07 14:22:33 -0700612}
613EXPORT_SYMBOL(skb_add_rx_frag);
614
Jason Wangf8e617e2013-11-01 14:07:47 +0800615void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
616 unsigned int truesize)
617{
618 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
619
620 skb_frag_size_add(frag, size);
621 skb->len += size;
622 skb->data_len += size;
623 skb->truesize += truesize;
624}
625EXPORT_SYMBOL(skb_coalesce_rx_frag);
626
Herbert Xu27b437c2006-07-13 19:26:39 -0700627static void skb_drop_list(struct sk_buff **listp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700628{
Eric Dumazetbd8a7032013-06-24 06:26:00 -0700629 kfree_skb_list(*listp);
Herbert Xu27b437c2006-07-13 19:26:39 -0700630 *listp = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631}
632
Herbert Xu27b437c2006-07-13 19:26:39 -0700633static inline void skb_drop_fraglist(struct sk_buff *skb)
634{
635 skb_drop_list(&skb_shinfo(skb)->frag_list);
636}
637
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638static void skb_clone_fraglist(struct sk_buff *skb)
639{
640 struct sk_buff *list;
641
David S. Millerfbb398a2009-06-09 00:18:59 -0700642 skb_walk_frags(skb, list)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700643 skb_get(list);
644}
645
Eric Dumazetd3836f22012-04-27 00:33:38 +0000646static void skb_free_head(struct sk_buff *skb)
647{
Alexander Duyck181edb22015-05-06 21:12:03 -0700648 unsigned char *head = skb->head;
649
Ilias Apalodimas6a5bcd82021-06-07 21:02:38 +0200650 if (skb->head_frag) {
651 if (skb_pp_recycle(skb, head))
652 return;
Alexander Duyck181edb22015-05-06 21:12:03 -0700653 skb_free_frag(head);
Ilias Apalodimas6a5bcd82021-06-07 21:02:38 +0200654 } else {
Alexander Duyck181edb22015-05-06 21:12:03 -0700655 kfree(head);
Ilias Apalodimas6a5bcd82021-06-07 21:02:38 +0200656 }
Eric Dumazetd3836f22012-04-27 00:33:38 +0000657}
658
Adrian Bunk5bba1712006-06-29 13:02:35 -0700659static void skb_release_data(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700660{
Eric Dumazetff04a772014-09-23 18:39:30 -0700661 struct skb_shared_info *shinfo = skb_shinfo(skb);
662 int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700663
Eric Dumazetff04a772014-09-23 18:39:30 -0700664 if (skb->cloned &&
665 atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
666 &shinfo->dataref))
Ilias Apalodimas2cc3aeb2021-07-16 10:02:18 +0300667 goto exit;
Shirley Maa6686f22011-07-06 12:22:12 +0000668
Jonathan Lemon70c43162021-01-06 14:18:36 -0800669 skb_zcopy_clear(skb, true);
670
Eric Dumazetff04a772014-09-23 18:39:30 -0700671 for (i = 0; i < shinfo->nr_frags; i++)
Ilias Apalodimas6a5bcd82021-06-07 21:02:38 +0200672 __skb_frag_unref(&shinfo->frags[i], skb->pp_recycle);
Shirley Maa6686f22011-07-06 12:22:12 +0000673
Eric Dumazetff04a772014-09-23 18:39:30 -0700674 if (shinfo->frag_list)
675 kfree_skb_list(shinfo->frag_list);
676
677 skb_free_head(skb);
Ilias Apalodimas2cc3aeb2021-07-16 10:02:18 +0300678exit:
679 /* When we clone an SKB we copy the reycling bit. The pp_recycle
680 * bit is only set on the head though, so in order to avoid races
681 * while trying to recycle fragments on __skb_frag_unref() we need
682 * to make one SKB responsible for triggering the recycle path.
683 * So disable the recycling bit if an SKB is cloned and we have
Tom Rix58e61e42022-02-09 07:02:42 -0800684 * additional references to the fragmented part of the SKB.
Ilias Apalodimas2cc3aeb2021-07-16 10:02:18 +0300685 * Eventually the last SKB will have the recycling bit set and it's
686 * dataref set to 0, which will trigger the recycling
687 */
688 skb->pp_recycle = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700689}
690
691/*
692 * Free an skbuff by memory without cleaning the state.
693 */
Herbert Xu2d4baff2007-11-26 23:11:19 +0800694static void kfree_skbmem(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700695{
Eric Dumazetd0bf4a92014-09-29 13:29:15 -0700696 struct sk_buff_fclones *fclones;
David S. Millerd179cd12005-08-17 14:57:30 -0700697
David S. Millerd179cd12005-08-17 14:57:30 -0700698 switch (skb->fclone) {
699 case SKB_FCLONE_UNAVAILABLE:
700 kmem_cache_free(skbuff_head_cache, skb);
Eric Dumazet6ffe75eb2014-12-03 17:04:39 -0800701 return;
David S. Millerd179cd12005-08-17 14:57:30 -0700702
703 case SKB_FCLONE_ORIG:
Eric Dumazetd0bf4a92014-09-29 13:29:15 -0700704 fclones = container_of(skb, struct sk_buff_fclones, skb1);
Eric Dumazet6ffe75eb2014-12-03 17:04:39 -0800705
706 /* We usually free the clone (TX completion) before original skb
707 * This test would have no chance to be true for the clone,
708 * while here, branch prediction will be good.
709 */
Reshetova, Elena26385952017-06-30 13:07:59 +0300710 if (refcount_read(&fclones->fclone_ref) == 1)
Eric Dumazet6ffe75eb2014-12-03 17:04:39 -0800711 goto fastpath;
David S. Millerd179cd12005-08-17 14:57:30 -0700712 break;
713
Eric Dumazet6ffe75eb2014-12-03 17:04:39 -0800714 default: /* SKB_FCLONE_CLONE */
Eric Dumazetd0bf4a92014-09-29 13:29:15 -0700715 fclones = container_of(skb, struct sk_buff_fclones, skb2);
David S. Millerd179cd12005-08-17 14:57:30 -0700716 break;
Stephen Hemminger3ff50b72007-04-20 17:09:22 -0700717 }
Reshetova, Elena26385952017-06-30 13:07:59 +0300718 if (!refcount_dec_and_test(&fclones->fclone_ref))
Eric Dumazet6ffe75eb2014-12-03 17:04:39 -0800719 return;
720fastpath:
721 kmem_cache_free(skbuff_fclone_cache, fclones);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700722}
723
Paolo Abeni0a463c72017-06-12 11:23:42 +0200724void skb_release_head_state(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700725{
Eric Dumazetadf30902009-06-02 05:19:30 +0000726 skb_dst_drop(skb);
Stephen Hemminger9c2b3322005-04-19 22:39:42 -0700727 if (skb->destructor) {
Changbin Duafa79d02021-08-13 22:57:49 +0800728 WARN_ON(in_hardirq());
Linus Torvalds1da177e2005-04-16 15:20:36 -0700729 skb->destructor(skb);
730 }
Igor Maravića3bf7ae2011-12-12 02:58:22 +0000731#if IS_ENABLED(CONFIG_NF_CONNTRACK)
Florian Westphalcb9c6832017-01-23 18:21:56 +0100732 nf_conntrack_put(skb_nfct(skb));
KOVACS Krisztian2fc72c72011-01-12 20:25:08 +0100733#endif
Florian Westphaldf5042f2018-12-18 17:15:16 +0100734 skb_ext_put(skb);
Lennert Buytenhek04a4bb52008-10-01 02:33:12 -0700735}
736
737/* Free everything but the sk_buff shell. */
738static void skb_release_all(struct sk_buff *skb)
739{
740 skb_release_head_state(skb);
Florian Westphala28b1b92017-07-23 19:54:47 +0200741 if (likely(skb->head))
742 skb_release_data(skb);
Herbert Xu2d4baff2007-11-26 23:11:19 +0800743}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700744
Herbert Xu2d4baff2007-11-26 23:11:19 +0800745/**
746 * __kfree_skb - private function
747 * @skb: buffer
748 *
749 * Free an sk_buff. Release anything attached to the buffer.
750 * Clean the state. This is an internal helper function. Users should
751 * always call kfree_skb
752 */
753
754void __kfree_skb(struct sk_buff *skb)
755{
756 skb_release_all(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700757 kfree_skbmem(skb);
758}
David S. Millerb4ac530fc2009-02-10 02:09:24 -0800759EXPORT_SYMBOL(__kfree_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700760
761/**
Menglong Dongc504e5c2022-01-09 14:36:26 +0800762 * kfree_skb_reason - free an sk_buff with special reason
Jörn Engel231d06a2006-03-20 21:28:35 -0800763 * @skb: buffer to free
Menglong Dongc504e5c2022-01-09 14:36:26 +0800764 * @reason: reason why this skb is dropped
Jörn Engel231d06a2006-03-20 21:28:35 -0800765 *
766 * Drop a reference to the buffer and free it if the usage count has
Menglong Dongc504e5c2022-01-09 14:36:26 +0800767 * hit zero. Meanwhile, pass the drop reason to 'kfree_skb'
768 * tracepoint.
Jörn Engel231d06a2006-03-20 21:28:35 -0800769 */
Menglong Dongc504e5c2022-01-09 14:36:26 +0800770void kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason)
Jörn Engel231d06a2006-03-20 21:28:35 -0800771{
Paolo Abeni3889a8032017-06-12 11:23:41 +0200772 if (!skb_unref(skb))
Jörn Engel231d06a2006-03-20 21:28:35 -0800773 return;
Paolo Abeni3889a8032017-06-12 11:23:41 +0200774
Menglong Dongc504e5c2022-01-09 14:36:26 +0800775 trace_kfree_skb(skb, __builtin_return_address(0), reason);
Jörn Engel231d06a2006-03-20 21:28:35 -0800776 __kfree_skb(skb);
777}
Menglong Dongc504e5c2022-01-09 14:36:26 +0800778EXPORT_SYMBOL(kfree_skb_reason);
Jörn Engel231d06a2006-03-20 21:28:35 -0800779
Eric Dumazetbd8a7032013-06-24 06:26:00 -0700780void kfree_skb_list(struct sk_buff *segs)
781{
782 while (segs) {
783 struct sk_buff *next = segs->next;
784
785 kfree_skb(segs);
786 segs = next;
787 }
788}
789EXPORT_SYMBOL(kfree_skb_list);
790
Willem de Bruijn64131392019-07-07 05:51:55 -0400791/* Dump skb information and contents.
792 *
793 * Must only be called from net_ratelimit()-ed paths.
794 *
Vladimir Oltean302af7c2020-10-05 17:48:38 +0300795 * Dumps whole packets if full_pkt, only headers otherwise.
Willem de Bruijn64131392019-07-07 05:51:55 -0400796 */
797void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt)
798{
Willem de Bruijn64131392019-07-07 05:51:55 -0400799 struct skb_shared_info *sh = skb_shinfo(skb);
800 struct net_device *dev = skb->dev;
801 struct sock *sk = skb->sk;
802 struct sk_buff *list_skb;
803 bool has_mac, has_trans;
804 int headroom, tailroom;
805 int i, len, seg_len;
806
807 if (full_pkt)
Willem de Bruijn64131392019-07-07 05:51:55 -0400808 len = skb->len;
809 else
810 len = min_t(int, skb->len, MAX_HEADER + 128);
811
812 headroom = skb_headroom(skb);
813 tailroom = skb_tailroom(skb);
814
815 has_mac = skb_mac_header_was_set(skb);
816 has_trans = skb_transport_header_was_set(skb);
817
818 printk("%sskb len=%u headroom=%u headlen=%u tailroom=%u\n"
819 "mac=(%d,%d) net=(%d,%d) trans=%d\n"
820 "shinfo(txflags=%u nr_frags=%u gso(size=%hu type=%u segs=%hu))\n"
821 "csum(0x%x ip_summed=%u complete_sw=%u valid=%u level=%u)\n"
822 "hash(0x%x sw=%u l4=%u) proto=0x%04x pkttype=%u iif=%d\n",
823 level, skb->len, headroom, skb_headlen(skb), tailroom,
824 has_mac ? skb->mac_header : -1,
825 has_mac ? skb_mac_header_len(skb) : -1,
826 skb->network_header,
827 has_trans ? skb_network_header_len(skb) : -1,
828 has_trans ? skb->transport_header : -1,
829 sh->tx_flags, sh->nr_frags,
830 sh->gso_size, sh->gso_type, sh->gso_segs,
831 skb->csum, skb->ip_summed, skb->csum_complete_sw,
832 skb->csum_valid, skb->csum_level,
833 skb->hash, skb->sw_hash, skb->l4_hash,
834 ntohs(skb->protocol), skb->pkt_type, skb->skb_iif);
835
836 if (dev)
Gal Pressman8a03ef62021-12-16 11:28:25 +0200837 printk("%sdev name=%s feat=%pNF\n",
Willem de Bruijn64131392019-07-07 05:51:55 -0400838 level, dev->name, &dev->features);
839 if (sk)
Qian Caidb8051f2019-07-16 11:43:05 -0400840 printk("%ssk family=%hu type=%u proto=%u\n",
Willem de Bruijn64131392019-07-07 05:51:55 -0400841 level, sk->sk_family, sk->sk_type, sk->sk_protocol);
842
843 if (full_pkt && headroom)
844 print_hex_dump(level, "skb headroom: ", DUMP_PREFIX_OFFSET,
845 16, 1, skb->head, headroom, false);
846
847 seg_len = min_t(int, skb_headlen(skb), len);
848 if (seg_len)
849 print_hex_dump(level, "skb linear: ", DUMP_PREFIX_OFFSET,
850 16, 1, skb->data, seg_len, false);
851 len -= seg_len;
852
853 if (full_pkt && tailroom)
854 print_hex_dump(level, "skb tailroom: ", DUMP_PREFIX_OFFSET,
855 16, 1, skb_tail_pointer(skb), tailroom, false);
856
857 for (i = 0; len && i < skb_shinfo(skb)->nr_frags; i++) {
858 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
859 u32 p_off, p_len, copied;
860 struct page *p;
861 u8 *vaddr;
862
Jonathan Lemonb54c9d52019-07-30 07:40:33 -0700863 skb_frag_foreach_page(frag, skb_frag_off(frag),
Willem de Bruijn64131392019-07-07 05:51:55 -0400864 skb_frag_size(frag), p, p_off, p_len,
865 copied) {
866 seg_len = min_t(int, p_len, len);
867 vaddr = kmap_atomic(p);
868 print_hex_dump(level, "skb frag: ",
869 DUMP_PREFIX_OFFSET,
870 16, 1, vaddr + p_off, seg_len, false);
871 kunmap_atomic(vaddr);
872 len -= seg_len;
873 if (!len)
874 break;
875 }
876 }
877
878 if (full_pkt && skb_has_frag_list(skb)) {
879 printk("skb fraglist:\n");
880 skb_walk_frags(skb, list_skb)
881 skb_dump(level, list_skb, true);
882 }
883}
884EXPORT_SYMBOL(skb_dump);
885
Stephen Hemmingerd1a203e2008-11-01 21:01:09 -0700886/**
Michael S. Tsirkin25121172012-11-01 09:16:28 +0000887 * skb_tx_error - report an sk_buff xmit error
888 * @skb: buffer that triggered an error
889 *
890 * Report xmit error if a device callback is tracking this skb.
891 * skb must be freed afterwards.
892 */
893void skb_tx_error(struct sk_buff *skb)
894{
Willem de Bruijn1f8b9772017-08-03 16:29:41 -0400895 skb_zcopy_clear(skb, true);
Michael S. Tsirkin25121172012-11-01 09:16:28 +0000896}
897EXPORT_SYMBOL(skb_tx_error);
898
Herbert Xube769db2020-08-22 08:23:29 +1000899#ifdef CONFIG_TRACEPOINTS
Michael S. Tsirkin25121172012-11-01 09:16:28 +0000900/**
Neil Hormanead2ceb2009-03-11 09:49:55 +0000901 * consume_skb - free an skbuff
902 * @skb: buffer to free
903 *
904 * Drop a ref to the buffer and free it if the usage count has hit zero
905 * Functions identically to kfree_skb, but kfree_skb assumes that the frame
906 * is being dropped after a failure and notes that
907 */
908void consume_skb(struct sk_buff *skb)
909{
Paolo Abeni3889a8032017-06-12 11:23:41 +0200910 if (!skb_unref(skb))
Neil Hormanead2ceb2009-03-11 09:49:55 +0000911 return;
Paolo Abeni3889a8032017-06-12 11:23:41 +0200912
Koki Sanagi07dc22e2010-08-23 18:46:12 +0900913 trace_consume_skb(skb);
Neil Hormanead2ceb2009-03-11 09:49:55 +0000914 __kfree_skb(skb);
915}
916EXPORT_SYMBOL(consume_skb);
Herbert Xube769db2020-08-22 08:23:29 +1000917#endif
Neil Hormanead2ceb2009-03-11 09:49:55 +0000918
Paolo Abeni0a463c72017-06-12 11:23:42 +0200919/**
Mauro Carvalho Chehabc1639be2020-11-16 11:17:58 +0100920 * __consume_stateless_skb - free an skbuff, assuming it is stateless
Paolo Abeni0a463c72017-06-12 11:23:42 +0200921 * @skb: buffer to free
922 *
Paolo Abenica2c1412017-09-06 14:44:36 +0200923 * Alike consume_skb(), but this variant assumes that this is the last
924 * skb reference and all the head states have been already dropped
Paolo Abeni0a463c72017-06-12 11:23:42 +0200925 */
Paolo Abenica2c1412017-09-06 14:44:36 +0200926void __consume_stateless_skb(struct sk_buff *skb)
Paolo Abeni0a463c72017-06-12 11:23:42 +0200927{
Paolo Abeni0a463c72017-06-12 11:23:42 +0200928 trace_consume_skb(skb);
Florian Westphal06dc75a2017-07-17 18:56:54 +0200929 skb_release_data(skb);
Paolo Abeni0a463c72017-06-12 11:23:42 +0200930 kfree_skbmem(skb);
931}
932
Alexander Lobakinf450d532021-02-13 14:12:25 +0000933static void napi_skb_cache_put(struct sk_buff *skb)
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100934{
935 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
Alexander Lobakinf450d532021-02-13 14:12:25 +0000936 u32 i;
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100937
Alexander Lobakinf450d532021-02-13 14:12:25 +0000938 kasan_poison_object_data(skbuff_head_cache, skb);
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100939 nc->skb_cache[nc->skb_count++] = skb;
940
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100941 if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) {
Alexander Lobakinf450d532021-02-13 14:12:25 +0000942 for (i = NAPI_SKB_CACHE_HALF; i < NAPI_SKB_CACHE_SIZE; i++)
943 kasan_unpoison_object_data(skbuff_head_cache,
944 nc->skb_cache[i]);
945
946 kmem_cache_free_bulk(skbuff_head_cache, NAPI_SKB_CACHE_HALF,
947 nc->skb_cache + NAPI_SKB_CACHE_HALF);
948 nc->skb_count = NAPI_SKB_CACHE_HALF;
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100949 }
950}
Alexander Lobakinf450d532021-02-13 14:12:25 +0000951
Jesper Dangaard Brouer15fad712016-02-08 13:15:04 +0100952void __kfree_skb_defer(struct sk_buff *skb)
953{
Alexander Lobakin9243adf2021-02-13 14:13:09 +0000954 skb_release_all(skb);
955 napi_skb_cache_put(skb);
956}
957
958void napi_skb_free_stolen_head(struct sk_buff *skb)
959{
Paolo Abeni9efb4b52021-07-28 18:24:02 +0200960 if (unlikely(skb->slow_gro)) {
961 nf_reset_ct(skb);
962 skb_dst_drop(skb);
963 skb_ext_put(skb);
Paolo Abeni5e10da52021-07-28 18:24:03 +0200964 skb_orphan(skb);
Paolo Abeni9efb4b52021-07-28 18:24:02 +0200965 skb->slow_gro = 0;
966 }
Alexander Lobakinf450d532021-02-13 14:12:25 +0000967 napi_skb_cache_put(skb);
Jesper Dangaard Brouer15fad712016-02-08 13:15:04 +0100968}
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100969
970void napi_consume_skb(struct sk_buff *skb, int budget)
971{
Jesper Dangaard Brouer885eb0a2016-03-11 09:43:58 +0100972 /* Zero budget indicate non-NAPI context called us, like netpoll */
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100973 if (unlikely(!budget)) {
Jesper Dangaard Brouer885eb0a2016-03-11 09:43:58 +0100974 dev_consume_skb_any(skb);
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100975 return;
976 }
977
Yunsheng Lin6454eca2020-11-24 18:49:29 +0800978 lockdep_assert_in_softirq();
979
Paolo Abeni76088942017-06-14 11:48:48 +0200980 if (!skb_unref(skb))
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100981 return;
Paolo Abeni76088942017-06-14 11:48:48 +0200982
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100983 /* if reaching here SKB is ready to free */
984 trace_consume_skb(skb);
985
986 /* if SKB is a clone, don't handle this case */
Eric Dumazetabbdb5a2016-03-20 11:27:47 -0700987 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) {
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100988 __kfree_skb(skb);
989 return;
990 }
991
Alexander Lobakin9243adf2021-02-13 14:13:09 +0000992 skb_release_all(skb);
Alexander Lobakinf450d532021-02-13 14:12:25 +0000993 napi_skb_cache_put(skb);
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100994}
995EXPORT_SYMBOL(napi_consume_skb);
996
Kees Cook03f61042021-11-20 16:31:49 -0800997/* Make sure a field is contained by headers group */
Eric Dumazetb1937222014-09-28 22:18:47 -0700998#define CHECK_SKB_FIELD(field) \
Kees Cook03f61042021-11-20 16:31:49 -0800999 BUILD_BUG_ON(offsetof(struct sk_buff, field) != \
1000 offsetof(struct sk_buff, headers.field)); \
Eric Dumazetb1937222014-09-28 22:18:47 -07001001
Herbert Xudec18812007-10-14 00:37:30 -07001002static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
1003{
1004 new->tstamp = old->tstamp;
Eric Dumazetb1937222014-09-28 22:18:47 -07001005 /* We do not copy old->sk */
Herbert Xudec18812007-10-14 00:37:30 -07001006 new->dev = old->dev;
Eric Dumazetb1937222014-09-28 22:18:47 -07001007 memcpy(new->cb, old->cb, sizeof(old->cb));
Eric Dumazet7fee2262010-05-11 23:19:48 +00001008 skb_dst_copy(new, old);
Florian Westphaldf5042f2018-12-18 17:15:16 +01001009 __skb_ext_copy(new, old);
Eric Dumazetb1937222014-09-28 22:18:47 -07001010 __nf_copy(new, old, false);
Patrick McHardy6aa895b2008-07-14 22:49:06 -07001011
Kees Cook03f61042021-11-20 16:31:49 -08001012 /* Note : this field could be in the headers group.
Eric Dumazetb1937222014-09-28 22:18:47 -07001013 * It is not yet because we do not want to have a 16 bit hole
1014 */
1015 new->queue_mapping = old->queue_mapping;
Eliezer Tamir06021292013-06-10 11:39:50 +03001016
Kees Cook03f61042021-11-20 16:31:49 -08001017 memcpy(&new->headers, &old->headers, sizeof(new->headers));
Eric Dumazetb1937222014-09-28 22:18:47 -07001018 CHECK_SKB_FIELD(protocol);
1019 CHECK_SKB_FIELD(csum);
1020 CHECK_SKB_FIELD(hash);
1021 CHECK_SKB_FIELD(priority);
1022 CHECK_SKB_FIELD(skb_iif);
1023 CHECK_SKB_FIELD(vlan_proto);
1024 CHECK_SKB_FIELD(vlan_tci);
1025 CHECK_SKB_FIELD(transport_header);
1026 CHECK_SKB_FIELD(network_header);
1027 CHECK_SKB_FIELD(mac_header);
1028 CHECK_SKB_FIELD(inner_protocol);
1029 CHECK_SKB_FIELD(inner_transport_header);
1030 CHECK_SKB_FIELD(inner_network_header);
1031 CHECK_SKB_FIELD(inner_mac_header);
1032 CHECK_SKB_FIELD(mark);
1033#ifdef CONFIG_NETWORK_SECMARK
1034 CHECK_SKB_FIELD(secmark);
1035#endif
Cong Wange0d10952013-08-01 11:10:25 +08001036#ifdef CONFIG_NET_RX_BUSY_POLL
Eric Dumazetb1937222014-09-28 22:18:47 -07001037 CHECK_SKB_FIELD(napi_id);
Eliezer Tamir06021292013-06-10 11:39:50 +03001038#endif
Eric Dumazet2bd82482015-02-03 23:48:24 -08001039#ifdef CONFIG_XPS
1040 CHECK_SKB_FIELD(sender_cpu);
1041#endif
Eric Dumazetb1937222014-09-28 22:18:47 -07001042#ifdef CONFIG_NET_SCHED
1043 CHECK_SKB_FIELD(tc_index);
Eric Dumazetb1937222014-09-28 22:18:47 -07001044#endif
1045
Herbert Xudec18812007-10-14 00:37:30 -07001046}
1047
Herbert Xu82c49a32009-05-22 22:11:37 +00001048/*
1049 * You should not add any new code to this function. Add it to
1050 * __copy_skb_header above instead.
1051 */
Herbert Xue0053ec2007-10-14 00:37:52 -07001052static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001053{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001054#define C(x) n->x = skb->x
1055
1056 n->next = n->prev = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001057 n->sk = NULL;
Herbert Xudec18812007-10-14 00:37:30 -07001058 __copy_skb_header(n, skb);
1059
Linus Torvalds1da177e2005-04-16 15:20:36 -07001060 C(len);
1061 C(data_len);
Alexey Dobriyan3e6b3b22007-03-16 15:00:46 -07001062 C(mac_len);
Patrick McHardy334a8132007-06-25 04:35:20 -07001063 n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
Paul Moore02f1c892008-01-07 21:56:41 -08001064 n->cloned = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001065 n->nohdr = 0;
Eric Dumazetb13dda92018-04-07 13:42:39 -07001066 n->peeked = 0;
Stefano Brivioe78bfb02018-07-13 13:21:07 +02001067 C(pfmemalloc);
Ilias Apalodimas6a5bcd82021-06-07 21:02:38 +02001068 C(pp_recycle);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001069 n->destructor = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001070 C(tail);
1071 C(end);
Paul Moore02f1c892008-01-07 21:56:41 -08001072 C(head);
Eric Dumazetd3836f22012-04-27 00:33:38 +00001073 C(head_frag);
Paul Moore02f1c892008-01-07 21:56:41 -08001074 C(data);
1075 C(truesize);
Reshetova, Elena63354792017-06-30 13:07:58 +03001076 refcount_set(&n->users, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001077
1078 atomic_inc(&(skb_shinfo(skb)->dataref));
1079 skb->cloned = 1;
1080
1081 return n;
Herbert Xue0053ec2007-10-14 00:37:52 -07001082#undef C
1083}
1084
1085/**
Jakub Kicinskida29e4b2019-06-03 15:16:58 -07001086 * alloc_skb_for_msg() - allocate sk_buff to wrap frag list forming a msg
1087 * @first: first sk_buff of the msg
1088 */
1089struct sk_buff *alloc_skb_for_msg(struct sk_buff *first)
1090{
1091 struct sk_buff *n;
1092
1093 n = alloc_skb(0, GFP_ATOMIC);
1094 if (!n)
1095 return NULL;
1096
1097 n->len = first->len;
1098 n->data_len = first->len;
1099 n->truesize = first->truesize;
1100
1101 skb_shinfo(n)->frag_list = first;
1102
1103 __copy_skb_header(n, first);
1104 n->destructor = NULL;
1105
1106 return n;
1107}
1108EXPORT_SYMBOL_GPL(alloc_skb_for_msg);
1109
1110/**
Herbert Xue0053ec2007-10-14 00:37:52 -07001111 * skb_morph - morph one skb into another
1112 * @dst: the skb to receive the contents
1113 * @src: the skb to supply the contents
1114 *
1115 * This is identical to skb_clone except that the target skb is
1116 * supplied by the user.
1117 *
1118 * The target skb is returned upon exit.
1119 */
1120struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
1121{
Herbert Xu2d4baff2007-11-26 23:11:19 +08001122 skb_release_all(dst);
Herbert Xue0053ec2007-10-14 00:37:52 -07001123 return __skb_clone(dst, src);
1124}
1125EXPORT_SYMBOL_GPL(skb_morph);
1126
Sowmini Varadhan6f89dbc2018-02-15 10:49:32 -08001127int mm_account_pinned_pages(struct mmpin *mmp, size_t size)
Willem de Bruijna91dbff2017-08-03 16:29:43 -04001128{
1129 unsigned long max_pg, num_pg, new_pg, old_pg;
1130 struct user_struct *user;
1131
1132 if (capable(CAP_IPC_LOCK) || !size)
1133 return 0;
1134
1135 num_pg = (size >> PAGE_SHIFT) + 2; /* worst case */
1136 max_pg = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
1137 user = mmp->user ? : current_user();
1138
1139 do {
1140 old_pg = atomic_long_read(&user->locked_vm);
1141 new_pg = old_pg + num_pg;
1142 if (new_pg > max_pg)
1143 return -ENOBUFS;
1144 } while (atomic_long_cmpxchg(&user->locked_vm, old_pg, new_pg) !=
1145 old_pg);
1146
1147 if (!mmp->user) {
1148 mmp->user = get_uid(user);
1149 mmp->num_pg = num_pg;
1150 } else {
1151 mmp->num_pg += num_pg;
1152 }
1153
1154 return 0;
1155}
Sowmini Varadhan6f89dbc2018-02-15 10:49:32 -08001156EXPORT_SYMBOL_GPL(mm_account_pinned_pages);
Willem de Bruijna91dbff2017-08-03 16:29:43 -04001157
Sowmini Varadhan6f89dbc2018-02-15 10:49:32 -08001158void mm_unaccount_pinned_pages(struct mmpin *mmp)
Willem de Bruijna91dbff2017-08-03 16:29:43 -04001159{
1160 if (mmp->user) {
1161 atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm);
1162 free_uid(mmp->user);
1163 }
1164}
Sowmini Varadhan6f89dbc2018-02-15 10:49:32 -08001165EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages);
Willem de Bruijna91dbff2017-08-03 16:29:43 -04001166
Jonathan Lemon8c793822021-01-06 14:18:37 -08001167struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size)
Willem de Bruijn52267792017-08-03 16:29:39 -04001168{
1169 struct ubuf_info *uarg;
1170 struct sk_buff *skb;
1171
1172 WARN_ON_ONCE(!in_task());
1173
1174 skb = sock_omalloc(sk, 0, GFP_KERNEL);
1175 if (!skb)
1176 return NULL;
1177
1178 BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb));
1179 uarg = (void *)skb->cb;
Willem de Bruijna91dbff2017-08-03 16:29:43 -04001180 uarg->mmp.user = NULL;
1181
1182 if (mm_account_pinned_pages(&uarg->mmp, size)) {
1183 kfree_skb(skb);
1184 return NULL;
1185 }
Willem de Bruijn52267792017-08-03 16:29:39 -04001186
Jonathan Lemon8c793822021-01-06 14:18:37 -08001187 uarg->callback = msg_zerocopy_callback;
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001188 uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;
1189 uarg->len = 1;
1190 uarg->bytelen = size;
Willem de Bruijn52267792017-08-03 16:29:39 -04001191 uarg->zerocopy = 1;
Jonathan Lemon04c2d332021-01-06 14:18:39 -08001192 uarg->flags = SKBFL_ZEROCOPY_FRAG;
Eric Dumazetc1d1b432017-08-31 16:48:22 -07001193 refcount_set(&uarg->refcnt, 1);
Willem de Bruijn52267792017-08-03 16:29:39 -04001194 sock_hold(sk);
1195
1196 return uarg;
1197}
Jonathan Lemon8c793822021-01-06 14:18:37 -08001198EXPORT_SYMBOL_GPL(msg_zerocopy_alloc);
Willem de Bruijn52267792017-08-03 16:29:39 -04001199
1200static inline struct sk_buff *skb_from_uarg(struct ubuf_info *uarg)
1201{
1202 return container_of((void *)uarg, struct sk_buff, cb);
1203}
1204
Jonathan Lemon8c793822021-01-06 14:18:37 -08001205struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
1206 struct ubuf_info *uarg)
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001207{
1208 if (uarg) {
1209 const u32 byte_limit = 1 << 19; /* limit to a few TSO */
1210 u32 bytelen, next;
1211
1212 /* realloc only when socket is locked (TCP, UDP cork),
1213 * so uarg->len and sk_zckey access is serialized
1214 */
1215 if (!sock_owned_by_user(sk)) {
1216 WARN_ON_ONCE(1);
1217 return NULL;
1218 }
1219
1220 bytelen = uarg->bytelen + size;
1221 if (uarg->len == USHRT_MAX - 1 || bytelen > byte_limit) {
1222 /* TCP can create new skb to attach new uarg */
1223 if (sk->sk_type == SOCK_STREAM)
1224 goto new_alloc;
1225 return NULL;
1226 }
1227
1228 next = (u32)atomic_read(&sk->sk_zckey);
1229 if ((u32)(uarg->id + uarg->len) == next) {
Willem de Bruijna91dbff2017-08-03 16:29:43 -04001230 if (mm_account_pinned_pages(&uarg->mmp, size))
1231 return NULL;
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001232 uarg->len++;
1233 uarg->bytelen = bytelen;
1234 atomic_set(&sk->sk_zckey, ++next);
Willem de Bruijn100f6d82019-05-30 18:01:21 -04001235
1236 /* no extra ref when appending to datagram (MSG_MORE) */
1237 if (sk->sk_type == SOCK_STREAM)
Jonathan Lemon8e044912021-01-06 14:18:41 -08001238 net_zcopy_get(uarg);
Willem de Bruijn100f6d82019-05-30 18:01:21 -04001239
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001240 return uarg;
1241 }
1242 }
1243
1244new_alloc:
Jonathan Lemon8c793822021-01-06 14:18:37 -08001245 return msg_zerocopy_alloc(sk, size);
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001246}
Jonathan Lemon8c793822021-01-06 14:18:37 -08001247EXPORT_SYMBOL_GPL(msg_zerocopy_realloc);
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001248
1249static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len)
1250{
1251 struct sock_exterr_skb *serr = SKB_EXT_ERR(skb);
1252 u32 old_lo, old_hi;
1253 u64 sum_len;
1254
1255 old_lo = serr->ee.ee_info;
1256 old_hi = serr->ee.ee_data;
1257 sum_len = old_hi - old_lo + 1ULL + len;
1258
1259 if (sum_len >= (1ULL << 32))
1260 return false;
1261
1262 if (lo != old_hi + 1)
1263 return false;
1264
1265 serr->ee.ee_data += len;
1266 return true;
1267}
1268
Jonathan Lemon8c793822021-01-06 14:18:37 -08001269static void __msg_zerocopy_callback(struct ubuf_info *uarg)
Willem de Bruijn52267792017-08-03 16:29:39 -04001270{
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001271 struct sk_buff *tail, *skb = skb_from_uarg(uarg);
Willem de Bruijn52267792017-08-03 16:29:39 -04001272 struct sock_exterr_skb *serr;
1273 struct sock *sk = skb->sk;
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001274 struct sk_buff_head *q;
1275 unsigned long flags;
Willem de Bruijn3bdd5ee2021-06-09 18:41:57 -04001276 bool is_zerocopy;
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001277 u32 lo, hi;
1278 u16 len;
Willem de Bruijn52267792017-08-03 16:29:39 -04001279
Willem de Bruijnccaffff2017-08-09 19:09:43 -04001280 mm_unaccount_pinned_pages(&uarg->mmp);
1281
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001282 /* if !len, there was only 1 call, and it was aborted
1283 * so do not queue a completion notification
1284 */
1285 if (!uarg->len || sock_flag(sk, SOCK_DEAD))
Willem de Bruijn52267792017-08-03 16:29:39 -04001286 goto release;
1287
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001288 len = uarg->len;
1289 lo = uarg->id;
1290 hi = uarg->id + len - 1;
Willem de Bruijn3bdd5ee2021-06-09 18:41:57 -04001291 is_zerocopy = uarg->zerocopy;
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001292
Willem de Bruijn52267792017-08-03 16:29:39 -04001293 serr = SKB_EXT_ERR(skb);
1294 memset(serr, 0, sizeof(*serr));
1295 serr->ee.ee_errno = 0;
1296 serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY;
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001297 serr->ee.ee_data = hi;
1298 serr->ee.ee_info = lo;
Willem de Bruijn3bdd5ee2021-06-09 18:41:57 -04001299 if (!is_zerocopy)
Willem de Bruijn52267792017-08-03 16:29:39 -04001300 serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED;
1301
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001302 q = &sk->sk_error_queue;
1303 spin_lock_irqsave(&q->lock, flags);
1304 tail = skb_peek_tail(q);
1305 if (!tail || SKB_EXT_ERR(tail)->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY ||
1306 !skb_zerocopy_notify_extend(tail, lo, len)) {
1307 __skb_queue_tail(q, skb);
1308 skb = NULL;
1309 }
1310 spin_unlock_irqrestore(&q->lock, flags);
Willem de Bruijn52267792017-08-03 16:29:39 -04001311
Alexander Aringe3ae2362021-06-27 18:48:21 -04001312 sk_error_report(sk);
Willem de Bruijn52267792017-08-03 16:29:39 -04001313
1314release:
1315 consume_skb(skb);
1316 sock_put(sk);
1317}
Jonathan Lemon75518852021-01-06 14:18:31 -08001318
Jonathan Lemon8c793822021-01-06 14:18:37 -08001319void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg,
1320 bool success)
Jonathan Lemon75518852021-01-06 14:18:31 -08001321{
1322 uarg->zerocopy = uarg->zerocopy & success;
1323
1324 if (refcount_dec_and_test(&uarg->refcnt))
Jonathan Lemon8c793822021-01-06 14:18:37 -08001325 __msg_zerocopy_callback(uarg);
Jonathan Lemon75518852021-01-06 14:18:31 -08001326}
Jonathan Lemon8c793822021-01-06 14:18:37 -08001327EXPORT_SYMBOL_GPL(msg_zerocopy_callback);
Willem de Bruijn52267792017-08-03 16:29:39 -04001328
Jonathan Lemon8c793822021-01-06 14:18:37 -08001329void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
Willem de Bruijn52267792017-08-03 16:29:39 -04001330{
Jonathan Lemon236a6b12021-01-06 14:18:35 -08001331 struct sock *sk = skb_from_uarg(uarg)->sk;
Willem de Bruijn52267792017-08-03 16:29:39 -04001332
Jonathan Lemon236a6b12021-01-06 14:18:35 -08001333 atomic_dec(&sk->sk_zckey);
1334 uarg->len--;
Willem de Bruijn52267792017-08-03 16:29:39 -04001335
Jonathan Lemon236a6b12021-01-06 14:18:35 -08001336 if (have_uref)
Jonathan Lemon8c793822021-01-06 14:18:37 -08001337 msg_zerocopy_callback(NULL, uarg, true);
Willem de Bruijn52267792017-08-03 16:29:39 -04001338}
Jonathan Lemon8c793822021-01-06 14:18:37 -08001339EXPORT_SYMBOL_GPL(msg_zerocopy_put_abort);
Willem de Bruijn52267792017-08-03 16:29:39 -04001340
Willem de Bruijnb5947e52018-11-30 15:32:39 -05001341int skb_zerocopy_iter_dgram(struct sk_buff *skb, struct msghdr *msg, int len)
1342{
1343 return __zerocopy_sg_from_iter(skb->sk, skb, &msg->msg_iter, len);
1344}
1345EXPORT_SYMBOL_GPL(skb_zerocopy_iter_dgram);
1346
Willem de Bruijn52267792017-08-03 16:29:39 -04001347int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
1348 struct msghdr *msg, int len,
1349 struct ubuf_info *uarg)
1350{
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001351 struct ubuf_info *orig_uarg = skb_zcopy(skb);
Willem de Bruijn52267792017-08-03 16:29:39 -04001352 struct iov_iter orig_iter = msg->msg_iter;
1353 int err, orig_len = skb->len;
1354
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001355 /* An skb can only point to one uarg. This edge case happens when
1356 * TCP appends to an skb, but zerocopy_realloc triggered a new alloc.
1357 */
1358 if (orig_uarg && uarg != orig_uarg)
1359 return -EEXIST;
1360
Willem de Bruijn52267792017-08-03 16:29:39 -04001361 err = __zerocopy_sg_from_iter(sk, skb, &msg->msg_iter, len);
1362 if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) {
Willem de Bruijn54d431172017-10-19 12:40:39 -04001363 struct sock *save_sk = skb->sk;
1364
Willem de Bruijn52267792017-08-03 16:29:39 -04001365 /* Streams do not free skb on error. Reset to prev state. */
1366 msg->msg_iter = orig_iter;
Willem de Bruijn54d431172017-10-19 12:40:39 -04001367 skb->sk = sk;
Willem de Bruijn52267792017-08-03 16:29:39 -04001368 ___pskb_trim(skb, orig_len);
Willem de Bruijn54d431172017-10-19 12:40:39 -04001369 skb->sk = save_sk;
Willem de Bruijn52267792017-08-03 16:29:39 -04001370 return err;
1371 }
1372
Willem de Bruijn52900d22018-11-30 15:32:40 -05001373 skb_zcopy_set(skb, uarg, NULL);
Willem de Bruijn52267792017-08-03 16:29:39 -04001374 return skb->len - orig_len;
1375}
1376EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream);
1377
Willem de Bruijn1f8b9772017-08-03 16:29:41 -04001378static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig,
Willem de Bruijn52267792017-08-03 16:29:39 -04001379 gfp_t gfp_mask)
1380{
1381 if (skb_zcopy(orig)) {
1382 if (skb_zcopy(nskb)) {
1383 /* !gfp_mask callers are verified to !skb_zcopy(nskb) */
1384 if (!gfp_mask) {
1385 WARN_ON_ONCE(1);
1386 return -ENOMEM;
1387 }
1388 if (skb_uarg(nskb) == skb_uarg(orig))
1389 return 0;
1390 if (skb_copy_ubufs(nskb, GFP_ATOMIC))
1391 return -EIO;
1392 }
Willem de Bruijn52900d22018-11-30 15:32:40 -05001393 skb_zcopy_set(nskb, skb_uarg(orig), NULL);
Willem de Bruijn52267792017-08-03 16:29:39 -04001394 }
1395 return 0;
1396}
1397
Ben Hutchings2c530402012-07-10 10:55:09 +00001398/**
1399 * skb_copy_ubufs - copy userspace skb frags buffers to kernel
Michael S. Tsirkin48c83012011-08-31 08:03:29 +00001400 * @skb: the skb to modify
1401 * @gfp_mask: allocation priority
1402 *
Jonathan Lemon06b4feb2021-01-06 14:18:38 -08001403 * This must be called on skb with SKBFL_ZEROCOPY_ENABLE.
Michael S. Tsirkin48c83012011-08-31 08:03:29 +00001404 * It will copy all frags into kernel and drop the reference
1405 * to userspace pages.
1406 *
1407 * If this function is called from an interrupt gfp_mask() must be
1408 * %GFP_ATOMIC.
1409 *
1410 * Returns 0 on success or a negative error code on failure
1411 * to allocate kernel memory to copy to.
1412 */
1413int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
Shirley Maa6686f22011-07-06 12:22:12 +00001414{
Shirley Maa6686f22011-07-06 12:22:12 +00001415 int num_frags = skb_shinfo(skb)->nr_frags;
1416 struct page *page, *head = NULL;
Willem de Bruijn3ece7822017-08-03 16:29:38 -04001417 int i, new_frags;
1418 u32 d_off;
Shirley Maa6686f22011-07-06 12:22:12 +00001419
Willem de Bruijn3ece7822017-08-03 16:29:38 -04001420 if (skb_shared(skb) || skb_unclone(skb, gfp_mask))
1421 return -EINVAL;
1422
Willem de Bruijnf72c4ac2017-12-28 12:38:13 -05001423 if (!num_frags)
1424 goto release;
1425
Willem de Bruijn3ece7822017-08-03 16:29:38 -04001426 new_frags = (__skb_pagelen(skb) + PAGE_SIZE - 1) >> PAGE_SHIFT;
1427 for (i = 0; i < new_frags; i++) {
Krishna Kumar02756ed2012-07-17 02:05:29 +00001428 page = alloc_page(gfp_mask);
Shirley Maa6686f22011-07-06 12:22:12 +00001429 if (!page) {
1430 while (head) {
Sunghan Suh40dadff2013-07-12 16:17:23 +09001431 struct page *next = (struct page *)page_private(head);
Shirley Maa6686f22011-07-06 12:22:12 +00001432 put_page(head);
1433 head = next;
1434 }
1435 return -ENOMEM;
1436 }
Willem de Bruijn3ece7822017-08-03 16:29:38 -04001437 set_page_private(page, (unsigned long)head);
1438 head = page;
1439 }
1440
1441 page = head;
1442 d_off = 0;
1443 for (i = 0; i < num_frags; i++) {
1444 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
1445 u32 p_off, p_len, copied;
1446 struct page *p;
1447 u8 *vaddr;
Willem de Bruijnc613c202017-07-31 08:15:47 -04001448
Jonathan Lemonb54c9d52019-07-30 07:40:33 -07001449 skb_frag_foreach_page(f, skb_frag_off(f), skb_frag_size(f),
Willem de Bruijnc613c202017-07-31 08:15:47 -04001450 p, p_off, p_len, copied) {
Willem de Bruijn3ece7822017-08-03 16:29:38 -04001451 u32 copy, done = 0;
Willem de Bruijnc613c202017-07-31 08:15:47 -04001452 vaddr = kmap_atomic(p);
Willem de Bruijn3ece7822017-08-03 16:29:38 -04001453
1454 while (done < p_len) {
1455 if (d_off == PAGE_SIZE) {
1456 d_off = 0;
1457 page = (struct page *)page_private(page);
1458 }
1459 copy = min_t(u32, PAGE_SIZE - d_off, p_len - done);
1460 memcpy(page_address(page) + d_off,
1461 vaddr + p_off + done, copy);
1462 done += copy;
1463 d_off += copy;
1464 }
Willem de Bruijnc613c202017-07-31 08:15:47 -04001465 kunmap_atomic(vaddr);
1466 }
Shirley Maa6686f22011-07-06 12:22:12 +00001467 }
1468
1469 /* skb frags release userspace buffers */
Krishna Kumar02756ed2012-07-17 02:05:29 +00001470 for (i = 0; i < num_frags; i++)
Ian Campbella8605c62011-10-19 23:01:49 +00001471 skb_frag_unref(skb, i);
Shirley Maa6686f22011-07-06 12:22:12 +00001472
Shirley Maa6686f22011-07-06 12:22:12 +00001473 /* skb frags point to kernel buffers */
Willem de Bruijn3ece7822017-08-03 16:29:38 -04001474 for (i = 0; i < new_frags - 1; i++) {
1475 __skb_fill_page_desc(skb, i, head, 0, PAGE_SIZE);
Sunghan Suh40dadff2013-07-12 16:17:23 +09001476 head = (struct page *)page_private(head);
Shirley Maa6686f22011-07-06 12:22:12 +00001477 }
Willem de Bruijn3ece7822017-08-03 16:29:38 -04001478 __skb_fill_page_desc(skb, new_frags - 1, head, 0, d_off);
1479 skb_shinfo(skb)->nr_frags = new_frags;
Michael S. Tsirkin48c83012011-08-31 08:03:29 +00001480
Willem de Bruijnb90ddd52017-12-20 17:37:50 -05001481release:
Willem de Bruijn1f8b9772017-08-03 16:29:41 -04001482 skb_zcopy_clear(skb, false);
Shirley Maa6686f22011-07-06 12:22:12 +00001483 return 0;
1484}
Michael S. Tsirkindcc0fb72012-07-20 09:23:20 +00001485EXPORT_SYMBOL_GPL(skb_copy_ubufs);
Shirley Maa6686f22011-07-06 12:22:12 +00001486
Herbert Xue0053ec2007-10-14 00:37:52 -07001487/**
1488 * skb_clone - duplicate an sk_buff
1489 * @skb: buffer to clone
1490 * @gfp_mask: allocation priority
1491 *
1492 * Duplicate an &sk_buff. The new one is not owned by a socket. Both
1493 * copies share the same packet data but not structure. The new
1494 * buffer has a reference count of 1. If the allocation fails the
1495 * function returns %NULL otherwise the new buffer is returned.
1496 *
1497 * If this function is called from an interrupt gfp_mask() must be
1498 * %GFP_ATOMIC.
1499 */
1500
1501struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
1502{
Eric Dumazetd0bf4a92014-09-29 13:29:15 -07001503 struct sk_buff_fclones *fclones = container_of(skb,
1504 struct sk_buff_fclones,
1505 skb1);
Eric Dumazet6ffe75eb2014-12-03 17:04:39 -08001506 struct sk_buff *n;
Herbert Xue0053ec2007-10-14 00:37:52 -07001507
Michael S. Tsirkin70008aa2012-07-20 09:23:10 +00001508 if (skb_orphan_frags(skb, gfp_mask))
1509 return NULL;
Shirley Maa6686f22011-07-06 12:22:12 +00001510
Herbert Xue0053ec2007-10-14 00:37:52 -07001511 if (skb->fclone == SKB_FCLONE_ORIG &&
Reshetova, Elena26385952017-06-30 13:07:59 +03001512 refcount_read(&fclones->fclone_ref) == 1) {
Eric Dumazet6ffe75eb2014-12-03 17:04:39 -08001513 n = &fclones->skb2;
Reshetova, Elena26385952017-06-30 13:07:59 +03001514 refcount_set(&fclones->fclone_ref, 2);
Herbert Xue0053ec2007-10-14 00:37:52 -07001515 } else {
Mel Gormanc93bdd02012-07-31 16:44:19 -07001516 if (skb_pfmemalloc(skb))
1517 gfp_mask |= __GFP_MEMALLOC;
1518
Herbert Xue0053ec2007-10-14 00:37:52 -07001519 n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
1520 if (!n)
1521 return NULL;
Vegard Nossumfe55f6d2008-08-30 12:16:35 +02001522
Herbert Xue0053ec2007-10-14 00:37:52 -07001523 n->fclone = SKB_FCLONE_UNAVAILABLE;
1524 }
1525
1526 return __skb_clone(n, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001527}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001528EXPORT_SYMBOL(skb_clone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001529
Toshiaki Makitab0768a82018-08-03 16:58:09 +09001530void skb_headers_offset_update(struct sk_buff *skb, int off)
Pravin B Shelarf5b17292013-03-07 13:21:40 +00001531{
Eric Dumazet030737b2013-10-19 11:42:54 -07001532 /* Only adjust this if it actually is csum_start rather than csum */
1533 if (skb->ip_summed == CHECKSUM_PARTIAL)
1534 skb->csum_start += off;
Pravin B Shelarf5b17292013-03-07 13:21:40 +00001535 /* {transport,network,mac}_header and tail are relative to skb->head */
1536 skb->transport_header += off;
1537 skb->network_header += off;
1538 if (skb_mac_header_was_set(skb))
1539 skb->mac_header += off;
1540 skb->inner_transport_header += off;
1541 skb->inner_network_header += off;
Pravin B Shelaraefbd2b2013-03-07 13:21:46 +00001542 skb->inner_mac_header += off;
Pravin B Shelarf5b17292013-03-07 13:21:40 +00001543}
Toshiaki Makitab0768a82018-08-03 16:58:09 +09001544EXPORT_SYMBOL(skb_headers_offset_update);
Pravin B Shelarf5b17292013-03-07 13:21:40 +00001545
Ilya Lesokhin08303c12018-04-30 10:16:11 +03001546void skb_copy_header(struct sk_buff *new, const struct sk_buff *old)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001547{
Herbert Xudec18812007-10-14 00:37:30 -07001548 __copy_skb_header(new, old);
1549
Herbert Xu79671682006-06-22 02:40:14 -07001550 skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
1551 skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
1552 skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001553}
Ilya Lesokhin08303c12018-04-30 10:16:11 +03001554EXPORT_SYMBOL(skb_copy_header);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001555
Mel Gormanc93bdd02012-07-31 16:44:19 -07001556static inline int skb_alloc_rx_flag(const struct sk_buff *skb)
1557{
1558 if (skb_pfmemalloc(skb))
1559 return SKB_ALLOC_RX;
1560 return 0;
1561}
1562
Linus Torvalds1da177e2005-04-16 15:20:36 -07001563/**
1564 * skb_copy - create private copy of an sk_buff
1565 * @skb: buffer to copy
1566 * @gfp_mask: allocation priority
1567 *
1568 * Make a copy of both an &sk_buff and its data. This is used when the
1569 * caller wishes to modify the data and needs a private copy of the
1570 * data to alter. Returns %NULL on failure or the pointer to the buffer
1571 * on success. The returned buffer has a reference count of 1.
1572 *
1573 * As by-product this function converts non-linear &sk_buff to linear
1574 * one, so that &sk_buff becomes completely private and caller is allowed
1575 * to modify all the data of returned buffer. This means that this
1576 * function is not recommended for use in circumstances when only
1577 * header is going to be modified. Use pskb_copy() instead.
1578 */
1579
Al Virodd0fc662005-10-07 07:46:04 +01001580struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001581{
Eric Dumazet6602ceb2010-09-01 05:25:10 +00001582 int headerlen = skb_headroom(skb);
Alexander Duyckec47ea82012-05-04 14:26:56 +00001583 unsigned int size = skb_end_offset(skb) + skb->data_len;
Mel Gormanc93bdd02012-07-31 16:44:19 -07001584 struct sk_buff *n = __alloc_skb(size, gfp_mask,
1585 skb_alloc_rx_flag(skb), NUMA_NO_NODE);
Eric Dumazet6602ceb2010-09-01 05:25:10 +00001586
Linus Torvalds1da177e2005-04-16 15:20:36 -07001587 if (!n)
1588 return NULL;
1589
1590 /* Set the data pointer */
1591 skb_reserve(n, headerlen);
1592 /* Set the tail pointer and length */
1593 skb_put(n, skb->len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001594
Tim Hansen9f77fad2017-10-09 11:37:59 -04001595 BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001596
Ilya Lesokhin08303c12018-04-30 10:16:11 +03001597 skb_copy_header(n, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001598 return n;
1599}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001600EXPORT_SYMBOL(skb_copy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001601
1602/**
Octavian Purdilabad93e92014-06-12 01:36:26 +03001603 * __pskb_copy_fclone - create copy of an sk_buff with private head.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001604 * @skb: buffer to copy
Eric Dumazet117632e2011-12-03 21:39:53 +00001605 * @headroom: headroom of new skb
Linus Torvalds1da177e2005-04-16 15:20:36 -07001606 * @gfp_mask: allocation priority
Octavian Purdilabad93e92014-06-12 01:36:26 +03001607 * @fclone: if true allocate the copy of the skb from the fclone
1608 * cache instead of the head cache; it is recommended to set this
1609 * to true for the cases where the copy will likely be cloned
Linus Torvalds1da177e2005-04-16 15:20:36 -07001610 *
1611 * Make a copy of both an &sk_buff and part of its data, located
1612 * in header. Fragmented data remain shared. This is used when
1613 * the caller wishes to modify only header of &sk_buff and needs
1614 * private copy of the header to alter. Returns %NULL on failure
1615 * or the pointer to the buffer on success.
1616 * The returned buffer has a reference count of 1.
1617 */
1618
Octavian Purdilabad93e92014-06-12 01:36:26 +03001619struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
1620 gfp_t gfp_mask, bool fclone)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001621{
Eric Dumazet117632e2011-12-03 21:39:53 +00001622 unsigned int size = skb_headlen(skb) + headroom;
Octavian Purdilabad93e92014-06-12 01:36:26 +03001623 int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0);
1624 struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE);
Eric Dumazet6602ceb2010-09-01 05:25:10 +00001625
Linus Torvalds1da177e2005-04-16 15:20:36 -07001626 if (!n)
1627 goto out;
1628
1629 /* Set the data pointer */
Eric Dumazet117632e2011-12-03 21:39:53 +00001630 skb_reserve(n, headroom);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001631 /* Set the tail pointer and length */
1632 skb_put(n, skb_headlen(skb));
1633 /* Copy the bytes */
Arnaldo Carvalho de Melod626f622007-03-27 18:55:52 -03001634 skb_copy_from_linear_data(skb, n->data, n->len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001635
Herbert Xu25f484a2006-11-07 14:57:15 -08001636 n->truesize += skb->data_len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001637 n->data_len = skb->data_len;
1638 n->len = skb->len;
1639
1640 if (skb_shinfo(skb)->nr_frags) {
1641 int i;
1642
Willem de Bruijn1f8b9772017-08-03 16:29:41 -04001643 if (skb_orphan_frags(skb, gfp_mask) ||
1644 skb_zerocopy_clone(n, skb, gfp_mask)) {
Michael S. Tsirkin70008aa2012-07-20 09:23:10 +00001645 kfree_skb(n);
1646 n = NULL;
1647 goto out;
Shirley Maa6686f22011-07-06 12:22:12 +00001648 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001649 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1650 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
Ian Campbellea2ab692011-08-22 23:44:58 +00001651 skb_frag_ref(skb, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001652 }
1653 skb_shinfo(n)->nr_frags = i;
1654 }
1655
David S. Miller21dc3302010-08-23 00:13:46 -07001656 if (skb_has_frag_list(skb)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001657 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
1658 skb_clone_fraglist(n);
1659 }
1660
Ilya Lesokhin08303c12018-04-30 10:16:11 +03001661 skb_copy_header(n, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001662out:
1663 return n;
1664}
Octavian Purdilabad93e92014-06-12 01:36:26 +03001665EXPORT_SYMBOL(__pskb_copy_fclone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001666
1667/**
1668 * pskb_expand_head - reallocate header of &sk_buff
1669 * @skb: buffer to reallocate
1670 * @nhead: room to add at head
1671 * @ntail: room to add at tail
1672 * @gfp_mask: allocation priority
1673 *
Mathias Krausebc323832013-11-07 14:18:26 +01001674 * Expands (or creates identical copy, if @nhead and @ntail are zero)
1675 * header of @skb. &sk_buff itself is not changed. &sk_buff MUST have
Linus Torvalds1da177e2005-04-16 15:20:36 -07001676 * reference count of 1. Returns zero in the case of success or error,
1677 * if expansion failed. In the last case, &sk_buff is not changed.
1678 *
1679 * All the pointers pointing into skb header may change and must be
1680 * reloaded after call to this function.
1681 */
1682
Victor Fusco86a76ca2005-07-08 14:57:47 -07001683int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
Al Virodd0fc662005-10-07 07:46:04 +01001684 gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001685{
Eric Dumazet158f3232017-01-27 07:11:27 -08001686 int i, osize = skb_end_offset(skb);
1687 int size = osize + nhead + ntail;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001688 long off;
Eric Dumazet158f3232017-01-27 07:11:27 -08001689 u8 *data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001690
Herbert Xu4edd87a2008-10-01 07:09:38 -07001691 BUG_ON(nhead < 0);
1692
Tim Hansen9f77fad2017-10-09 11:37:59 -04001693 BUG_ON(skb_shared(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001694
1695 size = SKB_DATA_ALIGN(size);
1696
Mel Gormanc93bdd02012-07-31 16:44:19 -07001697 if (skb_pfmemalloc(skb))
1698 gfp_mask |= __GFP_MEMALLOC;
1699 data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
1700 gfp_mask, NUMA_NO_NODE, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001701 if (!data)
1702 goto nodata;
Eric Dumazet87151b82012-04-10 20:08:39 +00001703 size = SKB_WITH_OVERHEAD(ksize(data));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001704
1705 /* Copy only real data... and, alas, header. This should be
Eric Dumazet6602ceb2010-09-01 05:25:10 +00001706 * optimized for the cases when header is void.
1707 */
1708 memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head);
1709
1710 memcpy((struct skb_shared_info *)(data + size),
1711 skb_shinfo(skb),
Eric Dumazetfed66382010-07-22 19:09:08 +00001712 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags]));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001713
Alexander Duyck3e245912012-05-04 14:26:51 +00001714 /*
1715 * if shinfo is shared we must drop the old head gracefully, but if it
1716 * is not we can just drop the old head and let the existing refcount
1717 * be since all we did is relocate the values
1718 */
1719 if (skb_cloned(skb)) {
Michael S. Tsirkin70008aa2012-07-20 09:23:10 +00001720 if (skb_orphan_frags(skb, gfp_mask))
1721 goto nofrags;
Willem de Bruijn1f8b9772017-08-03 16:29:41 -04001722 if (skb_zcopy(skb))
Eric Dumazetc1d1b432017-08-31 16:48:22 -07001723 refcount_inc(&skb_uarg(skb)->refcnt);
Eric Dumazet1fd63042010-09-02 23:09:32 +00001724 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
Ian Campbellea2ab692011-08-22 23:44:58 +00001725 skb_frag_ref(skb, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001726
Eric Dumazet1fd63042010-09-02 23:09:32 +00001727 if (skb_has_frag_list(skb))
1728 skb_clone_fraglist(skb);
1729
1730 skb_release_data(skb);
Alexander Duyck3e245912012-05-04 14:26:51 +00001731 } else {
1732 skb_free_head(skb);
Eric Dumazet1fd63042010-09-02 23:09:32 +00001733 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001734 off = (data + nhead) - skb->head;
1735
1736 skb->head = data;
Eric Dumazetd3836f22012-04-27 00:33:38 +00001737 skb->head_frag = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001738 skb->data += off;
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -07001739#ifdef NET_SKBUFF_DATA_USES_OFFSET
1740 skb->end = size;
Patrick McHardy56eb8882007-04-09 11:45:04 -07001741 off = nhead;
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -07001742#else
1743 skb->end = skb->head + size;
Patrick McHardy56eb8882007-04-09 11:45:04 -07001744#endif
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07001745 skb->tail += off;
Peter Pan(潘卫平)b41abb42013-06-06 21:27:21 +08001746 skb_headers_offset_update(skb, nhead);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001747 skb->cloned = 0;
Patrick McHardy334a8132007-06-25 04:35:20 -07001748 skb->hdr_len = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001749 skb->nohdr = 0;
1750 atomic_set(&skb_shinfo(skb)->dataref, 1);
Eric Dumazet158f3232017-01-27 07:11:27 -08001751
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02001752 skb_metadata_clear(skb);
1753
Eric Dumazet158f3232017-01-27 07:11:27 -08001754 /* It is not generally safe to change skb->truesize.
1755 * For the moment, we really care of rx path, or
1756 * when skb is orphaned (not attached to a socket).
1757 */
1758 if (!skb->sk || skb->destructor == sock_edemux)
1759 skb->truesize += size - osize;
1760
Linus Torvalds1da177e2005-04-16 15:20:36 -07001761 return 0;
1762
Shirley Maa6686f22011-07-06 12:22:12 +00001763nofrags:
1764 kfree(data);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001765nodata:
1766 return -ENOMEM;
1767}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001768EXPORT_SYMBOL(pskb_expand_head);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001769
1770/* Make private copy of skb with writable head and some headroom */
1771
1772struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
1773{
1774 struct sk_buff *skb2;
1775 int delta = headroom - skb_headroom(skb);
1776
1777 if (delta <= 0)
1778 skb2 = pskb_copy(skb, GFP_ATOMIC);
1779 else {
1780 skb2 = skb_clone(skb, GFP_ATOMIC);
1781 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
1782 GFP_ATOMIC)) {
1783 kfree_skb(skb2);
1784 skb2 = NULL;
1785 }
1786 }
1787 return skb2;
1788}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001789EXPORT_SYMBOL(skb_realloc_headroom);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001790
1791/**
Vasily Averinf1260ff2021-08-02 11:52:15 +03001792 * skb_expand_head - reallocate header of &sk_buff
1793 * @skb: buffer to reallocate
1794 * @headroom: needed headroom
1795 *
1796 * Unlike skb_realloc_headroom, this one does not allocate a new skb
1797 * if possible; copies skb->sk to new skb as needed
1798 * and frees original skb in case of failures.
1799 *
1800 * It expect increased headroom and generates warning otherwise.
1801 */
1802
1803struct sk_buff *skb_expand_head(struct sk_buff *skb, unsigned int headroom)
1804{
1805 int delta = headroom - skb_headroom(skb);
Vasily Averin7f678de2021-10-22 13:28:37 +03001806 int osize = skb_end_offset(skb);
1807 struct sock *sk = skb->sk;
Vasily Averinf1260ff2021-08-02 11:52:15 +03001808
1809 if (WARN_ONCE(delta <= 0,
1810 "%s is expecting an increase in the headroom", __func__))
1811 return skb;
1812
Vasily Averin7f678de2021-10-22 13:28:37 +03001813 delta = SKB_DATA_ALIGN(delta);
1814 /* pskb_expand_head() might crash, if skb is shared. */
1815 if (skb_shared(skb) || !is_skb_wmem(skb)) {
Vasily Averinf1260ff2021-08-02 11:52:15 +03001816 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1817
Vasily Averin7f678de2021-10-22 13:28:37 +03001818 if (unlikely(!nskb))
1819 goto fail;
1820
1821 if (sk)
1822 skb_set_owner_w(nskb, sk);
1823 consume_skb(skb);
Vasily Averinf1260ff2021-08-02 11:52:15 +03001824 skb = nskb;
1825 }
Vasily Averin7f678de2021-10-22 13:28:37 +03001826 if (pskb_expand_head(skb, delta, 0, GFP_ATOMIC))
1827 goto fail;
1828
1829 if (sk && is_skb_wmem(skb)) {
1830 delta = skb_end_offset(skb) - osize;
1831 refcount_add(delta, &sk->sk_wmem_alloc);
1832 skb->truesize += delta;
Vasily Averinf1260ff2021-08-02 11:52:15 +03001833 }
1834 return skb;
Vasily Averin7f678de2021-10-22 13:28:37 +03001835
1836fail:
1837 kfree_skb(skb);
1838 return NULL;
Vasily Averinf1260ff2021-08-02 11:52:15 +03001839}
1840EXPORT_SYMBOL(skb_expand_head);
1841
1842/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001843 * skb_copy_expand - copy and expand sk_buff
1844 * @skb: buffer to copy
1845 * @newheadroom: new free bytes at head
1846 * @newtailroom: new free bytes at tail
1847 * @gfp_mask: allocation priority
1848 *
1849 * Make a copy of both an &sk_buff and its data and while doing so
1850 * allocate additional space.
1851 *
1852 * This is used when the caller wishes to modify the data and needs a
1853 * private copy of the data to alter as well as more space for new fields.
1854 * Returns %NULL on failure or the pointer to the buffer
1855 * on success. The returned buffer has a reference count of 1.
1856 *
1857 * You must pass %GFP_ATOMIC as the allocation priority if this function
1858 * is called from an interrupt.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001859 */
1860struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
Victor Fusco86a76ca2005-07-08 14:57:47 -07001861 int newheadroom, int newtailroom,
Al Virodd0fc662005-10-07 07:46:04 +01001862 gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001863{
1864 /*
1865 * Allocate the copy buffer
1866 */
Mel Gormanc93bdd02012-07-31 16:44:19 -07001867 struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom,
1868 gfp_mask, skb_alloc_rx_flag(skb),
1869 NUMA_NO_NODE);
Patrick McHardyefd1e8d2007-04-10 18:30:09 -07001870 int oldheadroom = skb_headroom(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001871 int head_copy_len, head_copy_off;
1872
1873 if (!n)
1874 return NULL;
1875
1876 skb_reserve(n, newheadroom);
1877
1878 /* Set the tail pointer and length */
1879 skb_put(n, skb->len);
1880
Patrick McHardyefd1e8d2007-04-10 18:30:09 -07001881 head_copy_len = oldheadroom;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001882 head_copy_off = 0;
1883 if (newheadroom <= head_copy_len)
1884 head_copy_len = newheadroom;
1885 else
1886 head_copy_off = newheadroom - head_copy_len;
1887
1888 /* Copy the linear header and data. */
Tim Hansen9f77fad2017-10-09 11:37:59 -04001889 BUG_ON(skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
1890 skb->len + head_copy_len));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001891
Ilya Lesokhin08303c12018-04-30 10:16:11 +03001892 skb_copy_header(n, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001893
Eric Dumazet030737b2013-10-19 11:42:54 -07001894 skb_headers_offset_update(n, newheadroom - oldheadroom);
Patrick McHardyefd1e8d2007-04-10 18:30:09 -07001895
Linus Torvalds1da177e2005-04-16 15:20:36 -07001896 return n;
1897}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001898EXPORT_SYMBOL(skb_copy_expand);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001899
1900/**
Florian Fainellicd0a1372017-08-22 15:12:14 -07001901 * __skb_pad - zero pad the tail of an skb
Linus Torvalds1da177e2005-04-16 15:20:36 -07001902 * @skb: buffer to pad
1903 * @pad: space to pad
Florian Fainellicd0a1372017-08-22 15:12:14 -07001904 * @free_on_error: free buffer on error
Linus Torvalds1da177e2005-04-16 15:20:36 -07001905 *
1906 * Ensure that a buffer is followed by a padding area that is zero
1907 * filled. Used by network drivers which may DMA or transfer data
1908 * beyond the buffer end onto the wire.
1909 *
Florian Fainellicd0a1372017-08-22 15:12:14 -07001910 * May return error in out of memory cases. The skb is freed on error
1911 * if @free_on_error is true.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001912 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001913
Florian Fainellicd0a1372017-08-22 15:12:14 -07001914int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001915{
Herbert Xu5b057c62006-06-23 02:06:41 -07001916 int err;
1917 int ntail;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001918
Linus Torvalds1da177e2005-04-16 15:20:36 -07001919 /* If the skbuff is non linear tailroom is always zero.. */
Herbert Xu5b057c62006-06-23 02:06:41 -07001920 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001921 memset(skb->data+skb->len, 0, pad);
Herbert Xu5b057c62006-06-23 02:06:41 -07001922 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001923 }
Herbert Xu5b057c62006-06-23 02:06:41 -07001924
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -07001925 ntail = skb->data_len + pad - (skb->end - skb->tail);
Herbert Xu5b057c62006-06-23 02:06:41 -07001926 if (likely(skb_cloned(skb) || ntail > 0)) {
1927 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
1928 if (unlikely(err))
1929 goto free_skb;
1930 }
1931
1932 /* FIXME: The use of this function with non-linear skb's really needs
1933 * to be audited.
1934 */
1935 err = skb_linearize(skb);
1936 if (unlikely(err))
1937 goto free_skb;
1938
1939 memset(skb->data + skb->len, 0, pad);
1940 return 0;
1941
1942free_skb:
Florian Fainellicd0a1372017-08-22 15:12:14 -07001943 if (free_on_error)
1944 kfree_skb(skb);
Herbert Xu5b057c62006-06-23 02:06:41 -07001945 return err;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001946}
Florian Fainellicd0a1372017-08-22 15:12:14 -07001947EXPORT_SYMBOL(__skb_pad);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001948
Ilpo Järvinen0dde3e12008-03-27 17:43:41 -07001949/**
Mathias Krause0c7ddf32013-11-07 14:18:24 +01001950 * pskb_put - add data to the tail of a potentially fragmented buffer
1951 * @skb: start of the buffer to use
1952 * @tail: tail fragment of the buffer to use
1953 * @len: amount of data to add
1954 *
1955 * This function extends the used data area of the potentially
1956 * fragmented buffer. @tail must be the last fragment of @skb -- or
1957 * @skb itself. If this would exceed the total buffer size the kernel
1958 * will panic. A pointer to the first byte of the extra data is
1959 * returned.
1960 */
1961
Johannes Berg4df864c2017-06-16 14:29:21 +02001962void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len)
Mathias Krause0c7ddf32013-11-07 14:18:24 +01001963{
1964 if (tail != skb) {
1965 skb->data_len += len;
1966 skb->len += len;
1967 }
1968 return skb_put(tail, len);
1969}
1970EXPORT_SYMBOL_GPL(pskb_put);
1971
1972/**
Ilpo Järvinen0dde3e12008-03-27 17:43:41 -07001973 * skb_put - add data to a buffer
1974 * @skb: buffer to use
1975 * @len: amount of data to add
1976 *
1977 * This function extends the used data area of the buffer. If this would
1978 * exceed the total buffer size the kernel will panic. A pointer to the
1979 * first byte of the extra data is returned.
1980 */
Johannes Berg4df864c2017-06-16 14:29:21 +02001981void *skb_put(struct sk_buff *skb, unsigned int len)
Ilpo Järvinen0dde3e12008-03-27 17:43:41 -07001982{
Johannes Berg4df864c2017-06-16 14:29:21 +02001983 void *tmp = skb_tail_pointer(skb);
Ilpo Järvinen0dde3e12008-03-27 17:43:41 -07001984 SKB_LINEAR_ASSERT(skb);
1985 skb->tail += len;
1986 skb->len += len;
1987 if (unlikely(skb->tail > skb->end))
1988 skb_over_panic(skb, len, __builtin_return_address(0));
1989 return tmp;
1990}
1991EXPORT_SYMBOL(skb_put);
1992
Ilpo Järvinen6be8ac22008-03-27 17:47:24 -07001993/**
Ilpo Järvinenc2aa2702008-03-27 17:52:40 -07001994 * skb_push - add data to the start of a buffer
1995 * @skb: buffer to use
1996 * @len: amount of data to add
1997 *
1998 * This function extends the used data area of the buffer at the buffer
1999 * start. If this would exceed the total buffer headroom the kernel will
2000 * panic. A pointer to the first byte of the extra data is returned.
2001 */
Johannes Bergd58ff352017-06-16 14:29:23 +02002002void *skb_push(struct sk_buff *skb, unsigned int len)
Ilpo Järvinenc2aa2702008-03-27 17:52:40 -07002003{
2004 skb->data -= len;
2005 skb->len += len;
Ganesh Goudar9aba2f82018-08-02 15:34:52 +05302006 if (unlikely(skb->data < skb->head))
Ilpo Järvinenc2aa2702008-03-27 17:52:40 -07002007 skb_under_panic(skb, len, __builtin_return_address(0));
2008 return skb->data;
2009}
2010EXPORT_SYMBOL(skb_push);
2011
2012/**
Ilpo Järvinen6be8ac22008-03-27 17:47:24 -07002013 * skb_pull - remove data from the start of a buffer
2014 * @skb: buffer to use
2015 * @len: amount of data to remove
2016 *
2017 * This function removes data from the start of a buffer, returning
2018 * the memory to the headroom. A pointer to the next data in the buffer
2019 * is returned. Once the data has been pulled future pushes will overwrite
2020 * the old data.
2021 */
Johannes Bergaf728682017-06-16 14:29:22 +02002022void *skb_pull(struct sk_buff *skb, unsigned int len)
Ilpo Järvinen6be8ac22008-03-27 17:47:24 -07002023{
David S. Miller47d29642010-05-02 02:21:44 -07002024 return skb_pull_inline(skb, len);
Ilpo Järvinen6be8ac22008-03-27 17:47:24 -07002025}
2026EXPORT_SYMBOL(skb_pull);
2027
Ilpo Järvinen419ae742008-03-27 17:54:01 -07002028/**
Luiz Augusto von Dentz13244cc2021-12-01 10:54:52 -08002029 * skb_pull_data - remove data from the start of a buffer returning its
2030 * original position.
2031 * @skb: buffer to use
2032 * @len: amount of data to remove
2033 *
2034 * This function removes data from the start of a buffer, returning
2035 * the memory to the headroom. A pointer to the original data in the buffer
2036 * is returned after checking if there is enough data to pull. Once the
2037 * data has been pulled future pushes will overwrite the old data.
2038 */
2039void *skb_pull_data(struct sk_buff *skb, size_t len)
2040{
2041 void *data = skb->data;
2042
2043 if (skb->len < len)
2044 return NULL;
2045
2046 skb_pull(skb, len);
2047
2048 return data;
2049}
2050EXPORT_SYMBOL(skb_pull_data);
2051
2052/**
Ilpo Järvinen419ae742008-03-27 17:54:01 -07002053 * skb_trim - remove end from a buffer
2054 * @skb: buffer to alter
2055 * @len: new length
2056 *
2057 * Cut the length of a buffer down by removing data from the tail. If
2058 * the buffer is already under the length specified it is not modified.
2059 * The skb must be linear.
2060 */
2061void skb_trim(struct sk_buff *skb, unsigned int len)
2062{
2063 if (skb->len > len)
2064 __skb_trim(skb, len);
2065}
2066EXPORT_SYMBOL(skb_trim);
2067
Herbert Xu3cc0e872006-06-09 16:13:38 -07002068/* Trims skb to length len. It can change skb pointers.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002069 */
2070
Herbert Xu3cc0e872006-06-09 16:13:38 -07002071int ___pskb_trim(struct sk_buff *skb, unsigned int len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002072{
Herbert Xu27b437c2006-07-13 19:26:39 -07002073 struct sk_buff **fragp;
2074 struct sk_buff *frag;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002075 int offset = skb_headlen(skb);
2076 int nfrags = skb_shinfo(skb)->nr_frags;
2077 int i;
Herbert Xu27b437c2006-07-13 19:26:39 -07002078 int err;
2079
2080 if (skb_cloned(skb) &&
2081 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
2082 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002083
Herbert Xuf4d26fb2006-07-30 20:20:28 -07002084 i = 0;
2085 if (offset >= len)
2086 goto drop_pages;
2087
2088 for (; i < nfrags; i++) {
Eric Dumazet9e903e02011-10-18 21:00:24 +00002089 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]);
Herbert Xu27b437c2006-07-13 19:26:39 -07002090
2091 if (end < len) {
2092 offset = end;
2093 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002094 }
Herbert Xu27b437c2006-07-13 19:26:39 -07002095
Eric Dumazet9e903e02011-10-18 21:00:24 +00002096 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset);
Herbert Xu27b437c2006-07-13 19:26:39 -07002097
Herbert Xuf4d26fb2006-07-30 20:20:28 -07002098drop_pages:
Herbert Xu27b437c2006-07-13 19:26:39 -07002099 skb_shinfo(skb)->nr_frags = i;
2100
2101 for (; i < nfrags; i++)
Ian Campbellea2ab692011-08-22 23:44:58 +00002102 skb_frag_unref(skb, i);
Herbert Xu27b437c2006-07-13 19:26:39 -07002103
David S. Miller21dc3302010-08-23 00:13:46 -07002104 if (skb_has_frag_list(skb))
Herbert Xu27b437c2006-07-13 19:26:39 -07002105 skb_drop_fraglist(skb);
Herbert Xuf4d26fb2006-07-30 20:20:28 -07002106 goto done;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002107 }
2108
Herbert Xu27b437c2006-07-13 19:26:39 -07002109 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
2110 fragp = &frag->next) {
2111 int end = offset + frag->len;
2112
2113 if (skb_shared(frag)) {
2114 struct sk_buff *nfrag;
2115
2116 nfrag = skb_clone(frag, GFP_ATOMIC);
2117 if (unlikely(!nfrag))
2118 return -ENOMEM;
2119
2120 nfrag->next = frag->next;
Eric Dumazet85bb2a62012-04-19 02:24:53 +00002121 consume_skb(frag);
Herbert Xu27b437c2006-07-13 19:26:39 -07002122 frag = nfrag;
2123 *fragp = frag;
2124 }
2125
2126 if (end < len) {
2127 offset = end;
2128 continue;
2129 }
2130
2131 if (end > len &&
2132 unlikely((err = pskb_trim(frag, len - offset))))
2133 return err;
2134
2135 if (frag->next)
2136 skb_drop_list(&frag->next);
2137 break;
2138 }
2139
Herbert Xuf4d26fb2006-07-30 20:20:28 -07002140done:
Herbert Xu27b437c2006-07-13 19:26:39 -07002141 if (len > skb_headlen(skb)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002142 skb->data_len -= skb->len - len;
2143 skb->len = len;
2144 } else {
Herbert Xu27b437c2006-07-13 19:26:39 -07002145 skb->len = len;
2146 skb->data_len = 0;
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07002147 skb_set_tail_pointer(skb, len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002148 }
2149
Eric Dumazetc21b48c2017-04-26 09:07:46 -07002150 if (!skb->sk || skb->destructor == sock_edemux)
2151 skb_condense(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002152 return 0;
2153}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002154EXPORT_SYMBOL(___pskb_trim);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002155
Eric Dumazet88078d92018-04-18 11:43:15 -07002156/* Note : use pskb_trim_rcsum() instead of calling this directly
2157 */
2158int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len)
2159{
2160 if (skb->ip_summed == CHECKSUM_COMPLETE) {
2161 int delta = skb->len - len;
2162
Dimitris Michailidisd55bef502018-10-19 17:07:13 -07002163 skb->csum = csum_block_sub(skb->csum,
2164 skb_checksum(skb, len, delta, 0),
2165 len);
Vasily Averin54970a22020-12-14 22:07:39 +03002166 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
2167 int hdlen = (len > skb_headlen(skb)) ? skb_headlen(skb) : len;
2168 int offset = skb_checksum_start_offset(skb) + skb->csum_offset;
2169
2170 if (offset + sizeof(__sum16) > hdlen)
2171 return -EINVAL;
Eric Dumazet88078d92018-04-18 11:43:15 -07002172 }
2173 return __pskb_trim(skb, len);
2174}
2175EXPORT_SYMBOL(pskb_trim_rcsum_slow);
2176
Linus Torvalds1da177e2005-04-16 15:20:36 -07002177/**
2178 * __pskb_pull_tail - advance tail of skb header
2179 * @skb: buffer to reallocate
2180 * @delta: number of bytes to advance tail
2181 *
2182 * The function makes a sense only on a fragmented &sk_buff,
2183 * it expands header moving its tail forward and copying necessary
2184 * data from fragmented part.
2185 *
2186 * &sk_buff MUST have reference count of 1.
2187 *
2188 * Returns %NULL (and &sk_buff does not change) if pull failed
2189 * or value of new tail of skb in the case of success.
2190 *
2191 * All the pointers pointing into skb header may change and must be
2192 * reloaded after call to this function.
2193 */
2194
2195/* Moves tail of skb head forward, copying data from fragmented part,
2196 * when it is necessary.
2197 * 1. It may fail due to malloc failure.
2198 * 2. It may change skb pointers.
2199 *
2200 * It is pretty complicated. Luckily, it is called only in exceptional cases.
2201 */
Johannes Bergaf728682017-06-16 14:29:22 +02002202void *__pskb_pull_tail(struct sk_buff *skb, int delta)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002203{
2204 /* If skb has not enough free space at tail, get new one
2205 * plus 128 bytes for future expansions. If we have enough
2206 * room at tail, reallocate without expansion only if skb is cloned.
2207 */
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -07002208 int i, k, eat = (skb->tail + delta) - skb->end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002209
2210 if (eat > 0 || skb_cloned(skb)) {
2211 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
2212 GFP_ATOMIC))
2213 return NULL;
2214 }
2215
Tim Hansen9f77fad2017-10-09 11:37:59 -04002216 BUG_ON(skb_copy_bits(skb, skb_headlen(skb),
2217 skb_tail_pointer(skb), delta));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002218
2219 /* Optimization: no fragments, no reasons to preestimate
2220 * size of pulled pages. Superb.
2221 */
David S. Miller21dc3302010-08-23 00:13:46 -07002222 if (!skb_has_frag_list(skb))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002223 goto pull_pages;
2224
2225 /* Estimate size of pulled pages. */
2226 eat = delta;
2227 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
Eric Dumazet9e903e02011-10-18 21:00:24 +00002228 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
2229
2230 if (size >= eat)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002231 goto pull_pages;
Eric Dumazet9e903e02011-10-18 21:00:24 +00002232 eat -= size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002233 }
2234
2235 /* If we need update frag list, we are in troubles.
Wenhua Shi09001b02017-10-14 18:51:36 +02002236 * Certainly, it is possible to add an offset to skb data,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002237 * but taking into account that pulling is expected to
2238 * be very rare operation, it is worth to fight against
2239 * further bloating skb head and crucify ourselves here instead.
2240 * Pure masohism, indeed. 8)8)
2241 */
2242 if (eat) {
2243 struct sk_buff *list = skb_shinfo(skb)->frag_list;
2244 struct sk_buff *clone = NULL;
2245 struct sk_buff *insp = NULL;
2246
2247 do {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002248 if (list->len <= eat) {
2249 /* Eaten as whole. */
2250 eat -= list->len;
2251 list = list->next;
2252 insp = list;
2253 } else {
2254 /* Eaten partially. */
2255
2256 if (skb_shared(list)) {
2257 /* Sucks! We need to fork list. :-( */
2258 clone = skb_clone(list, GFP_ATOMIC);
2259 if (!clone)
2260 return NULL;
2261 insp = list->next;
2262 list = clone;
2263 } else {
2264 /* This may be pulled without
2265 * problems. */
2266 insp = list;
2267 }
2268 if (!pskb_pull(list, eat)) {
Wei Yongjunf3fbbe02009-02-25 00:37:32 +00002269 kfree_skb(clone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002270 return NULL;
2271 }
2272 break;
2273 }
2274 } while (eat);
2275
2276 /* Free pulled out fragments. */
2277 while ((list = skb_shinfo(skb)->frag_list) != insp) {
2278 skb_shinfo(skb)->frag_list = list->next;
2279 kfree_skb(list);
2280 }
2281 /* And insert new clone at head. */
2282 if (clone) {
2283 clone->next = list;
2284 skb_shinfo(skb)->frag_list = clone;
2285 }
2286 }
2287 /* Success! Now we may commit changes to skb data. */
2288
2289pull_pages:
2290 eat = delta;
2291 k = 0;
2292 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
Eric Dumazet9e903e02011-10-18 21:00:24 +00002293 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
2294
2295 if (size <= eat) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002296 skb_frag_unref(skb, i);
Eric Dumazet9e903e02011-10-18 21:00:24 +00002297 eat -= size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002298 } else {
Jonathan Lemonb54c9d52019-07-30 07:40:33 -07002299 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2300
2301 *frag = skb_shinfo(skb)->frags[i];
Linus Torvalds1da177e2005-04-16 15:20:36 -07002302 if (eat) {
Jonathan Lemonb54c9d52019-07-30 07:40:33 -07002303 skb_frag_off_add(frag, eat);
2304 skb_frag_size_sub(frag, eat);
linzhang3ccc6c62017-07-17 17:25:02 +08002305 if (!i)
2306 goto end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002307 eat = 0;
2308 }
2309 k++;
2310 }
2311 }
2312 skb_shinfo(skb)->nr_frags = k;
2313
linzhang3ccc6c62017-07-17 17:25:02 +08002314end:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002315 skb->tail += delta;
2316 skb->data_len -= delta;
2317
Willem de Bruijn1f8b9772017-08-03 16:29:41 -04002318 if (!skb->data_len)
2319 skb_zcopy_clear(skb, false);
2320
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07002321 return skb_tail_pointer(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002322}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002323EXPORT_SYMBOL(__pskb_pull_tail);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002324
Eric Dumazet22019b12011-07-29 18:37:31 +00002325/**
2326 * skb_copy_bits - copy bits from skb to kernel buffer
2327 * @skb: source skb
2328 * @offset: offset in source
2329 * @to: destination buffer
2330 * @len: number of bytes to copy
2331 *
2332 * Copy the specified number of bytes from the source skb to the
2333 * destination buffer.
2334 *
2335 * CAUTION ! :
2336 * If its prototype is ever changed,
2337 * check arch/{*}/net/{*}.S files,
2338 * since it is called from BPF assembly code.
2339 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002340int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
2341{
David S. Miller1a028e52007-04-27 15:21:23 -07002342 int start = skb_headlen(skb);
David S. Millerfbb398a2009-06-09 00:18:59 -07002343 struct sk_buff *frag_iter;
2344 int i, copy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002345
2346 if (offset > (int)skb->len - len)
2347 goto fault;
2348
2349 /* Copy header. */
David S. Miller1a028e52007-04-27 15:21:23 -07002350 if ((copy = start - offset) > 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002351 if (copy > len)
2352 copy = len;
Arnaldo Carvalho de Melod626f622007-03-27 18:55:52 -03002353 skb_copy_from_linear_data_offset(skb, offset, to, copy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002354 if ((len -= copy) == 0)
2355 return 0;
2356 offset += copy;
2357 to += copy;
2358 }
2359
2360 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
David S. Miller1a028e52007-04-27 15:21:23 -07002361 int end;
Eric Dumazet51c56b02012-04-05 11:35:15 +02002362 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
Linus Torvalds1da177e2005-04-16 15:20:36 -07002363
Ilpo Järvinen547b7922008-07-25 21:43:18 -07002364 WARN_ON(start > offset + len);
David S. Miller1a028e52007-04-27 15:21:23 -07002365
Eric Dumazet51c56b02012-04-05 11:35:15 +02002366 end = start + skb_frag_size(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002367 if ((copy = end - offset) > 0) {
Willem de Bruijnc613c202017-07-31 08:15:47 -04002368 u32 p_off, p_len, copied;
2369 struct page *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002370 u8 *vaddr;
2371
2372 if (copy > len)
2373 copy = len;
2374
Willem de Bruijnc613c202017-07-31 08:15:47 -04002375 skb_frag_foreach_page(f,
Jonathan Lemonb54c9d52019-07-30 07:40:33 -07002376 skb_frag_off(f) + offset - start,
Willem de Bruijnc613c202017-07-31 08:15:47 -04002377 copy, p, p_off, p_len, copied) {
2378 vaddr = kmap_atomic(p);
2379 memcpy(to + copied, vaddr + p_off, p_len);
2380 kunmap_atomic(vaddr);
2381 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002382
2383 if ((len -= copy) == 0)
2384 return 0;
2385 offset += copy;
2386 to += copy;
2387 }
David S. Miller1a028e52007-04-27 15:21:23 -07002388 start = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002389 }
2390
David S. Millerfbb398a2009-06-09 00:18:59 -07002391 skb_walk_frags(skb, frag_iter) {
2392 int end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002393
David S. Millerfbb398a2009-06-09 00:18:59 -07002394 WARN_ON(start > offset + len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002395
David S. Millerfbb398a2009-06-09 00:18:59 -07002396 end = start + frag_iter->len;
2397 if ((copy = end - offset) > 0) {
2398 if (copy > len)
2399 copy = len;
2400 if (skb_copy_bits(frag_iter, offset - start, to, copy))
2401 goto fault;
2402 if ((len -= copy) == 0)
2403 return 0;
2404 offset += copy;
2405 to += copy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002406 }
David S. Millerfbb398a2009-06-09 00:18:59 -07002407 start = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002408 }
Shirley Maa6686f22011-07-06 12:22:12 +00002409
Linus Torvalds1da177e2005-04-16 15:20:36 -07002410 if (!len)
2411 return 0;
2412
2413fault:
2414 return -EFAULT;
2415}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002416EXPORT_SYMBOL(skb_copy_bits);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002417
Jens Axboe9c55e012007-11-06 23:30:13 -08002418/*
2419 * Callback from splice_to_pipe(), if we need to release some pages
2420 * at the end of the spd in case we error'ed out in filling the pipe.
2421 */
2422static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
2423{
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08002424 put_page(spd->pages[i]);
2425}
Jens Axboe9c55e012007-11-06 23:30:13 -08002426
David S. Millera108d5f2012-04-23 23:06:11 -04002427static struct page *linear_to_page(struct page *page, unsigned int *len,
2428 unsigned int *offset,
Eric Dumazet18aafc62013-01-11 14:46:37 +00002429 struct sock *sk)
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08002430{
Eric Dumazet5640f762012-09-23 23:04:42 +00002431 struct page_frag *pfrag = sk_page_frag(sk);
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08002432
Eric Dumazet5640f762012-09-23 23:04:42 +00002433 if (!sk_page_frag_refill(sk, pfrag))
2434 return NULL;
Jarek Poplawski4fb66992009-02-01 00:41:42 -08002435
Eric Dumazet5640f762012-09-23 23:04:42 +00002436 *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset);
Jarek Poplawski4fb66992009-02-01 00:41:42 -08002437
Eric Dumazet5640f762012-09-23 23:04:42 +00002438 memcpy(page_address(pfrag->page) + pfrag->offset,
2439 page_address(page) + *offset, *len);
2440 *offset = pfrag->offset;
2441 pfrag->offset += *len;
Jarek Poplawski4fb66992009-02-01 00:41:42 -08002442
Eric Dumazet5640f762012-09-23 23:04:42 +00002443 return pfrag->page;
Jens Axboe9c55e012007-11-06 23:30:13 -08002444}
2445
Eric Dumazet41c73a02012-04-22 12:26:16 +00002446static bool spd_can_coalesce(const struct splice_pipe_desc *spd,
2447 struct page *page,
2448 unsigned int offset)
2449{
2450 return spd->nr_pages &&
2451 spd->pages[spd->nr_pages - 1] == page &&
2452 (spd->partial[spd->nr_pages - 1].offset +
2453 spd->partial[spd->nr_pages - 1].len == offset);
2454}
2455
Jens Axboe9c55e012007-11-06 23:30:13 -08002456/*
2457 * Fill page/offset/length into spd, if it can hold more pages.
2458 */
David S. Millera108d5f2012-04-23 23:06:11 -04002459static bool spd_fill_page(struct splice_pipe_desc *spd,
2460 struct pipe_inode_info *pipe, struct page *page,
2461 unsigned int *len, unsigned int offset,
Eric Dumazet18aafc62013-01-11 14:46:37 +00002462 bool linear,
David S. Millera108d5f2012-04-23 23:06:11 -04002463 struct sock *sk)
Jens Axboe9c55e012007-11-06 23:30:13 -08002464{
Eric Dumazet41c73a02012-04-22 12:26:16 +00002465 if (unlikely(spd->nr_pages == MAX_SKB_FRAGS))
David S. Millera108d5f2012-04-23 23:06:11 -04002466 return true;
Jens Axboe9c55e012007-11-06 23:30:13 -08002467
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08002468 if (linear) {
Eric Dumazet18aafc62013-01-11 14:46:37 +00002469 page = linear_to_page(page, len, &offset, sk);
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08002470 if (!page)
David S. Millera108d5f2012-04-23 23:06:11 -04002471 return true;
Eric Dumazet41c73a02012-04-22 12:26:16 +00002472 }
2473 if (spd_can_coalesce(spd, page, offset)) {
2474 spd->partial[spd->nr_pages - 1].len += *len;
David S. Millera108d5f2012-04-23 23:06:11 -04002475 return false;
Eric Dumazet41c73a02012-04-22 12:26:16 +00002476 }
2477 get_page(page);
Jens Axboe9c55e012007-11-06 23:30:13 -08002478 spd->pages[spd->nr_pages] = page;
Jarek Poplawski4fb66992009-02-01 00:41:42 -08002479 spd->partial[spd->nr_pages].len = *len;
Jens Axboe9c55e012007-11-06 23:30:13 -08002480 spd->partial[spd->nr_pages].offset = offset;
Jens Axboe9c55e012007-11-06 23:30:13 -08002481 spd->nr_pages++;
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08002482
David S. Millera108d5f2012-04-23 23:06:11 -04002483 return false;
Jens Axboe9c55e012007-11-06 23:30:13 -08002484}
2485
David S. Millera108d5f2012-04-23 23:06:11 -04002486static bool __splice_segment(struct page *page, unsigned int poff,
2487 unsigned int plen, unsigned int *off,
Eric Dumazet18aafc62013-01-11 14:46:37 +00002488 unsigned int *len,
Eric Dumazetd7ccf7c2012-04-23 23:35:04 -04002489 struct splice_pipe_desc *spd, bool linear,
David S. Millera108d5f2012-04-23 23:06:11 -04002490 struct sock *sk,
2491 struct pipe_inode_info *pipe)
Octavian Purdila2870c432008-07-15 00:49:11 -07002492{
2493 if (!*len)
David S. Millera108d5f2012-04-23 23:06:11 -04002494 return true;
Octavian Purdila2870c432008-07-15 00:49:11 -07002495
2496 /* skip this segment if already processed */
2497 if (*off >= plen) {
2498 *off -= plen;
David S. Millera108d5f2012-04-23 23:06:11 -04002499 return false;
Octavian Purdiladb43a282008-06-27 17:27:21 -07002500 }
Jens Axboe9c55e012007-11-06 23:30:13 -08002501
Octavian Purdila2870c432008-07-15 00:49:11 -07002502 /* ignore any bits we already processed */
Eric Dumazet9ca1b222013-01-05 21:31:18 +00002503 poff += *off;
2504 plen -= *off;
2505 *off = 0;
Octavian Purdila2870c432008-07-15 00:49:11 -07002506
Eric Dumazet18aafc62013-01-11 14:46:37 +00002507 do {
2508 unsigned int flen = min(*len, plen);
Octavian Purdila2870c432008-07-15 00:49:11 -07002509
Eric Dumazet18aafc62013-01-11 14:46:37 +00002510 if (spd_fill_page(spd, pipe, page, &flen, poff,
2511 linear, sk))
2512 return true;
2513 poff += flen;
2514 plen -= flen;
2515 *len -= flen;
2516 } while (*len && plen);
Octavian Purdila2870c432008-07-15 00:49:11 -07002517
David S. Millera108d5f2012-04-23 23:06:11 -04002518 return false;
Octavian Purdila2870c432008-07-15 00:49:11 -07002519}
2520
2521/*
David S. Millera108d5f2012-04-23 23:06:11 -04002522 * Map linear and fragment data from the skb to spd. It reports true if the
Octavian Purdila2870c432008-07-15 00:49:11 -07002523 * pipe is full or if we already spliced the requested length.
2524 */
David S. Millera108d5f2012-04-23 23:06:11 -04002525static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
2526 unsigned int *offset, unsigned int *len,
2527 struct splice_pipe_desc *spd, struct sock *sk)
Octavian Purdila2870c432008-07-15 00:49:11 -07002528{
2529 int seg;
Tom Herbertfa9835e2016-03-07 14:11:04 -08002530 struct sk_buff *iter;
Octavian Purdila2870c432008-07-15 00:49:11 -07002531
Eric Dumazet1d0c0b32012-04-27 02:10:03 +00002532 /* map the linear part :
Alexander Duyck2996d312012-05-02 18:18:42 +00002533 * If skb->head_frag is set, this 'linear' part is backed by a
2534 * fragment, and if the head is not shared with any clones then
2535 * we can avoid a copy since we own the head portion of this page.
Jens Axboe9c55e012007-11-06 23:30:13 -08002536 */
Octavian Purdila2870c432008-07-15 00:49:11 -07002537 if (__splice_segment(virt_to_page(skb->data),
2538 (unsigned long) skb->data & (PAGE_SIZE - 1),
2539 skb_headlen(skb),
Eric Dumazet18aafc62013-01-11 14:46:37 +00002540 offset, len, spd,
Alexander Duyck3a7c1ee42012-05-03 01:09:42 +00002541 skb_head_is_locked(skb),
Eric Dumazet1d0c0b32012-04-27 02:10:03 +00002542 sk, pipe))
David S. Millera108d5f2012-04-23 23:06:11 -04002543 return true;
Jens Axboe9c55e012007-11-06 23:30:13 -08002544
2545 /*
2546 * then map the fragments
2547 */
Jens Axboe9c55e012007-11-06 23:30:13 -08002548 for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
2549 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
2550
Ian Campbellea2ab692011-08-22 23:44:58 +00002551 if (__splice_segment(skb_frag_page(f),
Jonathan Lemonb54c9d52019-07-30 07:40:33 -07002552 skb_frag_off(f), skb_frag_size(f),
Eric Dumazet18aafc62013-01-11 14:46:37 +00002553 offset, len, spd, false, sk, pipe))
David S. Millera108d5f2012-04-23 23:06:11 -04002554 return true;
Jens Axboe9c55e012007-11-06 23:30:13 -08002555 }
2556
Tom Herbertfa9835e2016-03-07 14:11:04 -08002557 skb_walk_frags(skb, iter) {
2558 if (*offset >= iter->len) {
2559 *offset -= iter->len;
2560 continue;
2561 }
2562 /* __skb_splice_bits() only fails if the output has no room
2563 * left, so no point in going over the frag_list for the error
2564 * case.
2565 */
2566 if (__skb_splice_bits(iter, pipe, offset, len, spd, sk))
2567 return true;
2568 }
2569
David S. Millera108d5f2012-04-23 23:06:11 -04002570 return false;
Jens Axboe9c55e012007-11-06 23:30:13 -08002571}
2572
2573/*
2574 * Map data from the skb to a pipe. Should handle both the linear part,
Tom Herbertfa9835e2016-03-07 14:11:04 -08002575 * the fragments, and the frag list.
Jens Axboe9c55e012007-11-06 23:30:13 -08002576 */
Hannes Frederic Sowaa60e3cc2015-05-21 17:00:00 +02002577int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
Jens Axboe9c55e012007-11-06 23:30:13 -08002578 struct pipe_inode_info *pipe, unsigned int tlen,
Al Viro25869262016-09-17 21:02:10 -04002579 unsigned int flags)
Jens Axboe9c55e012007-11-06 23:30:13 -08002580{
Eric Dumazet41c73a02012-04-22 12:26:16 +00002581 struct partial_page partial[MAX_SKB_FRAGS];
2582 struct page *pages[MAX_SKB_FRAGS];
Jens Axboe9c55e012007-11-06 23:30:13 -08002583 struct splice_pipe_desc spd = {
2584 .pages = pages,
2585 .partial = partial,
Eric Dumazet047fe362012-06-12 15:24:40 +02002586 .nr_pages_max = MAX_SKB_FRAGS,
Miklos Szeredi28a625c2014-01-22 19:36:57 +01002587 .ops = &nosteal_pipe_buf_ops,
Jens Axboe9c55e012007-11-06 23:30:13 -08002588 .spd_release = sock_spd_release,
2589 };
Jens Axboe35f3d142010-05-20 10:43:18 +02002590 int ret = 0;
2591
Tom Herbertfa9835e2016-03-07 14:11:04 -08002592 __skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk);
Jens Axboe9c55e012007-11-06 23:30:13 -08002593
Hannes Frederic Sowaa60e3cc2015-05-21 17:00:00 +02002594 if (spd.nr_pages)
Al Viro25869262016-09-17 21:02:10 -04002595 ret = splice_to_pipe(pipe, &spd);
Jens Axboe9c55e012007-11-06 23:30:13 -08002596
Jens Axboe35f3d142010-05-20 10:43:18 +02002597 return ret;
Jens Axboe9c55e012007-11-06 23:30:13 -08002598}
Hannes Frederic Sowa2b514572015-05-21 17:00:01 +02002599EXPORT_SYMBOL_GPL(skb_splice_bits);
Jens Axboe9c55e012007-11-06 23:30:13 -08002600
Cong Wang0739cd22021-03-30 19:32:24 -07002601static int sendmsg_unlocked(struct sock *sk, struct msghdr *msg,
2602 struct kvec *vec, size_t num, size_t size)
2603{
2604 struct socket *sock = sk->sk_socket;
2605
2606 if (!sock)
2607 return -EINVAL;
2608 return kernel_sendmsg(sock, msg, vec, num, size);
2609}
2610
2611static int sendpage_unlocked(struct sock *sk, struct page *page, int offset,
2612 size_t size, int flags)
2613{
2614 struct socket *sock = sk->sk_socket;
2615
2616 if (!sock)
2617 return -EINVAL;
2618 return kernel_sendpage(sock, page, offset, size, flags);
2619}
2620
2621typedef int (*sendmsg_func)(struct sock *sk, struct msghdr *msg,
2622 struct kvec *vec, size_t num, size_t size);
2623typedef int (*sendpage_func)(struct sock *sk, struct page *page, int offset,
2624 size_t size, int flags);
2625static int __skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset,
2626 int len, sendmsg_func sendmsg, sendpage_func sendpage)
Tom Herbert20bf50d2017-07-28 16:22:42 -07002627{
2628 unsigned int orig_len = len;
2629 struct sk_buff *head = skb;
2630 unsigned short fragidx;
2631 int slen, ret;
2632
2633do_frag_list:
2634
2635 /* Deal with head data */
2636 while (offset < skb_headlen(skb) && len) {
2637 struct kvec kv;
2638 struct msghdr msg;
2639
2640 slen = min_t(int, len, skb_headlen(skb) - offset);
2641 kv.iov_base = skb->data + offset;
John Fastabenddb5980d2017-08-15 22:31:34 -07002642 kv.iov_len = slen;
Tom Herbert20bf50d2017-07-28 16:22:42 -07002643 memset(&msg, 0, sizeof(msg));
John Fastabendbd95e6782019-05-24 08:01:00 -07002644 msg.msg_flags = MSG_DONTWAIT;
Tom Herbert20bf50d2017-07-28 16:22:42 -07002645
Cong Wang0739cd22021-03-30 19:32:24 -07002646 ret = INDIRECT_CALL_2(sendmsg, kernel_sendmsg_locked,
2647 sendmsg_unlocked, sk, &msg, &kv, 1, slen);
Tom Herbert20bf50d2017-07-28 16:22:42 -07002648 if (ret <= 0)
2649 goto error;
2650
2651 offset += ret;
2652 len -= ret;
2653 }
2654
2655 /* All the data was skb head? */
2656 if (!len)
2657 goto out;
2658
2659 /* Make offset relative to start of frags */
2660 offset -= skb_headlen(skb);
2661
2662 /* Find where we are in frag list */
2663 for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
2664 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx];
2665
Matthew Wilcox (Oracle)d8e18a52019-07-22 20:08:26 -07002666 if (offset < skb_frag_size(frag))
Tom Herbert20bf50d2017-07-28 16:22:42 -07002667 break;
2668
Matthew Wilcox (Oracle)d8e18a52019-07-22 20:08:26 -07002669 offset -= skb_frag_size(frag);
Tom Herbert20bf50d2017-07-28 16:22:42 -07002670 }
2671
2672 for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
2673 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx];
2674
Matthew Wilcox (Oracle)d8e18a52019-07-22 20:08:26 -07002675 slen = min_t(size_t, len, skb_frag_size(frag) - offset);
Tom Herbert20bf50d2017-07-28 16:22:42 -07002676
2677 while (slen) {
Cong Wang0739cd22021-03-30 19:32:24 -07002678 ret = INDIRECT_CALL_2(sendpage, kernel_sendpage_locked,
2679 sendpage_unlocked, sk,
2680 skb_frag_page(frag),
2681 skb_frag_off(frag) + offset,
2682 slen, MSG_DONTWAIT);
Tom Herbert20bf50d2017-07-28 16:22:42 -07002683 if (ret <= 0)
2684 goto error;
2685
2686 len -= ret;
2687 offset += ret;
2688 slen -= ret;
2689 }
2690
2691 offset = 0;
2692 }
2693
2694 if (len) {
2695 /* Process any frag lists */
2696
2697 if (skb == head) {
2698 if (skb_has_frag_list(skb)) {
2699 skb = skb_shinfo(skb)->frag_list;
2700 goto do_frag_list;
2701 }
2702 } else if (skb->next) {
2703 skb = skb->next;
2704 goto do_frag_list;
2705 }
2706 }
2707
2708out:
2709 return orig_len - len;
2710
2711error:
2712 return orig_len == len ? ret : orig_len - len;
2713}
Cong Wang0739cd22021-03-30 19:32:24 -07002714
2715/* Send skb data on a socket. Socket must be locked. */
2716int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
2717 int len)
2718{
2719 return __skb_send_sock(sk, skb, offset, len, kernel_sendmsg_locked,
2720 kernel_sendpage_locked);
2721}
Tom Herbert20bf50d2017-07-28 16:22:42 -07002722EXPORT_SYMBOL_GPL(skb_send_sock_locked);
2723
Cong Wang0739cd22021-03-30 19:32:24 -07002724/* Send skb data on a socket. Socket must be unlocked. */
2725int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len)
2726{
2727 return __skb_send_sock(sk, skb, offset, len, sendmsg_unlocked,
2728 sendpage_unlocked);
2729}
2730
Herbert Xu357b40a2005-04-19 22:30:14 -07002731/**
2732 * skb_store_bits - store bits from kernel buffer to skb
2733 * @skb: destination buffer
2734 * @offset: offset in destination
2735 * @from: source buffer
2736 * @len: number of bytes to copy
2737 *
2738 * Copy the specified number of bytes from the source buffer to the
2739 * destination skb. This function handles all the messy bits of
2740 * traversing fragment lists and such.
2741 */
2742
Stephen Hemminger0c6fcc82007-04-20 16:40:01 -07002743int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
Herbert Xu357b40a2005-04-19 22:30:14 -07002744{
David S. Miller1a028e52007-04-27 15:21:23 -07002745 int start = skb_headlen(skb);
David S. Millerfbb398a2009-06-09 00:18:59 -07002746 struct sk_buff *frag_iter;
2747 int i, copy;
Herbert Xu357b40a2005-04-19 22:30:14 -07002748
2749 if (offset > (int)skb->len - len)
2750 goto fault;
2751
David S. Miller1a028e52007-04-27 15:21:23 -07002752 if ((copy = start - offset) > 0) {
Herbert Xu357b40a2005-04-19 22:30:14 -07002753 if (copy > len)
2754 copy = len;
Arnaldo Carvalho de Melo27d7ff42007-03-31 11:55:19 -03002755 skb_copy_to_linear_data_offset(skb, offset, from, copy);
Herbert Xu357b40a2005-04-19 22:30:14 -07002756 if ((len -= copy) == 0)
2757 return 0;
2758 offset += copy;
2759 from += copy;
2760 }
2761
2762 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2763 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
David S. Miller1a028e52007-04-27 15:21:23 -07002764 int end;
Herbert Xu357b40a2005-04-19 22:30:14 -07002765
Ilpo Järvinen547b7922008-07-25 21:43:18 -07002766 WARN_ON(start > offset + len);
David S. Miller1a028e52007-04-27 15:21:23 -07002767
Eric Dumazet9e903e02011-10-18 21:00:24 +00002768 end = start + skb_frag_size(frag);
Herbert Xu357b40a2005-04-19 22:30:14 -07002769 if ((copy = end - offset) > 0) {
Willem de Bruijnc613c202017-07-31 08:15:47 -04002770 u32 p_off, p_len, copied;
2771 struct page *p;
Herbert Xu357b40a2005-04-19 22:30:14 -07002772 u8 *vaddr;
2773
2774 if (copy > len)
2775 copy = len;
2776
Willem de Bruijnc613c202017-07-31 08:15:47 -04002777 skb_frag_foreach_page(frag,
Jonathan Lemonb54c9d52019-07-30 07:40:33 -07002778 skb_frag_off(frag) + offset - start,
Willem de Bruijnc613c202017-07-31 08:15:47 -04002779 copy, p, p_off, p_len, copied) {
2780 vaddr = kmap_atomic(p);
2781 memcpy(vaddr + p_off, from + copied, p_len);
2782 kunmap_atomic(vaddr);
2783 }
Herbert Xu357b40a2005-04-19 22:30:14 -07002784
2785 if ((len -= copy) == 0)
2786 return 0;
2787 offset += copy;
2788 from += copy;
2789 }
David S. Miller1a028e52007-04-27 15:21:23 -07002790 start = end;
Herbert Xu357b40a2005-04-19 22:30:14 -07002791 }
2792
David S. Millerfbb398a2009-06-09 00:18:59 -07002793 skb_walk_frags(skb, frag_iter) {
2794 int end;
Herbert Xu357b40a2005-04-19 22:30:14 -07002795
David S. Millerfbb398a2009-06-09 00:18:59 -07002796 WARN_ON(start > offset + len);
Herbert Xu357b40a2005-04-19 22:30:14 -07002797
David S. Millerfbb398a2009-06-09 00:18:59 -07002798 end = start + frag_iter->len;
2799 if ((copy = end - offset) > 0) {
2800 if (copy > len)
2801 copy = len;
2802 if (skb_store_bits(frag_iter, offset - start,
2803 from, copy))
2804 goto fault;
2805 if ((len -= copy) == 0)
2806 return 0;
2807 offset += copy;
2808 from += copy;
Herbert Xu357b40a2005-04-19 22:30:14 -07002809 }
David S. Millerfbb398a2009-06-09 00:18:59 -07002810 start = end;
Herbert Xu357b40a2005-04-19 22:30:14 -07002811 }
2812 if (!len)
2813 return 0;
2814
2815fault:
2816 return -EFAULT;
2817}
Herbert Xu357b40a2005-04-19 22:30:14 -07002818EXPORT_SYMBOL(skb_store_bits);
2819
Linus Torvalds1da177e2005-04-16 15:20:36 -07002820/* Checksum skb data. */
Daniel Borkmann2817a332013-10-30 11:50:51 +01002821__wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
2822 __wsum csum, const struct skb_checksum_ops *ops)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002823{
David S. Miller1a028e52007-04-27 15:21:23 -07002824 int start = skb_headlen(skb);
2825 int i, copy = start - offset;
David S. Millerfbb398a2009-06-09 00:18:59 -07002826 struct sk_buff *frag_iter;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002827 int pos = 0;
2828
2829 /* Checksum header. */
2830 if (copy > 0) {
2831 if (copy > len)
2832 copy = len;
Matteo Croce2544af02019-05-29 17:13:48 +02002833 csum = INDIRECT_CALL_1(ops->update, csum_partial_ext,
2834 skb->data + offset, copy, csum);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002835 if ((len -= copy) == 0)
2836 return csum;
2837 offset += copy;
2838 pos = copy;
2839 }
2840
2841 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
David S. Miller1a028e52007-04-27 15:21:23 -07002842 int end;
Eric Dumazet51c56b02012-04-05 11:35:15 +02002843 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
Linus Torvalds1da177e2005-04-16 15:20:36 -07002844
Ilpo Järvinen547b7922008-07-25 21:43:18 -07002845 WARN_ON(start > offset + len);
David S. Miller1a028e52007-04-27 15:21:23 -07002846
Eric Dumazet51c56b02012-04-05 11:35:15 +02002847 end = start + skb_frag_size(frag);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002848 if ((copy = end - offset) > 0) {
Willem de Bruijnc613c202017-07-31 08:15:47 -04002849 u32 p_off, p_len, copied;
2850 struct page *p;
Al Viro44bb9362006-11-14 21:36:14 -08002851 __wsum csum2;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002852 u8 *vaddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002853
2854 if (copy > len)
2855 copy = len;
Willem de Bruijnc613c202017-07-31 08:15:47 -04002856
2857 skb_frag_foreach_page(frag,
Jonathan Lemonb54c9d52019-07-30 07:40:33 -07002858 skb_frag_off(frag) + offset - start,
Willem de Bruijnc613c202017-07-31 08:15:47 -04002859 copy, p, p_off, p_len, copied) {
2860 vaddr = kmap_atomic(p);
Matteo Croce2544af02019-05-29 17:13:48 +02002861 csum2 = INDIRECT_CALL_1(ops->update,
2862 csum_partial_ext,
2863 vaddr + p_off, p_len, 0);
Willem de Bruijnc613c202017-07-31 08:15:47 -04002864 kunmap_atomic(vaddr);
Matteo Croce2544af02019-05-29 17:13:48 +02002865 csum = INDIRECT_CALL_1(ops->combine,
2866 csum_block_add_ext, csum,
2867 csum2, pos, p_len);
Willem de Bruijnc613c202017-07-31 08:15:47 -04002868 pos += p_len;
2869 }
2870
Linus Torvalds1da177e2005-04-16 15:20:36 -07002871 if (!(len -= copy))
2872 return csum;
2873 offset += copy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002874 }
David S. Miller1a028e52007-04-27 15:21:23 -07002875 start = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002876 }
2877
David S. Millerfbb398a2009-06-09 00:18:59 -07002878 skb_walk_frags(skb, frag_iter) {
2879 int end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002880
David S. Millerfbb398a2009-06-09 00:18:59 -07002881 WARN_ON(start > offset + len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002882
David S. Millerfbb398a2009-06-09 00:18:59 -07002883 end = start + frag_iter->len;
2884 if ((copy = end - offset) > 0) {
2885 __wsum csum2;
2886 if (copy > len)
2887 copy = len;
Daniel Borkmann2817a332013-10-30 11:50:51 +01002888 csum2 = __skb_checksum(frag_iter, offset - start,
2889 copy, 0, ops);
Matteo Croce2544af02019-05-29 17:13:48 +02002890 csum = INDIRECT_CALL_1(ops->combine, csum_block_add_ext,
2891 csum, csum2, pos, copy);
David S. Millerfbb398a2009-06-09 00:18:59 -07002892 if ((len -= copy) == 0)
2893 return csum;
2894 offset += copy;
2895 pos += copy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002896 }
David S. Millerfbb398a2009-06-09 00:18:59 -07002897 start = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002898 }
Kris Katterjohn09a62662006-01-08 22:24:28 -08002899 BUG_ON(len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002900
2901 return csum;
2902}
Daniel Borkmann2817a332013-10-30 11:50:51 +01002903EXPORT_SYMBOL(__skb_checksum);
2904
2905__wsum skb_checksum(const struct sk_buff *skb, int offset,
2906 int len, __wsum csum)
2907{
2908 const struct skb_checksum_ops ops = {
Daniel Borkmanncea80ea2013-11-04 17:10:25 +01002909 .update = csum_partial_ext,
Daniel Borkmann2817a332013-10-30 11:50:51 +01002910 .combine = csum_block_add_ext,
2911 };
2912
2913 return __skb_checksum(skb, offset, len, csum, &ops);
2914}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002915EXPORT_SYMBOL(skb_checksum);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002916
2917/* Both of above in one bottle. */
2918
Al Viro81d77662006-11-14 21:37:33 -08002919__wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
Al Viro8d5930d2020-07-10 20:07:10 -04002920 u8 *to, int len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002921{
David S. Miller1a028e52007-04-27 15:21:23 -07002922 int start = skb_headlen(skb);
2923 int i, copy = start - offset;
David S. Millerfbb398a2009-06-09 00:18:59 -07002924 struct sk_buff *frag_iter;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002925 int pos = 0;
Al Viro8d5930d2020-07-10 20:07:10 -04002926 __wsum csum = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002927
2928 /* Copy header. */
2929 if (copy > 0) {
2930 if (copy > len)
2931 copy = len;
2932 csum = csum_partial_copy_nocheck(skb->data + offset, to,
Al Virocc44c172020-07-11 00:12:07 -04002933 copy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002934 if ((len -= copy) == 0)
2935 return csum;
2936 offset += copy;
2937 to += copy;
2938 pos = copy;
2939 }
2940
2941 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
David S. Miller1a028e52007-04-27 15:21:23 -07002942 int end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002943
Ilpo Järvinen547b7922008-07-25 21:43:18 -07002944 WARN_ON(start > offset + len);
David S. Miller1a028e52007-04-27 15:21:23 -07002945
Eric Dumazet9e903e02011-10-18 21:00:24 +00002946 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002947 if ((copy = end - offset) > 0) {
Willem de Bruijnc613c202017-07-31 08:15:47 -04002948 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2949 u32 p_off, p_len, copied;
2950 struct page *p;
Al Viro50842052006-11-14 21:36:34 -08002951 __wsum csum2;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002952 u8 *vaddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002953
2954 if (copy > len)
2955 copy = len;
Willem de Bruijnc613c202017-07-31 08:15:47 -04002956
2957 skb_frag_foreach_page(frag,
Jonathan Lemonb54c9d52019-07-30 07:40:33 -07002958 skb_frag_off(frag) + offset - start,
Willem de Bruijnc613c202017-07-31 08:15:47 -04002959 copy, p, p_off, p_len, copied) {
2960 vaddr = kmap_atomic(p);
2961 csum2 = csum_partial_copy_nocheck(vaddr + p_off,
2962 to + copied,
Al Virocc44c172020-07-11 00:12:07 -04002963 p_len);
Willem de Bruijnc613c202017-07-31 08:15:47 -04002964 kunmap_atomic(vaddr);
2965 csum = csum_block_add(csum, csum2, pos);
2966 pos += p_len;
2967 }
2968
Linus Torvalds1da177e2005-04-16 15:20:36 -07002969 if (!(len -= copy))
2970 return csum;
2971 offset += copy;
2972 to += copy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002973 }
David S. Miller1a028e52007-04-27 15:21:23 -07002974 start = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002975 }
2976
David S. Millerfbb398a2009-06-09 00:18:59 -07002977 skb_walk_frags(skb, frag_iter) {
2978 __wsum csum2;
2979 int end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002980
David S. Millerfbb398a2009-06-09 00:18:59 -07002981 WARN_ON(start > offset + len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002982
David S. Millerfbb398a2009-06-09 00:18:59 -07002983 end = start + frag_iter->len;
2984 if ((copy = end - offset) > 0) {
2985 if (copy > len)
2986 copy = len;
2987 csum2 = skb_copy_and_csum_bits(frag_iter,
2988 offset - start,
Al Viro8d5930d2020-07-10 20:07:10 -04002989 to, copy);
David S. Millerfbb398a2009-06-09 00:18:59 -07002990 csum = csum_block_add(csum, csum2, pos);
2991 if ((len -= copy) == 0)
2992 return csum;
2993 offset += copy;
2994 to += copy;
2995 pos += copy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002996 }
David S. Millerfbb398a2009-06-09 00:18:59 -07002997 start = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002998 }
Kris Katterjohn09a62662006-01-08 22:24:28 -08002999 BUG_ON(len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003000 return csum;
3001}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003002EXPORT_SYMBOL(skb_copy_and_csum_bits);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003003
Cong Wang49f8e832018-11-08 14:05:42 -08003004__sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len)
3005{
3006 __sum16 sum;
3007
3008 sum = csum_fold(skb_checksum(skb, 0, len, skb->csum));
Cong Wang14641932018-11-26 09:31:26 -08003009 /* See comments in __skb_checksum_complete(). */
Cong Wang49f8e832018-11-08 14:05:42 -08003010 if (likely(!sum)) {
3011 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
3012 !skb->csum_complete_sw)
Cong Wang7fe50ac2018-11-12 14:47:18 -08003013 netdev_rx_csum_fault(skb->dev, skb);
Cong Wang49f8e832018-11-08 14:05:42 -08003014 }
3015 if (!skb_shared(skb))
3016 skb->csum_valid = !sum;
3017 return sum;
3018}
3019EXPORT_SYMBOL(__skb_checksum_complete_head);
3020
Cong Wang14641932018-11-26 09:31:26 -08003021/* This function assumes skb->csum already holds pseudo header's checksum,
3022 * which has been changed from the hardware checksum, for example, by
3023 * __skb_checksum_validate_complete(). And, the original skb->csum must
3024 * have been validated unsuccessfully for CHECKSUM_COMPLETE case.
3025 *
3026 * It returns non-zero if the recomputed checksum is still invalid, otherwise
3027 * zero. The new checksum is stored back into skb->csum unless the skb is
3028 * shared.
3029 */
Cong Wang49f8e832018-11-08 14:05:42 -08003030__sum16 __skb_checksum_complete(struct sk_buff *skb)
3031{
3032 __wsum csum;
3033 __sum16 sum;
3034
3035 csum = skb_checksum(skb, 0, skb->len, 0);
3036
Cong Wang49f8e832018-11-08 14:05:42 -08003037 sum = csum_fold(csum_add(skb->csum, csum));
Cong Wang14641932018-11-26 09:31:26 -08003038 /* This check is inverted, because we already knew the hardware
3039 * checksum is invalid before calling this function. So, if the
3040 * re-computed checksum is valid instead, then we have a mismatch
3041 * between the original skb->csum and skb_checksum(). This means either
3042 * the original hardware checksum is incorrect or we screw up skb->csum
3043 * when moving skb->data around.
3044 */
Cong Wang49f8e832018-11-08 14:05:42 -08003045 if (likely(!sum)) {
3046 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
3047 !skb->csum_complete_sw)
Cong Wang7fe50ac2018-11-12 14:47:18 -08003048 netdev_rx_csum_fault(skb->dev, skb);
Cong Wang49f8e832018-11-08 14:05:42 -08003049 }
3050
3051 if (!skb_shared(skb)) {
3052 /* Save full packet checksum */
3053 skb->csum = csum;
3054 skb->ip_summed = CHECKSUM_COMPLETE;
3055 skb->csum_complete_sw = 1;
3056 skb->csum_valid = !sum;
3057 }
3058
3059 return sum;
3060}
3061EXPORT_SYMBOL(__skb_checksum_complete);
3062
Davide Caratti96178132017-05-18 15:44:37 +02003063static __wsum warn_crc32c_csum_update(const void *buff, int len, __wsum sum)
3064{
3065 net_warn_ratelimited(
3066 "%s: attempt to compute crc32c without libcrc32c.ko\n",
3067 __func__);
3068 return 0;
3069}
3070
3071static __wsum warn_crc32c_csum_combine(__wsum csum, __wsum csum2,
3072 int offset, int len)
3073{
3074 net_warn_ratelimited(
3075 "%s: attempt to compute crc32c without libcrc32c.ko\n",
3076 __func__);
3077 return 0;
3078}
3079
3080static const struct skb_checksum_ops default_crc32c_ops = {
3081 .update = warn_crc32c_csum_update,
3082 .combine = warn_crc32c_csum_combine,
3083};
3084
3085const struct skb_checksum_ops *crc32c_csum_stub __read_mostly =
3086 &default_crc32c_ops;
3087EXPORT_SYMBOL(crc32c_csum_stub);
3088
Thomas Grafaf2806f2013-12-13 15:22:17 +01003089 /**
3090 * skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy()
3091 * @from: source buffer
3092 *
3093 * Calculates the amount of linear headroom needed in the 'to' skb passed
3094 * into skb_zerocopy().
3095 */
3096unsigned int
3097skb_zerocopy_headlen(const struct sk_buff *from)
3098{
3099 unsigned int hlen = 0;
3100
3101 if (!from->head_frag ||
3102 skb_headlen(from) < L1_CACHE_BYTES ||
Pravin B Shelara17ad092021-07-15 16:59:00 -07003103 skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS) {
Thomas Grafaf2806f2013-12-13 15:22:17 +01003104 hlen = skb_headlen(from);
Pravin B Shelara17ad092021-07-15 16:59:00 -07003105 if (!hlen)
3106 hlen = from->len;
3107 }
Thomas Grafaf2806f2013-12-13 15:22:17 +01003108
3109 if (skb_has_frag_list(from))
3110 hlen = from->len;
3111
3112 return hlen;
3113}
3114EXPORT_SYMBOL_GPL(skb_zerocopy_headlen);
3115
3116/**
3117 * skb_zerocopy - Zero copy skb to skb
3118 * @to: destination buffer
Masanari Iida7fceb4d2014-01-29 01:05:28 +09003119 * @from: source buffer
Thomas Grafaf2806f2013-12-13 15:22:17 +01003120 * @len: number of bytes to copy from source buffer
3121 * @hlen: size of linear headroom in destination buffer
3122 *
3123 * Copies up to `len` bytes from `from` to `to` by creating references
3124 * to the frags in the source buffer.
3125 *
3126 * The `hlen` as calculated by skb_zerocopy_headlen() specifies the
3127 * headroom in the `to` buffer.
Zoltan Kiss36d5fe62014-03-26 22:37:45 +00003128 *
3129 * Return value:
3130 * 0: everything is OK
3131 * -ENOMEM: couldn't orphan frags of @from due to lack of memory
3132 * -EFAULT: skb_copy_bits() found some problem with skb geometry
Thomas Grafaf2806f2013-12-13 15:22:17 +01003133 */
Zoltan Kiss36d5fe62014-03-26 22:37:45 +00003134int
3135skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen)
Thomas Grafaf2806f2013-12-13 15:22:17 +01003136{
3137 int i, j = 0;
3138 int plen = 0; /* length of skb->head fragment */
Zoltan Kiss36d5fe62014-03-26 22:37:45 +00003139 int ret;
Thomas Grafaf2806f2013-12-13 15:22:17 +01003140 struct page *page;
3141 unsigned int offset;
3142
3143 BUG_ON(!from->head_frag && !hlen);
3144
3145 /* dont bother with small payloads */
Zoltan Kiss36d5fe62014-03-26 22:37:45 +00003146 if (len <= skb_tailroom(to))
3147 return skb_copy_bits(from, 0, skb_put(to, len), len);
Thomas Grafaf2806f2013-12-13 15:22:17 +01003148
3149 if (hlen) {
Zoltan Kiss36d5fe62014-03-26 22:37:45 +00003150 ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen);
3151 if (unlikely(ret))
3152 return ret;
Thomas Grafaf2806f2013-12-13 15:22:17 +01003153 len -= hlen;
3154 } else {
3155 plen = min_t(int, skb_headlen(from), len);
3156 if (plen) {
3157 page = virt_to_head_page(from->head);
3158 offset = from->data - (unsigned char *)page_address(page);
3159 __skb_fill_page_desc(to, 0, page, offset, plen);
3160 get_page(page);
3161 j = 1;
3162 len -= plen;
3163 }
3164 }
3165
3166 to->truesize += len + plen;
3167 to->len += len + plen;
3168 to->data_len += len + plen;
3169
Zoltan Kiss36d5fe62014-03-26 22:37:45 +00003170 if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) {
3171 skb_tx_error(from);
3172 return -ENOMEM;
3173 }
Willem de Bruijn1f8b9772017-08-03 16:29:41 -04003174 skb_zerocopy_clone(to, from, GFP_ATOMIC);
Zoltan Kiss36d5fe62014-03-26 22:37:45 +00003175
Thomas Grafaf2806f2013-12-13 15:22:17 +01003176 for (i = 0; i < skb_shinfo(from)->nr_frags; i++) {
Matthew Wilcox (Oracle)d8e18a52019-07-22 20:08:26 -07003177 int size;
3178
Thomas Grafaf2806f2013-12-13 15:22:17 +01003179 if (!len)
3180 break;
3181 skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i];
Matthew Wilcox (Oracle)d8e18a52019-07-22 20:08:26 -07003182 size = min_t(int, skb_frag_size(&skb_shinfo(to)->frags[j]),
3183 len);
3184 skb_frag_size_set(&skb_shinfo(to)->frags[j], size);
3185 len -= size;
Thomas Grafaf2806f2013-12-13 15:22:17 +01003186 skb_frag_ref(to, j);
3187 j++;
3188 }
3189 skb_shinfo(to)->nr_frags = j;
Zoltan Kiss36d5fe62014-03-26 22:37:45 +00003190
3191 return 0;
Thomas Grafaf2806f2013-12-13 15:22:17 +01003192}
3193EXPORT_SYMBOL_GPL(skb_zerocopy);
3194
Linus Torvalds1da177e2005-04-16 15:20:36 -07003195void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
3196{
Al Virod3bc23e2006-11-14 21:24:49 -08003197 __wsum csum;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003198 long csstart;
3199
Patrick McHardy84fa7932006-08-29 16:44:56 -07003200 if (skb->ip_summed == CHECKSUM_PARTIAL)
Michał Mirosław55508d62010-12-14 15:24:08 +00003201 csstart = skb_checksum_start_offset(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003202 else
3203 csstart = skb_headlen(skb);
3204
Kris Katterjohn09a62662006-01-08 22:24:28 -08003205 BUG_ON(csstart > skb_headlen(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003206
Arnaldo Carvalho de Melod626f622007-03-27 18:55:52 -03003207 skb_copy_from_linear_data(skb, to, csstart);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003208
3209 csum = 0;
3210 if (csstart != skb->len)
3211 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
Al Viro8d5930d2020-07-10 20:07:10 -04003212 skb->len - csstart);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003213
Patrick McHardy84fa7932006-08-29 16:44:56 -07003214 if (skb->ip_summed == CHECKSUM_PARTIAL) {
Al Viroff1dcad2006-11-20 18:07:29 -08003215 long csstuff = csstart + skb->csum_offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003216
Al Virod3bc23e2006-11-14 21:24:49 -08003217 *((__sum16 *)(to + csstuff)) = csum_fold(csum);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003218 }
3219}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003220EXPORT_SYMBOL(skb_copy_and_csum_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003221
3222/**
3223 * skb_dequeue - remove from the head of the queue
3224 * @list: list to dequeue from
3225 *
3226 * Remove the head of the list. The list lock is taken so the function
3227 * may be used safely with other locking list functions. The head item is
3228 * returned or %NULL if the list is empty.
3229 */
3230
3231struct sk_buff *skb_dequeue(struct sk_buff_head *list)
3232{
3233 unsigned long flags;
3234 struct sk_buff *result;
3235
3236 spin_lock_irqsave(&list->lock, flags);
3237 result = __skb_dequeue(list);
3238 spin_unlock_irqrestore(&list->lock, flags);
3239 return result;
3240}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003241EXPORT_SYMBOL(skb_dequeue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003242
3243/**
3244 * skb_dequeue_tail - remove from the tail of the queue
3245 * @list: list to dequeue from
3246 *
3247 * Remove the tail of the list. The list lock is taken so the function
3248 * may be used safely with other locking list functions. The tail item is
3249 * returned or %NULL if the list is empty.
3250 */
3251struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
3252{
3253 unsigned long flags;
3254 struct sk_buff *result;
3255
3256 spin_lock_irqsave(&list->lock, flags);
3257 result = __skb_dequeue_tail(list);
3258 spin_unlock_irqrestore(&list->lock, flags);
3259 return result;
3260}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003261EXPORT_SYMBOL(skb_dequeue_tail);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003262
3263/**
3264 * skb_queue_purge - empty a list
3265 * @list: list to empty
3266 *
3267 * Delete all buffers on an &sk_buff list. Each buffer is removed from
3268 * the list and one reference dropped. This function takes the list
3269 * lock and is atomic with respect to other list locking functions.
3270 */
3271void skb_queue_purge(struct sk_buff_head *list)
3272{
3273 struct sk_buff *skb;
3274 while ((skb = skb_dequeue(list)) != NULL)
3275 kfree_skb(skb);
3276}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003277EXPORT_SYMBOL(skb_queue_purge);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003278
3279/**
Yaogong Wang9f5afea2016-09-07 14:49:28 -07003280 * skb_rbtree_purge - empty a skb rbtree
3281 * @root: root of the rbtree to empty
Peter Oskolkov385114d2018-08-02 23:34:38 +00003282 * Return value: the sum of truesizes of all purged skbs.
Yaogong Wang9f5afea2016-09-07 14:49:28 -07003283 *
3284 * Delete all buffers on an &sk_buff rbtree. Each buffer is removed from
3285 * the list and one reference dropped. This function does not take
3286 * any lock. Synchronization should be handled by the caller (e.g., TCP
3287 * out-of-order queue is protected by the socket lock).
3288 */
Peter Oskolkov385114d2018-08-02 23:34:38 +00003289unsigned int skb_rbtree_purge(struct rb_root *root)
Yaogong Wang9f5afea2016-09-07 14:49:28 -07003290{
Eric Dumazet7c905842017-09-23 12:39:12 -07003291 struct rb_node *p = rb_first(root);
Peter Oskolkov385114d2018-08-02 23:34:38 +00003292 unsigned int sum = 0;
Yaogong Wang9f5afea2016-09-07 14:49:28 -07003293
Eric Dumazet7c905842017-09-23 12:39:12 -07003294 while (p) {
3295 struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode);
3296
3297 p = rb_next(p);
3298 rb_erase(&skb->rbnode, root);
Peter Oskolkov385114d2018-08-02 23:34:38 +00003299 sum += skb->truesize;
Yaogong Wang9f5afea2016-09-07 14:49:28 -07003300 kfree_skb(skb);
Eric Dumazet7c905842017-09-23 12:39:12 -07003301 }
Peter Oskolkov385114d2018-08-02 23:34:38 +00003302 return sum;
Yaogong Wang9f5afea2016-09-07 14:49:28 -07003303}
3304
3305/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07003306 * skb_queue_head - queue a buffer at the list head
3307 * @list: list to use
3308 * @newsk: buffer to queue
3309 *
3310 * Queue a buffer at the start of the list. This function takes the
3311 * list lock and can be used safely with other locking &sk_buff functions
3312 * safely.
3313 *
3314 * A buffer cannot be placed on two lists at the same time.
3315 */
3316void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
3317{
3318 unsigned long flags;
3319
3320 spin_lock_irqsave(&list->lock, flags);
3321 __skb_queue_head(list, newsk);
3322 spin_unlock_irqrestore(&list->lock, flags);
3323}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003324EXPORT_SYMBOL(skb_queue_head);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003325
3326/**
3327 * skb_queue_tail - queue a buffer at the list tail
3328 * @list: list to use
3329 * @newsk: buffer to queue
3330 *
3331 * Queue a buffer at the tail of the list. This function takes the
3332 * list lock and can be used safely with other locking &sk_buff functions
3333 * safely.
3334 *
3335 * A buffer cannot be placed on two lists at the same time.
3336 */
3337void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
3338{
3339 unsigned long flags;
3340
3341 spin_lock_irqsave(&list->lock, flags);
3342 __skb_queue_tail(list, newsk);
3343 spin_unlock_irqrestore(&list->lock, flags);
3344}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003345EXPORT_SYMBOL(skb_queue_tail);
David S. Miller8728b832005-08-09 19:25:21 -07003346
Linus Torvalds1da177e2005-04-16 15:20:36 -07003347/**
3348 * skb_unlink - remove a buffer from a list
3349 * @skb: buffer to remove
David S. Miller8728b832005-08-09 19:25:21 -07003350 * @list: list to use
Linus Torvalds1da177e2005-04-16 15:20:36 -07003351 *
David S. Miller8728b832005-08-09 19:25:21 -07003352 * Remove a packet from a list. The list locks are taken and this
3353 * function is atomic with respect to other list locked calls
Linus Torvalds1da177e2005-04-16 15:20:36 -07003354 *
David S. Miller8728b832005-08-09 19:25:21 -07003355 * You must know what list the SKB is on.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003356 */
David S. Miller8728b832005-08-09 19:25:21 -07003357void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003358{
David S. Miller8728b832005-08-09 19:25:21 -07003359 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003360
David S. Miller8728b832005-08-09 19:25:21 -07003361 spin_lock_irqsave(&list->lock, flags);
3362 __skb_unlink(skb, list);
3363 spin_unlock_irqrestore(&list->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003364}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003365EXPORT_SYMBOL(skb_unlink);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003366
Linus Torvalds1da177e2005-04-16 15:20:36 -07003367/**
3368 * skb_append - append a buffer
3369 * @old: buffer to insert after
3370 * @newsk: buffer to insert
David S. Miller8728b832005-08-09 19:25:21 -07003371 * @list: list to use
Linus Torvalds1da177e2005-04-16 15:20:36 -07003372 *
3373 * Place a packet after a given packet in a list. The list locks are taken
3374 * and this function is atomic with respect to other list locked calls.
3375 * A buffer cannot be placed on two lists at the same time.
3376 */
David S. Miller8728b832005-08-09 19:25:21 -07003377void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003378{
3379 unsigned long flags;
3380
David S. Miller8728b832005-08-09 19:25:21 -07003381 spin_lock_irqsave(&list->lock, flags);
Gerrit Renker7de6c032008-04-14 00:05:09 -07003382 __skb_queue_after(list, old, newsk);
David S. Miller8728b832005-08-09 19:25:21 -07003383 spin_unlock_irqrestore(&list->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003384}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003385EXPORT_SYMBOL(skb_append);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003386
Linus Torvalds1da177e2005-04-16 15:20:36 -07003387static inline void skb_split_inside_header(struct sk_buff *skb,
3388 struct sk_buff* skb1,
3389 const u32 len, const int pos)
3390{
3391 int i;
3392
Arnaldo Carvalho de Melod626f622007-03-27 18:55:52 -03003393 skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
3394 pos - len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003395 /* And move data appendix as is. */
3396 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
3397 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
3398
3399 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
3400 skb_shinfo(skb)->nr_frags = 0;
3401 skb1->data_len = skb->data_len;
3402 skb1->len += skb1->data_len;
3403 skb->data_len = 0;
3404 skb->len = len;
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07003405 skb_set_tail_pointer(skb, len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003406}
3407
3408static inline void skb_split_no_header(struct sk_buff *skb,
3409 struct sk_buff* skb1,
3410 const u32 len, int pos)
3411{
3412 int i, k = 0;
3413 const int nfrags = skb_shinfo(skb)->nr_frags;
3414
3415 skb_shinfo(skb)->nr_frags = 0;
3416 skb1->len = skb1->data_len = skb->len - len;
3417 skb->len = len;
3418 skb->data_len = len - pos;
3419
3420 for (i = 0; i < nfrags; i++) {
Eric Dumazet9e903e02011-10-18 21:00:24 +00003421 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003422
3423 if (pos + size > len) {
3424 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
3425
3426 if (pos < len) {
3427 /* Split frag.
3428 * We have two variants in this case:
3429 * 1. Move all the frag to the second
3430 * part, if it is possible. F.e.
3431 * this approach is mandatory for TUX,
3432 * where splitting is expensive.
3433 * 2. Split is accurately. We make this.
3434 */
Ian Campbellea2ab692011-08-22 23:44:58 +00003435 skb_frag_ref(skb, i);
Jonathan Lemonb54c9d52019-07-30 07:40:33 -07003436 skb_frag_off_add(&skb_shinfo(skb1)->frags[0], len - pos);
Eric Dumazet9e903e02011-10-18 21:00:24 +00003437 skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos);
3438 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003439 skb_shinfo(skb)->nr_frags++;
3440 }
3441 k++;
3442 } else
3443 skb_shinfo(skb)->nr_frags++;
3444 pos += size;
3445 }
3446 skb_shinfo(skb1)->nr_frags = k;
3447}
3448
3449/**
3450 * skb_split - Split fragmented skb to two parts at length len.
3451 * @skb: the buffer to split
3452 * @skb1: the buffer to receive the second part
3453 * @len: new length for skb
3454 */
3455void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
3456{
3457 int pos = skb_headlen(skb);
Talal Ahmad9b65b172021-11-02 22:58:44 -04003458 const int zc_flags = SKBFL_SHARED_FRAG | SKBFL_PURE_ZEROCOPY;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003459
Talal Ahmad9b65b172021-11-02 22:58:44 -04003460 skb_shinfo(skb1)->flags |= skb_shinfo(skb)->flags & zc_flags;
Willem de Bruijn1f8b9772017-08-03 16:29:41 -04003461 skb_zerocopy_clone(skb1, skb, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003462 if (len < pos) /* Split line is inside header. */
3463 skb_split_inside_header(skb, skb1, len, pos);
3464 else /* Second chunk has no header, nothing to copy. */
3465 skb_split_no_header(skb, skb1, len, pos);
3466}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003467EXPORT_SYMBOL(skb_split);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003468
Ilpo Järvinen9f782db2008-11-25 13:57:01 -08003469/* Shifting from/to a cloned skb is a no-go.
3470 *
3471 * Caller cannot keep skb_shinfo related pointers past calling here!
3472 */
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003473static int skb_prepare_for_shift(struct sk_buff *skb)
3474{
Eric Dumazetc4777ef2021-11-01 17:45:55 -07003475 return skb_unclone_keeptruesize(skb, GFP_ATOMIC);
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003476}
3477
3478/**
3479 * skb_shift - Shifts paged data partially from skb to another
3480 * @tgt: buffer into which tail data gets added
3481 * @skb: buffer from which the paged data comes from
3482 * @shiftlen: shift up to this many bytes
3483 *
3484 * Attempts to shift up to shiftlen worth of bytes, which may be less than
Feng King20e994a2011-11-21 01:47:11 +00003485 * the length of the skb, from skb to tgt. Returns number bytes shifted.
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003486 * It's up to caller to free skb if everything was shifted.
3487 *
3488 * If @tgt runs out of frags, the whole operation is aborted.
3489 *
3490 * Skb cannot include anything else but paged data while tgt is allowed
3491 * to have non-paged data as well.
3492 *
3493 * TODO: full sized shift could be optimized but that would need
3494 * specialized skb free'er to handle frags without up-to-date nr_frags.
3495 */
3496int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
3497{
3498 int from, to, merge, todo;
Matthew Wilcox (Oracle)d8e18a52019-07-22 20:08:26 -07003499 skb_frag_t *fragfrom, *fragto;
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003500
3501 BUG_ON(shiftlen > skb->len);
Eric Dumazetf8071cd2016-11-15 12:51:50 -08003502
3503 if (skb_headlen(skb))
3504 return 0;
Willem de Bruijn1f8b9772017-08-03 16:29:41 -04003505 if (skb_zcopy(tgt) || skb_zcopy(skb))
3506 return 0;
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003507
3508 todo = shiftlen;
3509 from = 0;
3510 to = skb_shinfo(tgt)->nr_frags;
3511 fragfrom = &skb_shinfo(skb)->frags[from];
3512
3513 /* Actual merge is delayed until the point when we know we can
3514 * commit all, so that we don't have to undo partial changes
3515 */
3516 if (!to ||
Ian Campbellea2ab692011-08-22 23:44:58 +00003517 !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom),
Jonathan Lemonb54c9d52019-07-30 07:40:33 -07003518 skb_frag_off(fragfrom))) {
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003519 merge = -1;
3520 } else {
3521 merge = to - 1;
3522
Eric Dumazet9e903e02011-10-18 21:00:24 +00003523 todo -= skb_frag_size(fragfrom);
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003524 if (todo < 0) {
3525 if (skb_prepare_for_shift(skb) ||
3526 skb_prepare_for_shift(tgt))
3527 return 0;
3528
Ilpo Järvinen9f782db2008-11-25 13:57:01 -08003529 /* All previous frag pointers might be stale! */
3530 fragfrom = &skb_shinfo(skb)->frags[from];
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003531 fragto = &skb_shinfo(tgt)->frags[merge];
3532
Eric Dumazet9e903e02011-10-18 21:00:24 +00003533 skb_frag_size_add(fragto, shiftlen);
3534 skb_frag_size_sub(fragfrom, shiftlen);
Jonathan Lemonb54c9d52019-07-30 07:40:33 -07003535 skb_frag_off_add(fragfrom, shiftlen);
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003536
3537 goto onlymerged;
3538 }
3539
3540 from++;
3541 }
3542
3543 /* Skip full, not-fitting skb to avoid expensive operations */
3544 if ((shiftlen == skb->len) &&
3545 (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to))
3546 return 0;
3547
3548 if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt))
3549 return 0;
3550
3551 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
3552 if (to == MAX_SKB_FRAGS)
3553 return 0;
3554
3555 fragfrom = &skb_shinfo(skb)->frags[from];
3556 fragto = &skb_shinfo(tgt)->frags[to];
3557
Eric Dumazet9e903e02011-10-18 21:00:24 +00003558 if (todo >= skb_frag_size(fragfrom)) {
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003559 *fragto = *fragfrom;
Eric Dumazet9e903e02011-10-18 21:00:24 +00003560 todo -= skb_frag_size(fragfrom);
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003561 from++;
3562 to++;
3563
3564 } else {
Ian Campbellea2ab692011-08-22 23:44:58 +00003565 __skb_frag_ref(fragfrom);
Jonathan Lemonb54c9d52019-07-30 07:40:33 -07003566 skb_frag_page_copy(fragto, fragfrom);
3567 skb_frag_off_copy(fragto, fragfrom);
Eric Dumazet9e903e02011-10-18 21:00:24 +00003568 skb_frag_size_set(fragto, todo);
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003569
Jonathan Lemonb54c9d52019-07-30 07:40:33 -07003570 skb_frag_off_add(fragfrom, todo);
Eric Dumazet9e903e02011-10-18 21:00:24 +00003571 skb_frag_size_sub(fragfrom, todo);
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003572 todo = 0;
3573
3574 to++;
3575 break;
3576 }
3577 }
3578
3579 /* Ready to "commit" this state change to tgt */
3580 skb_shinfo(tgt)->nr_frags = to;
3581
3582 if (merge >= 0) {
3583 fragfrom = &skb_shinfo(skb)->frags[0];
3584 fragto = &skb_shinfo(tgt)->frags[merge];
3585
Eric Dumazet9e903e02011-10-18 21:00:24 +00003586 skb_frag_size_add(fragto, skb_frag_size(fragfrom));
Ilias Apalodimas6a5bcd82021-06-07 21:02:38 +02003587 __skb_frag_unref(fragfrom, skb->pp_recycle);
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003588 }
3589
3590 /* Reposition in the original skb */
3591 to = 0;
3592 while (from < skb_shinfo(skb)->nr_frags)
3593 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
3594 skb_shinfo(skb)->nr_frags = to;
3595
3596 BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);
3597
3598onlymerged:
3599 /* Most likely the tgt won't ever need its checksum anymore, skb on
3600 * the other hand might need it if it needs to be resent
3601 */
3602 tgt->ip_summed = CHECKSUM_PARTIAL;
3603 skb->ip_summed = CHECKSUM_PARTIAL;
3604
3605 /* Yak, is it really working this way? Some helper please? */
3606 skb->len -= shiftlen;
3607 skb->data_len -= shiftlen;
3608 skb->truesize -= shiftlen;
3609 tgt->len += shiftlen;
3610 tgt->data_len += shiftlen;
3611 tgt->truesize += shiftlen;
3612
3613 return shiftlen;
3614}
3615
Thomas Graf677e90e2005-06-23 20:59:51 -07003616/**
3617 * skb_prepare_seq_read - Prepare a sequential read of skb data
3618 * @skb: the buffer to read
3619 * @from: lower offset of data to be read
3620 * @to: upper offset of data to be read
3621 * @st: state variable
3622 *
3623 * Initializes the specified state variable. Must be called before
3624 * invoking skb_seq_read() for the first time.
3625 */
3626void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
3627 unsigned int to, struct skb_seq_state *st)
3628{
3629 st->lower_offset = from;
3630 st->upper_offset = to;
3631 st->root_skb = st->cur_skb = skb;
3632 st->frag_idx = st->stepped_offset = 0;
3633 st->frag_data = NULL;
Willem de Bruijn97550f62021-01-09 17:18:33 -05003634 st->frag_off = 0;
Thomas Graf677e90e2005-06-23 20:59:51 -07003635}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003636EXPORT_SYMBOL(skb_prepare_seq_read);
Thomas Graf677e90e2005-06-23 20:59:51 -07003637
3638/**
3639 * skb_seq_read - Sequentially read skb data
3640 * @consumed: number of bytes consumed by the caller so far
3641 * @data: destination pointer for data to be returned
3642 * @st: state variable
3643 *
Mathias Krausebc323832013-11-07 14:18:26 +01003644 * Reads a block of skb data at @consumed relative to the
Thomas Graf677e90e2005-06-23 20:59:51 -07003645 * lower offset specified to skb_prepare_seq_read(). Assigns
Mathias Krausebc323832013-11-07 14:18:26 +01003646 * the head of the data block to @data and returns the length
Thomas Graf677e90e2005-06-23 20:59:51 -07003647 * of the block or 0 if the end of the skb data or the upper
3648 * offset has been reached.
3649 *
3650 * The caller is not required to consume all of the data
Mathias Krausebc323832013-11-07 14:18:26 +01003651 * returned, i.e. @consumed is typically set to the number
Thomas Graf677e90e2005-06-23 20:59:51 -07003652 * of bytes already consumed and the next call to
3653 * skb_seq_read() will return the remaining part of the block.
3654 *
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003655 * Note 1: The size of each block of data returned can be arbitrary,
Masanari Iidae793c0f2014-09-04 23:44:36 +09003656 * this limitation is the cost for zerocopy sequential
Thomas Graf677e90e2005-06-23 20:59:51 -07003657 * reads of potentially non linear data.
3658 *
Randy Dunlapbc2cda12008-02-13 15:03:25 -08003659 * Note 2: Fragment lists within fragments are not implemented
Thomas Graf677e90e2005-06-23 20:59:51 -07003660 * at the moment, state->root_skb could be replaced with
3661 * a stack for this purpose.
3662 */
3663unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
3664 struct skb_seq_state *st)
3665{
3666 unsigned int block_limit, abs_offset = consumed + st->lower_offset;
3667 skb_frag_t *frag;
3668
Wedson Almeida Filhoaeb193e2013-06-23 23:33:48 -07003669 if (unlikely(abs_offset >= st->upper_offset)) {
3670 if (st->frag_data) {
3671 kunmap_atomic(st->frag_data);
3672 st->frag_data = NULL;
3673 }
Thomas Graf677e90e2005-06-23 20:59:51 -07003674 return 0;
Wedson Almeida Filhoaeb193e2013-06-23 23:33:48 -07003675 }
Thomas Graf677e90e2005-06-23 20:59:51 -07003676
3677next_skb:
Herbert Xu95e3b242009-01-29 16:07:52 -08003678 block_limit = skb_headlen(st->cur_skb) + st->stepped_offset;
Thomas Graf677e90e2005-06-23 20:59:51 -07003679
Thomas Chenault995b3372009-05-18 21:43:27 -07003680 if (abs_offset < block_limit && !st->frag_data) {
Herbert Xu95e3b242009-01-29 16:07:52 -08003681 *data = st->cur_skb->data + (abs_offset - st->stepped_offset);
Thomas Graf677e90e2005-06-23 20:59:51 -07003682 return block_limit - abs_offset;
3683 }
3684
3685 if (st->frag_idx == 0 && !st->frag_data)
3686 st->stepped_offset += skb_headlen(st->cur_skb);
3687
3688 while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
Willem de Bruijn97550f62021-01-09 17:18:33 -05003689 unsigned int pg_idx, pg_off, pg_sz;
Thomas Graf677e90e2005-06-23 20:59:51 -07003690
Willem de Bruijn97550f62021-01-09 17:18:33 -05003691 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
3692
3693 pg_idx = 0;
3694 pg_off = skb_frag_off(frag);
3695 pg_sz = skb_frag_size(frag);
3696
3697 if (skb_frag_must_loop(skb_frag_page(frag))) {
3698 pg_idx = (pg_off + st->frag_off) >> PAGE_SHIFT;
3699 pg_off = offset_in_page(pg_off + st->frag_off);
3700 pg_sz = min_t(unsigned int, pg_sz - st->frag_off,
3701 PAGE_SIZE - pg_off);
3702 }
3703
3704 block_limit = pg_sz + st->stepped_offset;
Thomas Graf677e90e2005-06-23 20:59:51 -07003705 if (abs_offset < block_limit) {
3706 if (!st->frag_data)
Willem de Bruijn97550f62021-01-09 17:18:33 -05003707 st->frag_data = kmap_atomic(skb_frag_page(frag) + pg_idx);
Thomas Graf677e90e2005-06-23 20:59:51 -07003708
Willem de Bruijn97550f62021-01-09 17:18:33 -05003709 *data = (u8 *)st->frag_data + pg_off +
Thomas Graf677e90e2005-06-23 20:59:51 -07003710 (abs_offset - st->stepped_offset);
3711
3712 return block_limit - abs_offset;
3713 }
3714
3715 if (st->frag_data) {
Eric Dumazet51c56b02012-04-05 11:35:15 +02003716 kunmap_atomic(st->frag_data);
Thomas Graf677e90e2005-06-23 20:59:51 -07003717 st->frag_data = NULL;
3718 }
3719
Willem de Bruijn97550f62021-01-09 17:18:33 -05003720 st->stepped_offset += pg_sz;
3721 st->frag_off += pg_sz;
3722 if (st->frag_off == skb_frag_size(frag)) {
3723 st->frag_off = 0;
3724 st->frag_idx++;
3725 }
Thomas Graf677e90e2005-06-23 20:59:51 -07003726 }
3727
Olaf Kirch5b5a60d2007-06-23 23:11:52 -07003728 if (st->frag_data) {
Eric Dumazet51c56b02012-04-05 11:35:15 +02003729 kunmap_atomic(st->frag_data);
Olaf Kirch5b5a60d2007-06-23 23:11:52 -07003730 st->frag_data = NULL;
3731 }
3732
David S. Miller21dc3302010-08-23 00:13:46 -07003733 if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) {
Shyam Iyer71b33462009-01-29 16:12:42 -08003734 st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
Thomas Graf677e90e2005-06-23 20:59:51 -07003735 st->frag_idx = 0;
3736 goto next_skb;
Shyam Iyer71b33462009-01-29 16:12:42 -08003737 } else if (st->cur_skb->next) {
3738 st->cur_skb = st->cur_skb->next;
Herbert Xu95e3b242009-01-29 16:07:52 -08003739 st->frag_idx = 0;
Thomas Graf677e90e2005-06-23 20:59:51 -07003740 goto next_skb;
3741 }
3742
3743 return 0;
3744}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003745EXPORT_SYMBOL(skb_seq_read);
Thomas Graf677e90e2005-06-23 20:59:51 -07003746
3747/**
3748 * skb_abort_seq_read - Abort a sequential read of skb data
3749 * @st: state variable
3750 *
3751 * Must be called if skb_seq_read() was not called until it
3752 * returned 0.
3753 */
3754void skb_abort_seq_read(struct skb_seq_state *st)
3755{
3756 if (st->frag_data)
Eric Dumazet51c56b02012-04-05 11:35:15 +02003757 kunmap_atomic(st->frag_data);
Thomas Graf677e90e2005-06-23 20:59:51 -07003758}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003759EXPORT_SYMBOL(skb_abort_seq_read);
Thomas Graf677e90e2005-06-23 20:59:51 -07003760
Thomas Graf3fc7e8a2005-06-23 21:00:17 -07003761#define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb))
3762
3763static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
3764 struct ts_config *conf,
3765 struct ts_state *state)
3766{
3767 return skb_seq_read(offset, text, TS_SKB_CB(state));
3768}
3769
3770static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
3771{
3772 skb_abort_seq_read(TS_SKB_CB(state));
3773}
3774
3775/**
3776 * skb_find_text - Find a text pattern in skb data
3777 * @skb: the buffer to look in
3778 * @from: search offset
3779 * @to: search limit
3780 * @config: textsearch configuration
Thomas Graf3fc7e8a2005-06-23 21:00:17 -07003781 *
3782 * Finds a pattern in the skb data according to the specified
3783 * textsearch configuration. Use textsearch_next() to retrieve
3784 * subsequent occurrences of the pattern. Returns the offset
3785 * to the first occurrence or UINT_MAX if no match was found.
3786 */
3787unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
Bojan Prtvar059a2442015-02-22 11:46:35 +01003788 unsigned int to, struct ts_config *config)
Thomas Graf3fc7e8a2005-06-23 21:00:17 -07003789{
Bojan Prtvar059a2442015-02-22 11:46:35 +01003790 struct ts_state state;
Phil Oesterf72b9482006-06-26 00:00:57 -07003791 unsigned int ret;
3792
Willem de Bruijnb228c9b2021-03-01 15:09:44 +00003793 BUILD_BUG_ON(sizeof(struct skb_seq_state) > sizeof(state.cb));
3794
Thomas Graf3fc7e8a2005-06-23 21:00:17 -07003795 config->get_next_block = skb_ts_get_next_block;
3796 config->finish = skb_ts_finish;
3797
Bojan Prtvar059a2442015-02-22 11:46:35 +01003798 skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state));
Thomas Graf3fc7e8a2005-06-23 21:00:17 -07003799
Bojan Prtvar059a2442015-02-22 11:46:35 +01003800 ret = textsearch_find(config, &state);
Phil Oesterf72b9482006-06-26 00:00:57 -07003801 return (ret <= to - from ? ret : UINT_MAX);
Thomas Graf3fc7e8a2005-06-23 21:00:17 -07003802}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003803EXPORT_SYMBOL(skb_find_text);
Thomas Graf3fc7e8a2005-06-23 21:00:17 -07003804
Hannes Frederic Sowabe12a1f2015-05-21 16:59:58 +02003805int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
3806 int offset, size_t size)
3807{
3808 int i = skb_shinfo(skb)->nr_frags;
3809
3810 if (skb_can_coalesce(skb, i, page, offset)) {
3811 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size);
3812 } else if (i < MAX_SKB_FRAGS) {
3813 get_page(page);
3814 skb_fill_page_desc(skb, i, page, offset, size);
3815 } else {
3816 return -EMSGSIZE;
3817 }
3818
3819 return 0;
3820}
3821EXPORT_SYMBOL_GPL(skb_append_pagefrags);
3822
Herbert Xucbb042f2006-03-20 22:43:56 -08003823/**
3824 * skb_pull_rcsum - pull skb and update receive checksum
3825 * @skb: buffer to update
Herbert Xucbb042f2006-03-20 22:43:56 -08003826 * @len: length of data pulled
3827 *
3828 * This function performs an skb_pull on the packet and updates
Urs Thuermannfee54fa2008-02-12 22:03:25 -08003829 * the CHECKSUM_COMPLETE checksum. It should be used on
Patrick McHardy84fa7932006-08-29 16:44:56 -07003830 * receive path processing instead of skb_pull unless you know
3831 * that the checksum difference is zero (e.g., a valid IP header)
3832 * or you are setting ip_summed to CHECKSUM_NONE.
Herbert Xucbb042f2006-03-20 22:43:56 -08003833 */
Johannes Bergaf728682017-06-16 14:29:22 +02003834void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
Herbert Xucbb042f2006-03-20 22:43:56 -08003835{
Pravin B Shelar31b33df2015-09-28 17:24:25 -07003836 unsigned char *data = skb->data;
3837
Herbert Xucbb042f2006-03-20 22:43:56 -08003838 BUG_ON(len > skb->len);
Pravin B Shelar31b33df2015-09-28 17:24:25 -07003839 __skb_pull(skb, len);
3840 skb_postpull_rcsum(skb, data, len);
3841 return skb->data;
Herbert Xucbb042f2006-03-20 22:43:56 -08003842}
Arnaldo Carvalho de Melof94691a2006-03-20 22:47:55 -08003843EXPORT_SYMBOL_GPL(skb_pull_rcsum);
3844
Yonghong Song13acc942018-03-21 16:31:03 -07003845static inline skb_frag_t skb_head_frag_to_page_desc(struct sk_buff *frag_skb)
3846{
3847 skb_frag_t head_frag;
3848 struct page *page;
3849
3850 page = virt_to_head_page(frag_skb->head);
Matthew Wilcox (Oracle)d8e18a52019-07-22 20:08:26 -07003851 __skb_frag_set_page(&head_frag, page);
Jonathan Lemonb54c9d52019-07-30 07:40:33 -07003852 skb_frag_off_set(&head_frag, frag_skb->data -
3853 (unsigned char *)page_address(page));
Matthew Wilcox (Oracle)d8e18a52019-07-22 20:08:26 -07003854 skb_frag_size_set(&head_frag, skb_headlen(frag_skb));
Yonghong Song13acc942018-03-21 16:31:03 -07003855 return head_frag;
3856}
3857
Steffen Klassert3a1296a2020-01-25 11:26:44 +01003858struct sk_buff *skb_segment_list(struct sk_buff *skb,
3859 netdev_features_t features,
3860 unsigned int offset)
3861{
3862 struct sk_buff *list_skb = skb_shinfo(skb)->frag_list;
3863 unsigned int tnl_hlen = skb_tnl_header_len(skb);
3864 unsigned int delta_truesize = 0;
3865 unsigned int delta_len = 0;
3866 struct sk_buff *tail = NULL;
Dongseok Yi53475c52021-01-08 11:28:38 +09003867 struct sk_buff *nskb, *tmp;
3868 int err;
Steffen Klassert3a1296a2020-01-25 11:26:44 +01003869
3870 skb_push(skb, -skb_network_offset(skb) + offset);
3871
3872 skb_shinfo(skb)->frag_list = NULL;
3873
3874 do {
3875 nskb = list_skb;
3876 list_skb = list_skb->next;
3877
Dongseok Yi53475c52021-01-08 11:28:38 +09003878 err = 0;
3879 if (skb_shared(nskb)) {
3880 tmp = skb_clone(nskb, GFP_ATOMIC);
3881 if (tmp) {
3882 consume_skb(nskb);
3883 nskb = tmp;
3884 err = skb_unclone(nskb, GFP_ATOMIC);
3885 } else {
3886 err = -ENOMEM;
3887 }
3888 }
3889
Steffen Klassert3a1296a2020-01-25 11:26:44 +01003890 if (!tail)
3891 skb->next = nskb;
3892 else
3893 tail->next = nskb;
3894
Dongseok Yi53475c52021-01-08 11:28:38 +09003895 if (unlikely(err)) {
3896 nskb->next = list_skb;
3897 goto err_linearize;
3898 }
3899
Steffen Klassert3a1296a2020-01-25 11:26:44 +01003900 tail = nskb;
3901
3902 delta_len += nskb->len;
3903 delta_truesize += nskb->truesize;
3904
3905 skb_push(nskb, -skb_network_offset(nskb) + offset);
3906
Florian Westphalcf673ed2020-03-30 18:51:29 +02003907 skb_release_head_state(nskb);
Colin Ian Kingc645fe9b2021-09-02 23:56:23 +01003908 __copy_skb_header(nskb, skb);
Steffen Klassert3a1296a2020-01-25 11:26:44 +01003909
3910 skb_headers_offset_update(nskb, skb_headroom(nskb) - skb_headroom(skb));
3911 skb_copy_from_linear_data_offset(skb, -tnl_hlen,
3912 nskb->data - tnl_hlen,
3913 offset + tnl_hlen);
3914
3915 if (skb_needs_linearize(nskb, features) &&
3916 __skb_linearize(nskb))
3917 goto err_linearize;
3918
3919 } while (list_skb);
3920
3921 skb->truesize = skb->truesize - delta_truesize;
3922 skb->data_len = skb->data_len - delta_len;
3923 skb->len = skb->len - delta_len;
3924
3925 skb_gso_reset(skb);
3926
3927 skb->prev = tail;
3928
3929 if (skb_needs_linearize(skb, features) &&
3930 __skb_linearize(skb))
3931 goto err_linearize;
3932
3933 skb_get(skb);
3934
3935 return skb;
3936
3937err_linearize:
3938 kfree_skb_list(skb->next);
3939 skb->next = NULL;
3940 return ERR_PTR(-ENOMEM);
3941}
3942EXPORT_SYMBOL_GPL(skb_segment_list);
3943
Herbert Xuf4c50d92006-06-22 03:02:40 -07003944/**
3945 * skb_segment - Perform protocol segmentation on skb.
Michael S. Tsirkindf5771f2014-03-10 18:29:19 +02003946 * @head_skb: buffer to segment
Herbert Xu576a30e2006-06-27 13:22:38 -07003947 * @features: features for the output path (see dev->features)
Herbert Xuf4c50d92006-06-22 03:02:40 -07003948 *
3949 * This function performs segmentation on the given skb. It returns
Ben Hutchings4c821d72008-04-13 21:52:48 -07003950 * a pointer to the first in a list of new skbs for the segments.
3951 * In case of error it returns ERR_PTR(err).
Herbert Xuf4c50d92006-06-22 03:02:40 -07003952 */
Michael S. Tsirkindf5771f2014-03-10 18:29:19 +02003953struct sk_buff *skb_segment(struct sk_buff *head_skb,
3954 netdev_features_t features)
Herbert Xuf4c50d92006-06-22 03:02:40 -07003955{
3956 struct sk_buff *segs = NULL;
3957 struct sk_buff *tail = NULL;
Michael S. Tsirkin1a4ceda2014-03-10 19:27:59 +02003958 struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list;
Michael S. Tsirkindf5771f2014-03-10 18:29:19 +02003959 skb_frag_t *frag = skb_shinfo(head_skb)->frags;
3960 unsigned int mss = skb_shinfo(head_skb)->gso_size;
3961 unsigned int doffset = head_skb->data - skb_mac_header(head_skb);
Michael S. Tsirkin1fd819e2014-03-10 19:28:08 +02003962 struct sk_buff *frag_skb = head_skb;
Herbert Xuf4c50d92006-06-22 03:02:40 -07003963 unsigned int offset = doffset;
Michael S. Tsirkindf5771f2014-03-10 18:29:19 +02003964 unsigned int tnl_hlen = skb_tnl_header_len(head_skb);
Alexander Duyck802ab552016-04-10 21:45:03 -04003965 unsigned int partial_segs = 0;
Herbert Xuf4c50d92006-06-22 03:02:40 -07003966 unsigned int headroom;
Alexander Duyck802ab552016-04-10 21:45:03 -04003967 unsigned int len = head_skb->len;
Pravin B Shelarec5f0612013-03-07 09:28:01 +00003968 __be16 proto;
Alexander Duyck36c98382016-05-02 09:38:18 -07003969 bool csum, sg;
Michael S. Tsirkindf5771f2014-03-10 18:29:19 +02003970 int nfrags = skb_shinfo(head_skb)->nr_frags;
Herbert Xuf4c50d92006-06-22 03:02:40 -07003971 int err = -ENOMEM;
3972 int i = 0;
3973 int pos;
3974
Shmulik Ladkani3dcbdb12019-09-06 12:23:50 +03003975 if (list_skb && !list_skb->head_frag && skb_headlen(list_skb) &&
3976 (skb_shinfo(head_skb)->gso_type & SKB_GSO_DODGY)) {
3977 /* gso_size is untrusted, and we have a frag_list with a linear
3978 * non head_frag head.
3979 *
3980 * (we assume checking the first list_skb member suffices;
3981 * i.e if either of the list_skb members have non head_frag
3982 * head, then the first one has too).
3983 *
3984 * If head_skb's headlen does not fit requested gso_size, it
3985 * means that the frag_list members do NOT terminate on exact
3986 * gso_size boundaries. Hence we cannot perform skb_frag_t page
3987 * sharing. Therefore we must fallback to copying the frag_list
3988 * skbs; we do so by disabling SG.
3989 */
3990 if (mss != GSO_BY_FRAGS && mss != skb_headlen(head_skb))
3991 features &= ~NETIF_F_SG;
3992 }
3993
Wei-Chun Chao5882a072014-06-08 23:48:54 -07003994 __skb_push(head_skb, doffset);
Miaohe Lin2f631132020-08-01 17:36:05 +08003995 proto = skb_network_protocol(head_skb, NULL);
Pravin B Shelarec5f0612013-03-07 09:28:01 +00003996 if (unlikely(!proto))
3997 return ERR_PTR(-EINVAL);
3998
Alexander Duyck36c98382016-05-02 09:38:18 -07003999 sg = !!(features & NETIF_F_SG);
Alexander Duyckf245d072016-02-05 15:28:26 -08004000 csum = !!can_checksum_protocol(features, proto);
Tom Herbert7e2b10c2014-06-04 17:20:02 -07004001
Steffen Klassert07b26c92016-09-19 12:58:47 +02004002 if (sg && csum && (mss != GSO_BY_FRAGS)) {
4003 if (!(features & NETIF_F_GSO_PARTIAL)) {
4004 struct sk_buff *iter;
Ilan Tayari43170c42017-04-19 21:26:07 +03004005 unsigned int frag_len;
Steffen Klassert07b26c92016-09-19 12:58:47 +02004006
4007 if (!list_skb ||
4008 !net_gso_ok(features, skb_shinfo(head_skb)->gso_type))
4009 goto normal;
4010
Ilan Tayari43170c42017-04-19 21:26:07 +03004011 /* If we get here then all the required
4012 * GSO features except frag_list are supported.
4013 * Try to split the SKB to multiple GSO SKBs
4014 * with no frag_list.
4015 * Currently we can do that only when the buffers don't
4016 * have a linear part and all the buffers except
4017 * the last are of the same length.
Steffen Klassert07b26c92016-09-19 12:58:47 +02004018 */
Ilan Tayari43170c42017-04-19 21:26:07 +03004019 frag_len = list_skb->len;
Steffen Klassert07b26c92016-09-19 12:58:47 +02004020 skb_walk_frags(head_skb, iter) {
Ilan Tayari43170c42017-04-19 21:26:07 +03004021 if (frag_len != iter->len && iter->next)
4022 goto normal;
Ilan Tayarieaffadb2017-04-08 02:07:08 +03004023 if (skb_headlen(iter) && !iter->head_frag)
Steffen Klassert07b26c92016-09-19 12:58:47 +02004024 goto normal;
4025
4026 len -= iter->len;
4027 }
Ilan Tayari43170c42017-04-19 21:26:07 +03004028
4029 if (len != frag_len)
4030 goto normal;
Steffen Klassert07b26c92016-09-19 12:58:47 +02004031 }
4032
4033 /* GSO partial only requires that we trim off any excess that
4034 * doesn't fit into an MSS sized block, so take care of that
4035 * now.
4036 */
Alexander Duyck802ab552016-04-10 21:45:03 -04004037 partial_segs = len / mss;
Alexander Duyckd7fb5a82016-05-02 09:38:12 -07004038 if (partial_segs > 1)
4039 mss *= partial_segs;
4040 else
4041 partial_segs = 0;
Alexander Duyck802ab552016-04-10 21:45:03 -04004042 }
4043
Steffen Klassert07b26c92016-09-19 12:58:47 +02004044normal:
Michael S. Tsirkindf5771f2014-03-10 18:29:19 +02004045 headroom = skb_headroom(head_skb);
4046 pos = skb_headlen(head_skb);
Herbert Xuf4c50d92006-06-22 03:02:40 -07004047
4048 do {
4049 struct sk_buff *nskb;
Michael S. Tsirkin8cb19902014-03-10 18:29:04 +02004050 skb_frag_t *nskb_frag;
Herbert Xuc8884ed2006-10-29 15:59:41 -08004051 int hsize;
Herbert Xuf4c50d92006-06-22 03:02:40 -07004052 int size;
4053
Marcelo Ricardo Leitner3953c462016-06-02 15:05:40 -03004054 if (unlikely(mss == GSO_BY_FRAGS)) {
4055 len = list_skb->len;
4056 } else {
4057 len = head_skb->len - offset;
4058 if (len > mss)
4059 len = mss;
4060 }
Herbert Xuf4c50d92006-06-22 03:02:40 -07004061
Michael S. Tsirkindf5771f2014-03-10 18:29:19 +02004062 hsize = skb_headlen(head_skb) - offset;
Herbert Xuf4c50d92006-06-22 03:02:40 -07004063
Xin Longdbd50f22021-01-15 17:36:38 +08004064 if (hsize <= 0 && i >= nfrags && skb_headlen(list_skb) &&
Michael S. Tsirkin1a4ceda2014-03-10 19:27:59 +02004065 (skb_headlen(list_skb) == len || sg)) {
4066 BUG_ON(skb_headlen(list_skb) > len);
Herbert Xu89319d382008-12-15 23:26:06 -08004067
Herbert Xu9d8506c2013-11-21 11:10:04 -08004068 i = 0;
Michael S. Tsirkin1a4ceda2014-03-10 19:27:59 +02004069 nfrags = skb_shinfo(list_skb)->nr_frags;
4070 frag = skb_shinfo(list_skb)->frags;
Michael S. Tsirkin1fd819e2014-03-10 19:28:08 +02004071 frag_skb = list_skb;
Michael S. Tsirkin1a4ceda2014-03-10 19:27:59 +02004072 pos += skb_headlen(list_skb);
Herbert Xu9d8506c2013-11-21 11:10:04 -08004073
4074 while (pos < offset + len) {
4075 BUG_ON(i >= nfrags);
4076
Michael S. Tsirkin4e1beba2014-03-10 18:29:14 +02004077 size = skb_frag_size(frag);
Herbert Xu9d8506c2013-11-21 11:10:04 -08004078 if (pos + size > offset + len)
4079 break;
4080
4081 i++;
4082 pos += size;
Michael S. Tsirkin4e1beba2014-03-10 18:29:14 +02004083 frag++;
Herbert Xu9d8506c2013-11-21 11:10:04 -08004084 }
4085
Michael S. Tsirkin1a4ceda2014-03-10 19:27:59 +02004086 nskb = skb_clone(list_skb, GFP_ATOMIC);
4087 list_skb = list_skb->next;
Herbert Xu89319d382008-12-15 23:26:06 -08004088
4089 if (unlikely(!nskb))
4090 goto err;
4091
Herbert Xu9d8506c2013-11-21 11:10:04 -08004092 if (unlikely(pskb_trim(nskb, len))) {
4093 kfree_skb(nskb);
4094 goto err;
4095 }
4096
Alexander Duyckec47ea82012-05-04 14:26:56 +00004097 hsize = skb_end_offset(nskb);
Herbert Xu89319d382008-12-15 23:26:06 -08004098 if (skb_cow_head(nskb, doffset + headroom)) {
4099 kfree_skb(nskb);
4100 goto err;
4101 }
4102
Alexander Duyckec47ea82012-05-04 14:26:56 +00004103 nskb->truesize += skb_end_offset(nskb) - hsize;
Herbert Xu89319d382008-12-15 23:26:06 -08004104 skb_release_head_state(nskb);
4105 __skb_push(nskb, doffset);
4106 } else {
Paolo Abeni00b229f2021-01-19 17:56:56 +01004107 if (hsize < 0)
4108 hsize = 0;
Xin Longdbd50f22021-01-15 17:36:38 +08004109 if (hsize > len || !sg)
4110 hsize = len;
Xin Longdbd50f22021-01-15 17:36:38 +08004111
Mel Gormanc93bdd02012-07-31 16:44:19 -07004112 nskb = __alloc_skb(hsize + doffset + headroom,
Michael S. Tsirkindf5771f2014-03-10 18:29:19 +02004113 GFP_ATOMIC, skb_alloc_rx_flag(head_skb),
Mel Gormanc93bdd02012-07-31 16:44:19 -07004114 NUMA_NO_NODE);
Herbert Xu89319d382008-12-15 23:26:06 -08004115
4116 if (unlikely(!nskb))
4117 goto err;
4118
4119 skb_reserve(nskb, headroom);
4120 __skb_put(nskb, doffset);
4121 }
Herbert Xuf4c50d92006-06-22 03:02:40 -07004122
4123 if (segs)
4124 tail->next = nskb;
4125 else
4126 segs = nskb;
4127 tail = nskb;
4128
Michael S. Tsirkindf5771f2014-03-10 18:29:19 +02004129 __copy_skb_header(nskb, head_skb);
Herbert Xuf4c50d92006-06-22 03:02:40 -07004130
Eric Dumazet030737b2013-10-19 11:42:54 -07004131 skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom);
Vlad Yasevichfcdfe3a2014-07-31 10:33:06 -04004132 skb_reset_mac_len(nskb);
Pravin B Shelar68c33162013-02-14 14:02:41 +00004133
Michael S. Tsirkindf5771f2014-03-10 18:29:19 +02004134 skb_copy_from_linear_data_offset(head_skb, -tnl_hlen,
Pravin B Shelar68c33162013-02-14 14:02:41 +00004135 nskb->data - tnl_hlen,
4136 doffset + tnl_hlen);
Herbert Xu89319d382008-12-15 23:26:06 -08004137
Herbert Xu9d8506c2013-11-21 11:10:04 -08004138 if (nskb->len == len + doffset)
Simon Horman1cdbcb72013-05-19 15:46:49 +00004139 goto perform_csum_check;
Herbert Xu89319d382008-12-15 23:26:06 -08004140
Alexander Duyck7fbeffe2016-02-05 15:27:43 -08004141 if (!sg) {
Yadu Kishore1454c9f2020-03-17 14:08:38 +05304142 if (!csum) {
4143 if (!nskb->remcsum_offload)
4144 nskb->ip_summed = CHECKSUM_NONE;
4145 SKB_GSO_CB(nskb)->csum =
4146 skb_copy_and_csum_bits(head_skb, offset,
4147 skb_put(nskb,
4148 len),
Al Viro8d5930d2020-07-10 20:07:10 -04004149 len);
Yadu Kishore1454c9f2020-03-17 14:08:38 +05304150 SKB_GSO_CB(nskb)->csum_start =
4151 skb_headroom(nskb) + doffset;
4152 } else {
4153 skb_copy_bits(head_skb, offset,
4154 skb_put(nskb, len),
4155 len);
4156 }
Herbert Xuf4c50d92006-06-22 03:02:40 -07004157 continue;
4158 }
4159
Michael S. Tsirkin8cb19902014-03-10 18:29:04 +02004160 nskb_frag = skb_shinfo(nskb)->frags;
Herbert Xuf4c50d92006-06-22 03:02:40 -07004161
Michael S. Tsirkindf5771f2014-03-10 18:29:19 +02004162 skb_copy_from_linear_data_offset(head_skb, offset,
Arnaldo Carvalho de Melod626f622007-03-27 18:55:52 -03004163 skb_put(nskb, hsize), hsize);
Herbert Xuf4c50d92006-06-22 03:02:40 -07004164
Jonathan Lemon06b4feb2021-01-06 14:18:38 -08004165 skb_shinfo(nskb)->flags |= skb_shinfo(head_skb)->flags &
4166 SKBFL_SHARED_FRAG;
Eric Dumazetcef401d2013-01-25 20:34:37 +00004167
Willem de Bruijnbf5c25d2017-12-22 19:00:17 -05004168 if (skb_orphan_frags(frag_skb, GFP_ATOMIC) ||
4169 skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC))
4170 goto err;
4171
Herbert Xu9d8506c2013-11-21 11:10:04 -08004172 while (pos < offset + len) {
4173 if (i >= nfrags) {
Herbert Xu9d8506c2013-11-21 11:10:04 -08004174 i = 0;
Michael S. Tsirkin1a4ceda2014-03-10 19:27:59 +02004175 nfrags = skb_shinfo(list_skb)->nr_frags;
4176 frag = skb_shinfo(list_skb)->frags;
Michael S. Tsirkin1fd819e2014-03-10 19:28:08 +02004177 frag_skb = list_skb;
Yonghong Song13acc942018-03-21 16:31:03 -07004178 if (!skb_headlen(list_skb)) {
4179 BUG_ON(!nfrags);
4180 } else {
4181 BUG_ON(!list_skb->head_frag);
Herbert Xu9d8506c2013-11-21 11:10:04 -08004182
Yonghong Song13acc942018-03-21 16:31:03 -07004183 /* to make room for head_frag. */
4184 i--;
4185 frag--;
4186 }
Willem de Bruijnbf5c25d2017-12-22 19:00:17 -05004187 if (skb_orphan_frags(frag_skb, GFP_ATOMIC) ||
4188 skb_zerocopy_clone(nskb, frag_skb,
4189 GFP_ATOMIC))
4190 goto err;
4191
Michael S. Tsirkin1a4ceda2014-03-10 19:27:59 +02004192 list_skb = list_skb->next;
Herbert Xu9d8506c2013-11-21 11:10:04 -08004193 }
4194
4195 if (unlikely(skb_shinfo(nskb)->nr_frags >=
4196 MAX_SKB_FRAGS)) {
4197 net_warn_ratelimited(
4198 "skb_segment: too many frags: %u %u\n",
4199 pos, mss);
Eric Dumazetff907a12018-07-19 16:04:38 -07004200 err = -EINVAL;
Herbert Xu9d8506c2013-11-21 11:10:04 -08004201 goto err;
4202 }
4203
Yonghong Song13acc942018-03-21 16:31:03 -07004204 *nskb_frag = (i < 0) ? skb_head_frag_to_page_desc(frag_skb) : *frag;
Michael S. Tsirkin8cb19902014-03-10 18:29:04 +02004205 __skb_frag_ref(nskb_frag);
4206 size = skb_frag_size(nskb_frag);
Herbert Xuf4c50d92006-06-22 03:02:40 -07004207
4208 if (pos < offset) {
Jonathan Lemonb54c9d52019-07-30 07:40:33 -07004209 skb_frag_off_add(nskb_frag, offset - pos);
Michael S. Tsirkin8cb19902014-03-10 18:29:04 +02004210 skb_frag_size_sub(nskb_frag, offset - pos);
Herbert Xuf4c50d92006-06-22 03:02:40 -07004211 }
4212
Herbert Xu89319d382008-12-15 23:26:06 -08004213 skb_shinfo(nskb)->nr_frags++;
Herbert Xuf4c50d92006-06-22 03:02:40 -07004214
4215 if (pos + size <= offset + len) {
4216 i++;
Michael S. Tsirkin4e1beba2014-03-10 18:29:14 +02004217 frag++;
Herbert Xuf4c50d92006-06-22 03:02:40 -07004218 pos += size;
4219 } else {
Michael S. Tsirkin8cb19902014-03-10 18:29:04 +02004220 skb_frag_size_sub(nskb_frag, pos + size - (offset + len));
Herbert Xu89319d382008-12-15 23:26:06 -08004221 goto skip_fraglist;
Herbert Xuf4c50d92006-06-22 03:02:40 -07004222 }
4223
Michael S. Tsirkin8cb19902014-03-10 18:29:04 +02004224 nskb_frag++;
Herbert Xuf4c50d92006-06-22 03:02:40 -07004225 }
4226
Herbert Xu89319d382008-12-15 23:26:06 -08004227skip_fraglist:
Herbert Xuf4c50d92006-06-22 03:02:40 -07004228 nskb->data_len = len - hsize;
4229 nskb->len += nskb->data_len;
4230 nskb->truesize += nskb->data_len;
Pravin B Shelarec5f0612013-03-07 09:28:01 +00004231
Simon Horman1cdbcb72013-05-19 15:46:49 +00004232perform_csum_check:
Alexander Duyck7fbeffe2016-02-05 15:27:43 -08004233 if (!csum) {
Eric Dumazetff907a12018-07-19 16:04:38 -07004234 if (skb_has_shared_frag(nskb) &&
4235 __skb_linearize(nskb))
4236 goto err;
4237
Alexander Duyck7fbeffe2016-02-05 15:27:43 -08004238 if (!nskb->remcsum_offload)
4239 nskb->ip_summed = CHECKSUM_NONE;
Alexander Duyck76443452016-02-05 15:27:37 -08004240 SKB_GSO_CB(nskb)->csum =
4241 skb_checksum(nskb, doffset,
4242 nskb->len - doffset, 0);
Tom Herbert7e2b10c2014-06-04 17:20:02 -07004243 SKB_GSO_CB(nskb)->csum_start =
Alexander Duyck76443452016-02-05 15:27:37 -08004244 skb_headroom(nskb) + doffset;
Pravin B Shelarec5f0612013-03-07 09:28:01 +00004245 }
Michael S. Tsirkindf5771f2014-03-10 18:29:19 +02004246 } while ((offset += len) < head_skb->len);
Herbert Xuf4c50d92006-06-22 03:02:40 -07004247
Eric Dumazetbec3cfd2014-10-03 20:59:19 -07004248 /* Some callers want to get the end of the list.
4249 * Put it in segs->prev to avoid walking the list.
4250 * (see validate_xmit_skb_list() for example)
4251 */
4252 segs->prev = tail;
Toshiaki Makita432c8562014-10-27 10:30:51 -07004253
Alexander Duyck802ab552016-04-10 21:45:03 -04004254 if (partial_segs) {
Steffen Klassert07b26c92016-09-19 12:58:47 +02004255 struct sk_buff *iter;
Alexander Duyck802ab552016-04-10 21:45:03 -04004256 int type = skb_shinfo(head_skb)->gso_type;
Steffen Klassert07b26c92016-09-19 12:58:47 +02004257 unsigned short gso_size = skb_shinfo(head_skb)->gso_size;
Alexander Duyck802ab552016-04-10 21:45:03 -04004258
4259 /* Update type to add partial and then remove dodgy if set */
Steffen Klassert07b26c92016-09-19 12:58:47 +02004260 type |= (features & NETIF_F_GSO_PARTIAL) / NETIF_F_GSO_PARTIAL * SKB_GSO_PARTIAL;
Alexander Duyck802ab552016-04-10 21:45:03 -04004261 type &= ~SKB_GSO_DODGY;
4262
4263 /* Update GSO info and prepare to start updating headers on
4264 * our way back down the stack of protocols.
4265 */
Steffen Klassert07b26c92016-09-19 12:58:47 +02004266 for (iter = segs; iter; iter = iter->next) {
4267 skb_shinfo(iter)->gso_size = gso_size;
4268 skb_shinfo(iter)->gso_segs = partial_segs;
4269 skb_shinfo(iter)->gso_type = type;
4270 SKB_GSO_CB(iter)->data_offset = skb_headroom(iter) + doffset;
4271 }
4272
4273 if (tail->len - doffset <= gso_size)
4274 skb_shinfo(tail)->gso_size = 0;
4275 else if (tail != segs)
4276 skb_shinfo(tail)->gso_segs = DIV_ROUND_UP(tail->len - doffset, gso_size);
Alexander Duyck802ab552016-04-10 21:45:03 -04004277 }
4278
Toshiaki Makita432c8562014-10-27 10:30:51 -07004279 /* Following permits correct backpressure, for protocols
4280 * using skb_set_owner_w().
4281 * Idea is to tranfert ownership from head_skb to last segment.
4282 */
4283 if (head_skb->destructor == sock_wfree) {
4284 swap(tail->truesize, head_skb->truesize);
4285 swap(tail->destructor, head_skb->destructor);
4286 swap(tail->sk, head_skb->sk);
4287 }
Herbert Xuf4c50d92006-06-22 03:02:40 -07004288 return segs;
4289
4290err:
Eric Dumazet289dccb2013-12-20 14:29:08 -08004291 kfree_skb_list(segs);
Herbert Xuf4c50d92006-06-22 03:02:40 -07004292 return ERR_PTR(err);
4293}
Herbert Xuf4c50d92006-06-22 03:02:40 -07004294EXPORT_SYMBOL_GPL(skb_segment);
4295
Florian Westphaldf5042f2018-12-18 17:15:16 +01004296#ifdef CONFIG_SKB_EXTENSIONS
4297#define SKB_EXT_ALIGN_VALUE 8
4298#define SKB_EXT_CHUNKSIZEOF(x) (ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE)
4299
4300static const u8 skb_ext_type_len[] = {
4301#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
4302 [SKB_EXT_BRIDGE_NF] = SKB_EXT_CHUNKSIZEOF(struct nf_bridge_info),
4303#endif
Florian Westphal41650792018-12-18 17:15:27 +01004304#ifdef CONFIG_XFRM
4305 [SKB_EXT_SEC_PATH] = SKB_EXT_CHUNKSIZEOF(struct sec_path),
4306#endif
Paul Blakey95a72332019-09-04 16:56:37 +03004307#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
4308 [TC_SKB_EXT] = SKB_EXT_CHUNKSIZEOF(struct tc_skb_ext),
4309#endif
Mat Martineau3ee17bc2020-01-09 07:59:19 -08004310#if IS_ENABLED(CONFIG_MPTCP)
4311 [SKB_EXT_MPTCP] = SKB_EXT_CHUNKSIZEOF(struct mptcp_ext),
4312#endif
Jeremy Kerr78476d32021-10-29 11:01:44 +08004313#if IS_ENABLED(CONFIG_MCTP_FLOWS)
4314 [SKB_EXT_MCTP] = SKB_EXT_CHUNKSIZEOF(struct mctp_flow),
4315#endif
Florian Westphaldf5042f2018-12-18 17:15:16 +01004316};
4317
4318static __always_inline unsigned int skb_ext_total_length(void)
4319{
4320 return SKB_EXT_CHUNKSIZEOF(struct skb_ext) +
4321#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
4322 skb_ext_type_len[SKB_EXT_BRIDGE_NF] +
4323#endif
Florian Westphal41650792018-12-18 17:15:27 +01004324#ifdef CONFIG_XFRM
4325 skb_ext_type_len[SKB_EXT_SEC_PATH] +
4326#endif
Paul Blakey95a72332019-09-04 16:56:37 +03004327#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
4328 skb_ext_type_len[TC_SKB_EXT] +
4329#endif
Mat Martineau3ee17bc2020-01-09 07:59:19 -08004330#if IS_ENABLED(CONFIG_MPTCP)
4331 skb_ext_type_len[SKB_EXT_MPTCP] +
4332#endif
Jeremy Kerr78476d32021-10-29 11:01:44 +08004333#if IS_ENABLED(CONFIG_MCTP_FLOWS)
4334 skb_ext_type_len[SKB_EXT_MCTP] +
4335#endif
Florian Westphaldf5042f2018-12-18 17:15:16 +01004336 0;
4337}
4338
4339static void skb_extensions_init(void)
4340{
4341 BUILD_BUG_ON(SKB_EXT_NUM >= 8);
4342 BUILD_BUG_ON(skb_ext_total_length() > 255);
4343
4344 skbuff_ext_cache = kmem_cache_create("skbuff_ext_cache",
4345 SKB_EXT_ALIGN_VALUE * skb_ext_total_length(),
4346 0,
4347 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
4348 NULL);
4349}
4350#else
4351static void skb_extensions_init(void) {}
4352#endif
4353
Linus Torvalds1da177e2005-04-16 15:20:36 -07004354void __init skb_init(void)
4355{
Kees Cook79a8a642018-02-07 17:44:38 -08004356 skbuff_head_cache = kmem_cache_create_usercopy("skbuff_head_cache",
Linus Torvalds1da177e2005-04-16 15:20:36 -07004357 sizeof(struct sk_buff),
4358 0,
Alexey Dobriyane5d679f332006-08-26 19:25:52 -07004359 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
Kees Cook79a8a642018-02-07 17:44:38 -08004360 offsetof(struct sk_buff, cb),
4361 sizeof_field(struct sk_buff, cb),
Paul Mundt20c2df82007-07-20 10:11:58 +09004362 NULL);
David S. Millerd179cd12005-08-17 14:57:30 -07004363 skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
Eric Dumazetd0bf4a92014-09-29 13:29:15 -07004364 sizeof(struct sk_buff_fclones),
David S. Millerd179cd12005-08-17 14:57:30 -07004365 0,
Alexey Dobriyane5d679f332006-08-26 19:25:52 -07004366 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
Paul Mundt20c2df82007-07-20 10:11:58 +09004367 NULL);
Florian Westphaldf5042f2018-12-18 17:15:16 +01004368 skb_extensions_init();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004369}
4370
David S. Miller51c739d2007-10-30 21:29:29 -07004371static int
Jason A. Donenfeld48a1df62017-06-04 04:16:22 +02004372__skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len,
4373 unsigned int recursion_level)
David Howells716ea3a2007-04-02 20:19:53 -07004374{
David S. Miller1a028e52007-04-27 15:21:23 -07004375 int start = skb_headlen(skb);
4376 int i, copy = start - offset;
David S. Millerfbb398a2009-06-09 00:18:59 -07004377 struct sk_buff *frag_iter;
David Howells716ea3a2007-04-02 20:19:53 -07004378 int elt = 0;
4379
Jason A. Donenfeld48a1df62017-06-04 04:16:22 +02004380 if (unlikely(recursion_level >= 24))
4381 return -EMSGSIZE;
4382
David Howells716ea3a2007-04-02 20:19:53 -07004383 if (copy > 0) {
4384 if (copy > len)
4385 copy = len;
Jens Axboe642f149032007-10-24 11:20:47 +02004386 sg_set_buf(sg, skb->data + offset, copy);
David Howells716ea3a2007-04-02 20:19:53 -07004387 elt++;
4388 if ((len -= copy) == 0)
4389 return elt;
4390 offset += copy;
4391 }
4392
4393 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
David S. Miller1a028e52007-04-27 15:21:23 -07004394 int end;
David Howells716ea3a2007-04-02 20:19:53 -07004395
Ilpo Järvinen547b7922008-07-25 21:43:18 -07004396 WARN_ON(start > offset + len);
David S. Miller1a028e52007-04-27 15:21:23 -07004397
Eric Dumazet9e903e02011-10-18 21:00:24 +00004398 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
David Howells716ea3a2007-04-02 20:19:53 -07004399 if ((copy = end - offset) > 0) {
4400 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
Jason A. Donenfeld48a1df62017-06-04 04:16:22 +02004401 if (unlikely(elt && sg_is_last(&sg[elt - 1])))
4402 return -EMSGSIZE;
David Howells716ea3a2007-04-02 20:19:53 -07004403
4404 if (copy > len)
4405 copy = len;
Ian Campbellea2ab692011-08-22 23:44:58 +00004406 sg_set_page(&sg[elt], skb_frag_page(frag), copy,
Jonathan Lemonb54c9d52019-07-30 07:40:33 -07004407 skb_frag_off(frag) + offset - start);
David Howells716ea3a2007-04-02 20:19:53 -07004408 elt++;
4409 if (!(len -= copy))
4410 return elt;
4411 offset += copy;
4412 }
David S. Miller1a028e52007-04-27 15:21:23 -07004413 start = end;
David Howells716ea3a2007-04-02 20:19:53 -07004414 }
4415
David S. Millerfbb398a2009-06-09 00:18:59 -07004416 skb_walk_frags(skb, frag_iter) {
Jason A. Donenfeld48a1df62017-06-04 04:16:22 +02004417 int end, ret;
David Howells716ea3a2007-04-02 20:19:53 -07004418
David S. Millerfbb398a2009-06-09 00:18:59 -07004419 WARN_ON(start > offset + len);
David Howells716ea3a2007-04-02 20:19:53 -07004420
David S. Millerfbb398a2009-06-09 00:18:59 -07004421 end = start + frag_iter->len;
4422 if ((copy = end - offset) > 0) {
Jason A. Donenfeld48a1df62017-06-04 04:16:22 +02004423 if (unlikely(elt && sg_is_last(&sg[elt - 1])))
4424 return -EMSGSIZE;
4425
David S. Millerfbb398a2009-06-09 00:18:59 -07004426 if (copy > len)
4427 copy = len;
Jason A. Donenfeld48a1df62017-06-04 04:16:22 +02004428 ret = __skb_to_sgvec(frag_iter, sg+elt, offset - start,
4429 copy, recursion_level + 1);
4430 if (unlikely(ret < 0))
4431 return ret;
4432 elt += ret;
David S. Millerfbb398a2009-06-09 00:18:59 -07004433 if ((len -= copy) == 0)
4434 return elt;
4435 offset += copy;
David Howells716ea3a2007-04-02 20:19:53 -07004436 }
David S. Millerfbb398a2009-06-09 00:18:59 -07004437 start = end;
David Howells716ea3a2007-04-02 20:19:53 -07004438 }
4439 BUG_ON(len);
4440 return elt;
4441}
4442
Jason A. Donenfeld48a1df62017-06-04 04:16:22 +02004443/**
4444 * skb_to_sgvec - Fill a scatter-gather list from a socket buffer
4445 * @skb: Socket buffer containing the buffers to be mapped
4446 * @sg: The scatter-gather list to map into
4447 * @offset: The offset into the buffer's contents to start mapping
4448 * @len: Length of buffer space to be mapped
4449 *
4450 * Fill the specified scatter-gather list with mappings/pointers into a
4451 * region of the buffer space attached to a socket buffer. Returns either
4452 * the number of scatterlist items used, or -EMSGSIZE if the contents
4453 * could not fit.
4454 */
4455int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
4456{
4457 int nsg = __skb_to_sgvec(skb, sg, offset, len, 0);
4458
4459 if (nsg <= 0)
4460 return nsg;
4461
4462 sg_mark_end(&sg[nsg - 1]);
4463
4464 return nsg;
4465}
4466EXPORT_SYMBOL_GPL(skb_to_sgvec);
4467
Fan Du25a91d82014-01-18 09:54:23 +08004468/* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given
4469 * sglist without mark the sg which contain last skb data as the end.
4470 * So the caller can mannipulate sg list as will when padding new data after
4471 * the first call without calling sg_unmark_end to expend sg list.
4472 *
4473 * Scenario to use skb_to_sgvec_nomark:
4474 * 1. sg_init_table
4475 * 2. skb_to_sgvec_nomark(payload1)
4476 * 3. skb_to_sgvec_nomark(payload2)
4477 *
4478 * This is equivalent to:
4479 * 1. sg_init_table
4480 * 2. skb_to_sgvec(payload1)
4481 * 3. sg_unmark_end
4482 * 4. skb_to_sgvec(payload2)
4483 *
4484 * When mapping mutilple payload conditionally, skb_to_sgvec_nomark
4485 * is more preferable.
4486 */
4487int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
4488 int offset, int len)
4489{
Jason A. Donenfeld48a1df62017-06-04 04:16:22 +02004490 return __skb_to_sgvec(skb, sg, offset, len, 0);
Fan Du25a91d82014-01-18 09:54:23 +08004491}
4492EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark);
4493
David S. Miller51c739d2007-10-30 21:29:29 -07004494
David S. Miller51c739d2007-10-30 21:29:29 -07004495
David Howells716ea3a2007-04-02 20:19:53 -07004496/**
4497 * skb_cow_data - Check that a socket buffer's data buffers are writable
4498 * @skb: The socket buffer to check.
4499 * @tailbits: Amount of trailing space to be added
4500 * @trailer: Returned pointer to the skb where the @tailbits space begins
4501 *
4502 * Make sure that the data buffers attached to a socket buffer are
4503 * writable. If they are not, private copies are made of the data buffers
4504 * and the socket buffer is set to use these instead.
4505 *
4506 * If @tailbits is given, make sure that there is space to write @tailbits
4507 * bytes of data beyond current end of socket buffer. @trailer will be
4508 * set to point to the skb in which this space begins.
4509 *
4510 * The number of scatterlist elements required to completely map the
4511 * COW'd and extended socket buffer will be returned.
4512 */
4513int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
4514{
4515 int copyflag;
4516 int elt;
4517 struct sk_buff *skb1, **skb_p;
4518
4519 /* If skb is cloned or its head is paged, reallocate
4520 * head pulling out all the pages (pages are considered not writable
4521 * at the moment even if they are anonymous).
4522 */
4523 if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) &&
Miaohe Linc15fc192020-08-01 17:30:23 +08004524 !__pskb_pull_tail(skb, __skb_pagelen(skb)))
David Howells716ea3a2007-04-02 20:19:53 -07004525 return -ENOMEM;
4526
4527 /* Easy case. Most of packets will go this way. */
David S. Miller21dc3302010-08-23 00:13:46 -07004528 if (!skb_has_frag_list(skb)) {
David Howells716ea3a2007-04-02 20:19:53 -07004529 /* A little of trouble, not enough of space for trailer.
4530 * This should not happen, when stack is tuned to generate
4531 * good frames. OK, on miss we reallocate and reserve even more
4532 * space, 128 bytes is fair. */
4533
4534 if (skb_tailroom(skb) < tailbits &&
4535 pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC))
4536 return -ENOMEM;
4537
4538 /* Voila! */
4539 *trailer = skb;
4540 return 1;
4541 }
4542
4543 /* Misery. We are in troubles, going to mincer fragments... */
4544
4545 elt = 1;
4546 skb_p = &skb_shinfo(skb)->frag_list;
4547 copyflag = 0;
4548
4549 while ((skb1 = *skb_p) != NULL) {
4550 int ntail = 0;
4551
4552 /* The fragment is partially pulled by someone,
4553 * this can happen on input. Copy it and everything
4554 * after it. */
4555
4556 if (skb_shared(skb1))
4557 copyflag = 1;
4558
4559 /* If the skb is the last, worry about trailer. */
4560
4561 if (skb1->next == NULL && tailbits) {
4562 if (skb_shinfo(skb1)->nr_frags ||
David S. Miller21dc3302010-08-23 00:13:46 -07004563 skb_has_frag_list(skb1) ||
David Howells716ea3a2007-04-02 20:19:53 -07004564 skb_tailroom(skb1) < tailbits)
4565 ntail = tailbits + 128;
4566 }
4567
4568 if (copyflag ||
4569 skb_cloned(skb1) ||
4570 ntail ||
4571 skb_shinfo(skb1)->nr_frags ||
David S. Miller21dc3302010-08-23 00:13:46 -07004572 skb_has_frag_list(skb1)) {
David Howells716ea3a2007-04-02 20:19:53 -07004573 struct sk_buff *skb2;
4574
4575 /* Fuck, we are miserable poor guys... */
4576 if (ntail == 0)
4577 skb2 = skb_copy(skb1, GFP_ATOMIC);
4578 else
4579 skb2 = skb_copy_expand(skb1,
4580 skb_headroom(skb1),
4581 ntail,
4582 GFP_ATOMIC);
4583 if (unlikely(skb2 == NULL))
4584 return -ENOMEM;
4585
4586 if (skb1->sk)
4587 skb_set_owner_w(skb2, skb1->sk);
4588
4589 /* Looking around. Are we still alive?
4590 * OK, link new skb, drop old one */
4591
4592 skb2->next = skb1->next;
4593 *skb_p = skb2;
4594 kfree_skb(skb1);
4595 skb1 = skb2;
4596 }
4597 elt++;
4598 *trailer = skb1;
4599 skb_p = &skb1->next;
4600 }
4601
4602 return elt;
4603}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08004604EXPORT_SYMBOL_GPL(skb_cow_data);
David Howells716ea3a2007-04-02 20:19:53 -07004605
Eric Dumazetb1faf562010-05-31 23:44:05 -07004606static void sock_rmem_free(struct sk_buff *skb)
4607{
4608 struct sock *sk = skb->sk;
4609
4610 atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
4611}
4612
Soheil Hassas Yeganeh8605330a2017-03-18 17:02:59 -04004613static void skb_set_err_queue(struct sk_buff *skb)
4614{
4615 /* pkt_type of skbs received on local sockets is never PACKET_OUTGOING.
4616 * So, it is safe to (mis)use it to mark skbs on the error queue.
4617 */
4618 skb->pkt_type = PACKET_OUTGOING;
4619 BUILD_BUG_ON(PACKET_OUTGOING == 0);
4620}
4621
Eric Dumazetb1faf562010-05-31 23:44:05 -07004622/*
4623 * Note: We dont mem charge error packets (no sk_forward_alloc changes)
4624 */
4625int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb)
4626{
4627 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
Eric Dumazetebb3b782019-10-10 20:17:44 -07004628 (unsigned int)READ_ONCE(sk->sk_rcvbuf))
Eric Dumazetb1faf562010-05-31 23:44:05 -07004629 return -ENOMEM;
4630
4631 skb_orphan(skb);
4632 skb->sk = sk;
4633 skb->destructor = sock_rmem_free;
4634 atomic_add(skb->truesize, &sk->sk_rmem_alloc);
Soheil Hassas Yeganeh8605330a2017-03-18 17:02:59 -04004635 skb_set_err_queue(skb);
Eric Dumazetb1faf562010-05-31 23:44:05 -07004636
Eric Dumazetabb57ea2011-05-18 02:21:31 -04004637 /* before exiting rcu section, make sure dst is refcounted */
4638 skb_dst_force(skb);
4639
Eric Dumazetb1faf562010-05-31 23:44:05 -07004640 skb_queue_tail(&sk->sk_error_queue, skb);
4641 if (!sock_flag(sk, SOCK_DEAD))
Alexander Aringe3ae2362021-06-27 18:48:21 -04004642 sk_error_report(sk);
Eric Dumazetb1faf562010-05-31 23:44:05 -07004643 return 0;
4644}
4645EXPORT_SYMBOL(sock_queue_err_skb);
4646
Soheil Hassas Yeganeh83a1a1a2016-11-30 14:01:08 -05004647static bool is_icmp_err_skb(const struct sk_buff *skb)
4648{
4649 return skb && (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP ||
4650 SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP6);
4651}
4652
Willem de Bruijn364a9e92014-08-31 21:30:27 -04004653struct sk_buff *sock_dequeue_err_skb(struct sock *sk)
4654{
4655 struct sk_buff_head *q = &sk->sk_error_queue;
Soheil Hassas Yeganeh83a1a1a2016-11-30 14:01:08 -05004656 struct sk_buff *skb, *skb_next = NULL;
4657 bool icmp_next = false;
Eric Dumazet997d5c32015-02-18 05:47:55 -08004658 unsigned long flags;
Willem de Bruijn364a9e92014-08-31 21:30:27 -04004659
Eric Dumazet997d5c32015-02-18 05:47:55 -08004660 spin_lock_irqsave(&q->lock, flags);
Willem de Bruijn364a9e92014-08-31 21:30:27 -04004661 skb = __skb_dequeue(q);
Soheil Hassas Yeganeh38b25792017-06-02 12:38:22 -04004662 if (skb && (skb_next = skb_peek(q))) {
Soheil Hassas Yeganeh83a1a1a2016-11-30 14:01:08 -05004663 icmp_next = is_icmp_err_skb(skb_next);
Soheil Hassas Yeganeh38b25792017-06-02 12:38:22 -04004664 if (icmp_next)
Willem de Bruijn985f7332020-11-26 10:12:20 -05004665 sk->sk_err = SKB_EXT_ERR(skb_next)->ee.ee_errno;
Soheil Hassas Yeganeh38b25792017-06-02 12:38:22 -04004666 }
Eric Dumazet997d5c32015-02-18 05:47:55 -08004667 spin_unlock_irqrestore(&q->lock, flags);
Willem de Bruijn364a9e92014-08-31 21:30:27 -04004668
Soheil Hassas Yeganeh83a1a1a2016-11-30 14:01:08 -05004669 if (is_icmp_err_skb(skb) && !icmp_next)
4670 sk->sk_err = 0;
4671
4672 if (skb_next)
Alexander Aringe3ae2362021-06-27 18:48:21 -04004673 sk_error_report(sk);
Willem de Bruijn364a9e92014-08-31 21:30:27 -04004674
4675 return skb;
4676}
4677EXPORT_SYMBOL(sock_dequeue_err_skb);
4678
Alexander Duyckcab41c42014-09-10 18:05:26 -04004679/**
4680 * skb_clone_sk - create clone of skb, and take reference to socket
4681 * @skb: the skb to clone
4682 *
4683 * This function creates a clone of a buffer that holds a reference on
4684 * sk_refcnt. Buffers created via this function are meant to be
4685 * returned using sock_queue_err_skb, or free via kfree_skb.
4686 *
4687 * When passing buffers allocated with this function to sock_queue_err_skb
4688 * it is necessary to wrap the call with sock_hold/sock_put in order to
4689 * prevent the socket from being released prior to being enqueued on
4690 * the sk_error_queue.
4691 */
Alexander Duyck62bccb82014-09-04 13:31:35 -04004692struct sk_buff *skb_clone_sk(struct sk_buff *skb)
4693{
4694 struct sock *sk = skb->sk;
4695 struct sk_buff *clone;
4696
Reshetova, Elena41c6d652017-06-30 13:08:01 +03004697 if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt))
Alexander Duyck62bccb82014-09-04 13:31:35 -04004698 return NULL;
4699
4700 clone = skb_clone(skb, GFP_ATOMIC);
4701 if (!clone) {
4702 sock_put(sk);
4703 return NULL;
4704 }
4705
4706 clone->sk = sk;
4707 clone->destructor = sock_efree;
4708
4709 return clone;
4710}
4711EXPORT_SYMBOL(skb_clone_sk);
4712
Alexander Duyck37846ef2014-09-04 13:31:10 -04004713static void __skb_complete_tx_timestamp(struct sk_buff *skb,
4714 struct sock *sk,
Soheil Hassas Yeganeh4ef1b282017-03-18 17:03:00 -04004715 int tstype,
4716 bool opt_stats)
Patrick Ohlyac45f602009-02-12 05:03:37 +00004717{
Patrick Ohlyac45f602009-02-12 05:03:37 +00004718 struct sock_exterr_skb *serr;
Patrick Ohlyac45f602009-02-12 05:03:37 +00004719 int err;
4720
Soheil Hassas Yeganeh4ef1b282017-03-18 17:03:00 -04004721 BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb));
4722
Patrick Ohlyac45f602009-02-12 05:03:37 +00004723 serr = SKB_EXT_ERR(skb);
4724 memset(serr, 0, sizeof(*serr));
4725 serr->ee.ee_errno = ENOMSG;
4726 serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
Willem de Bruijne7fd2882014-08-04 22:11:48 -04004727 serr->ee.ee_info = tstype;
Soheil Hassas Yeganeh4ef1b282017-03-18 17:03:00 -04004728 serr->opt_stats = opt_stats;
Willem de Bruijn1862d622017-04-12 19:24:35 -04004729 serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0;
Willem de Bruijn4ed2d762014-08-04 22:11:49 -04004730 if (sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) {
Willem de Bruijn09c2d252014-08-04 22:11:47 -04004731 serr->ee.ee_data = skb_shinfo(skb)->tskey;
Eric Dumazet42f67ee2021-11-15 11:02:33 -08004732 if (sk_is_tcp(sk))
Willem de Bruijn4ed2d762014-08-04 22:11:49 -04004733 serr->ee.ee_data -= sk->sk_tskey;
4734 }
Eric Dumazet29030372010-05-29 00:20:48 -07004735
Patrick Ohlyac45f602009-02-12 05:03:37 +00004736 err = sock_queue_err_skb(sk, skb);
Eric Dumazet29030372010-05-29 00:20:48 -07004737
Patrick Ohlyac45f602009-02-12 05:03:37 +00004738 if (err)
4739 kfree_skb(skb);
4740}
Alexander Duyck37846ef2014-09-04 13:31:10 -04004741
Willem de Bruijnb245be12015-01-30 13:29:32 -05004742static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly)
4743{
4744 bool ret;
4745
4746 if (likely(sysctl_tstamp_allow_data || tsonly))
4747 return true;
4748
4749 read_lock_bh(&sk->sk_callback_lock);
4750 ret = sk->sk_socket && sk->sk_socket->file &&
4751 file_ns_capable(sk->sk_socket->file, &init_user_ns, CAP_NET_RAW);
4752 read_unlock_bh(&sk->sk_callback_lock);
4753 return ret;
4754}
4755
Alexander Duyck37846ef2014-09-04 13:31:10 -04004756void skb_complete_tx_timestamp(struct sk_buff *skb,
4757 struct skb_shared_hwtstamps *hwtstamps)
4758{
4759 struct sock *sk = skb->sk;
4760
Willem de Bruijnb245be12015-01-30 13:29:32 -05004761 if (!skb_may_tx_timestamp(sk, false))
Willem de Bruijn35b99df2017-12-13 14:41:06 -05004762 goto err;
Willem de Bruijnb245be12015-01-30 13:29:32 -05004763
Eric Dumazet9ac25fc2017-03-03 21:01:03 -08004764 /* Take a reference to prevent skb_orphan() from freeing the socket,
4765 * but only if the socket refcount is not zero.
4766 */
Reshetova, Elena41c6d652017-06-30 13:08:01 +03004767 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
Eric Dumazet9ac25fc2017-03-03 21:01:03 -08004768 *skb_hwtstamps(skb) = *hwtstamps;
Soheil Hassas Yeganeh4ef1b282017-03-18 17:03:00 -04004769 __skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND, false);
Eric Dumazet9ac25fc2017-03-03 21:01:03 -08004770 sock_put(sk);
Willem de Bruijn35b99df2017-12-13 14:41:06 -05004771 return;
Eric Dumazet9ac25fc2017-03-03 21:01:03 -08004772 }
Willem de Bruijn35b99df2017-12-13 14:41:06 -05004773
4774err:
4775 kfree_skb(skb);
Alexander Duyck37846ef2014-09-04 13:31:10 -04004776}
4777EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp);
4778
4779void __skb_tstamp_tx(struct sk_buff *orig_skb,
Yousuk Seunge7ed11e2021-01-20 12:41:55 -08004780 const struct sk_buff *ack_skb,
Alexander Duyck37846ef2014-09-04 13:31:10 -04004781 struct skb_shared_hwtstamps *hwtstamps,
4782 struct sock *sk, int tstype)
4783{
4784 struct sk_buff *skb;
Soheil Hassas Yeganeh4ef1b282017-03-18 17:03:00 -04004785 bool tsonly, opt_stats = false;
Alexander Duyck37846ef2014-09-04 13:31:10 -04004786
Willem de Bruijn3a8dd972015-03-11 15:43:55 -04004787 if (!sk)
4788 return;
4789
Miroslav Lichvarb50a5c72017-05-19 17:52:40 +02004790 if (!hwtstamps && !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TX_SWHW) &&
4791 skb_shinfo(orig_skb)->tx_flags & SKBTX_IN_PROGRESS)
4792 return;
4793
Willem de Bruijn3a8dd972015-03-11 15:43:55 -04004794 tsonly = sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TSONLY;
4795 if (!skb_may_tx_timestamp(sk, tsonly))
Alexander Duyck37846ef2014-09-04 13:31:10 -04004796 return;
4797
Francis Yan1c885802016-11-27 23:07:18 -08004798 if (tsonly) {
4799#ifdef CONFIG_INET
4800 if ((sk->sk_tsflags & SOF_TIMESTAMPING_OPT_STATS) &&
Eric Dumazet42f67ee2021-11-15 11:02:33 -08004801 sk_is_tcp(sk)) {
Yousuk Seunge7ed11e2021-01-20 12:41:55 -08004802 skb = tcp_get_timestamping_opt_stats(sk, orig_skb,
4803 ack_skb);
Soheil Hassas Yeganeh4ef1b282017-03-18 17:03:00 -04004804 opt_stats = true;
4805 } else
Francis Yan1c885802016-11-27 23:07:18 -08004806#endif
4807 skb = alloc_skb(0, GFP_ATOMIC);
4808 } else {
Willem de Bruijn49ca0d82015-01-30 13:29:31 -05004809 skb = skb_clone(orig_skb, GFP_ATOMIC);
Francis Yan1c885802016-11-27 23:07:18 -08004810 }
Alexander Duyck37846ef2014-09-04 13:31:10 -04004811 if (!skb)
4812 return;
4813
Willem de Bruijn49ca0d82015-01-30 13:29:31 -05004814 if (tsonly) {
Willem de Bruijnfff88032017-06-08 11:35:03 -04004815 skb_shinfo(skb)->tx_flags |= skb_shinfo(orig_skb)->tx_flags &
4816 SKBTX_ANY_TSTAMP;
Willem de Bruijn49ca0d82015-01-30 13:29:31 -05004817 skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey;
4818 }
4819
4820 if (hwtstamps)
4821 *skb_hwtstamps(skb) = *hwtstamps;
4822 else
4823 skb->tstamp = ktime_get_real();
4824
Soheil Hassas Yeganeh4ef1b282017-03-18 17:03:00 -04004825 __skb_complete_tx_timestamp(skb, sk, tstype, opt_stats);
Alexander Duyck37846ef2014-09-04 13:31:10 -04004826}
Willem de Bruijne7fd2882014-08-04 22:11:48 -04004827EXPORT_SYMBOL_GPL(__skb_tstamp_tx);
4828
4829void skb_tstamp_tx(struct sk_buff *orig_skb,
4830 struct skb_shared_hwtstamps *hwtstamps)
4831{
Yousuk Seunge7ed11e2021-01-20 12:41:55 -08004832 return __skb_tstamp_tx(orig_skb, NULL, hwtstamps, orig_skb->sk,
Willem de Bruijne7fd2882014-08-04 22:11:48 -04004833 SCM_TSTAMP_SND);
4834}
Patrick Ohlyac45f602009-02-12 05:03:37 +00004835EXPORT_SYMBOL_GPL(skb_tstamp_tx);
4836
Johannes Berg6e3e9392011-11-09 10:15:42 +01004837void skb_complete_wifi_ack(struct sk_buff *skb, bool acked)
4838{
4839 struct sock *sk = skb->sk;
4840 struct sock_exterr_skb *serr;
Eric Dumazetdd4f1072017-03-03 21:01:02 -08004841 int err = 1;
Johannes Berg6e3e9392011-11-09 10:15:42 +01004842
4843 skb->wifi_acked_valid = 1;
4844 skb->wifi_acked = acked;
4845
4846 serr = SKB_EXT_ERR(skb);
4847 memset(serr, 0, sizeof(*serr));
4848 serr->ee.ee_errno = ENOMSG;
4849 serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS;
4850
Eric Dumazetdd4f1072017-03-03 21:01:02 -08004851 /* Take a reference to prevent skb_orphan() from freeing the socket,
4852 * but only if the socket refcount is not zero.
4853 */
Reshetova, Elena41c6d652017-06-30 13:08:01 +03004854 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
Eric Dumazetdd4f1072017-03-03 21:01:02 -08004855 err = sock_queue_err_skb(sk, skb);
4856 sock_put(sk);
4857 }
Johannes Berg6e3e9392011-11-09 10:15:42 +01004858 if (err)
4859 kfree_skb(skb);
4860}
4861EXPORT_SYMBOL_GPL(skb_complete_wifi_ack);
4862
Rusty Russellf35d9d82008-02-04 23:49:54 -05004863/**
4864 * skb_partial_csum_set - set up and verify partial csum values for packet
4865 * @skb: the skb to set
4866 * @start: the number of bytes after skb->data to start checksumming.
4867 * @off: the offset from start to place the checksum.
4868 *
4869 * For untrusted partially-checksummed packets, we need to make sure the values
4870 * for skb->csum_start and skb->csum_offset are valid so we don't oops.
4871 *
4872 * This function checks and sets those values and skb->ip_summed: if this
4873 * returns false you should drop the packet.
4874 */
4875bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
4876{
Eric Dumazet52b5d6f2018-10-10 06:59:35 -07004877 u32 csum_end = (u32)start + (u32)off + sizeof(__sum16);
4878 u32 csum_start = skb_headroom(skb) + (u32)start;
4879
4880 if (unlikely(csum_start > U16_MAX || csum_end > skb_headlen(skb))) {
4881 net_warn_ratelimited("bad partial csum: csum=%u/%u headroom=%u headlen=%u\n",
4882 start, off, skb_headroom(skb), skb_headlen(skb));
Rusty Russellf35d9d82008-02-04 23:49:54 -05004883 return false;
4884 }
4885 skb->ip_summed = CHECKSUM_PARTIAL;
Eric Dumazet52b5d6f2018-10-10 06:59:35 -07004886 skb->csum_start = csum_start;
Rusty Russellf35d9d82008-02-04 23:49:54 -05004887 skb->csum_offset = off;
Jason Wange5d5dec2013-03-26 23:11:20 +00004888 skb_set_transport_header(skb, start);
Rusty Russellf35d9d82008-02-04 23:49:54 -05004889 return true;
4890}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08004891EXPORT_SYMBOL_GPL(skb_partial_csum_set);
Rusty Russellf35d9d82008-02-04 23:49:54 -05004892
Paul Durranted1f50c2014-01-09 10:02:46 +00004893static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len,
4894 unsigned int max)
4895{
4896 if (skb_headlen(skb) >= len)
4897 return 0;
4898
4899 /* If we need to pullup then pullup to the max, so we
4900 * won't need to do it again.
4901 */
4902 if (max > skb->len)
4903 max = skb->len;
4904
4905 if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL)
4906 return -ENOMEM;
4907
4908 if (skb_headlen(skb) < len)
4909 return -EPROTO;
4910
4911 return 0;
4912}
4913
Jan Beulichf9708b42014-03-11 13:56:05 +00004914#define MAX_TCP_HDR_LEN (15 * 4)
4915
4916static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb,
4917 typeof(IPPROTO_IP) proto,
4918 unsigned int off)
4919{
Kees Cook161d1792020-02-19 22:23:04 -08004920 int err;
Jan Beulichf9708b42014-03-11 13:56:05 +00004921
Kees Cook161d1792020-02-19 22:23:04 -08004922 switch (proto) {
Jan Beulichf9708b42014-03-11 13:56:05 +00004923 case IPPROTO_TCP:
4924 err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr),
4925 off + MAX_TCP_HDR_LEN);
4926 if (!err && !skb_partial_csum_set(skb, off,
4927 offsetof(struct tcphdr,
4928 check)))
4929 err = -EPROTO;
4930 return err ? ERR_PTR(err) : &tcp_hdr(skb)->check;
4931
4932 case IPPROTO_UDP:
4933 err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr),
4934 off + sizeof(struct udphdr));
4935 if (!err && !skb_partial_csum_set(skb, off,
4936 offsetof(struct udphdr,
4937 check)))
4938 err = -EPROTO;
4939 return err ? ERR_PTR(err) : &udp_hdr(skb)->check;
4940 }
4941
4942 return ERR_PTR(-EPROTO);
4943}
4944
Paul Durranted1f50c2014-01-09 10:02:46 +00004945/* This value should be large enough to cover a tagged ethernet header plus
4946 * maximally sized IP and TCP or UDP headers.
4947 */
4948#define MAX_IP_HDR_LEN 128
4949
Jan Beulichf9708b42014-03-11 13:56:05 +00004950static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate)
Paul Durranted1f50c2014-01-09 10:02:46 +00004951{
4952 unsigned int off;
4953 bool fragment;
Jan Beulichf9708b42014-03-11 13:56:05 +00004954 __sum16 *csum;
Paul Durranted1f50c2014-01-09 10:02:46 +00004955 int err;
4956
4957 fragment = false;
4958
4959 err = skb_maybe_pull_tail(skb,
4960 sizeof(struct iphdr),
4961 MAX_IP_HDR_LEN);
4962 if (err < 0)
4963 goto out;
4964
Miaohe Lin11f920d2020-08-06 19:57:18 +08004965 if (ip_is_fragment(ip_hdr(skb)))
Paul Durranted1f50c2014-01-09 10:02:46 +00004966 fragment = true;
4967
4968 off = ip_hdrlen(skb);
4969
4970 err = -EPROTO;
4971
4972 if (fragment)
4973 goto out;
4974
Jan Beulichf9708b42014-03-11 13:56:05 +00004975 csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off);
4976 if (IS_ERR(csum))
4977 return PTR_ERR(csum);
Paul Durranted1f50c2014-01-09 10:02:46 +00004978
Jan Beulichf9708b42014-03-11 13:56:05 +00004979 if (recalculate)
4980 *csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
4981 ip_hdr(skb)->daddr,
4982 skb->len - off,
4983 ip_hdr(skb)->protocol, 0);
Paul Durranted1f50c2014-01-09 10:02:46 +00004984 err = 0;
4985
4986out:
4987 return err;
4988}
4989
4990/* This value should be large enough to cover a tagged ethernet header plus
4991 * an IPv6 header, all options, and a maximal TCP or UDP header.
4992 */
4993#define MAX_IPV6_HDR_LEN 256
4994
4995#define OPT_HDR(type, skb, off) \
4996 (type *)(skb_network_header(skb) + (off))
4997
4998static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate)
4999{
5000 int err;
5001 u8 nexthdr;
5002 unsigned int off;
5003 unsigned int len;
5004 bool fragment;
5005 bool done;
Jan Beulichf9708b42014-03-11 13:56:05 +00005006 __sum16 *csum;
Paul Durranted1f50c2014-01-09 10:02:46 +00005007
5008 fragment = false;
5009 done = false;
5010
5011 off = sizeof(struct ipv6hdr);
5012
5013 err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN);
5014 if (err < 0)
5015 goto out;
5016
5017 nexthdr = ipv6_hdr(skb)->nexthdr;
5018
5019 len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len);
5020 while (off <= len && !done) {
5021 switch (nexthdr) {
5022 case IPPROTO_DSTOPTS:
5023 case IPPROTO_HOPOPTS:
5024 case IPPROTO_ROUTING: {
5025 struct ipv6_opt_hdr *hp;
5026
5027 err = skb_maybe_pull_tail(skb,
5028 off +
5029 sizeof(struct ipv6_opt_hdr),
5030 MAX_IPV6_HDR_LEN);
5031 if (err < 0)
5032 goto out;
5033
5034 hp = OPT_HDR(struct ipv6_opt_hdr, skb, off);
5035 nexthdr = hp->nexthdr;
5036 off += ipv6_optlen(hp);
5037 break;
5038 }
5039 case IPPROTO_AH: {
5040 struct ip_auth_hdr *hp;
5041
5042 err = skb_maybe_pull_tail(skb,
5043 off +
5044 sizeof(struct ip_auth_hdr),
5045 MAX_IPV6_HDR_LEN);
5046 if (err < 0)
5047 goto out;
5048
5049 hp = OPT_HDR(struct ip_auth_hdr, skb, off);
5050 nexthdr = hp->nexthdr;
5051 off += ipv6_authlen(hp);
5052 break;
5053 }
5054 case IPPROTO_FRAGMENT: {
5055 struct frag_hdr *hp;
5056
5057 err = skb_maybe_pull_tail(skb,
5058 off +
5059 sizeof(struct frag_hdr),
5060 MAX_IPV6_HDR_LEN);
5061 if (err < 0)
5062 goto out;
5063
5064 hp = OPT_HDR(struct frag_hdr, skb, off);
5065
5066 if (hp->frag_off & htons(IP6_OFFSET | IP6_MF))
5067 fragment = true;
5068
5069 nexthdr = hp->nexthdr;
5070 off += sizeof(struct frag_hdr);
5071 break;
5072 }
5073 default:
5074 done = true;
5075 break;
5076 }
5077 }
5078
5079 err = -EPROTO;
5080
5081 if (!done || fragment)
5082 goto out;
5083
Jan Beulichf9708b42014-03-11 13:56:05 +00005084 csum = skb_checksum_setup_ip(skb, nexthdr, off);
5085 if (IS_ERR(csum))
5086 return PTR_ERR(csum);
Paul Durranted1f50c2014-01-09 10:02:46 +00005087
Jan Beulichf9708b42014-03-11 13:56:05 +00005088 if (recalculate)
5089 *csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
5090 &ipv6_hdr(skb)->daddr,
5091 skb->len - off, nexthdr, 0);
Paul Durranted1f50c2014-01-09 10:02:46 +00005092 err = 0;
5093
5094out:
5095 return err;
5096}
5097
5098/**
5099 * skb_checksum_setup - set up partial checksum offset
5100 * @skb: the skb to set up
5101 * @recalculate: if true the pseudo-header checksum will be recalculated
5102 */
5103int skb_checksum_setup(struct sk_buff *skb, bool recalculate)
5104{
5105 int err;
5106
5107 switch (skb->protocol) {
5108 case htons(ETH_P_IP):
Jan Beulichf9708b42014-03-11 13:56:05 +00005109 err = skb_checksum_setup_ipv4(skb, recalculate);
Paul Durranted1f50c2014-01-09 10:02:46 +00005110 break;
5111
5112 case htons(ETH_P_IPV6):
5113 err = skb_checksum_setup_ipv6(skb, recalculate);
5114 break;
5115
5116 default:
5117 err = -EPROTO;
5118 break;
5119 }
5120
5121 return err;
5122}
5123EXPORT_SYMBOL(skb_checksum_setup);
5124
Linus Lüssing9afd85c2015-05-02 14:01:07 +02005125/**
5126 * skb_checksum_maybe_trim - maybe trims the given skb
5127 * @skb: the skb to check
5128 * @transport_len: the data length beyond the network header
5129 *
5130 * Checks whether the given skb has data beyond the given transport length.
5131 * If so, returns a cloned skb trimmed to this transport length.
5132 * Otherwise returns the provided skb. Returns NULL in error cases
5133 * (e.g. transport_len exceeds skb length or out-of-memory).
5134 *
Linus Lüssinga5169932015-08-13 05:54:07 +02005135 * Caller needs to set the skb transport header and free any returned skb if it
5136 * differs from the provided skb.
Linus Lüssing9afd85c2015-05-02 14:01:07 +02005137 */
5138static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb,
5139 unsigned int transport_len)
5140{
5141 struct sk_buff *skb_chk;
5142 unsigned int len = skb_transport_offset(skb) + transport_len;
5143 int ret;
5144
Linus Lüssinga5169932015-08-13 05:54:07 +02005145 if (skb->len < len)
Linus Lüssing9afd85c2015-05-02 14:01:07 +02005146 return NULL;
Linus Lüssinga5169932015-08-13 05:54:07 +02005147 else if (skb->len == len)
Linus Lüssing9afd85c2015-05-02 14:01:07 +02005148 return skb;
Linus Lüssing9afd85c2015-05-02 14:01:07 +02005149
5150 skb_chk = skb_clone(skb, GFP_ATOMIC);
Linus Lüssing9afd85c2015-05-02 14:01:07 +02005151 if (!skb_chk)
5152 return NULL;
5153
5154 ret = pskb_trim_rcsum(skb_chk, len);
5155 if (ret) {
5156 kfree_skb(skb_chk);
5157 return NULL;
5158 }
5159
5160 return skb_chk;
5161}
5162
5163/**
5164 * skb_checksum_trimmed - validate checksum of an skb
5165 * @skb: the skb to check
5166 * @transport_len: the data length beyond the network header
5167 * @skb_chkf: checksum function to use
5168 *
5169 * Applies the given checksum function skb_chkf to the provided skb.
5170 * Returns a checked and maybe trimmed skb. Returns NULL on error.
5171 *
5172 * If the skb has data beyond the given transport length, then a
5173 * trimmed & cloned skb is checked and returned.
5174 *
Linus Lüssinga5169932015-08-13 05:54:07 +02005175 * Caller needs to set the skb transport header and free any returned skb if it
5176 * differs from the provided skb.
Linus Lüssing9afd85c2015-05-02 14:01:07 +02005177 */
5178struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb,
5179 unsigned int transport_len,
5180 __sum16(*skb_chkf)(struct sk_buff *skb))
5181{
5182 struct sk_buff *skb_chk;
5183 unsigned int offset = skb_transport_offset(skb);
Linus Lüssingfcba67c2015-05-05 00:19:35 +02005184 __sum16 ret;
Linus Lüssing9afd85c2015-05-02 14:01:07 +02005185
5186 skb_chk = skb_checksum_maybe_trim(skb, transport_len);
5187 if (!skb_chk)
Linus Lüssinga5169932015-08-13 05:54:07 +02005188 goto err;
Linus Lüssing9afd85c2015-05-02 14:01:07 +02005189
Linus Lüssinga5169932015-08-13 05:54:07 +02005190 if (!pskb_may_pull(skb_chk, offset))
5191 goto err;
Linus Lüssing9afd85c2015-05-02 14:01:07 +02005192
Linus Lüssing9b368812016-02-24 04:21:42 +01005193 skb_pull_rcsum(skb_chk, offset);
Linus Lüssing9afd85c2015-05-02 14:01:07 +02005194 ret = skb_chkf(skb_chk);
Linus Lüssing9b368812016-02-24 04:21:42 +01005195 skb_push_rcsum(skb_chk, offset);
Linus Lüssing9afd85c2015-05-02 14:01:07 +02005196
Linus Lüssinga5169932015-08-13 05:54:07 +02005197 if (ret)
5198 goto err;
Linus Lüssing9afd85c2015-05-02 14:01:07 +02005199
5200 return skb_chk;
Linus Lüssinga5169932015-08-13 05:54:07 +02005201
5202err:
5203 if (skb_chk && skb_chk != skb)
5204 kfree_skb(skb_chk);
5205
5206 return NULL;
5207
Linus Lüssing9afd85c2015-05-02 14:01:07 +02005208}
5209EXPORT_SYMBOL(skb_checksum_trimmed);
5210
Ben Hutchings4497b072008-06-19 16:22:28 -07005211void __skb_warn_lro_forwarding(const struct sk_buff *skb)
5212{
Joe Perchese87cc472012-05-13 21:56:26 +00005213 net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n",
5214 skb->dev->name);
Ben Hutchings4497b072008-06-19 16:22:28 -07005215}
Ben Hutchings4497b072008-06-19 16:22:28 -07005216EXPORT_SYMBOL(__skb_warn_lro_forwarding);
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005217
5218void kfree_skb_partial(struct sk_buff *skb, bool head_stolen)
5219{
Eric Dumazet3d861f62012-10-22 09:03:40 +00005220 if (head_stolen) {
5221 skb_release_head_state(skb);
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005222 kmem_cache_free(skbuff_head_cache, skb);
Eric Dumazet3d861f62012-10-22 09:03:40 +00005223 } else {
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005224 __kfree_skb(skb);
Eric Dumazet3d861f62012-10-22 09:03:40 +00005225 }
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005226}
5227EXPORT_SYMBOL(kfree_skb_partial);
5228
5229/**
5230 * skb_try_coalesce - try to merge skb to prior one
5231 * @to: prior buffer
5232 * @from: buffer to add
5233 * @fragstolen: pointer to boolean
Randy Dunlapc6c4b972012-06-08 14:01:44 +00005234 * @delta_truesize: how much more was allocated than was requested
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005235 */
5236bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
5237 bool *fragstolen, int *delta_truesize)
5238{
Eric Dumazetc818fa92017-10-04 10:48:35 -07005239 struct skb_shared_info *to_shinfo, *from_shinfo;
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005240 int i, delta, len = from->len;
5241
5242 *fragstolen = false;
5243
5244 if (skb_cloned(to))
5245 return false;
5246
Ilias Apalodimas6a5bcd82021-06-07 21:02:38 +02005247 /* The page pool signature of struct page will eventually figure out
5248 * which pages can be recycled or not but for now let's prohibit slab
5249 * allocated and page_pool allocated SKBs from being coalesced.
5250 */
5251 if (to->pp_recycle != from->pp_recycle)
5252 return false;
5253
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005254 if (len <= skb_tailroom(to)) {
Eric Dumazete93a0432014-09-15 04:19:52 -07005255 if (len)
5256 BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len));
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005257 *delta_truesize = 0;
5258 return true;
5259 }
5260
Eric Dumazetc818fa92017-10-04 10:48:35 -07005261 to_shinfo = skb_shinfo(to);
5262 from_shinfo = skb_shinfo(from);
5263 if (to_shinfo->frag_list || from_shinfo->frag_list)
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005264 return false;
Willem de Bruijn1f8b9772017-08-03 16:29:41 -04005265 if (skb_zcopy(to) || skb_zcopy(from))
5266 return false;
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005267
5268 if (skb_headlen(from) != 0) {
5269 struct page *page;
5270 unsigned int offset;
5271
Eric Dumazetc818fa92017-10-04 10:48:35 -07005272 if (to_shinfo->nr_frags +
5273 from_shinfo->nr_frags >= MAX_SKB_FRAGS)
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005274 return false;
5275
5276 if (skb_head_is_locked(from))
5277 return false;
5278
5279 delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
5280
5281 page = virt_to_head_page(from->head);
5282 offset = from->data - (unsigned char *)page_address(page);
5283
Eric Dumazetc818fa92017-10-04 10:48:35 -07005284 skb_fill_page_desc(to, to_shinfo->nr_frags,
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005285 page, offset, skb_headlen(from));
5286 *fragstolen = true;
5287 } else {
Eric Dumazetc818fa92017-10-04 10:48:35 -07005288 if (to_shinfo->nr_frags +
5289 from_shinfo->nr_frags > MAX_SKB_FRAGS)
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005290 return false;
5291
Weiping Panf4b549a2012-09-28 20:15:30 +00005292 delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from));
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005293 }
5294
5295 WARN_ON_ONCE(delta < len);
5296
Eric Dumazetc818fa92017-10-04 10:48:35 -07005297 memcpy(to_shinfo->frags + to_shinfo->nr_frags,
5298 from_shinfo->frags,
5299 from_shinfo->nr_frags * sizeof(skb_frag_t));
5300 to_shinfo->nr_frags += from_shinfo->nr_frags;
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005301
5302 if (!skb_cloned(from))
Eric Dumazetc818fa92017-10-04 10:48:35 -07005303 from_shinfo->nr_frags = 0;
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005304
Li RongQing8ea853f2012-09-18 16:53:21 +00005305 /* if the skb is not cloned this does nothing
5306 * since we set nr_frags to 0.
5307 */
Eric Dumazetc818fa92017-10-04 10:48:35 -07005308 for (i = 0; i < from_shinfo->nr_frags; i++)
5309 __skb_frag_ref(&from_shinfo->frags[i]);
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005310
5311 to->truesize += delta;
5312 to->len += len;
5313 to->data_len += len;
5314
5315 *delta_truesize = delta;
5316 return true;
5317}
5318EXPORT_SYMBOL(skb_try_coalesce);
Nicolas Dichtel621e84d2013-06-26 16:11:27 +02005319
5320/**
Nicolas Dichtel8b27f272013-09-02 15:34:56 +02005321 * skb_scrub_packet - scrub an skb
Nicolas Dichtel621e84d2013-06-26 16:11:27 +02005322 *
5323 * @skb: buffer to clean
Nicolas Dichtel8b27f272013-09-02 15:34:56 +02005324 * @xnet: packet is crossing netns
Nicolas Dichtel621e84d2013-06-26 16:11:27 +02005325 *
Nicolas Dichtel8b27f272013-09-02 15:34:56 +02005326 * skb_scrub_packet can be used after encapsulating or decapsulting a packet
5327 * into/from a tunnel. Some information have to be cleared during these
5328 * operations.
5329 * skb_scrub_packet can also be used to clean a skb before injecting it in
5330 * another namespace (@xnet == true). We have to clear all information in the
5331 * skb that could impact namespace isolation.
Nicolas Dichtel621e84d2013-06-26 16:11:27 +02005332 */
Nicolas Dichtel8b27f272013-09-02 15:34:56 +02005333void skb_scrub_packet(struct sk_buff *skb, bool xnet)
Nicolas Dichtel621e84d2013-06-26 16:11:27 +02005334{
Nicolas Dichtel621e84d2013-06-26 16:11:27 +02005335 skb->pkt_type = PACKET_HOST;
5336 skb->skb_iif = 0;
WANG Cong60ff7462014-05-04 16:39:18 -07005337 skb->ignore_df = 0;
Nicolas Dichtel621e84d2013-06-26 16:11:27 +02005338 skb_dst_drop(skb);
Florian Westphal174e2382019-09-26 20:37:05 +02005339 skb_ext_reset(skb);
Florian Westphal895b5c92019-09-29 20:54:03 +02005340 nf_reset_ct(skb);
Nicolas Dichtel621e84d2013-06-26 16:11:27 +02005341 nf_reset_trace(skb);
Herbert Xu213dd742015-04-16 09:03:27 +08005342
Petr Machata6f9a5062018-11-19 16:11:07 +00005343#ifdef CONFIG_NET_SWITCHDEV
5344 skb->offload_fwd_mark = 0;
Ido Schimmel875e8932018-12-04 08:15:10 +00005345 skb->offload_l3_fwd_mark = 0;
Petr Machata6f9a5062018-11-19 16:11:07 +00005346#endif
5347
Herbert Xu213dd742015-04-16 09:03:27 +08005348 if (!xnet)
5349 return;
5350
Ye Yin2b5ec1a2017-10-26 16:57:05 +08005351 ipvs_reset(skb);
Herbert Xu213dd742015-04-16 09:03:27 +08005352 skb->mark = 0;
Jesus Sanchez-Palenciac47d8c22018-07-03 15:42:47 -07005353 skb->tstamp = 0;
Nicolas Dichtel621e84d2013-06-26 16:11:27 +02005354}
5355EXPORT_SYMBOL_GPL(skb_scrub_packet);
Florian Westphalde960aa2014-01-26 10:58:16 +01005356
5357/**
5358 * skb_gso_transport_seglen - Return length of individual segments of a gso packet
5359 *
5360 * @skb: GSO skb
5361 *
5362 * skb_gso_transport_seglen is used to determine the real size of the
5363 * individual segments, including Layer4 headers (TCP/UDP).
5364 *
5365 * The MAC/L2 or network (IP, IPv6) headers are not accounted for.
5366 */
Daniel Axtensa4a77712018-03-01 17:13:40 +11005367static unsigned int skb_gso_transport_seglen(const struct sk_buff *skb)
Florian Westphalde960aa2014-01-26 10:58:16 +01005368{
5369 const struct skb_shared_info *shinfo = skb_shinfo(skb);
Florian Westphalf993bc22014-10-20 13:49:18 +02005370 unsigned int thlen = 0;
Florian Westphalde960aa2014-01-26 10:58:16 +01005371
Florian Westphalf993bc22014-10-20 13:49:18 +02005372 if (skb->encapsulation) {
5373 thlen = skb_inner_transport_header(skb) -
5374 skb_transport_header(skb);
Florian Westphal6d39d582014-04-09 10:28:50 +02005375
Florian Westphalf993bc22014-10-20 13:49:18 +02005376 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
5377 thlen += inner_tcp_hdrlen(skb);
5378 } else if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
5379 thlen = tcp_hdrlen(skb);
Daniel Axtens1dd27cd2018-03-09 14:06:09 +11005380 } else if (unlikely(skb_is_gso_sctp(skb))) {
Marcelo Ricardo Leitner90017ac2016-06-02 15:05:43 -03005381 thlen = sizeof(struct sctphdr);
Willem de Bruijnee80d1e2018-04-26 13:42:16 -04005382 } else if (shinfo->gso_type & SKB_GSO_UDP_L4) {
5383 thlen = sizeof(struct udphdr);
Florian Westphalf993bc22014-10-20 13:49:18 +02005384 }
Florian Westphal6d39d582014-04-09 10:28:50 +02005385 /* UFO sets gso_size to the size of the fragmentation
5386 * payload, i.e. the size of the L4 (UDP) header is already
5387 * accounted for.
5388 */
Florian Westphalf993bc22014-10-20 13:49:18 +02005389 return thlen + shinfo->gso_size;
Florian Westphalde960aa2014-01-26 10:58:16 +01005390}
Daniel Axtensa4a77712018-03-01 17:13:40 +11005391
5392/**
5393 * skb_gso_network_seglen - Return length of individual segments of a gso packet
5394 *
5395 * @skb: GSO skb
5396 *
5397 * skb_gso_network_seglen is used to determine the real size of the
5398 * individual segments, including Layer3 (IP, IPv6) and L4 headers (TCP/UDP).
5399 *
5400 * The MAC/L2 header is not accounted for.
5401 */
5402static unsigned int skb_gso_network_seglen(const struct sk_buff *skb)
5403{
5404 unsigned int hdr_len = skb_transport_header(skb) -
5405 skb_network_header(skb);
5406
5407 return hdr_len + skb_gso_transport_seglen(skb);
5408}
5409
5410/**
5411 * skb_gso_mac_seglen - Return length of individual segments of a gso packet
5412 *
5413 * @skb: GSO skb
5414 *
5415 * skb_gso_mac_seglen is used to determine the real size of the
5416 * individual segments, including MAC/L2, Layer3 (IP, IPv6) and L4
5417 * headers (TCP/UDP).
5418 */
5419static unsigned int skb_gso_mac_seglen(const struct sk_buff *skb)
5420{
5421 unsigned int hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
5422
5423 return hdr_len + skb_gso_transport_seglen(skb);
5424}
Vlad Yasevich0d5501c2014-08-08 14:42:13 -04005425
Marcelo Ricardo Leitnerae7ef812016-06-02 15:05:41 -03005426/**
Daniel Axtens2b16f042018-01-31 14:15:33 +11005427 * skb_gso_size_check - check the skb size, considering GSO_BY_FRAGS
5428 *
5429 * There are a couple of instances where we have a GSO skb, and we
5430 * want to determine what size it would be after it is segmented.
5431 *
5432 * We might want to check:
5433 * - L3+L4+payload size (e.g. IP forwarding)
5434 * - L2+L3+L4+payload size (e.g. sanity check before passing to driver)
5435 *
5436 * This is a helper to do that correctly considering GSO_BY_FRAGS.
5437 *
Mathieu Malaterre49682bf2018-10-31 13:16:58 +01005438 * @skb: GSO skb
5439 *
Daniel Axtens2b16f042018-01-31 14:15:33 +11005440 * @seg_len: The segmented length (from skb_gso_*_seglen). In the
5441 * GSO_BY_FRAGS case this will be [header sizes + GSO_BY_FRAGS].
5442 *
5443 * @max_len: The maximum permissible length.
5444 *
5445 * Returns true if the segmented length <= max length.
5446 */
5447static inline bool skb_gso_size_check(const struct sk_buff *skb,
5448 unsigned int seg_len,
5449 unsigned int max_len) {
5450 const struct skb_shared_info *shinfo = skb_shinfo(skb);
5451 const struct sk_buff *iter;
5452
5453 if (shinfo->gso_size != GSO_BY_FRAGS)
5454 return seg_len <= max_len;
5455
5456 /* Undo this so we can re-use header sizes */
5457 seg_len -= GSO_BY_FRAGS;
5458
5459 skb_walk_frags(skb, iter) {
5460 if (seg_len + skb_headlen(iter) > max_len)
5461 return false;
5462 }
5463
5464 return true;
5465}
5466
5467/**
Daniel Axtens779b7932018-03-01 17:13:37 +11005468 * skb_gso_validate_network_len - Will a split GSO skb fit into a given MTU?
Marcelo Ricardo Leitnerae7ef812016-06-02 15:05:41 -03005469 *
5470 * @skb: GSO skb
David S. Miller76f21b92016-06-03 22:56:28 -07005471 * @mtu: MTU to validate against
Marcelo Ricardo Leitnerae7ef812016-06-02 15:05:41 -03005472 *
Daniel Axtens779b7932018-03-01 17:13:37 +11005473 * skb_gso_validate_network_len validates if a given skb will fit a
5474 * wanted MTU once split. It considers L3 headers, L4 headers, and the
5475 * payload.
Marcelo Ricardo Leitnerae7ef812016-06-02 15:05:41 -03005476 */
Daniel Axtens779b7932018-03-01 17:13:37 +11005477bool skb_gso_validate_network_len(const struct sk_buff *skb, unsigned int mtu)
Marcelo Ricardo Leitnerae7ef812016-06-02 15:05:41 -03005478{
Daniel Axtens2b16f042018-01-31 14:15:33 +11005479 return skb_gso_size_check(skb, skb_gso_network_seglen(skb), mtu);
Marcelo Ricardo Leitnerae7ef812016-06-02 15:05:41 -03005480}
Daniel Axtens779b7932018-03-01 17:13:37 +11005481EXPORT_SYMBOL_GPL(skb_gso_validate_network_len);
Marcelo Ricardo Leitnerae7ef812016-06-02 15:05:41 -03005482
Daniel Axtens2b16f042018-01-31 14:15:33 +11005483/**
5484 * skb_gso_validate_mac_len - Will a split GSO skb fit in a given length?
5485 *
5486 * @skb: GSO skb
5487 * @len: length to validate against
5488 *
5489 * skb_gso_validate_mac_len validates if a given skb will fit a wanted
5490 * length once split, including L2, L3 and L4 headers and the payload.
5491 */
5492bool skb_gso_validate_mac_len(const struct sk_buff *skb, unsigned int len)
5493{
5494 return skb_gso_size_check(skb, skb_gso_mac_seglen(skb), len);
5495}
5496EXPORT_SYMBOL_GPL(skb_gso_validate_mac_len);
5497
Vlad Yasevich0d5501c2014-08-08 14:42:13 -04005498static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb)
5499{
Yuya Kusakabed85e8be2019-04-16 10:22:28 +09005500 int mac_len, meta_len;
5501 void *meta;
Toshiaki Makita4bbb3e02018-03-13 14:51:27 +09005502
Vlad Yasevich0d5501c2014-08-08 14:42:13 -04005503 if (skb_cow(skb, skb_headroom(skb)) < 0) {
5504 kfree_skb(skb);
5505 return NULL;
5506 }
5507
Toshiaki Makita4bbb3e02018-03-13 14:51:27 +09005508 mac_len = skb->data - skb_mac_header(skb);
Toshiaki Makitaae474572018-03-29 19:05:29 +09005509 if (likely(mac_len > VLAN_HLEN + ETH_TLEN)) {
5510 memmove(skb_mac_header(skb) + VLAN_HLEN, skb_mac_header(skb),
5511 mac_len - VLAN_HLEN - ETH_TLEN);
5512 }
Yuya Kusakabed85e8be2019-04-16 10:22:28 +09005513
5514 meta_len = skb_metadata_len(skb);
5515 if (meta_len) {
5516 meta = skb_metadata_end(skb) - meta_len;
5517 memmove(meta + VLAN_HLEN, meta, meta_len);
5518 }
5519
Vlad Yasevich0d5501c2014-08-08 14:42:13 -04005520 skb->mac_header += VLAN_HLEN;
5521 return skb;
5522}
5523
5524struct sk_buff *skb_vlan_untag(struct sk_buff *skb)
5525{
5526 struct vlan_hdr *vhdr;
5527 u16 vlan_tci;
5528
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01005529 if (unlikely(skb_vlan_tag_present(skb))) {
Vlad Yasevich0d5501c2014-08-08 14:42:13 -04005530 /* vlan_tci is already set-up so leave this for another time */
5531 return skb;
5532 }
5533
5534 skb = skb_share_check(skb, GFP_ATOMIC);
5535 if (unlikely(!skb))
5536 goto err_free;
Miaohe Lin55eff0e2020-08-15 04:44:31 -04005537 /* We may access the two bytes after vlan_hdr in vlan_set_encap_proto(). */
5538 if (unlikely(!pskb_may_pull(skb, VLAN_HLEN + sizeof(unsigned short))))
Vlad Yasevich0d5501c2014-08-08 14:42:13 -04005539 goto err_free;
5540
5541 vhdr = (struct vlan_hdr *)skb->data;
5542 vlan_tci = ntohs(vhdr->h_vlan_TCI);
5543 __vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci);
5544
5545 skb_pull_rcsum(skb, VLAN_HLEN);
5546 vlan_set_encap_proto(skb, vhdr);
5547
5548 skb = skb_reorder_vlan_header(skb);
5549 if (unlikely(!skb))
5550 goto err_free;
5551
5552 skb_reset_network_header(skb);
Alexander Lobakin8be33ec2020-11-09 23:47:23 +00005553 if (!skb_transport_header_was_set(skb))
5554 skb_reset_transport_header(skb);
Vlad Yasevich0d5501c2014-08-08 14:42:13 -04005555 skb_reset_mac_len(skb);
5556
5557 return skb;
5558
5559err_free:
5560 kfree_skb(skb);
5561 return NULL;
5562}
5563EXPORT_SYMBOL(skb_vlan_untag);
Eric Dumazet2e4e4412014-09-17 04:49:49 -07005564
Jiri Pirkoe2195122014-11-19 14:05:01 +01005565int skb_ensure_writable(struct sk_buff *skb, int write_len)
5566{
5567 if (!pskb_may_pull(skb, write_len))
5568 return -ENOMEM;
5569
5570 if (!skb_cloned(skb) || skb_clone_writable(skb, write_len))
5571 return 0;
5572
5573 return pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
5574}
5575EXPORT_SYMBOL(skb_ensure_writable);
5576
Shmulik Ladkanibfca4c52016-09-19 19:11:09 +03005577/* remove VLAN header from packet and update csum accordingly.
5578 * expects a non skb_vlan_tag_present skb with a vlan tag payload
5579 */
5580int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci)
Jiri Pirko93515d52014-11-19 14:05:02 +01005581{
5582 struct vlan_hdr *vhdr;
Shmulik Ladkanib6a79202016-09-29 12:10:41 +03005583 int offset = skb->data - skb_mac_header(skb);
Jiri Pirko93515d52014-11-19 14:05:02 +01005584 int err;
5585
Shmulik Ladkanib6a79202016-09-29 12:10:41 +03005586 if (WARN_ONCE(offset,
5587 "__skb_vlan_pop got skb with skb->data not at mac header (offset %d)\n",
5588 offset)) {
5589 return -EINVAL;
5590 }
5591
Jiri Pirko93515d52014-11-19 14:05:02 +01005592 err = skb_ensure_writable(skb, VLAN_ETH_HLEN);
5593 if (unlikely(err))
Shmulik Ladkanib6a79202016-09-29 12:10:41 +03005594 return err;
Jiri Pirko93515d52014-11-19 14:05:02 +01005595
5596 skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
5597
5598 vhdr = (struct vlan_hdr *)(skb->data + ETH_HLEN);
5599 *vlan_tci = ntohs(vhdr->h_vlan_TCI);
5600
5601 memmove(skb->data + VLAN_HLEN, skb->data, 2 * ETH_ALEN);
5602 __skb_pull(skb, VLAN_HLEN);
5603
5604 vlan_set_encap_proto(skb, vhdr);
5605 skb->mac_header += VLAN_HLEN;
5606
5607 if (skb_network_offset(skb) < ETH_HLEN)
5608 skb_set_network_header(skb, ETH_HLEN);
5609
5610 skb_reset_mac_len(skb);
Jiri Pirko93515d52014-11-19 14:05:02 +01005611
5612 return err;
5613}
Shmulik Ladkanibfca4c52016-09-19 19:11:09 +03005614EXPORT_SYMBOL(__skb_vlan_pop);
Jiri Pirko93515d52014-11-19 14:05:02 +01005615
Shmulik Ladkanib6a79202016-09-29 12:10:41 +03005616/* Pop a vlan tag either from hwaccel or from payload.
5617 * Expects skb->data at mac header.
5618 */
Jiri Pirko93515d52014-11-19 14:05:02 +01005619int skb_vlan_pop(struct sk_buff *skb)
5620{
5621 u16 vlan_tci;
5622 __be16 vlan_proto;
5623 int err;
5624
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01005625 if (likely(skb_vlan_tag_present(skb))) {
Michał Mirosławb18175242018-11-09 00:18:02 +01005626 __vlan_hwaccel_clear_tag(skb);
Jiri Pirko93515d52014-11-19 14:05:02 +01005627 } else {
Shmulik Ladkaniecf4ee42016-09-20 12:48:37 +03005628 if (unlikely(!eth_type_vlan(skb->protocol)))
Jiri Pirko93515d52014-11-19 14:05:02 +01005629 return 0;
5630
5631 err = __skb_vlan_pop(skb, &vlan_tci);
5632 if (err)
5633 return err;
5634 }
5635 /* move next vlan tag to hw accel tag */
Shmulik Ladkaniecf4ee42016-09-20 12:48:37 +03005636 if (likely(!eth_type_vlan(skb->protocol)))
Jiri Pirko93515d52014-11-19 14:05:02 +01005637 return 0;
5638
5639 vlan_proto = skb->protocol;
5640 err = __skb_vlan_pop(skb, &vlan_tci);
5641 if (unlikely(err))
5642 return err;
5643
5644 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
5645 return 0;
5646}
5647EXPORT_SYMBOL(skb_vlan_pop);
5648
Shmulik Ladkanib6a79202016-09-29 12:10:41 +03005649/* Push a vlan tag either into hwaccel or into payload (if hwaccel tag present).
5650 * Expects skb->data at mac header.
5651 */
Jiri Pirko93515d52014-11-19 14:05:02 +01005652int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci)
5653{
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01005654 if (skb_vlan_tag_present(skb)) {
Shmulik Ladkanib6a79202016-09-29 12:10:41 +03005655 int offset = skb->data - skb_mac_header(skb);
Jiri Pirko93515d52014-11-19 14:05:02 +01005656 int err;
5657
Shmulik Ladkanib6a79202016-09-29 12:10:41 +03005658 if (WARN_ONCE(offset,
5659 "skb_vlan_push got skb with skb->data not at mac header (offset %d)\n",
5660 offset)) {
5661 return -EINVAL;
5662 }
5663
Jiri Pirko93515d52014-11-19 14:05:02 +01005664 err = __vlan_insert_tag(skb, skb->vlan_proto,
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01005665 skb_vlan_tag_get(skb));
Shmulik Ladkanib6a79202016-09-29 12:10:41 +03005666 if (err)
Jiri Pirko93515d52014-11-19 14:05:02 +01005667 return err;
Daniel Borkmann9241e2d2016-04-16 02:27:58 +02005668
Jiri Pirko93515d52014-11-19 14:05:02 +01005669 skb->protocol = skb->vlan_proto;
5670 skb->mac_len += VLAN_HLEN;
Jiri Pirko93515d52014-11-19 14:05:02 +01005671
Daniel Borkmann6b83d282016-02-20 00:29:30 +01005672 skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
Jiri Pirko93515d52014-11-19 14:05:02 +01005673 }
5674 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
5675 return 0;
5676}
5677EXPORT_SYMBOL(skb_vlan_push);
5678
Guillaume Nault19fbcb32020-10-03 00:44:28 +02005679/**
5680 * skb_eth_pop() - Drop the Ethernet header at the head of a packet
5681 *
5682 * @skb: Socket buffer to modify
5683 *
5684 * Drop the Ethernet header of @skb.
5685 *
5686 * Expects that skb->data points to the mac header and that no VLAN tags are
5687 * present.
5688 *
5689 * Returns 0 on success, -errno otherwise.
5690 */
5691int skb_eth_pop(struct sk_buff *skb)
5692{
5693 if (!pskb_may_pull(skb, ETH_HLEN) || skb_vlan_tagged(skb) ||
5694 skb_network_offset(skb) < ETH_HLEN)
5695 return -EPROTO;
5696
5697 skb_pull_rcsum(skb, ETH_HLEN);
5698 skb_reset_mac_header(skb);
5699 skb_reset_mac_len(skb);
5700
5701 return 0;
5702}
5703EXPORT_SYMBOL(skb_eth_pop);
5704
5705/**
5706 * skb_eth_push() - Add a new Ethernet header at the head of a packet
5707 *
5708 * @skb: Socket buffer to modify
5709 * @dst: Destination MAC address of the new header
5710 * @src: Source MAC address of the new header
5711 *
5712 * Prepend @skb with a new Ethernet header.
5713 *
5714 * Expects that skb->data points to the mac header, which must be empty.
5715 *
5716 * Returns 0 on success, -errno otherwise.
5717 */
5718int skb_eth_push(struct sk_buff *skb, const unsigned char *dst,
5719 const unsigned char *src)
5720{
5721 struct ethhdr *eth;
5722 int err;
5723
5724 if (skb_network_offset(skb) || skb_vlan_tag_present(skb))
5725 return -EPROTO;
5726
5727 err = skb_cow_head(skb, sizeof(*eth));
5728 if (err < 0)
5729 return err;
5730
5731 skb_push(skb, sizeof(*eth));
5732 skb_reset_mac_header(skb);
5733 skb_reset_mac_len(skb);
5734
5735 eth = eth_hdr(skb);
5736 ether_addr_copy(eth->h_dest, dst);
5737 ether_addr_copy(eth->h_source, src);
5738 eth->h_proto = skb->protocol;
5739
5740 skb_postpush_rcsum(skb, eth, sizeof(*eth));
5741
5742 return 0;
5743}
5744EXPORT_SYMBOL(skb_eth_push);
5745
John Hurley8822e272019-07-07 15:01:54 +01005746/* Update the ethertype of hdr and the skb csum value if required. */
5747static void skb_mod_eth_type(struct sk_buff *skb, struct ethhdr *hdr,
5748 __be16 ethertype)
5749{
5750 if (skb->ip_summed == CHECKSUM_COMPLETE) {
5751 __be16 diff[] = { ~hdr->h_proto, ethertype };
5752
5753 skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum);
5754 }
5755
5756 hdr->h_proto = ethertype;
5757}
5758
5759/**
Martin Varghesee7dbfed2019-12-21 08:50:01 +05305760 * skb_mpls_push() - push a new MPLS header after mac_len bytes from start of
5761 * the packet
John Hurley8822e272019-07-07 15:01:54 +01005762 *
5763 * @skb: buffer
5764 * @mpls_lse: MPLS label stack entry to push
5765 * @mpls_proto: ethertype of the new MPLS header (expects 0x8847 or 0x8848)
Davide Carattifa4e0f82019-10-12 13:55:07 +02005766 * @mac_len: length of the MAC header
Martin Varghesee7dbfed2019-12-21 08:50:01 +05305767 * @ethernet: flag to indicate if the resulting packet after skb_mpls_push is
5768 * ethernet
John Hurley8822e272019-07-07 15:01:54 +01005769 *
5770 * Expects skb->data at mac header.
5771 *
5772 * Returns 0 on success, -errno otherwise.
5773 */
Davide Carattifa4e0f82019-10-12 13:55:07 +02005774int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto,
Martin Varghesed04ac222019-12-05 05:57:22 +05305775 int mac_len, bool ethernet)
John Hurley8822e272019-07-07 15:01:54 +01005776{
5777 struct mpls_shim_hdr *lse;
5778 int err;
5779
5780 if (unlikely(!eth_p_mpls(mpls_proto)))
5781 return -EINVAL;
5782
5783 /* Networking stack does not allow simultaneous Tunnel and MPLS GSO. */
5784 if (skb->encapsulation)
5785 return -EINVAL;
5786
5787 err = skb_cow_head(skb, MPLS_HLEN);
5788 if (unlikely(err))
5789 return err;
5790
5791 if (!skb->inner_protocol) {
Martin Varghesee7dbfed2019-12-21 08:50:01 +05305792 skb_set_inner_network_header(skb, skb_network_offset(skb));
John Hurley8822e272019-07-07 15:01:54 +01005793 skb_set_inner_protocol(skb, skb->protocol);
5794 }
5795
5796 skb_push(skb, MPLS_HLEN);
5797 memmove(skb_mac_header(skb) - MPLS_HLEN, skb_mac_header(skb),
Davide Carattifa4e0f82019-10-12 13:55:07 +02005798 mac_len);
John Hurley8822e272019-07-07 15:01:54 +01005799 skb_reset_mac_header(skb);
Davide Carattifa4e0f82019-10-12 13:55:07 +02005800 skb_set_network_header(skb, mac_len);
Martin Varghesee7dbfed2019-12-21 08:50:01 +05305801 skb_reset_mac_len(skb);
John Hurley8822e272019-07-07 15:01:54 +01005802
5803 lse = mpls_hdr(skb);
5804 lse->label_stack_entry = mpls_lse;
5805 skb_postpush_rcsum(skb, lse, MPLS_HLEN);
5806
Guillaume Nault4296adc2020-10-02 21:53:08 +02005807 if (ethernet && mac_len >= ETH_HLEN)
John Hurley8822e272019-07-07 15:01:54 +01005808 skb_mod_eth_type(skb, eth_hdr(skb), mpls_proto);
5809 skb->protocol = mpls_proto;
5810
5811 return 0;
5812}
5813EXPORT_SYMBOL_GPL(skb_mpls_push);
5814
Eric Dumazet2e4e4412014-09-17 04:49:49 -07005815/**
John Hurleyed246ce2019-07-07 15:01:55 +01005816 * skb_mpls_pop() - pop the outermost MPLS header
5817 *
5818 * @skb: buffer
5819 * @next_proto: ethertype of header after popped MPLS header
Davide Carattifa4e0f82019-10-12 13:55:07 +02005820 * @mac_len: length of the MAC header
Martin Varghese76f99f92019-12-21 08:50:23 +05305821 * @ethernet: flag to indicate if the packet is ethernet
John Hurleyed246ce2019-07-07 15:01:55 +01005822 *
5823 * Expects skb->data at mac header.
5824 *
5825 * Returns 0 on success, -errno otherwise.
5826 */
Martin Varghese040b5cf2019-12-02 10:49:51 +05305827int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len,
5828 bool ethernet)
John Hurleyed246ce2019-07-07 15:01:55 +01005829{
5830 int err;
5831
5832 if (unlikely(!eth_p_mpls(skb->protocol)))
Davide Carattidedc5a02019-10-12 13:55:06 +02005833 return 0;
John Hurleyed246ce2019-07-07 15:01:55 +01005834
Davide Carattifa4e0f82019-10-12 13:55:07 +02005835 err = skb_ensure_writable(skb, mac_len + MPLS_HLEN);
John Hurleyed246ce2019-07-07 15:01:55 +01005836 if (unlikely(err))
5837 return err;
5838
5839 skb_postpull_rcsum(skb, mpls_hdr(skb), MPLS_HLEN);
5840 memmove(skb_mac_header(skb) + MPLS_HLEN, skb_mac_header(skb),
Davide Carattifa4e0f82019-10-12 13:55:07 +02005841 mac_len);
John Hurleyed246ce2019-07-07 15:01:55 +01005842
5843 __skb_pull(skb, MPLS_HLEN);
5844 skb_reset_mac_header(skb);
Davide Carattifa4e0f82019-10-12 13:55:07 +02005845 skb_set_network_header(skb, mac_len);
John Hurleyed246ce2019-07-07 15:01:55 +01005846
Guillaume Nault4296adc2020-10-02 21:53:08 +02005847 if (ethernet && mac_len >= ETH_HLEN) {
John Hurleyed246ce2019-07-07 15:01:55 +01005848 struct ethhdr *hdr;
5849
5850 /* use mpls_hdr() to get ethertype to account for VLANs. */
5851 hdr = (struct ethhdr *)((void *)mpls_hdr(skb) - ETH_HLEN);
5852 skb_mod_eth_type(skb, hdr, next_proto);
5853 }
5854 skb->protocol = next_proto;
5855
5856 return 0;
5857}
5858EXPORT_SYMBOL_GPL(skb_mpls_pop);
5859
5860/**
John Hurleyd27cf5c2019-07-07 15:01:56 +01005861 * skb_mpls_update_lse() - modify outermost MPLS header and update csum
5862 *
5863 * @skb: buffer
5864 * @mpls_lse: new MPLS label stack entry to update to
5865 *
5866 * Expects skb->data at mac header.
5867 *
5868 * Returns 0 on success, -errno otherwise.
5869 */
5870int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse)
5871{
5872 int err;
5873
5874 if (unlikely(!eth_p_mpls(skb->protocol)))
5875 return -EINVAL;
5876
5877 err = skb_ensure_writable(skb, skb->mac_len + MPLS_HLEN);
5878 if (unlikely(err))
5879 return err;
5880
5881 if (skb->ip_summed == CHECKSUM_COMPLETE) {
5882 __be32 diff[] = { ~mpls_hdr(skb)->label_stack_entry, mpls_lse };
5883
5884 skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum);
5885 }
5886
5887 mpls_hdr(skb)->label_stack_entry = mpls_lse;
5888
5889 return 0;
5890}
5891EXPORT_SYMBOL_GPL(skb_mpls_update_lse);
5892
5893/**
John Hurley2a2ea502019-07-07 15:01:57 +01005894 * skb_mpls_dec_ttl() - decrement the TTL of the outermost MPLS header
5895 *
5896 * @skb: buffer
5897 *
5898 * Expects skb->data at mac header.
5899 *
5900 * Returns 0 on success, -errno otherwise.
5901 */
5902int skb_mpls_dec_ttl(struct sk_buff *skb)
5903{
5904 u32 lse;
5905 u8 ttl;
5906
5907 if (unlikely(!eth_p_mpls(skb->protocol)))
5908 return -EINVAL;
5909
Davide Caratti13de4ed2020-12-03 10:58:21 +01005910 if (!pskb_may_pull(skb, skb_network_offset(skb) + MPLS_HLEN))
5911 return -ENOMEM;
5912
John Hurley2a2ea502019-07-07 15:01:57 +01005913 lse = be32_to_cpu(mpls_hdr(skb)->label_stack_entry);
5914 ttl = (lse & MPLS_LS_TTL_MASK) >> MPLS_LS_TTL_SHIFT;
5915 if (!--ttl)
5916 return -EINVAL;
5917
5918 lse &= ~MPLS_LS_TTL_MASK;
5919 lse |= ttl << MPLS_LS_TTL_SHIFT;
5920
5921 return skb_mpls_update_lse(skb, cpu_to_be32(lse));
5922}
5923EXPORT_SYMBOL_GPL(skb_mpls_dec_ttl);
5924
5925/**
Eric Dumazet2e4e4412014-09-17 04:49:49 -07005926 * alloc_skb_with_frags - allocate skb with page frags
5927 *
Masanari Iidade3f0d02014-10-09 12:58:08 +09005928 * @header_len: size of linear part
5929 * @data_len: needed length in frags
5930 * @max_page_order: max page order desired.
5931 * @errcode: pointer to error code if any
5932 * @gfp_mask: allocation mask
Eric Dumazet2e4e4412014-09-17 04:49:49 -07005933 *
5934 * This can be used to allocate a paged skb, given a maximal order for frags.
5935 */
5936struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
5937 unsigned long data_len,
5938 int max_page_order,
5939 int *errcode,
5940 gfp_t gfp_mask)
5941{
5942 int npages = (data_len + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
5943 unsigned long chunk;
5944 struct sk_buff *skb;
5945 struct page *page;
Eric Dumazet2e4e4412014-09-17 04:49:49 -07005946 int i;
5947
5948 *errcode = -EMSGSIZE;
5949 /* Note this test could be relaxed, if we succeed to allocate
5950 * high order pages...
5951 */
5952 if (npages > MAX_SKB_FRAGS)
5953 return NULL;
5954
Eric Dumazet2e4e4412014-09-17 04:49:49 -07005955 *errcode = -ENOBUFS;
David Rientjesf8c468e2019-01-02 13:01:43 -08005956 skb = alloc_skb(header_len, gfp_mask);
Eric Dumazet2e4e4412014-09-17 04:49:49 -07005957 if (!skb)
5958 return NULL;
5959
5960 skb->truesize += npages << PAGE_SHIFT;
5961
5962 for (i = 0; npages > 0; i++) {
5963 int order = max_page_order;
5964
5965 while (order) {
5966 if (npages >= 1 << order) {
Mel Gormand0164ad2015-11-06 16:28:21 -08005967 page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) |
Eric Dumazet2e4e4412014-09-17 04:49:49 -07005968 __GFP_COMP |
Michal Hockod14b56f2018-06-28 17:53:06 +02005969 __GFP_NOWARN,
Eric Dumazet2e4e4412014-09-17 04:49:49 -07005970 order);
5971 if (page)
5972 goto fill_page;
5973 /* Do not retry other high order allocations */
5974 order = 1;
5975 max_page_order = 0;
5976 }
5977 order--;
5978 }
5979 page = alloc_page(gfp_mask);
5980 if (!page)
5981 goto failure;
5982fill_page:
5983 chunk = min_t(unsigned long, data_len,
5984 PAGE_SIZE << order);
5985 skb_fill_page_desc(skb, i, page, 0, chunk);
5986 data_len -= chunk;
5987 npages -= 1 << order;
5988 }
5989 return skb;
5990
5991failure:
5992 kfree_skb(skb);
5993 return NULL;
5994}
5995EXPORT_SYMBOL(alloc_skb_with_frags);
Sowmini Varadhan6fa01cc2016-04-22 18:36:35 -07005996
5997/* carve out the first off bytes from skb when off < headlen */
5998static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off,
5999 const int headlen, gfp_t gfp_mask)
6000{
6001 int i;
6002 int size = skb_end_offset(skb);
6003 int new_hlen = headlen - off;
6004 u8 *data;
Sowmini Varadhan6fa01cc2016-04-22 18:36:35 -07006005
6006 size = SKB_DATA_ALIGN(size);
6007
6008 if (skb_pfmemalloc(skb))
6009 gfp_mask |= __GFP_MEMALLOC;
6010 data = kmalloc_reserve(size +
6011 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
6012 gfp_mask, NUMA_NO_NODE, NULL);
6013 if (!data)
6014 return -ENOMEM;
6015
6016 size = SKB_WITH_OVERHEAD(ksize(data));
6017
6018 /* Copy real data, and all frags */
6019 skb_copy_from_linear_data_offset(skb, off, data, new_hlen);
6020 skb->len -= off;
6021
6022 memcpy((struct skb_shared_info *)(data + size),
6023 skb_shinfo(skb),
6024 offsetof(struct skb_shared_info,
6025 frags[skb_shinfo(skb)->nr_frags]));
6026 if (skb_cloned(skb)) {
6027 /* drop the old head gracefully */
6028 if (skb_orphan_frags(skb, gfp_mask)) {
6029 kfree(data);
6030 return -ENOMEM;
6031 }
6032 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
6033 skb_frag_ref(skb, i);
6034 if (skb_has_frag_list(skb))
6035 skb_clone_fraglist(skb);
6036 skb_release_data(skb);
6037 } else {
6038 /* we can reuse existing recount- all we did was
6039 * relocate values
6040 */
6041 skb_free_head(skb);
6042 }
6043
Sowmini Varadhan6fa01cc2016-04-22 18:36:35 -07006044 skb->head = data;
6045 skb->data = data;
6046 skb->head_frag = 0;
6047#ifdef NET_SKBUFF_DATA_USES_OFFSET
6048 skb->end = size;
Sowmini Varadhan6fa01cc2016-04-22 18:36:35 -07006049#else
6050 skb->end = skb->head + size;
6051#endif
6052 skb_set_tail_pointer(skb, skb_headlen(skb));
6053 skb_headers_offset_update(skb, 0);
6054 skb->cloned = 0;
6055 skb->hdr_len = 0;
6056 skb->nohdr = 0;
6057 atomic_set(&skb_shinfo(skb)->dataref, 1);
6058
6059 return 0;
6060}
6061
6062static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp);
6063
6064/* carve out the first eat bytes from skb's frag_list. May recurse into
6065 * pskb_carve()
6066 */
6067static int pskb_carve_frag_list(struct sk_buff *skb,
6068 struct skb_shared_info *shinfo, int eat,
6069 gfp_t gfp_mask)
6070{
6071 struct sk_buff *list = shinfo->frag_list;
6072 struct sk_buff *clone = NULL;
6073 struct sk_buff *insp = NULL;
6074
6075 do {
6076 if (!list) {
6077 pr_err("Not enough bytes to eat. Want %d\n", eat);
6078 return -EFAULT;
6079 }
6080 if (list->len <= eat) {
6081 /* Eaten as whole. */
6082 eat -= list->len;
6083 list = list->next;
6084 insp = list;
6085 } else {
6086 /* Eaten partially. */
6087 if (skb_shared(list)) {
6088 clone = skb_clone(list, gfp_mask);
6089 if (!clone)
6090 return -ENOMEM;
6091 insp = list->next;
6092 list = clone;
6093 } else {
6094 /* This may be pulled without problems. */
6095 insp = list;
6096 }
6097 if (pskb_carve(list, eat, gfp_mask) < 0) {
6098 kfree_skb(clone);
6099 return -ENOMEM;
6100 }
6101 break;
6102 }
6103 } while (eat);
6104
6105 /* Free pulled out fragments. */
6106 while ((list = shinfo->frag_list) != insp) {
6107 shinfo->frag_list = list->next;
6108 kfree_skb(list);
6109 }
6110 /* And insert new clone at head. */
6111 if (clone) {
6112 clone->next = list;
6113 shinfo->frag_list = clone;
6114 }
6115 return 0;
6116}
6117
6118/* carve off first len bytes from skb. Split line (off) is in the
6119 * non-linear part of skb
6120 */
6121static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off,
6122 int pos, gfp_t gfp_mask)
6123{
6124 int i, k = 0;
6125 int size = skb_end_offset(skb);
6126 u8 *data;
6127 const int nfrags = skb_shinfo(skb)->nr_frags;
6128 struct skb_shared_info *shinfo;
Sowmini Varadhan6fa01cc2016-04-22 18:36:35 -07006129
6130 size = SKB_DATA_ALIGN(size);
6131
6132 if (skb_pfmemalloc(skb))
6133 gfp_mask |= __GFP_MEMALLOC;
6134 data = kmalloc_reserve(size +
6135 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
6136 gfp_mask, NUMA_NO_NODE, NULL);
6137 if (!data)
6138 return -ENOMEM;
6139
6140 size = SKB_WITH_OVERHEAD(ksize(data));
6141
6142 memcpy((struct skb_shared_info *)(data + size),
Miaohe Line3ec1e82020-08-15 04:48:53 -04006143 skb_shinfo(skb), offsetof(struct skb_shared_info, frags[0]));
Sowmini Varadhan6fa01cc2016-04-22 18:36:35 -07006144 if (skb_orphan_frags(skb, gfp_mask)) {
6145 kfree(data);
6146 return -ENOMEM;
6147 }
6148 shinfo = (struct skb_shared_info *)(data + size);
6149 for (i = 0; i < nfrags; i++) {
6150 int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]);
6151
6152 if (pos + fsize > off) {
6153 shinfo->frags[k] = skb_shinfo(skb)->frags[i];
6154
6155 if (pos < off) {
6156 /* Split frag.
6157 * We have two variants in this case:
6158 * 1. Move all the frag to the second
6159 * part, if it is possible. F.e.
6160 * this approach is mandatory for TUX,
6161 * where splitting is expensive.
6162 * 2. Split is accurately. We make this.
6163 */
Jonathan Lemonb54c9d52019-07-30 07:40:33 -07006164 skb_frag_off_add(&shinfo->frags[0], off - pos);
Sowmini Varadhan6fa01cc2016-04-22 18:36:35 -07006165 skb_frag_size_sub(&shinfo->frags[0], off - pos);
6166 }
6167 skb_frag_ref(skb, i);
6168 k++;
6169 }
6170 pos += fsize;
6171 }
6172 shinfo->nr_frags = k;
6173 if (skb_has_frag_list(skb))
6174 skb_clone_fraglist(skb);
6175
Miaohe Lineabe8612020-08-15 04:46:41 -04006176 /* split line is in frag list */
6177 if (k == 0 && pskb_carve_frag_list(skb, shinfo, off - pos, gfp_mask)) {
6178 /* skb_frag_unref() is not needed here as shinfo->nr_frags = 0. */
6179 if (skb_has_frag_list(skb))
6180 kfree_skb_list(skb_shinfo(skb)->frag_list);
6181 kfree(data);
6182 return -ENOMEM;
Sowmini Varadhan6fa01cc2016-04-22 18:36:35 -07006183 }
6184 skb_release_data(skb);
6185
Sowmini Varadhan6fa01cc2016-04-22 18:36:35 -07006186 skb->head = data;
6187 skb->head_frag = 0;
6188 skb->data = data;
6189#ifdef NET_SKBUFF_DATA_USES_OFFSET
6190 skb->end = size;
Sowmini Varadhan6fa01cc2016-04-22 18:36:35 -07006191#else
6192 skb->end = skb->head + size;
6193#endif
6194 skb_reset_tail_pointer(skb);
6195 skb_headers_offset_update(skb, 0);
6196 skb->cloned = 0;
6197 skb->hdr_len = 0;
6198 skb->nohdr = 0;
6199 skb->len -= off;
6200 skb->data_len = skb->len;
6201 atomic_set(&skb_shinfo(skb)->dataref, 1);
6202 return 0;
6203}
6204
6205/* remove len bytes from the beginning of the skb */
6206static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp)
6207{
6208 int headlen = skb_headlen(skb);
6209
6210 if (len < headlen)
6211 return pskb_carve_inside_header(skb, len, headlen, gfp);
6212 else
6213 return pskb_carve_inside_nonlinear(skb, len, headlen, gfp);
6214}
6215
6216/* Extract to_copy bytes starting at off from skb, and return this in
6217 * a new skb
6218 */
6219struct sk_buff *pskb_extract(struct sk_buff *skb, int off,
6220 int to_copy, gfp_t gfp)
6221{
6222 struct sk_buff *clone = skb_clone(skb, gfp);
6223
6224 if (!clone)
6225 return NULL;
6226
6227 if (pskb_carve(clone, off, gfp) < 0 ||
6228 pskb_trim(clone, to_copy)) {
6229 kfree_skb(clone);
6230 return NULL;
6231 }
6232 return clone;
6233}
6234EXPORT_SYMBOL(pskb_extract);
Eric Dumazetc8c8b122016-12-07 09:19:33 -08006235
6236/**
6237 * skb_condense - try to get rid of fragments/frag_list if possible
6238 * @skb: buffer
6239 *
6240 * Can be used to save memory before skb is added to a busy queue.
6241 * If packet has bytes in frags and enough tail room in skb->head,
6242 * pull all of them, so that we can free the frags right now and adjust
6243 * truesize.
6244 * Notes:
6245 * We do not reallocate skb->head thus can not fail.
6246 * Caller must re-evaluate skb->truesize if needed.
6247 */
6248void skb_condense(struct sk_buff *skb)
6249{
Eric Dumazet3174fed2016-12-09 08:02:05 -08006250 if (skb->data_len) {
6251 if (skb->data_len > skb->end - skb->tail ||
6252 skb_cloned(skb))
6253 return;
Eric Dumazetc8c8b122016-12-07 09:19:33 -08006254
Eric Dumazet3174fed2016-12-09 08:02:05 -08006255 /* Nice, we can free page frag(s) right now */
6256 __pskb_pull_tail(skb, skb->data_len);
6257 }
6258 /* At this point, skb->truesize might be over estimated,
6259 * because skb had a fragment, and fragments do not tell
6260 * their truesize.
6261 * When we pulled its content into skb->head, fragment
6262 * was freed, but __pskb_pull_tail() could not possibly
6263 * adjust skb->truesize, not knowing the frag truesize.
Eric Dumazetc8c8b122016-12-07 09:19:33 -08006264 */
6265 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
6266}
Florian Westphaldf5042f2018-12-18 17:15:16 +01006267
6268#ifdef CONFIG_SKB_EXTENSIONS
6269static void *skb_ext_get_ptr(struct skb_ext *ext, enum skb_ext_id id)
6270{
6271 return (void *)ext + (ext->offset[id] * SKB_EXT_ALIGN_VALUE);
6272}
6273
Paolo Abeni8b69a802020-01-09 07:59:24 -08006274/**
6275 * __skb_ext_alloc - allocate a new skb extensions storage
6276 *
Florian Westphal4930f482020-05-16 10:46:23 +02006277 * @flags: See kmalloc().
6278 *
Paolo Abeni8b69a802020-01-09 07:59:24 -08006279 * Returns the newly allocated pointer. The pointer can later attached to a
6280 * skb via __skb_ext_set().
6281 * Note: caller must handle the skb_ext as an opaque data.
6282 */
Florian Westphal4930f482020-05-16 10:46:23 +02006283struct skb_ext *__skb_ext_alloc(gfp_t flags)
Florian Westphaldf5042f2018-12-18 17:15:16 +01006284{
Florian Westphal4930f482020-05-16 10:46:23 +02006285 struct skb_ext *new = kmem_cache_alloc(skbuff_ext_cache, flags);
Florian Westphaldf5042f2018-12-18 17:15:16 +01006286
6287 if (new) {
6288 memset(new->offset, 0, sizeof(new->offset));
6289 refcount_set(&new->refcnt, 1);
6290 }
6291
6292 return new;
6293}
6294
Florian Westphal41650792018-12-18 17:15:27 +01006295static struct skb_ext *skb_ext_maybe_cow(struct skb_ext *old,
6296 unsigned int old_active)
Florian Westphaldf5042f2018-12-18 17:15:16 +01006297{
6298 struct skb_ext *new;
6299
6300 if (refcount_read(&old->refcnt) == 1)
6301 return old;
6302
6303 new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC);
6304 if (!new)
6305 return NULL;
6306
6307 memcpy(new, old, old->chunks * SKB_EXT_ALIGN_VALUE);
6308 refcount_set(&new->refcnt, 1);
6309
Florian Westphal41650792018-12-18 17:15:27 +01006310#ifdef CONFIG_XFRM
6311 if (old_active & (1 << SKB_EXT_SEC_PATH)) {
6312 struct sec_path *sp = skb_ext_get_ptr(old, SKB_EXT_SEC_PATH);
6313 unsigned int i;
6314
6315 for (i = 0; i < sp->len; i++)
6316 xfrm_state_hold(sp->xvec[i]);
6317 }
6318#endif
Florian Westphaldf5042f2018-12-18 17:15:16 +01006319 __skb_ext_put(old);
6320 return new;
6321}
6322
6323/**
Paolo Abeni8b69a802020-01-09 07:59:24 -08006324 * __skb_ext_set - attach the specified extension storage to this skb
6325 * @skb: buffer
6326 * @id: extension id
6327 * @ext: extension storage previously allocated via __skb_ext_alloc()
6328 *
6329 * Existing extensions, if any, are cleared.
6330 *
6331 * Returns the pointer to the extension.
6332 */
6333void *__skb_ext_set(struct sk_buff *skb, enum skb_ext_id id,
6334 struct skb_ext *ext)
6335{
6336 unsigned int newlen, newoff = SKB_EXT_CHUNKSIZEOF(*ext);
6337
6338 skb_ext_put(skb);
6339 newlen = newoff + skb_ext_type_len[id];
6340 ext->chunks = newlen;
6341 ext->offset[id] = newoff;
6342 skb->extensions = ext;
6343 skb->active_extensions = 1 << id;
6344 return skb_ext_get_ptr(ext, id);
6345}
6346
6347/**
Florian Westphaldf5042f2018-12-18 17:15:16 +01006348 * skb_ext_add - allocate space for given extension, COW if needed
6349 * @skb: buffer
6350 * @id: extension to allocate space for
6351 *
6352 * Allocates enough space for the given extension.
6353 * If the extension is already present, a pointer to that extension
6354 * is returned.
6355 *
6356 * If the skb was cloned, COW applies and the returned memory can be
6357 * modified without changing the extension space of clones buffers.
6358 *
6359 * Returns pointer to the extension or NULL on allocation failure.
6360 */
6361void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id)
6362{
6363 struct skb_ext *new, *old = NULL;
6364 unsigned int newlen, newoff;
6365
6366 if (skb->active_extensions) {
6367 old = skb->extensions;
6368
Florian Westphal41650792018-12-18 17:15:27 +01006369 new = skb_ext_maybe_cow(old, skb->active_extensions);
Florian Westphaldf5042f2018-12-18 17:15:16 +01006370 if (!new)
6371 return NULL;
6372
Paolo Abeni682ec852018-12-21 19:03:15 +01006373 if (__skb_ext_exist(new, id))
Florian Westphaldf5042f2018-12-18 17:15:16 +01006374 goto set_active;
Florian Westphaldf5042f2018-12-18 17:15:16 +01006375
Paolo Abenie94e50b2018-12-21 19:03:13 +01006376 newoff = new->chunks;
Florian Westphaldf5042f2018-12-18 17:15:16 +01006377 } else {
6378 newoff = SKB_EXT_CHUNKSIZEOF(*new);
6379
Florian Westphal4930f482020-05-16 10:46:23 +02006380 new = __skb_ext_alloc(GFP_ATOMIC);
Florian Westphaldf5042f2018-12-18 17:15:16 +01006381 if (!new)
6382 return NULL;
6383 }
6384
6385 newlen = newoff + skb_ext_type_len[id];
6386 new->chunks = newlen;
6387 new->offset[id] = newoff;
Florian Westphaldf5042f2018-12-18 17:15:16 +01006388set_active:
Paolo Abenib0999f32021-07-28 18:24:01 +02006389 skb->slow_gro = 1;
Paolo Abeni682ec852018-12-21 19:03:15 +01006390 skb->extensions = new;
Florian Westphaldf5042f2018-12-18 17:15:16 +01006391 skb->active_extensions |= 1 << id;
6392 return skb_ext_get_ptr(new, id);
6393}
6394EXPORT_SYMBOL(skb_ext_add);
6395
Florian Westphal41650792018-12-18 17:15:27 +01006396#ifdef CONFIG_XFRM
6397static void skb_ext_put_sp(struct sec_path *sp)
6398{
6399 unsigned int i;
6400
6401 for (i = 0; i < sp->len; i++)
6402 xfrm_state_put(sp->xvec[i]);
6403}
6404#endif
6405
Jeremy Kerr78476d32021-10-29 11:01:44 +08006406#ifdef CONFIG_MCTP_FLOWS
6407static void skb_ext_put_mctp(struct mctp_flow *flow)
6408{
6409 if (flow->key)
6410 mctp_key_unref(flow->key);
6411}
6412#endif
6413
Florian Westphaldf5042f2018-12-18 17:15:16 +01006414void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id)
6415{
6416 struct skb_ext *ext = skb->extensions;
6417
6418 skb->active_extensions &= ~(1 << id);
6419 if (skb->active_extensions == 0) {
6420 skb->extensions = NULL;
6421 __skb_ext_put(ext);
Florian Westphal41650792018-12-18 17:15:27 +01006422#ifdef CONFIG_XFRM
6423 } else if (id == SKB_EXT_SEC_PATH &&
6424 refcount_read(&ext->refcnt) == 1) {
6425 struct sec_path *sp = skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH);
6426
6427 skb_ext_put_sp(sp);
6428 sp->len = 0;
6429#endif
Florian Westphaldf5042f2018-12-18 17:15:16 +01006430 }
6431}
6432EXPORT_SYMBOL(__skb_ext_del);
6433
6434void __skb_ext_put(struct skb_ext *ext)
6435{
6436 /* If this is last clone, nothing can increment
6437 * it after check passes. Avoids one atomic op.
6438 */
6439 if (refcount_read(&ext->refcnt) == 1)
6440 goto free_now;
6441
6442 if (!refcount_dec_and_test(&ext->refcnt))
6443 return;
6444free_now:
Florian Westphal41650792018-12-18 17:15:27 +01006445#ifdef CONFIG_XFRM
6446 if (__skb_ext_exist(ext, SKB_EXT_SEC_PATH))
6447 skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH));
6448#endif
Jeremy Kerr78476d32021-10-29 11:01:44 +08006449#ifdef CONFIG_MCTP_FLOWS
6450 if (__skb_ext_exist(ext, SKB_EXT_MCTP))
6451 skb_ext_put_mctp(skb_ext_get_ptr(ext, SKB_EXT_MCTP));
6452#endif
Florian Westphal41650792018-12-18 17:15:27 +01006453
Florian Westphaldf5042f2018-12-18 17:15:16 +01006454 kmem_cache_free(skbuff_ext_cache, ext);
6455}
6456EXPORT_SYMBOL(__skb_ext_put);
6457#endif /* CONFIG_SKB_EXTENSIONS */