blob: 93443a01ab39ecf865426cef381d2c11c7967f1f [file] [log] [blame]
Thomas Gleixner2874c5f2019-05-27 08:55:01 +02001// SPDX-License-Identifier: GPL-2.0-or-later
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
3 * Routines having to do with the 'struct sk_buff' memory handlers.
4 *
Alan Cox113aa832008-10-13 19:01:08 -07005 * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk>
Linus Torvalds1da177e2005-04-16 15:20:36 -07006 * Florian La Roche <rzsfl@rz.uni-sb.de>
7 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07008 * Fixes:
9 * Alan Cox : Fixed the worst of the load
10 * balancer bugs.
11 * Dave Platt : Interrupt stacking fix.
12 * Richard Kooijman : Timestamp fixes.
13 * Alan Cox : Changed buffer format.
14 * Alan Cox : destructor hook for AF_UNIX etc.
15 * Linus Torvalds : Better skb_clone.
16 * Alan Cox : Added skb_copy.
17 * Alan Cox : Added all the changed routines Linus
18 * only put in the headers
19 * Ray VanTassle : Fixed --skb->lock in free
20 * Alan Cox : skb_copy copy arp field
21 * Andi Kleen : slabified it.
22 * Robert Olsson : Removed skb_head_pool
23 *
24 * NOTE:
25 * The __skb_ routines should be called with interrupts
26 * disabled, or you better be *real* sure that the operation is atomic
27 * with respect to whatever list is being frobbed (e.g. via lock_sock()
28 * or via disabling bottom half handlers, etc).
Linus Torvalds1da177e2005-04-16 15:20:36 -070029 */
30
31/*
32 * The functions in this file will not compile correctly with gcc 2.4.x
33 */
34
Joe Perchese005d192012-05-16 19:58:40 +000035#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36
Linus Torvalds1da177e2005-04-16 15:20:36 -070037#include <linux/module.h>
38#include <linux/types.h>
39#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070040#include <linux/mm.h>
41#include <linux/interrupt.h>
42#include <linux/in.h>
43#include <linux/inet.h>
44#include <linux/slab.h>
Florian Westphalde960aa2014-01-26 10:58:16 +010045#include <linux/tcp.h>
46#include <linux/udp.h>
Marcelo Ricardo Leitner90017ac2016-06-02 15:05:43 -030047#include <linux/sctp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070048#include <linux/netdevice.h>
49#ifdef CONFIG_NET_CLS_ACT
50#include <net/pkt_sched.h>
51#endif
52#include <linux/string.h>
53#include <linux/skbuff.h>
Jens Axboe9c55e012007-11-06 23:30:13 -080054#include <linux/splice.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070055#include <linux/cache.h>
56#include <linux/rtnetlink.h>
57#include <linux/init.h>
David Howells716ea3a2007-04-02 20:19:53 -070058#include <linux/scatterlist.h>
Patrick Ohlyac45f602009-02-12 05:03:37 +000059#include <linux/errqueue.h>
Linus Torvalds268bb0c2011-05-20 12:50:29 -070060#include <linux/prefetch.h>
Vlad Yasevich0d5501c2014-08-08 14:42:13 -040061#include <linux/if_vlan.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070062
63#include <net/protocol.h>
64#include <net/dst.h>
65#include <net/sock.h>
66#include <net/checksum.h>
Paul Durranted1f50c2014-01-09 10:02:46 +000067#include <net/ip6_checksum.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070068#include <net/xfrm.h>
John Hurley8822e272019-07-07 15:01:54 +010069#include <net/mpls.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070070
Linus Torvalds7c0f6ba2016-12-24 11:46:01 -080071#include <linux/uaccess.h>
Steven Rostedtad8d75f2009-04-14 19:39:12 -040072#include <trace/events/skb.h>
Eric Dumazet51c56b02012-04-05 11:35:15 +020073#include <linux/highmem.h>
Willem de Bruijnb245be12015-01-30 13:29:32 -050074#include <linux/capability.h>
75#include <linux/user_namespace.h>
Matteo Croce2544af02019-05-29 17:13:48 +020076#include <linux/indirect_call_wrapper.h>
Al Viroa1f8e7f72006-10-19 16:08:53 -040077
Bart Van Assche7b7ed882019-03-25 09:17:23 -070078#include "datagram.h"
79
Alexey Dobriyan08009a72018-02-24 21:20:33 +030080struct kmem_cache *skbuff_head_cache __ro_after_init;
81static struct kmem_cache *skbuff_fclone_cache __ro_after_init;
Florian Westphaldf5042f2018-12-18 17:15:16 +010082#ifdef CONFIG_SKB_EXTENSIONS
83static struct kmem_cache *skbuff_ext_cache __ro_after_init;
84#endif
Hans Westgaard Ry5f74f822016-02-03 09:26:57 +010085int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS;
86EXPORT_SYMBOL(sysctl_max_skb_frags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070087
Linus Torvalds1da177e2005-04-16 15:20:36 -070088/**
Jean Sacrenf05de732013-02-11 13:30:38 +000089 * skb_panic - private function for out-of-line support
90 * @skb: buffer
91 * @sz: size
92 * @addr: address
James Hogan99d58512013-02-13 11:20:27 +000093 * @msg: skb_over_panic or skb_under_panic
Linus Torvalds1da177e2005-04-16 15:20:36 -070094 *
Jean Sacrenf05de732013-02-11 13:30:38 +000095 * Out-of-line support for skb_put() and skb_push().
96 * Called via the wrapper skb_over_panic() or skb_under_panic().
97 * Keep out of line to prevent kernel bloat.
98 * __builtin_return_address is not used because it is not always reliable.
Linus Torvalds1da177e2005-04-16 15:20:36 -070099 */
Jean Sacrenf05de732013-02-11 13:30:38 +0000100static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr,
James Hogan99d58512013-02-13 11:20:27 +0000101 const char msg[])
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102{
Joe Perchese005d192012-05-16 19:58:40 +0000103 pr_emerg("%s: text:%p len:%d put:%d head:%p data:%p tail:%#lx end:%#lx dev:%s\n",
James Hogan99d58512013-02-13 11:20:27 +0000104 msg, addr, skb->len, sz, skb->head, skb->data,
Joe Perchese005d192012-05-16 19:58:40 +0000105 (unsigned long)skb->tail, (unsigned long)skb->end,
106 skb->dev ? skb->dev->name : "<NULL>");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107 BUG();
108}
109
Jean Sacrenf05de732013-02-11 13:30:38 +0000110static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111{
Jean Sacrenf05de732013-02-11 13:30:38 +0000112 skb_panic(skb, sz, addr, __func__);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113}
114
Jean Sacrenf05de732013-02-11 13:30:38 +0000115static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr)
116{
117 skb_panic(skb, sz, addr, __func__);
118}
Mel Gormanc93bdd02012-07-31 16:44:19 -0700119
120/*
121 * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells
122 * the caller if emergency pfmemalloc reserves are being used. If it is and
123 * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves
124 * may be used. Otherwise, the packet data may be discarded until enough
125 * memory is free
126 */
127#define kmalloc_reserve(size, gfp, node, pfmemalloc) \
128 __kmalloc_reserve(size, gfp, node, _RET_IP_, pfmemalloc)
stephen hemminger61c5e882012-12-28 18:24:28 +0000129
130static void *__kmalloc_reserve(size_t size, gfp_t flags, int node,
131 unsigned long ip, bool *pfmemalloc)
Mel Gormanc93bdd02012-07-31 16:44:19 -0700132{
133 void *obj;
134 bool ret_pfmemalloc = false;
135
136 /*
137 * Try a regular allocation, when that fails and we're not entitled
138 * to the reserves, fail.
139 */
140 obj = kmalloc_node_track_caller(size,
141 flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
142 node);
143 if (obj || !(gfp_pfmemalloc_allowed(flags)))
144 goto out;
145
146 /* Try again but now we are using pfmemalloc reserves */
147 ret_pfmemalloc = true;
148 obj = kmalloc_node_track_caller(size, flags, node);
149
150out:
151 if (pfmemalloc)
152 *pfmemalloc = ret_pfmemalloc;
153
154 return obj;
155}
156
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157/* Allocate a new skbuff. We do this ourselves so we can fill in a few
158 * 'private' fields and also do memory statistics to find all the
159 * [BEEP] leaks.
160 *
161 */
162
163/**
David S. Millerd179cd12005-08-17 14:57:30 -0700164 * __alloc_skb - allocate a network buffer
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165 * @size: size to allocate
166 * @gfp_mask: allocation mask
Mel Gormanc93bdd02012-07-31 16:44:19 -0700167 * @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache
168 * instead of head cache and allocate a cloned (child) skb.
169 * If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for
170 * allocations in case the data is required for writeback
Christoph Hellwigb30973f2006-12-06 20:32:36 -0800171 * @node: numa node to allocate memory on
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172 *
173 * Allocate a new &sk_buff. The returned buffer has no headroom and a
Ben Hutchings94b60422012-06-06 15:23:37 +0000174 * tail room of at least size bytes. The object has a reference count
175 * of one. The return is the buffer. On a failure the return is %NULL.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700176 *
177 * Buffers may only be allocated from interrupts using a @gfp_mask of
178 * %GFP_ATOMIC.
179 */
Al Virodd0fc662005-10-07 07:46:04 +0100180struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
Mel Gormanc93bdd02012-07-31 16:44:19 -0700181 int flags, int node)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182{
Christoph Lametere18b8902006-12-06 20:33:20 -0800183 struct kmem_cache *cache;
Benjamin LaHaise4947d3e2006-01-03 14:06:50 -0800184 struct skb_shared_info *shinfo;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185 struct sk_buff *skb;
186 u8 *data;
Mel Gormanc93bdd02012-07-31 16:44:19 -0700187 bool pfmemalloc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188
Mel Gormanc93bdd02012-07-31 16:44:19 -0700189 cache = (flags & SKB_ALLOC_FCLONE)
190 ? skbuff_fclone_cache : skbuff_head_cache;
191
192 if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))
193 gfp_mask |= __GFP_MEMALLOC;
Herbert Xu8798b3f2006-01-23 16:32:45 -0800194
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195 /* Get the HEAD */
Christoph Hellwigb30973f2006-12-06 20:32:36 -0800196 skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700197 if (!skb)
198 goto out;
Eric Dumazetec7d2f22010-05-05 01:07:37 -0700199 prefetchw(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200
Eric Dumazet87fb4b72011-10-13 07:28:54 +0000201 /* We do our best to align skb_shared_info on a separate cache
202 * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives
203 * aligned memory blocks, unless SLUB/SLAB debug is enabled.
204 * Both skb->head and skb_shared_info are cache line aligned.
205 */
Tony Lindgrenbc417e32011-11-02 13:40:28 +0000206 size = SKB_DATA_ALIGN(size);
Eric Dumazet87fb4b72011-10-13 07:28:54 +0000207 size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
Mel Gormanc93bdd02012-07-31 16:44:19 -0700208 data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700209 if (!data)
210 goto nodata;
Eric Dumazet87fb4b72011-10-13 07:28:54 +0000211 /* kmalloc(size) might give us more room than requested.
212 * Put skb_shared_info exactly at the end of allocated zone,
213 * to allow max possible filling before reallocation.
214 */
215 size = SKB_WITH_OVERHEAD(ksize(data));
Eric Dumazetec7d2f22010-05-05 01:07:37 -0700216 prefetchw(data + size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217
Arnaldo Carvalho de Meloca0605a2007-03-19 10:48:59 -0300218 /*
Johannes Bergc8005782008-05-03 20:56:42 -0700219 * Only clear those fields we need to clear, not those that we will
220 * actually initialise below. Hence, don't put any more fields after
221 * the tail pointer in struct sk_buff!
Arnaldo Carvalho de Meloca0605a2007-03-19 10:48:59 -0300222 */
223 memset(skb, 0, offsetof(struct sk_buff, tail));
Eric Dumazet87fb4b72011-10-13 07:28:54 +0000224 /* Account for allocated memory : skb + skb->head */
225 skb->truesize = SKB_TRUESIZE(size);
Mel Gormanc93bdd02012-07-31 16:44:19 -0700226 skb->pfmemalloc = pfmemalloc;
Reshetova, Elena63354792017-06-30 13:07:58 +0300227 refcount_set(&skb->users, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700228 skb->head = data;
229 skb->data = data;
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -0700230 skb_reset_tail_pointer(skb);
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -0700231 skb->end = skb->tail + size;
Cong Wang35d04612013-05-29 15:16:05 +0800232 skb->mac_header = (typeof(skb->mac_header))~0U;
233 skb->transport_header = (typeof(skb->transport_header))~0U;
Stephen Hemminger19633e12009-06-17 05:23:27 +0000234
Benjamin LaHaise4947d3e2006-01-03 14:06:50 -0800235 /* make sure we initialize shinfo sequentially */
236 shinfo = skb_shinfo(skb);
Eric Dumazetec7d2f22010-05-05 01:07:37 -0700237 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
Benjamin LaHaise4947d3e2006-01-03 14:06:50 -0800238 atomic_set(&shinfo->dataref, 1);
Benjamin LaHaise4947d3e2006-01-03 14:06:50 -0800239
Mel Gormanc93bdd02012-07-31 16:44:19 -0700240 if (flags & SKB_ALLOC_FCLONE) {
Eric Dumazetd0bf4a92014-09-29 13:29:15 -0700241 struct sk_buff_fclones *fclones;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242
Eric Dumazetd0bf4a92014-09-29 13:29:15 -0700243 fclones = container_of(skb, struct sk_buff_fclones, skb1);
244
David S. Millerd179cd12005-08-17 14:57:30 -0700245 skb->fclone = SKB_FCLONE_ORIG;
Reshetova, Elena26385952017-06-30 13:07:59 +0300246 refcount_set(&fclones->fclone_ref, 1);
David S. Millerd179cd12005-08-17 14:57:30 -0700247
Eric Dumazet6ffe75e2014-12-03 17:04:39 -0800248 fclones->skb2.fclone = SKB_FCLONE_CLONE;
David S. Millerd179cd12005-08-17 14:57:30 -0700249 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250out:
251 return skb;
252nodata:
Herbert Xu8798b3f2006-01-23 16:32:45 -0800253 kmem_cache_free(cache, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254 skb = NULL;
255 goto out;
256}
David S. Millerb4ac530fc2009-02-10 02:09:24 -0800257EXPORT_SYMBOL(__alloc_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258
Jesper Dangaard Brouerba0509b2019-04-12 17:07:37 +0200259/* Caller must provide SKB that is memset cleared */
260static struct sk_buff *__build_skb_around(struct sk_buff *skb,
261 void *data, unsigned int frag_size)
262{
263 struct skb_shared_info *shinfo;
264 unsigned int size = frag_size ? : ksize(data);
265
266 size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
267
268 /* Assumes caller memset cleared SKB */
269 skb->truesize = SKB_TRUESIZE(size);
270 refcount_set(&skb->users, 1);
271 skb->head = data;
272 skb->data = data;
273 skb_reset_tail_pointer(skb);
274 skb->end = skb->tail + size;
275 skb->mac_header = (typeof(skb->mac_header))~0U;
276 skb->transport_header = (typeof(skb->transport_header))~0U;
277
278 /* make sure we initialize shinfo sequentially */
279 shinfo = skb_shinfo(skb);
280 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
281 atomic_set(&shinfo->dataref, 1);
282
283 return skb;
284}
285
Linus Torvalds1da177e2005-04-16 15:20:36 -0700286/**
Eric Dumazet2ea2f622015-04-24 16:05:01 -0700287 * __build_skb - build a network buffer
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000288 * @data: data buffer provided by caller
Eric Dumazet2ea2f622015-04-24 16:05:01 -0700289 * @frag_size: size of data, or 0 if head was kmalloced
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000290 *
291 * Allocate a new &sk_buff. Caller provides space holding head and
Florian Fainellideceb4c2013-07-23 20:22:39 +0100292 * skb_shared_info. @data must have been allocated by kmalloc() only if
Eric Dumazet2ea2f622015-04-24 16:05:01 -0700293 * @frag_size is 0, otherwise data should come from the page allocator
294 * or vmalloc()
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000295 * The return is the new skb buffer.
296 * On a failure the return is %NULL, and @data is not freed.
297 * Notes :
298 * Before IO, driver allocates only data buffer where NIC put incoming frame
299 * Driver should add room at head (NET_SKB_PAD) and
300 * MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info))
301 * After IO, driver calls build_skb(), to allocate sk_buff and populate it
302 * before giving packet to stack.
303 * RX rings only contains data buffers, not full skbs.
304 */
Eric Dumazet2ea2f622015-04-24 16:05:01 -0700305struct sk_buff *__build_skb(void *data, unsigned int frag_size)
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000306{
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000307 struct sk_buff *skb;
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000308
309 skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC);
Jesper Dangaard Brouerba0509b2019-04-12 17:07:37 +0200310 if (unlikely(!skb))
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000311 return NULL;
312
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000313 memset(skb, 0, offsetof(struct sk_buff, tail));
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000314
Jesper Dangaard Brouerba0509b2019-04-12 17:07:37 +0200315 return __build_skb_around(skb, data, frag_size);
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000316}
Eric Dumazet2ea2f622015-04-24 16:05:01 -0700317
318/* build_skb() is wrapper over __build_skb(), that specifically
319 * takes care of skb->head and skb->pfmemalloc
320 * This means that if @frag_size is not zero, then @data must be backed
321 * by a page fragment, not kmalloc() or vmalloc()
322 */
323struct sk_buff *build_skb(void *data, unsigned int frag_size)
324{
325 struct sk_buff *skb = __build_skb(data, frag_size);
326
327 if (skb && frag_size) {
328 skb->head_frag = 1;
Michal Hocko2f064f32015-08-21 14:11:51 -0700329 if (page_is_pfmemalloc(virt_to_head_page(data)))
Eric Dumazet2ea2f622015-04-24 16:05:01 -0700330 skb->pfmemalloc = 1;
331 }
332 return skb;
333}
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000334EXPORT_SYMBOL(build_skb);
335
Jesper Dangaard Brouerba0509b2019-04-12 17:07:37 +0200336/**
337 * build_skb_around - build a network buffer around provided skb
338 * @skb: sk_buff provide by caller, must be memset cleared
339 * @data: data buffer provided by caller
340 * @frag_size: size of data, or 0 if head was kmalloced
341 */
342struct sk_buff *build_skb_around(struct sk_buff *skb,
343 void *data, unsigned int frag_size)
344{
345 if (unlikely(!skb))
346 return NULL;
347
348 skb = __build_skb_around(skb, data, frag_size);
349
350 if (skb && frag_size) {
351 skb->head_frag = 1;
352 if (page_is_pfmemalloc(virt_to_head_page(data)))
353 skb->pfmemalloc = 1;
354 }
355 return skb;
356}
357EXPORT_SYMBOL(build_skb_around);
358
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100359#define NAPI_SKB_CACHE_SIZE 64
360
361struct napi_alloc_cache {
362 struct page_frag_cache page;
Alexey Dobriyane0d79242016-11-19 03:47:56 +0300363 unsigned int skb_count;
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100364 void *skb_cache[NAPI_SKB_CACHE_SIZE];
365};
366
Alexander Duyckb63ae8c2015-05-06 21:11:57 -0700367static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache);
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100368static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache);
Alexander Duyckffde7322014-12-09 19:40:42 -0800369
Alexander Duyckffde7322014-12-09 19:40:42 -0800370static void *__napi_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
371{
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100372 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
Alexander Duyck94519802015-05-06 21:11:40 -0700373
Alexander Duyck8c2dd3e2017-01-10 16:58:06 -0800374 return page_frag_alloc(&nc->page, fragsz, gfp_mask);
Alexander Duyckffde7322014-12-09 19:40:42 -0800375}
376
377void *napi_alloc_frag(unsigned int fragsz)
378{
Alexander Duyck3bed3cc2019-02-15 14:44:18 -0800379 fragsz = SKB_DATA_ALIGN(fragsz);
380
Mel Gorman453f85d2017-11-15 17:38:03 -0800381 return __napi_alloc_frag(fragsz, GFP_ATOMIC);
Alexander Duyckffde7322014-12-09 19:40:42 -0800382}
383EXPORT_SYMBOL(napi_alloc_frag);
384
Eric Dumazet6f532612012-05-18 05:12:12 +0000385/**
Sebastian Andrzej Siewior7ba7aea2019-06-07 21:20:34 +0200386 * netdev_alloc_frag - allocate a page fragment
387 * @fragsz: fragment size
388 *
389 * Allocates a frag from a page for receive buffer.
390 * Uses GFP_ATOMIC allocations.
391 */
392void *netdev_alloc_frag(unsigned int fragsz)
393{
394 struct page_frag_cache *nc;
395 void *data;
396
397 fragsz = SKB_DATA_ALIGN(fragsz);
398 if (in_irq() || irqs_disabled()) {
399 nc = this_cpu_ptr(&netdev_alloc_cache);
400 data = page_frag_alloc(nc, fragsz, GFP_ATOMIC);
401 } else {
402 local_bh_disable();
403 data = __napi_alloc_frag(fragsz, GFP_ATOMIC);
404 local_bh_enable();
405 }
406 return data;
407}
408EXPORT_SYMBOL(netdev_alloc_frag);
409
410/**
Alexander Duyckfd11a832014-12-09 19:40:49 -0800411 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device
412 * @dev: network device to receive on
Masanari Iidad7499162015-08-24 22:56:54 +0900413 * @len: length to allocate
Alexander Duyckfd11a832014-12-09 19:40:49 -0800414 * @gfp_mask: get_free_pages mask, passed to alloc_skb
415 *
416 * Allocate a new &sk_buff and assign it a usage count of one. The
417 * buffer has NET_SKB_PAD headroom built in. Users should allocate
418 * the headroom they think they need without accounting for the
419 * built in space. The built in space is used for optimisations.
420 *
421 * %NULL is returned if there is no free memory.
422 */
Alexander Duyck94519802015-05-06 21:11:40 -0700423struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len,
424 gfp_t gfp_mask)
Alexander Duyckfd11a832014-12-09 19:40:49 -0800425{
Alexander Duyckb63ae8c2015-05-06 21:11:57 -0700426 struct page_frag_cache *nc;
Alexander Duyckfd11a832014-12-09 19:40:49 -0800427 struct sk_buff *skb;
Alexander Duyck94519802015-05-06 21:11:40 -0700428 bool pfmemalloc;
429 void *data;
Alexander Duyckfd11a832014-12-09 19:40:49 -0800430
Alexander Duyck94519802015-05-06 21:11:40 -0700431 len += NET_SKB_PAD;
Alexander Duyckfd11a832014-12-09 19:40:49 -0800432
Alexander Duyck94519802015-05-06 21:11:40 -0700433 if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) ||
Mel Gormand0164ad2015-11-06 16:28:21 -0800434 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
Alexander Duycka080e7b2015-05-13 13:34:13 -0700435 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
436 if (!skb)
437 goto skb_fail;
438 goto skb_success;
439 }
Alexander Duyck94519802015-05-06 21:11:40 -0700440
441 len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
442 len = SKB_DATA_ALIGN(len);
443
444 if (sk_memalloc_socks())
445 gfp_mask |= __GFP_MEMALLOC;
446
Sebastian Andrzej Siewior92dcabd2019-06-07 21:20:35 +0200447 if (in_irq() || irqs_disabled()) {
448 nc = this_cpu_ptr(&netdev_alloc_cache);
449 data = page_frag_alloc(nc, len, gfp_mask);
450 pfmemalloc = nc->pfmemalloc;
451 } else {
452 local_bh_disable();
453 nc = this_cpu_ptr(&napi_alloc_cache.page);
454 data = page_frag_alloc(nc, len, gfp_mask);
455 pfmemalloc = nc->pfmemalloc;
456 local_bh_enable();
457 }
Alexander Duyck94519802015-05-06 21:11:40 -0700458
459 if (unlikely(!data))
460 return NULL;
461
462 skb = __build_skb(data, len);
463 if (unlikely(!skb)) {
Alexander Duyck181edb22015-05-06 21:12:03 -0700464 skb_free_frag(data);
Alexander Duyck94519802015-05-06 21:11:40 -0700465 return NULL;
Christoph Hellwig7b2e4972006-08-07 16:09:04 -0700466 }
Alexander Duyckfd11a832014-12-09 19:40:49 -0800467
Alexander Duyck94519802015-05-06 21:11:40 -0700468 /* use OR instead of assignment to avoid clearing of bits in mask */
469 if (pfmemalloc)
470 skb->pfmemalloc = 1;
471 skb->head_frag = 1;
472
Alexander Duycka080e7b2015-05-13 13:34:13 -0700473skb_success:
Alexander Duyck94519802015-05-06 21:11:40 -0700474 skb_reserve(skb, NET_SKB_PAD);
475 skb->dev = dev;
476
Alexander Duycka080e7b2015-05-13 13:34:13 -0700477skb_fail:
Christoph Hellwig8af27452006-07-31 22:35:23 -0700478 return skb;
479}
David S. Millerb4ac530fc2009-02-10 02:09:24 -0800480EXPORT_SYMBOL(__netdev_alloc_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700481
Alexander Duyckfd11a832014-12-09 19:40:49 -0800482/**
483 * __napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance
484 * @napi: napi instance this buffer was allocated for
Masanari Iidad7499162015-08-24 22:56:54 +0900485 * @len: length to allocate
Alexander Duyckfd11a832014-12-09 19:40:49 -0800486 * @gfp_mask: get_free_pages mask, passed to alloc_skb and alloc_pages
487 *
488 * Allocate a new sk_buff for use in NAPI receive. This buffer will
489 * attempt to allocate the head from a special reserved region used
490 * only for NAPI Rx allocation. By doing this we can save several
491 * CPU cycles by avoiding having to disable and re-enable IRQs.
492 *
493 * %NULL is returned if there is no free memory.
494 */
Alexander Duyck94519802015-05-06 21:11:40 -0700495struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len,
496 gfp_t gfp_mask)
Alexander Duyckfd11a832014-12-09 19:40:49 -0800497{
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100498 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
Alexander Duyckfd11a832014-12-09 19:40:49 -0800499 struct sk_buff *skb;
Alexander Duyck94519802015-05-06 21:11:40 -0700500 void *data;
Alexander Duyckfd11a832014-12-09 19:40:49 -0800501
Alexander Duyck94519802015-05-06 21:11:40 -0700502 len += NET_SKB_PAD + NET_IP_ALIGN;
Alexander Duyckfd11a832014-12-09 19:40:49 -0800503
Alexander Duyck94519802015-05-06 21:11:40 -0700504 if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) ||
Mel Gormand0164ad2015-11-06 16:28:21 -0800505 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
Alexander Duycka080e7b2015-05-13 13:34:13 -0700506 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
507 if (!skb)
508 goto skb_fail;
509 goto skb_success;
510 }
Alexander Duyck94519802015-05-06 21:11:40 -0700511
512 len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
513 len = SKB_DATA_ALIGN(len);
514
515 if (sk_memalloc_socks())
516 gfp_mask |= __GFP_MEMALLOC;
517
Alexander Duyck8c2dd3e2017-01-10 16:58:06 -0800518 data = page_frag_alloc(&nc->page, len, gfp_mask);
Alexander Duyck94519802015-05-06 21:11:40 -0700519 if (unlikely(!data))
520 return NULL;
521
522 skb = __build_skb(data, len);
523 if (unlikely(!skb)) {
Alexander Duyck181edb22015-05-06 21:12:03 -0700524 skb_free_frag(data);
Alexander Duyck94519802015-05-06 21:11:40 -0700525 return NULL;
Alexander Duyckfd11a832014-12-09 19:40:49 -0800526 }
527
Alexander Duyck94519802015-05-06 21:11:40 -0700528 /* use OR instead of assignment to avoid clearing of bits in mask */
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100529 if (nc->page.pfmemalloc)
Alexander Duyck94519802015-05-06 21:11:40 -0700530 skb->pfmemalloc = 1;
531 skb->head_frag = 1;
532
Alexander Duycka080e7b2015-05-13 13:34:13 -0700533skb_success:
Alexander Duyck94519802015-05-06 21:11:40 -0700534 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
535 skb->dev = napi->dev;
536
Alexander Duycka080e7b2015-05-13 13:34:13 -0700537skb_fail:
Alexander Duyckfd11a832014-12-09 19:40:49 -0800538 return skb;
539}
540EXPORT_SYMBOL(__napi_alloc_skb);
541
Peter Zijlstra654bed12008-10-07 14:22:33 -0700542void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
Eric Dumazet50269e12012-03-23 23:59:33 +0000543 int size, unsigned int truesize)
Peter Zijlstra654bed12008-10-07 14:22:33 -0700544{
545 skb_fill_page_desc(skb, i, page, off, size);
546 skb->len += size;
547 skb->data_len += size;
Eric Dumazet50269e12012-03-23 23:59:33 +0000548 skb->truesize += truesize;
Peter Zijlstra654bed12008-10-07 14:22:33 -0700549}
550EXPORT_SYMBOL(skb_add_rx_frag);
551
Jason Wangf8e617e2013-11-01 14:07:47 +0800552void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
553 unsigned int truesize)
554{
555 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
556
557 skb_frag_size_add(frag, size);
558 skb->len += size;
559 skb->data_len += size;
560 skb->truesize += truesize;
561}
562EXPORT_SYMBOL(skb_coalesce_rx_frag);
563
Herbert Xu27b437c2006-07-13 19:26:39 -0700564static void skb_drop_list(struct sk_buff **listp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700565{
Eric Dumazetbd8a7032013-06-24 06:26:00 -0700566 kfree_skb_list(*listp);
Herbert Xu27b437c2006-07-13 19:26:39 -0700567 *listp = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700568}
569
Herbert Xu27b437c2006-07-13 19:26:39 -0700570static inline void skb_drop_fraglist(struct sk_buff *skb)
571{
572 skb_drop_list(&skb_shinfo(skb)->frag_list);
573}
574
Linus Torvalds1da177e2005-04-16 15:20:36 -0700575static void skb_clone_fraglist(struct sk_buff *skb)
576{
577 struct sk_buff *list;
578
David S. Millerfbb398a2009-06-09 00:18:59 -0700579 skb_walk_frags(skb, list)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580 skb_get(list);
581}
582
Eric Dumazetd3836f22012-04-27 00:33:38 +0000583static void skb_free_head(struct sk_buff *skb)
584{
Alexander Duyck181edb22015-05-06 21:12:03 -0700585 unsigned char *head = skb->head;
586
Eric Dumazetd3836f22012-04-27 00:33:38 +0000587 if (skb->head_frag)
Alexander Duyck181edb22015-05-06 21:12:03 -0700588 skb_free_frag(head);
Eric Dumazetd3836f22012-04-27 00:33:38 +0000589 else
Alexander Duyck181edb22015-05-06 21:12:03 -0700590 kfree(head);
Eric Dumazetd3836f22012-04-27 00:33:38 +0000591}
592
Adrian Bunk5bba1712006-06-29 13:02:35 -0700593static void skb_release_data(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594{
Eric Dumazetff04a772014-09-23 18:39:30 -0700595 struct skb_shared_info *shinfo = skb_shinfo(skb);
596 int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700597
Eric Dumazetff04a772014-09-23 18:39:30 -0700598 if (skb->cloned &&
599 atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
600 &shinfo->dataref))
601 return;
Shirley Maa6686f22011-07-06 12:22:12 +0000602
Eric Dumazetff04a772014-09-23 18:39:30 -0700603 for (i = 0; i < shinfo->nr_frags; i++)
604 __skb_frag_unref(&shinfo->frags[i]);
Shirley Maa6686f22011-07-06 12:22:12 +0000605
Eric Dumazetff04a772014-09-23 18:39:30 -0700606 if (shinfo->frag_list)
607 kfree_skb_list(shinfo->frag_list);
608
Willem de Bruijn1f8b9772017-08-03 16:29:41 -0400609 skb_zcopy_clear(skb, true);
Eric Dumazetff04a772014-09-23 18:39:30 -0700610 skb_free_head(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700611}
612
613/*
614 * Free an skbuff by memory without cleaning the state.
615 */
Herbert Xu2d4baff2007-11-26 23:11:19 +0800616static void kfree_skbmem(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617{
Eric Dumazetd0bf4a92014-09-29 13:29:15 -0700618 struct sk_buff_fclones *fclones;
David S. Millerd179cd12005-08-17 14:57:30 -0700619
David S. Millerd179cd12005-08-17 14:57:30 -0700620 switch (skb->fclone) {
621 case SKB_FCLONE_UNAVAILABLE:
622 kmem_cache_free(skbuff_head_cache, skb);
Eric Dumazet6ffe75e2014-12-03 17:04:39 -0800623 return;
David S. Millerd179cd12005-08-17 14:57:30 -0700624
625 case SKB_FCLONE_ORIG:
Eric Dumazetd0bf4a92014-09-29 13:29:15 -0700626 fclones = container_of(skb, struct sk_buff_fclones, skb1);
Eric Dumazet6ffe75e2014-12-03 17:04:39 -0800627
628 /* We usually free the clone (TX completion) before original skb
629 * This test would have no chance to be true for the clone,
630 * while here, branch prediction will be good.
631 */
Reshetova, Elena26385952017-06-30 13:07:59 +0300632 if (refcount_read(&fclones->fclone_ref) == 1)
Eric Dumazet6ffe75e2014-12-03 17:04:39 -0800633 goto fastpath;
David S. Millerd179cd12005-08-17 14:57:30 -0700634 break;
635
Eric Dumazet6ffe75e2014-12-03 17:04:39 -0800636 default: /* SKB_FCLONE_CLONE */
Eric Dumazetd0bf4a92014-09-29 13:29:15 -0700637 fclones = container_of(skb, struct sk_buff_fclones, skb2);
David S. Millerd179cd12005-08-17 14:57:30 -0700638 break;
Stephen Hemminger3ff50b72007-04-20 17:09:22 -0700639 }
Reshetova, Elena26385952017-06-30 13:07:59 +0300640 if (!refcount_dec_and_test(&fclones->fclone_ref))
Eric Dumazet6ffe75e2014-12-03 17:04:39 -0800641 return;
642fastpath:
643 kmem_cache_free(skbuff_fclone_cache, fclones);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700644}
645
Paolo Abeni0a463c72017-06-12 11:23:42 +0200646void skb_release_head_state(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700647{
Eric Dumazetadf30902009-06-02 05:19:30 +0000648 skb_dst_drop(skb);
Stephen Hemminger9c2b3322005-04-19 22:39:42 -0700649 if (skb->destructor) {
650 WARN_ON(in_irq());
Linus Torvalds1da177e2005-04-16 15:20:36 -0700651 skb->destructor(skb);
652 }
Igor Maravića3bf7ae2011-12-12 02:58:22 +0000653#if IS_ENABLED(CONFIG_NF_CONNTRACK)
Florian Westphalcb9c6832017-01-23 18:21:56 +0100654 nf_conntrack_put(skb_nfct(skb));
KOVACS Krisztian2fc72c72011-01-12 20:25:08 +0100655#endif
Florian Westphaldf5042f2018-12-18 17:15:16 +0100656 skb_ext_put(skb);
Lennert Buytenhek04a4bb52008-10-01 02:33:12 -0700657}
658
659/* Free everything but the sk_buff shell. */
660static void skb_release_all(struct sk_buff *skb)
661{
662 skb_release_head_state(skb);
Florian Westphala28b1b92017-07-23 19:54:47 +0200663 if (likely(skb->head))
664 skb_release_data(skb);
Herbert Xu2d4baff2007-11-26 23:11:19 +0800665}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666
Herbert Xu2d4baff2007-11-26 23:11:19 +0800667/**
668 * __kfree_skb - private function
669 * @skb: buffer
670 *
671 * Free an sk_buff. Release anything attached to the buffer.
672 * Clean the state. This is an internal helper function. Users should
673 * always call kfree_skb
674 */
675
676void __kfree_skb(struct sk_buff *skb)
677{
678 skb_release_all(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700679 kfree_skbmem(skb);
680}
David S. Millerb4ac530fc2009-02-10 02:09:24 -0800681EXPORT_SYMBOL(__kfree_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700682
683/**
Jörn Engel231d06a2006-03-20 21:28:35 -0800684 * kfree_skb - free an sk_buff
685 * @skb: buffer to free
686 *
687 * Drop a reference to the buffer and free it if the usage count has
688 * hit zero.
689 */
690void kfree_skb(struct sk_buff *skb)
691{
Paolo Abeni3889a8032017-06-12 11:23:41 +0200692 if (!skb_unref(skb))
Jörn Engel231d06a2006-03-20 21:28:35 -0800693 return;
Paolo Abeni3889a8032017-06-12 11:23:41 +0200694
Neil Hormanead2ceb2009-03-11 09:49:55 +0000695 trace_kfree_skb(skb, __builtin_return_address(0));
Jörn Engel231d06a2006-03-20 21:28:35 -0800696 __kfree_skb(skb);
697}
David S. Millerb4ac530fc2009-02-10 02:09:24 -0800698EXPORT_SYMBOL(kfree_skb);
Jörn Engel231d06a2006-03-20 21:28:35 -0800699
Eric Dumazetbd8a7032013-06-24 06:26:00 -0700700void kfree_skb_list(struct sk_buff *segs)
701{
702 while (segs) {
703 struct sk_buff *next = segs->next;
704
705 kfree_skb(segs);
706 segs = next;
707 }
708}
709EXPORT_SYMBOL(kfree_skb_list);
710
Willem de Bruijn64131392019-07-07 05:51:55 -0400711/* Dump skb information and contents.
712 *
713 * Must only be called from net_ratelimit()-ed paths.
714 *
715 * Dumps up to can_dump_full whole packets if full_pkt, headers otherwise.
716 */
717void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt)
718{
719 static atomic_t can_dump_full = ATOMIC_INIT(5);
720 struct skb_shared_info *sh = skb_shinfo(skb);
721 struct net_device *dev = skb->dev;
722 struct sock *sk = skb->sk;
723 struct sk_buff *list_skb;
724 bool has_mac, has_trans;
725 int headroom, tailroom;
726 int i, len, seg_len;
727
728 if (full_pkt)
729 full_pkt = atomic_dec_if_positive(&can_dump_full) >= 0;
730
731 if (full_pkt)
732 len = skb->len;
733 else
734 len = min_t(int, skb->len, MAX_HEADER + 128);
735
736 headroom = skb_headroom(skb);
737 tailroom = skb_tailroom(skb);
738
739 has_mac = skb_mac_header_was_set(skb);
740 has_trans = skb_transport_header_was_set(skb);
741
742 printk("%sskb len=%u headroom=%u headlen=%u tailroom=%u\n"
743 "mac=(%d,%d) net=(%d,%d) trans=%d\n"
744 "shinfo(txflags=%u nr_frags=%u gso(size=%hu type=%u segs=%hu))\n"
745 "csum(0x%x ip_summed=%u complete_sw=%u valid=%u level=%u)\n"
746 "hash(0x%x sw=%u l4=%u) proto=0x%04x pkttype=%u iif=%d\n",
747 level, skb->len, headroom, skb_headlen(skb), tailroom,
748 has_mac ? skb->mac_header : -1,
749 has_mac ? skb_mac_header_len(skb) : -1,
750 skb->network_header,
751 has_trans ? skb_network_header_len(skb) : -1,
752 has_trans ? skb->transport_header : -1,
753 sh->tx_flags, sh->nr_frags,
754 sh->gso_size, sh->gso_type, sh->gso_segs,
755 skb->csum, skb->ip_summed, skb->csum_complete_sw,
756 skb->csum_valid, skb->csum_level,
757 skb->hash, skb->sw_hash, skb->l4_hash,
758 ntohs(skb->protocol), skb->pkt_type, skb->skb_iif);
759
760 if (dev)
761 printk("%sdev name=%s feat=0x%pNF\n",
762 level, dev->name, &dev->features);
763 if (sk)
764 printk("%ssk family=%hu type=%hu proto=%hu\n",
765 level, sk->sk_family, sk->sk_type, sk->sk_protocol);
766
767 if (full_pkt && headroom)
768 print_hex_dump(level, "skb headroom: ", DUMP_PREFIX_OFFSET,
769 16, 1, skb->head, headroom, false);
770
771 seg_len = min_t(int, skb_headlen(skb), len);
772 if (seg_len)
773 print_hex_dump(level, "skb linear: ", DUMP_PREFIX_OFFSET,
774 16, 1, skb->data, seg_len, false);
775 len -= seg_len;
776
777 if (full_pkt && tailroom)
778 print_hex_dump(level, "skb tailroom: ", DUMP_PREFIX_OFFSET,
779 16, 1, skb_tail_pointer(skb), tailroom, false);
780
781 for (i = 0; len && i < skb_shinfo(skb)->nr_frags; i++) {
782 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
783 u32 p_off, p_len, copied;
784 struct page *p;
785 u8 *vaddr;
786
787 skb_frag_foreach_page(frag, frag->page_offset,
788 skb_frag_size(frag), p, p_off, p_len,
789 copied) {
790 seg_len = min_t(int, p_len, len);
791 vaddr = kmap_atomic(p);
792 print_hex_dump(level, "skb frag: ",
793 DUMP_PREFIX_OFFSET,
794 16, 1, vaddr + p_off, seg_len, false);
795 kunmap_atomic(vaddr);
796 len -= seg_len;
797 if (!len)
798 break;
799 }
800 }
801
802 if (full_pkt && skb_has_frag_list(skb)) {
803 printk("skb fraglist:\n");
804 skb_walk_frags(skb, list_skb)
805 skb_dump(level, list_skb, true);
806 }
807}
808EXPORT_SYMBOL(skb_dump);
809
Stephen Hemmingerd1a203e2008-11-01 21:01:09 -0700810/**
Michael S. Tsirkin25121172012-11-01 09:16:28 +0000811 * skb_tx_error - report an sk_buff xmit error
812 * @skb: buffer that triggered an error
813 *
814 * Report xmit error if a device callback is tracking this skb.
815 * skb must be freed afterwards.
816 */
817void skb_tx_error(struct sk_buff *skb)
818{
Willem de Bruijn1f8b9772017-08-03 16:29:41 -0400819 skb_zcopy_clear(skb, true);
Michael S. Tsirkin25121172012-11-01 09:16:28 +0000820}
821EXPORT_SYMBOL(skb_tx_error);
822
823/**
Neil Hormanead2ceb2009-03-11 09:49:55 +0000824 * consume_skb - free an skbuff
825 * @skb: buffer to free
826 *
827 * Drop a ref to the buffer and free it if the usage count has hit zero
828 * Functions identically to kfree_skb, but kfree_skb assumes that the frame
829 * is being dropped after a failure and notes that
830 */
831void consume_skb(struct sk_buff *skb)
832{
Paolo Abeni3889a8032017-06-12 11:23:41 +0200833 if (!skb_unref(skb))
Neil Hormanead2ceb2009-03-11 09:49:55 +0000834 return;
Paolo Abeni3889a8032017-06-12 11:23:41 +0200835
Koki Sanagi07dc22e2010-08-23 18:46:12 +0900836 trace_consume_skb(skb);
Neil Hormanead2ceb2009-03-11 09:49:55 +0000837 __kfree_skb(skb);
838}
839EXPORT_SYMBOL(consume_skb);
840
Paolo Abeni0a463c72017-06-12 11:23:42 +0200841/**
842 * consume_stateless_skb - free an skbuff, assuming it is stateless
843 * @skb: buffer to free
844 *
Paolo Abenica2c1412017-09-06 14:44:36 +0200845 * Alike consume_skb(), but this variant assumes that this is the last
846 * skb reference and all the head states have been already dropped
Paolo Abeni0a463c72017-06-12 11:23:42 +0200847 */
Paolo Abenica2c1412017-09-06 14:44:36 +0200848void __consume_stateless_skb(struct sk_buff *skb)
Paolo Abeni0a463c72017-06-12 11:23:42 +0200849{
Paolo Abeni0a463c72017-06-12 11:23:42 +0200850 trace_consume_skb(skb);
Florian Westphal06dc75a2017-07-17 18:56:54 +0200851 skb_release_data(skb);
Paolo Abeni0a463c72017-06-12 11:23:42 +0200852 kfree_skbmem(skb);
853}
854
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100855void __kfree_skb_flush(void)
856{
857 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
858
859 /* flush skb_cache if containing objects */
860 if (nc->skb_count) {
861 kmem_cache_free_bulk(skbuff_head_cache, nc->skb_count,
862 nc->skb_cache);
863 nc->skb_count = 0;
864 }
865}
866
Jesper Dangaard Brouer15fad712016-02-08 13:15:04 +0100867static inline void _kfree_skb_defer(struct sk_buff *skb)
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100868{
869 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
870
871 /* drop skb->head and call any destructors for packet */
872 skb_release_all(skb);
873
874 /* record skb to CPU local list */
875 nc->skb_cache[nc->skb_count++] = skb;
876
877#ifdef CONFIG_SLUB
878 /* SLUB writes into objects when freeing */
879 prefetchw(skb);
880#endif
881
882 /* flush skb_cache if it is filled */
883 if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) {
884 kmem_cache_free_bulk(skbuff_head_cache, NAPI_SKB_CACHE_SIZE,
885 nc->skb_cache);
886 nc->skb_count = 0;
887 }
888}
Jesper Dangaard Brouer15fad712016-02-08 13:15:04 +0100889void __kfree_skb_defer(struct sk_buff *skb)
890{
891 _kfree_skb_defer(skb);
892}
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100893
894void napi_consume_skb(struct sk_buff *skb, int budget)
895{
896 if (unlikely(!skb))
897 return;
898
Jesper Dangaard Brouer885eb0a2016-03-11 09:43:58 +0100899 /* Zero budget indicate non-NAPI context called us, like netpoll */
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100900 if (unlikely(!budget)) {
Jesper Dangaard Brouer885eb0a2016-03-11 09:43:58 +0100901 dev_consume_skb_any(skb);
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100902 return;
903 }
904
Paolo Abeni76088942017-06-14 11:48:48 +0200905 if (!skb_unref(skb))
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100906 return;
Paolo Abeni76088942017-06-14 11:48:48 +0200907
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100908 /* if reaching here SKB is ready to free */
909 trace_consume_skb(skb);
910
911 /* if SKB is a clone, don't handle this case */
Eric Dumazetabbdb5a2016-03-20 11:27:47 -0700912 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) {
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100913 __kfree_skb(skb);
914 return;
915 }
916
Jesper Dangaard Brouer15fad712016-02-08 13:15:04 +0100917 _kfree_skb_defer(skb);
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +0100918}
919EXPORT_SYMBOL(napi_consume_skb);
920
Eric Dumazetb1937222014-09-28 22:18:47 -0700921/* Make sure a field is enclosed inside headers_start/headers_end section */
922#define CHECK_SKB_FIELD(field) \
923 BUILD_BUG_ON(offsetof(struct sk_buff, field) < \
924 offsetof(struct sk_buff, headers_start)); \
925 BUILD_BUG_ON(offsetof(struct sk_buff, field) > \
926 offsetof(struct sk_buff, headers_end)); \
927
Herbert Xudec18812007-10-14 00:37:30 -0700928static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
929{
930 new->tstamp = old->tstamp;
Eric Dumazetb1937222014-09-28 22:18:47 -0700931 /* We do not copy old->sk */
Herbert Xudec18812007-10-14 00:37:30 -0700932 new->dev = old->dev;
Eric Dumazetb1937222014-09-28 22:18:47 -0700933 memcpy(new->cb, old->cb, sizeof(old->cb));
Eric Dumazet7fee2262010-05-11 23:19:48 +0000934 skb_dst_copy(new, old);
Florian Westphaldf5042f2018-12-18 17:15:16 +0100935 __skb_ext_copy(new, old);
Eric Dumazetb1937222014-09-28 22:18:47 -0700936 __nf_copy(new, old, false);
Patrick McHardy6aa895b2008-07-14 22:49:06 -0700937
Eric Dumazetb1937222014-09-28 22:18:47 -0700938 /* Note : this field could be in headers_start/headers_end section
939 * It is not yet because we do not want to have a 16 bit hole
940 */
941 new->queue_mapping = old->queue_mapping;
Eliezer Tamir06021292013-06-10 11:39:50 +0300942
Eric Dumazetb1937222014-09-28 22:18:47 -0700943 memcpy(&new->headers_start, &old->headers_start,
944 offsetof(struct sk_buff, headers_end) -
945 offsetof(struct sk_buff, headers_start));
946 CHECK_SKB_FIELD(protocol);
947 CHECK_SKB_FIELD(csum);
948 CHECK_SKB_FIELD(hash);
949 CHECK_SKB_FIELD(priority);
950 CHECK_SKB_FIELD(skb_iif);
951 CHECK_SKB_FIELD(vlan_proto);
952 CHECK_SKB_FIELD(vlan_tci);
953 CHECK_SKB_FIELD(transport_header);
954 CHECK_SKB_FIELD(network_header);
955 CHECK_SKB_FIELD(mac_header);
956 CHECK_SKB_FIELD(inner_protocol);
957 CHECK_SKB_FIELD(inner_transport_header);
958 CHECK_SKB_FIELD(inner_network_header);
959 CHECK_SKB_FIELD(inner_mac_header);
960 CHECK_SKB_FIELD(mark);
961#ifdef CONFIG_NETWORK_SECMARK
962 CHECK_SKB_FIELD(secmark);
963#endif
Cong Wange0d10952013-08-01 11:10:25 +0800964#ifdef CONFIG_NET_RX_BUSY_POLL
Eric Dumazetb1937222014-09-28 22:18:47 -0700965 CHECK_SKB_FIELD(napi_id);
Eliezer Tamir06021292013-06-10 11:39:50 +0300966#endif
Eric Dumazet2bd82482015-02-03 23:48:24 -0800967#ifdef CONFIG_XPS
968 CHECK_SKB_FIELD(sender_cpu);
969#endif
Eric Dumazetb1937222014-09-28 22:18:47 -0700970#ifdef CONFIG_NET_SCHED
971 CHECK_SKB_FIELD(tc_index);
Eric Dumazetb1937222014-09-28 22:18:47 -0700972#endif
973
Herbert Xudec18812007-10-14 00:37:30 -0700974}
975
Herbert Xu82c49a32009-05-22 22:11:37 +0000976/*
977 * You should not add any new code to this function. Add it to
978 * __copy_skb_header above instead.
979 */
Herbert Xue0053ec2007-10-14 00:37:52 -0700980static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700981{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982#define C(x) n->x = skb->x
983
984 n->next = n->prev = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700985 n->sk = NULL;
Herbert Xudec18812007-10-14 00:37:30 -0700986 __copy_skb_header(n, skb);
987
Linus Torvalds1da177e2005-04-16 15:20:36 -0700988 C(len);
989 C(data_len);
Alexey Dobriyan3e6b3b22007-03-16 15:00:46 -0700990 C(mac_len);
Patrick McHardy334a8132007-06-25 04:35:20 -0700991 n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
Paul Moore02f1c892008-01-07 21:56:41 -0800992 n->cloned = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700993 n->nohdr = 0;
Eric Dumazetb13dda92018-04-07 13:42:39 -0700994 n->peeked = 0;
Stefano Brivioe78bfb02018-07-13 13:21:07 +0200995 C(pfmemalloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700996 n->destructor = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700997 C(tail);
998 C(end);
Paul Moore02f1c892008-01-07 21:56:41 -0800999 C(head);
Eric Dumazetd3836f22012-04-27 00:33:38 +00001000 C(head_frag);
Paul Moore02f1c892008-01-07 21:56:41 -08001001 C(data);
1002 C(truesize);
Reshetova, Elena63354792017-06-30 13:07:58 +03001003 refcount_set(&n->users, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001004
1005 atomic_inc(&(skb_shinfo(skb)->dataref));
1006 skb->cloned = 1;
1007
1008 return n;
Herbert Xue0053ec2007-10-14 00:37:52 -07001009#undef C
1010}
1011
1012/**
Jakub Kicinskida29e4b2019-06-03 15:16:58 -07001013 * alloc_skb_for_msg() - allocate sk_buff to wrap frag list forming a msg
1014 * @first: first sk_buff of the msg
1015 */
1016struct sk_buff *alloc_skb_for_msg(struct sk_buff *first)
1017{
1018 struct sk_buff *n;
1019
1020 n = alloc_skb(0, GFP_ATOMIC);
1021 if (!n)
1022 return NULL;
1023
1024 n->len = first->len;
1025 n->data_len = first->len;
1026 n->truesize = first->truesize;
1027
1028 skb_shinfo(n)->frag_list = first;
1029
1030 __copy_skb_header(n, first);
1031 n->destructor = NULL;
1032
1033 return n;
1034}
1035EXPORT_SYMBOL_GPL(alloc_skb_for_msg);
1036
1037/**
Herbert Xue0053ec2007-10-14 00:37:52 -07001038 * skb_morph - morph one skb into another
1039 * @dst: the skb to receive the contents
1040 * @src: the skb to supply the contents
1041 *
1042 * This is identical to skb_clone except that the target skb is
1043 * supplied by the user.
1044 *
1045 * The target skb is returned upon exit.
1046 */
1047struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
1048{
Herbert Xu2d4baff2007-11-26 23:11:19 +08001049 skb_release_all(dst);
Herbert Xue0053ec2007-10-14 00:37:52 -07001050 return __skb_clone(dst, src);
1051}
1052EXPORT_SYMBOL_GPL(skb_morph);
1053
Sowmini Varadhan6f89dbc2018-02-15 10:49:32 -08001054int mm_account_pinned_pages(struct mmpin *mmp, size_t size)
Willem de Bruijna91dbff2017-08-03 16:29:43 -04001055{
1056 unsigned long max_pg, num_pg, new_pg, old_pg;
1057 struct user_struct *user;
1058
1059 if (capable(CAP_IPC_LOCK) || !size)
1060 return 0;
1061
1062 num_pg = (size >> PAGE_SHIFT) + 2; /* worst case */
1063 max_pg = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
1064 user = mmp->user ? : current_user();
1065
1066 do {
1067 old_pg = atomic_long_read(&user->locked_vm);
1068 new_pg = old_pg + num_pg;
1069 if (new_pg > max_pg)
1070 return -ENOBUFS;
1071 } while (atomic_long_cmpxchg(&user->locked_vm, old_pg, new_pg) !=
1072 old_pg);
1073
1074 if (!mmp->user) {
1075 mmp->user = get_uid(user);
1076 mmp->num_pg = num_pg;
1077 } else {
1078 mmp->num_pg += num_pg;
1079 }
1080
1081 return 0;
1082}
Sowmini Varadhan6f89dbc2018-02-15 10:49:32 -08001083EXPORT_SYMBOL_GPL(mm_account_pinned_pages);
Willem de Bruijna91dbff2017-08-03 16:29:43 -04001084
Sowmini Varadhan6f89dbc2018-02-15 10:49:32 -08001085void mm_unaccount_pinned_pages(struct mmpin *mmp)
Willem de Bruijna91dbff2017-08-03 16:29:43 -04001086{
1087 if (mmp->user) {
1088 atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm);
1089 free_uid(mmp->user);
1090 }
1091}
Sowmini Varadhan6f89dbc2018-02-15 10:49:32 -08001092EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages);
Willem de Bruijna91dbff2017-08-03 16:29:43 -04001093
Willem de Bruijn52267792017-08-03 16:29:39 -04001094struct ubuf_info *sock_zerocopy_alloc(struct sock *sk, size_t size)
1095{
1096 struct ubuf_info *uarg;
1097 struct sk_buff *skb;
1098
1099 WARN_ON_ONCE(!in_task());
1100
1101 skb = sock_omalloc(sk, 0, GFP_KERNEL);
1102 if (!skb)
1103 return NULL;
1104
1105 BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb));
1106 uarg = (void *)skb->cb;
Willem de Bruijna91dbff2017-08-03 16:29:43 -04001107 uarg->mmp.user = NULL;
1108
1109 if (mm_account_pinned_pages(&uarg->mmp, size)) {
1110 kfree_skb(skb);
1111 return NULL;
1112 }
Willem de Bruijn52267792017-08-03 16:29:39 -04001113
1114 uarg->callback = sock_zerocopy_callback;
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001115 uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;
1116 uarg->len = 1;
1117 uarg->bytelen = size;
Willem de Bruijn52267792017-08-03 16:29:39 -04001118 uarg->zerocopy = 1;
Eric Dumazetc1d1b432017-08-31 16:48:22 -07001119 refcount_set(&uarg->refcnt, 1);
Willem de Bruijn52267792017-08-03 16:29:39 -04001120 sock_hold(sk);
1121
1122 return uarg;
1123}
1124EXPORT_SYMBOL_GPL(sock_zerocopy_alloc);
1125
1126static inline struct sk_buff *skb_from_uarg(struct ubuf_info *uarg)
1127{
1128 return container_of((void *)uarg, struct sk_buff, cb);
1129}
1130
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001131struct ubuf_info *sock_zerocopy_realloc(struct sock *sk, size_t size,
1132 struct ubuf_info *uarg)
1133{
1134 if (uarg) {
1135 const u32 byte_limit = 1 << 19; /* limit to a few TSO */
1136 u32 bytelen, next;
1137
1138 /* realloc only when socket is locked (TCP, UDP cork),
1139 * so uarg->len and sk_zckey access is serialized
1140 */
1141 if (!sock_owned_by_user(sk)) {
1142 WARN_ON_ONCE(1);
1143 return NULL;
1144 }
1145
1146 bytelen = uarg->bytelen + size;
1147 if (uarg->len == USHRT_MAX - 1 || bytelen > byte_limit) {
1148 /* TCP can create new skb to attach new uarg */
1149 if (sk->sk_type == SOCK_STREAM)
1150 goto new_alloc;
1151 return NULL;
1152 }
1153
1154 next = (u32)atomic_read(&sk->sk_zckey);
1155 if ((u32)(uarg->id + uarg->len) == next) {
Willem de Bruijna91dbff2017-08-03 16:29:43 -04001156 if (mm_account_pinned_pages(&uarg->mmp, size))
1157 return NULL;
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001158 uarg->len++;
1159 uarg->bytelen = bytelen;
1160 atomic_set(&sk->sk_zckey, ++next);
Willem de Bruijn100f6d82019-05-30 18:01:21 -04001161
1162 /* no extra ref when appending to datagram (MSG_MORE) */
1163 if (sk->sk_type == SOCK_STREAM)
1164 sock_zerocopy_get(uarg);
1165
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001166 return uarg;
1167 }
1168 }
1169
1170new_alloc:
1171 return sock_zerocopy_alloc(sk, size);
1172}
1173EXPORT_SYMBOL_GPL(sock_zerocopy_realloc);
1174
1175static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len)
1176{
1177 struct sock_exterr_skb *serr = SKB_EXT_ERR(skb);
1178 u32 old_lo, old_hi;
1179 u64 sum_len;
1180
1181 old_lo = serr->ee.ee_info;
1182 old_hi = serr->ee.ee_data;
1183 sum_len = old_hi - old_lo + 1ULL + len;
1184
1185 if (sum_len >= (1ULL << 32))
1186 return false;
1187
1188 if (lo != old_hi + 1)
1189 return false;
1190
1191 serr->ee.ee_data += len;
1192 return true;
1193}
1194
Willem de Bruijn52267792017-08-03 16:29:39 -04001195void sock_zerocopy_callback(struct ubuf_info *uarg, bool success)
1196{
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001197 struct sk_buff *tail, *skb = skb_from_uarg(uarg);
Willem de Bruijn52267792017-08-03 16:29:39 -04001198 struct sock_exterr_skb *serr;
1199 struct sock *sk = skb->sk;
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001200 struct sk_buff_head *q;
1201 unsigned long flags;
1202 u32 lo, hi;
1203 u16 len;
Willem de Bruijn52267792017-08-03 16:29:39 -04001204
Willem de Bruijnccaffff2017-08-09 19:09:43 -04001205 mm_unaccount_pinned_pages(&uarg->mmp);
1206
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001207 /* if !len, there was only 1 call, and it was aborted
1208 * so do not queue a completion notification
1209 */
1210 if (!uarg->len || sock_flag(sk, SOCK_DEAD))
Willem de Bruijn52267792017-08-03 16:29:39 -04001211 goto release;
1212
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001213 len = uarg->len;
1214 lo = uarg->id;
1215 hi = uarg->id + len - 1;
1216
Willem de Bruijn52267792017-08-03 16:29:39 -04001217 serr = SKB_EXT_ERR(skb);
1218 memset(serr, 0, sizeof(*serr));
1219 serr->ee.ee_errno = 0;
1220 serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY;
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001221 serr->ee.ee_data = hi;
1222 serr->ee.ee_info = lo;
Willem de Bruijn52267792017-08-03 16:29:39 -04001223 if (!success)
1224 serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED;
1225
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001226 q = &sk->sk_error_queue;
1227 spin_lock_irqsave(&q->lock, flags);
1228 tail = skb_peek_tail(q);
1229 if (!tail || SKB_EXT_ERR(tail)->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY ||
1230 !skb_zerocopy_notify_extend(tail, lo, len)) {
1231 __skb_queue_tail(q, skb);
1232 skb = NULL;
1233 }
1234 spin_unlock_irqrestore(&q->lock, flags);
Willem de Bruijn52267792017-08-03 16:29:39 -04001235
1236 sk->sk_error_report(sk);
1237
1238release:
1239 consume_skb(skb);
1240 sock_put(sk);
1241}
1242EXPORT_SYMBOL_GPL(sock_zerocopy_callback);
1243
1244void sock_zerocopy_put(struct ubuf_info *uarg)
1245{
Eric Dumazetc1d1b432017-08-31 16:48:22 -07001246 if (uarg && refcount_dec_and_test(&uarg->refcnt)) {
Willem de Bruijn52267792017-08-03 16:29:39 -04001247 if (uarg->callback)
1248 uarg->callback(uarg, uarg->zerocopy);
1249 else
1250 consume_skb(skb_from_uarg(uarg));
1251 }
1252}
1253EXPORT_SYMBOL_GPL(sock_zerocopy_put);
1254
Willem de Bruijn52900d22018-11-30 15:32:40 -05001255void sock_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
Willem de Bruijn52267792017-08-03 16:29:39 -04001256{
1257 if (uarg) {
1258 struct sock *sk = skb_from_uarg(uarg)->sk;
1259
1260 atomic_dec(&sk->sk_zckey);
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001261 uarg->len--;
Willem de Bruijn52267792017-08-03 16:29:39 -04001262
Willem de Bruijn52900d22018-11-30 15:32:40 -05001263 if (have_uref)
1264 sock_zerocopy_put(uarg);
Willem de Bruijn52267792017-08-03 16:29:39 -04001265 }
1266}
1267EXPORT_SYMBOL_GPL(sock_zerocopy_put_abort);
1268
Willem de Bruijnb5947e52018-11-30 15:32:39 -05001269int skb_zerocopy_iter_dgram(struct sk_buff *skb, struct msghdr *msg, int len)
1270{
1271 return __zerocopy_sg_from_iter(skb->sk, skb, &msg->msg_iter, len);
1272}
1273EXPORT_SYMBOL_GPL(skb_zerocopy_iter_dgram);
1274
Willem de Bruijn52267792017-08-03 16:29:39 -04001275int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
1276 struct msghdr *msg, int len,
1277 struct ubuf_info *uarg)
1278{
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001279 struct ubuf_info *orig_uarg = skb_zcopy(skb);
Willem de Bruijn52267792017-08-03 16:29:39 -04001280 struct iov_iter orig_iter = msg->msg_iter;
1281 int err, orig_len = skb->len;
1282
Willem de Bruijn4ab6c992017-08-03 16:29:42 -04001283 /* An skb can only point to one uarg. This edge case happens when
1284 * TCP appends to an skb, but zerocopy_realloc triggered a new alloc.
1285 */
1286 if (orig_uarg && uarg != orig_uarg)
1287 return -EEXIST;
1288
Willem de Bruijn52267792017-08-03 16:29:39 -04001289 err = __zerocopy_sg_from_iter(sk, skb, &msg->msg_iter, len);
1290 if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) {
Willem de Bruijn54d431172017-10-19 12:40:39 -04001291 struct sock *save_sk = skb->sk;
1292
Willem de Bruijn52267792017-08-03 16:29:39 -04001293 /* Streams do not free skb on error. Reset to prev state. */
1294 msg->msg_iter = orig_iter;
Willem de Bruijn54d431172017-10-19 12:40:39 -04001295 skb->sk = sk;
Willem de Bruijn52267792017-08-03 16:29:39 -04001296 ___pskb_trim(skb, orig_len);
Willem de Bruijn54d431172017-10-19 12:40:39 -04001297 skb->sk = save_sk;
Willem de Bruijn52267792017-08-03 16:29:39 -04001298 return err;
1299 }
1300
Willem de Bruijn52900d22018-11-30 15:32:40 -05001301 skb_zcopy_set(skb, uarg, NULL);
Willem de Bruijn52267792017-08-03 16:29:39 -04001302 return skb->len - orig_len;
1303}
1304EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream);
1305
Willem de Bruijn1f8b9772017-08-03 16:29:41 -04001306static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig,
Willem de Bruijn52267792017-08-03 16:29:39 -04001307 gfp_t gfp_mask)
1308{
1309 if (skb_zcopy(orig)) {
1310 if (skb_zcopy(nskb)) {
1311 /* !gfp_mask callers are verified to !skb_zcopy(nskb) */
1312 if (!gfp_mask) {
1313 WARN_ON_ONCE(1);
1314 return -ENOMEM;
1315 }
1316 if (skb_uarg(nskb) == skb_uarg(orig))
1317 return 0;
1318 if (skb_copy_ubufs(nskb, GFP_ATOMIC))
1319 return -EIO;
1320 }
Willem de Bruijn52900d22018-11-30 15:32:40 -05001321 skb_zcopy_set(nskb, skb_uarg(orig), NULL);
Willem de Bruijn52267792017-08-03 16:29:39 -04001322 }
1323 return 0;
1324}
1325
Ben Hutchings2c530402012-07-10 10:55:09 +00001326/**
1327 * skb_copy_ubufs - copy userspace skb frags buffers to kernel
Michael S. Tsirkin48c83012011-08-31 08:03:29 +00001328 * @skb: the skb to modify
1329 * @gfp_mask: allocation priority
1330 *
1331 * This must be called on SKBTX_DEV_ZEROCOPY skb.
1332 * It will copy all frags into kernel and drop the reference
1333 * to userspace pages.
1334 *
1335 * If this function is called from an interrupt gfp_mask() must be
1336 * %GFP_ATOMIC.
1337 *
1338 * Returns 0 on success or a negative error code on failure
1339 * to allocate kernel memory to copy to.
1340 */
1341int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
Shirley Maa6686f22011-07-06 12:22:12 +00001342{
Shirley Maa6686f22011-07-06 12:22:12 +00001343 int num_frags = skb_shinfo(skb)->nr_frags;
1344 struct page *page, *head = NULL;
Willem de Bruijn3ece7822017-08-03 16:29:38 -04001345 int i, new_frags;
1346 u32 d_off;
Shirley Maa6686f22011-07-06 12:22:12 +00001347
Willem de Bruijn3ece7822017-08-03 16:29:38 -04001348 if (skb_shared(skb) || skb_unclone(skb, gfp_mask))
1349 return -EINVAL;
1350
Willem de Bruijnf72c4ac2017-12-28 12:38:13 -05001351 if (!num_frags)
1352 goto release;
1353
Willem de Bruijn3ece7822017-08-03 16:29:38 -04001354 new_frags = (__skb_pagelen(skb) + PAGE_SIZE - 1) >> PAGE_SHIFT;
1355 for (i = 0; i < new_frags; i++) {
Krishna Kumar02756ed2012-07-17 02:05:29 +00001356 page = alloc_page(gfp_mask);
Shirley Maa6686f22011-07-06 12:22:12 +00001357 if (!page) {
1358 while (head) {
Sunghan Suh40dadff2013-07-12 16:17:23 +09001359 struct page *next = (struct page *)page_private(head);
Shirley Maa6686f22011-07-06 12:22:12 +00001360 put_page(head);
1361 head = next;
1362 }
1363 return -ENOMEM;
1364 }
Willem de Bruijn3ece7822017-08-03 16:29:38 -04001365 set_page_private(page, (unsigned long)head);
1366 head = page;
1367 }
1368
1369 page = head;
1370 d_off = 0;
1371 for (i = 0; i < num_frags; i++) {
1372 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
1373 u32 p_off, p_len, copied;
1374 struct page *p;
1375 u8 *vaddr;
Willem de Bruijnc613c202017-07-31 08:15:47 -04001376
1377 skb_frag_foreach_page(f, f->page_offset, skb_frag_size(f),
1378 p, p_off, p_len, copied) {
Willem de Bruijn3ece7822017-08-03 16:29:38 -04001379 u32 copy, done = 0;
Willem de Bruijnc613c202017-07-31 08:15:47 -04001380 vaddr = kmap_atomic(p);
Willem de Bruijn3ece7822017-08-03 16:29:38 -04001381
1382 while (done < p_len) {
1383 if (d_off == PAGE_SIZE) {
1384 d_off = 0;
1385 page = (struct page *)page_private(page);
1386 }
1387 copy = min_t(u32, PAGE_SIZE - d_off, p_len - done);
1388 memcpy(page_address(page) + d_off,
1389 vaddr + p_off + done, copy);
1390 done += copy;
1391 d_off += copy;
1392 }
Willem de Bruijnc613c202017-07-31 08:15:47 -04001393 kunmap_atomic(vaddr);
1394 }
Shirley Maa6686f22011-07-06 12:22:12 +00001395 }
1396
1397 /* skb frags release userspace buffers */
Krishna Kumar02756ed2012-07-17 02:05:29 +00001398 for (i = 0; i < num_frags; i++)
Ian Campbella8605c62011-10-19 23:01:49 +00001399 skb_frag_unref(skb, i);
Shirley Maa6686f22011-07-06 12:22:12 +00001400
Shirley Maa6686f22011-07-06 12:22:12 +00001401 /* skb frags point to kernel buffers */
Willem de Bruijn3ece7822017-08-03 16:29:38 -04001402 for (i = 0; i < new_frags - 1; i++) {
1403 __skb_fill_page_desc(skb, i, head, 0, PAGE_SIZE);
Sunghan Suh40dadff2013-07-12 16:17:23 +09001404 head = (struct page *)page_private(head);
Shirley Maa6686f22011-07-06 12:22:12 +00001405 }
Willem de Bruijn3ece7822017-08-03 16:29:38 -04001406 __skb_fill_page_desc(skb, new_frags - 1, head, 0, d_off);
1407 skb_shinfo(skb)->nr_frags = new_frags;
Michael S. Tsirkin48c83012011-08-31 08:03:29 +00001408
Willem de Bruijnb90ddd52017-12-20 17:37:50 -05001409release:
Willem de Bruijn1f8b9772017-08-03 16:29:41 -04001410 skb_zcopy_clear(skb, false);
Shirley Maa6686f22011-07-06 12:22:12 +00001411 return 0;
1412}
Michael S. Tsirkindcc0fb72012-07-20 09:23:20 +00001413EXPORT_SYMBOL_GPL(skb_copy_ubufs);
Shirley Maa6686f22011-07-06 12:22:12 +00001414
Herbert Xue0053ec2007-10-14 00:37:52 -07001415/**
1416 * skb_clone - duplicate an sk_buff
1417 * @skb: buffer to clone
1418 * @gfp_mask: allocation priority
1419 *
1420 * Duplicate an &sk_buff. The new one is not owned by a socket. Both
1421 * copies share the same packet data but not structure. The new
1422 * buffer has a reference count of 1. If the allocation fails the
1423 * function returns %NULL otherwise the new buffer is returned.
1424 *
1425 * If this function is called from an interrupt gfp_mask() must be
1426 * %GFP_ATOMIC.
1427 */
1428
1429struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
1430{
Eric Dumazetd0bf4a92014-09-29 13:29:15 -07001431 struct sk_buff_fclones *fclones = container_of(skb,
1432 struct sk_buff_fclones,
1433 skb1);
Eric Dumazet6ffe75e2014-12-03 17:04:39 -08001434 struct sk_buff *n;
Herbert Xue0053ec2007-10-14 00:37:52 -07001435
Michael S. Tsirkin70008aa2012-07-20 09:23:10 +00001436 if (skb_orphan_frags(skb, gfp_mask))
1437 return NULL;
Shirley Maa6686f22011-07-06 12:22:12 +00001438
Herbert Xue0053ec2007-10-14 00:37:52 -07001439 if (skb->fclone == SKB_FCLONE_ORIG &&
Reshetova, Elena26385952017-06-30 13:07:59 +03001440 refcount_read(&fclones->fclone_ref) == 1) {
Eric Dumazet6ffe75e2014-12-03 17:04:39 -08001441 n = &fclones->skb2;
Reshetova, Elena26385952017-06-30 13:07:59 +03001442 refcount_set(&fclones->fclone_ref, 2);
Herbert Xue0053ec2007-10-14 00:37:52 -07001443 } else {
Mel Gormanc93bdd02012-07-31 16:44:19 -07001444 if (skb_pfmemalloc(skb))
1445 gfp_mask |= __GFP_MEMALLOC;
1446
Herbert Xue0053ec2007-10-14 00:37:52 -07001447 n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
1448 if (!n)
1449 return NULL;
Vegard Nossumfe55f6d2008-08-30 12:16:35 +02001450
Herbert Xue0053ec2007-10-14 00:37:52 -07001451 n->fclone = SKB_FCLONE_UNAVAILABLE;
1452 }
1453
1454 return __skb_clone(n, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001455}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001456EXPORT_SYMBOL(skb_clone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001457
Toshiaki Makitab0768a82018-08-03 16:58:09 +09001458void skb_headers_offset_update(struct sk_buff *skb, int off)
Pravin B Shelarf5b17292013-03-07 13:21:40 +00001459{
Eric Dumazet030737b2013-10-19 11:42:54 -07001460 /* Only adjust this if it actually is csum_start rather than csum */
1461 if (skb->ip_summed == CHECKSUM_PARTIAL)
1462 skb->csum_start += off;
Pravin B Shelarf5b17292013-03-07 13:21:40 +00001463 /* {transport,network,mac}_header and tail are relative to skb->head */
1464 skb->transport_header += off;
1465 skb->network_header += off;
1466 if (skb_mac_header_was_set(skb))
1467 skb->mac_header += off;
1468 skb->inner_transport_header += off;
1469 skb->inner_network_header += off;
Pravin B Shelaraefbd2b2013-03-07 13:21:46 +00001470 skb->inner_mac_header += off;
Pravin B Shelarf5b17292013-03-07 13:21:40 +00001471}
Toshiaki Makitab0768a82018-08-03 16:58:09 +09001472EXPORT_SYMBOL(skb_headers_offset_update);
Pravin B Shelarf5b17292013-03-07 13:21:40 +00001473
Ilya Lesokhin08303c12018-04-30 10:16:11 +03001474void skb_copy_header(struct sk_buff *new, const struct sk_buff *old)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001475{
Herbert Xudec18812007-10-14 00:37:30 -07001476 __copy_skb_header(new, old);
1477
Herbert Xu79671682006-06-22 02:40:14 -07001478 skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
1479 skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
1480 skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001481}
Ilya Lesokhin08303c12018-04-30 10:16:11 +03001482EXPORT_SYMBOL(skb_copy_header);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001483
Mel Gormanc93bdd02012-07-31 16:44:19 -07001484static inline int skb_alloc_rx_flag(const struct sk_buff *skb)
1485{
1486 if (skb_pfmemalloc(skb))
1487 return SKB_ALLOC_RX;
1488 return 0;
1489}
1490
Linus Torvalds1da177e2005-04-16 15:20:36 -07001491/**
1492 * skb_copy - create private copy of an sk_buff
1493 * @skb: buffer to copy
1494 * @gfp_mask: allocation priority
1495 *
1496 * Make a copy of both an &sk_buff and its data. This is used when the
1497 * caller wishes to modify the data and needs a private copy of the
1498 * data to alter. Returns %NULL on failure or the pointer to the buffer
1499 * on success. The returned buffer has a reference count of 1.
1500 *
1501 * As by-product this function converts non-linear &sk_buff to linear
1502 * one, so that &sk_buff becomes completely private and caller is allowed
1503 * to modify all the data of returned buffer. This means that this
1504 * function is not recommended for use in circumstances when only
1505 * header is going to be modified. Use pskb_copy() instead.
1506 */
1507
Al Virodd0fc662005-10-07 07:46:04 +01001508struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001509{
Eric Dumazet6602ceb2010-09-01 05:25:10 +00001510 int headerlen = skb_headroom(skb);
Alexander Duyckec47ea82012-05-04 14:26:56 +00001511 unsigned int size = skb_end_offset(skb) + skb->data_len;
Mel Gormanc93bdd02012-07-31 16:44:19 -07001512 struct sk_buff *n = __alloc_skb(size, gfp_mask,
1513 skb_alloc_rx_flag(skb), NUMA_NO_NODE);
Eric Dumazet6602ceb2010-09-01 05:25:10 +00001514
Linus Torvalds1da177e2005-04-16 15:20:36 -07001515 if (!n)
1516 return NULL;
1517
1518 /* Set the data pointer */
1519 skb_reserve(n, headerlen);
1520 /* Set the tail pointer and length */
1521 skb_put(n, skb->len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001522
Tim Hansen9f77fad2017-10-09 11:37:59 -04001523 BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001524
Ilya Lesokhin08303c12018-04-30 10:16:11 +03001525 skb_copy_header(n, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001526 return n;
1527}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001528EXPORT_SYMBOL(skb_copy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001529
1530/**
Octavian Purdilabad93e92014-06-12 01:36:26 +03001531 * __pskb_copy_fclone - create copy of an sk_buff with private head.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001532 * @skb: buffer to copy
Eric Dumazet117632e2011-12-03 21:39:53 +00001533 * @headroom: headroom of new skb
Linus Torvalds1da177e2005-04-16 15:20:36 -07001534 * @gfp_mask: allocation priority
Octavian Purdilabad93e92014-06-12 01:36:26 +03001535 * @fclone: if true allocate the copy of the skb from the fclone
1536 * cache instead of the head cache; it is recommended to set this
1537 * to true for the cases where the copy will likely be cloned
Linus Torvalds1da177e2005-04-16 15:20:36 -07001538 *
1539 * Make a copy of both an &sk_buff and part of its data, located
1540 * in header. Fragmented data remain shared. This is used when
1541 * the caller wishes to modify only header of &sk_buff and needs
1542 * private copy of the header to alter. Returns %NULL on failure
1543 * or the pointer to the buffer on success.
1544 * The returned buffer has a reference count of 1.
1545 */
1546
Octavian Purdilabad93e92014-06-12 01:36:26 +03001547struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
1548 gfp_t gfp_mask, bool fclone)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001549{
Eric Dumazet117632e2011-12-03 21:39:53 +00001550 unsigned int size = skb_headlen(skb) + headroom;
Octavian Purdilabad93e92014-06-12 01:36:26 +03001551 int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0);
1552 struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE);
Eric Dumazet6602ceb2010-09-01 05:25:10 +00001553
Linus Torvalds1da177e2005-04-16 15:20:36 -07001554 if (!n)
1555 goto out;
1556
1557 /* Set the data pointer */
Eric Dumazet117632e2011-12-03 21:39:53 +00001558 skb_reserve(n, headroom);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001559 /* Set the tail pointer and length */
1560 skb_put(n, skb_headlen(skb));
1561 /* Copy the bytes */
Arnaldo Carvalho de Melod626f622007-03-27 18:55:52 -03001562 skb_copy_from_linear_data(skb, n->data, n->len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001563
Herbert Xu25f484a2006-11-07 14:57:15 -08001564 n->truesize += skb->data_len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001565 n->data_len = skb->data_len;
1566 n->len = skb->len;
1567
1568 if (skb_shinfo(skb)->nr_frags) {
1569 int i;
1570
Willem de Bruijn1f8b9772017-08-03 16:29:41 -04001571 if (skb_orphan_frags(skb, gfp_mask) ||
1572 skb_zerocopy_clone(n, skb, gfp_mask)) {
Michael S. Tsirkin70008aa2012-07-20 09:23:10 +00001573 kfree_skb(n);
1574 n = NULL;
1575 goto out;
Shirley Maa6686f22011-07-06 12:22:12 +00001576 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001577 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1578 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
Ian Campbellea2ab692011-08-22 23:44:58 +00001579 skb_frag_ref(skb, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001580 }
1581 skb_shinfo(n)->nr_frags = i;
1582 }
1583
David S. Miller21dc3302010-08-23 00:13:46 -07001584 if (skb_has_frag_list(skb)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001585 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
1586 skb_clone_fraglist(n);
1587 }
1588
Ilya Lesokhin08303c12018-04-30 10:16:11 +03001589 skb_copy_header(n, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001590out:
1591 return n;
1592}
Octavian Purdilabad93e92014-06-12 01:36:26 +03001593EXPORT_SYMBOL(__pskb_copy_fclone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001594
1595/**
1596 * pskb_expand_head - reallocate header of &sk_buff
1597 * @skb: buffer to reallocate
1598 * @nhead: room to add at head
1599 * @ntail: room to add at tail
1600 * @gfp_mask: allocation priority
1601 *
Mathias Krausebc323832013-11-07 14:18:26 +01001602 * Expands (or creates identical copy, if @nhead and @ntail are zero)
1603 * header of @skb. &sk_buff itself is not changed. &sk_buff MUST have
Linus Torvalds1da177e2005-04-16 15:20:36 -07001604 * reference count of 1. Returns zero in the case of success or error,
1605 * if expansion failed. In the last case, &sk_buff is not changed.
1606 *
1607 * All the pointers pointing into skb header may change and must be
1608 * reloaded after call to this function.
1609 */
1610
Victor Fusco86a76ca2005-07-08 14:57:47 -07001611int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
Al Virodd0fc662005-10-07 07:46:04 +01001612 gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001613{
Eric Dumazet158f3232017-01-27 07:11:27 -08001614 int i, osize = skb_end_offset(skb);
1615 int size = osize + nhead + ntail;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001616 long off;
Eric Dumazet158f3232017-01-27 07:11:27 -08001617 u8 *data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001618
Herbert Xu4edd87a2008-10-01 07:09:38 -07001619 BUG_ON(nhead < 0);
1620
Tim Hansen9f77fad2017-10-09 11:37:59 -04001621 BUG_ON(skb_shared(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001622
1623 size = SKB_DATA_ALIGN(size);
1624
Mel Gormanc93bdd02012-07-31 16:44:19 -07001625 if (skb_pfmemalloc(skb))
1626 gfp_mask |= __GFP_MEMALLOC;
1627 data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
1628 gfp_mask, NUMA_NO_NODE, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001629 if (!data)
1630 goto nodata;
Eric Dumazet87151b82012-04-10 20:08:39 +00001631 size = SKB_WITH_OVERHEAD(ksize(data));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001632
1633 /* Copy only real data... and, alas, header. This should be
Eric Dumazet6602ceb2010-09-01 05:25:10 +00001634 * optimized for the cases when header is void.
1635 */
1636 memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head);
1637
1638 memcpy((struct skb_shared_info *)(data + size),
1639 skb_shinfo(skb),
Eric Dumazetfed66382010-07-22 19:09:08 +00001640 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags]));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001641
Alexander Duyck3e245912012-05-04 14:26:51 +00001642 /*
1643 * if shinfo is shared we must drop the old head gracefully, but if it
1644 * is not we can just drop the old head and let the existing refcount
1645 * be since all we did is relocate the values
1646 */
1647 if (skb_cloned(skb)) {
Michael S. Tsirkin70008aa2012-07-20 09:23:10 +00001648 if (skb_orphan_frags(skb, gfp_mask))
1649 goto nofrags;
Willem de Bruijn1f8b9772017-08-03 16:29:41 -04001650 if (skb_zcopy(skb))
Eric Dumazetc1d1b432017-08-31 16:48:22 -07001651 refcount_inc(&skb_uarg(skb)->refcnt);
Eric Dumazet1fd63042010-09-02 23:09:32 +00001652 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
Ian Campbellea2ab692011-08-22 23:44:58 +00001653 skb_frag_ref(skb, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001654
Eric Dumazet1fd63042010-09-02 23:09:32 +00001655 if (skb_has_frag_list(skb))
1656 skb_clone_fraglist(skb);
1657
1658 skb_release_data(skb);
Alexander Duyck3e245912012-05-04 14:26:51 +00001659 } else {
1660 skb_free_head(skb);
Eric Dumazet1fd63042010-09-02 23:09:32 +00001661 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001662 off = (data + nhead) - skb->head;
1663
1664 skb->head = data;
Eric Dumazetd3836f22012-04-27 00:33:38 +00001665 skb->head_frag = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001666 skb->data += off;
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -07001667#ifdef NET_SKBUFF_DATA_USES_OFFSET
1668 skb->end = size;
Patrick McHardy56eb8882007-04-09 11:45:04 -07001669 off = nhead;
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -07001670#else
1671 skb->end = skb->head + size;
Patrick McHardy56eb8882007-04-09 11:45:04 -07001672#endif
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07001673 skb->tail += off;
Peter Pan(潘卫平)b41abb42013-06-06 21:27:21 +08001674 skb_headers_offset_update(skb, nhead);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001675 skb->cloned = 0;
Patrick McHardy334a8132007-06-25 04:35:20 -07001676 skb->hdr_len = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001677 skb->nohdr = 0;
1678 atomic_set(&skb_shinfo(skb)->dataref, 1);
Eric Dumazet158f3232017-01-27 07:11:27 -08001679
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02001680 skb_metadata_clear(skb);
1681
Eric Dumazet158f3232017-01-27 07:11:27 -08001682 /* It is not generally safe to change skb->truesize.
1683 * For the moment, we really care of rx path, or
1684 * when skb is orphaned (not attached to a socket).
1685 */
1686 if (!skb->sk || skb->destructor == sock_edemux)
1687 skb->truesize += size - osize;
1688
Linus Torvalds1da177e2005-04-16 15:20:36 -07001689 return 0;
1690
Shirley Maa6686f22011-07-06 12:22:12 +00001691nofrags:
1692 kfree(data);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001693nodata:
1694 return -ENOMEM;
1695}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001696EXPORT_SYMBOL(pskb_expand_head);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001697
1698/* Make private copy of skb with writable head and some headroom */
1699
1700struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
1701{
1702 struct sk_buff *skb2;
1703 int delta = headroom - skb_headroom(skb);
1704
1705 if (delta <= 0)
1706 skb2 = pskb_copy(skb, GFP_ATOMIC);
1707 else {
1708 skb2 = skb_clone(skb, GFP_ATOMIC);
1709 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
1710 GFP_ATOMIC)) {
1711 kfree_skb(skb2);
1712 skb2 = NULL;
1713 }
1714 }
1715 return skb2;
1716}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001717EXPORT_SYMBOL(skb_realloc_headroom);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001718
1719/**
1720 * skb_copy_expand - copy and expand sk_buff
1721 * @skb: buffer to copy
1722 * @newheadroom: new free bytes at head
1723 * @newtailroom: new free bytes at tail
1724 * @gfp_mask: allocation priority
1725 *
1726 * Make a copy of both an &sk_buff and its data and while doing so
1727 * allocate additional space.
1728 *
1729 * This is used when the caller wishes to modify the data and needs a
1730 * private copy of the data to alter as well as more space for new fields.
1731 * Returns %NULL on failure or the pointer to the buffer
1732 * on success. The returned buffer has a reference count of 1.
1733 *
1734 * You must pass %GFP_ATOMIC as the allocation priority if this function
1735 * is called from an interrupt.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001736 */
1737struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
Victor Fusco86a76ca2005-07-08 14:57:47 -07001738 int newheadroom, int newtailroom,
Al Virodd0fc662005-10-07 07:46:04 +01001739 gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001740{
1741 /*
1742 * Allocate the copy buffer
1743 */
Mel Gormanc93bdd02012-07-31 16:44:19 -07001744 struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom,
1745 gfp_mask, skb_alloc_rx_flag(skb),
1746 NUMA_NO_NODE);
Patrick McHardyefd1e8d2007-04-10 18:30:09 -07001747 int oldheadroom = skb_headroom(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001748 int head_copy_len, head_copy_off;
1749
1750 if (!n)
1751 return NULL;
1752
1753 skb_reserve(n, newheadroom);
1754
1755 /* Set the tail pointer and length */
1756 skb_put(n, skb->len);
1757
Patrick McHardyefd1e8d2007-04-10 18:30:09 -07001758 head_copy_len = oldheadroom;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001759 head_copy_off = 0;
1760 if (newheadroom <= head_copy_len)
1761 head_copy_len = newheadroom;
1762 else
1763 head_copy_off = newheadroom - head_copy_len;
1764
1765 /* Copy the linear header and data. */
Tim Hansen9f77fad2017-10-09 11:37:59 -04001766 BUG_ON(skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
1767 skb->len + head_copy_len));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001768
Ilya Lesokhin08303c12018-04-30 10:16:11 +03001769 skb_copy_header(n, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001770
Eric Dumazet030737b2013-10-19 11:42:54 -07001771 skb_headers_offset_update(n, newheadroom - oldheadroom);
Patrick McHardyefd1e8d2007-04-10 18:30:09 -07001772
Linus Torvalds1da177e2005-04-16 15:20:36 -07001773 return n;
1774}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001775EXPORT_SYMBOL(skb_copy_expand);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001776
1777/**
Florian Fainellicd0a1372017-08-22 15:12:14 -07001778 * __skb_pad - zero pad the tail of an skb
Linus Torvalds1da177e2005-04-16 15:20:36 -07001779 * @skb: buffer to pad
1780 * @pad: space to pad
Florian Fainellicd0a1372017-08-22 15:12:14 -07001781 * @free_on_error: free buffer on error
Linus Torvalds1da177e2005-04-16 15:20:36 -07001782 *
1783 * Ensure that a buffer is followed by a padding area that is zero
1784 * filled. Used by network drivers which may DMA or transfer data
1785 * beyond the buffer end onto the wire.
1786 *
Florian Fainellicd0a1372017-08-22 15:12:14 -07001787 * May return error in out of memory cases. The skb is freed on error
1788 * if @free_on_error is true.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001789 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001790
Florian Fainellicd0a1372017-08-22 15:12:14 -07001791int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001792{
Herbert Xu5b057c62006-06-23 02:06:41 -07001793 int err;
1794 int ntail;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001795
Linus Torvalds1da177e2005-04-16 15:20:36 -07001796 /* If the skbuff is non linear tailroom is always zero.. */
Herbert Xu5b057c62006-06-23 02:06:41 -07001797 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001798 memset(skb->data+skb->len, 0, pad);
Herbert Xu5b057c62006-06-23 02:06:41 -07001799 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001800 }
Herbert Xu5b057c62006-06-23 02:06:41 -07001801
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -07001802 ntail = skb->data_len + pad - (skb->end - skb->tail);
Herbert Xu5b057c62006-06-23 02:06:41 -07001803 if (likely(skb_cloned(skb) || ntail > 0)) {
1804 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
1805 if (unlikely(err))
1806 goto free_skb;
1807 }
1808
1809 /* FIXME: The use of this function with non-linear skb's really needs
1810 * to be audited.
1811 */
1812 err = skb_linearize(skb);
1813 if (unlikely(err))
1814 goto free_skb;
1815
1816 memset(skb->data + skb->len, 0, pad);
1817 return 0;
1818
1819free_skb:
Florian Fainellicd0a1372017-08-22 15:12:14 -07001820 if (free_on_error)
1821 kfree_skb(skb);
Herbert Xu5b057c62006-06-23 02:06:41 -07001822 return err;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001823}
Florian Fainellicd0a1372017-08-22 15:12:14 -07001824EXPORT_SYMBOL(__skb_pad);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001825
Ilpo Järvinen0dde3e12008-03-27 17:43:41 -07001826/**
Mathias Krause0c7ddf32013-11-07 14:18:24 +01001827 * pskb_put - add data to the tail of a potentially fragmented buffer
1828 * @skb: start of the buffer to use
1829 * @tail: tail fragment of the buffer to use
1830 * @len: amount of data to add
1831 *
1832 * This function extends the used data area of the potentially
1833 * fragmented buffer. @tail must be the last fragment of @skb -- or
1834 * @skb itself. If this would exceed the total buffer size the kernel
1835 * will panic. A pointer to the first byte of the extra data is
1836 * returned.
1837 */
1838
Johannes Berg4df864c2017-06-16 14:29:21 +02001839void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len)
Mathias Krause0c7ddf32013-11-07 14:18:24 +01001840{
1841 if (tail != skb) {
1842 skb->data_len += len;
1843 skb->len += len;
1844 }
1845 return skb_put(tail, len);
1846}
1847EXPORT_SYMBOL_GPL(pskb_put);
1848
1849/**
Ilpo Järvinen0dde3e12008-03-27 17:43:41 -07001850 * skb_put - add data to a buffer
1851 * @skb: buffer to use
1852 * @len: amount of data to add
1853 *
1854 * This function extends the used data area of the buffer. If this would
1855 * exceed the total buffer size the kernel will panic. A pointer to the
1856 * first byte of the extra data is returned.
1857 */
Johannes Berg4df864c2017-06-16 14:29:21 +02001858void *skb_put(struct sk_buff *skb, unsigned int len)
Ilpo Järvinen0dde3e12008-03-27 17:43:41 -07001859{
Johannes Berg4df864c2017-06-16 14:29:21 +02001860 void *tmp = skb_tail_pointer(skb);
Ilpo Järvinen0dde3e12008-03-27 17:43:41 -07001861 SKB_LINEAR_ASSERT(skb);
1862 skb->tail += len;
1863 skb->len += len;
1864 if (unlikely(skb->tail > skb->end))
1865 skb_over_panic(skb, len, __builtin_return_address(0));
1866 return tmp;
1867}
1868EXPORT_SYMBOL(skb_put);
1869
Ilpo Järvinen6be8ac22008-03-27 17:47:24 -07001870/**
Ilpo Järvinenc2aa2702008-03-27 17:52:40 -07001871 * skb_push - add data to the start of a buffer
1872 * @skb: buffer to use
1873 * @len: amount of data to add
1874 *
1875 * This function extends the used data area of the buffer at the buffer
1876 * start. If this would exceed the total buffer headroom the kernel will
1877 * panic. A pointer to the first byte of the extra data is returned.
1878 */
Johannes Bergd58ff352017-06-16 14:29:23 +02001879void *skb_push(struct sk_buff *skb, unsigned int len)
Ilpo Järvinenc2aa2702008-03-27 17:52:40 -07001880{
1881 skb->data -= len;
1882 skb->len += len;
Ganesh Goudar9aba2f82018-08-02 15:34:52 +05301883 if (unlikely(skb->data < skb->head))
Ilpo Järvinenc2aa2702008-03-27 17:52:40 -07001884 skb_under_panic(skb, len, __builtin_return_address(0));
1885 return skb->data;
1886}
1887EXPORT_SYMBOL(skb_push);
1888
1889/**
Ilpo Järvinen6be8ac22008-03-27 17:47:24 -07001890 * skb_pull - remove data from the start of a buffer
1891 * @skb: buffer to use
1892 * @len: amount of data to remove
1893 *
1894 * This function removes data from the start of a buffer, returning
1895 * the memory to the headroom. A pointer to the next data in the buffer
1896 * is returned. Once the data has been pulled future pushes will overwrite
1897 * the old data.
1898 */
Johannes Bergaf728682017-06-16 14:29:22 +02001899void *skb_pull(struct sk_buff *skb, unsigned int len)
Ilpo Järvinen6be8ac22008-03-27 17:47:24 -07001900{
David S. Miller47d29642010-05-02 02:21:44 -07001901 return skb_pull_inline(skb, len);
Ilpo Järvinen6be8ac22008-03-27 17:47:24 -07001902}
1903EXPORT_SYMBOL(skb_pull);
1904
Ilpo Järvinen419ae742008-03-27 17:54:01 -07001905/**
1906 * skb_trim - remove end from a buffer
1907 * @skb: buffer to alter
1908 * @len: new length
1909 *
1910 * Cut the length of a buffer down by removing data from the tail. If
1911 * the buffer is already under the length specified it is not modified.
1912 * The skb must be linear.
1913 */
1914void skb_trim(struct sk_buff *skb, unsigned int len)
1915{
1916 if (skb->len > len)
1917 __skb_trim(skb, len);
1918}
1919EXPORT_SYMBOL(skb_trim);
1920
Herbert Xu3cc0e872006-06-09 16:13:38 -07001921/* Trims skb to length len. It can change skb pointers.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001922 */
1923
Herbert Xu3cc0e872006-06-09 16:13:38 -07001924int ___pskb_trim(struct sk_buff *skb, unsigned int len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001925{
Herbert Xu27b437c2006-07-13 19:26:39 -07001926 struct sk_buff **fragp;
1927 struct sk_buff *frag;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001928 int offset = skb_headlen(skb);
1929 int nfrags = skb_shinfo(skb)->nr_frags;
1930 int i;
Herbert Xu27b437c2006-07-13 19:26:39 -07001931 int err;
1932
1933 if (skb_cloned(skb) &&
1934 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
1935 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001936
Herbert Xuf4d26fb2006-07-30 20:20:28 -07001937 i = 0;
1938 if (offset >= len)
1939 goto drop_pages;
1940
1941 for (; i < nfrags; i++) {
Eric Dumazet9e903e02011-10-18 21:00:24 +00001942 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]);
Herbert Xu27b437c2006-07-13 19:26:39 -07001943
1944 if (end < len) {
1945 offset = end;
1946 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001947 }
Herbert Xu27b437c2006-07-13 19:26:39 -07001948
Eric Dumazet9e903e02011-10-18 21:00:24 +00001949 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset);
Herbert Xu27b437c2006-07-13 19:26:39 -07001950
Herbert Xuf4d26fb2006-07-30 20:20:28 -07001951drop_pages:
Herbert Xu27b437c2006-07-13 19:26:39 -07001952 skb_shinfo(skb)->nr_frags = i;
1953
1954 for (; i < nfrags; i++)
Ian Campbellea2ab692011-08-22 23:44:58 +00001955 skb_frag_unref(skb, i);
Herbert Xu27b437c2006-07-13 19:26:39 -07001956
David S. Miller21dc3302010-08-23 00:13:46 -07001957 if (skb_has_frag_list(skb))
Herbert Xu27b437c2006-07-13 19:26:39 -07001958 skb_drop_fraglist(skb);
Herbert Xuf4d26fb2006-07-30 20:20:28 -07001959 goto done;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001960 }
1961
Herbert Xu27b437c2006-07-13 19:26:39 -07001962 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
1963 fragp = &frag->next) {
1964 int end = offset + frag->len;
1965
1966 if (skb_shared(frag)) {
1967 struct sk_buff *nfrag;
1968
1969 nfrag = skb_clone(frag, GFP_ATOMIC);
1970 if (unlikely(!nfrag))
1971 return -ENOMEM;
1972
1973 nfrag->next = frag->next;
Eric Dumazet85bb2a62012-04-19 02:24:53 +00001974 consume_skb(frag);
Herbert Xu27b437c2006-07-13 19:26:39 -07001975 frag = nfrag;
1976 *fragp = frag;
1977 }
1978
1979 if (end < len) {
1980 offset = end;
1981 continue;
1982 }
1983
1984 if (end > len &&
1985 unlikely((err = pskb_trim(frag, len - offset))))
1986 return err;
1987
1988 if (frag->next)
1989 skb_drop_list(&frag->next);
1990 break;
1991 }
1992
Herbert Xuf4d26fb2006-07-30 20:20:28 -07001993done:
Herbert Xu27b437c2006-07-13 19:26:39 -07001994 if (len > skb_headlen(skb)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001995 skb->data_len -= skb->len - len;
1996 skb->len = len;
1997 } else {
Herbert Xu27b437c2006-07-13 19:26:39 -07001998 skb->len = len;
1999 skb->data_len = 0;
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07002000 skb_set_tail_pointer(skb, len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002001 }
2002
Eric Dumazetc21b48c2017-04-26 09:07:46 -07002003 if (!skb->sk || skb->destructor == sock_edemux)
2004 skb_condense(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002005 return 0;
2006}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002007EXPORT_SYMBOL(___pskb_trim);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002008
Eric Dumazet88078d92018-04-18 11:43:15 -07002009/* Note : use pskb_trim_rcsum() instead of calling this directly
2010 */
2011int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len)
2012{
2013 if (skb->ip_summed == CHECKSUM_COMPLETE) {
2014 int delta = skb->len - len;
2015
Dimitris Michailidisd55bef502018-10-19 17:07:13 -07002016 skb->csum = csum_block_sub(skb->csum,
2017 skb_checksum(skb, len, delta, 0),
2018 len);
Eric Dumazet88078d92018-04-18 11:43:15 -07002019 }
2020 return __pskb_trim(skb, len);
2021}
2022EXPORT_SYMBOL(pskb_trim_rcsum_slow);
2023
Linus Torvalds1da177e2005-04-16 15:20:36 -07002024/**
2025 * __pskb_pull_tail - advance tail of skb header
2026 * @skb: buffer to reallocate
2027 * @delta: number of bytes to advance tail
2028 *
2029 * The function makes a sense only on a fragmented &sk_buff,
2030 * it expands header moving its tail forward and copying necessary
2031 * data from fragmented part.
2032 *
2033 * &sk_buff MUST have reference count of 1.
2034 *
2035 * Returns %NULL (and &sk_buff does not change) if pull failed
2036 * or value of new tail of skb in the case of success.
2037 *
2038 * All the pointers pointing into skb header may change and must be
2039 * reloaded after call to this function.
2040 */
2041
2042/* Moves tail of skb head forward, copying data from fragmented part,
2043 * when it is necessary.
2044 * 1. It may fail due to malloc failure.
2045 * 2. It may change skb pointers.
2046 *
2047 * It is pretty complicated. Luckily, it is called only in exceptional cases.
2048 */
Johannes Bergaf728682017-06-16 14:29:22 +02002049void *__pskb_pull_tail(struct sk_buff *skb, int delta)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002050{
2051 /* If skb has not enough free space at tail, get new one
2052 * plus 128 bytes for future expansions. If we have enough
2053 * room at tail, reallocate without expansion only if skb is cloned.
2054 */
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -07002055 int i, k, eat = (skb->tail + delta) - skb->end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002056
2057 if (eat > 0 || skb_cloned(skb)) {
2058 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
2059 GFP_ATOMIC))
2060 return NULL;
2061 }
2062
Tim Hansen9f77fad2017-10-09 11:37:59 -04002063 BUG_ON(skb_copy_bits(skb, skb_headlen(skb),
2064 skb_tail_pointer(skb), delta));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002065
2066 /* Optimization: no fragments, no reasons to preestimate
2067 * size of pulled pages. Superb.
2068 */
David S. Miller21dc3302010-08-23 00:13:46 -07002069 if (!skb_has_frag_list(skb))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002070 goto pull_pages;
2071
2072 /* Estimate size of pulled pages. */
2073 eat = delta;
2074 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
Eric Dumazet9e903e02011-10-18 21:00:24 +00002075 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
2076
2077 if (size >= eat)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002078 goto pull_pages;
Eric Dumazet9e903e02011-10-18 21:00:24 +00002079 eat -= size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002080 }
2081
2082 /* If we need update frag list, we are in troubles.
Wenhua Shi09001b02017-10-14 18:51:36 +02002083 * Certainly, it is possible to add an offset to skb data,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002084 * but taking into account that pulling is expected to
2085 * be very rare operation, it is worth to fight against
2086 * further bloating skb head and crucify ourselves here instead.
2087 * Pure masohism, indeed. 8)8)
2088 */
2089 if (eat) {
2090 struct sk_buff *list = skb_shinfo(skb)->frag_list;
2091 struct sk_buff *clone = NULL;
2092 struct sk_buff *insp = NULL;
2093
2094 do {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002095 if (list->len <= eat) {
2096 /* Eaten as whole. */
2097 eat -= list->len;
2098 list = list->next;
2099 insp = list;
2100 } else {
2101 /* Eaten partially. */
2102
2103 if (skb_shared(list)) {
2104 /* Sucks! We need to fork list. :-( */
2105 clone = skb_clone(list, GFP_ATOMIC);
2106 if (!clone)
2107 return NULL;
2108 insp = list->next;
2109 list = clone;
2110 } else {
2111 /* This may be pulled without
2112 * problems. */
2113 insp = list;
2114 }
2115 if (!pskb_pull(list, eat)) {
Wei Yongjunf3fbbe02009-02-25 00:37:32 +00002116 kfree_skb(clone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002117 return NULL;
2118 }
2119 break;
2120 }
2121 } while (eat);
2122
2123 /* Free pulled out fragments. */
2124 while ((list = skb_shinfo(skb)->frag_list) != insp) {
2125 skb_shinfo(skb)->frag_list = list->next;
2126 kfree_skb(list);
2127 }
2128 /* And insert new clone at head. */
2129 if (clone) {
2130 clone->next = list;
2131 skb_shinfo(skb)->frag_list = clone;
2132 }
2133 }
2134 /* Success! Now we may commit changes to skb data. */
2135
2136pull_pages:
2137 eat = delta;
2138 k = 0;
2139 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
Eric Dumazet9e903e02011-10-18 21:00:24 +00002140 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
2141
2142 if (size <= eat) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002143 skb_frag_unref(skb, i);
Eric Dumazet9e903e02011-10-18 21:00:24 +00002144 eat -= size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002145 } else {
2146 skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i];
2147 if (eat) {
2148 skb_shinfo(skb)->frags[k].page_offset += eat;
Eric Dumazet9e903e02011-10-18 21:00:24 +00002149 skb_frag_size_sub(&skb_shinfo(skb)->frags[k], eat);
linzhang3ccc6c62017-07-17 17:25:02 +08002150 if (!i)
2151 goto end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002152 eat = 0;
2153 }
2154 k++;
2155 }
2156 }
2157 skb_shinfo(skb)->nr_frags = k;
2158
linzhang3ccc6c62017-07-17 17:25:02 +08002159end:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002160 skb->tail += delta;
2161 skb->data_len -= delta;
2162
Willem de Bruijn1f8b9772017-08-03 16:29:41 -04002163 if (!skb->data_len)
2164 skb_zcopy_clear(skb, false);
2165
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07002166 return skb_tail_pointer(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002167}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002168EXPORT_SYMBOL(__pskb_pull_tail);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002169
Eric Dumazet22019b12011-07-29 18:37:31 +00002170/**
2171 * skb_copy_bits - copy bits from skb to kernel buffer
2172 * @skb: source skb
2173 * @offset: offset in source
2174 * @to: destination buffer
2175 * @len: number of bytes to copy
2176 *
2177 * Copy the specified number of bytes from the source skb to the
2178 * destination buffer.
2179 *
2180 * CAUTION ! :
2181 * If its prototype is ever changed,
2182 * check arch/{*}/net/{*}.S files,
2183 * since it is called from BPF assembly code.
2184 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002185int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
2186{
David S. Miller1a028e52007-04-27 15:21:23 -07002187 int start = skb_headlen(skb);
David S. Millerfbb398a2009-06-09 00:18:59 -07002188 struct sk_buff *frag_iter;
2189 int i, copy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002190
2191 if (offset > (int)skb->len - len)
2192 goto fault;
2193
2194 /* Copy header. */
David S. Miller1a028e52007-04-27 15:21:23 -07002195 if ((copy = start - offset) > 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002196 if (copy > len)
2197 copy = len;
Arnaldo Carvalho de Melod626f622007-03-27 18:55:52 -03002198 skb_copy_from_linear_data_offset(skb, offset, to, copy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002199 if ((len -= copy) == 0)
2200 return 0;
2201 offset += copy;
2202 to += copy;
2203 }
2204
2205 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
David S. Miller1a028e52007-04-27 15:21:23 -07002206 int end;
Eric Dumazet51c56b02012-04-05 11:35:15 +02002207 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
Linus Torvalds1da177e2005-04-16 15:20:36 -07002208
Ilpo Järvinen547b7922008-07-25 21:43:18 -07002209 WARN_ON(start > offset + len);
David S. Miller1a028e52007-04-27 15:21:23 -07002210
Eric Dumazet51c56b02012-04-05 11:35:15 +02002211 end = start + skb_frag_size(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002212 if ((copy = end - offset) > 0) {
Willem de Bruijnc613c202017-07-31 08:15:47 -04002213 u32 p_off, p_len, copied;
2214 struct page *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002215 u8 *vaddr;
2216
2217 if (copy > len)
2218 copy = len;
2219
Willem de Bruijnc613c202017-07-31 08:15:47 -04002220 skb_frag_foreach_page(f,
2221 f->page_offset + offset - start,
2222 copy, p, p_off, p_len, copied) {
2223 vaddr = kmap_atomic(p);
2224 memcpy(to + copied, vaddr + p_off, p_len);
2225 kunmap_atomic(vaddr);
2226 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002227
2228 if ((len -= copy) == 0)
2229 return 0;
2230 offset += copy;
2231 to += copy;
2232 }
David S. Miller1a028e52007-04-27 15:21:23 -07002233 start = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002234 }
2235
David S. Millerfbb398a2009-06-09 00:18:59 -07002236 skb_walk_frags(skb, frag_iter) {
2237 int end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002238
David S. Millerfbb398a2009-06-09 00:18:59 -07002239 WARN_ON(start > offset + len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002240
David S. Millerfbb398a2009-06-09 00:18:59 -07002241 end = start + frag_iter->len;
2242 if ((copy = end - offset) > 0) {
2243 if (copy > len)
2244 copy = len;
2245 if (skb_copy_bits(frag_iter, offset - start, to, copy))
2246 goto fault;
2247 if ((len -= copy) == 0)
2248 return 0;
2249 offset += copy;
2250 to += copy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002251 }
David S. Millerfbb398a2009-06-09 00:18:59 -07002252 start = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002253 }
Shirley Maa6686f22011-07-06 12:22:12 +00002254
Linus Torvalds1da177e2005-04-16 15:20:36 -07002255 if (!len)
2256 return 0;
2257
2258fault:
2259 return -EFAULT;
2260}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002261EXPORT_SYMBOL(skb_copy_bits);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002262
Jens Axboe9c55e012007-11-06 23:30:13 -08002263/*
2264 * Callback from splice_to_pipe(), if we need to release some pages
2265 * at the end of the spd in case we error'ed out in filling the pipe.
2266 */
2267static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
2268{
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08002269 put_page(spd->pages[i]);
2270}
Jens Axboe9c55e012007-11-06 23:30:13 -08002271
David S. Millera108d5f2012-04-23 23:06:11 -04002272static struct page *linear_to_page(struct page *page, unsigned int *len,
2273 unsigned int *offset,
Eric Dumazet18aafc62013-01-11 14:46:37 +00002274 struct sock *sk)
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08002275{
Eric Dumazet5640f762012-09-23 23:04:42 +00002276 struct page_frag *pfrag = sk_page_frag(sk);
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08002277
Eric Dumazet5640f762012-09-23 23:04:42 +00002278 if (!sk_page_frag_refill(sk, pfrag))
2279 return NULL;
Jarek Poplawski4fb66992009-02-01 00:41:42 -08002280
Eric Dumazet5640f762012-09-23 23:04:42 +00002281 *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset);
Jarek Poplawski4fb66992009-02-01 00:41:42 -08002282
Eric Dumazet5640f762012-09-23 23:04:42 +00002283 memcpy(page_address(pfrag->page) + pfrag->offset,
2284 page_address(page) + *offset, *len);
2285 *offset = pfrag->offset;
2286 pfrag->offset += *len;
Jarek Poplawski4fb66992009-02-01 00:41:42 -08002287
Eric Dumazet5640f762012-09-23 23:04:42 +00002288 return pfrag->page;
Jens Axboe9c55e012007-11-06 23:30:13 -08002289}
2290
Eric Dumazet41c73a02012-04-22 12:26:16 +00002291static bool spd_can_coalesce(const struct splice_pipe_desc *spd,
2292 struct page *page,
2293 unsigned int offset)
2294{
2295 return spd->nr_pages &&
2296 spd->pages[spd->nr_pages - 1] == page &&
2297 (spd->partial[spd->nr_pages - 1].offset +
2298 spd->partial[spd->nr_pages - 1].len == offset);
2299}
2300
Jens Axboe9c55e012007-11-06 23:30:13 -08002301/*
2302 * Fill page/offset/length into spd, if it can hold more pages.
2303 */
David S. Millera108d5f2012-04-23 23:06:11 -04002304static bool spd_fill_page(struct splice_pipe_desc *spd,
2305 struct pipe_inode_info *pipe, struct page *page,
2306 unsigned int *len, unsigned int offset,
Eric Dumazet18aafc62013-01-11 14:46:37 +00002307 bool linear,
David S. Millera108d5f2012-04-23 23:06:11 -04002308 struct sock *sk)
Jens Axboe9c55e012007-11-06 23:30:13 -08002309{
Eric Dumazet41c73a02012-04-22 12:26:16 +00002310 if (unlikely(spd->nr_pages == MAX_SKB_FRAGS))
David S. Millera108d5f2012-04-23 23:06:11 -04002311 return true;
Jens Axboe9c55e012007-11-06 23:30:13 -08002312
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08002313 if (linear) {
Eric Dumazet18aafc62013-01-11 14:46:37 +00002314 page = linear_to_page(page, len, &offset, sk);
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08002315 if (!page)
David S. Millera108d5f2012-04-23 23:06:11 -04002316 return true;
Eric Dumazet41c73a02012-04-22 12:26:16 +00002317 }
2318 if (spd_can_coalesce(spd, page, offset)) {
2319 spd->partial[spd->nr_pages - 1].len += *len;
David S. Millera108d5f2012-04-23 23:06:11 -04002320 return false;
Eric Dumazet41c73a02012-04-22 12:26:16 +00002321 }
2322 get_page(page);
Jens Axboe9c55e012007-11-06 23:30:13 -08002323 spd->pages[spd->nr_pages] = page;
Jarek Poplawski4fb66992009-02-01 00:41:42 -08002324 spd->partial[spd->nr_pages].len = *len;
Jens Axboe9c55e012007-11-06 23:30:13 -08002325 spd->partial[spd->nr_pages].offset = offset;
Jens Axboe9c55e012007-11-06 23:30:13 -08002326 spd->nr_pages++;
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08002327
David S. Millera108d5f2012-04-23 23:06:11 -04002328 return false;
Jens Axboe9c55e012007-11-06 23:30:13 -08002329}
2330
David S. Millera108d5f2012-04-23 23:06:11 -04002331static bool __splice_segment(struct page *page, unsigned int poff,
2332 unsigned int plen, unsigned int *off,
Eric Dumazet18aafc62013-01-11 14:46:37 +00002333 unsigned int *len,
Eric Dumazetd7ccf7c2012-04-23 23:35:04 -04002334 struct splice_pipe_desc *spd, bool linear,
David S. Millera108d5f2012-04-23 23:06:11 -04002335 struct sock *sk,
2336 struct pipe_inode_info *pipe)
Octavian Purdila2870c432008-07-15 00:49:11 -07002337{
2338 if (!*len)
David S. Millera108d5f2012-04-23 23:06:11 -04002339 return true;
Octavian Purdila2870c432008-07-15 00:49:11 -07002340
2341 /* skip this segment if already processed */
2342 if (*off >= plen) {
2343 *off -= plen;
David S. Millera108d5f2012-04-23 23:06:11 -04002344 return false;
Octavian Purdiladb43a282008-06-27 17:27:21 -07002345 }
Jens Axboe9c55e012007-11-06 23:30:13 -08002346
Octavian Purdila2870c432008-07-15 00:49:11 -07002347 /* ignore any bits we already processed */
Eric Dumazet9ca1b222013-01-05 21:31:18 +00002348 poff += *off;
2349 plen -= *off;
2350 *off = 0;
Octavian Purdila2870c432008-07-15 00:49:11 -07002351
Eric Dumazet18aafc62013-01-11 14:46:37 +00002352 do {
2353 unsigned int flen = min(*len, plen);
Octavian Purdila2870c432008-07-15 00:49:11 -07002354
Eric Dumazet18aafc62013-01-11 14:46:37 +00002355 if (spd_fill_page(spd, pipe, page, &flen, poff,
2356 linear, sk))
2357 return true;
2358 poff += flen;
2359 plen -= flen;
2360 *len -= flen;
2361 } while (*len && plen);
Octavian Purdila2870c432008-07-15 00:49:11 -07002362
David S. Millera108d5f2012-04-23 23:06:11 -04002363 return false;
Octavian Purdila2870c432008-07-15 00:49:11 -07002364}
2365
2366/*
David S. Millera108d5f2012-04-23 23:06:11 -04002367 * Map linear and fragment data from the skb to spd. It reports true if the
Octavian Purdila2870c432008-07-15 00:49:11 -07002368 * pipe is full or if we already spliced the requested length.
2369 */
David S. Millera108d5f2012-04-23 23:06:11 -04002370static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
2371 unsigned int *offset, unsigned int *len,
2372 struct splice_pipe_desc *spd, struct sock *sk)
Octavian Purdila2870c432008-07-15 00:49:11 -07002373{
2374 int seg;
Tom Herbertfa9835e2016-03-07 14:11:04 -08002375 struct sk_buff *iter;
Octavian Purdila2870c432008-07-15 00:49:11 -07002376
Eric Dumazet1d0c0b32012-04-27 02:10:03 +00002377 /* map the linear part :
Alexander Duyck2996d312012-05-02 18:18:42 +00002378 * If skb->head_frag is set, this 'linear' part is backed by a
2379 * fragment, and if the head is not shared with any clones then
2380 * we can avoid a copy since we own the head portion of this page.
Jens Axboe9c55e012007-11-06 23:30:13 -08002381 */
Octavian Purdila2870c432008-07-15 00:49:11 -07002382 if (__splice_segment(virt_to_page(skb->data),
2383 (unsigned long) skb->data & (PAGE_SIZE - 1),
2384 skb_headlen(skb),
Eric Dumazet18aafc62013-01-11 14:46:37 +00002385 offset, len, spd,
Alexander Duyck3a7c1ee42012-05-03 01:09:42 +00002386 skb_head_is_locked(skb),
Eric Dumazet1d0c0b32012-04-27 02:10:03 +00002387 sk, pipe))
David S. Millera108d5f2012-04-23 23:06:11 -04002388 return true;
Jens Axboe9c55e012007-11-06 23:30:13 -08002389
2390 /*
2391 * then map the fragments
2392 */
Jens Axboe9c55e012007-11-06 23:30:13 -08002393 for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
2394 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
2395
Ian Campbellea2ab692011-08-22 23:44:58 +00002396 if (__splice_segment(skb_frag_page(f),
Eric Dumazet9e903e02011-10-18 21:00:24 +00002397 f->page_offset, skb_frag_size(f),
Eric Dumazet18aafc62013-01-11 14:46:37 +00002398 offset, len, spd, false, sk, pipe))
David S. Millera108d5f2012-04-23 23:06:11 -04002399 return true;
Jens Axboe9c55e012007-11-06 23:30:13 -08002400 }
2401
Tom Herbertfa9835e2016-03-07 14:11:04 -08002402 skb_walk_frags(skb, iter) {
2403 if (*offset >= iter->len) {
2404 *offset -= iter->len;
2405 continue;
2406 }
2407 /* __skb_splice_bits() only fails if the output has no room
2408 * left, so no point in going over the frag_list for the error
2409 * case.
2410 */
2411 if (__skb_splice_bits(iter, pipe, offset, len, spd, sk))
2412 return true;
2413 }
2414
David S. Millera108d5f2012-04-23 23:06:11 -04002415 return false;
Jens Axboe9c55e012007-11-06 23:30:13 -08002416}
2417
2418/*
2419 * Map data from the skb to a pipe. Should handle both the linear part,
Tom Herbertfa9835e2016-03-07 14:11:04 -08002420 * the fragments, and the frag list.
Jens Axboe9c55e012007-11-06 23:30:13 -08002421 */
Hannes Frederic Sowaa60e3cc2015-05-21 17:00:00 +02002422int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
Jens Axboe9c55e012007-11-06 23:30:13 -08002423 struct pipe_inode_info *pipe, unsigned int tlen,
Al Viro25869262016-09-17 21:02:10 -04002424 unsigned int flags)
Jens Axboe9c55e012007-11-06 23:30:13 -08002425{
Eric Dumazet41c73a02012-04-22 12:26:16 +00002426 struct partial_page partial[MAX_SKB_FRAGS];
2427 struct page *pages[MAX_SKB_FRAGS];
Jens Axboe9c55e012007-11-06 23:30:13 -08002428 struct splice_pipe_desc spd = {
2429 .pages = pages,
2430 .partial = partial,
Eric Dumazet047fe362012-06-12 15:24:40 +02002431 .nr_pages_max = MAX_SKB_FRAGS,
Miklos Szeredi28a625c2014-01-22 19:36:57 +01002432 .ops = &nosteal_pipe_buf_ops,
Jens Axboe9c55e012007-11-06 23:30:13 -08002433 .spd_release = sock_spd_release,
2434 };
Jens Axboe35f3d142010-05-20 10:43:18 +02002435 int ret = 0;
2436
Tom Herbertfa9835e2016-03-07 14:11:04 -08002437 __skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk);
Jens Axboe9c55e012007-11-06 23:30:13 -08002438
Hannes Frederic Sowaa60e3cc2015-05-21 17:00:00 +02002439 if (spd.nr_pages)
Al Viro25869262016-09-17 21:02:10 -04002440 ret = splice_to_pipe(pipe, &spd);
Jens Axboe9c55e012007-11-06 23:30:13 -08002441
Jens Axboe35f3d142010-05-20 10:43:18 +02002442 return ret;
Jens Axboe9c55e012007-11-06 23:30:13 -08002443}
Hannes Frederic Sowa2b514572015-05-21 17:00:01 +02002444EXPORT_SYMBOL_GPL(skb_splice_bits);
Jens Axboe9c55e012007-11-06 23:30:13 -08002445
Tom Herbert20bf50d2017-07-28 16:22:42 -07002446/* Send skb data on a socket. Socket must be locked. */
2447int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
2448 int len)
2449{
2450 unsigned int orig_len = len;
2451 struct sk_buff *head = skb;
2452 unsigned short fragidx;
2453 int slen, ret;
2454
2455do_frag_list:
2456
2457 /* Deal with head data */
2458 while (offset < skb_headlen(skb) && len) {
2459 struct kvec kv;
2460 struct msghdr msg;
2461
2462 slen = min_t(int, len, skb_headlen(skb) - offset);
2463 kv.iov_base = skb->data + offset;
John Fastabenddb5980d2017-08-15 22:31:34 -07002464 kv.iov_len = slen;
Tom Herbert20bf50d2017-07-28 16:22:42 -07002465 memset(&msg, 0, sizeof(msg));
John Fastabendbd95e6782019-05-24 08:01:00 -07002466 msg.msg_flags = MSG_DONTWAIT;
Tom Herbert20bf50d2017-07-28 16:22:42 -07002467
2468 ret = kernel_sendmsg_locked(sk, &msg, &kv, 1, slen);
2469 if (ret <= 0)
2470 goto error;
2471
2472 offset += ret;
2473 len -= ret;
2474 }
2475
2476 /* All the data was skb head? */
2477 if (!len)
2478 goto out;
2479
2480 /* Make offset relative to start of frags */
2481 offset -= skb_headlen(skb);
2482
2483 /* Find where we are in frag list */
2484 for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
2485 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx];
2486
2487 if (offset < frag->size)
2488 break;
2489
2490 offset -= frag->size;
2491 }
2492
2493 for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
2494 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx];
2495
2496 slen = min_t(size_t, len, frag->size - offset);
2497
2498 while (slen) {
2499 ret = kernel_sendpage_locked(sk, frag->page.p,
2500 frag->page_offset + offset,
2501 slen, MSG_DONTWAIT);
2502 if (ret <= 0)
2503 goto error;
2504
2505 len -= ret;
2506 offset += ret;
2507 slen -= ret;
2508 }
2509
2510 offset = 0;
2511 }
2512
2513 if (len) {
2514 /* Process any frag lists */
2515
2516 if (skb == head) {
2517 if (skb_has_frag_list(skb)) {
2518 skb = skb_shinfo(skb)->frag_list;
2519 goto do_frag_list;
2520 }
2521 } else if (skb->next) {
2522 skb = skb->next;
2523 goto do_frag_list;
2524 }
2525 }
2526
2527out:
2528 return orig_len - len;
2529
2530error:
2531 return orig_len == len ? ret : orig_len - len;
2532}
2533EXPORT_SYMBOL_GPL(skb_send_sock_locked);
2534
Herbert Xu357b40a2005-04-19 22:30:14 -07002535/**
2536 * skb_store_bits - store bits from kernel buffer to skb
2537 * @skb: destination buffer
2538 * @offset: offset in destination
2539 * @from: source buffer
2540 * @len: number of bytes to copy
2541 *
2542 * Copy the specified number of bytes from the source buffer to the
2543 * destination skb. This function handles all the messy bits of
2544 * traversing fragment lists and such.
2545 */
2546
Stephen Hemminger0c6fcc82007-04-20 16:40:01 -07002547int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
Herbert Xu357b40a2005-04-19 22:30:14 -07002548{
David S. Miller1a028e52007-04-27 15:21:23 -07002549 int start = skb_headlen(skb);
David S. Millerfbb398a2009-06-09 00:18:59 -07002550 struct sk_buff *frag_iter;
2551 int i, copy;
Herbert Xu357b40a2005-04-19 22:30:14 -07002552
2553 if (offset > (int)skb->len - len)
2554 goto fault;
2555
David S. Miller1a028e52007-04-27 15:21:23 -07002556 if ((copy = start - offset) > 0) {
Herbert Xu357b40a2005-04-19 22:30:14 -07002557 if (copy > len)
2558 copy = len;
Arnaldo Carvalho de Melo27d7ff42007-03-31 11:55:19 -03002559 skb_copy_to_linear_data_offset(skb, offset, from, copy);
Herbert Xu357b40a2005-04-19 22:30:14 -07002560 if ((len -= copy) == 0)
2561 return 0;
2562 offset += copy;
2563 from += copy;
2564 }
2565
2566 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2567 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
David S. Miller1a028e52007-04-27 15:21:23 -07002568 int end;
Herbert Xu357b40a2005-04-19 22:30:14 -07002569
Ilpo Järvinen547b7922008-07-25 21:43:18 -07002570 WARN_ON(start > offset + len);
David S. Miller1a028e52007-04-27 15:21:23 -07002571
Eric Dumazet9e903e02011-10-18 21:00:24 +00002572 end = start + skb_frag_size(frag);
Herbert Xu357b40a2005-04-19 22:30:14 -07002573 if ((copy = end - offset) > 0) {
Willem de Bruijnc613c202017-07-31 08:15:47 -04002574 u32 p_off, p_len, copied;
2575 struct page *p;
Herbert Xu357b40a2005-04-19 22:30:14 -07002576 u8 *vaddr;
2577
2578 if (copy > len)
2579 copy = len;
2580
Willem de Bruijnc613c202017-07-31 08:15:47 -04002581 skb_frag_foreach_page(frag,
2582 frag->page_offset + offset - start,
2583 copy, p, p_off, p_len, copied) {
2584 vaddr = kmap_atomic(p);
2585 memcpy(vaddr + p_off, from + copied, p_len);
2586 kunmap_atomic(vaddr);
2587 }
Herbert Xu357b40a2005-04-19 22:30:14 -07002588
2589 if ((len -= copy) == 0)
2590 return 0;
2591 offset += copy;
2592 from += copy;
2593 }
David S. Miller1a028e52007-04-27 15:21:23 -07002594 start = end;
Herbert Xu357b40a2005-04-19 22:30:14 -07002595 }
2596
David S. Millerfbb398a2009-06-09 00:18:59 -07002597 skb_walk_frags(skb, frag_iter) {
2598 int end;
Herbert Xu357b40a2005-04-19 22:30:14 -07002599
David S. Millerfbb398a2009-06-09 00:18:59 -07002600 WARN_ON(start > offset + len);
Herbert Xu357b40a2005-04-19 22:30:14 -07002601
David S. Millerfbb398a2009-06-09 00:18:59 -07002602 end = start + frag_iter->len;
2603 if ((copy = end - offset) > 0) {
2604 if (copy > len)
2605 copy = len;
2606 if (skb_store_bits(frag_iter, offset - start,
2607 from, copy))
2608 goto fault;
2609 if ((len -= copy) == 0)
2610 return 0;
2611 offset += copy;
2612 from += copy;
Herbert Xu357b40a2005-04-19 22:30:14 -07002613 }
David S. Millerfbb398a2009-06-09 00:18:59 -07002614 start = end;
Herbert Xu357b40a2005-04-19 22:30:14 -07002615 }
2616 if (!len)
2617 return 0;
2618
2619fault:
2620 return -EFAULT;
2621}
Herbert Xu357b40a2005-04-19 22:30:14 -07002622EXPORT_SYMBOL(skb_store_bits);
2623
Linus Torvalds1da177e2005-04-16 15:20:36 -07002624/* Checksum skb data. */
Daniel Borkmann2817a332013-10-30 11:50:51 +01002625__wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
2626 __wsum csum, const struct skb_checksum_ops *ops)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002627{
David S. Miller1a028e52007-04-27 15:21:23 -07002628 int start = skb_headlen(skb);
2629 int i, copy = start - offset;
David S. Millerfbb398a2009-06-09 00:18:59 -07002630 struct sk_buff *frag_iter;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002631 int pos = 0;
2632
2633 /* Checksum header. */
2634 if (copy > 0) {
2635 if (copy > len)
2636 copy = len;
Matteo Croce2544af02019-05-29 17:13:48 +02002637 csum = INDIRECT_CALL_1(ops->update, csum_partial_ext,
2638 skb->data + offset, copy, csum);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002639 if ((len -= copy) == 0)
2640 return csum;
2641 offset += copy;
2642 pos = copy;
2643 }
2644
2645 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
David S. Miller1a028e52007-04-27 15:21:23 -07002646 int end;
Eric Dumazet51c56b02012-04-05 11:35:15 +02002647 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
Linus Torvalds1da177e2005-04-16 15:20:36 -07002648
Ilpo Järvinen547b7922008-07-25 21:43:18 -07002649 WARN_ON(start > offset + len);
David S. Miller1a028e52007-04-27 15:21:23 -07002650
Eric Dumazet51c56b02012-04-05 11:35:15 +02002651 end = start + skb_frag_size(frag);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002652 if ((copy = end - offset) > 0) {
Willem de Bruijnc613c202017-07-31 08:15:47 -04002653 u32 p_off, p_len, copied;
2654 struct page *p;
Al Viro44bb9362006-11-14 21:36:14 -08002655 __wsum csum2;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002656 u8 *vaddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002657
2658 if (copy > len)
2659 copy = len;
Willem de Bruijnc613c202017-07-31 08:15:47 -04002660
2661 skb_frag_foreach_page(frag,
2662 frag->page_offset + offset - start,
2663 copy, p, p_off, p_len, copied) {
2664 vaddr = kmap_atomic(p);
Matteo Croce2544af02019-05-29 17:13:48 +02002665 csum2 = INDIRECT_CALL_1(ops->update,
2666 csum_partial_ext,
2667 vaddr + p_off, p_len, 0);
Willem de Bruijnc613c202017-07-31 08:15:47 -04002668 kunmap_atomic(vaddr);
Matteo Croce2544af02019-05-29 17:13:48 +02002669 csum = INDIRECT_CALL_1(ops->combine,
2670 csum_block_add_ext, csum,
2671 csum2, pos, p_len);
Willem de Bruijnc613c202017-07-31 08:15:47 -04002672 pos += p_len;
2673 }
2674
Linus Torvalds1da177e2005-04-16 15:20:36 -07002675 if (!(len -= copy))
2676 return csum;
2677 offset += copy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002678 }
David S. Miller1a028e52007-04-27 15:21:23 -07002679 start = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002680 }
2681
David S. Millerfbb398a2009-06-09 00:18:59 -07002682 skb_walk_frags(skb, frag_iter) {
2683 int end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002684
David S. Millerfbb398a2009-06-09 00:18:59 -07002685 WARN_ON(start > offset + len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002686
David S. Millerfbb398a2009-06-09 00:18:59 -07002687 end = start + frag_iter->len;
2688 if ((copy = end - offset) > 0) {
2689 __wsum csum2;
2690 if (copy > len)
2691 copy = len;
Daniel Borkmann2817a332013-10-30 11:50:51 +01002692 csum2 = __skb_checksum(frag_iter, offset - start,
2693 copy, 0, ops);
Matteo Croce2544af02019-05-29 17:13:48 +02002694 csum = INDIRECT_CALL_1(ops->combine, csum_block_add_ext,
2695 csum, csum2, pos, copy);
David S. Millerfbb398a2009-06-09 00:18:59 -07002696 if ((len -= copy) == 0)
2697 return csum;
2698 offset += copy;
2699 pos += copy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002700 }
David S. Millerfbb398a2009-06-09 00:18:59 -07002701 start = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002702 }
Kris Katterjohn09a62662006-01-08 22:24:28 -08002703 BUG_ON(len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002704
2705 return csum;
2706}
Daniel Borkmann2817a332013-10-30 11:50:51 +01002707EXPORT_SYMBOL(__skb_checksum);
2708
2709__wsum skb_checksum(const struct sk_buff *skb, int offset,
2710 int len, __wsum csum)
2711{
2712 const struct skb_checksum_ops ops = {
Daniel Borkmanncea80ea2013-11-04 17:10:25 +01002713 .update = csum_partial_ext,
Daniel Borkmann2817a332013-10-30 11:50:51 +01002714 .combine = csum_block_add_ext,
2715 };
2716
2717 return __skb_checksum(skb, offset, len, csum, &ops);
2718}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002719EXPORT_SYMBOL(skb_checksum);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002720
2721/* Both of above in one bottle. */
2722
Al Viro81d77662006-11-14 21:37:33 -08002723__wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
2724 u8 *to, int len, __wsum csum)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002725{
David S. Miller1a028e52007-04-27 15:21:23 -07002726 int start = skb_headlen(skb);
2727 int i, copy = start - offset;
David S. Millerfbb398a2009-06-09 00:18:59 -07002728 struct sk_buff *frag_iter;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002729 int pos = 0;
2730
2731 /* Copy header. */
2732 if (copy > 0) {
2733 if (copy > len)
2734 copy = len;
2735 csum = csum_partial_copy_nocheck(skb->data + offset, to,
2736 copy, csum);
2737 if ((len -= copy) == 0)
2738 return csum;
2739 offset += copy;
2740 to += copy;
2741 pos = copy;
2742 }
2743
2744 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
David S. Miller1a028e52007-04-27 15:21:23 -07002745 int end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002746
Ilpo Järvinen547b7922008-07-25 21:43:18 -07002747 WARN_ON(start > offset + len);
David S. Miller1a028e52007-04-27 15:21:23 -07002748
Eric Dumazet9e903e02011-10-18 21:00:24 +00002749 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002750 if ((copy = end - offset) > 0) {
Willem de Bruijnc613c202017-07-31 08:15:47 -04002751 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2752 u32 p_off, p_len, copied;
2753 struct page *p;
Al Viro50842052006-11-14 21:36:34 -08002754 __wsum csum2;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002755 u8 *vaddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002756
2757 if (copy > len)
2758 copy = len;
Willem de Bruijnc613c202017-07-31 08:15:47 -04002759
2760 skb_frag_foreach_page(frag,
2761 frag->page_offset + offset - start,
2762 copy, p, p_off, p_len, copied) {
2763 vaddr = kmap_atomic(p);
2764 csum2 = csum_partial_copy_nocheck(vaddr + p_off,
2765 to + copied,
2766 p_len, 0);
2767 kunmap_atomic(vaddr);
2768 csum = csum_block_add(csum, csum2, pos);
2769 pos += p_len;
2770 }
2771
Linus Torvalds1da177e2005-04-16 15:20:36 -07002772 if (!(len -= copy))
2773 return csum;
2774 offset += copy;
2775 to += copy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002776 }
David S. Miller1a028e52007-04-27 15:21:23 -07002777 start = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002778 }
2779
David S. Millerfbb398a2009-06-09 00:18:59 -07002780 skb_walk_frags(skb, frag_iter) {
2781 __wsum csum2;
2782 int end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002783
David S. Millerfbb398a2009-06-09 00:18:59 -07002784 WARN_ON(start > offset + len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002785
David S. Millerfbb398a2009-06-09 00:18:59 -07002786 end = start + frag_iter->len;
2787 if ((copy = end - offset) > 0) {
2788 if (copy > len)
2789 copy = len;
2790 csum2 = skb_copy_and_csum_bits(frag_iter,
2791 offset - start,
2792 to, copy, 0);
2793 csum = csum_block_add(csum, csum2, pos);
2794 if ((len -= copy) == 0)
2795 return csum;
2796 offset += copy;
2797 to += copy;
2798 pos += copy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002799 }
David S. Millerfbb398a2009-06-09 00:18:59 -07002800 start = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002801 }
Kris Katterjohn09a62662006-01-08 22:24:28 -08002802 BUG_ON(len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002803 return csum;
2804}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002805EXPORT_SYMBOL(skb_copy_and_csum_bits);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002806
Cong Wang49f8e832018-11-08 14:05:42 -08002807__sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len)
2808{
2809 __sum16 sum;
2810
2811 sum = csum_fold(skb_checksum(skb, 0, len, skb->csum));
Cong Wang14641932018-11-26 09:31:26 -08002812 /* See comments in __skb_checksum_complete(). */
Cong Wang49f8e832018-11-08 14:05:42 -08002813 if (likely(!sum)) {
2814 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
2815 !skb->csum_complete_sw)
Cong Wang7fe50ac2018-11-12 14:47:18 -08002816 netdev_rx_csum_fault(skb->dev, skb);
Cong Wang49f8e832018-11-08 14:05:42 -08002817 }
2818 if (!skb_shared(skb))
2819 skb->csum_valid = !sum;
2820 return sum;
2821}
2822EXPORT_SYMBOL(__skb_checksum_complete_head);
2823
Cong Wang14641932018-11-26 09:31:26 -08002824/* This function assumes skb->csum already holds pseudo header's checksum,
2825 * which has been changed from the hardware checksum, for example, by
2826 * __skb_checksum_validate_complete(). And, the original skb->csum must
2827 * have been validated unsuccessfully for CHECKSUM_COMPLETE case.
2828 *
2829 * It returns non-zero if the recomputed checksum is still invalid, otherwise
2830 * zero. The new checksum is stored back into skb->csum unless the skb is
2831 * shared.
2832 */
Cong Wang49f8e832018-11-08 14:05:42 -08002833__sum16 __skb_checksum_complete(struct sk_buff *skb)
2834{
2835 __wsum csum;
2836 __sum16 sum;
2837
2838 csum = skb_checksum(skb, 0, skb->len, 0);
2839
Cong Wang49f8e832018-11-08 14:05:42 -08002840 sum = csum_fold(csum_add(skb->csum, csum));
Cong Wang14641932018-11-26 09:31:26 -08002841 /* This check is inverted, because we already knew the hardware
2842 * checksum is invalid before calling this function. So, if the
2843 * re-computed checksum is valid instead, then we have a mismatch
2844 * between the original skb->csum and skb_checksum(). This means either
2845 * the original hardware checksum is incorrect or we screw up skb->csum
2846 * when moving skb->data around.
2847 */
Cong Wang49f8e832018-11-08 14:05:42 -08002848 if (likely(!sum)) {
2849 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
2850 !skb->csum_complete_sw)
Cong Wang7fe50ac2018-11-12 14:47:18 -08002851 netdev_rx_csum_fault(skb->dev, skb);
Cong Wang49f8e832018-11-08 14:05:42 -08002852 }
2853
2854 if (!skb_shared(skb)) {
2855 /* Save full packet checksum */
2856 skb->csum = csum;
2857 skb->ip_summed = CHECKSUM_COMPLETE;
2858 skb->csum_complete_sw = 1;
2859 skb->csum_valid = !sum;
2860 }
2861
2862 return sum;
2863}
2864EXPORT_SYMBOL(__skb_checksum_complete);
2865
Davide Caratti96178132017-05-18 15:44:37 +02002866static __wsum warn_crc32c_csum_update(const void *buff, int len, __wsum sum)
2867{
2868 net_warn_ratelimited(
2869 "%s: attempt to compute crc32c without libcrc32c.ko\n",
2870 __func__);
2871 return 0;
2872}
2873
2874static __wsum warn_crc32c_csum_combine(__wsum csum, __wsum csum2,
2875 int offset, int len)
2876{
2877 net_warn_ratelimited(
2878 "%s: attempt to compute crc32c without libcrc32c.ko\n",
2879 __func__);
2880 return 0;
2881}
2882
2883static const struct skb_checksum_ops default_crc32c_ops = {
2884 .update = warn_crc32c_csum_update,
2885 .combine = warn_crc32c_csum_combine,
2886};
2887
2888const struct skb_checksum_ops *crc32c_csum_stub __read_mostly =
2889 &default_crc32c_ops;
2890EXPORT_SYMBOL(crc32c_csum_stub);
2891
Thomas Grafaf2806f2013-12-13 15:22:17 +01002892 /**
2893 * skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy()
2894 * @from: source buffer
2895 *
2896 * Calculates the amount of linear headroom needed in the 'to' skb passed
2897 * into skb_zerocopy().
2898 */
2899unsigned int
2900skb_zerocopy_headlen(const struct sk_buff *from)
2901{
2902 unsigned int hlen = 0;
2903
2904 if (!from->head_frag ||
2905 skb_headlen(from) < L1_CACHE_BYTES ||
2906 skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS)
2907 hlen = skb_headlen(from);
2908
2909 if (skb_has_frag_list(from))
2910 hlen = from->len;
2911
2912 return hlen;
2913}
2914EXPORT_SYMBOL_GPL(skb_zerocopy_headlen);
2915
2916/**
2917 * skb_zerocopy - Zero copy skb to skb
2918 * @to: destination buffer
Masanari Iida7fceb4d2014-01-29 01:05:28 +09002919 * @from: source buffer
Thomas Grafaf2806f2013-12-13 15:22:17 +01002920 * @len: number of bytes to copy from source buffer
2921 * @hlen: size of linear headroom in destination buffer
2922 *
2923 * Copies up to `len` bytes from `from` to `to` by creating references
2924 * to the frags in the source buffer.
2925 *
2926 * The `hlen` as calculated by skb_zerocopy_headlen() specifies the
2927 * headroom in the `to` buffer.
Zoltan Kiss36d5fe62014-03-26 22:37:45 +00002928 *
2929 * Return value:
2930 * 0: everything is OK
2931 * -ENOMEM: couldn't orphan frags of @from due to lack of memory
2932 * -EFAULT: skb_copy_bits() found some problem with skb geometry
Thomas Grafaf2806f2013-12-13 15:22:17 +01002933 */
Zoltan Kiss36d5fe62014-03-26 22:37:45 +00002934int
2935skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen)
Thomas Grafaf2806f2013-12-13 15:22:17 +01002936{
2937 int i, j = 0;
2938 int plen = 0; /* length of skb->head fragment */
Zoltan Kiss36d5fe62014-03-26 22:37:45 +00002939 int ret;
Thomas Grafaf2806f2013-12-13 15:22:17 +01002940 struct page *page;
2941 unsigned int offset;
2942
2943 BUG_ON(!from->head_frag && !hlen);
2944
2945 /* dont bother with small payloads */
Zoltan Kiss36d5fe62014-03-26 22:37:45 +00002946 if (len <= skb_tailroom(to))
2947 return skb_copy_bits(from, 0, skb_put(to, len), len);
Thomas Grafaf2806f2013-12-13 15:22:17 +01002948
2949 if (hlen) {
Zoltan Kiss36d5fe62014-03-26 22:37:45 +00002950 ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen);
2951 if (unlikely(ret))
2952 return ret;
Thomas Grafaf2806f2013-12-13 15:22:17 +01002953 len -= hlen;
2954 } else {
2955 plen = min_t(int, skb_headlen(from), len);
2956 if (plen) {
2957 page = virt_to_head_page(from->head);
2958 offset = from->data - (unsigned char *)page_address(page);
2959 __skb_fill_page_desc(to, 0, page, offset, plen);
2960 get_page(page);
2961 j = 1;
2962 len -= plen;
2963 }
2964 }
2965
2966 to->truesize += len + plen;
2967 to->len += len + plen;
2968 to->data_len += len + plen;
2969
Zoltan Kiss36d5fe62014-03-26 22:37:45 +00002970 if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) {
2971 skb_tx_error(from);
2972 return -ENOMEM;
2973 }
Willem de Bruijn1f8b9772017-08-03 16:29:41 -04002974 skb_zerocopy_clone(to, from, GFP_ATOMIC);
Zoltan Kiss36d5fe62014-03-26 22:37:45 +00002975
Thomas Grafaf2806f2013-12-13 15:22:17 +01002976 for (i = 0; i < skb_shinfo(from)->nr_frags; i++) {
2977 if (!len)
2978 break;
2979 skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i];
2980 skb_shinfo(to)->frags[j].size = min_t(int, skb_shinfo(to)->frags[j].size, len);
2981 len -= skb_shinfo(to)->frags[j].size;
2982 skb_frag_ref(to, j);
2983 j++;
2984 }
2985 skb_shinfo(to)->nr_frags = j;
Zoltan Kiss36d5fe62014-03-26 22:37:45 +00002986
2987 return 0;
Thomas Grafaf2806f2013-12-13 15:22:17 +01002988}
2989EXPORT_SYMBOL_GPL(skb_zerocopy);
2990
Linus Torvalds1da177e2005-04-16 15:20:36 -07002991void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
2992{
Al Virod3bc23e2006-11-14 21:24:49 -08002993 __wsum csum;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002994 long csstart;
2995
Patrick McHardy84fa7932006-08-29 16:44:56 -07002996 if (skb->ip_summed == CHECKSUM_PARTIAL)
Michał Mirosław55508d62010-12-14 15:24:08 +00002997 csstart = skb_checksum_start_offset(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002998 else
2999 csstart = skb_headlen(skb);
3000
Kris Katterjohn09a62662006-01-08 22:24:28 -08003001 BUG_ON(csstart > skb_headlen(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003002
Arnaldo Carvalho de Melod626f622007-03-27 18:55:52 -03003003 skb_copy_from_linear_data(skb, to, csstart);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003004
3005 csum = 0;
3006 if (csstart != skb->len)
3007 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
3008 skb->len - csstart, 0);
3009
Patrick McHardy84fa7932006-08-29 16:44:56 -07003010 if (skb->ip_summed == CHECKSUM_PARTIAL) {
Al Viroff1dcad2006-11-20 18:07:29 -08003011 long csstuff = csstart + skb->csum_offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003012
Al Virod3bc23e2006-11-14 21:24:49 -08003013 *((__sum16 *)(to + csstuff)) = csum_fold(csum);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003014 }
3015}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003016EXPORT_SYMBOL(skb_copy_and_csum_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003017
3018/**
3019 * skb_dequeue - remove from the head of the queue
3020 * @list: list to dequeue from
3021 *
3022 * Remove the head of the list. The list lock is taken so the function
3023 * may be used safely with other locking list functions. The head item is
3024 * returned or %NULL if the list is empty.
3025 */
3026
3027struct sk_buff *skb_dequeue(struct sk_buff_head *list)
3028{
3029 unsigned long flags;
3030 struct sk_buff *result;
3031
3032 spin_lock_irqsave(&list->lock, flags);
3033 result = __skb_dequeue(list);
3034 spin_unlock_irqrestore(&list->lock, flags);
3035 return result;
3036}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003037EXPORT_SYMBOL(skb_dequeue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003038
3039/**
3040 * skb_dequeue_tail - remove from the tail of the queue
3041 * @list: list to dequeue from
3042 *
3043 * Remove the tail of the list. The list lock is taken so the function
3044 * may be used safely with other locking list functions. The tail item is
3045 * returned or %NULL if the list is empty.
3046 */
3047struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
3048{
3049 unsigned long flags;
3050 struct sk_buff *result;
3051
3052 spin_lock_irqsave(&list->lock, flags);
3053 result = __skb_dequeue_tail(list);
3054 spin_unlock_irqrestore(&list->lock, flags);
3055 return result;
3056}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003057EXPORT_SYMBOL(skb_dequeue_tail);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003058
3059/**
3060 * skb_queue_purge - empty a list
3061 * @list: list to empty
3062 *
3063 * Delete all buffers on an &sk_buff list. Each buffer is removed from
3064 * the list and one reference dropped. This function takes the list
3065 * lock and is atomic with respect to other list locking functions.
3066 */
3067void skb_queue_purge(struct sk_buff_head *list)
3068{
3069 struct sk_buff *skb;
3070 while ((skb = skb_dequeue(list)) != NULL)
3071 kfree_skb(skb);
3072}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003073EXPORT_SYMBOL(skb_queue_purge);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003074
3075/**
Yaogong Wang9f5afea2016-09-07 14:49:28 -07003076 * skb_rbtree_purge - empty a skb rbtree
3077 * @root: root of the rbtree to empty
Peter Oskolkov385114d2018-08-02 23:34:38 +00003078 * Return value: the sum of truesizes of all purged skbs.
Yaogong Wang9f5afea2016-09-07 14:49:28 -07003079 *
3080 * Delete all buffers on an &sk_buff rbtree. Each buffer is removed from
3081 * the list and one reference dropped. This function does not take
3082 * any lock. Synchronization should be handled by the caller (e.g., TCP
3083 * out-of-order queue is protected by the socket lock).
3084 */
Peter Oskolkov385114d2018-08-02 23:34:38 +00003085unsigned int skb_rbtree_purge(struct rb_root *root)
Yaogong Wang9f5afea2016-09-07 14:49:28 -07003086{
Eric Dumazet7c905842017-09-23 12:39:12 -07003087 struct rb_node *p = rb_first(root);
Peter Oskolkov385114d2018-08-02 23:34:38 +00003088 unsigned int sum = 0;
Yaogong Wang9f5afea2016-09-07 14:49:28 -07003089
Eric Dumazet7c905842017-09-23 12:39:12 -07003090 while (p) {
3091 struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode);
3092
3093 p = rb_next(p);
3094 rb_erase(&skb->rbnode, root);
Peter Oskolkov385114d2018-08-02 23:34:38 +00003095 sum += skb->truesize;
Yaogong Wang9f5afea2016-09-07 14:49:28 -07003096 kfree_skb(skb);
Eric Dumazet7c905842017-09-23 12:39:12 -07003097 }
Peter Oskolkov385114d2018-08-02 23:34:38 +00003098 return sum;
Yaogong Wang9f5afea2016-09-07 14:49:28 -07003099}
3100
3101/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07003102 * skb_queue_head - queue a buffer at the list head
3103 * @list: list to use
3104 * @newsk: buffer to queue
3105 *
3106 * Queue a buffer at the start of the list. This function takes the
3107 * list lock and can be used safely with other locking &sk_buff functions
3108 * safely.
3109 *
3110 * A buffer cannot be placed on two lists at the same time.
3111 */
3112void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
3113{
3114 unsigned long flags;
3115
3116 spin_lock_irqsave(&list->lock, flags);
3117 __skb_queue_head(list, newsk);
3118 spin_unlock_irqrestore(&list->lock, flags);
3119}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003120EXPORT_SYMBOL(skb_queue_head);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003121
3122/**
3123 * skb_queue_tail - queue a buffer at the list tail
3124 * @list: list to use
3125 * @newsk: buffer to queue
3126 *
3127 * Queue a buffer at the tail of the list. This function takes the
3128 * list lock and can be used safely with other locking &sk_buff functions
3129 * safely.
3130 *
3131 * A buffer cannot be placed on two lists at the same time.
3132 */
3133void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
3134{
3135 unsigned long flags;
3136
3137 spin_lock_irqsave(&list->lock, flags);
3138 __skb_queue_tail(list, newsk);
3139 spin_unlock_irqrestore(&list->lock, flags);
3140}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003141EXPORT_SYMBOL(skb_queue_tail);
David S. Miller8728b832005-08-09 19:25:21 -07003142
Linus Torvalds1da177e2005-04-16 15:20:36 -07003143/**
3144 * skb_unlink - remove a buffer from a list
3145 * @skb: buffer to remove
David S. Miller8728b832005-08-09 19:25:21 -07003146 * @list: list to use
Linus Torvalds1da177e2005-04-16 15:20:36 -07003147 *
David S. Miller8728b832005-08-09 19:25:21 -07003148 * Remove a packet from a list. The list locks are taken and this
3149 * function is atomic with respect to other list locked calls
Linus Torvalds1da177e2005-04-16 15:20:36 -07003150 *
David S. Miller8728b832005-08-09 19:25:21 -07003151 * You must know what list the SKB is on.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003152 */
David S. Miller8728b832005-08-09 19:25:21 -07003153void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003154{
David S. Miller8728b832005-08-09 19:25:21 -07003155 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003156
David S. Miller8728b832005-08-09 19:25:21 -07003157 spin_lock_irqsave(&list->lock, flags);
3158 __skb_unlink(skb, list);
3159 spin_unlock_irqrestore(&list->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003160}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003161EXPORT_SYMBOL(skb_unlink);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003162
Linus Torvalds1da177e2005-04-16 15:20:36 -07003163/**
3164 * skb_append - append a buffer
3165 * @old: buffer to insert after
3166 * @newsk: buffer to insert
David S. Miller8728b832005-08-09 19:25:21 -07003167 * @list: list to use
Linus Torvalds1da177e2005-04-16 15:20:36 -07003168 *
3169 * Place a packet after a given packet in a list. The list locks are taken
3170 * and this function is atomic with respect to other list locked calls.
3171 * A buffer cannot be placed on two lists at the same time.
3172 */
David S. Miller8728b832005-08-09 19:25:21 -07003173void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003174{
3175 unsigned long flags;
3176
David S. Miller8728b832005-08-09 19:25:21 -07003177 spin_lock_irqsave(&list->lock, flags);
Gerrit Renker7de6c032008-04-14 00:05:09 -07003178 __skb_queue_after(list, old, newsk);
David S. Miller8728b832005-08-09 19:25:21 -07003179 spin_unlock_irqrestore(&list->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003180}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003181EXPORT_SYMBOL(skb_append);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003182
Linus Torvalds1da177e2005-04-16 15:20:36 -07003183static inline void skb_split_inside_header(struct sk_buff *skb,
3184 struct sk_buff* skb1,
3185 const u32 len, const int pos)
3186{
3187 int i;
3188
Arnaldo Carvalho de Melod626f622007-03-27 18:55:52 -03003189 skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
3190 pos - len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003191 /* And move data appendix as is. */
3192 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
3193 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
3194
3195 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
3196 skb_shinfo(skb)->nr_frags = 0;
3197 skb1->data_len = skb->data_len;
3198 skb1->len += skb1->data_len;
3199 skb->data_len = 0;
3200 skb->len = len;
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07003201 skb_set_tail_pointer(skb, len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003202}
3203
3204static inline void skb_split_no_header(struct sk_buff *skb,
3205 struct sk_buff* skb1,
3206 const u32 len, int pos)
3207{
3208 int i, k = 0;
3209 const int nfrags = skb_shinfo(skb)->nr_frags;
3210
3211 skb_shinfo(skb)->nr_frags = 0;
3212 skb1->len = skb1->data_len = skb->len - len;
3213 skb->len = len;
3214 skb->data_len = len - pos;
3215
3216 for (i = 0; i < nfrags; i++) {
Eric Dumazet9e903e02011-10-18 21:00:24 +00003217 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003218
3219 if (pos + size > len) {
3220 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
3221
3222 if (pos < len) {
3223 /* Split frag.
3224 * We have two variants in this case:
3225 * 1. Move all the frag to the second
3226 * part, if it is possible. F.e.
3227 * this approach is mandatory for TUX,
3228 * where splitting is expensive.
3229 * 2. Split is accurately. We make this.
3230 */
Ian Campbellea2ab692011-08-22 23:44:58 +00003231 skb_frag_ref(skb, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003232 skb_shinfo(skb1)->frags[0].page_offset += len - pos;
Eric Dumazet9e903e02011-10-18 21:00:24 +00003233 skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos);
3234 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003235 skb_shinfo(skb)->nr_frags++;
3236 }
3237 k++;
3238 } else
3239 skb_shinfo(skb)->nr_frags++;
3240 pos += size;
3241 }
3242 skb_shinfo(skb1)->nr_frags = k;
3243}
3244
3245/**
3246 * skb_split - Split fragmented skb to two parts at length len.
3247 * @skb: the buffer to split
3248 * @skb1: the buffer to receive the second part
3249 * @len: new length for skb
3250 */
3251void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
3252{
3253 int pos = skb_headlen(skb);
3254
Willem de Bruijnfff88032017-06-08 11:35:03 -04003255 skb_shinfo(skb1)->tx_flags |= skb_shinfo(skb)->tx_flags &
3256 SKBTX_SHARED_FRAG;
Willem de Bruijn1f8b9772017-08-03 16:29:41 -04003257 skb_zerocopy_clone(skb1, skb, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003258 if (len < pos) /* Split line is inside header. */
3259 skb_split_inside_header(skb, skb1, len, pos);
3260 else /* Second chunk has no header, nothing to copy. */
3261 skb_split_no_header(skb, skb1, len, pos);
3262}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003263EXPORT_SYMBOL(skb_split);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003264
Ilpo Järvinen9f782db2008-11-25 13:57:01 -08003265/* Shifting from/to a cloned skb is a no-go.
3266 *
3267 * Caller cannot keep skb_shinfo related pointers past calling here!
3268 */
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003269static int skb_prepare_for_shift(struct sk_buff *skb)
3270{
Ilpo Järvinen0ace2852008-11-24 21:30:21 -08003271 return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003272}
3273
3274/**
3275 * skb_shift - Shifts paged data partially from skb to another
3276 * @tgt: buffer into which tail data gets added
3277 * @skb: buffer from which the paged data comes from
3278 * @shiftlen: shift up to this many bytes
3279 *
3280 * Attempts to shift up to shiftlen worth of bytes, which may be less than
Feng King20e994a2011-11-21 01:47:11 +00003281 * the length of the skb, from skb to tgt. Returns number bytes shifted.
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003282 * It's up to caller to free skb if everything was shifted.
3283 *
3284 * If @tgt runs out of frags, the whole operation is aborted.
3285 *
3286 * Skb cannot include anything else but paged data while tgt is allowed
3287 * to have non-paged data as well.
3288 *
3289 * TODO: full sized shift could be optimized but that would need
3290 * specialized skb free'er to handle frags without up-to-date nr_frags.
3291 */
3292int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
3293{
3294 int from, to, merge, todo;
3295 struct skb_frag_struct *fragfrom, *fragto;
3296
3297 BUG_ON(shiftlen > skb->len);
Eric Dumazetf8071cd2016-11-15 12:51:50 -08003298
3299 if (skb_headlen(skb))
3300 return 0;
Willem de Bruijn1f8b9772017-08-03 16:29:41 -04003301 if (skb_zcopy(tgt) || skb_zcopy(skb))
3302 return 0;
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003303
3304 todo = shiftlen;
3305 from = 0;
3306 to = skb_shinfo(tgt)->nr_frags;
3307 fragfrom = &skb_shinfo(skb)->frags[from];
3308
3309 /* Actual merge is delayed until the point when we know we can
3310 * commit all, so that we don't have to undo partial changes
3311 */
3312 if (!to ||
Ian Campbellea2ab692011-08-22 23:44:58 +00003313 !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom),
3314 fragfrom->page_offset)) {
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003315 merge = -1;
3316 } else {
3317 merge = to - 1;
3318
Eric Dumazet9e903e02011-10-18 21:00:24 +00003319 todo -= skb_frag_size(fragfrom);
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003320 if (todo < 0) {
3321 if (skb_prepare_for_shift(skb) ||
3322 skb_prepare_for_shift(tgt))
3323 return 0;
3324
Ilpo Järvinen9f782db2008-11-25 13:57:01 -08003325 /* All previous frag pointers might be stale! */
3326 fragfrom = &skb_shinfo(skb)->frags[from];
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003327 fragto = &skb_shinfo(tgt)->frags[merge];
3328
Eric Dumazet9e903e02011-10-18 21:00:24 +00003329 skb_frag_size_add(fragto, shiftlen);
3330 skb_frag_size_sub(fragfrom, shiftlen);
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003331 fragfrom->page_offset += shiftlen;
3332
3333 goto onlymerged;
3334 }
3335
3336 from++;
3337 }
3338
3339 /* Skip full, not-fitting skb to avoid expensive operations */
3340 if ((shiftlen == skb->len) &&
3341 (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to))
3342 return 0;
3343
3344 if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt))
3345 return 0;
3346
3347 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
3348 if (to == MAX_SKB_FRAGS)
3349 return 0;
3350
3351 fragfrom = &skb_shinfo(skb)->frags[from];
3352 fragto = &skb_shinfo(tgt)->frags[to];
3353
Eric Dumazet9e903e02011-10-18 21:00:24 +00003354 if (todo >= skb_frag_size(fragfrom)) {
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003355 *fragto = *fragfrom;
Eric Dumazet9e903e02011-10-18 21:00:24 +00003356 todo -= skb_frag_size(fragfrom);
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003357 from++;
3358 to++;
3359
3360 } else {
Ian Campbellea2ab692011-08-22 23:44:58 +00003361 __skb_frag_ref(fragfrom);
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003362 fragto->page = fragfrom->page;
3363 fragto->page_offset = fragfrom->page_offset;
Eric Dumazet9e903e02011-10-18 21:00:24 +00003364 skb_frag_size_set(fragto, todo);
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003365
3366 fragfrom->page_offset += todo;
Eric Dumazet9e903e02011-10-18 21:00:24 +00003367 skb_frag_size_sub(fragfrom, todo);
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003368 todo = 0;
3369
3370 to++;
3371 break;
3372 }
3373 }
3374
3375 /* Ready to "commit" this state change to tgt */
3376 skb_shinfo(tgt)->nr_frags = to;
3377
3378 if (merge >= 0) {
3379 fragfrom = &skb_shinfo(skb)->frags[0];
3380 fragto = &skb_shinfo(tgt)->frags[merge];
3381
Eric Dumazet9e903e02011-10-18 21:00:24 +00003382 skb_frag_size_add(fragto, skb_frag_size(fragfrom));
Ian Campbellea2ab692011-08-22 23:44:58 +00003383 __skb_frag_unref(fragfrom);
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08003384 }
3385
3386 /* Reposition in the original skb */
3387 to = 0;
3388 while (from < skb_shinfo(skb)->nr_frags)
3389 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
3390 skb_shinfo(skb)->nr_frags = to;
3391
3392 BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);
3393
3394onlymerged:
3395 /* Most likely the tgt won't ever need its checksum anymore, skb on
3396 * the other hand might need it if it needs to be resent
3397 */
3398 tgt->ip_summed = CHECKSUM_PARTIAL;
3399 skb->ip_summed = CHECKSUM_PARTIAL;
3400
3401 /* Yak, is it really working this way? Some helper please? */
3402 skb->len -= shiftlen;
3403 skb->data_len -= shiftlen;
3404 skb->truesize -= shiftlen;
3405 tgt->len += shiftlen;
3406 tgt->data_len += shiftlen;
3407 tgt->truesize += shiftlen;
3408
3409 return shiftlen;
3410}
3411
Thomas Graf677e90e2005-06-23 20:59:51 -07003412/**
3413 * skb_prepare_seq_read - Prepare a sequential read of skb data
3414 * @skb: the buffer to read
3415 * @from: lower offset of data to be read
3416 * @to: upper offset of data to be read
3417 * @st: state variable
3418 *
3419 * Initializes the specified state variable. Must be called before
3420 * invoking skb_seq_read() for the first time.
3421 */
3422void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
3423 unsigned int to, struct skb_seq_state *st)
3424{
3425 st->lower_offset = from;
3426 st->upper_offset = to;
3427 st->root_skb = st->cur_skb = skb;
3428 st->frag_idx = st->stepped_offset = 0;
3429 st->frag_data = NULL;
3430}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003431EXPORT_SYMBOL(skb_prepare_seq_read);
Thomas Graf677e90e2005-06-23 20:59:51 -07003432
3433/**
3434 * skb_seq_read - Sequentially read skb data
3435 * @consumed: number of bytes consumed by the caller so far
3436 * @data: destination pointer for data to be returned
3437 * @st: state variable
3438 *
Mathias Krausebc323832013-11-07 14:18:26 +01003439 * Reads a block of skb data at @consumed relative to the
Thomas Graf677e90e2005-06-23 20:59:51 -07003440 * lower offset specified to skb_prepare_seq_read(). Assigns
Mathias Krausebc323832013-11-07 14:18:26 +01003441 * the head of the data block to @data and returns the length
Thomas Graf677e90e2005-06-23 20:59:51 -07003442 * of the block or 0 if the end of the skb data or the upper
3443 * offset has been reached.
3444 *
3445 * The caller is not required to consume all of the data
Mathias Krausebc323832013-11-07 14:18:26 +01003446 * returned, i.e. @consumed is typically set to the number
Thomas Graf677e90e2005-06-23 20:59:51 -07003447 * of bytes already consumed and the next call to
3448 * skb_seq_read() will return the remaining part of the block.
3449 *
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003450 * Note 1: The size of each block of data returned can be arbitrary,
Masanari Iidae793c0f2014-09-04 23:44:36 +09003451 * this limitation is the cost for zerocopy sequential
Thomas Graf677e90e2005-06-23 20:59:51 -07003452 * reads of potentially non linear data.
3453 *
Randy Dunlapbc2cda12008-02-13 15:03:25 -08003454 * Note 2: Fragment lists within fragments are not implemented
Thomas Graf677e90e2005-06-23 20:59:51 -07003455 * at the moment, state->root_skb could be replaced with
3456 * a stack for this purpose.
3457 */
3458unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
3459 struct skb_seq_state *st)
3460{
3461 unsigned int block_limit, abs_offset = consumed + st->lower_offset;
3462 skb_frag_t *frag;
3463
Wedson Almeida Filhoaeb193e2013-06-23 23:33:48 -07003464 if (unlikely(abs_offset >= st->upper_offset)) {
3465 if (st->frag_data) {
3466 kunmap_atomic(st->frag_data);
3467 st->frag_data = NULL;
3468 }
Thomas Graf677e90e2005-06-23 20:59:51 -07003469 return 0;
Wedson Almeida Filhoaeb193e2013-06-23 23:33:48 -07003470 }
Thomas Graf677e90e2005-06-23 20:59:51 -07003471
3472next_skb:
Herbert Xu95e3b242009-01-29 16:07:52 -08003473 block_limit = skb_headlen(st->cur_skb) + st->stepped_offset;
Thomas Graf677e90e2005-06-23 20:59:51 -07003474
Thomas Chenault995b3372009-05-18 21:43:27 -07003475 if (abs_offset < block_limit && !st->frag_data) {
Herbert Xu95e3b242009-01-29 16:07:52 -08003476 *data = st->cur_skb->data + (abs_offset - st->stepped_offset);
Thomas Graf677e90e2005-06-23 20:59:51 -07003477 return block_limit - abs_offset;
3478 }
3479
3480 if (st->frag_idx == 0 && !st->frag_data)
3481 st->stepped_offset += skb_headlen(st->cur_skb);
3482
3483 while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
3484 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
Eric Dumazet9e903e02011-10-18 21:00:24 +00003485 block_limit = skb_frag_size(frag) + st->stepped_offset;
Thomas Graf677e90e2005-06-23 20:59:51 -07003486
3487 if (abs_offset < block_limit) {
3488 if (!st->frag_data)
Eric Dumazet51c56b02012-04-05 11:35:15 +02003489 st->frag_data = kmap_atomic(skb_frag_page(frag));
Thomas Graf677e90e2005-06-23 20:59:51 -07003490
3491 *data = (u8 *) st->frag_data + frag->page_offset +
3492 (abs_offset - st->stepped_offset);
3493
3494 return block_limit - abs_offset;
3495 }
3496
3497 if (st->frag_data) {
Eric Dumazet51c56b02012-04-05 11:35:15 +02003498 kunmap_atomic(st->frag_data);
Thomas Graf677e90e2005-06-23 20:59:51 -07003499 st->frag_data = NULL;
3500 }
3501
3502 st->frag_idx++;
Eric Dumazet9e903e02011-10-18 21:00:24 +00003503 st->stepped_offset += skb_frag_size(frag);
Thomas Graf677e90e2005-06-23 20:59:51 -07003504 }
3505
Olaf Kirch5b5a60d2007-06-23 23:11:52 -07003506 if (st->frag_data) {
Eric Dumazet51c56b02012-04-05 11:35:15 +02003507 kunmap_atomic(st->frag_data);
Olaf Kirch5b5a60d2007-06-23 23:11:52 -07003508 st->frag_data = NULL;
3509 }
3510
David S. Miller21dc3302010-08-23 00:13:46 -07003511 if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) {
Shyam Iyer71b33462009-01-29 16:12:42 -08003512 st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
Thomas Graf677e90e2005-06-23 20:59:51 -07003513 st->frag_idx = 0;
3514 goto next_skb;
Shyam Iyer71b33462009-01-29 16:12:42 -08003515 } else if (st->cur_skb->next) {
3516 st->cur_skb = st->cur_skb->next;
Herbert Xu95e3b242009-01-29 16:07:52 -08003517 st->frag_idx = 0;
Thomas Graf677e90e2005-06-23 20:59:51 -07003518 goto next_skb;
3519 }
3520
3521 return 0;
3522}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003523EXPORT_SYMBOL(skb_seq_read);
Thomas Graf677e90e2005-06-23 20:59:51 -07003524
3525/**
3526 * skb_abort_seq_read - Abort a sequential read of skb data
3527 * @st: state variable
3528 *
3529 * Must be called if skb_seq_read() was not called until it
3530 * returned 0.
3531 */
3532void skb_abort_seq_read(struct skb_seq_state *st)
3533{
3534 if (st->frag_data)
Eric Dumazet51c56b02012-04-05 11:35:15 +02003535 kunmap_atomic(st->frag_data);
Thomas Graf677e90e2005-06-23 20:59:51 -07003536}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003537EXPORT_SYMBOL(skb_abort_seq_read);
Thomas Graf677e90e2005-06-23 20:59:51 -07003538
Thomas Graf3fc7e8a2005-06-23 21:00:17 -07003539#define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb))
3540
3541static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
3542 struct ts_config *conf,
3543 struct ts_state *state)
3544{
3545 return skb_seq_read(offset, text, TS_SKB_CB(state));
3546}
3547
3548static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
3549{
3550 skb_abort_seq_read(TS_SKB_CB(state));
3551}
3552
3553/**
3554 * skb_find_text - Find a text pattern in skb data
3555 * @skb: the buffer to look in
3556 * @from: search offset
3557 * @to: search limit
3558 * @config: textsearch configuration
Thomas Graf3fc7e8a2005-06-23 21:00:17 -07003559 *
3560 * Finds a pattern in the skb data according to the specified
3561 * textsearch configuration. Use textsearch_next() to retrieve
3562 * subsequent occurrences of the pattern. Returns the offset
3563 * to the first occurrence or UINT_MAX if no match was found.
3564 */
3565unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
Bojan Prtvar059a2442015-02-22 11:46:35 +01003566 unsigned int to, struct ts_config *config)
Thomas Graf3fc7e8a2005-06-23 21:00:17 -07003567{
Bojan Prtvar059a2442015-02-22 11:46:35 +01003568 struct ts_state state;
Phil Oesterf72b9482006-06-26 00:00:57 -07003569 unsigned int ret;
3570
Thomas Graf3fc7e8a2005-06-23 21:00:17 -07003571 config->get_next_block = skb_ts_get_next_block;
3572 config->finish = skb_ts_finish;
3573
Bojan Prtvar059a2442015-02-22 11:46:35 +01003574 skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state));
Thomas Graf3fc7e8a2005-06-23 21:00:17 -07003575
Bojan Prtvar059a2442015-02-22 11:46:35 +01003576 ret = textsearch_find(config, &state);
Phil Oesterf72b9482006-06-26 00:00:57 -07003577 return (ret <= to - from ? ret : UINT_MAX);
Thomas Graf3fc7e8a2005-06-23 21:00:17 -07003578}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003579EXPORT_SYMBOL(skb_find_text);
Thomas Graf3fc7e8a2005-06-23 21:00:17 -07003580
Hannes Frederic Sowabe12a1f2015-05-21 16:59:58 +02003581int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
3582 int offset, size_t size)
3583{
3584 int i = skb_shinfo(skb)->nr_frags;
3585
3586 if (skb_can_coalesce(skb, i, page, offset)) {
3587 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size);
3588 } else if (i < MAX_SKB_FRAGS) {
3589 get_page(page);
3590 skb_fill_page_desc(skb, i, page, offset, size);
3591 } else {
3592 return -EMSGSIZE;
3593 }
3594
3595 return 0;
3596}
3597EXPORT_SYMBOL_GPL(skb_append_pagefrags);
3598
Herbert Xucbb042f2006-03-20 22:43:56 -08003599/**
3600 * skb_pull_rcsum - pull skb and update receive checksum
3601 * @skb: buffer to update
Herbert Xucbb042f2006-03-20 22:43:56 -08003602 * @len: length of data pulled
3603 *
3604 * This function performs an skb_pull on the packet and updates
Urs Thuermannfee54fa2008-02-12 22:03:25 -08003605 * the CHECKSUM_COMPLETE checksum. It should be used on
Patrick McHardy84fa7932006-08-29 16:44:56 -07003606 * receive path processing instead of skb_pull unless you know
3607 * that the checksum difference is zero (e.g., a valid IP header)
3608 * or you are setting ip_summed to CHECKSUM_NONE.
Herbert Xucbb042f2006-03-20 22:43:56 -08003609 */
Johannes Bergaf728682017-06-16 14:29:22 +02003610void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
Herbert Xucbb042f2006-03-20 22:43:56 -08003611{
Pravin B Shelar31b33df2015-09-28 17:24:25 -07003612 unsigned char *data = skb->data;
3613
Herbert Xucbb042f2006-03-20 22:43:56 -08003614 BUG_ON(len > skb->len);
Pravin B Shelar31b33df2015-09-28 17:24:25 -07003615 __skb_pull(skb, len);
3616 skb_postpull_rcsum(skb, data, len);
3617 return skb->data;
Herbert Xucbb042f2006-03-20 22:43:56 -08003618}
Arnaldo Carvalho de Melof94691a2006-03-20 22:47:55 -08003619EXPORT_SYMBOL_GPL(skb_pull_rcsum);
3620
Yonghong Song13acc942018-03-21 16:31:03 -07003621static inline skb_frag_t skb_head_frag_to_page_desc(struct sk_buff *frag_skb)
3622{
3623 skb_frag_t head_frag;
3624 struct page *page;
3625
3626 page = virt_to_head_page(frag_skb->head);
3627 head_frag.page.p = page;
3628 head_frag.page_offset = frag_skb->data -
3629 (unsigned char *)page_address(page);
3630 head_frag.size = skb_headlen(frag_skb);
3631 return head_frag;
3632}
3633
Herbert Xuf4c50d92006-06-22 03:02:40 -07003634/**
3635 * skb_segment - Perform protocol segmentation on skb.
Michael S. Tsirkindf5771f2014-03-10 18:29:19 +02003636 * @head_skb: buffer to segment
Herbert Xu576a30e2006-06-27 13:22:38 -07003637 * @features: features for the output path (see dev->features)
Herbert Xuf4c50d92006-06-22 03:02:40 -07003638 *
3639 * This function performs segmentation on the given skb. It returns
Ben Hutchings4c821d72008-04-13 21:52:48 -07003640 * a pointer to the first in a list of new skbs for the segments.
3641 * In case of error it returns ERR_PTR(err).
Herbert Xuf4c50d92006-06-22 03:02:40 -07003642 */
Michael S. Tsirkindf5771f2014-03-10 18:29:19 +02003643struct sk_buff *skb_segment(struct sk_buff *head_skb,
3644 netdev_features_t features)
Herbert Xuf4c50d92006-06-22 03:02:40 -07003645{
3646 struct sk_buff *segs = NULL;
3647 struct sk_buff *tail = NULL;
Michael S. Tsirkin1a4ceda2014-03-10 19:27:59 +02003648 struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list;
Michael S. Tsirkindf5771f2014-03-10 18:29:19 +02003649 skb_frag_t *frag = skb_shinfo(head_skb)->frags;
3650 unsigned int mss = skb_shinfo(head_skb)->gso_size;
3651 unsigned int doffset = head_skb->data - skb_mac_header(head_skb);
Michael S. Tsirkin1fd819e2014-03-10 19:28:08 +02003652 struct sk_buff *frag_skb = head_skb;
Herbert Xuf4c50d92006-06-22 03:02:40 -07003653 unsigned int offset = doffset;
Michael S. Tsirkindf5771f2014-03-10 18:29:19 +02003654 unsigned int tnl_hlen = skb_tnl_header_len(head_skb);
Alexander Duyck802ab552016-04-10 21:45:03 -04003655 unsigned int partial_segs = 0;
Herbert Xuf4c50d92006-06-22 03:02:40 -07003656 unsigned int headroom;
Alexander Duyck802ab552016-04-10 21:45:03 -04003657 unsigned int len = head_skb->len;
Pravin B Shelarec5f0612013-03-07 09:28:01 +00003658 __be16 proto;
Alexander Duyck36c98382016-05-02 09:38:18 -07003659 bool csum, sg;
Michael S. Tsirkindf5771f2014-03-10 18:29:19 +02003660 int nfrags = skb_shinfo(head_skb)->nr_frags;
Herbert Xuf4c50d92006-06-22 03:02:40 -07003661 int err = -ENOMEM;
3662 int i = 0;
3663 int pos;
Vlad Yasevich53d64712014-03-27 17:26:18 -04003664 int dummy;
Herbert Xuf4c50d92006-06-22 03:02:40 -07003665
Wei-Chun Chao5882a072014-06-08 23:48:54 -07003666 __skb_push(head_skb, doffset);
Vlad Yasevich53d64712014-03-27 17:26:18 -04003667 proto = skb_network_protocol(head_skb, &dummy);
Pravin B Shelarec5f0612013-03-07 09:28:01 +00003668 if (unlikely(!proto))
3669 return ERR_PTR(-EINVAL);
3670
Alexander Duyck36c98382016-05-02 09:38:18 -07003671 sg = !!(features & NETIF_F_SG);
Alexander Duyckf245d072016-02-05 15:28:26 -08003672 csum = !!can_checksum_protocol(features, proto);
Tom Herbert7e2b10c2014-06-04 17:20:02 -07003673
Steffen Klassert07b26c92016-09-19 12:58:47 +02003674 if (sg && csum && (mss != GSO_BY_FRAGS)) {
3675 if (!(features & NETIF_F_GSO_PARTIAL)) {
3676 struct sk_buff *iter;
Ilan Tayari43170c42017-04-19 21:26:07 +03003677 unsigned int frag_len;
Steffen Klassert07b26c92016-09-19 12:58:47 +02003678
3679 if (!list_skb ||
3680 !net_gso_ok(features, skb_shinfo(head_skb)->gso_type))
3681 goto normal;
3682
Ilan Tayari43170c42017-04-19 21:26:07 +03003683 /* If we get here then all the required
3684 * GSO features except frag_list are supported.
3685 * Try to split the SKB to multiple GSO SKBs
3686 * with no frag_list.
3687 * Currently we can do that only when the buffers don't
3688 * have a linear part and all the buffers except
3689 * the last are of the same length.
Steffen Klassert07b26c92016-09-19 12:58:47 +02003690 */
Ilan Tayari43170c42017-04-19 21:26:07 +03003691 frag_len = list_skb->len;
Steffen Klassert07b26c92016-09-19 12:58:47 +02003692 skb_walk_frags(head_skb, iter) {
Ilan Tayari43170c42017-04-19 21:26:07 +03003693 if (frag_len != iter->len && iter->next)
3694 goto normal;
Ilan Tayarieaffadb2017-04-08 02:07:08 +03003695 if (skb_headlen(iter) && !iter->head_frag)
Steffen Klassert07b26c92016-09-19 12:58:47 +02003696 goto normal;
3697
3698 len -= iter->len;
3699 }
Ilan Tayari43170c42017-04-19 21:26:07 +03003700
3701 if (len != frag_len)
3702 goto normal;
Steffen Klassert07b26c92016-09-19 12:58:47 +02003703 }
3704
3705 /* GSO partial only requires that we trim off any excess that
3706 * doesn't fit into an MSS sized block, so take care of that
3707 * now.
3708 */
Alexander Duyck802ab552016-04-10 21:45:03 -04003709 partial_segs = len / mss;
Alexander Duyckd7fb5a82016-05-02 09:38:12 -07003710 if (partial_segs > 1)
3711 mss *= partial_segs;
3712 else
3713 partial_segs = 0;
Alexander Duyck802ab552016-04-10 21:45:03 -04003714 }
3715
Steffen Klassert07b26c92016-09-19 12:58:47 +02003716normal:
Michael S. Tsirkindf5771f2014-03-10 18:29:19 +02003717 headroom = skb_headroom(head_skb);
3718 pos = skb_headlen(head_skb);
Herbert Xuf4c50d92006-06-22 03:02:40 -07003719
3720 do {
3721 struct sk_buff *nskb;
Michael S. Tsirkin8cb19902014-03-10 18:29:04 +02003722 skb_frag_t *nskb_frag;
Herbert Xuc8884ed2006-10-29 15:59:41 -08003723 int hsize;
Herbert Xuf4c50d92006-06-22 03:02:40 -07003724 int size;
3725
Marcelo Ricardo Leitner3953c462016-06-02 15:05:40 -03003726 if (unlikely(mss == GSO_BY_FRAGS)) {
3727 len = list_skb->len;
3728 } else {
3729 len = head_skb->len - offset;
3730 if (len > mss)
3731 len = mss;
3732 }
Herbert Xuf4c50d92006-06-22 03:02:40 -07003733
Michael S. Tsirkindf5771f2014-03-10 18:29:19 +02003734 hsize = skb_headlen(head_skb) - offset;
Herbert Xuf4c50d92006-06-22 03:02:40 -07003735 if (hsize < 0)
3736 hsize = 0;
Herbert Xuc8884ed2006-10-29 15:59:41 -08003737 if (hsize > len || !sg)
3738 hsize = len;
Herbert Xuf4c50d92006-06-22 03:02:40 -07003739
Michael S. Tsirkin1a4ceda2014-03-10 19:27:59 +02003740 if (!hsize && i >= nfrags && skb_headlen(list_skb) &&
3741 (skb_headlen(list_skb) == len || sg)) {
3742 BUG_ON(skb_headlen(list_skb) > len);
Herbert Xu89319d382008-12-15 23:26:06 -08003743
Herbert Xu9d8506c2013-11-21 11:10:04 -08003744 i = 0;
Michael S. Tsirkin1a4ceda2014-03-10 19:27:59 +02003745 nfrags = skb_shinfo(list_skb)->nr_frags;
3746 frag = skb_shinfo(list_skb)->frags;
Michael S. Tsirkin1fd819e2014-03-10 19:28:08 +02003747 frag_skb = list_skb;
Michael S. Tsirkin1a4ceda2014-03-10 19:27:59 +02003748 pos += skb_headlen(list_skb);
Herbert Xu9d8506c2013-11-21 11:10:04 -08003749
3750 while (pos < offset + len) {
3751 BUG_ON(i >= nfrags);
3752
Michael S. Tsirkin4e1beba2014-03-10 18:29:14 +02003753 size = skb_frag_size(frag);
Herbert Xu9d8506c2013-11-21 11:10:04 -08003754 if (pos + size > offset + len)
3755 break;
3756
3757 i++;
3758 pos += size;
Michael S. Tsirkin4e1beba2014-03-10 18:29:14 +02003759 frag++;
Herbert Xu9d8506c2013-11-21 11:10:04 -08003760 }
3761
Michael S. Tsirkin1a4ceda2014-03-10 19:27:59 +02003762 nskb = skb_clone(list_skb, GFP_ATOMIC);
3763 list_skb = list_skb->next;
Herbert Xu89319d382008-12-15 23:26:06 -08003764
3765 if (unlikely(!nskb))
3766 goto err;
3767
Herbert Xu9d8506c2013-11-21 11:10:04 -08003768 if (unlikely(pskb_trim(nskb, len))) {
3769 kfree_skb(nskb);
3770 goto err;
3771 }
3772
Alexander Duyckec47ea82012-05-04 14:26:56 +00003773 hsize = skb_end_offset(nskb);
Herbert Xu89319d382008-12-15 23:26:06 -08003774 if (skb_cow_head(nskb, doffset + headroom)) {
3775 kfree_skb(nskb);
3776 goto err;
3777 }
3778
Alexander Duyckec47ea82012-05-04 14:26:56 +00003779 nskb->truesize += skb_end_offset(nskb) - hsize;
Herbert Xu89319d382008-12-15 23:26:06 -08003780 skb_release_head_state(nskb);
3781 __skb_push(nskb, doffset);
3782 } else {
Mel Gormanc93bdd02012-07-31 16:44:19 -07003783 nskb = __alloc_skb(hsize + doffset + headroom,
Michael S. Tsirkindf5771f2014-03-10 18:29:19 +02003784 GFP_ATOMIC, skb_alloc_rx_flag(head_skb),
Mel Gormanc93bdd02012-07-31 16:44:19 -07003785 NUMA_NO_NODE);
Herbert Xu89319d382008-12-15 23:26:06 -08003786
3787 if (unlikely(!nskb))
3788 goto err;
3789
3790 skb_reserve(nskb, headroom);
3791 __skb_put(nskb, doffset);
3792 }
Herbert Xuf4c50d92006-06-22 03:02:40 -07003793
3794 if (segs)
3795 tail->next = nskb;
3796 else
3797 segs = nskb;
3798 tail = nskb;
3799
Michael S. Tsirkindf5771f2014-03-10 18:29:19 +02003800 __copy_skb_header(nskb, head_skb);
Herbert Xuf4c50d92006-06-22 03:02:40 -07003801
Eric Dumazet030737b2013-10-19 11:42:54 -07003802 skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom);
Vlad Yasevichfcdfe3a2014-07-31 10:33:06 -04003803 skb_reset_mac_len(nskb);
Pravin B Shelar68c33162013-02-14 14:02:41 +00003804
Michael S. Tsirkindf5771f2014-03-10 18:29:19 +02003805 skb_copy_from_linear_data_offset(head_skb, -tnl_hlen,
Pravin B Shelar68c33162013-02-14 14:02:41 +00003806 nskb->data - tnl_hlen,
3807 doffset + tnl_hlen);
Herbert Xu89319d382008-12-15 23:26:06 -08003808
Herbert Xu9d8506c2013-11-21 11:10:04 -08003809 if (nskb->len == len + doffset)
Simon Horman1cdbcb72013-05-19 15:46:49 +00003810 goto perform_csum_check;
Herbert Xu89319d382008-12-15 23:26:06 -08003811
Alexander Duyck7fbeffe2016-02-05 15:27:43 -08003812 if (!sg) {
3813 if (!nskb->remcsum_offload)
3814 nskb->ip_summed = CHECKSUM_NONE;
Alexander Duyck76443452016-02-05 15:27:37 -08003815 SKB_GSO_CB(nskb)->csum =
3816 skb_copy_and_csum_bits(head_skb, offset,
3817 skb_put(nskb, len),
3818 len, 0);
Tom Herbert7e2b10c2014-06-04 17:20:02 -07003819 SKB_GSO_CB(nskb)->csum_start =
Alexander Duyck76443452016-02-05 15:27:37 -08003820 skb_headroom(nskb) + doffset;
Herbert Xuf4c50d92006-06-22 03:02:40 -07003821 continue;
3822 }
3823
Michael S. Tsirkin8cb19902014-03-10 18:29:04 +02003824 nskb_frag = skb_shinfo(nskb)->frags;
Herbert Xuf4c50d92006-06-22 03:02:40 -07003825
Michael S. Tsirkindf5771f2014-03-10 18:29:19 +02003826 skb_copy_from_linear_data_offset(head_skb, offset,
Arnaldo Carvalho de Melod626f622007-03-27 18:55:52 -03003827 skb_put(nskb, hsize), hsize);
Herbert Xuf4c50d92006-06-22 03:02:40 -07003828
Willem de Bruijnfff88032017-06-08 11:35:03 -04003829 skb_shinfo(nskb)->tx_flags |= skb_shinfo(head_skb)->tx_flags &
3830 SKBTX_SHARED_FRAG;
Eric Dumazetcef401d2013-01-25 20:34:37 +00003831
Willem de Bruijnbf5c25d2017-12-22 19:00:17 -05003832 if (skb_orphan_frags(frag_skb, GFP_ATOMIC) ||
3833 skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC))
3834 goto err;
3835
Herbert Xu9d8506c2013-11-21 11:10:04 -08003836 while (pos < offset + len) {
3837 if (i >= nfrags) {
Herbert Xu9d8506c2013-11-21 11:10:04 -08003838 i = 0;
Michael S. Tsirkin1a4ceda2014-03-10 19:27:59 +02003839 nfrags = skb_shinfo(list_skb)->nr_frags;
3840 frag = skb_shinfo(list_skb)->frags;
Michael S. Tsirkin1fd819e2014-03-10 19:28:08 +02003841 frag_skb = list_skb;
Yonghong Song13acc942018-03-21 16:31:03 -07003842 if (!skb_headlen(list_skb)) {
3843 BUG_ON(!nfrags);
3844 } else {
3845 BUG_ON(!list_skb->head_frag);
Herbert Xu9d8506c2013-11-21 11:10:04 -08003846
Yonghong Song13acc942018-03-21 16:31:03 -07003847 /* to make room for head_frag. */
3848 i--;
3849 frag--;
3850 }
Willem de Bruijnbf5c25d2017-12-22 19:00:17 -05003851 if (skb_orphan_frags(frag_skb, GFP_ATOMIC) ||
3852 skb_zerocopy_clone(nskb, frag_skb,
3853 GFP_ATOMIC))
3854 goto err;
3855
Michael S. Tsirkin1a4ceda2014-03-10 19:27:59 +02003856 list_skb = list_skb->next;
Herbert Xu9d8506c2013-11-21 11:10:04 -08003857 }
3858
3859 if (unlikely(skb_shinfo(nskb)->nr_frags >=
3860 MAX_SKB_FRAGS)) {
3861 net_warn_ratelimited(
3862 "skb_segment: too many frags: %u %u\n",
3863 pos, mss);
Eric Dumazetff907a12018-07-19 16:04:38 -07003864 err = -EINVAL;
Herbert Xu9d8506c2013-11-21 11:10:04 -08003865 goto err;
3866 }
3867
Yonghong Song13acc942018-03-21 16:31:03 -07003868 *nskb_frag = (i < 0) ? skb_head_frag_to_page_desc(frag_skb) : *frag;
Michael S. Tsirkin8cb19902014-03-10 18:29:04 +02003869 __skb_frag_ref(nskb_frag);
3870 size = skb_frag_size(nskb_frag);
Herbert Xuf4c50d92006-06-22 03:02:40 -07003871
3872 if (pos < offset) {
Michael S. Tsirkin8cb19902014-03-10 18:29:04 +02003873 nskb_frag->page_offset += offset - pos;
3874 skb_frag_size_sub(nskb_frag, offset - pos);
Herbert Xuf4c50d92006-06-22 03:02:40 -07003875 }
3876
Herbert Xu89319d382008-12-15 23:26:06 -08003877 skb_shinfo(nskb)->nr_frags++;
Herbert Xuf4c50d92006-06-22 03:02:40 -07003878
3879 if (pos + size <= offset + len) {
3880 i++;
Michael S. Tsirkin4e1beba2014-03-10 18:29:14 +02003881 frag++;
Herbert Xuf4c50d92006-06-22 03:02:40 -07003882 pos += size;
3883 } else {
Michael S. Tsirkin8cb19902014-03-10 18:29:04 +02003884 skb_frag_size_sub(nskb_frag, pos + size - (offset + len));
Herbert Xu89319d382008-12-15 23:26:06 -08003885 goto skip_fraglist;
Herbert Xuf4c50d92006-06-22 03:02:40 -07003886 }
3887
Michael S. Tsirkin8cb19902014-03-10 18:29:04 +02003888 nskb_frag++;
Herbert Xuf4c50d92006-06-22 03:02:40 -07003889 }
3890
Herbert Xu89319d382008-12-15 23:26:06 -08003891skip_fraglist:
Herbert Xuf4c50d92006-06-22 03:02:40 -07003892 nskb->data_len = len - hsize;
3893 nskb->len += nskb->data_len;
3894 nskb->truesize += nskb->data_len;
Pravin B Shelarec5f0612013-03-07 09:28:01 +00003895
Simon Horman1cdbcb72013-05-19 15:46:49 +00003896perform_csum_check:
Alexander Duyck7fbeffe2016-02-05 15:27:43 -08003897 if (!csum) {
Eric Dumazetff907a12018-07-19 16:04:38 -07003898 if (skb_has_shared_frag(nskb) &&
3899 __skb_linearize(nskb))
3900 goto err;
3901
Alexander Duyck7fbeffe2016-02-05 15:27:43 -08003902 if (!nskb->remcsum_offload)
3903 nskb->ip_summed = CHECKSUM_NONE;
Alexander Duyck76443452016-02-05 15:27:37 -08003904 SKB_GSO_CB(nskb)->csum =
3905 skb_checksum(nskb, doffset,
3906 nskb->len - doffset, 0);
Tom Herbert7e2b10c2014-06-04 17:20:02 -07003907 SKB_GSO_CB(nskb)->csum_start =
Alexander Duyck76443452016-02-05 15:27:37 -08003908 skb_headroom(nskb) + doffset;
Pravin B Shelarec5f0612013-03-07 09:28:01 +00003909 }
Michael S. Tsirkindf5771f2014-03-10 18:29:19 +02003910 } while ((offset += len) < head_skb->len);
Herbert Xuf4c50d92006-06-22 03:02:40 -07003911
Eric Dumazetbec3cfd2014-10-03 20:59:19 -07003912 /* Some callers want to get the end of the list.
3913 * Put it in segs->prev to avoid walking the list.
3914 * (see validate_xmit_skb_list() for example)
3915 */
3916 segs->prev = tail;
Toshiaki Makita432c8562014-10-27 10:30:51 -07003917
Alexander Duyck802ab552016-04-10 21:45:03 -04003918 if (partial_segs) {
Steffen Klassert07b26c92016-09-19 12:58:47 +02003919 struct sk_buff *iter;
Alexander Duyck802ab552016-04-10 21:45:03 -04003920 int type = skb_shinfo(head_skb)->gso_type;
Steffen Klassert07b26c92016-09-19 12:58:47 +02003921 unsigned short gso_size = skb_shinfo(head_skb)->gso_size;
Alexander Duyck802ab552016-04-10 21:45:03 -04003922
3923 /* Update type to add partial and then remove dodgy if set */
Steffen Klassert07b26c92016-09-19 12:58:47 +02003924 type |= (features & NETIF_F_GSO_PARTIAL) / NETIF_F_GSO_PARTIAL * SKB_GSO_PARTIAL;
Alexander Duyck802ab552016-04-10 21:45:03 -04003925 type &= ~SKB_GSO_DODGY;
3926
3927 /* Update GSO info and prepare to start updating headers on
3928 * our way back down the stack of protocols.
3929 */
Steffen Klassert07b26c92016-09-19 12:58:47 +02003930 for (iter = segs; iter; iter = iter->next) {
3931 skb_shinfo(iter)->gso_size = gso_size;
3932 skb_shinfo(iter)->gso_segs = partial_segs;
3933 skb_shinfo(iter)->gso_type = type;
3934 SKB_GSO_CB(iter)->data_offset = skb_headroom(iter) + doffset;
3935 }
3936
3937 if (tail->len - doffset <= gso_size)
3938 skb_shinfo(tail)->gso_size = 0;
3939 else if (tail != segs)
3940 skb_shinfo(tail)->gso_segs = DIV_ROUND_UP(tail->len - doffset, gso_size);
Alexander Duyck802ab552016-04-10 21:45:03 -04003941 }
3942
Toshiaki Makita432c8562014-10-27 10:30:51 -07003943 /* Following permits correct backpressure, for protocols
3944 * using skb_set_owner_w().
3945 * Idea is to tranfert ownership from head_skb to last segment.
3946 */
3947 if (head_skb->destructor == sock_wfree) {
3948 swap(tail->truesize, head_skb->truesize);
3949 swap(tail->destructor, head_skb->destructor);
3950 swap(tail->sk, head_skb->sk);
3951 }
Herbert Xuf4c50d92006-06-22 03:02:40 -07003952 return segs;
3953
3954err:
Eric Dumazet289dccb2013-12-20 14:29:08 -08003955 kfree_skb_list(segs);
Herbert Xuf4c50d92006-06-22 03:02:40 -07003956 return ERR_PTR(err);
3957}
Herbert Xuf4c50d92006-06-22 03:02:40 -07003958EXPORT_SYMBOL_GPL(skb_segment);
3959
David Millerd4546c22018-06-24 14:13:49 +09003960int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb)
Herbert Xu71d93b32008-12-15 23:42:33 -08003961{
Eric Dumazet8a291112013-10-08 09:02:23 -07003962 struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb);
Herbert Xu67147ba2009-05-26 18:50:22 +00003963 unsigned int offset = skb_gro_offset(skb);
3964 unsigned int headlen = skb_headlen(skb);
Eric Dumazet8a291112013-10-08 09:02:23 -07003965 unsigned int len = skb_gro_len(skb);
Eric Dumazet715dc1f2012-05-02 23:33:21 +00003966 unsigned int delta_truesize;
David Millerd4546c22018-06-24 14:13:49 +09003967 struct sk_buff *lp;
Herbert Xu71d93b32008-12-15 23:42:33 -08003968
Steffen Klassert0ab03f32019-04-02 08:16:03 +02003969 if (unlikely(p->len + len >= 65536 || NAPI_GRO_CB(skb)->flush))
Herbert Xu71d93b32008-12-15 23:42:33 -08003970 return -E2BIG;
3971
Eric Dumazet29e98242014-05-16 11:34:37 -07003972 lp = NAPI_GRO_CB(p)->last;
Eric Dumazet8a291112013-10-08 09:02:23 -07003973 pinfo = skb_shinfo(lp);
3974
3975 if (headlen <= offset) {
Herbert Xu42da6992009-05-26 18:50:19 +00003976 skb_frag_t *frag;
Herbert Xu66e92fc2009-05-26 18:50:32 +00003977 skb_frag_t *frag2;
Herbert Xu9aaa1562009-05-26 18:50:33 +00003978 int i = skbinfo->nr_frags;
3979 int nr_frags = pinfo->nr_frags + i;
Herbert Xu42da6992009-05-26 18:50:19 +00003980
Herbert Xu66e92fc2009-05-26 18:50:32 +00003981 if (nr_frags > MAX_SKB_FRAGS)
Eric Dumazet8a291112013-10-08 09:02:23 -07003982 goto merge;
Herbert Xu81705ad2009-01-29 14:19:51 +00003983
Eric Dumazet8a291112013-10-08 09:02:23 -07003984 offset -= headlen;
Herbert Xu9aaa1562009-05-26 18:50:33 +00003985 pinfo->nr_frags = nr_frags;
3986 skbinfo->nr_frags = 0;
Herbert Xuf5572062009-01-14 20:40:03 -08003987
Herbert Xu9aaa1562009-05-26 18:50:33 +00003988 frag = pinfo->frags + nr_frags;
3989 frag2 = skbinfo->frags + i;
Herbert Xu66e92fc2009-05-26 18:50:32 +00003990 do {
3991 *--frag = *--frag2;
3992 } while (--i);
3993
3994 frag->page_offset += offset;
Eric Dumazet9e903e02011-10-18 21:00:24 +00003995 skb_frag_size_sub(frag, offset);
Herbert Xu66e92fc2009-05-26 18:50:32 +00003996
Eric Dumazet715dc1f2012-05-02 23:33:21 +00003997 /* all fragments truesize : remove (head size + sk_buff) */
Alexander Duyckec47ea82012-05-04 14:26:56 +00003998 delta_truesize = skb->truesize -
3999 SKB_TRUESIZE(skb_end_offset(skb));
Eric Dumazet715dc1f2012-05-02 23:33:21 +00004000
Herbert Xuf5572062009-01-14 20:40:03 -08004001 skb->truesize -= skb->data_len;
4002 skb->len -= skb->data_len;
4003 skb->data_len = 0;
4004
Eric Dumazet715dc1f2012-05-02 23:33:21 +00004005 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE;
Herbert Xu5d38a072009-01-04 16:13:40 -08004006 goto done;
Eric Dumazetd7e88832012-04-30 08:10:34 +00004007 } else if (skb->head_frag) {
4008 int nr_frags = pinfo->nr_frags;
4009 skb_frag_t *frag = pinfo->frags + nr_frags;
4010 struct page *page = virt_to_head_page(skb->head);
4011 unsigned int first_size = headlen - offset;
4012 unsigned int first_offset;
4013
4014 if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS)
Eric Dumazet8a291112013-10-08 09:02:23 -07004015 goto merge;
Eric Dumazetd7e88832012-04-30 08:10:34 +00004016
4017 first_offset = skb->data -
4018 (unsigned char *)page_address(page) +
4019 offset;
4020
4021 pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags;
4022
4023 frag->page.p = page;
4024 frag->page_offset = first_offset;
4025 skb_frag_size_set(frag, first_size);
4026
4027 memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags);
4028 /* We dont need to clear skbinfo->nr_frags here */
4029
Eric Dumazet715dc1f2012-05-02 23:33:21 +00004030 delta_truesize = skb->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
Eric Dumazetd7e88832012-04-30 08:10:34 +00004031 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD;
4032 goto done;
Eric Dumazet8a291112013-10-08 09:02:23 -07004033 }
Herbert Xu71d93b32008-12-15 23:42:33 -08004034
4035merge:
Eric Dumazet715dc1f2012-05-02 23:33:21 +00004036 delta_truesize = skb->truesize;
Herbert Xu67147ba2009-05-26 18:50:22 +00004037 if (offset > headlen) {
Michal Schmidtd1dc7ab2011-01-24 12:08:48 +00004038 unsigned int eat = offset - headlen;
4039
4040 skbinfo->frags[0].page_offset += eat;
Eric Dumazet9e903e02011-10-18 21:00:24 +00004041 skb_frag_size_sub(&skbinfo->frags[0], eat);
Michal Schmidtd1dc7ab2011-01-24 12:08:48 +00004042 skb->data_len -= eat;
4043 skb->len -= eat;
Herbert Xu67147ba2009-05-26 18:50:22 +00004044 offset = headlen;
Herbert Xu56035022009-02-05 21:26:52 -08004045 }
4046
Herbert Xu67147ba2009-05-26 18:50:22 +00004047 __skb_pull(skb, offset);
Herbert Xu56035022009-02-05 21:26:52 -08004048
Eric Dumazet29e98242014-05-16 11:34:37 -07004049 if (NAPI_GRO_CB(p)->last == p)
Eric Dumazet8a291112013-10-08 09:02:23 -07004050 skb_shinfo(p)->frag_list = skb;
4051 else
4052 NAPI_GRO_CB(p)->last->next = skb;
Eric Dumazetc3c7c252012-12-06 13:54:59 +00004053 NAPI_GRO_CB(p)->last = skb;
Eric Dumazetf4a775d2014-09-22 16:29:32 -07004054 __skb_header_release(skb);
Eric Dumazet8a291112013-10-08 09:02:23 -07004055 lp = p;
Herbert Xu71d93b32008-12-15 23:42:33 -08004056
Herbert Xu5d38a072009-01-04 16:13:40 -08004057done:
4058 NAPI_GRO_CB(p)->count++;
Herbert Xu37fe4732009-01-17 19:48:13 +00004059 p->data_len += len;
Eric Dumazet715dc1f2012-05-02 23:33:21 +00004060 p->truesize += delta_truesize;
Herbert Xu37fe4732009-01-17 19:48:13 +00004061 p->len += len;
Eric Dumazet8a291112013-10-08 09:02:23 -07004062 if (lp != p) {
4063 lp->data_len += len;
4064 lp->truesize += delta_truesize;
4065 lp->len += len;
4066 }
Herbert Xu71d93b32008-12-15 23:42:33 -08004067 NAPI_GRO_CB(skb)->same_flow = 1;
4068 return 0;
4069}
Marcelo Ricardo Leitner57c056502016-06-02 15:05:39 -03004070EXPORT_SYMBOL_GPL(skb_gro_receive);
Herbert Xu71d93b32008-12-15 23:42:33 -08004071
Florian Westphaldf5042f2018-12-18 17:15:16 +01004072#ifdef CONFIG_SKB_EXTENSIONS
4073#define SKB_EXT_ALIGN_VALUE 8
4074#define SKB_EXT_CHUNKSIZEOF(x) (ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE)
4075
4076static const u8 skb_ext_type_len[] = {
4077#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
4078 [SKB_EXT_BRIDGE_NF] = SKB_EXT_CHUNKSIZEOF(struct nf_bridge_info),
4079#endif
Florian Westphal41650792018-12-18 17:15:27 +01004080#ifdef CONFIG_XFRM
4081 [SKB_EXT_SEC_PATH] = SKB_EXT_CHUNKSIZEOF(struct sec_path),
4082#endif
Florian Westphaldf5042f2018-12-18 17:15:16 +01004083};
4084
4085static __always_inline unsigned int skb_ext_total_length(void)
4086{
4087 return SKB_EXT_CHUNKSIZEOF(struct skb_ext) +
4088#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
4089 skb_ext_type_len[SKB_EXT_BRIDGE_NF] +
4090#endif
Florian Westphal41650792018-12-18 17:15:27 +01004091#ifdef CONFIG_XFRM
4092 skb_ext_type_len[SKB_EXT_SEC_PATH] +
4093#endif
Florian Westphaldf5042f2018-12-18 17:15:16 +01004094 0;
4095}
4096
4097static void skb_extensions_init(void)
4098{
4099 BUILD_BUG_ON(SKB_EXT_NUM >= 8);
4100 BUILD_BUG_ON(skb_ext_total_length() > 255);
4101
4102 skbuff_ext_cache = kmem_cache_create("skbuff_ext_cache",
4103 SKB_EXT_ALIGN_VALUE * skb_ext_total_length(),
4104 0,
4105 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
4106 NULL);
4107}
4108#else
4109static void skb_extensions_init(void) {}
4110#endif
4111
Linus Torvalds1da177e2005-04-16 15:20:36 -07004112void __init skb_init(void)
4113{
Kees Cook79a8a642018-02-07 17:44:38 -08004114 skbuff_head_cache = kmem_cache_create_usercopy("skbuff_head_cache",
Linus Torvalds1da177e2005-04-16 15:20:36 -07004115 sizeof(struct sk_buff),
4116 0,
Alexey Dobriyane5d679f332006-08-26 19:25:52 -07004117 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
Kees Cook79a8a642018-02-07 17:44:38 -08004118 offsetof(struct sk_buff, cb),
4119 sizeof_field(struct sk_buff, cb),
Paul Mundt20c2df82007-07-20 10:11:58 +09004120 NULL);
David S. Millerd179cd12005-08-17 14:57:30 -07004121 skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
Eric Dumazetd0bf4a92014-09-29 13:29:15 -07004122 sizeof(struct sk_buff_fclones),
David S. Millerd179cd12005-08-17 14:57:30 -07004123 0,
Alexey Dobriyane5d679f332006-08-26 19:25:52 -07004124 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
Paul Mundt20c2df82007-07-20 10:11:58 +09004125 NULL);
Florian Westphaldf5042f2018-12-18 17:15:16 +01004126 skb_extensions_init();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004127}
4128
David S. Miller51c739d2007-10-30 21:29:29 -07004129static int
Jason A. Donenfeld48a1df62017-06-04 04:16:22 +02004130__skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len,
4131 unsigned int recursion_level)
David Howells716ea3a2007-04-02 20:19:53 -07004132{
David S. Miller1a028e52007-04-27 15:21:23 -07004133 int start = skb_headlen(skb);
4134 int i, copy = start - offset;
David S. Millerfbb398a2009-06-09 00:18:59 -07004135 struct sk_buff *frag_iter;
David Howells716ea3a2007-04-02 20:19:53 -07004136 int elt = 0;
4137
Jason A. Donenfeld48a1df62017-06-04 04:16:22 +02004138 if (unlikely(recursion_level >= 24))
4139 return -EMSGSIZE;
4140
David Howells716ea3a2007-04-02 20:19:53 -07004141 if (copy > 0) {
4142 if (copy > len)
4143 copy = len;
Jens Axboe642f149032007-10-24 11:20:47 +02004144 sg_set_buf(sg, skb->data + offset, copy);
David Howells716ea3a2007-04-02 20:19:53 -07004145 elt++;
4146 if ((len -= copy) == 0)
4147 return elt;
4148 offset += copy;
4149 }
4150
4151 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
David S. Miller1a028e52007-04-27 15:21:23 -07004152 int end;
David Howells716ea3a2007-04-02 20:19:53 -07004153
Ilpo Järvinen547b7922008-07-25 21:43:18 -07004154 WARN_ON(start > offset + len);
David S. Miller1a028e52007-04-27 15:21:23 -07004155
Eric Dumazet9e903e02011-10-18 21:00:24 +00004156 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
David Howells716ea3a2007-04-02 20:19:53 -07004157 if ((copy = end - offset) > 0) {
4158 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
Jason A. Donenfeld48a1df62017-06-04 04:16:22 +02004159 if (unlikely(elt && sg_is_last(&sg[elt - 1])))
4160 return -EMSGSIZE;
David Howells716ea3a2007-04-02 20:19:53 -07004161
4162 if (copy > len)
4163 copy = len;
Ian Campbellea2ab692011-08-22 23:44:58 +00004164 sg_set_page(&sg[elt], skb_frag_page(frag), copy,
Jens Axboe642f149032007-10-24 11:20:47 +02004165 frag->page_offset+offset-start);
David Howells716ea3a2007-04-02 20:19:53 -07004166 elt++;
4167 if (!(len -= copy))
4168 return elt;
4169 offset += copy;
4170 }
David S. Miller1a028e52007-04-27 15:21:23 -07004171 start = end;
David Howells716ea3a2007-04-02 20:19:53 -07004172 }
4173
David S. Millerfbb398a2009-06-09 00:18:59 -07004174 skb_walk_frags(skb, frag_iter) {
Jason A. Donenfeld48a1df62017-06-04 04:16:22 +02004175 int end, ret;
David Howells716ea3a2007-04-02 20:19:53 -07004176
David S. Millerfbb398a2009-06-09 00:18:59 -07004177 WARN_ON(start > offset + len);
David Howells716ea3a2007-04-02 20:19:53 -07004178
David S. Millerfbb398a2009-06-09 00:18:59 -07004179 end = start + frag_iter->len;
4180 if ((copy = end - offset) > 0) {
Jason A. Donenfeld48a1df62017-06-04 04:16:22 +02004181 if (unlikely(elt && sg_is_last(&sg[elt - 1])))
4182 return -EMSGSIZE;
4183
David S. Millerfbb398a2009-06-09 00:18:59 -07004184 if (copy > len)
4185 copy = len;
Jason A. Donenfeld48a1df62017-06-04 04:16:22 +02004186 ret = __skb_to_sgvec(frag_iter, sg+elt, offset - start,
4187 copy, recursion_level + 1);
4188 if (unlikely(ret < 0))
4189 return ret;
4190 elt += ret;
David S. Millerfbb398a2009-06-09 00:18:59 -07004191 if ((len -= copy) == 0)
4192 return elt;
4193 offset += copy;
David Howells716ea3a2007-04-02 20:19:53 -07004194 }
David S. Millerfbb398a2009-06-09 00:18:59 -07004195 start = end;
David Howells716ea3a2007-04-02 20:19:53 -07004196 }
4197 BUG_ON(len);
4198 return elt;
4199}
4200
Jason A. Donenfeld48a1df62017-06-04 04:16:22 +02004201/**
4202 * skb_to_sgvec - Fill a scatter-gather list from a socket buffer
4203 * @skb: Socket buffer containing the buffers to be mapped
4204 * @sg: The scatter-gather list to map into
4205 * @offset: The offset into the buffer's contents to start mapping
4206 * @len: Length of buffer space to be mapped
4207 *
4208 * Fill the specified scatter-gather list with mappings/pointers into a
4209 * region of the buffer space attached to a socket buffer. Returns either
4210 * the number of scatterlist items used, or -EMSGSIZE if the contents
4211 * could not fit.
4212 */
4213int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
4214{
4215 int nsg = __skb_to_sgvec(skb, sg, offset, len, 0);
4216
4217 if (nsg <= 0)
4218 return nsg;
4219
4220 sg_mark_end(&sg[nsg - 1]);
4221
4222 return nsg;
4223}
4224EXPORT_SYMBOL_GPL(skb_to_sgvec);
4225
Fan Du25a91d82014-01-18 09:54:23 +08004226/* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given
4227 * sglist without mark the sg which contain last skb data as the end.
4228 * So the caller can mannipulate sg list as will when padding new data after
4229 * the first call without calling sg_unmark_end to expend sg list.
4230 *
4231 * Scenario to use skb_to_sgvec_nomark:
4232 * 1. sg_init_table
4233 * 2. skb_to_sgvec_nomark(payload1)
4234 * 3. skb_to_sgvec_nomark(payload2)
4235 *
4236 * This is equivalent to:
4237 * 1. sg_init_table
4238 * 2. skb_to_sgvec(payload1)
4239 * 3. sg_unmark_end
4240 * 4. skb_to_sgvec(payload2)
4241 *
4242 * When mapping mutilple payload conditionally, skb_to_sgvec_nomark
4243 * is more preferable.
4244 */
4245int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
4246 int offset, int len)
4247{
Jason A. Donenfeld48a1df62017-06-04 04:16:22 +02004248 return __skb_to_sgvec(skb, sg, offset, len, 0);
Fan Du25a91d82014-01-18 09:54:23 +08004249}
4250EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark);
4251
David S. Miller51c739d2007-10-30 21:29:29 -07004252
David S. Miller51c739d2007-10-30 21:29:29 -07004253
David Howells716ea3a2007-04-02 20:19:53 -07004254/**
4255 * skb_cow_data - Check that a socket buffer's data buffers are writable
4256 * @skb: The socket buffer to check.
4257 * @tailbits: Amount of trailing space to be added
4258 * @trailer: Returned pointer to the skb where the @tailbits space begins
4259 *
4260 * Make sure that the data buffers attached to a socket buffer are
4261 * writable. If they are not, private copies are made of the data buffers
4262 * and the socket buffer is set to use these instead.
4263 *
4264 * If @tailbits is given, make sure that there is space to write @tailbits
4265 * bytes of data beyond current end of socket buffer. @trailer will be
4266 * set to point to the skb in which this space begins.
4267 *
4268 * The number of scatterlist elements required to completely map the
4269 * COW'd and extended socket buffer will be returned.
4270 */
4271int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
4272{
4273 int copyflag;
4274 int elt;
4275 struct sk_buff *skb1, **skb_p;
4276
4277 /* If skb is cloned or its head is paged, reallocate
4278 * head pulling out all the pages (pages are considered not writable
4279 * at the moment even if they are anonymous).
4280 */
4281 if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) &&
4282 __pskb_pull_tail(skb, skb_pagelen(skb)-skb_headlen(skb)) == NULL)
4283 return -ENOMEM;
4284
4285 /* Easy case. Most of packets will go this way. */
David S. Miller21dc3302010-08-23 00:13:46 -07004286 if (!skb_has_frag_list(skb)) {
David Howells716ea3a2007-04-02 20:19:53 -07004287 /* A little of trouble, not enough of space for trailer.
4288 * This should not happen, when stack is tuned to generate
4289 * good frames. OK, on miss we reallocate and reserve even more
4290 * space, 128 bytes is fair. */
4291
4292 if (skb_tailroom(skb) < tailbits &&
4293 pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC))
4294 return -ENOMEM;
4295
4296 /* Voila! */
4297 *trailer = skb;
4298 return 1;
4299 }
4300
4301 /* Misery. We are in troubles, going to mincer fragments... */
4302
4303 elt = 1;
4304 skb_p = &skb_shinfo(skb)->frag_list;
4305 copyflag = 0;
4306
4307 while ((skb1 = *skb_p) != NULL) {
4308 int ntail = 0;
4309
4310 /* The fragment is partially pulled by someone,
4311 * this can happen on input. Copy it and everything
4312 * after it. */
4313
4314 if (skb_shared(skb1))
4315 copyflag = 1;
4316
4317 /* If the skb is the last, worry about trailer. */
4318
4319 if (skb1->next == NULL && tailbits) {
4320 if (skb_shinfo(skb1)->nr_frags ||
David S. Miller21dc3302010-08-23 00:13:46 -07004321 skb_has_frag_list(skb1) ||
David Howells716ea3a2007-04-02 20:19:53 -07004322 skb_tailroom(skb1) < tailbits)
4323 ntail = tailbits + 128;
4324 }
4325
4326 if (copyflag ||
4327 skb_cloned(skb1) ||
4328 ntail ||
4329 skb_shinfo(skb1)->nr_frags ||
David S. Miller21dc3302010-08-23 00:13:46 -07004330 skb_has_frag_list(skb1)) {
David Howells716ea3a2007-04-02 20:19:53 -07004331 struct sk_buff *skb2;
4332
4333 /* Fuck, we are miserable poor guys... */
4334 if (ntail == 0)
4335 skb2 = skb_copy(skb1, GFP_ATOMIC);
4336 else
4337 skb2 = skb_copy_expand(skb1,
4338 skb_headroom(skb1),
4339 ntail,
4340 GFP_ATOMIC);
4341 if (unlikely(skb2 == NULL))
4342 return -ENOMEM;
4343
4344 if (skb1->sk)
4345 skb_set_owner_w(skb2, skb1->sk);
4346
4347 /* Looking around. Are we still alive?
4348 * OK, link new skb, drop old one */
4349
4350 skb2->next = skb1->next;
4351 *skb_p = skb2;
4352 kfree_skb(skb1);
4353 skb1 = skb2;
4354 }
4355 elt++;
4356 *trailer = skb1;
4357 skb_p = &skb1->next;
4358 }
4359
4360 return elt;
4361}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08004362EXPORT_SYMBOL_GPL(skb_cow_data);
David Howells716ea3a2007-04-02 20:19:53 -07004363
Eric Dumazetb1faf562010-05-31 23:44:05 -07004364static void sock_rmem_free(struct sk_buff *skb)
4365{
4366 struct sock *sk = skb->sk;
4367
4368 atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
4369}
4370
Soheil Hassas Yeganeh8605330a2017-03-18 17:02:59 -04004371static void skb_set_err_queue(struct sk_buff *skb)
4372{
4373 /* pkt_type of skbs received on local sockets is never PACKET_OUTGOING.
4374 * So, it is safe to (mis)use it to mark skbs on the error queue.
4375 */
4376 skb->pkt_type = PACKET_OUTGOING;
4377 BUILD_BUG_ON(PACKET_OUTGOING == 0);
4378}
4379
Eric Dumazetb1faf562010-05-31 23:44:05 -07004380/*
4381 * Note: We dont mem charge error packets (no sk_forward_alloc changes)
4382 */
4383int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb)
4384{
4385 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
Eric Dumazet95c96172012-04-15 05:58:06 +00004386 (unsigned int)sk->sk_rcvbuf)
Eric Dumazetb1faf562010-05-31 23:44:05 -07004387 return -ENOMEM;
4388
4389 skb_orphan(skb);
4390 skb->sk = sk;
4391 skb->destructor = sock_rmem_free;
4392 atomic_add(skb->truesize, &sk->sk_rmem_alloc);
Soheil Hassas Yeganeh8605330a2017-03-18 17:02:59 -04004393 skb_set_err_queue(skb);
Eric Dumazetb1faf562010-05-31 23:44:05 -07004394
Eric Dumazetabb57ea2011-05-18 02:21:31 -04004395 /* before exiting rcu section, make sure dst is refcounted */
4396 skb_dst_force(skb);
4397
Eric Dumazetb1faf562010-05-31 23:44:05 -07004398 skb_queue_tail(&sk->sk_error_queue, skb);
4399 if (!sock_flag(sk, SOCK_DEAD))
Vinicius Costa Gomes6e5d58f2018-03-14 13:32:09 -07004400 sk->sk_error_report(sk);
Eric Dumazetb1faf562010-05-31 23:44:05 -07004401 return 0;
4402}
4403EXPORT_SYMBOL(sock_queue_err_skb);
4404
Soheil Hassas Yeganeh83a1a1a2016-11-30 14:01:08 -05004405static bool is_icmp_err_skb(const struct sk_buff *skb)
4406{
4407 return skb && (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP ||
4408 SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP6);
4409}
4410
Willem de Bruijn364a9e92014-08-31 21:30:27 -04004411struct sk_buff *sock_dequeue_err_skb(struct sock *sk)
4412{
4413 struct sk_buff_head *q = &sk->sk_error_queue;
Soheil Hassas Yeganeh83a1a1a2016-11-30 14:01:08 -05004414 struct sk_buff *skb, *skb_next = NULL;
4415 bool icmp_next = false;
Eric Dumazet997d5c32015-02-18 05:47:55 -08004416 unsigned long flags;
Willem de Bruijn364a9e92014-08-31 21:30:27 -04004417
Eric Dumazet997d5c32015-02-18 05:47:55 -08004418 spin_lock_irqsave(&q->lock, flags);
Willem de Bruijn364a9e92014-08-31 21:30:27 -04004419 skb = __skb_dequeue(q);
Soheil Hassas Yeganeh38b25792017-06-02 12:38:22 -04004420 if (skb && (skb_next = skb_peek(q))) {
Soheil Hassas Yeganeh83a1a1a2016-11-30 14:01:08 -05004421 icmp_next = is_icmp_err_skb(skb_next);
Soheil Hassas Yeganeh38b25792017-06-02 12:38:22 -04004422 if (icmp_next)
4423 sk->sk_err = SKB_EXT_ERR(skb_next)->ee.ee_origin;
4424 }
Eric Dumazet997d5c32015-02-18 05:47:55 -08004425 spin_unlock_irqrestore(&q->lock, flags);
Willem de Bruijn364a9e92014-08-31 21:30:27 -04004426
Soheil Hassas Yeganeh83a1a1a2016-11-30 14:01:08 -05004427 if (is_icmp_err_skb(skb) && !icmp_next)
4428 sk->sk_err = 0;
4429
4430 if (skb_next)
Willem de Bruijn364a9e92014-08-31 21:30:27 -04004431 sk->sk_error_report(sk);
4432
4433 return skb;
4434}
4435EXPORT_SYMBOL(sock_dequeue_err_skb);
4436
Alexander Duyckcab41c42014-09-10 18:05:26 -04004437/**
4438 * skb_clone_sk - create clone of skb, and take reference to socket
4439 * @skb: the skb to clone
4440 *
4441 * This function creates a clone of a buffer that holds a reference on
4442 * sk_refcnt. Buffers created via this function are meant to be
4443 * returned using sock_queue_err_skb, or free via kfree_skb.
4444 *
4445 * When passing buffers allocated with this function to sock_queue_err_skb
4446 * it is necessary to wrap the call with sock_hold/sock_put in order to
4447 * prevent the socket from being released prior to being enqueued on
4448 * the sk_error_queue.
4449 */
Alexander Duyck62bccb82014-09-04 13:31:35 -04004450struct sk_buff *skb_clone_sk(struct sk_buff *skb)
4451{
4452 struct sock *sk = skb->sk;
4453 struct sk_buff *clone;
4454
Reshetova, Elena41c6d652017-06-30 13:08:01 +03004455 if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt))
Alexander Duyck62bccb82014-09-04 13:31:35 -04004456 return NULL;
4457
4458 clone = skb_clone(skb, GFP_ATOMIC);
4459 if (!clone) {
4460 sock_put(sk);
4461 return NULL;
4462 }
4463
4464 clone->sk = sk;
4465 clone->destructor = sock_efree;
4466
4467 return clone;
4468}
4469EXPORT_SYMBOL(skb_clone_sk);
4470
Alexander Duyck37846ef2014-09-04 13:31:10 -04004471static void __skb_complete_tx_timestamp(struct sk_buff *skb,
4472 struct sock *sk,
Soheil Hassas Yeganeh4ef1b282017-03-18 17:03:00 -04004473 int tstype,
4474 bool opt_stats)
Patrick Ohlyac45f602009-02-12 05:03:37 +00004475{
Patrick Ohlyac45f602009-02-12 05:03:37 +00004476 struct sock_exterr_skb *serr;
Patrick Ohlyac45f602009-02-12 05:03:37 +00004477 int err;
4478
Soheil Hassas Yeganeh4ef1b282017-03-18 17:03:00 -04004479 BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb));
4480
Patrick Ohlyac45f602009-02-12 05:03:37 +00004481 serr = SKB_EXT_ERR(skb);
4482 memset(serr, 0, sizeof(*serr));
4483 serr->ee.ee_errno = ENOMSG;
4484 serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
Willem de Bruijne7fd2882014-08-04 22:11:48 -04004485 serr->ee.ee_info = tstype;
Soheil Hassas Yeganeh4ef1b282017-03-18 17:03:00 -04004486 serr->opt_stats = opt_stats;
Willem de Bruijn1862d622017-04-12 19:24:35 -04004487 serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0;
Willem de Bruijn4ed2d762014-08-04 22:11:49 -04004488 if (sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) {
Willem de Bruijn09c2d252014-08-04 22:11:47 -04004489 serr->ee.ee_data = skb_shinfo(skb)->tskey;
WANG Congac5cc972015-12-16 23:39:04 -08004490 if (sk->sk_protocol == IPPROTO_TCP &&
4491 sk->sk_type == SOCK_STREAM)
Willem de Bruijn4ed2d762014-08-04 22:11:49 -04004492 serr->ee.ee_data -= sk->sk_tskey;
4493 }
Eric Dumazet29030372010-05-29 00:20:48 -07004494
Patrick Ohlyac45f602009-02-12 05:03:37 +00004495 err = sock_queue_err_skb(sk, skb);
Eric Dumazet29030372010-05-29 00:20:48 -07004496
Patrick Ohlyac45f602009-02-12 05:03:37 +00004497 if (err)
4498 kfree_skb(skb);
4499}
Alexander Duyck37846ef2014-09-04 13:31:10 -04004500
Willem de Bruijnb245be12015-01-30 13:29:32 -05004501static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly)
4502{
4503 bool ret;
4504
4505 if (likely(sysctl_tstamp_allow_data || tsonly))
4506 return true;
4507
4508 read_lock_bh(&sk->sk_callback_lock);
4509 ret = sk->sk_socket && sk->sk_socket->file &&
4510 file_ns_capable(sk->sk_socket->file, &init_user_ns, CAP_NET_RAW);
4511 read_unlock_bh(&sk->sk_callback_lock);
4512 return ret;
4513}
4514
Alexander Duyck37846ef2014-09-04 13:31:10 -04004515void skb_complete_tx_timestamp(struct sk_buff *skb,
4516 struct skb_shared_hwtstamps *hwtstamps)
4517{
4518 struct sock *sk = skb->sk;
4519
Willem de Bruijnb245be12015-01-30 13:29:32 -05004520 if (!skb_may_tx_timestamp(sk, false))
Willem de Bruijn35b99df2017-12-13 14:41:06 -05004521 goto err;
Willem de Bruijnb245be12015-01-30 13:29:32 -05004522
Eric Dumazet9ac25fc2017-03-03 21:01:03 -08004523 /* Take a reference to prevent skb_orphan() from freeing the socket,
4524 * but only if the socket refcount is not zero.
4525 */
Reshetova, Elena41c6d652017-06-30 13:08:01 +03004526 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
Eric Dumazet9ac25fc2017-03-03 21:01:03 -08004527 *skb_hwtstamps(skb) = *hwtstamps;
Soheil Hassas Yeganeh4ef1b282017-03-18 17:03:00 -04004528 __skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND, false);
Eric Dumazet9ac25fc2017-03-03 21:01:03 -08004529 sock_put(sk);
Willem de Bruijn35b99df2017-12-13 14:41:06 -05004530 return;
Eric Dumazet9ac25fc2017-03-03 21:01:03 -08004531 }
Willem de Bruijn35b99df2017-12-13 14:41:06 -05004532
4533err:
4534 kfree_skb(skb);
Alexander Duyck37846ef2014-09-04 13:31:10 -04004535}
4536EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp);
4537
4538void __skb_tstamp_tx(struct sk_buff *orig_skb,
4539 struct skb_shared_hwtstamps *hwtstamps,
4540 struct sock *sk, int tstype)
4541{
4542 struct sk_buff *skb;
Soheil Hassas Yeganeh4ef1b282017-03-18 17:03:00 -04004543 bool tsonly, opt_stats = false;
Alexander Duyck37846ef2014-09-04 13:31:10 -04004544
Willem de Bruijn3a8dd972015-03-11 15:43:55 -04004545 if (!sk)
4546 return;
4547
Miroslav Lichvarb50a5c72017-05-19 17:52:40 +02004548 if (!hwtstamps && !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TX_SWHW) &&
4549 skb_shinfo(orig_skb)->tx_flags & SKBTX_IN_PROGRESS)
4550 return;
4551
Willem de Bruijn3a8dd972015-03-11 15:43:55 -04004552 tsonly = sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TSONLY;
4553 if (!skb_may_tx_timestamp(sk, tsonly))
Alexander Duyck37846ef2014-09-04 13:31:10 -04004554 return;
4555
Francis Yan1c885802016-11-27 23:07:18 -08004556 if (tsonly) {
4557#ifdef CONFIG_INET
4558 if ((sk->sk_tsflags & SOF_TIMESTAMPING_OPT_STATS) &&
4559 sk->sk_protocol == IPPROTO_TCP &&
Soheil Hassas Yeganeh4ef1b282017-03-18 17:03:00 -04004560 sk->sk_type == SOCK_STREAM) {
Francis Yan1c885802016-11-27 23:07:18 -08004561 skb = tcp_get_timestamping_opt_stats(sk);
Soheil Hassas Yeganeh4ef1b282017-03-18 17:03:00 -04004562 opt_stats = true;
4563 } else
Francis Yan1c885802016-11-27 23:07:18 -08004564#endif
4565 skb = alloc_skb(0, GFP_ATOMIC);
4566 } else {
Willem de Bruijn49ca0d82015-01-30 13:29:31 -05004567 skb = skb_clone(orig_skb, GFP_ATOMIC);
Francis Yan1c885802016-11-27 23:07:18 -08004568 }
Alexander Duyck37846ef2014-09-04 13:31:10 -04004569 if (!skb)
4570 return;
4571
Willem de Bruijn49ca0d82015-01-30 13:29:31 -05004572 if (tsonly) {
Willem de Bruijnfff88032017-06-08 11:35:03 -04004573 skb_shinfo(skb)->tx_flags |= skb_shinfo(orig_skb)->tx_flags &
4574 SKBTX_ANY_TSTAMP;
Willem de Bruijn49ca0d82015-01-30 13:29:31 -05004575 skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey;
4576 }
4577
4578 if (hwtstamps)
4579 *skb_hwtstamps(skb) = *hwtstamps;
4580 else
4581 skb->tstamp = ktime_get_real();
4582
Soheil Hassas Yeganeh4ef1b282017-03-18 17:03:00 -04004583 __skb_complete_tx_timestamp(skb, sk, tstype, opt_stats);
Alexander Duyck37846ef2014-09-04 13:31:10 -04004584}
Willem de Bruijne7fd2882014-08-04 22:11:48 -04004585EXPORT_SYMBOL_GPL(__skb_tstamp_tx);
4586
4587void skb_tstamp_tx(struct sk_buff *orig_skb,
4588 struct skb_shared_hwtstamps *hwtstamps)
4589{
4590 return __skb_tstamp_tx(orig_skb, hwtstamps, orig_skb->sk,
4591 SCM_TSTAMP_SND);
4592}
Patrick Ohlyac45f602009-02-12 05:03:37 +00004593EXPORT_SYMBOL_GPL(skb_tstamp_tx);
4594
Johannes Berg6e3e9392011-11-09 10:15:42 +01004595void skb_complete_wifi_ack(struct sk_buff *skb, bool acked)
4596{
4597 struct sock *sk = skb->sk;
4598 struct sock_exterr_skb *serr;
Eric Dumazetdd4f1072017-03-03 21:01:02 -08004599 int err = 1;
Johannes Berg6e3e9392011-11-09 10:15:42 +01004600
4601 skb->wifi_acked_valid = 1;
4602 skb->wifi_acked = acked;
4603
4604 serr = SKB_EXT_ERR(skb);
4605 memset(serr, 0, sizeof(*serr));
4606 serr->ee.ee_errno = ENOMSG;
4607 serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS;
4608
Eric Dumazetdd4f1072017-03-03 21:01:02 -08004609 /* Take a reference to prevent skb_orphan() from freeing the socket,
4610 * but only if the socket refcount is not zero.
4611 */
Reshetova, Elena41c6d652017-06-30 13:08:01 +03004612 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
Eric Dumazetdd4f1072017-03-03 21:01:02 -08004613 err = sock_queue_err_skb(sk, skb);
4614 sock_put(sk);
4615 }
Johannes Berg6e3e9392011-11-09 10:15:42 +01004616 if (err)
4617 kfree_skb(skb);
4618}
4619EXPORT_SYMBOL_GPL(skb_complete_wifi_ack);
4620
Rusty Russellf35d9d82008-02-04 23:49:54 -05004621/**
4622 * skb_partial_csum_set - set up and verify partial csum values for packet
4623 * @skb: the skb to set
4624 * @start: the number of bytes after skb->data to start checksumming.
4625 * @off: the offset from start to place the checksum.
4626 *
4627 * For untrusted partially-checksummed packets, we need to make sure the values
4628 * for skb->csum_start and skb->csum_offset are valid so we don't oops.
4629 *
4630 * This function checks and sets those values and skb->ip_summed: if this
4631 * returns false you should drop the packet.
4632 */
4633bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
4634{
Eric Dumazet52b5d6f2018-10-10 06:59:35 -07004635 u32 csum_end = (u32)start + (u32)off + sizeof(__sum16);
4636 u32 csum_start = skb_headroom(skb) + (u32)start;
4637
4638 if (unlikely(csum_start > U16_MAX || csum_end > skb_headlen(skb))) {
4639 net_warn_ratelimited("bad partial csum: csum=%u/%u headroom=%u headlen=%u\n",
4640 start, off, skb_headroom(skb), skb_headlen(skb));
Rusty Russellf35d9d82008-02-04 23:49:54 -05004641 return false;
4642 }
4643 skb->ip_summed = CHECKSUM_PARTIAL;
Eric Dumazet52b5d6f2018-10-10 06:59:35 -07004644 skb->csum_start = csum_start;
Rusty Russellf35d9d82008-02-04 23:49:54 -05004645 skb->csum_offset = off;
Jason Wange5d5dec2013-03-26 23:11:20 +00004646 skb_set_transport_header(skb, start);
Rusty Russellf35d9d82008-02-04 23:49:54 -05004647 return true;
4648}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08004649EXPORT_SYMBOL_GPL(skb_partial_csum_set);
Rusty Russellf35d9d82008-02-04 23:49:54 -05004650
Paul Durranted1f50c2014-01-09 10:02:46 +00004651static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len,
4652 unsigned int max)
4653{
4654 if (skb_headlen(skb) >= len)
4655 return 0;
4656
4657 /* If we need to pullup then pullup to the max, so we
4658 * won't need to do it again.
4659 */
4660 if (max > skb->len)
4661 max = skb->len;
4662
4663 if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL)
4664 return -ENOMEM;
4665
4666 if (skb_headlen(skb) < len)
4667 return -EPROTO;
4668
4669 return 0;
4670}
4671
Jan Beulichf9708b42014-03-11 13:56:05 +00004672#define MAX_TCP_HDR_LEN (15 * 4)
4673
4674static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb,
4675 typeof(IPPROTO_IP) proto,
4676 unsigned int off)
4677{
4678 switch (proto) {
4679 int err;
4680
4681 case IPPROTO_TCP:
4682 err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr),
4683 off + MAX_TCP_HDR_LEN);
4684 if (!err && !skb_partial_csum_set(skb, off,
4685 offsetof(struct tcphdr,
4686 check)))
4687 err = -EPROTO;
4688 return err ? ERR_PTR(err) : &tcp_hdr(skb)->check;
4689
4690 case IPPROTO_UDP:
4691 err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr),
4692 off + sizeof(struct udphdr));
4693 if (!err && !skb_partial_csum_set(skb, off,
4694 offsetof(struct udphdr,
4695 check)))
4696 err = -EPROTO;
4697 return err ? ERR_PTR(err) : &udp_hdr(skb)->check;
4698 }
4699
4700 return ERR_PTR(-EPROTO);
4701}
4702
Paul Durranted1f50c2014-01-09 10:02:46 +00004703/* This value should be large enough to cover a tagged ethernet header plus
4704 * maximally sized IP and TCP or UDP headers.
4705 */
4706#define MAX_IP_HDR_LEN 128
4707
Jan Beulichf9708b42014-03-11 13:56:05 +00004708static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate)
Paul Durranted1f50c2014-01-09 10:02:46 +00004709{
4710 unsigned int off;
4711 bool fragment;
Jan Beulichf9708b42014-03-11 13:56:05 +00004712 __sum16 *csum;
Paul Durranted1f50c2014-01-09 10:02:46 +00004713 int err;
4714
4715 fragment = false;
4716
4717 err = skb_maybe_pull_tail(skb,
4718 sizeof(struct iphdr),
4719 MAX_IP_HDR_LEN);
4720 if (err < 0)
4721 goto out;
4722
4723 if (ip_hdr(skb)->frag_off & htons(IP_OFFSET | IP_MF))
4724 fragment = true;
4725
4726 off = ip_hdrlen(skb);
4727
4728 err = -EPROTO;
4729
4730 if (fragment)
4731 goto out;
4732
Jan Beulichf9708b42014-03-11 13:56:05 +00004733 csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off);
4734 if (IS_ERR(csum))
4735 return PTR_ERR(csum);
Paul Durranted1f50c2014-01-09 10:02:46 +00004736
Jan Beulichf9708b42014-03-11 13:56:05 +00004737 if (recalculate)
4738 *csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
4739 ip_hdr(skb)->daddr,
4740 skb->len - off,
4741 ip_hdr(skb)->protocol, 0);
Paul Durranted1f50c2014-01-09 10:02:46 +00004742 err = 0;
4743
4744out:
4745 return err;
4746}
4747
4748/* This value should be large enough to cover a tagged ethernet header plus
4749 * an IPv6 header, all options, and a maximal TCP or UDP header.
4750 */
4751#define MAX_IPV6_HDR_LEN 256
4752
4753#define OPT_HDR(type, skb, off) \
4754 (type *)(skb_network_header(skb) + (off))
4755
4756static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate)
4757{
4758 int err;
4759 u8 nexthdr;
4760 unsigned int off;
4761 unsigned int len;
4762 bool fragment;
4763 bool done;
Jan Beulichf9708b42014-03-11 13:56:05 +00004764 __sum16 *csum;
Paul Durranted1f50c2014-01-09 10:02:46 +00004765
4766 fragment = false;
4767 done = false;
4768
4769 off = sizeof(struct ipv6hdr);
4770
4771 err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN);
4772 if (err < 0)
4773 goto out;
4774
4775 nexthdr = ipv6_hdr(skb)->nexthdr;
4776
4777 len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len);
4778 while (off <= len && !done) {
4779 switch (nexthdr) {
4780 case IPPROTO_DSTOPTS:
4781 case IPPROTO_HOPOPTS:
4782 case IPPROTO_ROUTING: {
4783 struct ipv6_opt_hdr *hp;
4784
4785 err = skb_maybe_pull_tail(skb,
4786 off +
4787 sizeof(struct ipv6_opt_hdr),
4788 MAX_IPV6_HDR_LEN);
4789 if (err < 0)
4790 goto out;
4791
4792 hp = OPT_HDR(struct ipv6_opt_hdr, skb, off);
4793 nexthdr = hp->nexthdr;
4794 off += ipv6_optlen(hp);
4795 break;
4796 }
4797 case IPPROTO_AH: {
4798 struct ip_auth_hdr *hp;
4799
4800 err = skb_maybe_pull_tail(skb,
4801 off +
4802 sizeof(struct ip_auth_hdr),
4803 MAX_IPV6_HDR_LEN);
4804 if (err < 0)
4805 goto out;
4806
4807 hp = OPT_HDR(struct ip_auth_hdr, skb, off);
4808 nexthdr = hp->nexthdr;
4809 off += ipv6_authlen(hp);
4810 break;
4811 }
4812 case IPPROTO_FRAGMENT: {
4813 struct frag_hdr *hp;
4814
4815 err = skb_maybe_pull_tail(skb,
4816 off +
4817 sizeof(struct frag_hdr),
4818 MAX_IPV6_HDR_LEN);
4819 if (err < 0)
4820 goto out;
4821
4822 hp = OPT_HDR(struct frag_hdr, skb, off);
4823
4824 if (hp->frag_off & htons(IP6_OFFSET | IP6_MF))
4825 fragment = true;
4826
4827 nexthdr = hp->nexthdr;
4828 off += sizeof(struct frag_hdr);
4829 break;
4830 }
4831 default:
4832 done = true;
4833 break;
4834 }
4835 }
4836
4837 err = -EPROTO;
4838
4839 if (!done || fragment)
4840 goto out;
4841
Jan Beulichf9708b42014-03-11 13:56:05 +00004842 csum = skb_checksum_setup_ip(skb, nexthdr, off);
4843 if (IS_ERR(csum))
4844 return PTR_ERR(csum);
Paul Durranted1f50c2014-01-09 10:02:46 +00004845
Jan Beulichf9708b42014-03-11 13:56:05 +00004846 if (recalculate)
4847 *csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4848 &ipv6_hdr(skb)->daddr,
4849 skb->len - off, nexthdr, 0);
Paul Durranted1f50c2014-01-09 10:02:46 +00004850 err = 0;
4851
4852out:
4853 return err;
4854}
4855
4856/**
4857 * skb_checksum_setup - set up partial checksum offset
4858 * @skb: the skb to set up
4859 * @recalculate: if true the pseudo-header checksum will be recalculated
4860 */
4861int skb_checksum_setup(struct sk_buff *skb, bool recalculate)
4862{
4863 int err;
4864
4865 switch (skb->protocol) {
4866 case htons(ETH_P_IP):
Jan Beulichf9708b42014-03-11 13:56:05 +00004867 err = skb_checksum_setup_ipv4(skb, recalculate);
Paul Durranted1f50c2014-01-09 10:02:46 +00004868 break;
4869
4870 case htons(ETH_P_IPV6):
4871 err = skb_checksum_setup_ipv6(skb, recalculate);
4872 break;
4873
4874 default:
4875 err = -EPROTO;
4876 break;
4877 }
4878
4879 return err;
4880}
4881EXPORT_SYMBOL(skb_checksum_setup);
4882
Linus Lüssing9afd85c2015-05-02 14:01:07 +02004883/**
4884 * skb_checksum_maybe_trim - maybe trims the given skb
4885 * @skb: the skb to check
4886 * @transport_len: the data length beyond the network header
4887 *
4888 * Checks whether the given skb has data beyond the given transport length.
4889 * If so, returns a cloned skb trimmed to this transport length.
4890 * Otherwise returns the provided skb. Returns NULL in error cases
4891 * (e.g. transport_len exceeds skb length or out-of-memory).
4892 *
Linus Lüssinga5169932015-08-13 05:54:07 +02004893 * Caller needs to set the skb transport header and free any returned skb if it
4894 * differs from the provided skb.
Linus Lüssing9afd85c2015-05-02 14:01:07 +02004895 */
4896static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb,
4897 unsigned int transport_len)
4898{
4899 struct sk_buff *skb_chk;
4900 unsigned int len = skb_transport_offset(skb) + transport_len;
4901 int ret;
4902
Linus Lüssinga5169932015-08-13 05:54:07 +02004903 if (skb->len < len)
Linus Lüssing9afd85c2015-05-02 14:01:07 +02004904 return NULL;
Linus Lüssinga5169932015-08-13 05:54:07 +02004905 else if (skb->len == len)
Linus Lüssing9afd85c2015-05-02 14:01:07 +02004906 return skb;
Linus Lüssing9afd85c2015-05-02 14:01:07 +02004907
4908 skb_chk = skb_clone(skb, GFP_ATOMIC);
Linus Lüssing9afd85c2015-05-02 14:01:07 +02004909 if (!skb_chk)
4910 return NULL;
4911
4912 ret = pskb_trim_rcsum(skb_chk, len);
4913 if (ret) {
4914 kfree_skb(skb_chk);
4915 return NULL;
4916 }
4917
4918 return skb_chk;
4919}
4920
4921/**
4922 * skb_checksum_trimmed - validate checksum of an skb
4923 * @skb: the skb to check
4924 * @transport_len: the data length beyond the network header
4925 * @skb_chkf: checksum function to use
4926 *
4927 * Applies the given checksum function skb_chkf to the provided skb.
4928 * Returns a checked and maybe trimmed skb. Returns NULL on error.
4929 *
4930 * If the skb has data beyond the given transport length, then a
4931 * trimmed & cloned skb is checked and returned.
4932 *
Linus Lüssinga5169932015-08-13 05:54:07 +02004933 * Caller needs to set the skb transport header and free any returned skb if it
4934 * differs from the provided skb.
Linus Lüssing9afd85c2015-05-02 14:01:07 +02004935 */
4936struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb,
4937 unsigned int transport_len,
4938 __sum16(*skb_chkf)(struct sk_buff *skb))
4939{
4940 struct sk_buff *skb_chk;
4941 unsigned int offset = skb_transport_offset(skb);
Linus Lüssingfcba67c2015-05-05 00:19:35 +02004942 __sum16 ret;
Linus Lüssing9afd85c2015-05-02 14:01:07 +02004943
4944 skb_chk = skb_checksum_maybe_trim(skb, transport_len);
4945 if (!skb_chk)
Linus Lüssinga5169932015-08-13 05:54:07 +02004946 goto err;
Linus Lüssing9afd85c2015-05-02 14:01:07 +02004947
Linus Lüssinga5169932015-08-13 05:54:07 +02004948 if (!pskb_may_pull(skb_chk, offset))
4949 goto err;
Linus Lüssing9afd85c2015-05-02 14:01:07 +02004950
Linus Lüssing9b368812016-02-24 04:21:42 +01004951 skb_pull_rcsum(skb_chk, offset);
Linus Lüssing9afd85c2015-05-02 14:01:07 +02004952 ret = skb_chkf(skb_chk);
Linus Lüssing9b368812016-02-24 04:21:42 +01004953 skb_push_rcsum(skb_chk, offset);
Linus Lüssing9afd85c2015-05-02 14:01:07 +02004954
Linus Lüssinga5169932015-08-13 05:54:07 +02004955 if (ret)
4956 goto err;
Linus Lüssing9afd85c2015-05-02 14:01:07 +02004957
4958 return skb_chk;
Linus Lüssinga5169932015-08-13 05:54:07 +02004959
4960err:
4961 if (skb_chk && skb_chk != skb)
4962 kfree_skb(skb_chk);
4963
4964 return NULL;
4965
Linus Lüssing9afd85c2015-05-02 14:01:07 +02004966}
4967EXPORT_SYMBOL(skb_checksum_trimmed);
4968
Ben Hutchings4497b072008-06-19 16:22:28 -07004969void __skb_warn_lro_forwarding(const struct sk_buff *skb)
4970{
Joe Perchese87cc472012-05-13 21:56:26 +00004971 net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n",
4972 skb->dev->name);
Ben Hutchings4497b072008-06-19 16:22:28 -07004973}
Ben Hutchings4497b072008-06-19 16:22:28 -07004974EXPORT_SYMBOL(__skb_warn_lro_forwarding);
Eric Dumazetbad43ca2012-05-19 03:02:02 +00004975
4976void kfree_skb_partial(struct sk_buff *skb, bool head_stolen)
4977{
Eric Dumazet3d861f62012-10-22 09:03:40 +00004978 if (head_stolen) {
4979 skb_release_head_state(skb);
Eric Dumazetbad43ca2012-05-19 03:02:02 +00004980 kmem_cache_free(skbuff_head_cache, skb);
Eric Dumazet3d861f62012-10-22 09:03:40 +00004981 } else {
Eric Dumazetbad43ca2012-05-19 03:02:02 +00004982 __kfree_skb(skb);
Eric Dumazet3d861f62012-10-22 09:03:40 +00004983 }
Eric Dumazetbad43ca2012-05-19 03:02:02 +00004984}
4985EXPORT_SYMBOL(kfree_skb_partial);
4986
4987/**
4988 * skb_try_coalesce - try to merge skb to prior one
4989 * @to: prior buffer
4990 * @from: buffer to add
4991 * @fragstolen: pointer to boolean
Randy Dunlapc6c4b972012-06-08 14:01:44 +00004992 * @delta_truesize: how much more was allocated than was requested
Eric Dumazetbad43ca2012-05-19 03:02:02 +00004993 */
4994bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
4995 bool *fragstolen, int *delta_truesize)
4996{
Eric Dumazetc818fa92017-10-04 10:48:35 -07004997 struct skb_shared_info *to_shinfo, *from_shinfo;
Eric Dumazetbad43ca2012-05-19 03:02:02 +00004998 int i, delta, len = from->len;
4999
5000 *fragstolen = false;
5001
5002 if (skb_cloned(to))
5003 return false;
5004
5005 if (len <= skb_tailroom(to)) {
Eric Dumazete93a0432014-09-15 04:19:52 -07005006 if (len)
5007 BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len));
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005008 *delta_truesize = 0;
5009 return true;
5010 }
5011
Eric Dumazetc818fa92017-10-04 10:48:35 -07005012 to_shinfo = skb_shinfo(to);
5013 from_shinfo = skb_shinfo(from);
5014 if (to_shinfo->frag_list || from_shinfo->frag_list)
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005015 return false;
Willem de Bruijn1f8b9772017-08-03 16:29:41 -04005016 if (skb_zcopy(to) || skb_zcopy(from))
5017 return false;
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005018
5019 if (skb_headlen(from) != 0) {
5020 struct page *page;
5021 unsigned int offset;
5022
Eric Dumazetc818fa92017-10-04 10:48:35 -07005023 if (to_shinfo->nr_frags +
5024 from_shinfo->nr_frags >= MAX_SKB_FRAGS)
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005025 return false;
5026
5027 if (skb_head_is_locked(from))
5028 return false;
5029
5030 delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
5031
5032 page = virt_to_head_page(from->head);
5033 offset = from->data - (unsigned char *)page_address(page);
5034
Eric Dumazetc818fa92017-10-04 10:48:35 -07005035 skb_fill_page_desc(to, to_shinfo->nr_frags,
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005036 page, offset, skb_headlen(from));
5037 *fragstolen = true;
5038 } else {
Eric Dumazetc818fa92017-10-04 10:48:35 -07005039 if (to_shinfo->nr_frags +
5040 from_shinfo->nr_frags > MAX_SKB_FRAGS)
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005041 return false;
5042
Weiping Panf4b549a2012-09-28 20:15:30 +00005043 delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from));
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005044 }
5045
5046 WARN_ON_ONCE(delta < len);
5047
Eric Dumazetc818fa92017-10-04 10:48:35 -07005048 memcpy(to_shinfo->frags + to_shinfo->nr_frags,
5049 from_shinfo->frags,
5050 from_shinfo->nr_frags * sizeof(skb_frag_t));
5051 to_shinfo->nr_frags += from_shinfo->nr_frags;
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005052
5053 if (!skb_cloned(from))
Eric Dumazetc818fa92017-10-04 10:48:35 -07005054 from_shinfo->nr_frags = 0;
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005055
Li RongQing8ea853f2012-09-18 16:53:21 +00005056 /* if the skb is not cloned this does nothing
5057 * since we set nr_frags to 0.
5058 */
Eric Dumazetc818fa92017-10-04 10:48:35 -07005059 for (i = 0; i < from_shinfo->nr_frags; i++)
5060 __skb_frag_ref(&from_shinfo->frags[i]);
Eric Dumazetbad43ca2012-05-19 03:02:02 +00005061
5062 to->truesize += delta;
5063 to->len += len;
5064 to->data_len += len;
5065
5066 *delta_truesize = delta;
5067 return true;
5068}
5069EXPORT_SYMBOL(skb_try_coalesce);
Nicolas Dichtel621e84d2013-06-26 16:11:27 +02005070
5071/**
Nicolas Dichtel8b27f272013-09-02 15:34:56 +02005072 * skb_scrub_packet - scrub an skb
Nicolas Dichtel621e84d2013-06-26 16:11:27 +02005073 *
5074 * @skb: buffer to clean
Nicolas Dichtel8b27f272013-09-02 15:34:56 +02005075 * @xnet: packet is crossing netns
Nicolas Dichtel621e84d2013-06-26 16:11:27 +02005076 *
Nicolas Dichtel8b27f272013-09-02 15:34:56 +02005077 * skb_scrub_packet can be used after encapsulating or decapsulting a packet
5078 * into/from a tunnel. Some information have to be cleared during these
5079 * operations.
5080 * skb_scrub_packet can also be used to clean a skb before injecting it in
5081 * another namespace (@xnet == true). We have to clear all information in the
5082 * skb that could impact namespace isolation.
Nicolas Dichtel621e84d2013-06-26 16:11:27 +02005083 */
Nicolas Dichtel8b27f272013-09-02 15:34:56 +02005084void skb_scrub_packet(struct sk_buff *skb, bool xnet)
Nicolas Dichtel621e84d2013-06-26 16:11:27 +02005085{
Nicolas Dichtel621e84d2013-06-26 16:11:27 +02005086 skb->pkt_type = PACKET_HOST;
5087 skb->skb_iif = 0;
WANG Cong60ff7462014-05-04 16:39:18 -07005088 skb->ignore_df = 0;
Nicolas Dichtel621e84d2013-06-26 16:11:27 +02005089 skb_dst_drop(skb);
Nicolas Dichtel621e84d2013-06-26 16:11:27 +02005090 secpath_reset(skb);
5091 nf_reset(skb);
5092 nf_reset_trace(skb);
Herbert Xu213dd742015-04-16 09:03:27 +08005093
Petr Machata6f9a5062018-11-19 16:11:07 +00005094#ifdef CONFIG_NET_SWITCHDEV
5095 skb->offload_fwd_mark = 0;
Ido Schimmel875e8932018-12-04 08:15:10 +00005096 skb->offload_l3_fwd_mark = 0;
Petr Machata6f9a5062018-11-19 16:11:07 +00005097#endif
5098
Herbert Xu213dd742015-04-16 09:03:27 +08005099 if (!xnet)
5100 return;
5101
Ye Yin2b5ec1a2017-10-26 16:57:05 +08005102 ipvs_reset(skb);
Herbert Xu213dd742015-04-16 09:03:27 +08005103 skb->mark = 0;
Jesus Sanchez-Palenciac47d8c22018-07-03 15:42:47 -07005104 skb->tstamp = 0;
Nicolas Dichtel621e84d2013-06-26 16:11:27 +02005105}
5106EXPORT_SYMBOL_GPL(skb_scrub_packet);
Florian Westphalde960aa2014-01-26 10:58:16 +01005107
5108/**
5109 * skb_gso_transport_seglen - Return length of individual segments of a gso packet
5110 *
5111 * @skb: GSO skb
5112 *
5113 * skb_gso_transport_seglen is used to determine the real size of the
5114 * individual segments, including Layer4 headers (TCP/UDP).
5115 *
5116 * The MAC/L2 or network (IP, IPv6) headers are not accounted for.
5117 */
Daniel Axtensa4a77712018-03-01 17:13:40 +11005118static unsigned int skb_gso_transport_seglen(const struct sk_buff *skb)
Florian Westphalde960aa2014-01-26 10:58:16 +01005119{
5120 const struct skb_shared_info *shinfo = skb_shinfo(skb);
Florian Westphalf993bc22014-10-20 13:49:18 +02005121 unsigned int thlen = 0;
Florian Westphalde960aa2014-01-26 10:58:16 +01005122
Florian Westphalf993bc22014-10-20 13:49:18 +02005123 if (skb->encapsulation) {
5124 thlen = skb_inner_transport_header(skb) -
5125 skb_transport_header(skb);
Florian Westphal6d39d582014-04-09 10:28:50 +02005126
Florian Westphalf993bc22014-10-20 13:49:18 +02005127 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
5128 thlen += inner_tcp_hdrlen(skb);
5129 } else if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
5130 thlen = tcp_hdrlen(skb);
Daniel Axtens1dd27cd2018-03-09 14:06:09 +11005131 } else if (unlikely(skb_is_gso_sctp(skb))) {
Marcelo Ricardo Leitner90017ac2016-06-02 15:05:43 -03005132 thlen = sizeof(struct sctphdr);
Willem de Bruijnee80d1e2018-04-26 13:42:16 -04005133 } else if (shinfo->gso_type & SKB_GSO_UDP_L4) {
5134 thlen = sizeof(struct udphdr);
Florian Westphalf993bc22014-10-20 13:49:18 +02005135 }
Florian Westphal6d39d582014-04-09 10:28:50 +02005136 /* UFO sets gso_size to the size of the fragmentation
5137 * payload, i.e. the size of the L4 (UDP) header is already
5138 * accounted for.
5139 */
Florian Westphalf993bc22014-10-20 13:49:18 +02005140 return thlen + shinfo->gso_size;
Florian Westphalde960aa2014-01-26 10:58:16 +01005141}
Daniel Axtensa4a77712018-03-01 17:13:40 +11005142
5143/**
5144 * skb_gso_network_seglen - Return length of individual segments of a gso packet
5145 *
5146 * @skb: GSO skb
5147 *
5148 * skb_gso_network_seglen is used to determine the real size of the
5149 * individual segments, including Layer3 (IP, IPv6) and L4 headers (TCP/UDP).
5150 *
5151 * The MAC/L2 header is not accounted for.
5152 */
5153static unsigned int skb_gso_network_seglen(const struct sk_buff *skb)
5154{
5155 unsigned int hdr_len = skb_transport_header(skb) -
5156 skb_network_header(skb);
5157
5158 return hdr_len + skb_gso_transport_seglen(skb);
5159}
5160
5161/**
5162 * skb_gso_mac_seglen - Return length of individual segments of a gso packet
5163 *
5164 * @skb: GSO skb
5165 *
5166 * skb_gso_mac_seglen is used to determine the real size of the
5167 * individual segments, including MAC/L2, Layer3 (IP, IPv6) and L4
5168 * headers (TCP/UDP).
5169 */
5170static unsigned int skb_gso_mac_seglen(const struct sk_buff *skb)
5171{
5172 unsigned int hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
5173
5174 return hdr_len + skb_gso_transport_seglen(skb);
5175}
Vlad Yasevich0d5501c2014-08-08 14:42:13 -04005176
Marcelo Ricardo Leitnerae7ef812016-06-02 15:05:41 -03005177/**
Daniel Axtens2b16f042018-01-31 14:15:33 +11005178 * skb_gso_size_check - check the skb size, considering GSO_BY_FRAGS
5179 *
5180 * There are a couple of instances where we have a GSO skb, and we
5181 * want to determine what size it would be after it is segmented.
5182 *
5183 * We might want to check:
5184 * - L3+L4+payload size (e.g. IP forwarding)
5185 * - L2+L3+L4+payload size (e.g. sanity check before passing to driver)
5186 *
5187 * This is a helper to do that correctly considering GSO_BY_FRAGS.
5188 *
Mathieu Malaterre49682bf2018-10-31 13:16:58 +01005189 * @skb: GSO skb
5190 *
Daniel Axtens2b16f042018-01-31 14:15:33 +11005191 * @seg_len: The segmented length (from skb_gso_*_seglen). In the
5192 * GSO_BY_FRAGS case this will be [header sizes + GSO_BY_FRAGS].
5193 *
5194 * @max_len: The maximum permissible length.
5195 *
5196 * Returns true if the segmented length <= max length.
5197 */
5198static inline bool skb_gso_size_check(const struct sk_buff *skb,
5199 unsigned int seg_len,
5200 unsigned int max_len) {
5201 const struct skb_shared_info *shinfo = skb_shinfo(skb);
5202 const struct sk_buff *iter;
5203
5204 if (shinfo->gso_size != GSO_BY_FRAGS)
5205 return seg_len <= max_len;
5206
5207 /* Undo this so we can re-use header sizes */
5208 seg_len -= GSO_BY_FRAGS;
5209
5210 skb_walk_frags(skb, iter) {
5211 if (seg_len + skb_headlen(iter) > max_len)
5212 return false;
5213 }
5214
5215 return true;
5216}
5217
5218/**
Daniel Axtens779b7932018-03-01 17:13:37 +11005219 * skb_gso_validate_network_len - Will a split GSO skb fit into a given MTU?
Marcelo Ricardo Leitnerae7ef812016-06-02 15:05:41 -03005220 *
5221 * @skb: GSO skb
David S. Miller76f21b92016-06-03 22:56:28 -07005222 * @mtu: MTU to validate against
Marcelo Ricardo Leitnerae7ef812016-06-02 15:05:41 -03005223 *
Daniel Axtens779b7932018-03-01 17:13:37 +11005224 * skb_gso_validate_network_len validates if a given skb will fit a
5225 * wanted MTU once split. It considers L3 headers, L4 headers, and the
5226 * payload.
Marcelo Ricardo Leitnerae7ef812016-06-02 15:05:41 -03005227 */
Daniel Axtens779b7932018-03-01 17:13:37 +11005228bool skb_gso_validate_network_len(const struct sk_buff *skb, unsigned int mtu)
Marcelo Ricardo Leitnerae7ef812016-06-02 15:05:41 -03005229{
Daniel Axtens2b16f042018-01-31 14:15:33 +11005230 return skb_gso_size_check(skb, skb_gso_network_seglen(skb), mtu);
Marcelo Ricardo Leitnerae7ef812016-06-02 15:05:41 -03005231}
Daniel Axtens779b7932018-03-01 17:13:37 +11005232EXPORT_SYMBOL_GPL(skb_gso_validate_network_len);
Marcelo Ricardo Leitnerae7ef812016-06-02 15:05:41 -03005233
Daniel Axtens2b16f042018-01-31 14:15:33 +11005234/**
5235 * skb_gso_validate_mac_len - Will a split GSO skb fit in a given length?
5236 *
5237 * @skb: GSO skb
5238 * @len: length to validate against
5239 *
5240 * skb_gso_validate_mac_len validates if a given skb will fit a wanted
5241 * length once split, including L2, L3 and L4 headers and the payload.
5242 */
5243bool skb_gso_validate_mac_len(const struct sk_buff *skb, unsigned int len)
5244{
5245 return skb_gso_size_check(skb, skb_gso_mac_seglen(skb), len);
5246}
5247EXPORT_SYMBOL_GPL(skb_gso_validate_mac_len);
5248
Vlad Yasevich0d5501c2014-08-08 14:42:13 -04005249static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb)
5250{
Yuya Kusakabed85e8be2019-04-16 10:22:28 +09005251 int mac_len, meta_len;
5252 void *meta;
Toshiaki Makita4bbb3e02018-03-13 14:51:27 +09005253
Vlad Yasevich0d5501c2014-08-08 14:42:13 -04005254 if (skb_cow(skb, skb_headroom(skb)) < 0) {
5255 kfree_skb(skb);
5256 return NULL;
5257 }
5258
Toshiaki Makita4bbb3e02018-03-13 14:51:27 +09005259 mac_len = skb->data - skb_mac_header(skb);
Toshiaki Makitaae474572018-03-29 19:05:29 +09005260 if (likely(mac_len > VLAN_HLEN + ETH_TLEN)) {
5261 memmove(skb_mac_header(skb) + VLAN_HLEN, skb_mac_header(skb),
5262 mac_len - VLAN_HLEN - ETH_TLEN);
5263 }
Yuya Kusakabed85e8be2019-04-16 10:22:28 +09005264
5265 meta_len = skb_metadata_len(skb);
5266 if (meta_len) {
5267 meta = skb_metadata_end(skb) - meta_len;
5268 memmove(meta + VLAN_HLEN, meta, meta_len);
5269 }
5270
Vlad Yasevich0d5501c2014-08-08 14:42:13 -04005271 skb->mac_header += VLAN_HLEN;
5272 return skb;
5273}
5274
5275struct sk_buff *skb_vlan_untag(struct sk_buff *skb)
5276{
5277 struct vlan_hdr *vhdr;
5278 u16 vlan_tci;
5279
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01005280 if (unlikely(skb_vlan_tag_present(skb))) {
Vlad Yasevich0d5501c2014-08-08 14:42:13 -04005281 /* vlan_tci is already set-up so leave this for another time */
5282 return skb;
5283 }
5284
5285 skb = skb_share_check(skb, GFP_ATOMIC);
5286 if (unlikely(!skb))
5287 goto err_free;
5288
5289 if (unlikely(!pskb_may_pull(skb, VLAN_HLEN)))
5290 goto err_free;
5291
5292 vhdr = (struct vlan_hdr *)skb->data;
5293 vlan_tci = ntohs(vhdr->h_vlan_TCI);
5294 __vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci);
5295
5296 skb_pull_rcsum(skb, VLAN_HLEN);
5297 vlan_set_encap_proto(skb, vhdr);
5298
5299 skb = skb_reorder_vlan_header(skb);
5300 if (unlikely(!skb))
5301 goto err_free;
5302
5303 skb_reset_network_header(skb);
5304 skb_reset_transport_header(skb);
5305 skb_reset_mac_len(skb);
5306
5307 return skb;
5308
5309err_free:
5310 kfree_skb(skb);
5311 return NULL;
5312}
5313EXPORT_SYMBOL(skb_vlan_untag);
Eric Dumazet2e4e4412014-09-17 04:49:49 -07005314
Jiri Pirkoe2195122014-11-19 14:05:01 +01005315int skb_ensure_writable(struct sk_buff *skb, int write_len)
5316{
5317 if (!pskb_may_pull(skb, write_len))
5318 return -ENOMEM;
5319
5320 if (!skb_cloned(skb) || skb_clone_writable(skb, write_len))
5321 return 0;
5322
5323 return pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
5324}
5325EXPORT_SYMBOL(skb_ensure_writable);
5326
Shmulik Ladkanibfca4c52016-09-19 19:11:09 +03005327/* remove VLAN header from packet and update csum accordingly.
5328 * expects a non skb_vlan_tag_present skb with a vlan tag payload
5329 */
5330int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci)
Jiri Pirko93515d52014-11-19 14:05:02 +01005331{
5332 struct vlan_hdr *vhdr;
Shmulik Ladkanib6a79202016-09-29 12:10:41 +03005333 int offset = skb->data - skb_mac_header(skb);
Jiri Pirko93515d52014-11-19 14:05:02 +01005334 int err;
5335
Shmulik Ladkanib6a79202016-09-29 12:10:41 +03005336 if (WARN_ONCE(offset,
5337 "__skb_vlan_pop got skb with skb->data not at mac header (offset %d)\n",
5338 offset)) {
5339 return -EINVAL;
5340 }
5341
Jiri Pirko93515d52014-11-19 14:05:02 +01005342 err = skb_ensure_writable(skb, VLAN_ETH_HLEN);
5343 if (unlikely(err))
Shmulik Ladkanib6a79202016-09-29 12:10:41 +03005344 return err;
Jiri Pirko93515d52014-11-19 14:05:02 +01005345
5346 skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
5347
5348 vhdr = (struct vlan_hdr *)(skb->data + ETH_HLEN);
5349 *vlan_tci = ntohs(vhdr->h_vlan_TCI);
5350
5351 memmove(skb->data + VLAN_HLEN, skb->data, 2 * ETH_ALEN);
5352 __skb_pull(skb, VLAN_HLEN);
5353
5354 vlan_set_encap_proto(skb, vhdr);
5355 skb->mac_header += VLAN_HLEN;
5356
5357 if (skb_network_offset(skb) < ETH_HLEN)
5358 skb_set_network_header(skb, ETH_HLEN);
5359
5360 skb_reset_mac_len(skb);
Jiri Pirko93515d52014-11-19 14:05:02 +01005361
5362 return err;
5363}
Shmulik Ladkanibfca4c52016-09-19 19:11:09 +03005364EXPORT_SYMBOL(__skb_vlan_pop);
Jiri Pirko93515d52014-11-19 14:05:02 +01005365
Shmulik Ladkanib6a79202016-09-29 12:10:41 +03005366/* Pop a vlan tag either from hwaccel or from payload.
5367 * Expects skb->data at mac header.
5368 */
Jiri Pirko93515d52014-11-19 14:05:02 +01005369int skb_vlan_pop(struct sk_buff *skb)
5370{
5371 u16 vlan_tci;
5372 __be16 vlan_proto;
5373 int err;
5374
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01005375 if (likely(skb_vlan_tag_present(skb))) {
Michał Mirosławb18175242018-11-09 00:18:02 +01005376 __vlan_hwaccel_clear_tag(skb);
Jiri Pirko93515d52014-11-19 14:05:02 +01005377 } else {
Shmulik Ladkaniecf4ee42016-09-20 12:48:37 +03005378 if (unlikely(!eth_type_vlan(skb->protocol)))
Jiri Pirko93515d52014-11-19 14:05:02 +01005379 return 0;
5380
5381 err = __skb_vlan_pop(skb, &vlan_tci);
5382 if (err)
5383 return err;
5384 }
5385 /* move next vlan tag to hw accel tag */
Shmulik Ladkaniecf4ee42016-09-20 12:48:37 +03005386 if (likely(!eth_type_vlan(skb->protocol)))
Jiri Pirko93515d52014-11-19 14:05:02 +01005387 return 0;
5388
5389 vlan_proto = skb->protocol;
5390 err = __skb_vlan_pop(skb, &vlan_tci);
5391 if (unlikely(err))
5392 return err;
5393
5394 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
5395 return 0;
5396}
5397EXPORT_SYMBOL(skb_vlan_pop);
5398
Shmulik Ladkanib6a79202016-09-29 12:10:41 +03005399/* Push a vlan tag either into hwaccel or into payload (if hwaccel tag present).
5400 * Expects skb->data at mac header.
5401 */
Jiri Pirko93515d52014-11-19 14:05:02 +01005402int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci)
5403{
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01005404 if (skb_vlan_tag_present(skb)) {
Shmulik Ladkanib6a79202016-09-29 12:10:41 +03005405 int offset = skb->data - skb_mac_header(skb);
Jiri Pirko93515d52014-11-19 14:05:02 +01005406 int err;
5407
Shmulik Ladkanib6a79202016-09-29 12:10:41 +03005408 if (WARN_ONCE(offset,
5409 "skb_vlan_push got skb with skb->data not at mac header (offset %d)\n",
5410 offset)) {
5411 return -EINVAL;
5412 }
5413
Jiri Pirko93515d52014-11-19 14:05:02 +01005414 err = __vlan_insert_tag(skb, skb->vlan_proto,
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01005415 skb_vlan_tag_get(skb));
Shmulik Ladkanib6a79202016-09-29 12:10:41 +03005416 if (err)
Jiri Pirko93515d52014-11-19 14:05:02 +01005417 return err;
Daniel Borkmann9241e2d2016-04-16 02:27:58 +02005418
Jiri Pirko93515d52014-11-19 14:05:02 +01005419 skb->protocol = skb->vlan_proto;
5420 skb->mac_len += VLAN_HLEN;
Jiri Pirko93515d52014-11-19 14:05:02 +01005421
Daniel Borkmann6b83d282016-02-20 00:29:30 +01005422 skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
Jiri Pirko93515d52014-11-19 14:05:02 +01005423 }
5424 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
5425 return 0;
5426}
5427EXPORT_SYMBOL(skb_vlan_push);
5428
John Hurley8822e272019-07-07 15:01:54 +01005429/* Update the ethertype of hdr and the skb csum value if required. */
5430static void skb_mod_eth_type(struct sk_buff *skb, struct ethhdr *hdr,
5431 __be16 ethertype)
5432{
5433 if (skb->ip_summed == CHECKSUM_COMPLETE) {
5434 __be16 diff[] = { ~hdr->h_proto, ethertype };
5435
5436 skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum);
5437 }
5438
5439 hdr->h_proto = ethertype;
5440}
5441
5442/**
5443 * skb_mpls_push() - push a new MPLS header after the mac header
5444 *
5445 * @skb: buffer
5446 * @mpls_lse: MPLS label stack entry to push
5447 * @mpls_proto: ethertype of the new MPLS header (expects 0x8847 or 0x8848)
5448 *
5449 * Expects skb->data at mac header.
5450 *
5451 * Returns 0 on success, -errno otherwise.
5452 */
5453int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto)
5454{
5455 struct mpls_shim_hdr *lse;
5456 int err;
5457
5458 if (unlikely(!eth_p_mpls(mpls_proto)))
5459 return -EINVAL;
5460
5461 /* Networking stack does not allow simultaneous Tunnel and MPLS GSO. */
5462 if (skb->encapsulation)
5463 return -EINVAL;
5464
5465 err = skb_cow_head(skb, MPLS_HLEN);
5466 if (unlikely(err))
5467 return err;
5468
5469 if (!skb->inner_protocol) {
5470 skb_set_inner_network_header(skb, skb->mac_len);
5471 skb_set_inner_protocol(skb, skb->protocol);
5472 }
5473
5474 skb_push(skb, MPLS_HLEN);
5475 memmove(skb_mac_header(skb) - MPLS_HLEN, skb_mac_header(skb),
5476 skb->mac_len);
5477 skb_reset_mac_header(skb);
5478 skb_set_network_header(skb, skb->mac_len);
5479
5480 lse = mpls_hdr(skb);
5481 lse->label_stack_entry = mpls_lse;
5482 skb_postpush_rcsum(skb, lse, MPLS_HLEN);
5483
5484 if (skb->dev && skb->dev->type == ARPHRD_ETHER)
5485 skb_mod_eth_type(skb, eth_hdr(skb), mpls_proto);
5486 skb->protocol = mpls_proto;
5487
5488 return 0;
5489}
5490EXPORT_SYMBOL_GPL(skb_mpls_push);
5491
Eric Dumazet2e4e4412014-09-17 04:49:49 -07005492/**
John Hurleyed246ce2019-07-07 15:01:55 +01005493 * skb_mpls_pop() - pop the outermost MPLS header
5494 *
5495 * @skb: buffer
5496 * @next_proto: ethertype of header after popped MPLS header
5497 *
5498 * Expects skb->data at mac header.
5499 *
5500 * Returns 0 on success, -errno otherwise.
5501 */
5502int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto)
5503{
5504 int err;
5505
5506 if (unlikely(!eth_p_mpls(skb->protocol)))
5507 return -EINVAL;
5508
5509 err = skb_ensure_writable(skb, skb->mac_len + MPLS_HLEN);
5510 if (unlikely(err))
5511 return err;
5512
5513 skb_postpull_rcsum(skb, mpls_hdr(skb), MPLS_HLEN);
5514 memmove(skb_mac_header(skb) + MPLS_HLEN, skb_mac_header(skb),
5515 skb->mac_len);
5516
5517 __skb_pull(skb, MPLS_HLEN);
5518 skb_reset_mac_header(skb);
5519 skb_set_network_header(skb, skb->mac_len);
5520
5521 if (skb->dev && skb->dev->type == ARPHRD_ETHER) {
5522 struct ethhdr *hdr;
5523
5524 /* use mpls_hdr() to get ethertype to account for VLANs. */
5525 hdr = (struct ethhdr *)((void *)mpls_hdr(skb) - ETH_HLEN);
5526 skb_mod_eth_type(skb, hdr, next_proto);
5527 }
5528 skb->protocol = next_proto;
5529
5530 return 0;
5531}
5532EXPORT_SYMBOL_GPL(skb_mpls_pop);
5533
5534/**
John Hurleyd27cf5c2019-07-07 15:01:56 +01005535 * skb_mpls_update_lse() - modify outermost MPLS header and update csum
5536 *
5537 * @skb: buffer
5538 * @mpls_lse: new MPLS label stack entry to update to
5539 *
5540 * Expects skb->data at mac header.
5541 *
5542 * Returns 0 on success, -errno otherwise.
5543 */
5544int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse)
5545{
5546 int err;
5547
5548 if (unlikely(!eth_p_mpls(skb->protocol)))
5549 return -EINVAL;
5550
5551 err = skb_ensure_writable(skb, skb->mac_len + MPLS_HLEN);
5552 if (unlikely(err))
5553 return err;
5554
5555 if (skb->ip_summed == CHECKSUM_COMPLETE) {
5556 __be32 diff[] = { ~mpls_hdr(skb)->label_stack_entry, mpls_lse };
5557
5558 skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum);
5559 }
5560
5561 mpls_hdr(skb)->label_stack_entry = mpls_lse;
5562
5563 return 0;
5564}
5565EXPORT_SYMBOL_GPL(skb_mpls_update_lse);
5566
5567/**
Eric Dumazet2e4e4412014-09-17 04:49:49 -07005568 * alloc_skb_with_frags - allocate skb with page frags
5569 *
Masanari Iidade3f0d02014-10-09 12:58:08 +09005570 * @header_len: size of linear part
5571 * @data_len: needed length in frags
5572 * @max_page_order: max page order desired.
5573 * @errcode: pointer to error code if any
5574 * @gfp_mask: allocation mask
Eric Dumazet2e4e4412014-09-17 04:49:49 -07005575 *
5576 * This can be used to allocate a paged skb, given a maximal order for frags.
5577 */
5578struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
5579 unsigned long data_len,
5580 int max_page_order,
5581 int *errcode,
5582 gfp_t gfp_mask)
5583{
5584 int npages = (data_len + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
5585 unsigned long chunk;
5586 struct sk_buff *skb;
5587 struct page *page;
Eric Dumazet2e4e4412014-09-17 04:49:49 -07005588 int i;
5589
5590 *errcode = -EMSGSIZE;
5591 /* Note this test could be relaxed, if we succeed to allocate
5592 * high order pages...
5593 */
5594 if (npages > MAX_SKB_FRAGS)
5595 return NULL;
5596
Eric Dumazet2e4e4412014-09-17 04:49:49 -07005597 *errcode = -ENOBUFS;
David Rientjesf8c468e2019-01-02 13:01:43 -08005598 skb = alloc_skb(header_len, gfp_mask);
Eric Dumazet2e4e4412014-09-17 04:49:49 -07005599 if (!skb)
5600 return NULL;
5601
5602 skb->truesize += npages << PAGE_SHIFT;
5603
5604 for (i = 0; npages > 0; i++) {
5605 int order = max_page_order;
5606
5607 while (order) {
5608 if (npages >= 1 << order) {
Mel Gormand0164ad2015-11-06 16:28:21 -08005609 page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) |
Eric Dumazet2e4e4412014-09-17 04:49:49 -07005610 __GFP_COMP |
Michal Hockod14b56f2018-06-28 17:53:06 +02005611 __GFP_NOWARN,
Eric Dumazet2e4e4412014-09-17 04:49:49 -07005612 order);
5613 if (page)
5614 goto fill_page;
5615 /* Do not retry other high order allocations */
5616 order = 1;
5617 max_page_order = 0;
5618 }
5619 order--;
5620 }
5621 page = alloc_page(gfp_mask);
5622 if (!page)
5623 goto failure;
5624fill_page:
5625 chunk = min_t(unsigned long, data_len,
5626 PAGE_SIZE << order);
5627 skb_fill_page_desc(skb, i, page, 0, chunk);
5628 data_len -= chunk;
5629 npages -= 1 << order;
5630 }
5631 return skb;
5632
5633failure:
5634 kfree_skb(skb);
5635 return NULL;
5636}
5637EXPORT_SYMBOL(alloc_skb_with_frags);
Sowmini Varadhan6fa01cc2016-04-22 18:36:35 -07005638
5639/* carve out the first off bytes from skb when off < headlen */
5640static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off,
5641 const int headlen, gfp_t gfp_mask)
5642{
5643 int i;
5644 int size = skb_end_offset(skb);
5645 int new_hlen = headlen - off;
5646 u8 *data;
Sowmini Varadhan6fa01cc2016-04-22 18:36:35 -07005647
5648 size = SKB_DATA_ALIGN(size);
5649
5650 if (skb_pfmemalloc(skb))
5651 gfp_mask |= __GFP_MEMALLOC;
5652 data = kmalloc_reserve(size +
5653 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
5654 gfp_mask, NUMA_NO_NODE, NULL);
5655 if (!data)
5656 return -ENOMEM;
5657
5658 size = SKB_WITH_OVERHEAD(ksize(data));
5659
5660 /* Copy real data, and all frags */
5661 skb_copy_from_linear_data_offset(skb, off, data, new_hlen);
5662 skb->len -= off;
5663
5664 memcpy((struct skb_shared_info *)(data + size),
5665 skb_shinfo(skb),
5666 offsetof(struct skb_shared_info,
5667 frags[skb_shinfo(skb)->nr_frags]));
5668 if (skb_cloned(skb)) {
5669 /* drop the old head gracefully */
5670 if (skb_orphan_frags(skb, gfp_mask)) {
5671 kfree(data);
5672 return -ENOMEM;
5673 }
5674 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
5675 skb_frag_ref(skb, i);
5676 if (skb_has_frag_list(skb))
5677 skb_clone_fraglist(skb);
5678 skb_release_data(skb);
5679 } else {
5680 /* we can reuse existing recount- all we did was
5681 * relocate values
5682 */
5683 skb_free_head(skb);
5684 }
5685
Sowmini Varadhan6fa01cc2016-04-22 18:36:35 -07005686 skb->head = data;
5687 skb->data = data;
5688 skb->head_frag = 0;
5689#ifdef NET_SKBUFF_DATA_USES_OFFSET
5690 skb->end = size;
Sowmini Varadhan6fa01cc2016-04-22 18:36:35 -07005691#else
5692 skb->end = skb->head + size;
5693#endif
5694 skb_set_tail_pointer(skb, skb_headlen(skb));
5695 skb_headers_offset_update(skb, 0);
5696 skb->cloned = 0;
5697 skb->hdr_len = 0;
5698 skb->nohdr = 0;
5699 atomic_set(&skb_shinfo(skb)->dataref, 1);
5700
5701 return 0;
5702}
5703
5704static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp);
5705
5706/* carve out the first eat bytes from skb's frag_list. May recurse into
5707 * pskb_carve()
5708 */
5709static int pskb_carve_frag_list(struct sk_buff *skb,
5710 struct skb_shared_info *shinfo, int eat,
5711 gfp_t gfp_mask)
5712{
5713 struct sk_buff *list = shinfo->frag_list;
5714 struct sk_buff *clone = NULL;
5715 struct sk_buff *insp = NULL;
5716
5717 do {
5718 if (!list) {
5719 pr_err("Not enough bytes to eat. Want %d\n", eat);
5720 return -EFAULT;
5721 }
5722 if (list->len <= eat) {
5723 /* Eaten as whole. */
5724 eat -= list->len;
5725 list = list->next;
5726 insp = list;
5727 } else {
5728 /* Eaten partially. */
5729 if (skb_shared(list)) {
5730 clone = skb_clone(list, gfp_mask);
5731 if (!clone)
5732 return -ENOMEM;
5733 insp = list->next;
5734 list = clone;
5735 } else {
5736 /* This may be pulled without problems. */
5737 insp = list;
5738 }
5739 if (pskb_carve(list, eat, gfp_mask) < 0) {
5740 kfree_skb(clone);
5741 return -ENOMEM;
5742 }
5743 break;
5744 }
5745 } while (eat);
5746
5747 /* Free pulled out fragments. */
5748 while ((list = shinfo->frag_list) != insp) {
5749 shinfo->frag_list = list->next;
5750 kfree_skb(list);
5751 }
5752 /* And insert new clone at head. */
5753 if (clone) {
5754 clone->next = list;
5755 shinfo->frag_list = clone;
5756 }
5757 return 0;
5758}
5759
5760/* carve off first len bytes from skb. Split line (off) is in the
5761 * non-linear part of skb
5762 */
5763static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off,
5764 int pos, gfp_t gfp_mask)
5765{
5766 int i, k = 0;
5767 int size = skb_end_offset(skb);
5768 u8 *data;
5769 const int nfrags = skb_shinfo(skb)->nr_frags;
5770 struct skb_shared_info *shinfo;
Sowmini Varadhan6fa01cc2016-04-22 18:36:35 -07005771
5772 size = SKB_DATA_ALIGN(size);
5773
5774 if (skb_pfmemalloc(skb))
5775 gfp_mask |= __GFP_MEMALLOC;
5776 data = kmalloc_reserve(size +
5777 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
5778 gfp_mask, NUMA_NO_NODE, NULL);
5779 if (!data)
5780 return -ENOMEM;
5781
5782 size = SKB_WITH_OVERHEAD(ksize(data));
5783
5784 memcpy((struct skb_shared_info *)(data + size),
5785 skb_shinfo(skb), offsetof(struct skb_shared_info,
5786 frags[skb_shinfo(skb)->nr_frags]));
5787 if (skb_orphan_frags(skb, gfp_mask)) {
5788 kfree(data);
5789 return -ENOMEM;
5790 }
5791 shinfo = (struct skb_shared_info *)(data + size);
5792 for (i = 0; i < nfrags; i++) {
5793 int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]);
5794
5795 if (pos + fsize > off) {
5796 shinfo->frags[k] = skb_shinfo(skb)->frags[i];
5797
5798 if (pos < off) {
5799 /* Split frag.
5800 * We have two variants in this case:
5801 * 1. Move all the frag to the second
5802 * part, if it is possible. F.e.
5803 * this approach is mandatory for TUX,
5804 * where splitting is expensive.
5805 * 2. Split is accurately. We make this.
5806 */
5807 shinfo->frags[0].page_offset += off - pos;
5808 skb_frag_size_sub(&shinfo->frags[0], off - pos);
5809 }
5810 skb_frag_ref(skb, i);
5811 k++;
5812 }
5813 pos += fsize;
5814 }
5815 shinfo->nr_frags = k;
5816 if (skb_has_frag_list(skb))
5817 skb_clone_fraglist(skb);
5818
5819 if (k == 0) {
5820 /* split line is in frag list */
5821 pskb_carve_frag_list(skb, shinfo, off - pos, gfp_mask);
5822 }
5823 skb_release_data(skb);
5824
Sowmini Varadhan6fa01cc2016-04-22 18:36:35 -07005825 skb->head = data;
5826 skb->head_frag = 0;
5827 skb->data = data;
5828#ifdef NET_SKBUFF_DATA_USES_OFFSET
5829 skb->end = size;
Sowmini Varadhan6fa01cc2016-04-22 18:36:35 -07005830#else
5831 skb->end = skb->head + size;
5832#endif
5833 skb_reset_tail_pointer(skb);
5834 skb_headers_offset_update(skb, 0);
5835 skb->cloned = 0;
5836 skb->hdr_len = 0;
5837 skb->nohdr = 0;
5838 skb->len -= off;
5839 skb->data_len = skb->len;
5840 atomic_set(&skb_shinfo(skb)->dataref, 1);
5841 return 0;
5842}
5843
5844/* remove len bytes from the beginning of the skb */
5845static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp)
5846{
5847 int headlen = skb_headlen(skb);
5848
5849 if (len < headlen)
5850 return pskb_carve_inside_header(skb, len, headlen, gfp);
5851 else
5852 return pskb_carve_inside_nonlinear(skb, len, headlen, gfp);
5853}
5854
5855/* Extract to_copy bytes starting at off from skb, and return this in
5856 * a new skb
5857 */
5858struct sk_buff *pskb_extract(struct sk_buff *skb, int off,
5859 int to_copy, gfp_t gfp)
5860{
5861 struct sk_buff *clone = skb_clone(skb, gfp);
5862
5863 if (!clone)
5864 return NULL;
5865
5866 if (pskb_carve(clone, off, gfp) < 0 ||
5867 pskb_trim(clone, to_copy)) {
5868 kfree_skb(clone);
5869 return NULL;
5870 }
5871 return clone;
5872}
5873EXPORT_SYMBOL(pskb_extract);
Eric Dumazetc8c8b122016-12-07 09:19:33 -08005874
5875/**
5876 * skb_condense - try to get rid of fragments/frag_list if possible
5877 * @skb: buffer
5878 *
5879 * Can be used to save memory before skb is added to a busy queue.
5880 * If packet has bytes in frags and enough tail room in skb->head,
5881 * pull all of them, so that we can free the frags right now and adjust
5882 * truesize.
5883 * Notes:
5884 * We do not reallocate skb->head thus can not fail.
5885 * Caller must re-evaluate skb->truesize if needed.
5886 */
5887void skb_condense(struct sk_buff *skb)
5888{
Eric Dumazet3174fed2016-12-09 08:02:05 -08005889 if (skb->data_len) {
5890 if (skb->data_len > skb->end - skb->tail ||
5891 skb_cloned(skb))
5892 return;
Eric Dumazetc8c8b122016-12-07 09:19:33 -08005893
Eric Dumazet3174fed2016-12-09 08:02:05 -08005894 /* Nice, we can free page frag(s) right now */
5895 __pskb_pull_tail(skb, skb->data_len);
5896 }
5897 /* At this point, skb->truesize might be over estimated,
5898 * because skb had a fragment, and fragments do not tell
5899 * their truesize.
5900 * When we pulled its content into skb->head, fragment
5901 * was freed, but __pskb_pull_tail() could not possibly
5902 * adjust skb->truesize, not knowing the frag truesize.
Eric Dumazetc8c8b122016-12-07 09:19:33 -08005903 */
5904 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
5905}
Florian Westphaldf5042f2018-12-18 17:15:16 +01005906
5907#ifdef CONFIG_SKB_EXTENSIONS
5908static void *skb_ext_get_ptr(struct skb_ext *ext, enum skb_ext_id id)
5909{
5910 return (void *)ext + (ext->offset[id] * SKB_EXT_ALIGN_VALUE);
5911}
5912
5913static struct skb_ext *skb_ext_alloc(void)
5914{
5915 struct skb_ext *new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC);
5916
5917 if (new) {
5918 memset(new->offset, 0, sizeof(new->offset));
5919 refcount_set(&new->refcnt, 1);
5920 }
5921
5922 return new;
5923}
5924
Florian Westphal41650792018-12-18 17:15:27 +01005925static struct skb_ext *skb_ext_maybe_cow(struct skb_ext *old,
5926 unsigned int old_active)
Florian Westphaldf5042f2018-12-18 17:15:16 +01005927{
5928 struct skb_ext *new;
5929
5930 if (refcount_read(&old->refcnt) == 1)
5931 return old;
5932
5933 new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC);
5934 if (!new)
5935 return NULL;
5936
5937 memcpy(new, old, old->chunks * SKB_EXT_ALIGN_VALUE);
5938 refcount_set(&new->refcnt, 1);
5939
Florian Westphal41650792018-12-18 17:15:27 +01005940#ifdef CONFIG_XFRM
5941 if (old_active & (1 << SKB_EXT_SEC_PATH)) {
5942 struct sec_path *sp = skb_ext_get_ptr(old, SKB_EXT_SEC_PATH);
5943 unsigned int i;
5944
5945 for (i = 0; i < sp->len; i++)
5946 xfrm_state_hold(sp->xvec[i]);
5947 }
5948#endif
Florian Westphaldf5042f2018-12-18 17:15:16 +01005949 __skb_ext_put(old);
5950 return new;
5951}
5952
5953/**
5954 * skb_ext_add - allocate space for given extension, COW if needed
5955 * @skb: buffer
5956 * @id: extension to allocate space for
5957 *
5958 * Allocates enough space for the given extension.
5959 * If the extension is already present, a pointer to that extension
5960 * is returned.
5961 *
5962 * If the skb was cloned, COW applies and the returned memory can be
5963 * modified without changing the extension space of clones buffers.
5964 *
5965 * Returns pointer to the extension or NULL on allocation failure.
5966 */
5967void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id)
5968{
5969 struct skb_ext *new, *old = NULL;
5970 unsigned int newlen, newoff;
5971
5972 if (skb->active_extensions) {
5973 old = skb->extensions;
5974
Florian Westphal41650792018-12-18 17:15:27 +01005975 new = skb_ext_maybe_cow(old, skb->active_extensions);
Florian Westphaldf5042f2018-12-18 17:15:16 +01005976 if (!new)
5977 return NULL;
5978
Paolo Abeni682ec852018-12-21 19:03:15 +01005979 if (__skb_ext_exist(new, id))
Florian Westphaldf5042f2018-12-18 17:15:16 +01005980 goto set_active;
Florian Westphaldf5042f2018-12-18 17:15:16 +01005981
Paolo Abenie94e50b2018-12-21 19:03:13 +01005982 newoff = new->chunks;
Florian Westphaldf5042f2018-12-18 17:15:16 +01005983 } else {
5984 newoff = SKB_EXT_CHUNKSIZEOF(*new);
5985
5986 new = skb_ext_alloc();
5987 if (!new)
5988 return NULL;
5989 }
5990
5991 newlen = newoff + skb_ext_type_len[id];
5992 new->chunks = newlen;
5993 new->offset[id] = newoff;
Florian Westphaldf5042f2018-12-18 17:15:16 +01005994set_active:
Paolo Abeni682ec852018-12-21 19:03:15 +01005995 skb->extensions = new;
Florian Westphaldf5042f2018-12-18 17:15:16 +01005996 skb->active_extensions |= 1 << id;
5997 return skb_ext_get_ptr(new, id);
5998}
5999EXPORT_SYMBOL(skb_ext_add);
6000
Florian Westphal41650792018-12-18 17:15:27 +01006001#ifdef CONFIG_XFRM
6002static void skb_ext_put_sp(struct sec_path *sp)
6003{
6004 unsigned int i;
6005
6006 for (i = 0; i < sp->len; i++)
6007 xfrm_state_put(sp->xvec[i]);
6008}
6009#endif
6010
Florian Westphaldf5042f2018-12-18 17:15:16 +01006011void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id)
6012{
6013 struct skb_ext *ext = skb->extensions;
6014
6015 skb->active_extensions &= ~(1 << id);
6016 if (skb->active_extensions == 0) {
6017 skb->extensions = NULL;
6018 __skb_ext_put(ext);
Florian Westphal41650792018-12-18 17:15:27 +01006019#ifdef CONFIG_XFRM
6020 } else if (id == SKB_EXT_SEC_PATH &&
6021 refcount_read(&ext->refcnt) == 1) {
6022 struct sec_path *sp = skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH);
6023
6024 skb_ext_put_sp(sp);
6025 sp->len = 0;
6026#endif
Florian Westphaldf5042f2018-12-18 17:15:16 +01006027 }
6028}
6029EXPORT_SYMBOL(__skb_ext_del);
6030
6031void __skb_ext_put(struct skb_ext *ext)
6032{
6033 /* If this is last clone, nothing can increment
6034 * it after check passes. Avoids one atomic op.
6035 */
6036 if (refcount_read(&ext->refcnt) == 1)
6037 goto free_now;
6038
6039 if (!refcount_dec_and_test(&ext->refcnt))
6040 return;
6041free_now:
Florian Westphal41650792018-12-18 17:15:27 +01006042#ifdef CONFIG_XFRM
6043 if (__skb_ext_exist(ext, SKB_EXT_SEC_PATH))
6044 skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH));
6045#endif
6046
Florian Westphaldf5042f2018-12-18 17:15:16 +01006047 kmem_cache_free(skbuff_ext_cache, ext);
6048}
6049EXPORT_SYMBOL(__skb_ext_put);
6050#endif /* CONFIG_SKB_EXTENSIONS */