blob: 8400c1f33cd4e6c39426a676739508c05f0b146b [file] [log] [blame]
Alexei Starovoitov51580e72014-09-26 00:17:02 -07001/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
Alexei Starovoitov969bf052016-05-05 19:49:10 -07002 * Copyright (c) 2016 Facebook
Joe Stringerfd978bf72018-10-02 13:35:35 -07003 * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io
Alexei Starovoitov51580e72014-09-26 00:17:02 -07004 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of version 2 of the GNU General Public
7 * License as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License for more details.
13 */
Yonghong Song838e9692018-11-19 15:29:11 -080014#include <uapi/linux/btf.h>
Alexei Starovoitov51580e72014-09-26 00:17:02 -070015#include <linux/kernel.h>
16#include <linux/types.h>
17#include <linux/slab.h>
18#include <linux/bpf.h>
Yonghong Song838e9692018-11-19 15:29:11 -080019#include <linux/btf.h>
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +010020#include <linux/bpf_verifier.h>
Alexei Starovoitov51580e72014-09-26 00:17:02 -070021#include <linux/filter.h>
22#include <net/netlink.h>
23#include <linux/file.h>
24#include <linux/vmalloc.h>
Thomas Grafebb676d2016-10-27 11:23:51 +020025#include <linux/stringify.h>
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -080026#include <linux/bsearch.h>
27#include <linux/sort.h>
Yonghong Songc195651e2018-04-28 22:28:08 -070028#include <linux/perf_event.h>
Martin KaFai Laud9762e82018-12-13 10:41:48 -080029#include <linux/ctype.h>
Alexei Starovoitov51580e72014-09-26 00:17:02 -070030
Jakub Kicinskif4ac7e02017-10-09 10:30:12 -070031#include "disasm.h"
32
Jakub Kicinski00176a32017-10-16 16:40:54 -070033static const struct bpf_verifier_ops * const bpf_verifier_ops[] = {
34#define BPF_PROG_TYPE(_id, _name) \
35 [_id] = & _name ## _verifier_ops,
36#define BPF_MAP_TYPE(_id, _ops)
37#include <linux/bpf_types.h>
38#undef BPF_PROG_TYPE
39#undef BPF_MAP_TYPE
40};
41
Alexei Starovoitov51580e72014-09-26 00:17:02 -070042/* bpf_check() is a static code analyzer that walks eBPF program
43 * instruction by instruction and updates register/stack state.
44 * All paths of conditional branches are analyzed until 'bpf_exit' insn.
45 *
46 * The first pass is depth-first-search to check that the program is a DAG.
47 * It rejects the following programs:
48 * - larger than BPF_MAXINSNS insns
49 * - if loop is present (detected via back-edge)
50 * - unreachable insns exist (shouldn't be a forest. program = one function)
51 * - out of bounds or malformed jumps
52 * The second pass is all possible path descent from the 1st insn.
53 * Since it's analyzing all pathes through the program, the length of the
Gary Lineba38a92017-03-01 16:25:51 +080054 * analysis is limited to 64k insn, which may be hit even if total number of
Alexei Starovoitov51580e72014-09-26 00:17:02 -070055 * insn is less then 4K, but there are too many branches that change stack/regs.
56 * Number of 'branches to be analyzed' is limited to 1k
57 *
58 * On entry to each instruction, each register has a type, and the instruction
59 * changes the types of the registers depending on instruction semantics.
60 * If instruction is BPF_MOV64_REG(BPF_REG_1, BPF_REG_5), then type of R5 is
61 * copied to R1.
62 *
63 * All registers are 64-bit.
64 * R0 - return register
65 * R1-R5 argument passing registers
66 * R6-R9 callee saved registers
67 * R10 - frame pointer read-only
68 *
69 * At the start of BPF program the register R1 contains a pointer to bpf_context
70 * and has type PTR_TO_CTX.
71 *
72 * Verifier tracks arithmetic operations on pointers in case:
73 * BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
74 * BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -20),
75 * 1st insn copies R10 (which has FRAME_PTR) type into R1
76 * and 2nd arithmetic instruction is pattern matched to recognize
77 * that it wants to construct a pointer to some element within stack.
78 * So after 2nd insn, the register R1 has type PTR_TO_STACK
79 * (and -20 constant is saved for further stack bounds checking).
80 * Meaning that this reg is a pointer to stack plus known immediate constant.
81 *
Edward Creef1174f72017-08-07 15:26:19 +010082 * Most of the time the registers have SCALAR_VALUE type, which
Alexei Starovoitov51580e72014-09-26 00:17:02 -070083 * means the register has some value, but it's not a valid pointer.
Edward Creef1174f72017-08-07 15:26:19 +010084 * (like pointer plus pointer becomes SCALAR_VALUE type)
Alexei Starovoitov51580e72014-09-26 00:17:02 -070085 *
86 * When verifier sees load or store instructions the type of base register
Joe Stringerc64b7982018-10-02 13:35:33 -070087 * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are
88 * four pointer types recognized by check_mem_access() function.
Alexei Starovoitov51580e72014-09-26 00:17:02 -070089 *
90 * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value'
91 * and the range of [ptr, ptr + map's value_size) is accessible.
92 *
93 * registers used to pass values to function calls are checked against
94 * function argument constraints.
95 *
96 * ARG_PTR_TO_MAP_KEY is one of such argument constraints.
97 * It means that the register type passed to this function must be
98 * PTR_TO_STACK and it will be used inside the function as
99 * 'pointer to map element key'
100 *
101 * For example the argument constraints for bpf_map_lookup_elem():
102 * .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
103 * .arg1_type = ARG_CONST_MAP_PTR,
104 * .arg2_type = ARG_PTR_TO_MAP_KEY,
105 *
106 * ret_type says that this function returns 'pointer to map elem value or null'
107 * function expects 1st argument to be a const pointer to 'struct bpf_map' and
108 * 2nd argument should be a pointer to stack, which will be used inside
109 * the helper function as a pointer to map element key.
110 *
111 * On the kernel side the helper function looks like:
112 * u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
113 * {
114 * struct bpf_map *map = (struct bpf_map *) (unsigned long) r1;
115 * void *key = (void *) (unsigned long) r2;
116 * void *value;
117 *
118 * here kernel can access 'key' and 'map' pointers safely, knowing that
119 * [key, key + map->key_size) bytes are valid and were initialized on
120 * the stack of eBPF program.
121 * }
122 *
123 * Corresponding eBPF program may look like:
124 * BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), // after this insn R2 type is FRAME_PTR
125 * BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), // after this insn R2 type is PTR_TO_STACK
126 * BPF_LD_MAP_FD(BPF_REG_1, map_fd), // after this insn R1 type is CONST_PTR_TO_MAP
127 * BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem),
128 * here verifier looks at prototype of map_lookup_elem() and sees:
129 * .arg1_type == ARG_CONST_MAP_PTR and R1->type == CONST_PTR_TO_MAP, which is ok,
130 * Now verifier knows that this map has key of R1->map_ptr->key_size bytes
131 *
132 * Then .arg2_type == ARG_PTR_TO_MAP_KEY and R2->type == PTR_TO_STACK, ok so far,
133 * Now verifier checks that [R2, R2 + map's key_size) are within stack limits
134 * and were initialized prior to this call.
135 * If it's ok, then verifier allows this BPF_CALL insn and looks at
136 * .ret_type which is RET_PTR_TO_MAP_VALUE_OR_NULL, so it sets
137 * R0->type = PTR_TO_MAP_VALUE_OR_NULL which means bpf_map_lookup_elem() function
138 * returns ether pointer to map value or NULL.
139 *
140 * When type PTR_TO_MAP_VALUE_OR_NULL passes through 'if (reg != 0) goto +off'
141 * insn, the register holding that pointer in the true branch changes state to
142 * PTR_TO_MAP_VALUE and the same register changes state to CONST_IMM in the false
143 * branch. See check_cond_jmp_op().
144 *
145 * After the call R0 is set to return type of the function and registers R1-R5
146 * are set to NOT_INIT to indicate that they are no longer readable.
Joe Stringerfd978bf72018-10-02 13:35:35 -0700147 *
148 * The following reference types represent a potential reference to a kernel
149 * resource which, after first being allocated, must be checked and freed by
150 * the BPF program:
151 * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET
152 *
153 * When the verifier sees a helper call return a reference type, it allocates a
154 * pointer id for the reference and stores it in the current function state.
155 * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into
156 * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type
157 * passes through a NULL-check conditional. For the branch wherein the state is
158 * changed to CONST_IMM, the verifier releases the reference.
Joe Stringer6acc9b42018-10-02 13:35:36 -0700159 *
160 * For each helper function that allocates a reference, such as
161 * bpf_sk_lookup_tcp(), there is a corresponding release function, such as
162 * bpf_sk_release(). When a reference type passes into the release function,
163 * the verifier also releases the reference. If any unchecked or unreleased
164 * reference remains at the end of the program, the verifier rejects it.
Alexei Starovoitov51580e72014-09-26 00:17:02 -0700165 */
166
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700167/* verifier_state + insn_idx are pushed to stack when branch is encountered */
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +0100168struct bpf_verifier_stack_elem {
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700169 /* verifer state is 'st'
170 * before processing instruction 'insn_idx'
171 * and after processing instruction 'prev_insn_idx'
172 */
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +0100173 struct bpf_verifier_state st;
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700174 int insn_idx;
175 int prev_insn_idx;
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +0100176 struct bpf_verifier_stack_elem *next;
Alexei Starovoitovcbd35702014-09-26 00:17:03 -0700177};
178
Daniel Borkmann07016152016-04-05 22:33:17 +0200179#define BPF_COMPLEXITY_LIMIT_STACK 1024
Alexei Starovoitovceefbc92018-12-03 22:46:06 -0800180#define BPF_COMPLEXITY_LIMIT_STATES 64
Daniel Borkmann07016152016-04-05 22:33:17 +0200181
Daniel Borkmannc93552c2018-05-24 02:32:53 +0200182#define BPF_MAP_PTR_UNPRIV 1UL
183#define BPF_MAP_PTR_POISON ((void *)((0xeB9FUL << 1) + \
184 POISON_POINTER_DELTA))
185#define BPF_MAP_PTR(X) ((struct bpf_map *)((X) & ~BPF_MAP_PTR_UNPRIV))
186
187static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux)
188{
189 return BPF_MAP_PTR(aux->map_state) == BPF_MAP_PTR_POISON;
190}
191
192static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux)
193{
194 return aux->map_state & BPF_MAP_PTR_UNPRIV;
195}
196
197static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux,
198 const struct bpf_map *map, bool unpriv)
199{
200 BUILD_BUG_ON((unsigned long)BPF_MAP_PTR_POISON & BPF_MAP_PTR_UNPRIV);
201 unpriv |= bpf_map_ptr_unpriv(aux);
202 aux->map_state = (unsigned long)map |
203 (unpriv ? BPF_MAP_PTR_UNPRIV : 0UL);
204}
Martin KaFai Laufad73a12017-03-22 10:00:32 -0700205
Daniel Borkmann33ff9822016-04-13 00:10:50 +0200206struct bpf_call_arg_meta {
207 struct bpf_map *map_ptr;
Daniel Borkmann435faee12016-04-13 00:10:51 +0200208 bool raw_mode;
Daniel Borkmann36bbef52016-09-20 00:26:13 +0200209 bool pkt_access;
Daniel Borkmann435faee12016-04-13 00:10:51 +0200210 int regno;
211 int access_size;
Yonghong Song849fa502018-04-28 22:28:09 -0700212 s64 msize_smax_value;
213 u64 msize_umax_value;
Martin KaFai Lau1b986582019-03-12 10:23:02 -0700214 int ref_obj_id;
Alexei Starovoitovd83525c2019-01-31 15:40:04 -0800215 int func_id;
Daniel Borkmann33ff9822016-04-13 00:10:50 +0200216};
217
Alexei Starovoitovcbd35702014-09-26 00:17:03 -0700218static DEFINE_MUTEX(bpf_verifier_lock);
219
Martin KaFai Laud9762e82018-12-13 10:41:48 -0800220static const struct bpf_line_info *
221find_linfo(const struct bpf_verifier_env *env, u32 insn_off)
222{
223 const struct bpf_line_info *linfo;
224 const struct bpf_prog *prog;
225 u32 i, nr_linfo;
226
227 prog = env->prog;
228 nr_linfo = prog->aux->nr_linfo;
229
230 if (!nr_linfo || insn_off >= prog->len)
231 return NULL;
232
233 linfo = prog->aux->linfo;
234 for (i = 1; i < nr_linfo; i++)
235 if (insn_off < linfo[i].insn_off)
236 break;
237
238 return &linfo[i - 1];
239}
240
Martin KaFai Lau77d2e052018-03-24 11:44:23 -0700241void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt,
242 va_list args)
Alexei Starovoitovcbd35702014-09-26 00:17:03 -0700243{
Jakub Kicinskia2a7d572017-10-09 10:30:15 -0700244 unsigned int n;
Alexei Starovoitovcbd35702014-09-26 00:17:03 -0700245
Jakub Kicinskia2a7d572017-10-09 10:30:15 -0700246 n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args);
Jakub Kicinskia2a7d572017-10-09 10:30:15 -0700247
248 WARN_ONCE(n >= BPF_VERIFIER_TMP_LOG_SIZE - 1,
249 "verifier log line truncated - local buffer too short\n");
250
251 n = min(log->len_total - log->len_used - 1, n);
252 log->kbuf[n] = '\0';
253
254 if (!copy_to_user(log->ubuf + log->len_used, log->kbuf, n + 1))
255 log->len_used += n;
256 else
257 log->ubuf = NULL;
Alexei Starovoitovcbd35702014-09-26 00:17:03 -0700258}
Jiri Olsaabe08842018-03-23 11:41:28 +0100259
260/* log_level controls verbosity level of eBPF verifier.
261 * bpf_verifier_log_write() is used to dump the verification trace to the log,
262 * so the user can figure out what's wrong with the program
Quentin Monnet430e68d2018-01-10 12:26:06 +0000263 */
Jiri Olsaabe08842018-03-23 11:41:28 +0100264__printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env,
265 const char *fmt, ...)
266{
267 va_list args;
268
Martin KaFai Lau77d2e052018-03-24 11:44:23 -0700269 if (!bpf_verifier_log_needed(&env->log))
270 return;
271
Jiri Olsaabe08842018-03-23 11:41:28 +0100272 va_start(args, fmt);
Martin KaFai Lau77d2e052018-03-24 11:44:23 -0700273 bpf_verifier_vlog(&env->log, fmt, args);
Jiri Olsaabe08842018-03-23 11:41:28 +0100274 va_end(args);
275}
276EXPORT_SYMBOL_GPL(bpf_verifier_log_write);
277
278__printf(2, 3) static void verbose(void *private_data, const char *fmt, ...)
279{
Martin KaFai Lau77d2e052018-03-24 11:44:23 -0700280 struct bpf_verifier_env *env = private_data;
Jiri Olsaabe08842018-03-23 11:41:28 +0100281 va_list args;
282
Martin KaFai Lau77d2e052018-03-24 11:44:23 -0700283 if (!bpf_verifier_log_needed(&env->log))
284 return;
285
Jiri Olsaabe08842018-03-23 11:41:28 +0100286 va_start(args, fmt);
Martin KaFai Lau77d2e052018-03-24 11:44:23 -0700287 bpf_verifier_vlog(&env->log, fmt, args);
Jiri Olsaabe08842018-03-23 11:41:28 +0100288 va_end(args);
289}
Alexei Starovoitovcbd35702014-09-26 00:17:03 -0700290
Martin KaFai Laud9762e82018-12-13 10:41:48 -0800291static const char *ltrim(const char *s)
292{
293 while (isspace(*s))
294 s++;
295
296 return s;
297}
298
299__printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env,
300 u32 insn_off,
301 const char *prefix_fmt, ...)
302{
303 const struct bpf_line_info *linfo;
304
305 if (!bpf_verifier_log_needed(&env->log))
306 return;
307
308 linfo = find_linfo(env, insn_off);
309 if (!linfo || linfo == env->prev_linfo)
310 return;
311
312 if (prefix_fmt) {
313 va_list args;
314
315 va_start(args, prefix_fmt);
316 bpf_verifier_vlog(&env->log, prefix_fmt, args);
317 va_end(args);
318 }
319
320 verbose(env, "%s\n",
321 ltrim(btf_name_by_offset(env->prog->aux->btf,
322 linfo->line_off)));
323
324 env->prev_linfo = linfo;
325}
326
Daniel Borkmannde8f3a82017-09-25 02:25:51 +0200327static bool type_is_pkt_pointer(enum bpf_reg_type type)
328{
329 return type == PTR_TO_PACKET ||
330 type == PTR_TO_PACKET_META;
331}
332
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -0800333static bool type_is_sk_pointer(enum bpf_reg_type type)
334{
335 return type == PTR_TO_SOCKET ||
Martin KaFai Lau655a51e2019-02-09 23:22:24 -0800336 type == PTR_TO_SOCK_COMMON ||
337 type == PTR_TO_TCP_SOCK;
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -0800338}
339
Joe Stringer840b9612018-10-02 13:35:32 -0700340static bool reg_type_may_be_null(enum bpf_reg_type type)
341{
Joe Stringerfd978bf72018-10-02 13:35:35 -0700342 return type == PTR_TO_MAP_VALUE_OR_NULL ||
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -0800343 type == PTR_TO_SOCKET_OR_NULL ||
Martin KaFai Lau655a51e2019-02-09 23:22:24 -0800344 type == PTR_TO_SOCK_COMMON_OR_NULL ||
345 type == PTR_TO_TCP_SOCK_OR_NULL;
Joe Stringerfd978bf72018-10-02 13:35:35 -0700346}
347
Alexei Starovoitovd83525c2019-01-31 15:40:04 -0800348static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg)
349{
350 return reg->type == PTR_TO_MAP_VALUE &&
351 map_value_has_spin_lock(reg->map_ptr);
352}
353
Martin KaFai Laucba368c2019-03-18 10:37:13 -0700354static bool reg_type_may_be_refcounted_or_null(enum bpf_reg_type type)
355{
356 return type == PTR_TO_SOCKET ||
357 type == PTR_TO_SOCKET_OR_NULL ||
358 type == PTR_TO_TCP_SOCK ||
359 type == PTR_TO_TCP_SOCK_OR_NULL;
360}
361
Martin KaFai Lau1b986582019-03-12 10:23:02 -0700362static bool arg_type_may_be_refcounted(enum bpf_arg_type type)
Joe Stringerfd978bf72018-10-02 13:35:35 -0700363{
Martin KaFai Lau1b986582019-03-12 10:23:02 -0700364 return type == ARG_PTR_TO_SOCK_COMMON;
Joe Stringerfd978bf72018-10-02 13:35:35 -0700365}
366
367/* Determine whether the function releases some resources allocated by another
368 * function call. The first reference type argument will be assumed to be
369 * released by release_reference().
370 */
371static bool is_release_function(enum bpf_func_id func_id)
372{
Joe Stringer6acc9b42018-10-02 13:35:36 -0700373 return func_id == BPF_FUNC_sk_release;
Joe Stringer840b9612018-10-02 13:35:32 -0700374}
375
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -0800376static bool is_acquire_function(enum bpf_func_id func_id)
377{
378 return func_id == BPF_FUNC_sk_lookup_tcp ||
Lorenz Baueredbf8c02019-03-22 09:54:01 +0800379 func_id == BPF_FUNC_sk_lookup_udp ||
380 func_id == BPF_FUNC_skc_lookup_tcp;
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -0800381}
382
Martin KaFai Lau1b986582019-03-12 10:23:02 -0700383static bool is_ptr_cast_function(enum bpf_func_id func_id)
384{
385 return func_id == BPF_FUNC_tcp_sock ||
386 func_id == BPF_FUNC_sk_fullsock;
387}
388
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700389/* string representation of 'enum bpf_reg_type' */
390static const char * const reg_type_str[] = {
391 [NOT_INIT] = "?",
Edward Creef1174f72017-08-07 15:26:19 +0100392 [SCALAR_VALUE] = "inv",
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700393 [PTR_TO_CTX] = "ctx",
394 [CONST_PTR_TO_MAP] = "map_ptr",
395 [PTR_TO_MAP_VALUE] = "map_value",
396 [PTR_TO_MAP_VALUE_OR_NULL] = "map_value_or_null",
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700397 [PTR_TO_STACK] = "fp",
Alexei Starovoitov969bf052016-05-05 19:49:10 -0700398 [PTR_TO_PACKET] = "pkt",
Daniel Borkmannde8f3a82017-09-25 02:25:51 +0200399 [PTR_TO_PACKET_META] = "pkt_meta",
Alexei Starovoitov969bf052016-05-05 19:49:10 -0700400 [PTR_TO_PACKET_END] = "pkt_end",
Petar Penkovd58e4682018-09-14 07:46:18 -0700401 [PTR_TO_FLOW_KEYS] = "flow_keys",
Joe Stringerc64b7982018-10-02 13:35:33 -0700402 [PTR_TO_SOCKET] = "sock",
403 [PTR_TO_SOCKET_OR_NULL] = "sock_or_null",
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -0800404 [PTR_TO_SOCK_COMMON] = "sock_common",
405 [PTR_TO_SOCK_COMMON_OR_NULL] = "sock_common_or_null",
Martin KaFai Lau655a51e2019-02-09 23:22:24 -0800406 [PTR_TO_TCP_SOCK] = "tcp_sock",
407 [PTR_TO_TCP_SOCK_OR_NULL] = "tcp_sock_or_null",
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700408};
409
Edward Cree8efea212018-08-22 20:02:44 +0100410static char slot_type_char[] = {
411 [STACK_INVALID] = '?',
412 [STACK_SPILL] = 'r',
413 [STACK_MISC] = 'm',
414 [STACK_ZERO] = '0',
415};
416
Alexei Starovoitov4e920242017-11-30 21:31:36 -0800417static void print_liveness(struct bpf_verifier_env *env,
418 enum bpf_reg_liveness live)
419{
Alexei Starovoitov9242b5f2018-12-13 11:42:34 -0800420 if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE))
Alexei Starovoitov4e920242017-11-30 21:31:36 -0800421 verbose(env, "_");
422 if (live & REG_LIVE_READ)
423 verbose(env, "r");
424 if (live & REG_LIVE_WRITTEN)
425 verbose(env, "w");
Alexei Starovoitov9242b5f2018-12-13 11:42:34 -0800426 if (live & REG_LIVE_DONE)
427 verbose(env, "D");
Alexei Starovoitov4e920242017-11-30 21:31:36 -0800428}
429
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800430static struct bpf_func_state *func(struct bpf_verifier_env *env,
431 const struct bpf_reg_state *reg)
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700432{
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800433 struct bpf_verifier_state *cur = env->cur_state;
434
435 return cur->frame[reg->frameno];
436}
437
438static void print_verifier_state(struct bpf_verifier_env *env,
439 const struct bpf_func_state *state)
440{
441 const struct bpf_reg_state *reg;
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700442 enum bpf_reg_type t;
443 int i;
444
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800445 if (state->frameno)
446 verbose(env, " frame%d:", state->frameno);
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700447 for (i = 0; i < MAX_BPF_REG; i++) {
Alexei Starovoitov1a0dc1a2016-05-05 19:49:09 -0700448 reg = &state->regs[i];
449 t = reg->type;
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700450 if (t == NOT_INIT)
451 continue;
Alexei Starovoitov4e920242017-11-30 21:31:36 -0800452 verbose(env, " R%d", i);
453 print_liveness(env, reg->live);
454 verbose(env, "=%s", reg_type_str[t]);
Edward Creef1174f72017-08-07 15:26:19 +0100455 if ((t == SCALAR_VALUE || t == PTR_TO_STACK) &&
456 tnum_is_const(reg->var_off)) {
457 /* reg->off should be 0 for SCALAR_VALUE */
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700458 verbose(env, "%lld", reg->var_off.value + reg->off);
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800459 if (t == PTR_TO_STACK)
460 verbose(env, ",call_%d", func(env, reg)->callsite);
Edward Creef1174f72017-08-07 15:26:19 +0100461 } else {
Martin KaFai Laucba368c2019-03-18 10:37:13 -0700462 verbose(env, "(id=%d", reg->id);
463 if (reg_type_may_be_refcounted_or_null(t))
464 verbose(env, ",ref_obj_id=%d", reg->ref_obj_id);
Edward Creef1174f72017-08-07 15:26:19 +0100465 if (t != SCALAR_VALUE)
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700466 verbose(env, ",off=%d", reg->off);
Daniel Borkmannde8f3a82017-09-25 02:25:51 +0200467 if (type_is_pkt_pointer(t))
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700468 verbose(env, ",r=%d", reg->range);
Edward Creef1174f72017-08-07 15:26:19 +0100469 else if (t == CONST_PTR_TO_MAP ||
470 t == PTR_TO_MAP_VALUE ||
471 t == PTR_TO_MAP_VALUE_OR_NULL)
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700472 verbose(env, ",ks=%d,vs=%d",
Edward Creef1174f72017-08-07 15:26:19 +0100473 reg->map_ptr->key_size,
474 reg->map_ptr->value_size);
Edward Cree7d1238f2017-08-07 15:26:56 +0100475 if (tnum_is_const(reg->var_off)) {
476 /* Typically an immediate SCALAR_VALUE, but
477 * could be a pointer whose offset is too big
478 * for reg->off
479 */
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700480 verbose(env, ",imm=%llx", reg->var_off.value);
Edward Cree7d1238f2017-08-07 15:26:56 +0100481 } else {
482 if (reg->smin_value != reg->umin_value &&
483 reg->smin_value != S64_MIN)
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700484 verbose(env, ",smin_value=%lld",
Edward Cree7d1238f2017-08-07 15:26:56 +0100485 (long long)reg->smin_value);
486 if (reg->smax_value != reg->umax_value &&
487 reg->smax_value != S64_MAX)
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700488 verbose(env, ",smax_value=%lld",
Edward Cree7d1238f2017-08-07 15:26:56 +0100489 (long long)reg->smax_value);
490 if (reg->umin_value != 0)
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700491 verbose(env, ",umin_value=%llu",
Edward Cree7d1238f2017-08-07 15:26:56 +0100492 (unsigned long long)reg->umin_value);
493 if (reg->umax_value != U64_MAX)
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700494 verbose(env, ",umax_value=%llu",
Edward Cree7d1238f2017-08-07 15:26:56 +0100495 (unsigned long long)reg->umax_value);
496 if (!tnum_is_unknown(reg->var_off)) {
497 char tn_buf[48];
Edward Creef1174f72017-08-07 15:26:19 +0100498
Edward Cree7d1238f2017-08-07 15:26:56 +0100499 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700500 verbose(env, ",var_off=%s", tn_buf);
Edward Cree7d1238f2017-08-07 15:26:56 +0100501 }
Edward Creef1174f72017-08-07 15:26:19 +0100502 }
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700503 verbose(env, ")");
Edward Creef1174f72017-08-07 15:26:19 +0100504 }
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700505 }
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700506 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
Edward Cree8efea212018-08-22 20:02:44 +0100507 char types_buf[BPF_REG_SIZE + 1];
508 bool valid = false;
509 int j;
510
511 for (j = 0; j < BPF_REG_SIZE; j++) {
512 if (state->stack[i].slot_type[j] != STACK_INVALID)
513 valid = true;
514 types_buf[j] = slot_type_char[
515 state->stack[i].slot_type[j]];
516 }
517 types_buf[BPF_REG_SIZE] = 0;
518 if (!valid)
519 continue;
520 verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
521 print_liveness(env, state->stack[i].spilled_ptr.live);
522 if (state->stack[i].slot_type[0] == STACK_SPILL)
Alexei Starovoitov4e920242017-11-30 21:31:36 -0800523 verbose(env, "=%s",
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700524 reg_type_str[state->stack[i].spilled_ptr.type]);
Edward Cree8efea212018-08-22 20:02:44 +0100525 else
526 verbose(env, "=%s", types_buf);
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700527 }
Joe Stringerfd978bf72018-10-02 13:35:35 -0700528 if (state->acquired_refs && state->refs[0].id) {
529 verbose(env, " refs=%d", state->refs[0].id);
530 for (i = 1; i < state->acquired_refs; i++)
531 if (state->refs[i].id)
532 verbose(env, ",%d", state->refs[i].id);
533 }
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700534 verbose(env, "\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700535}
536
Joe Stringer84dbf352018-10-02 13:35:34 -0700537#define COPY_STATE_FN(NAME, COUNT, FIELD, SIZE) \
538static int copy_##NAME##_state(struct bpf_func_state *dst, \
539 const struct bpf_func_state *src) \
540{ \
541 if (!src->FIELD) \
542 return 0; \
543 if (WARN_ON_ONCE(dst->COUNT < src->COUNT)) { \
544 /* internal bug, make state invalid to reject the program */ \
545 memset(dst, 0, sizeof(*dst)); \
546 return -EFAULT; \
547 } \
548 memcpy(dst->FIELD, src->FIELD, \
549 sizeof(*src->FIELD) * (src->COUNT / SIZE)); \
550 return 0; \
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700551}
Joe Stringerfd978bf72018-10-02 13:35:35 -0700552/* copy_reference_state() */
553COPY_STATE_FN(reference, acquired_refs, refs, 1)
Joe Stringer84dbf352018-10-02 13:35:34 -0700554/* copy_stack_state() */
555COPY_STATE_FN(stack, allocated_stack, stack, BPF_REG_SIZE)
556#undef COPY_STATE_FN
557
558#define REALLOC_STATE_FN(NAME, COUNT, FIELD, SIZE) \
559static int realloc_##NAME##_state(struct bpf_func_state *state, int size, \
560 bool copy_old) \
561{ \
562 u32 old_size = state->COUNT; \
563 struct bpf_##NAME##_state *new_##FIELD; \
564 int slot = size / SIZE; \
565 \
566 if (size <= old_size || !size) { \
567 if (copy_old) \
568 return 0; \
569 state->COUNT = slot * SIZE; \
570 if (!size && old_size) { \
571 kfree(state->FIELD); \
572 state->FIELD = NULL; \
573 } \
574 return 0; \
575 } \
576 new_##FIELD = kmalloc_array(slot, sizeof(struct bpf_##NAME##_state), \
577 GFP_KERNEL); \
578 if (!new_##FIELD) \
579 return -ENOMEM; \
580 if (copy_old) { \
581 if (state->FIELD) \
582 memcpy(new_##FIELD, state->FIELD, \
583 sizeof(*new_##FIELD) * (old_size / SIZE)); \
584 memset(new_##FIELD + old_size / SIZE, 0, \
585 sizeof(*new_##FIELD) * (size - old_size) / SIZE); \
586 } \
587 state->COUNT = slot * SIZE; \
588 kfree(state->FIELD); \
589 state->FIELD = new_##FIELD; \
590 return 0; \
591}
Joe Stringerfd978bf72018-10-02 13:35:35 -0700592/* realloc_reference_state() */
593REALLOC_STATE_FN(reference, acquired_refs, refs, 1)
Joe Stringer84dbf352018-10-02 13:35:34 -0700594/* realloc_stack_state() */
595REALLOC_STATE_FN(stack, allocated_stack, stack, BPF_REG_SIZE)
596#undef REALLOC_STATE_FN
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700597
598/* do_check() starts with zero-sized stack in struct bpf_verifier_state to
599 * make it consume minimal amount of memory. check_stack_write() access from
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800600 * the program calls into realloc_func_state() to grow the stack size.
Joe Stringer84dbf352018-10-02 13:35:34 -0700601 * Note there is a non-zero 'parent' pointer inside bpf_verifier_state
602 * which realloc_stack_state() copies over. It points to previous
603 * bpf_verifier_state which is never reallocated.
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700604 */
Joe Stringerfd978bf72018-10-02 13:35:35 -0700605static int realloc_func_state(struct bpf_func_state *state, int stack_size,
606 int refs_size, bool copy_old)
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700607{
Joe Stringerfd978bf72018-10-02 13:35:35 -0700608 int err = realloc_reference_state(state, refs_size, copy_old);
609 if (err)
610 return err;
611 return realloc_stack_state(state, stack_size, copy_old);
612}
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700613
Joe Stringerfd978bf72018-10-02 13:35:35 -0700614/* Acquire a pointer id from the env and update the state->refs to include
615 * this new pointer reference.
616 * On success, returns a valid pointer id to associate with the register
617 * On failure, returns a negative errno.
618 */
619static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx)
620{
621 struct bpf_func_state *state = cur_func(env);
622 int new_ofs = state->acquired_refs;
623 int id, err;
624
625 err = realloc_reference_state(state, state->acquired_refs + 1, true);
626 if (err)
627 return err;
628 id = ++env->id_gen;
629 state->refs[new_ofs].id = id;
630 state->refs[new_ofs].insn_idx = insn_idx;
631
632 return id;
633}
634
635/* release function corresponding to acquire_reference_state(). Idempotent. */
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -0800636static int release_reference_state(struct bpf_func_state *state, int ptr_id)
Joe Stringerfd978bf72018-10-02 13:35:35 -0700637{
638 int i, last_idx;
639
Joe Stringerfd978bf72018-10-02 13:35:35 -0700640 last_idx = state->acquired_refs - 1;
641 for (i = 0; i < state->acquired_refs; i++) {
642 if (state->refs[i].id == ptr_id) {
643 if (last_idx && i != last_idx)
644 memcpy(&state->refs[i], &state->refs[last_idx],
645 sizeof(*state->refs));
646 memset(&state->refs[last_idx], 0, sizeof(*state->refs));
647 state->acquired_refs--;
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700648 return 0;
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700649 }
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700650 }
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -0800651 return -EINVAL;
Joe Stringerfd978bf72018-10-02 13:35:35 -0700652}
653
654static int transfer_reference_state(struct bpf_func_state *dst,
655 struct bpf_func_state *src)
656{
657 int err = realloc_reference_state(dst, src->acquired_refs, false);
658 if (err)
659 return err;
660 err = copy_reference_state(dst, src);
661 if (err)
662 return err;
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700663 return 0;
664}
665
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800666static void free_func_state(struct bpf_func_state *state)
667{
Alexei Starovoitov58963512018-01-08 07:51:17 -0800668 if (!state)
669 return;
Joe Stringerfd978bf72018-10-02 13:35:35 -0700670 kfree(state->refs);
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800671 kfree(state->stack);
672 kfree(state);
673}
674
Alexei Starovoitov1969db42017-11-01 00:08:04 -0700675static void free_verifier_state(struct bpf_verifier_state *state,
676 bool free_self)
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700677{
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800678 int i;
679
680 for (i = 0; i <= state->curframe; i++) {
681 free_func_state(state->frame[i]);
682 state->frame[i] = NULL;
683 }
Alexei Starovoitov1969db42017-11-01 00:08:04 -0700684 if (free_self)
685 kfree(state);
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700686}
687
688/* copy verifier state from src to dst growing dst stack space
689 * when necessary to accommodate larger src stack
690 */
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800691static int copy_func_state(struct bpf_func_state *dst,
692 const struct bpf_func_state *src)
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700693{
694 int err;
695
Joe Stringerfd978bf72018-10-02 13:35:35 -0700696 err = realloc_func_state(dst, src->allocated_stack, src->acquired_refs,
697 false);
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700698 if (err)
699 return err;
Joe Stringerfd978bf72018-10-02 13:35:35 -0700700 memcpy(dst, src, offsetof(struct bpf_func_state, acquired_refs));
701 err = copy_reference_state(dst, src);
702 if (err)
703 return err;
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700704 return copy_stack_state(dst, src);
705}
706
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800707static int copy_verifier_state(struct bpf_verifier_state *dst_state,
708 const struct bpf_verifier_state *src)
709{
710 struct bpf_func_state *dst;
711 int i, err;
712
713 /* if dst has more stack frames then src frame, free them */
714 for (i = src->curframe + 1; i <= dst_state->curframe; i++) {
715 free_func_state(dst_state->frame[i]);
716 dst_state->frame[i] = NULL;
717 }
Daniel Borkmann979d63d2019-01-03 00:58:34 +0100718 dst_state->speculative = src->speculative;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800719 dst_state->curframe = src->curframe;
Alexei Starovoitovd83525c2019-01-31 15:40:04 -0800720 dst_state->active_spin_lock = src->active_spin_lock;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800721 for (i = 0; i <= src->curframe; i++) {
722 dst = dst_state->frame[i];
723 if (!dst) {
724 dst = kzalloc(sizeof(*dst), GFP_KERNEL);
725 if (!dst)
726 return -ENOMEM;
727 dst_state->frame[i] = dst;
728 }
729 err = copy_func_state(dst, src->frame[i]);
730 if (err)
731 return err;
732 }
733 return 0;
734}
735
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700736static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx,
737 int *insn_idx)
738{
739 struct bpf_verifier_state *cur = env->cur_state;
740 struct bpf_verifier_stack_elem *elem, *head = env->head;
741 int err;
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700742
743 if (env->head == NULL)
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700744 return -ENOENT;
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700745
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700746 if (cur) {
747 err = copy_verifier_state(cur, &head->st);
748 if (err)
749 return err;
750 }
751 if (insn_idx)
752 *insn_idx = head->insn_idx;
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700753 if (prev_insn_idx)
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700754 *prev_insn_idx = head->prev_insn_idx;
755 elem = head->next;
Alexei Starovoitov1969db42017-11-01 00:08:04 -0700756 free_verifier_state(&head->st, false);
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700757 kfree(head);
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700758 env->head = elem;
759 env->stack_size--;
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700760 return 0;
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700761}
762
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +0100763static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env,
Daniel Borkmann979d63d2019-01-03 00:58:34 +0100764 int insn_idx, int prev_insn_idx,
765 bool speculative)
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700766{
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700767 struct bpf_verifier_state *cur = env->cur_state;
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +0100768 struct bpf_verifier_stack_elem *elem;
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700769 int err;
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700770
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700771 elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL);
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700772 if (!elem)
773 goto err;
774
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700775 elem->insn_idx = insn_idx;
776 elem->prev_insn_idx = prev_insn_idx;
777 elem->next = env->head;
778 env->head = elem;
779 env->stack_size++;
Alexei Starovoitov1969db42017-11-01 00:08:04 -0700780 err = copy_verifier_state(&elem->st, cur);
781 if (err)
782 goto err;
Daniel Borkmann979d63d2019-01-03 00:58:34 +0100783 elem->st.speculative |= speculative;
Daniel Borkmann07016152016-04-05 22:33:17 +0200784 if (env->stack_size > BPF_COMPLEXITY_LIMIT_STACK) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700785 verbose(env, "BPF program is too complex\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700786 goto err;
787 }
788 return &elem->st;
789err:
Alexei Starovoitov58963512018-01-08 07:51:17 -0800790 free_verifier_state(env->cur_state, true);
791 env->cur_state = NULL;
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700792 /* pop all elements and return */
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700793 while (!pop_stack(env, NULL, NULL));
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700794 return NULL;
795}
796
797#define CALLER_SAVED_REGS 6
798static const int caller_saved[CALLER_SAVED_REGS] = {
799 BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5
800};
801
Edward Creef1174f72017-08-07 15:26:19 +0100802static void __mark_reg_not_init(struct bpf_reg_state *reg);
803
Edward Creeb03c9f92017-08-07 15:26:36 +0100804/* Mark the unknown part of a register (variable offset or scalar value) as
805 * known to have the value @imm.
806 */
807static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm)
808{
Alexei Starovoitova9c676b2018-09-04 19:13:44 -0700809 /* Clear id, off, and union(map_ptr, range) */
810 memset(((u8 *)reg) + sizeof(reg->type), 0,
811 offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type));
Edward Creeb03c9f92017-08-07 15:26:36 +0100812 reg->var_off = tnum_const(imm);
813 reg->smin_value = (s64)imm;
814 reg->smax_value = (s64)imm;
815 reg->umin_value = imm;
816 reg->umax_value = imm;
817}
818
Edward Creef1174f72017-08-07 15:26:19 +0100819/* Mark the 'variable offset' part of a register as zero. This should be
820 * used only on registers holding a pointer type.
821 */
822static void __mark_reg_known_zero(struct bpf_reg_state *reg)
823{
Edward Creeb03c9f92017-08-07 15:26:36 +0100824 __mark_reg_known(reg, 0);
Edward Creef1174f72017-08-07 15:26:19 +0100825}
826
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -0800827static void __mark_reg_const_zero(struct bpf_reg_state *reg)
828{
829 __mark_reg_known(reg, 0);
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -0800830 reg->type = SCALAR_VALUE;
831}
832
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700833static void mark_reg_known_zero(struct bpf_verifier_env *env,
834 struct bpf_reg_state *regs, u32 regno)
Edward Creef1174f72017-08-07 15:26:19 +0100835{
836 if (WARN_ON(regno >= MAX_BPF_REG)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700837 verbose(env, "mark_reg_known_zero(regs, %u)\n", regno);
Edward Creef1174f72017-08-07 15:26:19 +0100838 /* Something bad happened, let's kill all regs */
839 for (regno = 0; regno < MAX_BPF_REG; regno++)
840 __mark_reg_not_init(regs + regno);
841 return;
842 }
843 __mark_reg_known_zero(regs + regno);
844}
845
Daniel Borkmannde8f3a82017-09-25 02:25:51 +0200846static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)
847{
848 return type_is_pkt_pointer(reg->type);
849}
850
851static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg)
852{
853 return reg_is_pkt_pointer(reg) ||
854 reg->type == PTR_TO_PACKET_END;
855}
856
857/* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */
858static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg,
859 enum bpf_reg_type which)
860{
861 /* The register can already have a range from prior markings.
862 * This is fine as long as it hasn't been advanced from its
863 * origin.
864 */
865 return reg->type == which &&
866 reg->id == 0 &&
867 reg->off == 0 &&
868 tnum_equals_const(reg->var_off, 0);
869}
870
Edward Creeb03c9f92017-08-07 15:26:36 +0100871/* Attempts to improve min/max values based on var_off information */
872static void __update_reg_bounds(struct bpf_reg_state *reg)
873{
874 /* min signed is max(sign bit) | min(other bits) */
875 reg->smin_value = max_t(s64, reg->smin_value,
876 reg->var_off.value | (reg->var_off.mask & S64_MIN));
877 /* max signed is min(sign bit) | max(other bits) */
878 reg->smax_value = min_t(s64, reg->smax_value,
879 reg->var_off.value | (reg->var_off.mask & S64_MAX));
880 reg->umin_value = max(reg->umin_value, reg->var_off.value);
881 reg->umax_value = min(reg->umax_value,
882 reg->var_off.value | reg->var_off.mask);
883}
884
885/* Uses signed min/max values to inform unsigned, and vice-versa */
886static void __reg_deduce_bounds(struct bpf_reg_state *reg)
887{
888 /* Learn sign from signed bounds.
889 * If we cannot cross the sign boundary, then signed and unsigned bounds
890 * are the same, so combine. This works even in the negative case, e.g.
891 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff.
892 */
893 if (reg->smin_value >= 0 || reg->smax_value < 0) {
894 reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value,
895 reg->umin_value);
896 reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value,
897 reg->umax_value);
898 return;
899 }
900 /* Learn sign from unsigned bounds. Signed bounds cross the sign
901 * boundary, so we must be careful.
902 */
903 if ((s64)reg->umax_value >= 0) {
904 /* Positive. We can't learn anything from the smin, but smax
905 * is positive, hence safe.
906 */
907 reg->smin_value = reg->umin_value;
908 reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value,
909 reg->umax_value);
910 } else if ((s64)reg->umin_value < 0) {
911 /* Negative. We can't learn anything from the smax, but smin
912 * is negative, hence safe.
913 */
914 reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value,
915 reg->umin_value);
916 reg->smax_value = reg->umax_value;
917 }
918}
919
920/* Attempts to improve var_off based on unsigned min/max information */
921static void __reg_bound_offset(struct bpf_reg_state *reg)
922{
923 reg->var_off = tnum_intersect(reg->var_off,
924 tnum_range(reg->umin_value,
925 reg->umax_value));
926}
927
928/* Reset the min/max bounds of a register */
929static void __mark_reg_unbounded(struct bpf_reg_state *reg)
930{
931 reg->smin_value = S64_MIN;
932 reg->smax_value = S64_MAX;
933 reg->umin_value = 0;
934 reg->umax_value = U64_MAX;
935}
936
Edward Creef1174f72017-08-07 15:26:19 +0100937/* Mark a register as having a completely unknown (scalar) value. */
938static void __mark_reg_unknown(struct bpf_reg_state *reg)
939{
Alexei Starovoitova9c676b2018-09-04 19:13:44 -0700940 /*
941 * Clear type, id, off, and union(map_ptr, range) and
942 * padding between 'type' and union
943 */
944 memset(reg, 0, offsetof(struct bpf_reg_state, var_off));
Edward Creef1174f72017-08-07 15:26:19 +0100945 reg->type = SCALAR_VALUE;
Edward Creef1174f72017-08-07 15:26:19 +0100946 reg->var_off = tnum_unknown;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800947 reg->frameno = 0;
Edward Creeb03c9f92017-08-07 15:26:36 +0100948 __mark_reg_unbounded(reg);
Edward Creef1174f72017-08-07 15:26:19 +0100949}
950
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700951static void mark_reg_unknown(struct bpf_verifier_env *env,
952 struct bpf_reg_state *regs, u32 regno)
Edward Creef1174f72017-08-07 15:26:19 +0100953{
954 if (WARN_ON(regno >= MAX_BPF_REG)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700955 verbose(env, "mark_reg_unknown(regs, %u)\n", regno);
Alexei Starovoitov19ceb412017-11-30 21:31:37 -0800956 /* Something bad happened, let's kill all regs except FP */
957 for (regno = 0; regno < BPF_REG_FP; regno++)
Edward Creef1174f72017-08-07 15:26:19 +0100958 __mark_reg_not_init(regs + regno);
959 return;
960 }
961 __mark_reg_unknown(regs + regno);
962}
963
964static void __mark_reg_not_init(struct bpf_reg_state *reg)
965{
966 __mark_reg_unknown(reg);
967 reg->type = NOT_INIT;
968}
969
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700970static void mark_reg_not_init(struct bpf_verifier_env *env,
971 struct bpf_reg_state *regs, u32 regno)
Daniel Borkmanna9789ef2017-05-25 01:05:06 +0200972{
Edward Creef1174f72017-08-07 15:26:19 +0100973 if (WARN_ON(regno >= MAX_BPF_REG)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700974 verbose(env, "mark_reg_not_init(regs, %u)\n", regno);
Alexei Starovoitov19ceb412017-11-30 21:31:37 -0800975 /* Something bad happened, let's kill all regs except FP */
976 for (regno = 0; regno < BPF_REG_FP; regno++)
Edward Creef1174f72017-08-07 15:26:19 +0100977 __mark_reg_not_init(regs + regno);
978 return;
979 }
980 __mark_reg_not_init(regs + regno);
Daniel Borkmanna9789ef2017-05-25 01:05:06 +0200981}
982
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700983static void init_reg_state(struct bpf_verifier_env *env,
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800984 struct bpf_func_state *state)
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700985{
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800986 struct bpf_reg_state *regs = state->regs;
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700987 int i;
988
Edward Creedc503a82017-08-15 20:34:35 +0100989 for (i = 0; i < MAX_BPF_REG; i++) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700990 mark_reg_not_init(env, regs, i);
Edward Creedc503a82017-08-15 20:34:35 +0100991 regs[i].live = REG_LIVE_NONE;
Edward Cree679c7822018-08-22 20:02:19 +0100992 regs[i].parent = NULL;
Edward Creedc503a82017-08-15 20:34:35 +0100993 }
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700994
995 /* frame pointer */
Edward Creef1174f72017-08-07 15:26:19 +0100996 regs[BPF_REG_FP].type = PTR_TO_STACK;
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700997 mark_reg_known_zero(env, regs, BPF_REG_FP);
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800998 regs[BPF_REG_FP].frameno = state->frameno;
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700999
1000 /* 1st arg to a function */
1001 regs[BPF_REG_1].type = PTR_TO_CTX;
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001002 mark_reg_known_zero(env, regs, BPF_REG_1);
Daniel Borkmann6760bf22016-12-18 01:52:59 +01001003}
1004
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001005#define BPF_MAIN_FUNC (-1)
1006static void init_func_state(struct bpf_verifier_env *env,
1007 struct bpf_func_state *state,
1008 int callsite, int frameno, int subprogno)
1009{
1010 state->callsite = callsite;
1011 state->frameno = frameno;
1012 state->subprogno = subprogno;
1013 init_reg_state(env, state);
1014}
1015
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001016enum reg_arg_type {
1017 SRC_OP, /* register is used as source operand */
1018 DST_OP, /* register is used as destination operand */
1019 DST_OP_NO_MARK /* same as above, check only, don't mark */
1020};
1021
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001022static int cmp_subprogs(const void *a, const void *b)
1023{
Jiong Wang9c8105b2018-05-02 16:17:18 -04001024 return ((struct bpf_subprog_info *)a)->start -
1025 ((struct bpf_subprog_info *)b)->start;
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001026}
1027
1028static int find_subprog(struct bpf_verifier_env *env, int off)
1029{
Jiong Wang9c8105b2018-05-02 16:17:18 -04001030 struct bpf_subprog_info *p;
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001031
Jiong Wang9c8105b2018-05-02 16:17:18 -04001032 p = bsearch(&off, env->subprog_info, env->subprog_cnt,
1033 sizeof(env->subprog_info[0]), cmp_subprogs);
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001034 if (!p)
1035 return -ENOENT;
Jiong Wang9c8105b2018-05-02 16:17:18 -04001036 return p - env->subprog_info;
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001037
1038}
1039
1040static int add_subprog(struct bpf_verifier_env *env, int off)
1041{
1042 int insn_cnt = env->prog->len;
1043 int ret;
1044
1045 if (off >= insn_cnt || off < 0) {
1046 verbose(env, "call to invalid destination\n");
1047 return -EINVAL;
1048 }
1049 ret = find_subprog(env, off);
1050 if (ret >= 0)
1051 return 0;
Jiong Wang4cb3d992018-05-02 16:17:19 -04001052 if (env->subprog_cnt >= BPF_MAX_SUBPROGS) {
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001053 verbose(env, "too many subprograms\n");
1054 return -E2BIG;
1055 }
Jiong Wang9c8105b2018-05-02 16:17:18 -04001056 env->subprog_info[env->subprog_cnt++].start = off;
1057 sort(env->subprog_info, env->subprog_cnt,
1058 sizeof(env->subprog_info[0]), cmp_subprogs, NULL);
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001059 return 0;
1060}
1061
1062static int check_subprogs(struct bpf_verifier_env *env)
1063{
1064 int i, ret, subprog_start, subprog_end, off, cur_subprog = 0;
Jiong Wang9c8105b2018-05-02 16:17:18 -04001065 struct bpf_subprog_info *subprog = env->subprog_info;
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001066 struct bpf_insn *insn = env->prog->insnsi;
1067 int insn_cnt = env->prog->len;
1068
Jiong Wangf910cef2018-05-02 16:17:17 -04001069 /* Add entry function. */
1070 ret = add_subprog(env, 0);
1071 if (ret < 0)
1072 return ret;
1073
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001074 /* determine subprog starts. The end is one before the next starts */
1075 for (i = 0; i < insn_cnt; i++) {
1076 if (insn[i].code != (BPF_JMP | BPF_CALL))
1077 continue;
1078 if (insn[i].src_reg != BPF_PSEUDO_CALL)
1079 continue;
1080 if (!env->allow_ptr_leaks) {
1081 verbose(env, "function calls to other bpf functions are allowed for root only\n");
1082 return -EPERM;
1083 }
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001084 ret = add_subprog(env, i + insn[i].imm + 1);
1085 if (ret < 0)
1086 return ret;
1087 }
1088
Jiong Wang4cb3d992018-05-02 16:17:19 -04001089 /* Add a fake 'exit' subprog which could simplify subprog iteration
1090 * logic. 'subprog_cnt' should not be increased.
1091 */
1092 subprog[env->subprog_cnt].start = insn_cnt;
1093
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07001094 if (env->log.level & BPF_LOG_LEVEL2)
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001095 for (i = 0; i < env->subprog_cnt; i++)
Jiong Wang9c8105b2018-05-02 16:17:18 -04001096 verbose(env, "func#%d @%d\n", i, subprog[i].start);
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001097
1098 /* now check that all jumps are within the same subprog */
Jiong Wang4cb3d992018-05-02 16:17:19 -04001099 subprog_start = subprog[cur_subprog].start;
1100 subprog_end = subprog[cur_subprog + 1].start;
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001101 for (i = 0; i < insn_cnt; i++) {
1102 u8 code = insn[i].code;
1103
Jiong Wang092ed092019-01-26 12:26:01 -05001104 if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32)
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001105 goto next;
1106 if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL)
1107 goto next;
1108 off = i + insn[i].off + 1;
1109 if (off < subprog_start || off >= subprog_end) {
1110 verbose(env, "jump out of range from insn %d to %d\n", i, off);
1111 return -EINVAL;
1112 }
1113next:
1114 if (i == subprog_end - 1) {
1115 /* to avoid fall-through from one subprog into another
1116 * the last insn of the subprog should be either exit
1117 * or unconditional jump back
1118 */
1119 if (code != (BPF_JMP | BPF_EXIT) &&
1120 code != (BPF_JMP | BPF_JA)) {
1121 verbose(env, "last insn is not an exit or jmp\n");
1122 return -EINVAL;
1123 }
1124 subprog_start = subprog_end;
Jiong Wang4cb3d992018-05-02 16:17:19 -04001125 cur_subprog++;
1126 if (cur_subprog < env->subprog_cnt)
Jiong Wang9c8105b2018-05-02 16:17:18 -04001127 subprog_end = subprog[cur_subprog + 1].start;
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001128 }
1129 }
1130 return 0;
1131}
1132
Edward Cree679c7822018-08-22 20:02:19 +01001133/* Parentage chain of this register (or stack slot) should take care of all
1134 * issues like callee-saved registers, stack slot allocation time, etc.
1135 */
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001136static int mark_reg_read(struct bpf_verifier_env *env,
Edward Cree679c7822018-08-22 20:02:19 +01001137 const struct bpf_reg_state *state,
1138 struct bpf_reg_state *parent)
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001139{
1140 bool writes = parent == state->parent; /* Observe write marks */
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07001141 int cnt = 0;
Edward Creedc503a82017-08-15 20:34:35 +01001142
1143 while (parent) {
1144 /* if read wasn't screened by an earlier write ... */
Edward Cree679c7822018-08-22 20:02:19 +01001145 if (writes && state->live & REG_LIVE_WRITTEN)
Edward Creedc503a82017-08-15 20:34:35 +01001146 break;
Alexei Starovoitov9242b5f2018-12-13 11:42:34 -08001147 if (parent->live & REG_LIVE_DONE) {
1148 verbose(env, "verifier BUG type %s var_off %lld off %d\n",
1149 reg_type_str[parent->type],
1150 parent->var_off.value, parent->off);
1151 return -EFAULT;
1152 }
Alexei Starovoitov25af32d2019-04-01 21:27:42 -07001153 if (parent->live & REG_LIVE_READ)
1154 /* The parentage chain never changes and
1155 * this parent was already marked as LIVE_READ.
1156 * There is no need to keep walking the chain again and
1157 * keep re-marking all parents as LIVE_READ.
1158 * This case happens when the same register is read
1159 * multiple times without writes into it in-between.
1160 */
1161 break;
Edward Creedc503a82017-08-15 20:34:35 +01001162 /* ... then we depend on parent's value */
Edward Cree679c7822018-08-22 20:02:19 +01001163 parent->live |= REG_LIVE_READ;
Edward Creedc503a82017-08-15 20:34:35 +01001164 state = parent;
1165 parent = state->parent;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001166 writes = true;
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07001167 cnt++;
Edward Creedc503a82017-08-15 20:34:35 +01001168 }
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07001169
1170 if (env->longest_mark_read_walk < cnt)
1171 env->longest_mark_read_walk = cnt;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001172 return 0;
Edward Creedc503a82017-08-15 20:34:35 +01001173}
1174
1175static int check_reg_arg(struct bpf_verifier_env *env, u32 regno,
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001176 enum reg_arg_type t)
1177{
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001178 struct bpf_verifier_state *vstate = env->cur_state;
1179 struct bpf_func_state *state = vstate->frame[vstate->curframe];
1180 struct bpf_reg_state *regs = state->regs;
Edward Creedc503a82017-08-15 20:34:35 +01001181
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001182 if (regno >= MAX_BPF_REG) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001183 verbose(env, "R%d is invalid\n", regno);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001184 return -EINVAL;
1185 }
1186
1187 if (t == SRC_OP) {
1188 /* check whether register used as source operand can be read */
1189 if (regs[regno].type == NOT_INIT) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001190 verbose(env, "R%d !read_ok\n", regno);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001191 return -EACCES;
1192 }
Edward Cree679c7822018-08-22 20:02:19 +01001193 /* We don't need to worry about FP liveness because it's read-only */
1194 if (regno != BPF_REG_FP)
1195 return mark_reg_read(env, &regs[regno],
1196 regs[regno].parent);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001197 } else {
1198 /* check whether register used as dest operand can be written to */
1199 if (regno == BPF_REG_FP) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001200 verbose(env, "frame pointer is read only\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001201 return -EACCES;
1202 }
Edward Creedc503a82017-08-15 20:34:35 +01001203 regs[regno].live |= REG_LIVE_WRITTEN;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001204 if (t == DST_OP)
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001205 mark_reg_unknown(env, regs, regno);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001206 }
1207 return 0;
1208}
1209
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07001210static bool is_spillable_regtype(enum bpf_reg_type type)
1211{
1212 switch (type) {
1213 case PTR_TO_MAP_VALUE:
1214 case PTR_TO_MAP_VALUE_OR_NULL:
1215 case PTR_TO_STACK:
1216 case PTR_TO_CTX:
Alexei Starovoitov969bf052016-05-05 19:49:10 -07001217 case PTR_TO_PACKET:
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02001218 case PTR_TO_PACKET_META:
Alexei Starovoitov969bf052016-05-05 19:49:10 -07001219 case PTR_TO_PACKET_END:
Petar Penkovd58e4682018-09-14 07:46:18 -07001220 case PTR_TO_FLOW_KEYS:
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07001221 case CONST_PTR_TO_MAP:
Joe Stringerc64b7982018-10-02 13:35:33 -07001222 case PTR_TO_SOCKET:
1223 case PTR_TO_SOCKET_OR_NULL:
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08001224 case PTR_TO_SOCK_COMMON:
1225 case PTR_TO_SOCK_COMMON_OR_NULL:
Martin KaFai Lau655a51e2019-02-09 23:22:24 -08001226 case PTR_TO_TCP_SOCK:
1227 case PTR_TO_TCP_SOCK_OR_NULL:
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07001228 return true;
1229 default:
1230 return false;
1231 }
1232}
1233
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08001234/* Does this register contain a constant zero? */
1235static bool register_is_null(struct bpf_reg_state *reg)
1236{
1237 return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0);
1238}
1239
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001240/* check_stack_read/write functions track spill/fill of registers,
1241 * stack boundary and alignment are checked in check_mem_access()
1242 */
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001243static int check_stack_write(struct bpf_verifier_env *env,
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001244 struct bpf_func_state *state, /* func where register points to */
Alexei Starovoitovaf86ca42018-05-15 09:27:05 -07001245 int off, int size, int value_regno, int insn_idx)
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001246{
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001247 struct bpf_func_state *cur; /* state of the current function */
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001248 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001249 enum bpf_reg_type type;
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001250
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001251 err = realloc_func_state(state, round_up(slot + 1, BPF_REG_SIZE),
Joe Stringerfd978bf72018-10-02 13:35:35 -07001252 state->acquired_refs, true);
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001253 if (err)
1254 return err;
Alexei Starovoitov9c3997602014-10-28 15:11:41 -07001255 /* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0,
1256 * so it's aligned access and [off, off + size) are within stack limits
1257 */
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001258 if (!env->allow_ptr_leaks &&
1259 state->stack[spi].slot_type[0] == STACK_SPILL &&
1260 size != BPF_REG_SIZE) {
1261 verbose(env, "attempt to corrupt spilled pointer on stack\n");
1262 return -EACCES;
1263 }
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001264
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001265 cur = env->cur_state->frame[env->cur_state->curframe];
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001266 if (value_regno >= 0 &&
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001267 is_spillable_regtype((type = cur->regs[value_regno].type))) {
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001268
1269 /* register containing pointer is being spilled into stack */
Alexei Starovoitov9c3997602014-10-28 15:11:41 -07001270 if (size != BPF_REG_SIZE) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001271 verbose(env, "invalid size of register spill\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001272 return -EACCES;
1273 }
1274
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001275 if (state != cur && type == PTR_TO_STACK) {
1276 verbose(env, "cannot spill pointers to stack into stack frame of the caller\n");
1277 return -EINVAL;
1278 }
1279
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001280 /* save register state */
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001281 state->stack[spi].spilled_ptr = cur->regs[value_regno];
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001282 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001283
Alexei Starovoitovaf86ca42018-05-15 09:27:05 -07001284 for (i = 0; i < BPF_REG_SIZE; i++) {
1285 if (state->stack[spi].slot_type[i] == STACK_MISC &&
1286 !env->allow_ptr_leaks) {
1287 int *poff = &env->insn_aux_data[insn_idx].sanitize_stack_off;
1288 int soff = (-spi - 1) * BPF_REG_SIZE;
1289
1290 /* detected reuse of integer stack slot with a pointer
1291 * which means either llvm is reusing stack slot or
1292 * an attacker is trying to exploit CVE-2018-3639
1293 * (speculative store bypass)
1294 * Have to sanitize that slot with preemptive
1295 * store of zero.
1296 */
1297 if (*poff && *poff != soff) {
1298 /* disallow programs where single insn stores
1299 * into two different stack slots, since verifier
1300 * cannot sanitize them
1301 */
1302 verbose(env,
1303 "insn %d cannot access two stack slots fp%d and fp%d",
1304 insn_idx, *poff, soff);
1305 return -EINVAL;
1306 }
1307 *poff = soff;
1308 }
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001309 state->stack[spi].slot_type[i] = STACK_SPILL;
Alexei Starovoitovaf86ca42018-05-15 09:27:05 -07001310 }
Alexei Starovoitov9c3997602014-10-28 15:11:41 -07001311 } else {
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08001312 u8 type = STACK_MISC;
1313
Edward Cree679c7822018-08-22 20:02:19 +01001314 /* regular write of data into stack destroys any spilled ptr */
1315 state->stack[spi].spilled_ptr.type = NOT_INIT;
Jiong Wang0bae2d42018-12-15 03:34:40 -05001316 /* Mark slots as STACK_MISC if they belonged to spilled ptr. */
1317 if (state->stack[spi].slot_type[0] == STACK_SPILL)
1318 for (i = 0; i < BPF_REG_SIZE; i++)
1319 state->stack[spi].slot_type[i] = STACK_MISC;
Alexei Starovoitov9c3997602014-10-28 15:11:41 -07001320
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08001321 /* only mark the slot as written if all 8 bytes were written
1322 * otherwise read propagation may incorrectly stop too soon
1323 * when stack slots are partially written.
1324 * This heuristic means that read propagation will be
1325 * conservative, since it will add reg_live_read marks
1326 * to stack slots all the way to first state when programs
1327 * writes+reads less than 8 bytes
1328 */
1329 if (size == BPF_REG_SIZE)
1330 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
1331
1332 /* when we zero initialize stack slots mark them as such */
1333 if (value_regno >= 0 &&
1334 register_is_null(&cur->regs[value_regno]))
1335 type = STACK_ZERO;
1336
Jiong Wang0bae2d42018-12-15 03:34:40 -05001337 /* Mark slots affected by this stack write. */
Alexei Starovoitov9c3997602014-10-28 15:11:41 -07001338 for (i = 0; i < size; i++)
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001339 state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] =
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08001340 type;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001341 }
1342 return 0;
1343}
1344
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001345static int check_stack_read(struct bpf_verifier_env *env,
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001346 struct bpf_func_state *reg_state /* func where register points to */,
1347 int off, int size, int value_regno)
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001348{
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001349 struct bpf_verifier_state *vstate = env->cur_state;
1350 struct bpf_func_state *state = vstate->frame[vstate->curframe];
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001351 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE;
1352 u8 *stype;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001353
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001354 if (reg_state->allocated_stack <= slot) {
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001355 verbose(env, "invalid read from stack off %d+0 size %d\n",
1356 off, size);
1357 return -EACCES;
1358 }
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001359 stype = reg_state->stack[spi].slot_type;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001360
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001361 if (stype[0] == STACK_SPILL) {
Alexei Starovoitov9c3997602014-10-28 15:11:41 -07001362 if (size != BPF_REG_SIZE) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001363 verbose(env, "invalid size of register spill\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001364 return -EACCES;
1365 }
Alexei Starovoitov9c3997602014-10-28 15:11:41 -07001366 for (i = 1; i < BPF_REG_SIZE; i++) {
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001367 if (stype[(slot - i) % BPF_REG_SIZE] != STACK_SPILL) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001368 verbose(env, "corrupted spill memory\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001369 return -EACCES;
1370 }
1371 }
1372
Edward Creedc503a82017-08-15 20:34:35 +01001373 if (value_regno >= 0) {
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001374 /* restore register state from stack */
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001375 state->regs[value_regno] = reg_state->stack[spi].spilled_ptr;
Alexei Starovoitov2f18f622017-11-30 21:31:38 -08001376 /* mark reg as written since spilled pointer state likely
1377 * has its liveness marks cleared by is_state_visited()
1378 * which resets stack/reg liveness for state transitions
1379 */
1380 state->regs[value_regno].live |= REG_LIVE_WRITTEN;
Edward Creedc503a82017-08-15 20:34:35 +01001381 }
Edward Cree679c7822018-08-22 20:02:19 +01001382 mark_reg_read(env, &reg_state->stack[spi].spilled_ptr,
1383 reg_state->stack[spi].spilled_ptr.parent);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001384 return 0;
1385 } else {
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08001386 int zeros = 0;
1387
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001388 for (i = 0; i < size; i++) {
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08001389 if (stype[(slot - i) % BPF_REG_SIZE] == STACK_MISC)
1390 continue;
1391 if (stype[(slot - i) % BPF_REG_SIZE] == STACK_ZERO) {
1392 zeros++;
1393 continue;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001394 }
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08001395 verbose(env, "invalid read from stack off %d+%d size %d\n",
1396 off, i, size);
1397 return -EACCES;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001398 }
Edward Cree679c7822018-08-22 20:02:19 +01001399 mark_reg_read(env, &reg_state->stack[spi].spilled_ptr,
1400 reg_state->stack[spi].spilled_ptr.parent);
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08001401 if (value_regno >= 0) {
1402 if (zeros == size) {
1403 /* any size read into register is zero extended,
1404 * so the whole register == const_zero
1405 */
1406 __mark_reg_const_zero(&state->regs[value_regno]);
1407 } else {
1408 /* have read misc data from the stack */
1409 mark_reg_unknown(env, state->regs, value_regno);
1410 }
1411 state->regs[value_regno].live |= REG_LIVE_WRITTEN;
1412 }
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001413 return 0;
1414 }
1415}
1416
Daniel Borkmanne4298d22019-01-03 00:58:31 +01001417static int check_stack_access(struct bpf_verifier_env *env,
1418 const struct bpf_reg_state *reg,
1419 int off, int size)
1420{
1421 /* Stack accesses must be at a fixed offset, so that we
1422 * can determine what type of data were returned. See
1423 * check_stack_read().
1424 */
1425 if (!tnum_is_const(reg->var_off)) {
1426 char tn_buf[48];
1427
1428 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
1429 verbose(env, "variable stack access var_off=%s off=%d size=%d",
1430 tn_buf, off, size);
1431 return -EACCES;
1432 }
1433
1434 if (off >= 0 || off < -MAX_BPF_STACK) {
1435 verbose(env, "invalid stack off=%d size=%d\n", off, size);
1436 return -EACCES;
1437 }
1438
1439 return 0;
1440}
1441
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001442/* check read/write into map element returned by bpf_map_lookup_elem() */
Edward Creef1174f72017-08-07 15:26:19 +01001443static int __check_map_access(struct bpf_verifier_env *env, u32 regno, int off,
Yonghong Song9fd29c02017-11-12 14:49:09 -08001444 int size, bool zero_size_allowed)
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001445{
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001446 struct bpf_reg_state *regs = cur_regs(env);
1447 struct bpf_map *map = regs[regno].map_ptr;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001448
Yonghong Song9fd29c02017-11-12 14:49:09 -08001449 if (off < 0 || size < 0 || (size == 0 && !zero_size_allowed) ||
1450 off + size > map->value_size) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001451 verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n",
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001452 map->value_size, off, size);
1453 return -EACCES;
1454 }
1455 return 0;
1456}
1457
Edward Creef1174f72017-08-07 15:26:19 +01001458/* check read/write into a map element with possible variable offset */
1459static int check_map_access(struct bpf_verifier_env *env, u32 regno,
Yonghong Song9fd29c02017-11-12 14:49:09 -08001460 int off, int size, bool zero_size_allowed)
Gianluca Borellodbcfe5f2017-01-09 10:19:46 -08001461{
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001462 struct bpf_verifier_state *vstate = env->cur_state;
1463 struct bpf_func_state *state = vstate->frame[vstate->curframe];
Gianluca Borellodbcfe5f2017-01-09 10:19:46 -08001464 struct bpf_reg_state *reg = &state->regs[regno];
1465 int err;
1466
Edward Creef1174f72017-08-07 15:26:19 +01001467 /* We may have adjusted the register to this map value, so we
1468 * need to try adding each of min_value and max_value to off
1469 * to make sure our theoretical access will be safe.
Gianluca Borellodbcfe5f2017-01-09 10:19:46 -08001470 */
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07001471 if (env->log.level & BPF_LOG_LEVEL)
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001472 print_verifier_state(env, state);
Daniel Borkmannb7137c42019-01-03 00:58:33 +01001473
Gianluca Borellodbcfe5f2017-01-09 10:19:46 -08001474 /* The minimum value is only important with signed
1475 * comparisons where we can't assume the floor of a
1476 * value is 0. If we are using signed variables for our
1477 * index'es we need to make sure that whatever we use
1478 * will have a set floor within our range.
1479 */
Daniel Borkmannb7137c42019-01-03 00:58:33 +01001480 if (reg->smin_value < 0 &&
1481 (reg->smin_value == S64_MIN ||
1482 (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) ||
1483 reg->smin_value + off < 0)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001484 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
Gianluca Borellodbcfe5f2017-01-09 10:19:46 -08001485 regno);
1486 return -EACCES;
1487 }
Yonghong Song9fd29c02017-11-12 14:49:09 -08001488 err = __check_map_access(env, regno, reg->smin_value + off, size,
1489 zero_size_allowed);
Gianluca Borellodbcfe5f2017-01-09 10:19:46 -08001490 if (err) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001491 verbose(env, "R%d min value is outside of the array range\n",
1492 regno);
Gianluca Borellodbcfe5f2017-01-09 10:19:46 -08001493 return err;
1494 }
1495
Edward Creeb03c9f92017-08-07 15:26:36 +01001496 /* If we haven't set a max value then we need to bail since we can't be
1497 * sure we won't do bad things.
1498 * If reg->umax_value + off could overflow, treat that as unbounded too.
Gianluca Borellodbcfe5f2017-01-09 10:19:46 -08001499 */
Edward Creeb03c9f92017-08-07 15:26:36 +01001500 if (reg->umax_value >= BPF_MAX_VAR_OFF) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001501 verbose(env, "R%d unbounded memory access, make sure to bounds check any array access into a map\n",
Gianluca Borellodbcfe5f2017-01-09 10:19:46 -08001502 regno);
1503 return -EACCES;
1504 }
Yonghong Song9fd29c02017-11-12 14:49:09 -08001505 err = __check_map_access(env, regno, reg->umax_value + off, size,
1506 zero_size_allowed);
Edward Creef1174f72017-08-07 15:26:19 +01001507 if (err)
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001508 verbose(env, "R%d max value is outside of the array range\n",
1509 regno);
Alexei Starovoitovd83525c2019-01-31 15:40:04 -08001510
1511 if (map_value_has_spin_lock(reg->map_ptr)) {
1512 u32 lock = reg->map_ptr->spin_lock_off;
1513
1514 /* if any part of struct bpf_spin_lock can be touched by
1515 * load/store reject this program.
1516 * To check that [x1, x2) overlaps with [y1, y2)
1517 * it is sufficient to check x1 < y2 && y1 < x2.
1518 */
1519 if (reg->smin_value + off < lock + sizeof(struct bpf_spin_lock) &&
1520 lock < reg->umax_value + off + size) {
1521 verbose(env, "bpf_spin_lock cannot be accessed directly by load/store\n");
1522 return -EACCES;
1523 }
1524 }
Edward Creef1174f72017-08-07 15:26:19 +01001525 return err;
Gianluca Borellodbcfe5f2017-01-09 10:19:46 -08001526}
1527
Alexei Starovoitov969bf052016-05-05 19:49:10 -07001528#define MAX_PACKET_OFF 0xffff
1529
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01001530static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,
Thomas Graf3a0af8f2016-11-30 17:10:10 +01001531 const struct bpf_call_arg_meta *meta,
1532 enum bpf_access_type t)
Brenden Blanco4acf6c02016-07-19 12:16:56 -07001533{
Daniel Borkmann36bbef52016-09-20 00:26:13 +02001534 switch (env->prog->type) {
Daniel Borkmann5d66fa72018-10-24 22:05:45 +02001535 /* Program types only with direct read access go here! */
Thomas Graf3a0af8f2016-11-30 17:10:10 +01001536 case BPF_PROG_TYPE_LWT_IN:
1537 case BPF_PROG_TYPE_LWT_OUT:
Mathieu Xhonneux004d4b22018-05-20 14:58:16 +01001538 case BPF_PROG_TYPE_LWT_SEG6LOCAL:
Martin KaFai Lau2dbb9b92018-08-08 01:01:25 -07001539 case BPF_PROG_TYPE_SK_REUSEPORT:
Daniel Borkmann5d66fa72018-10-24 22:05:45 +02001540 case BPF_PROG_TYPE_FLOW_DISSECTOR:
Daniel Borkmannd5563d32018-10-24 22:05:46 +02001541 case BPF_PROG_TYPE_CGROUP_SKB:
Thomas Graf3a0af8f2016-11-30 17:10:10 +01001542 if (t == BPF_WRITE)
1543 return false;
Alexander Alemayhu7e57fbb2017-02-14 00:02:35 +01001544 /* fallthrough */
Daniel Borkmann5d66fa72018-10-24 22:05:45 +02001545
1546 /* Program types with direct read + write access go here! */
Daniel Borkmann36bbef52016-09-20 00:26:13 +02001547 case BPF_PROG_TYPE_SCHED_CLS:
1548 case BPF_PROG_TYPE_SCHED_ACT:
Brenden Blanco4acf6c02016-07-19 12:16:56 -07001549 case BPF_PROG_TYPE_XDP:
Thomas Graf3a0af8f2016-11-30 17:10:10 +01001550 case BPF_PROG_TYPE_LWT_XMIT:
John Fastabend8a31db52017-08-15 22:33:09 -07001551 case BPF_PROG_TYPE_SK_SKB:
John Fastabend4f738ad2018-03-18 12:57:10 -07001552 case BPF_PROG_TYPE_SK_MSG:
Daniel Borkmann36bbef52016-09-20 00:26:13 +02001553 if (meta)
1554 return meta->pkt_access;
1555
1556 env->seen_direct_write = true;
Brenden Blanco4acf6c02016-07-19 12:16:56 -07001557 return true;
1558 default:
1559 return false;
1560 }
1561}
1562
Edward Creef1174f72017-08-07 15:26:19 +01001563static int __check_packet_access(struct bpf_verifier_env *env, u32 regno,
Yonghong Song9fd29c02017-11-12 14:49:09 -08001564 int off, int size, bool zero_size_allowed)
Alexei Starovoitov969bf052016-05-05 19:49:10 -07001565{
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001566 struct bpf_reg_state *regs = cur_regs(env);
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01001567 struct bpf_reg_state *reg = &regs[regno];
Alexei Starovoitov969bf052016-05-05 19:49:10 -07001568
Yonghong Song9fd29c02017-11-12 14:49:09 -08001569 if (off < 0 || size < 0 || (size == 0 && !zero_size_allowed) ||
1570 (u64)off + size > reg->range) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001571 verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n",
Alexei Starovoitovd91b28e2016-05-19 18:17:13 -07001572 off, size, regno, reg->id, reg->off, reg->range);
Alexei Starovoitov969bf052016-05-05 19:49:10 -07001573 return -EACCES;
1574 }
1575 return 0;
1576}
1577
Edward Creef1174f72017-08-07 15:26:19 +01001578static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off,
Yonghong Song9fd29c02017-11-12 14:49:09 -08001579 int size, bool zero_size_allowed)
Edward Creef1174f72017-08-07 15:26:19 +01001580{
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001581 struct bpf_reg_state *regs = cur_regs(env);
Edward Creef1174f72017-08-07 15:26:19 +01001582 struct bpf_reg_state *reg = &regs[regno];
1583 int err;
1584
1585 /* We may have added a variable offset to the packet pointer; but any
1586 * reg->range we have comes after that. We are only checking the fixed
1587 * offset.
1588 */
1589
1590 /* We don't allow negative numbers, because we aren't tracking enough
1591 * detail to prove they're safe.
1592 */
Edward Creeb03c9f92017-08-07 15:26:36 +01001593 if (reg->smin_value < 0) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001594 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
Edward Creef1174f72017-08-07 15:26:19 +01001595 regno);
1596 return -EACCES;
1597 }
Yonghong Song9fd29c02017-11-12 14:49:09 -08001598 err = __check_packet_access(env, regno, off, size, zero_size_allowed);
Edward Creef1174f72017-08-07 15:26:19 +01001599 if (err) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001600 verbose(env, "R%d offset is outside of the packet\n", regno);
Edward Creef1174f72017-08-07 15:26:19 +01001601 return err;
1602 }
Jiong Wange6478152018-11-08 04:08:42 -05001603
1604 /* __check_packet_access has made sure "off + size - 1" is within u16.
1605 * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff,
1606 * otherwise find_good_pkt_pointers would have refused to set range info
1607 * that __check_packet_access would have rejected this pkt access.
1608 * Therefore, "off + reg->umax_value + size - 1" won't overflow u32.
1609 */
1610 env->prog->aux->max_pkt_offset =
1611 max_t(u32, env->prog->aux->max_pkt_offset,
1612 off + reg->umax_value + size - 1);
1613
Edward Creef1174f72017-08-07 15:26:19 +01001614 return err;
1615}
1616
1617/* check access to 'struct bpf_context' fields. Supports fixed offsets only */
Yonghong Song31fd8582017-06-13 15:52:13 -07001618static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size,
Alexei Starovoitov19de99f2016-06-15 18:25:38 -07001619 enum bpf_access_type t, enum bpf_reg_type *reg_type)
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001620{
Daniel Borkmannf96da092017-07-02 02:13:27 +02001621 struct bpf_insn_access_aux info = {
1622 .reg_type = *reg_type,
1623 };
Yonghong Song31fd8582017-06-13 15:52:13 -07001624
Jakub Kicinski4f9218a2017-10-16 16:40:55 -07001625 if (env->ops->is_valid_access &&
Andrey Ignatov5e43f892018-03-30 15:08:00 -07001626 env->ops->is_valid_access(off, size, t, env->prog, &info)) {
Daniel Borkmannf96da092017-07-02 02:13:27 +02001627 /* A non zero info.ctx_field_size indicates that this field is a
1628 * candidate for later verifier transformation to load the whole
1629 * field and then apply a mask when accessed with a narrower
1630 * access than actual ctx access size. A zero info.ctx_field_size
1631 * will only allow for whole field access and rejects any other
1632 * type of narrower access.
Yonghong Song31fd8582017-06-13 15:52:13 -07001633 */
Yonghong Song23994632017-06-22 15:07:39 -07001634 *reg_type = info.reg_type;
Yonghong Song31fd8582017-06-13 15:52:13 -07001635
Jakub Kicinski4f9218a2017-10-16 16:40:55 -07001636 env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size;
Alexei Starovoitov32bbe002016-04-06 18:43:28 -07001637 /* remember the offset of last byte accessed in ctx */
1638 if (env->prog->aux->max_ctx_offset < off + size)
1639 env->prog->aux->max_ctx_offset = off + size;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001640 return 0;
Alexei Starovoitov32bbe002016-04-06 18:43:28 -07001641 }
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001642
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001643 verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001644 return -EACCES;
1645}
1646
Petar Penkovd58e4682018-09-14 07:46:18 -07001647static int check_flow_keys_access(struct bpf_verifier_env *env, int off,
1648 int size)
1649{
1650 if (size < 0 || off < 0 ||
1651 (u64)off + size > sizeof(struct bpf_flow_keys)) {
1652 verbose(env, "invalid access to flow keys off=%d size=%d\n",
1653 off, size);
1654 return -EACCES;
1655 }
1656 return 0;
1657}
1658
Martin KaFai Lau5f456642019-02-08 22:25:54 -08001659static int check_sock_access(struct bpf_verifier_env *env, int insn_idx,
1660 u32 regno, int off, int size,
1661 enum bpf_access_type t)
Joe Stringerc64b7982018-10-02 13:35:33 -07001662{
1663 struct bpf_reg_state *regs = cur_regs(env);
1664 struct bpf_reg_state *reg = &regs[regno];
Martin KaFai Lau5f456642019-02-08 22:25:54 -08001665 struct bpf_insn_access_aux info = {};
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08001666 bool valid;
Joe Stringerc64b7982018-10-02 13:35:33 -07001667
1668 if (reg->smin_value < 0) {
1669 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
1670 regno);
1671 return -EACCES;
1672 }
1673
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08001674 switch (reg->type) {
1675 case PTR_TO_SOCK_COMMON:
1676 valid = bpf_sock_common_is_valid_access(off, size, t, &info);
1677 break;
1678 case PTR_TO_SOCKET:
1679 valid = bpf_sock_is_valid_access(off, size, t, &info);
1680 break;
Martin KaFai Lau655a51e2019-02-09 23:22:24 -08001681 case PTR_TO_TCP_SOCK:
1682 valid = bpf_tcp_sock_is_valid_access(off, size, t, &info);
1683 break;
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08001684 default:
1685 valid = false;
Joe Stringerc64b7982018-10-02 13:35:33 -07001686 }
1687
Martin KaFai Lau5f456642019-02-08 22:25:54 -08001688
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08001689 if (valid) {
1690 env->insn_aux_data[insn_idx].ctx_field_size =
1691 info.ctx_field_size;
1692 return 0;
1693 }
1694
1695 verbose(env, "R%d invalid %s access off=%d size=%d\n",
1696 regno, reg_type_str[reg->type], off, size);
1697
1698 return -EACCES;
Joe Stringerc64b7982018-10-02 13:35:33 -07001699}
1700
Daniel Borkmann4cabc5b2017-07-21 00:00:21 +02001701static bool __is_pointer_value(bool allow_ptr_leaks,
1702 const struct bpf_reg_state *reg)
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07001703{
Daniel Borkmann4cabc5b2017-07-21 00:00:21 +02001704 if (allow_ptr_leaks)
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07001705 return false;
1706
Edward Creef1174f72017-08-07 15:26:19 +01001707 return reg->type != SCALAR_VALUE;
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07001708}
1709
Daniel Borkmann2a159c62018-10-21 02:09:24 +02001710static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)
1711{
1712 return cur_regs(env) + regno;
1713}
1714
Daniel Borkmann4cabc5b2017-07-21 00:00:21 +02001715static bool is_pointer_value(struct bpf_verifier_env *env, int regno)
1716{
Daniel Borkmann2a159c62018-10-21 02:09:24 +02001717 return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno));
Daniel Borkmann4cabc5b2017-07-21 00:00:21 +02001718}
1719
Daniel Borkmannf37a8cb2018-01-16 23:30:10 +01001720static bool is_ctx_reg(struct bpf_verifier_env *env, int regno)
1721{
Daniel Borkmann2a159c62018-10-21 02:09:24 +02001722 const struct bpf_reg_state *reg = reg_state(env, regno);
Daniel Borkmannf37a8cb2018-01-16 23:30:10 +01001723
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08001724 return reg->type == PTR_TO_CTX;
1725}
1726
1727static bool is_sk_reg(struct bpf_verifier_env *env, int regno)
1728{
1729 const struct bpf_reg_state *reg = reg_state(env, regno);
1730
1731 return type_is_sk_pointer(reg->type);
Daniel Borkmannf37a8cb2018-01-16 23:30:10 +01001732}
1733
Daniel Borkmannca369602018-02-23 22:29:05 +01001734static bool is_pkt_reg(struct bpf_verifier_env *env, int regno)
1735{
Daniel Borkmann2a159c62018-10-21 02:09:24 +02001736 const struct bpf_reg_state *reg = reg_state(env, regno);
Daniel Borkmannca369602018-02-23 22:29:05 +01001737
1738 return type_is_pkt_pointer(reg->type);
1739}
1740
Daniel Borkmann4b5defd2018-10-21 02:09:25 +02001741static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno)
1742{
1743 const struct bpf_reg_state *reg = reg_state(env, regno);
1744
1745 /* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */
1746 return reg->type == PTR_TO_FLOW_KEYS;
1747}
1748
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001749static int check_pkt_ptr_alignment(struct bpf_verifier_env *env,
1750 const struct bpf_reg_state *reg,
David S. Millerd1174412017-05-10 11:22:52 -07001751 int off, int size, bool strict)
Alexei Starovoitov969bf052016-05-05 19:49:10 -07001752{
Edward Creef1174f72017-08-07 15:26:19 +01001753 struct tnum reg_off;
David S. Millere07b98d2017-05-10 11:38:07 -07001754 int ip_align;
David S. Millerd1174412017-05-10 11:22:52 -07001755
1756 /* Byte size accesses are always allowed. */
1757 if (!strict || size == 1)
1758 return 0;
1759
David S. Millere4eda882017-05-22 12:27:07 -04001760 /* For platforms that do not have a Kconfig enabling
1761 * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of
1762 * NET_IP_ALIGN is universally set to '2'. And on platforms
1763 * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get
1764 * to this code only in strict mode where we want to emulate
1765 * the NET_IP_ALIGN==2 checking. Therefore use an
1766 * unconditional IP align value of '2'.
David S. Millere07b98d2017-05-10 11:38:07 -07001767 */
David S. Millere4eda882017-05-22 12:27:07 -04001768 ip_align = 2;
Edward Creef1174f72017-08-07 15:26:19 +01001769
1770 reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off));
1771 if (!tnum_is_aligned(reg_off, size)) {
1772 char tn_buf[48];
1773
1774 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001775 verbose(env,
1776 "misaligned packet access off %d+%s+%d+%d size %d\n",
Edward Creef1174f72017-08-07 15:26:19 +01001777 ip_align, tn_buf, reg->off, off, size);
Alexei Starovoitov969bf052016-05-05 19:49:10 -07001778 return -EACCES;
1779 }
Daniel Borkmann79adffc2017-03-31 02:24:03 +02001780
Alexei Starovoitov969bf052016-05-05 19:49:10 -07001781 return 0;
1782}
1783
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001784static int check_generic_ptr_alignment(struct bpf_verifier_env *env,
1785 const struct bpf_reg_state *reg,
Edward Creef1174f72017-08-07 15:26:19 +01001786 const char *pointer_desc,
1787 int off, int size, bool strict)
Daniel Borkmann79adffc2017-03-31 02:24:03 +02001788{
Edward Creef1174f72017-08-07 15:26:19 +01001789 struct tnum reg_off;
1790
1791 /* Byte size accesses are always allowed. */
1792 if (!strict || size == 1)
1793 return 0;
1794
1795 reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off));
1796 if (!tnum_is_aligned(reg_off, size)) {
1797 char tn_buf[48];
1798
1799 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001800 verbose(env, "misaligned %saccess off %s+%d+%d size %d\n",
Edward Creef1174f72017-08-07 15:26:19 +01001801 pointer_desc, tn_buf, reg->off, off, size);
Daniel Borkmann79adffc2017-03-31 02:24:03 +02001802 return -EACCES;
1803 }
1804
1805 return 0;
1806}
1807
David S. Millere07b98d2017-05-10 11:38:07 -07001808static int check_ptr_alignment(struct bpf_verifier_env *env,
Daniel Borkmannca369602018-02-23 22:29:05 +01001809 const struct bpf_reg_state *reg, int off,
1810 int size, bool strict_alignment_once)
Daniel Borkmann79adffc2017-03-31 02:24:03 +02001811{
Daniel Borkmannca369602018-02-23 22:29:05 +01001812 bool strict = env->strict_alignment || strict_alignment_once;
Edward Creef1174f72017-08-07 15:26:19 +01001813 const char *pointer_desc = "";
David S. Millerd1174412017-05-10 11:22:52 -07001814
Daniel Borkmann79adffc2017-03-31 02:24:03 +02001815 switch (reg->type) {
1816 case PTR_TO_PACKET:
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02001817 case PTR_TO_PACKET_META:
1818 /* Special case, because of NET_IP_ALIGN. Given metadata sits
1819 * right in front, treat it the very same way.
1820 */
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001821 return check_pkt_ptr_alignment(env, reg, off, size, strict);
Petar Penkovd58e4682018-09-14 07:46:18 -07001822 case PTR_TO_FLOW_KEYS:
1823 pointer_desc = "flow keys ";
1824 break;
Edward Creef1174f72017-08-07 15:26:19 +01001825 case PTR_TO_MAP_VALUE:
1826 pointer_desc = "value ";
1827 break;
1828 case PTR_TO_CTX:
1829 pointer_desc = "context ";
1830 break;
1831 case PTR_TO_STACK:
1832 pointer_desc = "stack ";
Jann Horna5ec6ae2017-12-18 20:11:58 -08001833 /* The stack spill tracking logic in check_stack_write()
1834 * and check_stack_read() relies on stack accesses being
1835 * aligned.
1836 */
1837 strict = true;
Edward Creef1174f72017-08-07 15:26:19 +01001838 break;
Joe Stringerc64b7982018-10-02 13:35:33 -07001839 case PTR_TO_SOCKET:
1840 pointer_desc = "sock ";
1841 break;
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08001842 case PTR_TO_SOCK_COMMON:
1843 pointer_desc = "sock_common ";
1844 break;
Martin KaFai Lau655a51e2019-02-09 23:22:24 -08001845 case PTR_TO_TCP_SOCK:
1846 pointer_desc = "tcp_sock ";
1847 break;
Daniel Borkmann79adffc2017-03-31 02:24:03 +02001848 default:
Edward Creef1174f72017-08-07 15:26:19 +01001849 break;
Daniel Borkmann79adffc2017-03-31 02:24:03 +02001850 }
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001851 return check_generic_ptr_alignment(env, reg, pointer_desc, off, size,
1852 strict);
Daniel Borkmann79adffc2017-03-31 02:24:03 +02001853}
1854
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001855static int update_stack_depth(struct bpf_verifier_env *env,
1856 const struct bpf_func_state *func,
1857 int off)
1858{
Jiong Wang9c8105b2018-05-02 16:17:18 -04001859 u16 stack = env->subprog_info[func->subprogno].stack_depth;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001860
1861 if (stack >= -off)
1862 return 0;
1863
1864 /* update known max for given subprogram */
Jiong Wang9c8105b2018-05-02 16:17:18 -04001865 env->subprog_info[func->subprogno].stack_depth = -off;
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001866 return 0;
1867}
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001868
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001869/* starting from main bpf function walk all instructions of the function
1870 * and recursively walk all callees that given function can call.
1871 * Ignore jump and exit insns.
1872 * Since recursion is prevented by check_cfg() this algorithm
1873 * only needs a local stack of MAX_CALL_FRAMES to remember callsites
1874 */
1875static int check_max_stack_depth(struct bpf_verifier_env *env)
1876{
Jiong Wang9c8105b2018-05-02 16:17:18 -04001877 int depth = 0, frame = 0, idx = 0, i = 0, subprog_end;
1878 struct bpf_subprog_info *subprog = env->subprog_info;
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001879 struct bpf_insn *insn = env->prog->insnsi;
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001880 int ret_insn[MAX_CALL_FRAMES];
1881 int ret_prog[MAX_CALL_FRAMES];
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001882
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001883process_func:
1884 /* round up to 32-bytes, since this is granularity
1885 * of interpreter stack size
1886 */
Jiong Wang9c8105b2018-05-02 16:17:18 -04001887 depth += round_up(max_t(u32, subprog[idx].stack_depth, 1), 32);
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001888 if (depth > MAX_BPF_STACK) {
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001889 verbose(env, "combined stack size of %d calls is %d. Too large\n",
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001890 frame + 1, depth);
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001891 return -EACCES;
1892 }
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001893continue_func:
Jiong Wang4cb3d992018-05-02 16:17:19 -04001894 subprog_end = subprog[idx + 1].start;
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001895 for (; i < subprog_end; i++) {
1896 if (insn[i].code != (BPF_JMP | BPF_CALL))
1897 continue;
1898 if (insn[i].src_reg != BPF_PSEUDO_CALL)
1899 continue;
1900 /* remember insn and function to return to */
1901 ret_insn[frame] = i + 1;
Jiong Wang9c8105b2018-05-02 16:17:18 -04001902 ret_prog[frame] = idx;
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001903
1904 /* find the callee */
1905 i = i + insn[i].imm + 1;
Jiong Wang9c8105b2018-05-02 16:17:18 -04001906 idx = find_subprog(env, i);
1907 if (idx < 0) {
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001908 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
1909 i);
1910 return -EFAULT;
1911 }
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001912 frame++;
1913 if (frame >= MAX_CALL_FRAMES) {
1914 WARN_ONCE(1, "verifier bug. Call stack is too deep\n");
1915 return -EFAULT;
1916 }
1917 goto process_func;
1918 }
1919 /* end of for() loop means the last insn of the 'subprog'
1920 * was reached. Doesn't matter whether it was JA or EXIT
1921 */
1922 if (frame == 0)
1923 return 0;
Jiong Wang9c8105b2018-05-02 16:17:18 -04001924 depth -= round_up(max_t(u32, subprog[idx].stack_depth, 1), 32);
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001925 frame--;
1926 i = ret_insn[frame];
Jiong Wang9c8105b2018-05-02 16:17:18 -04001927 idx = ret_prog[frame];
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001928 goto continue_func;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001929}
1930
David S. Miller19d28fb2018-01-11 21:27:54 -05001931#ifndef CONFIG_BPF_JIT_ALWAYS_ON
Alexei Starovoitov1ea47e02017-12-14 17:55:13 -08001932static int get_callee_stack_depth(struct bpf_verifier_env *env,
1933 const struct bpf_insn *insn, int idx)
1934{
1935 int start = idx + insn->imm + 1, subprog;
1936
1937 subprog = find_subprog(env, start);
1938 if (subprog < 0) {
1939 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
1940 start);
1941 return -EFAULT;
1942 }
Jiong Wang9c8105b2018-05-02 16:17:18 -04001943 return env->subprog_info[subprog].stack_depth;
Alexei Starovoitov1ea47e02017-12-14 17:55:13 -08001944}
David S. Miller19d28fb2018-01-11 21:27:54 -05001945#endif
Alexei Starovoitov1ea47e02017-12-14 17:55:13 -08001946
Daniel Borkmann58990d12018-06-07 17:40:03 +02001947static int check_ctx_reg(struct bpf_verifier_env *env,
1948 const struct bpf_reg_state *reg, int regno)
1949{
1950 /* Access to ctx or passing it to a helper is only allowed in
1951 * its original, unmodified form.
1952 */
1953
1954 if (reg->off) {
1955 verbose(env, "dereference of modified ctx ptr R%d off=%d disallowed\n",
1956 regno, reg->off);
1957 return -EACCES;
1958 }
1959
1960 if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
1961 char tn_buf[48];
1962
1963 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
1964 verbose(env, "variable ctx access var_off=%s disallowed\n", tn_buf);
1965 return -EACCES;
1966 }
1967
1968 return 0;
1969}
1970
Jann Horn0c17d1d2017-12-18 20:11:55 -08001971/* truncate register to smaller size (in bytes)
1972 * must be called with size < BPF_REG_SIZE
1973 */
1974static void coerce_reg_to_size(struct bpf_reg_state *reg, int size)
1975{
1976 u64 mask;
1977
1978 /* clear high bits in bit representation */
1979 reg->var_off = tnum_cast(reg->var_off, size);
1980
1981 /* fix arithmetic bounds */
1982 mask = ((u64)1 << (size * 8)) - 1;
1983 if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) {
1984 reg->umin_value &= mask;
1985 reg->umax_value &= mask;
1986 } else {
1987 reg->umin_value = 0;
1988 reg->umax_value = mask;
1989 }
1990 reg->smin_value = reg->umin_value;
1991 reg->smax_value = reg->umax_value;
1992}
1993
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001994/* check whether memory at (regno + off) is accessible for t = (read | write)
1995 * if t==write, value_regno is a register which value is stored into memory
1996 * if t==read, value_regno is a register which will receive the value from memory
1997 * if t==write && value_regno==-1, some unknown value is stored into memory
1998 * if t==read && value_regno==-1, don't care what we read from memory
1999 */
Daniel Borkmannca369602018-02-23 22:29:05 +01002000static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno,
2001 int off, int bpf_size, enum bpf_access_type t,
2002 int value_regno, bool strict_alignment_once)
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002003{
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07002004 struct bpf_reg_state *regs = cur_regs(env);
2005 struct bpf_reg_state *reg = regs + regno;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002006 struct bpf_func_state *state;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002007 int size, err = 0;
2008
2009 size = bpf_size_to_bytes(bpf_size);
2010 if (size < 0)
2011 return size;
2012
Edward Creef1174f72017-08-07 15:26:19 +01002013 /* alignment checks will add in reg->off themselves */
Daniel Borkmannca369602018-02-23 22:29:05 +01002014 err = check_ptr_alignment(env, reg, off, size, strict_alignment_once);
Alexei Starovoitov969bf052016-05-05 19:49:10 -07002015 if (err)
2016 return err;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002017
Edward Creef1174f72017-08-07 15:26:19 +01002018 /* for access checks, reg->off is just part of off */
2019 off += reg->off;
2020
2021 if (reg->type == PTR_TO_MAP_VALUE) {
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07002022 if (t == BPF_WRITE && value_regno >= 0 &&
2023 is_pointer_value(env, value_regno)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002024 verbose(env, "R%d leaks addr into map\n", value_regno);
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07002025 return -EACCES;
2026 }
Josef Bacik48461132016-09-28 10:54:32 -04002027
Yonghong Song9fd29c02017-11-12 14:49:09 -08002028 err = check_map_access(env, regno, off, size, false);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002029 if (!err && t == BPF_READ && value_regno >= 0)
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07002030 mark_reg_unknown(env, regs, value_regno);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002031
Alexei Starovoitov1a0dc1a2016-05-05 19:49:09 -07002032 } else if (reg->type == PTR_TO_CTX) {
Edward Creef1174f72017-08-07 15:26:19 +01002033 enum bpf_reg_type reg_type = SCALAR_VALUE;
Alexei Starovoitov19de99f2016-06-15 18:25:38 -07002034
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07002035 if (t == BPF_WRITE && value_regno >= 0 &&
2036 is_pointer_value(env, value_regno)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002037 verbose(env, "R%d leaks addr into ctx\n", value_regno);
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07002038 return -EACCES;
2039 }
Edward Creef1174f72017-08-07 15:26:19 +01002040
Daniel Borkmann58990d12018-06-07 17:40:03 +02002041 err = check_ctx_reg(env, reg, regno);
2042 if (err < 0)
2043 return err;
2044
Yonghong Song31fd8582017-06-13 15:52:13 -07002045 err = check_ctx_access(env, insn_idx, off, size, t, &reg_type);
Alexei Starovoitov969bf052016-05-05 19:49:10 -07002046 if (!err && t == BPF_READ && value_regno >= 0) {
Edward Creef1174f72017-08-07 15:26:19 +01002047 /* ctx access returns either a scalar, or a
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02002048 * PTR_TO_PACKET[_META,_END]. In the latter
2049 * case, we know the offset is zero.
Edward Creef1174f72017-08-07 15:26:19 +01002050 */
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08002051 if (reg_type == SCALAR_VALUE) {
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07002052 mark_reg_unknown(env, regs, value_regno);
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08002053 } else {
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07002054 mark_reg_known_zero(env, regs,
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002055 value_regno);
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08002056 if (reg_type_may_be_null(reg_type))
2057 regs[value_regno].id = ++env->id_gen;
2058 }
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07002059 regs[value_regno].type = reg_type;
Alexei Starovoitov969bf052016-05-05 19:49:10 -07002060 }
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002061
Edward Creef1174f72017-08-07 15:26:19 +01002062 } else if (reg->type == PTR_TO_STACK) {
Edward Creef1174f72017-08-07 15:26:19 +01002063 off += reg->var_off.value;
Daniel Borkmanne4298d22019-01-03 00:58:31 +01002064 err = check_stack_access(env, reg, off, size);
2065 if (err)
2066 return err;
Alexei Starovoitov87266792017-05-30 13:31:29 -07002067
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002068 state = func(env, reg);
2069 err = update_stack_depth(env, state, off);
2070 if (err)
2071 return err;
Alexei Starovoitov87266792017-05-30 13:31:29 -07002072
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07002073 if (t == BPF_WRITE)
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002074 err = check_stack_write(env, state, off, size,
Alexei Starovoitovaf86ca42018-05-15 09:27:05 -07002075 value_regno, insn_idx);
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07002076 else
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002077 err = check_stack_read(env, state, off, size,
2078 value_regno);
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02002079 } else if (reg_is_pkt_pointer(reg)) {
Thomas Graf3a0af8f2016-11-30 17:10:10 +01002080 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002081 verbose(env, "cannot write into packet\n");
Alexei Starovoitov969bf052016-05-05 19:49:10 -07002082 return -EACCES;
2083 }
Brenden Blanco4acf6c02016-07-19 12:16:56 -07002084 if (t == BPF_WRITE && value_regno >= 0 &&
2085 is_pointer_value(env, value_regno)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002086 verbose(env, "R%d leaks addr into packet\n",
2087 value_regno);
Brenden Blanco4acf6c02016-07-19 12:16:56 -07002088 return -EACCES;
2089 }
Yonghong Song9fd29c02017-11-12 14:49:09 -08002090 err = check_packet_access(env, regno, off, size, false);
Alexei Starovoitov969bf052016-05-05 19:49:10 -07002091 if (!err && t == BPF_READ && value_regno >= 0)
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07002092 mark_reg_unknown(env, regs, value_regno);
Petar Penkovd58e4682018-09-14 07:46:18 -07002093 } else if (reg->type == PTR_TO_FLOW_KEYS) {
2094 if (t == BPF_WRITE && value_regno >= 0 &&
2095 is_pointer_value(env, value_regno)) {
2096 verbose(env, "R%d leaks addr into flow keys\n",
2097 value_regno);
2098 return -EACCES;
2099 }
2100
2101 err = check_flow_keys_access(env, off, size);
2102 if (!err && t == BPF_READ && value_regno >= 0)
2103 mark_reg_unknown(env, regs, value_regno);
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08002104 } else if (type_is_sk_pointer(reg->type)) {
Joe Stringerc64b7982018-10-02 13:35:33 -07002105 if (t == BPF_WRITE) {
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08002106 verbose(env, "R%d cannot write into %s\n",
2107 regno, reg_type_str[reg->type]);
Joe Stringerc64b7982018-10-02 13:35:33 -07002108 return -EACCES;
2109 }
Martin KaFai Lau5f456642019-02-08 22:25:54 -08002110 err = check_sock_access(env, insn_idx, regno, off, size, t);
Joe Stringerc64b7982018-10-02 13:35:33 -07002111 if (!err && value_regno >= 0)
2112 mark_reg_unknown(env, regs, value_regno);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002113 } else {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002114 verbose(env, "R%d invalid mem access '%s'\n", regno,
2115 reg_type_str[reg->type]);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002116 return -EACCES;
2117 }
Alexei Starovoitov969bf052016-05-05 19:49:10 -07002118
Edward Creef1174f72017-08-07 15:26:19 +01002119 if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ &&
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07002120 regs[value_regno].type == SCALAR_VALUE) {
Edward Creef1174f72017-08-07 15:26:19 +01002121 /* b/h/w load zero-extends, mark upper bits as known 0 */
Jann Horn0c17d1d2017-12-18 20:11:55 -08002122 coerce_reg_to_size(&regs[value_regno], size);
Alexei Starovoitov969bf052016-05-05 19:49:10 -07002123 }
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002124 return err;
2125}
2126
Yonghong Song31fd8582017-06-13 15:52:13 -07002127static int check_xadd(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn)
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002128{
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002129 int err;
2130
2131 if ((BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) ||
2132 insn->imm != 0) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002133 verbose(env, "BPF_XADD uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002134 return -EINVAL;
2135 }
2136
2137 /* check src1 operand */
Edward Creedc503a82017-08-15 20:34:35 +01002138 err = check_reg_arg(env, insn->src_reg, SRC_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002139 if (err)
2140 return err;
2141
2142 /* check src2 operand */
Edward Creedc503a82017-08-15 20:34:35 +01002143 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002144 if (err)
2145 return err;
2146
Daniel Borkmann6bdf6ab2017-06-29 03:04:59 +02002147 if (is_pointer_value(env, insn->src_reg)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002148 verbose(env, "R%d leaks addr into mem\n", insn->src_reg);
Daniel Borkmann6bdf6ab2017-06-29 03:04:59 +02002149 return -EACCES;
2150 }
2151
Daniel Borkmannca369602018-02-23 22:29:05 +01002152 if (is_ctx_reg(env, insn->dst_reg) ||
Daniel Borkmann4b5defd2018-10-21 02:09:25 +02002153 is_pkt_reg(env, insn->dst_reg) ||
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08002154 is_flow_key_reg(env, insn->dst_reg) ||
2155 is_sk_reg(env, insn->dst_reg)) {
Daniel Borkmannca369602018-02-23 22:29:05 +01002156 verbose(env, "BPF_XADD stores into R%d %s is not allowed\n",
Daniel Borkmann2a159c62018-10-21 02:09:24 +02002157 insn->dst_reg,
2158 reg_type_str[reg_state(env, insn->dst_reg)->type]);
Daniel Borkmannf37a8cb2018-01-16 23:30:10 +01002159 return -EACCES;
2160 }
2161
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002162 /* check whether atomic_add can read the memory */
Yonghong Song31fd8582017-06-13 15:52:13 -07002163 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
Daniel Borkmannca369602018-02-23 22:29:05 +01002164 BPF_SIZE(insn->code), BPF_READ, -1, true);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002165 if (err)
2166 return err;
2167
2168 /* check whether atomic_add can write into the same memory */
Yonghong Song31fd8582017-06-13 15:52:13 -07002169 return check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
Daniel Borkmannca369602018-02-23 22:29:05 +01002170 BPF_SIZE(insn->code), BPF_WRITE, -1, true);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002171}
2172
Andrey Ignatov2011fcc2019-03-28 18:01:57 -07002173static int __check_stack_boundary(struct bpf_verifier_env *env, u32 regno,
2174 int off, int access_size,
2175 bool zero_size_allowed)
2176{
2177 struct bpf_reg_state *reg = reg_state(env, regno);
2178
2179 if (off >= 0 || off < -MAX_BPF_STACK || off + access_size > 0 ||
2180 access_size < 0 || (access_size == 0 && !zero_size_allowed)) {
2181 if (tnum_is_const(reg->var_off)) {
2182 verbose(env, "invalid stack type R%d off=%d access_size=%d\n",
2183 regno, off, access_size);
2184 } else {
2185 char tn_buf[48];
2186
2187 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
2188 verbose(env, "invalid stack type R%d var_off=%s access_size=%d\n",
2189 regno, tn_buf, access_size);
2190 }
2191 return -EACCES;
2192 }
2193 return 0;
2194}
2195
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002196/* when register 'regno' is passed into function that will read 'access_size'
2197 * bytes from that pointer, make sure that it's within stack boundary
Edward Creef1174f72017-08-07 15:26:19 +01002198 * and all elements of stack are initialized.
2199 * Unlike most pointer bounds-checking functions, this one doesn't take an
2200 * 'off' argument, so it has to add in reg->off itself.
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002201 */
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01002202static int check_stack_boundary(struct bpf_verifier_env *env, int regno,
Daniel Borkmann435faee12016-04-13 00:10:51 +02002203 int access_size, bool zero_size_allowed,
2204 struct bpf_call_arg_meta *meta)
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002205{
Daniel Borkmann2a159c62018-10-21 02:09:24 +02002206 struct bpf_reg_state *reg = reg_state(env, regno);
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002207 struct bpf_func_state *state = func(env, reg);
Andrey Ignatov2011fcc2019-03-28 18:01:57 -07002208 int err, min_off, max_off, i, slot, spi;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002209
Alexei Starovoitov914cb782017-11-30 21:31:40 -08002210 if (reg->type != PTR_TO_STACK) {
Edward Creef1174f72017-08-07 15:26:19 +01002211 /* Allow zero-byte read from NULL, regardless of pointer type */
Daniel Borkmann8e2fe1d92016-02-19 23:05:22 +01002212 if (zero_size_allowed && access_size == 0 &&
Alexei Starovoitov914cb782017-11-30 21:31:40 -08002213 register_is_null(reg))
Daniel Borkmann8e2fe1d92016-02-19 23:05:22 +01002214 return 0;
2215
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002216 verbose(env, "R%d type=%s expected=%s\n", regno,
Alexei Starovoitov914cb782017-11-30 21:31:40 -08002217 reg_type_str[reg->type],
Daniel Borkmann8e2fe1d92016-02-19 23:05:22 +01002218 reg_type_str[PTR_TO_STACK]);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002219 return -EACCES;
Daniel Borkmann8e2fe1d92016-02-19 23:05:22 +01002220 }
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002221
Andrey Ignatov2011fcc2019-03-28 18:01:57 -07002222 if (tnum_is_const(reg->var_off)) {
2223 min_off = max_off = reg->var_off.value + reg->off;
2224 err = __check_stack_boundary(env, regno, min_off, access_size,
2225 zero_size_allowed);
2226 if (err)
2227 return err;
2228 } else {
Andrey Ignatov088ec262019-04-03 23:22:39 -07002229 /* Variable offset is prohibited for unprivileged mode for
2230 * simplicity since it requires corresponding support in
2231 * Spectre masking for stack ALU.
2232 * See also retrieve_ptr_limit().
2233 */
2234 if (!env->allow_ptr_leaks) {
2235 char tn_buf[48];
2236
2237 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
2238 verbose(env, "R%d indirect variable offset stack access prohibited for !root, var_off=%s\n",
2239 regno, tn_buf);
2240 return -EACCES;
2241 }
Andrey Ignatovf2bcd052019-04-03 23:22:37 -07002242 /* Only initialized buffer on stack is allowed to be accessed
2243 * with variable offset. With uninitialized buffer it's hard to
2244 * guarantee that whole memory is marked as initialized on
2245 * helper return since specific bounds are unknown what may
2246 * cause uninitialized stack leaking.
2247 */
2248 if (meta && meta->raw_mode)
2249 meta = NULL;
2250
Andrey Ignatov2011fcc2019-03-28 18:01:57 -07002251 min_off = reg->smin_value + reg->off;
2252 max_off = reg->umax_value + reg->off;
2253 err = __check_stack_boundary(env, regno, min_off, access_size,
2254 zero_size_allowed);
2255 if (err)
2256 return err;
2257 err = __check_stack_boundary(env, regno, max_off, access_size,
2258 zero_size_allowed);
2259 if (err)
2260 return err;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002261 }
2262
Daniel Borkmann435faee12016-04-13 00:10:51 +02002263 if (meta && meta->raw_mode) {
2264 meta->access_size = access_size;
2265 meta->regno = regno;
2266 return 0;
2267 }
2268
Andrey Ignatov2011fcc2019-03-28 18:01:57 -07002269 for (i = min_off; i < max_off + access_size; i++) {
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08002270 u8 *stype;
2271
Andrey Ignatov2011fcc2019-03-28 18:01:57 -07002272 slot = -i - 1;
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07002273 spi = slot / BPF_REG_SIZE;
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08002274 if (state->allocated_stack <= slot)
2275 goto err;
2276 stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
2277 if (*stype == STACK_MISC)
2278 goto mark;
2279 if (*stype == STACK_ZERO) {
2280 /* helper can write anything into the stack */
2281 *stype = STACK_MISC;
2282 goto mark;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002283 }
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08002284err:
Andrey Ignatov2011fcc2019-03-28 18:01:57 -07002285 if (tnum_is_const(reg->var_off)) {
2286 verbose(env, "invalid indirect read from stack off %d+%d size %d\n",
2287 min_off, i - min_off, access_size);
2288 } else {
2289 char tn_buf[48];
2290
2291 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
2292 verbose(env, "invalid indirect read from stack var_off %s+%d size %d\n",
2293 tn_buf, i - min_off, access_size);
2294 }
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08002295 return -EACCES;
2296mark:
2297 /* reading any byte out of 8-byte 'spill_slot' will cause
2298 * the whole slot to be marked as 'read'
2299 */
Edward Cree679c7822018-08-22 20:02:19 +01002300 mark_reg_read(env, &state->stack[spi].spilled_ptr,
2301 state->stack[spi].spilled_ptr.parent);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002302 }
Andrey Ignatov2011fcc2019-03-28 18:01:57 -07002303 return update_stack_depth(env, state, min_off);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002304}
2305
Gianluca Borello06c1c042017-01-09 10:19:49 -08002306static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,
2307 int access_size, bool zero_size_allowed,
2308 struct bpf_call_arg_meta *meta)
2309{
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07002310 struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
Gianluca Borello06c1c042017-01-09 10:19:49 -08002311
Edward Creef1174f72017-08-07 15:26:19 +01002312 switch (reg->type) {
Gianluca Borello06c1c042017-01-09 10:19:49 -08002313 case PTR_TO_PACKET:
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02002314 case PTR_TO_PACKET_META:
Yonghong Song9fd29c02017-11-12 14:49:09 -08002315 return check_packet_access(env, regno, reg->off, access_size,
2316 zero_size_allowed);
Gianluca Borello06c1c042017-01-09 10:19:49 -08002317 case PTR_TO_MAP_VALUE:
Yonghong Song9fd29c02017-11-12 14:49:09 -08002318 return check_map_access(env, regno, reg->off, access_size,
2319 zero_size_allowed);
Edward Creef1174f72017-08-07 15:26:19 +01002320 default: /* scalar_value|ptr_to_stack or invalid ptr */
Gianluca Borello06c1c042017-01-09 10:19:49 -08002321 return check_stack_boundary(env, regno, access_size,
2322 zero_size_allowed, meta);
2323 }
2324}
2325
Alexei Starovoitovd83525c2019-01-31 15:40:04 -08002326/* Implementation details:
2327 * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL
2328 * Two bpf_map_lookups (even with the same key) will have different reg->id.
2329 * For traditional PTR_TO_MAP_VALUE the verifier clears reg->id after
2330 * value_or_null->value transition, since the verifier only cares about
2331 * the range of access to valid map value pointer and doesn't care about actual
2332 * address of the map element.
2333 * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps
2334 * reg->id > 0 after value_or_null->value transition. By doing so
2335 * two bpf_map_lookups will be considered two different pointers that
2336 * point to different bpf_spin_locks.
2337 * The verifier allows taking only one bpf_spin_lock at a time to avoid
2338 * dead-locks.
2339 * Since only one bpf_spin_lock is allowed the checks are simpler than
2340 * reg_is_refcounted() logic. The verifier needs to remember only
2341 * one spin_lock instead of array of acquired_refs.
2342 * cur_state->active_spin_lock remembers which map value element got locked
2343 * and clears it after bpf_spin_unlock.
2344 */
2345static int process_spin_lock(struct bpf_verifier_env *env, int regno,
2346 bool is_lock)
2347{
2348 struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
2349 struct bpf_verifier_state *cur = env->cur_state;
2350 bool is_const = tnum_is_const(reg->var_off);
2351 struct bpf_map *map = reg->map_ptr;
2352 u64 val = reg->var_off.value;
2353
2354 if (reg->type != PTR_TO_MAP_VALUE) {
2355 verbose(env, "R%d is not a pointer to map_value\n", regno);
2356 return -EINVAL;
2357 }
2358 if (!is_const) {
2359 verbose(env,
2360 "R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n",
2361 regno);
2362 return -EINVAL;
2363 }
2364 if (!map->btf) {
2365 verbose(env,
2366 "map '%s' has to have BTF in order to use bpf_spin_lock\n",
2367 map->name);
2368 return -EINVAL;
2369 }
2370 if (!map_value_has_spin_lock(map)) {
2371 if (map->spin_lock_off == -E2BIG)
2372 verbose(env,
2373 "map '%s' has more than one 'struct bpf_spin_lock'\n",
2374 map->name);
2375 else if (map->spin_lock_off == -ENOENT)
2376 verbose(env,
2377 "map '%s' doesn't have 'struct bpf_spin_lock'\n",
2378 map->name);
2379 else
2380 verbose(env,
2381 "map '%s' is not a struct type or bpf_spin_lock is mangled\n",
2382 map->name);
2383 return -EINVAL;
2384 }
2385 if (map->spin_lock_off != val + reg->off) {
2386 verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock'\n",
2387 val + reg->off);
2388 return -EINVAL;
2389 }
2390 if (is_lock) {
2391 if (cur->active_spin_lock) {
2392 verbose(env,
2393 "Locking two bpf_spin_locks are not allowed\n");
2394 return -EINVAL;
2395 }
2396 cur->active_spin_lock = reg->id;
2397 } else {
2398 if (!cur->active_spin_lock) {
2399 verbose(env, "bpf_spin_unlock without taking a lock\n");
2400 return -EINVAL;
2401 }
2402 if (cur->active_spin_lock != reg->id) {
2403 verbose(env, "bpf_spin_unlock of different lock\n");
2404 return -EINVAL;
2405 }
2406 cur->active_spin_lock = 0;
2407 }
2408 return 0;
2409}
2410
Daniel Borkmann90133412018-01-20 01:24:29 +01002411static bool arg_type_is_mem_ptr(enum bpf_arg_type type)
2412{
2413 return type == ARG_PTR_TO_MEM ||
2414 type == ARG_PTR_TO_MEM_OR_NULL ||
2415 type == ARG_PTR_TO_UNINIT_MEM;
2416}
2417
2418static bool arg_type_is_mem_size(enum bpf_arg_type type)
2419{
2420 return type == ARG_CONST_SIZE ||
2421 type == ARG_CONST_SIZE_OR_ZERO;
2422}
2423
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01002424static int check_func_arg(struct bpf_verifier_env *env, u32 regno,
Daniel Borkmann33ff9822016-04-13 00:10:50 +02002425 enum bpf_arg_type arg_type,
2426 struct bpf_call_arg_meta *meta)
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002427{
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07002428 struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
Alexei Starovoitov6841de82016-08-11 18:17:16 -07002429 enum bpf_reg_type expected_type, type = reg->type;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002430 int err = 0;
2431
Daniel Borkmann80f1d682015-03-12 17:21:42 +01002432 if (arg_type == ARG_DONTCARE)
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002433 return 0;
2434
Edward Creedc503a82017-08-15 20:34:35 +01002435 err = check_reg_arg(env, regno, SRC_OP);
2436 if (err)
2437 return err;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002438
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07002439 if (arg_type == ARG_ANYTHING) {
2440 if (is_pointer_value(env, regno)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002441 verbose(env, "R%d leaks addr into helper function\n",
2442 regno);
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07002443 return -EACCES;
2444 }
Daniel Borkmann80f1d682015-03-12 17:21:42 +01002445 return 0;
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07002446 }
Daniel Borkmann80f1d682015-03-12 17:21:42 +01002447
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02002448 if (type_is_pkt_pointer(type) &&
Thomas Graf3a0af8f2016-11-30 17:10:10 +01002449 !may_access_direct_pkt_data(env, meta, BPF_READ)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002450 verbose(env, "helper access to the packet is not allowed\n");
Alexei Starovoitov6841de82016-08-11 18:17:16 -07002451 return -EACCES;
2452 }
2453
Daniel Borkmann8e2fe1d92016-02-19 23:05:22 +01002454 if (arg_type == ARG_PTR_TO_MAP_KEY ||
Mauricio Vasquez B2ea864c2018-10-18 15:16:20 +02002455 arg_type == ARG_PTR_TO_MAP_VALUE ||
2456 arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) {
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002457 expected_type = PTR_TO_STACK;
Paul Chaignond71962f2018-04-24 15:07:54 +02002458 if (!type_is_pkt_pointer(type) && type != PTR_TO_MAP_VALUE &&
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02002459 type != expected_type)
Alexei Starovoitov6841de82016-08-11 18:17:16 -07002460 goto err_type;
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -08002461 } else if (arg_type == ARG_CONST_SIZE ||
2462 arg_type == ARG_CONST_SIZE_OR_ZERO) {
Edward Creef1174f72017-08-07 15:26:19 +01002463 expected_type = SCALAR_VALUE;
2464 if (type != expected_type)
Alexei Starovoitov6841de82016-08-11 18:17:16 -07002465 goto err_type;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002466 } else if (arg_type == ARG_CONST_MAP_PTR) {
2467 expected_type = CONST_PTR_TO_MAP;
Alexei Starovoitov6841de82016-08-11 18:17:16 -07002468 if (type != expected_type)
2469 goto err_type;
Alexei Starovoitov608cd712015-03-26 19:53:57 -07002470 } else if (arg_type == ARG_PTR_TO_CTX) {
2471 expected_type = PTR_TO_CTX;
Alexei Starovoitov6841de82016-08-11 18:17:16 -07002472 if (type != expected_type)
2473 goto err_type;
Daniel Borkmann58990d12018-06-07 17:40:03 +02002474 err = check_ctx_reg(env, reg, regno);
2475 if (err < 0)
2476 return err;
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08002477 } else if (arg_type == ARG_PTR_TO_SOCK_COMMON) {
2478 expected_type = PTR_TO_SOCK_COMMON;
2479 /* Any sk pointer can be ARG_PTR_TO_SOCK_COMMON */
2480 if (!type_is_sk_pointer(type))
2481 goto err_type;
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002482 if (reg->ref_obj_id) {
2483 if (meta->ref_obj_id) {
2484 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n",
2485 regno, reg->ref_obj_id,
2486 meta->ref_obj_id);
2487 return -EFAULT;
2488 }
2489 meta->ref_obj_id = reg->ref_obj_id;
Joe Stringerfd978bf72018-10-02 13:35:35 -07002490 }
Alexei Starovoitovd83525c2019-01-31 15:40:04 -08002491 } else if (arg_type == ARG_PTR_TO_SPIN_LOCK) {
2492 if (meta->func_id == BPF_FUNC_spin_lock) {
2493 if (process_spin_lock(env, regno, true))
2494 return -EACCES;
2495 } else if (meta->func_id == BPF_FUNC_spin_unlock) {
2496 if (process_spin_lock(env, regno, false))
2497 return -EACCES;
2498 } else {
2499 verbose(env, "verifier internal error\n");
2500 return -EFAULT;
2501 }
Daniel Borkmann90133412018-01-20 01:24:29 +01002502 } else if (arg_type_is_mem_ptr(arg_type)) {
Daniel Borkmann8e2fe1d92016-02-19 23:05:22 +01002503 expected_type = PTR_TO_STACK;
2504 /* One exception here. In case function allows for NULL to be
Edward Creef1174f72017-08-07 15:26:19 +01002505 * passed in as argument, it's a SCALAR_VALUE type. Final test
Daniel Borkmann8e2fe1d92016-02-19 23:05:22 +01002506 * happens during stack boundary checking.
2507 */
Alexei Starovoitov914cb782017-11-30 21:31:40 -08002508 if (register_is_null(reg) &&
Gianluca Borellodb1ac492017-11-22 18:32:53 +00002509 arg_type == ARG_PTR_TO_MEM_OR_NULL)
Alexei Starovoitov6841de82016-08-11 18:17:16 -07002510 /* final test in check_stack_boundary() */;
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02002511 else if (!type_is_pkt_pointer(type) &&
2512 type != PTR_TO_MAP_VALUE &&
Edward Creef1174f72017-08-07 15:26:19 +01002513 type != expected_type)
Alexei Starovoitov6841de82016-08-11 18:17:16 -07002514 goto err_type;
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -08002515 meta->raw_mode = arg_type == ARG_PTR_TO_UNINIT_MEM;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002516 } else {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002517 verbose(env, "unsupported arg_type %d\n", arg_type);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002518 return -EFAULT;
2519 }
2520
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002521 if (arg_type == ARG_CONST_MAP_PTR) {
2522 /* bpf_map_xxx(map_ptr) call: remember that map_ptr */
Daniel Borkmann33ff9822016-04-13 00:10:50 +02002523 meta->map_ptr = reg->map_ptr;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002524 } else if (arg_type == ARG_PTR_TO_MAP_KEY) {
2525 /* bpf_map_xxx(..., map_ptr, ..., key) call:
2526 * check that [key, key + map->key_size) are within
2527 * stack limits and initialized
2528 */
Daniel Borkmann33ff9822016-04-13 00:10:50 +02002529 if (!meta->map_ptr) {
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002530 /* in function declaration map_ptr must come before
2531 * map_key, so that it's verified and known before
2532 * we have to check map_key here. Otherwise it means
2533 * that kernel subsystem misconfigured verifier
2534 */
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002535 verbose(env, "invalid map_ptr to access map->key\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002536 return -EACCES;
2537 }
Paul Chaignond71962f2018-04-24 15:07:54 +02002538 err = check_helper_mem_access(env, regno,
2539 meta->map_ptr->key_size, false,
2540 NULL);
Mauricio Vasquez B2ea864c2018-10-18 15:16:20 +02002541 } else if (arg_type == ARG_PTR_TO_MAP_VALUE ||
2542 arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) {
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002543 /* bpf_map_xxx(..., map_ptr, ..., value) call:
2544 * check [value, value + map->value_size) validity
2545 */
Daniel Borkmann33ff9822016-04-13 00:10:50 +02002546 if (!meta->map_ptr) {
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002547 /* kernel subsystem misconfigured verifier */
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002548 verbose(env, "invalid map_ptr to access map->value\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002549 return -EACCES;
2550 }
Mauricio Vasquez B2ea864c2018-10-18 15:16:20 +02002551 meta->raw_mode = (arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE);
Paul Chaignond71962f2018-04-24 15:07:54 +02002552 err = check_helper_mem_access(env, regno,
2553 meta->map_ptr->value_size, false,
Mauricio Vasquez B2ea864c2018-10-18 15:16:20 +02002554 meta);
Daniel Borkmann90133412018-01-20 01:24:29 +01002555 } else if (arg_type_is_mem_size(arg_type)) {
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -08002556 bool zero_size_allowed = (arg_type == ARG_CONST_SIZE_OR_ZERO);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002557
Yonghong Song849fa502018-04-28 22:28:09 -07002558 /* remember the mem_size which may be used later
2559 * to refine return values.
2560 */
2561 meta->msize_smax_value = reg->smax_value;
2562 meta->msize_umax_value = reg->umax_value;
2563
Edward Creef1174f72017-08-07 15:26:19 +01002564 /* The register is SCALAR_VALUE; the access check
2565 * happens using its boundaries.
Gianluca Borello06c1c042017-01-09 10:19:49 -08002566 */
Edward Creef1174f72017-08-07 15:26:19 +01002567 if (!tnum_is_const(reg->var_off))
Gianluca Borello06c1c042017-01-09 10:19:49 -08002568 /* For unprivileged variable accesses, disable raw
2569 * mode so that the program is required to
2570 * initialize all the memory that the helper could
2571 * just partially fill up.
2572 */
2573 meta = NULL;
2574
Edward Creeb03c9f92017-08-07 15:26:36 +01002575 if (reg->smin_value < 0) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002576 verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n",
Edward Creef1174f72017-08-07 15:26:19 +01002577 regno);
2578 return -EACCES;
2579 }
Gianluca Borello06c1c042017-01-09 10:19:49 -08002580
Edward Creeb03c9f92017-08-07 15:26:36 +01002581 if (reg->umin_value == 0) {
Edward Creef1174f72017-08-07 15:26:19 +01002582 err = check_helper_mem_access(env, regno - 1, 0,
2583 zero_size_allowed,
2584 meta);
Gianluca Borello06c1c042017-01-09 10:19:49 -08002585 if (err)
2586 return err;
Gianluca Borello06c1c042017-01-09 10:19:49 -08002587 }
Edward Creef1174f72017-08-07 15:26:19 +01002588
Edward Creeb03c9f92017-08-07 15:26:36 +01002589 if (reg->umax_value >= BPF_MAX_VAR_SIZ) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002590 verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n",
Edward Creef1174f72017-08-07 15:26:19 +01002591 regno);
2592 return -EACCES;
2593 }
2594 err = check_helper_mem_access(env, regno - 1,
Edward Creeb03c9f92017-08-07 15:26:36 +01002595 reg->umax_value,
Edward Creef1174f72017-08-07 15:26:19 +01002596 zero_size_allowed, meta);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002597 }
2598
2599 return err;
Alexei Starovoitov6841de82016-08-11 18:17:16 -07002600err_type:
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002601 verbose(env, "R%d type=%s expected=%s\n", regno,
Alexei Starovoitov6841de82016-08-11 18:17:16 -07002602 reg_type_str[type], reg_type_str[expected_type]);
2603 return -EACCES;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002604}
2605
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002606static int check_map_func_compatibility(struct bpf_verifier_env *env,
2607 struct bpf_map *map, int func_id)
Kaixu Xia35578d72015-08-06 07:02:35 +00002608{
Kaixu Xia35578d72015-08-06 07:02:35 +00002609 if (!map)
2610 return 0;
2611
Alexei Starovoitov6aff67c2016-04-27 18:56:21 -07002612 /* We need a two way check, first is from map perspective ... */
2613 switch (map->map_type) {
2614 case BPF_MAP_TYPE_PROG_ARRAY:
2615 if (func_id != BPF_FUNC_tail_call)
2616 goto error;
2617 break;
2618 case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
2619 if (func_id != BPF_FUNC_perf_event_read &&
Yonghong Song908432c2017-10-05 09:19:20 -07002620 func_id != BPF_FUNC_perf_event_output &&
2621 func_id != BPF_FUNC_perf_event_read_value)
Alexei Starovoitov6aff67c2016-04-27 18:56:21 -07002622 goto error;
2623 break;
2624 case BPF_MAP_TYPE_STACK_TRACE:
2625 if (func_id != BPF_FUNC_get_stackid)
2626 goto error;
2627 break;
Martin KaFai Lau4ed8ec52016-06-30 10:28:43 -07002628 case BPF_MAP_TYPE_CGROUP_ARRAY:
David S. Miller60747ef2016-08-18 01:17:32 -04002629 if (func_id != BPF_FUNC_skb_under_cgroup &&
Sargun Dhillon60d20f92016-08-12 08:56:52 -07002630 func_id != BPF_FUNC_current_task_under_cgroup)
Martin KaFai Lau4a482f32016-06-30 10:28:44 -07002631 goto error;
2632 break;
Roman Gushchincd339432018-08-02 14:27:24 -07002633 case BPF_MAP_TYPE_CGROUP_STORAGE:
Roman Gushchinb741f162018-09-28 14:45:43 +00002634 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE:
Roman Gushchincd339432018-08-02 14:27:24 -07002635 if (func_id != BPF_FUNC_get_local_storage)
2636 goto error;
2637 break;
John Fastabend546ac1f2017-07-17 09:28:56 -07002638 /* devmap returns a pointer to a live net_device ifindex that we cannot
2639 * allow to be modified from bpf side. So do not allow lookup elements
2640 * for now.
2641 */
2642 case BPF_MAP_TYPE_DEVMAP:
John Fastabend2ddf71e2017-07-17 09:30:02 -07002643 if (func_id != BPF_FUNC_redirect_map)
John Fastabend546ac1f2017-07-17 09:28:56 -07002644 goto error;
2645 break;
Björn Töpelfbfc504a2018-05-02 13:01:28 +02002646 /* Restrict bpf side of cpumap and xskmap, open when use-cases
2647 * appear.
2648 */
Jesper Dangaard Brouer6710e112017-10-16 12:19:28 +02002649 case BPF_MAP_TYPE_CPUMAP:
Björn Töpelfbfc504a2018-05-02 13:01:28 +02002650 case BPF_MAP_TYPE_XSKMAP:
Jesper Dangaard Brouer6710e112017-10-16 12:19:28 +02002651 if (func_id != BPF_FUNC_redirect_map)
2652 goto error;
2653 break;
Martin KaFai Lau56f668d2017-03-22 10:00:33 -07002654 case BPF_MAP_TYPE_ARRAY_OF_MAPS:
Martin KaFai Laubcc6b1b2017-03-22 10:00:34 -07002655 case BPF_MAP_TYPE_HASH_OF_MAPS:
Martin KaFai Lau56f668d2017-03-22 10:00:33 -07002656 if (func_id != BPF_FUNC_map_lookup_elem)
2657 goto error;
Martin KaFai Lau16a43622017-08-17 18:14:43 -07002658 break;
John Fastabend174a79f2017-08-15 22:32:47 -07002659 case BPF_MAP_TYPE_SOCKMAP:
2660 if (func_id != BPF_FUNC_sk_redirect_map &&
2661 func_id != BPF_FUNC_sock_map_update &&
John Fastabend4f738ad2018-03-18 12:57:10 -07002662 func_id != BPF_FUNC_map_delete_elem &&
2663 func_id != BPF_FUNC_msg_redirect_map)
John Fastabend174a79f2017-08-15 22:32:47 -07002664 goto error;
2665 break;
John Fastabend81110382018-05-14 10:00:17 -07002666 case BPF_MAP_TYPE_SOCKHASH:
2667 if (func_id != BPF_FUNC_sk_redirect_hash &&
2668 func_id != BPF_FUNC_sock_hash_update &&
2669 func_id != BPF_FUNC_map_delete_elem &&
2670 func_id != BPF_FUNC_msg_redirect_hash)
2671 goto error;
2672 break;
Martin KaFai Lau2dbb9b92018-08-08 01:01:25 -07002673 case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY:
2674 if (func_id != BPF_FUNC_sk_select_reuseport)
2675 goto error;
2676 break;
Mauricio Vasquez Bf1a2e442018-10-18 15:16:25 +02002677 case BPF_MAP_TYPE_QUEUE:
2678 case BPF_MAP_TYPE_STACK:
2679 if (func_id != BPF_FUNC_map_peek_elem &&
2680 func_id != BPF_FUNC_map_pop_elem &&
2681 func_id != BPF_FUNC_map_push_elem)
2682 goto error;
2683 break;
Alexei Starovoitov6aff67c2016-04-27 18:56:21 -07002684 default:
2685 break;
2686 }
2687
2688 /* ... and second from the function itself. */
2689 switch (func_id) {
2690 case BPF_FUNC_tail_call:
2691 if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY)
2692 goto error;
Jiong Wangf910cef2018-05-02 16:17:17 -04002693 if (env->subprog_cnt > 1) {
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002694 verbose(env, "tail_calls are not allowed in programs with bpf-to-bpf calls\n");
2695 return -EINVAL;
2696 }
Alexei Starovoitov6aff67c2016-04-27 18:56:21 -07002697 break;
2698 case BPF_FUNC_perf_event_read:
2699 case BPF_FUNC_perf_event_output:
Yonghong Song908432c2017-10-05 09:19:20 -07002700 case BPF_FUNC_perf_event_read_value:
Alexei Starovoitov6aff67c2016-04-27 18:56:21 -07002701 if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY)
2702 goto error;
2703 break;
2704 case BPF_FUNC_get_stackid:
2705 if (map->map_type != BPF_MAP_TYPE_STACK_TRACE)
2706 goto error;
2707 break;
Sargun Dhillon60d20f92016-08-12 08:56:52 -07002708 case BPF_FUNC_current_task_under_cgroup:
Daniel Borkmann747ea552016-08-12 22:17:17 +02002709 case BPF_FUNC_skb_under_cgroup:
Martin KaFai Lau4a482f32016-06-30 10:28:44 -07002710 if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY)
2711 goto error;
2712 break;
John Fastabend97f91a72017-07-17 09:29:18 -07002713 case BPF_FUNC_redirect_map:
Jesper Dangaard Brouer9c270af2017-10-16 12:19:34 +02002714 if (map->map_type != BPF_MAP_TYPE_DEVMAP &&
Björn Töpelfbfc504a2018-05-02 13:01:28 +02002715 map->map_type != BPF_MAP_TYPE_CPUMAP &&
2716 map->map_type != BPF_MAP_TYPE_XSKMAP)
John Fastabend97f91a72017-07-17 09:29:18 -07002717 goto error;
2718 break;
John Fastabend174a79f2017-08-15 22:32:47 -07002719 case BPF_FUNC_sk_redirect_map:
John Fastabend4f738ad2018-03-18 12:57:10 -07002720 case BPF_FUNC_msg_redirect_map:
John Fastabend81110382018-05-14 10:00:17 -07002721 case BPF_FUNC_sock_map_update:
John Fastabend174a79f2017-08-15 22:32:47 -07002722 if (map->map_type != BPF_MAP_TYPE_SOCKMAP)
2723 goto error;
2724 break;
John Fastabend81110382018-05-14 10:00:17 -07002725 case BPF_FUNC_sk_redirect_hash:
2726 case BPF_FUNC_msg_redirect_hash:
2727 case BPF_FUNC_sock_hash_update:
2728 if (map->map_type != BPF_MAP_TYPE_SOCKHASH)
John Fastabend174a79f2017-08-15 22:32:47 -07002729 goto error;
2730 break;
Roman Gushchincd339432018-08-02 14:27:24 -07002731 case BPF_FUNC_get_local_storage:
Roman Gushchinb741f162018-09-28 14:45:43 +00002732 if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
2733 map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
Roman Gushchincd339432018-08-02 14:27:24 -07002734 goto error;
2735 break;
Martin KaFai Lau2dbb9b92018-08-08 01:01:25 -07002736 case BPF_FUNC_sk_select_reuseport:
2737 if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY)
2738 goto error;
2739 break;
Mauricio Vasquez Bf1a2e442018-10-18 15:16:25 +02002740 case BPF_FUNC_map_peek_elem:
2741 case BPF_FUNC_map_pop_elem:
2742 case BPF_FUNC_map_push_elem:
2743 if (map->map_type != BPF_MAP_TYPE_QUEUE &&
2744 map->map_type != BPF_MAP_TYPE_STACK)
2745 goto error;
2746 break;
Alexei Starovoitov6aff67c2016-04-27 18:56:21 -07002747 default:
2748 break;
Kaixu Xia35578d72015-08-06 07:02:35 +00002749 }
2750
2751 return 0;
Alexei Starovoitov6aff67c2016-04-27 18:56:21 -07002752error:
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002753 verbose(env, "cannot pass map_type %d into func %s#%d\n",
Thomas Grafebb676d2016-10-27 11:23:51 +02002754 map->map_type, func_id_name(func_id), func_id);
Alexei Starovoitov6aff67c2016-04-27 18:56:21 -07002755 return -EINVAL;
Kaixu Xia35578d72015-08-06 07:02:35 +00002756}
2757
Daniel Borkmann90133412018-01-20 01:24:29 +01002758static bool check_raw_mode_ok(const struct bpf_func_proto *fn)
Daniel Borkmann435faee12016-04-13 00:10:51 +02002759{
2760 int count = 0;
2761
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -08002762 if (fn->arg1_type == ARG_PTR_TO_UNINIT_MEM)
Daniel Borkmann435faee12016-04-13 00:10:51 +02002763 count++;
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -08002764 if (fn->arg2_type == ARG_PTR_TO_UNINIT_MEM)
Daniel Borkmann435faee12016-04-13 00:10:51 +02002765 count++;
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -08002766 if (fn->arg3_type == ARG_PTR_TO_UNINIT_MEM)
Daniel Borkmann435faee12016-04-13 00:10:51 +02002767 count++;
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -08002768 if (fn->arg4_type == ARG_PTR_TO_UNINIT_MEM)
Daniel Borkmann435faee12016-04-13 00:10:51 +02002769 count++;
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -08002770 if (fn->arg5_type == ARG_PTR_TO_UNINIT_MEM)
Daniel Borkmann435faee12016-04-13 00:10:51 +02002771 count++;
2772
Daniel Borkmann90133412018-01-20 01:24:29 +01002773 /* We only support one arg being in raw mode at the moment,
2774 * which is sufficient for the helper functions we have
2775 * right now.
2776 */
2777 return count <= 1;
2778}
2779
2780static bool check_args_pair_invalid(enum bpf_arg_type arg_curr,
2781 enum bpf_arg_type arg_next)
2782{
2783 return (arg_type_is_mem_ptr(arg_curr) &&
2784 !arg_type_is_mem_size(arg_next)) ||
2785 (!arg_type_is_mem_ptr(arg_curr) &&
2786 arg_type_is_mem_size(arg_next));
2787}
2788
2789static bool check_arg_pair_ok(const struct bpf_func_proto *fn)
2790{
2791 /* bpf_xxx(..., buf, len) call will access 'len'
2792 * bytes from memory 'buf'. Both arg types need
2793 * to be paired, so make sure there's no buggy
2794 * helper function specification.
2795 */
2796 if (arg_type_is_mem_size(fn->arg1_type) ||
2797 arg_type_is_mem_ptr(fn->arg5_type) ||
2798 check_args_pair_invalid(fn->arg1_type, fn->arg2_type) ||
2799 check_args_pair_invalid(fn->arg2_type, fn->arg3_type) ||
2800 check_args_pair_invalid(fn->arg3_type, fn->arg4_type) ||
2801 check_args_pair_invalid(fn->arg4_type, fn->arg5_type))
2802 return false;
2803
2804 return true;
2805}
2806
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002807static bool check_refcount_ok(const struct bpf_func_proto *fn, int func_id)
Joe Stringerfd978bf72018-10-02 13:35:35 -07002808{
2809 int count = 0;
2810
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002811 if (arg_type_may_be_refcounted(fn->arg1_type))
Joe Stringerfd978bf72018-10-02 13:35:35 -07002812 count++;
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002813 if (arg_type_may_be_refcounted(fn->arg2_type))
Joe Stringerfd978bf72018-10-02 13:35:35 -07002814 count++;
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002815 if (arg_type_may_be_refcounted(fn->arg3_type))
Joe Stringerfd978bf72018-10-02 13:35:35 -07002816 count++;
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002817 if (arg_type_may_be_refcounted(fn->arg4_type))
Joe Stringerfd978bf72018-10-02 13:35:35 -07002818 count++;
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002819 if (arg_type_may_be_refcounted(fn->arg5_type))
Joe Stringerfd978bf72018-10-02 13:35:35 -07002820 count++;
2821
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002822 /* A reference acquiring function cannot acquire
2823 * another refcounted ptr.
2824 */
2825 if (is_acquire_function(func_id) && count)
2826 return false;
2827
Joe Stringerfd978bf72018-10-02 13:35:35 -07002828 /* We only support one arg being unreferenced at the moment,
2829 * which is sufficient for the helper functions we have right now.
2830 */
2831 return count <= 1;
2832}
2833
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002834static int check_func_proto(const struct bpf_func_proto *fn, int func_id)
Daniel Borkmann90133412018-01-20 01:24:29 +01002835{
2836 return check_raw_mode_ok(fn) &&
Joe Stringerfd978bf72018-10-02 13:35:35 -07002837 check_arg_pair_ok(fn) &&
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002838 check_refcount_ok(fn, func_id) ? 0 : -EINVAL;
Daniel Borkmann435faee12016-04-13 00:10:51 +02002839}
2840
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02002841/* Packet data might have moved, any old PTR_TO_PACKET[_META,_END]
2842 * are now invalid, so turn them into unknown SCALAR_VALUE.
Edward Creef1174f72017-08-07 15:26:19 +01002843 */
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002844static void __clear_all_pkt_pointers(struct bpf_verifier_env *env,
2845 struct bpf_func_state *state)
Alexei Starovoitov969bf052016-05-05 19:49:10 -07002846{
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01002847 struct bpf_reg_state *regs = state->regs, *reg;
Alexei Starovoitov969bf052016-05-05 19:49:10 -07002848 int i;
2849
2850 for (i = 0; i < MAX_BPF_REG; i++)
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02002851 if (reg_is_pkt_pointer_any(&regs[i]))
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002852 mark_reg_unknown(env, regs, i);
Alexei Starovoitov969bf052016-05-05 19:49:10 -07002853
Joe Stringerf3709f62018-10-02 13:35:29 -07002854 bpf_for_each_spilled_reg(i, state, reg) {
2855 if (!reg)
Alexei Starovoitov969bf052016-05-05 19:49:10 -07002856 continue;
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02002857 if (reg_is_pkt_pointer_any(reg))
2858 __mark_reg_unknown(reg);
Alexei Starovoitov969bf052016-05-05 19:49:10 -07002859 }
2860}
2861
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002862static void clear_all_pkt_pointers(struct bpf_verifier_env *env)
2863{
2864 struct bpf_verifier_state *vstate = env->cur_state;
2865 int i;
2866
2867 for (i = 0; i <= vstate->curframe; i++)
2868 __clear_all_pkt_pointers(env, vstate->frame[i]);
2869}
2870
Joe Stringerfd978bf72018-10-02 13:35:35 -07002871static void release_reg_references(struct bpf_verifier_env *env,
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002872 struct bpf_func_state *state,
2873 int ref_obj_id)
Joe Stringerfd978bf72018-10-02 13:35:35 -07002874{
2875 struct bpf_reg_state *regs = state->regs, *reg;
2876 int i;
2877
2878 for (i = 0; i < MAX_BPF_REG; i++)
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002879 if (regs[i].ref_obj_id == ref_obj_id)
Joe Stringerfd978bf72018-10-02 13:35:35 -07002880 mark_reg_unknown(env, regs, i);
2881
2882 bpf_for_each_spilled_reg(i, state, reg) {
2883 if (!reg)
2884 continue;
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002885 if (reg->ref_obj_id == ref_obj_id)
Joe Stringerfd978bf72018-10-02 13:35:35 -07002886 __mark_reg_unknown(reg);
2887 }
2888}
2889
2890/* The pointer with the specified id has released its reference to kernel
2891 * resources. Identify all copies of the same pointer and clear the reference.
2892 */
2893static int release_reference(struct bpf_verifier_env *env,
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002894 int ref_obj_id)
Joe Stringerfd978bf72018-10-02 13:35:35 -07002895{
2896 struct bpf_verifier_state *vstate = env->cur_state;
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002897 int err;
Joe Stringerfd978bf72018-10-02 13:35:35 -07002898 int i;
2899
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002900 err = release_reference_state(cur_func(env), ref_obj_id);
2901 if (err)
2902 return err;
Joe Stringerfd978bf72018-10-02 13:35:35 -07002903
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002904 for (i = 0; i <= vstate->curframe; i++)
2905 release_reg_references(env, vstate->frame[i], ref_obj_id);
2906
2907 return 0;
Joe Stringerfd978bf72018-10-02 13:35:35 -07002908}
2909
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002910static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
2911 int *insn_idx)
2912{
2913 struct bpf_verifier_state *state = env->cur_state;
2914 struct bpf_func_state *caller, *callee;
Joe Stringerfd978bf72018-10-02 13:35:35 -07002915 int i, err, subprog, target_insn;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002916
Alexei Starovoitovaada9ce2017-12-25 13:15:42 -08002917 if (state->curframe + 1 >= MAX_CALL_FRAMES) {
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002918 verbose(env, "the call stack of %d frames is too deep\n",
Alexei Starovoitovaada9ce2017-12-25 13:15:42 -08002919 state->curframe + 2);
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002920 return -E2BIG;
2921 }
2922
2923 target_insn = *insn_idx + insn->imm;
2924 subprog = find_subprog(env, target_insn + 1);
2925 if (subprog < 0) {
2926 verbose(env, "verifier bug. No program starts at insn %d\n",
2927 target_insn + 1);
2928 return -EFAULT;
2929 }
2930
2931 caller = state->frame[state->curframe];
2932 if (state->frame[state->curframe + 1]) {
2933 verbose(env, "verifier bug. Frame %d already allocated\n",
2934 state->curframe + 1);
2935 return -EFAULT;
2936 }
2937
2938 callee = kzalloc(sizeof(*callee), GFP_KERNEL);
2939 if (!callee)
2940 return -ENOMEM;
2941 state->frame[state->curframe + 1] = callee;
2942
2943 /* callee cannot access r0, r6 - r9 for reading and has to write
2944 * into its own stack before reading from it.
2945 * callee can read/write into caller's stack
2946 */
2947 init_func_state(env, callee,
2948 /* remember the callsite, it will be used by bpf_exit */
2949 *insn_idx /* callsite */,
2950 state->curframe + 1 /* frameno within this callchain */,
Jiong Wangf910cef2018-05-02 16:17:17 -04002951 subprog /* subprog number within this prog */);
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002952
Joe Stringerfd978bf72018-10-02 13:35:35 -07002953 /* Transfer references to the callee */
2954 err = transfer_reference_state(callee, caller);
2955 if (err)
2956 return err;
2957
Edward Cree679c7822018-08-22 20:02:19 +01002958 /* copy r1 - r5 args that callee can access. The copy includes parent
2959 * pointers, which connects us up to the liveness chain
2960 */
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002961 for (i = BPF_REG_1; i <= BPF_REG_5; i++)
2962 callee->regs[i] = caller->regs[i];
2963
Edward Cree679c7822018-08-22 20:02:19 +01002964 /* after the call registers r0 - r5 were scratched */
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002965 for (i = 0; i < CALLER_SAVED_REGS; i++) {
2966 mark_reg_not_init(env, caller->regs, caller_saved[i]);
2967 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
2968 }
2969
2970 /* only increment it after check_reg_arg() finished */
2971 state->curframe++;
2972
2973 /* and go analyze first insn of the callee */
2974 *insn_idx = target_insn;
2975
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07002976 if (env->log.level & BPF_LOG_LEVEL) {
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002977 verbose(env, "caller:\n");
2978 print_verifier_state(env, caller);
2979 verbose(env, "callee:\n");
2980 print_verifier_state(env, callee);
2981 }
2982 return 0;
2983}
2984
2985static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
2986{
2987 struct bpf_verifier_state *state = env->cur_state;
2988 struct bpf_func_state *caller, *callee;
2989 struct bpf_reg_state *r0;
Joe Stringerfd978bf72018-10-02 13:35:35 -07002990 int err;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002991
2992 callee = state->frame[state->curframe];
2993 r0 = &callee->regs[BPF_REG_0];
2994 if (r0->type == PTR_TO_STACK) {
2995 /* technically it's ok to return caller's stack pointer
2996 * (or caller's caller's pointer) back to the caller,
2997 * since these pointers are valid. Only current stack
2998 * pointer will be invalid as soon as function exits,
2999 * but let's be conservative
3000 */
3001 verbose(env, "cannot return stack pointer to the caller\n");
3002 return -EINVAL;
3003 }
3004
3005 state->curframe--;
3006 caller = state->frame[state->curframe];
3007 /* return to the caller whatever r0 had in the callee */
3008 caller->regs[BPF_REG_0] = *r0;
3009
Joe Stringerfd978bf72018-10-02 13:35:35 -07003010 /* Transfer references to the caller */
3011 err = transfer_reference_state(caller, callee);
3012 if (err)
3013 return err;
3014
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08003015 *insn_idx = callee->callsite + 1;
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07003016 if (env->log.level & BPF_LOG_LEVEL) {
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08003017 verbose(env, "returning from callee:\n");
3018 print_verifier_state(env, callee);
3019 verbose(env, "to caller at %d:\n", *insn_idx);
3020 print_verifier_state(env, caller);
3021 }
3022 /* clear everything in the callee */
3023 free_func_state(callee);
3024 state->frame[state->curframe + 1] = NULL;
3025 return 0;
3026}
3027
Yonghong Song849fa502018-04-28 22:28:09 -07003028static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type,
3029 int func_id,
3030 struct bpf_call_arg_meta *meta)
3031{
3032 struct bpf_reg_state *ret_reg = &regs[BPF_REG_0];
3033
3034 if (ret_type != RET_INTEGER ||
3035 (func_id != BPF_FUNC_get_stack &&
3036 func_id != BPF_FUNC_probe_read_str))
3037 return;
3038
3039 ret_reg->smax_value = meta->msize_smax_value;
3040 ret_reg->umax_value = meta->msize_umax_value;
3041 __reg_deduce_bounds(ret_reg);
3042 __reg_bound_offset(ret_reg);
3043}
3044
Daniel Borkmannc93552c2018-05-24 02:32:53 +02003045static int
3046record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
3047 int func_id, int insn_idx)
3048{
3049 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
3050
3051 if (func_id != BPF_FUNC_tail_call &&
Daniel Borkmann09772d92018-06-02 23:06:35 +02003052 func_id != BPF_FUNC_map_lookup_elem &&
3053 func_id != BPF_FUNC_map_update_elem &&
Mauricio Vasquez Bf1a2e442018-10-18 15:16:25 +02003054 func_id != BPF_FUNC_map_delete_elem &&
3055 func_id != BPF_FUNC_map_push_elem &&
3056 func_id != BPF_FUNC_map_pop_elem &&
3057 func_id != BPF_FUNC_map_peek_elem)
Daniel Borkmannc93552c2018-05-24 02:32:53 +02003058 return 0;
Daniel Borkmann09772d92018-06-02 23:06:35 +02003059
Daniel Borkmannc93552c2018-05-24 02:32:53 +02003060 if (meta->map_ptr == NULL) {
3061 verbose(env, "kernel subsystem misconfigured verifier\n");
3062 return -EINVAL;
3063 }
3064
3065 if (!BPF_MAP_PTR(aux->map_state))
3066 bpf_map_ptr_store(aux, meta->map_ptr,
3067 meta->map_ptr->unpriv_array);
3068 else if (BPF_MAP_PTR(aux->map_state) != meta->map_ptr)
3069 bpf_map_ptr_store(aux, BPF_MAP_PTR_POISON,
3070 meta->map_ptr->unpriv_array);
3071 return 0;
3072}
3073
Joe Stringerfd978bf72018-10-02 13:35:35 -07003074static int check_reference_leak(struct bpf_verifier_env *env)
3075{
3076 struct bpf_func_state *state = cur_func(env);
3077 int i;
3078
3079 for (i = 0; i < state->acquired_refs; i++) {
3080 verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",
3081 state->refs[i].id, state->refs[i].insn_idx);
3082 }
3083 return state->acquired_refs ? -EINVAL : 0;
3084}
3085
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08003086static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn_idx)
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003087{
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003088 const struct bpf_func_proto *fn = NULL;
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07003089 struct bpf_reg_state *regs;
Daniel Borkmann33ff9822016-04-13 00:10:50 +02003090 struct bpf_call_arg_meta meta;
Alexei Starovoitov969bf052016-05-05 19:49:10 -07003091 bool changes_data;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003092 int i, err;
3093
3094 /* find function prototype */
3095 if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07003096 verbose(env, "invalid func %s#%d\n", func_id_name(func_id),
3097 func_id);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003098 return -EINVAL;
3099 }
3100
Jakub Kicinski00176a32017-10-16 16:40:54 -07003101 if (env->ops->get_func_proto)
Andrey Ignatov5e43f892018-03-30 15:08:00 -07003102 fn = env->ops->get_func_proto(func_id, env->prog);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003103 if (!fn) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07003104 verbose(env, "unknown func %s#%d\n", func_id_name(func_id),
3105 func_id);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003106 return -EINVAL;
3107 }
3108
3109 /* eBPF programs must be GPL compatible to use GPL-ed functions */
Daniel Borkmann24701ec2015-03-01 12:31:47 +01003110 if (!env->prog->gpl_compatible && fn->gpl_only) {
Daniel Borkmann3fe28672018-06-02 23:06:33 +02003111 verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003112 return -EINVAL;
3113 }
3114
Daniel Borkmann04514d12017-12-14 21:07:25 +01003115 /* With LD_ABS/IND some JITs save/restore skb from r1. */
Martin KaFai Lau17bedab2016-12-07 15:53:11 -08003116 changes_data = bpf_helper_changes_pkt_data(fn->func);
Daniel Borkmann04514d12017-12-14 21:07:25 +01003117 if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) {
3118 verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n",
3119 func_id_name(func_id), func_id);
3120 return -EINVAL;
3121 }
Alexei Starovoitov969bf052016-05-05 19:49:10 -07003122
Daniel Borkmann33ff9822016-04-13 00:10:50 +02003123 memset(&meta, 0, sizeof(meta));
Daniel Borkmann36bbef52016-09-20 00:26:13 +02003124 meta.pkt_access = fn->pkt_access;
Daniel Borkmann33ff9822016-04-13 00:10:50 +02003125
Martin KaFai Lau1b986582019-03-12 10:23:02 -07003126 err = check_func_proto(fn, func_id);
Daniel Borkmann435faee12016-04-13 00:10:51 +02003127 if (err) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07003128 verbose(env, "kernel subsystem misconfigured func %s#%d\n",
Thomas Grafebb676d2016-10-27 11:23:51 +02003129 func_id_name(func_id), func_id);
Daniel Borkmann435faee12016-04-13 00:10:51 +02003130 return err;
3131 }
3132
Alexei Starovoitovd83525c2019-01-31 15:40:04 -08003133 meta.func_id = func_id;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003134 /* check args */
Daniel Borkmann33ff9822016-04-13 00:10:50 +02003135 err = check_func_arg(env, BPF_REG_1, fn->arg1_type, &meta);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003136 if (err)
3137 return err;
Daniel Borkmann33ff9822016-04-13 00:10:50 +02003138 err = check_func_arg(env, BPF_REG_2, fn->arg2_type, &meta);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003139 if (err)
3140 return err;
Daniel Borkmann33ff9822016-04-13 00:10:50 +02003141 err = check_func_arg(env, BPF_REG_3, fn->arg3_type, &meta);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003142 if (err)
3143 return err;
Daniel Borkmann33ff9822016-04-13 00:10:50 +02003144 err = check_func_arg(env, BPF_REG_4, fn->arg4_type, &meta);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003145 if (err)
3146 return err;
Daniel Borkmann33ff9822016-04-13 00:10:50 +02003147 err = check_func_arg(env, BPF_REG_5, fn->arg5_type, &meta);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003148 if (err)
3149 return err;
3150
Daniel Borkmannc93552c2018-05-24 02:32:53 +02003151 err = record_func_map(env, &meta, func_id, insn_idx);
3152 if (err)
3153 return err;
3154
Daniel Borkmann435faee12016-04-13 00:10:51 +02003155 /* Mark slots with STACK_MISC in case of raw mode, stack offset
3156 * is inferred from register state.
3157 */
3158 for (i = 0; i < meta.access_size; i++) {
Daniel Borkmannca369602018-02-23 22:29:05 +01003159 err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B,
3160 BPF_WRITE, -1, false);
Daniel Borkmann435faee12016-04-13 00:10:51 +02003161 if (err)
3162 return err;
3163 }
3164
Joe Stringerfd978bf72018-10-02 13:35:35 -07003165 if (func_id == BPF_FUNC_tail_call) {
3166 err = check_reference_leak(env);
3167 if (err) {
3168 verbose(env, "tail_call would lead to reference leak\n");
3169 return err;
3170 }
3171 } else if (is_release_function(func_id)) {
Martin KaFai Lau1b986582019-03-12 10:23:02 -07003172 err = release_reference(env, meta.ref_obj_id);
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08003173 if (err) {
3174 verbose(env, "func %s#%d reference has not been acquired before\n",
3175 func_id_name(func_id), func_id);
Joe Stringerfd978bf72018-10-02 13:35:35 -07003176 return err;
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08003177 }
Joe Stringerfd978bf72018-10-02 13:35:35 -07003178 }
3179
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07003180 regs = cur_regs(env);
Roman Gushchincd339432018-08-02 14:27:24 -07003181
3182 /* check that flags argument in get_local_storage(map, flags) is 0,
3183 * this is required because get_local_storage() can't return an error.
3184 */
3185 if (func_id == BPF_FUNC_get_local_storage &&
3186 !register_is_null(&regs[BPF_REG_2])) {
3187 verbose(env, "get_local_storage() doesn't support non-zero flags\n");
3188 return -EINVAL;
3189 }
3190
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003191 /* reset caller saved regs */
Edward Creedc503a82017-08-15 20:34:35 +01003192 for (i = 0; i < CALLER_SAVED_REGS; i++) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07003193 mark_reg_not_init(env, regs, caller_saved[i]);
Edward Creedc503a82017-08-15 20:34:35 +01003194 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
3195 }
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003196
Edward Creedc503a82017-08-15 20:34:35 +01003197 /* update return register (already marked as written above) */
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003198 if (fn->ret_type == RET_INTEGER) {
Edward Creef1174f72017-08-07 15:26:19 +01003199 /* sets type to SCALAR_VALUE */
Jakub Kicinski61bd5212017-10-09 10:30:11 -07003200 mark_reg_unknown(env, regs, BPF_REG_0);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003201 } else if (fn->ret_type == RET_VOID) {
3202 regs[BPF_REG_0].type = NOT_INIT;
Roman Gushchin3e6a4b32018-08-02 14:27:22 -07003203 } else if (fn->ret_type == RET_PTR_TO_MAP_VALUE_OR_NULL ||
3204 fn->ret_type == RET_PTR_TO_MAP_VALUE) {
Edward Creef1174f72017-08-07 15:26:19 +01003205 /* There is no offset yet applied, variable or fixed */
Jakub Kicinski61bd5212017-10-09 10:30:11 -07003206 mark_reg_known_zero(env, regs, BPF_REG_0);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003207 /* remember map_ptr, so that check_map_access()
3208 * can check 'value_size' boundary of memory access
3209 * to map element returned from bpf_map_lookup_elem()
3210 */
Daniel Borkmann33ff9822016-04-13 00:10:50 +02003211 if (meta.map_ptr == NULL) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07003212 verbose(env,
3213 "kernel subsystem misconfigured verifier\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003214 return -EINVAL;
3215 }
Daniel Borkmann33ff9822016-04-13 00:10:50 +02003216 regs[BPF_REG_0].map_ptr = meta.map_ptr;
Daniel Borkmann4d31f302018-11-01 00:05:53 +01003217 if (fn->ret_type == RET_PTR_TO_MAP_VALUE) {
3218 regs[BPF_REG_0].type = PTR_TO_MAP_VALUE;
Alexei Starovoitove16d2f12019-01-31 15:40:05 -08003219 if (map_value_has_spin_lock(meta.map_ptr))
3220 regs[BPF_REG_0].id = ++env->id_gen;
Daniel Borkmann4d31f302018-11-01 00:05:53 +01003221 } else {
3222 regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL;
3223 regs[BPF_REG_0].id = ++env->id_gen;
3224 }
Joe Stringerc64b7982018-10-02 13:35:33 -07003225 } else if (fn->ret_type == RET_PTR_TO_SOCKET_OR_NULL) {
3226 mark_reg_known_zero(env, regs, BPF_REG_0);
3227 regs[BPF_REG_0].type = PTR_TO_SOCKET_OR_NULL;
Lorenz Bauer0f3adc22019-03-22 09:53:59 +08003228 regs[BPF_REG_0].id = ++env->id_gen;
Lorenz Bauer85a51f82019-03-22 09:54:00 +08003229 } else if (fn->ret_type == RET_PTR_TO_SOCK_COMMON_OR_NULL) {
3230 mark_reg_known_zero(env, regs, BPF_REG_0);
3231 regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON_OR_NULL;
3232 regs[BPF_REG_0].id = ++env->id_gen;
Martin KaFai Lau655a51e2019-02-09 23:22:24 -08003233 } else if (fn->ret_type == RET_PTR_TO_TCP_SOCK_OR_NULL) {
3234 mark_reg_known_zero(env, regs, BPF_REG_0);
3235 regs[BPF_REG_0].type = PTR_TO_TCP_SOCK_OR_NULL;
3236 regs[BPF_REG_0].id = ++env->id_gen;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003237 } else {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07003238 verbose(env, "unknown return type %d of func %s#%d\n",
Thomas Grafebb676d2016-10-27 11:23:51 +02003239 fn->ret_type, func_id_name(func_id), func_id);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003240 return -EINVAL;
3241 }
Alexei Starovoitov04fd61ab2015-05-19 16:59:03 -07003242
Lorenz Bauer0f3adc22019-03-22 09:53:59 +08003243 if (is_ptr_cast_function(func_id)) {
Martin KaFai Lau1b986582019-03-12 10:23:02 -07003244 /* For release_reference() */
3245 regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id;
Lorenz Bauer0f3adc22019-03-22 09:53:59 +08003246 } else if (is_acquire_function(func_id)) {
3247 int id = acquire_reference_state(env, insn_idx);
3248
3249 if (id < 0)
3250 return id;
3251 /* For mark_ptr_or_null_reg() */
3252 regs[BPF_REG_0].id = id;
3253 /* For release_reference() */
3254 regs[BPF_REG_0].ref_obj_id = id;
3255 }
Martin KaFai Lau1b986582019-03-12 10:23:02 -07003256
Yonghong Song849fa502018-04-28 22:28:09 -07003257 do_refine_retval_range(regs, fn->ret_type, func_id, &meta);
3258
Jakub Kicinski61bd5212017-10-09 10:30:11 -07003259 err = check_map_func_compatibility(env, meta.map_ptr, func_id);
Kaixu Xia35578d72015-08-06 07:02:35 +00003260 if (err)
3261 return err;
Alexei Starovoitov04fd61ab2015-05-19 16:59:03 -07003262
Yonghong Songc195651e2018-04-28 22:28:08 -07003263 if (func_id == BPF_FUNC_get_stack && !env->prog->has_callchain_buf) {
3264 const char *err_str;
3265
3266#ifdef CONFIG_PERF_EVENTS
3267 err = get_callchain_buffers(sysctl_perf_event_max_stack);
3268 err_str = "cannot get callchain buffer for func %s#%d\n";
3269#else
3270 err = -ENOTSUPP;
3271 err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n";
3272#endif
3273 if (err) {
3274 verbose(env, err_str, func_id_name(func_id), func_id);
3275 return err;
3276 }
3277
3278 env->prog->has_callchain_buf = true;
3279 }
3280
Alexei Starovoitov969bf052016-05-05 19:49:10 -07003281 if (changes_data)
3282 clear_all_pkt_pointers(env);
3283 return 0;
3284}
3285
Edward Creeb03c9f92017-08-07 15:26:36 +01003286static bool signed_add_overflows(s64 a, s64 b)
3287{
3288 /* Do the add in u64, where overflow is well-defined */
3289 s64 res = (s64)((u64)a + (u64)b);
3290
3291 if (b < 0)
3292 return res > a;
3293 return res < a;
3294}
3295
3296static bool signed_sub_overflows(s64 a, s64 b)
3297{
3298 /* Do the sub in u64, where overflow is well-defined */
3299 s64 res = (s64)((u64)a - (u64)b);
3300
3301 if (b < 0)
3302 return res < a;
3303 return res > a;
David S. Millerd1174412017-05-10 11:22:52 -07003304}
3305
Alexei Starovoitovbb7f0f92017-12-18 20:12:00 -08003306static bool check_reg_sane_offset(struct bpf_verifier_env *env,
3307 const struct bpf_reg_state *reg,
3308 enum bpf_reg_type type)
3309{
3310 bool known = tnum_is_const(reg->var_off);
3311 s64 val = reg->var_off.value;
3312 s64 smin = reg->smin_value;
3313
3314 if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) {
3315 verbose(env, "math between %s pointer and %lld is not allowed\n",
3316 reg_type_str[type], val);
3317 return false;
3318 }
3319
3320 if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) {
3321 verbose(env, "%s pointer offset %d is not allowed\n",
3322 reg_type_str[type], reg->off);
3323 return false;
3324 }
3325
3326 if (smin == S64_MIN) {
3327 verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n",
3328 reg_type_str[type]);
3329 return false;
3330 }
3331
3332 if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) {
3333 verbose(env, "value %lld makes %s pointer be out of bounds\n",
3334 smin, reg_type_str[type]);
3335 return false;
3336 }
3337
3338 return true;
3339}
3340
Daniel Borkmann979d63d2019-01-03 00:58:34 +01003341static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env)
3342{
3343 return &env->insn_aux_data[env->insn_idx];
3344}
3345
3346static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg,
3347 u32 *ptr_limit, u8 opcode, bool off_is_neg)
3348{
3349 bool mask_to_left = (opcode == BPF_ADD && off_is_neg) ||
3350 (opcode == BPF_SUB && !off_is_neg);
3351 u32 off;
3352
3353 switch (ptr_reg->type) {
3354 case PTR_TO_STACK:
Andrey Ignatov088ec262019-04-03 23:22:39 -07003355 /* Indirect variable offset stack access is prohibited in
3356 * unprivileged mode so it's not handled here.
3357 */
Daniel Borkmann979d63d2019-01-03 00:58:34 +01003358 off = ptr_reg->off + ptr_reg->var_off.value;
3359 if (mask_to_left)
3360 *ptr_limit = MAX_BPF_STACK + off;
3361 else
3362 *ptr_limit = -off;
3363 return 0;
3364 case PTR_TO_MAP_VALUE:
3365 if (mask_to_left) {
3366 *ptr_limit = ptr_reg->umax_value + ptr_reg->off;
3367 } else {
3368 off = ptr_reg->smin_value + ptr_reg->off;
3369 *ptr_limit = ptr_reg->map_ptr->value_size - off;
3370 }
3371 return 0;
3372 default:
3373 return -EINVAL;
3374 }
3375}
3376
Daniel Borkmannd3bd7412019-01-06 00:54:37 +01003377static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env,
3378 const struct bpf_insn *insn)
3379{
3380 return env->allow_ptr_leaks || BPF_SRC(insn->code) == BPF_K;
3381}
3382
3383static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux,
3384 u32 alu_state, u32 alu_limit)
3385{
3386 /* If we arrived here from different branches with different
3387 * state or limits to sanitize, then this won't work.
3388 */
3389 if (aux->alu_state &&
3390 (aux->alu_state != alu_state ||
3391 aux->alu_limit != alu_limit))
3392 return -EACCES;
3393
3394 /* Corresponding fixup done in fixup_bpf_calls(). */
3395 aux->alu_state = alu_state;
3396 aux->alu_limit = alu_limit;
3397 return 0;
3398}
3399
3400static int sanitize_val_alu(struct bpf_verifier_env *env,
3401 struct bpf_insn *insn)
3402{
3403 struct bpf_insn_aux_data *aux = cur_aux(env);
3404
3405 if (can_skip_alu_sanitation(env, insn))
3406 return 0;
3407
3408 return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0);
3409}
3410
Daniel Borkmann979d63d2019-01-03 00:58:34 +01003411static int sanitize_ptr_alu(struct bpf_verifier_env *env,
3412 struct bpf_insn *insn,
3413 const struct bpf_reg_state *ptr_reg,
3414 struct bpf_reg_state *dst_reg,
3415 bool off_is_neg)
3416{
3417 struct bpf_verifier_state *vstate = env->cur_state;
3418 struct bpf_insn_aux_data *aux = cur_aux(env);
3419 bool ptr_is_dst_reg = ptr_reg == dst_reg;
3420 u8 opcode = BPF_OP(insn->code);
3421 u32 alu_state, alu_limit;
3422 struct bpf_reg_state tmp;
3423 bool ret;
3424
Daniel Borkmannd3bd7412019-01-06 00:54:37 +01003425 if (can_skip_alu_sanitation(env, insn))
Daniel Borkmann979d63d2019-01-03 00:58:34 +01003426 return 0;
3427
3428 /* We already marked aux for masking from non-speculative
3429 * paths, thus we got here in the first place. We only care
3430 * to explore bad access from here.
3431 */
3432 if (vstate->speculative)
3433 goto do_sim;
3434
3435 alu_state = off_is_neg ? BPF_ALU_NEG_VALUE : 0;
3436 alu_state |= ptr_is_dst_reg ?
3437 BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST;
3438
3439 if (retrieve_ptr_limit(ptr_reg, &alu_limit, opcode, off_is_neg))
3440 return 0;
Daniel Borkmannd3bd7412019-01-06 00:54:37 +01003441 if (update_alu_sanitation_state(aux, alu_state, alu_limit))
Daniel Borkmann979d63d2019-01-03 00:58:34 +01003442 return -EACCES;
Daniel Borkmann979d63d2019-01-03 00:58:34 +01003443do_sim:
3444 /* Simulate and find potential out-of-bounds access under
3445 * speculative execution from truncation as a result of
3446 * masking when off was not within expected range. If off
3447 * sits in dst, then we temporarily need to move ptr there
3448 * to simulate dst (== 0) +/-= ptr. Needed, for example,
3449 * for cases where we use K-based arithmetic in one direction
3450 * and truncated reg-based in the other in order to explore
3451 * bad access.
3452 */
3453 if (!ptr_is_dst_reg) {
3454 tmp = *dst_reg;
3455 *dst_reg = *ptr_reg;
3456 }
3457 ret = push_stack(env, env->insn_idx + 1, env->insn_idx, true);
Xu Yu08032782019-03-21 18:00:35 +08003458 if (!ptr_is_dst_reg && ret)
Daniel Borkmann979d63d2019-01-03 00:58:34 +01003459 *dst_reg = tmp;
3460 return !ret ? -EFAULT : 0;
3461}
3462
Edward Creef1174f72017-08-07 15:26:19 +01003463/* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off.
Edward Creef1174f72017-08-07 15:26:19 +01003464 * Caller should also handle BPF_MOV case separately.
3465 * If we return -EACCES, caller may want to try again treating pointer as a
3466 * scalar. So we only emit a diagnostic if !env->allow_ptr_leaks.
3467 */
3468static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
3469 struct bpf_insn *insn,
3470 const struct bpf_reg_state *ptr_reg,
3471 const struct bpf_reg_state *off_reg)
Josef Bacik48461132016-09-28 10:54:32 -04003472{
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08003473 struct bpf_verifier_state *vstate = env->cur_state;
3474 struct bpf_func_state *state = vstate->frame[vstate->curframe];
3475 struct bpf_reg_state *regs = state->regs, *dst_reg;
Edward Creef1174f72017-08-07 15:26:19 +01003476 bool known = tnum_is_const(off_reg->var_off);
Edward Creeb03c9f92017-08-07 15:26:36 +01003477 s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value,
3478 smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value;
3479 u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value,
3480 umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value;
Daniel Borkmann9d7ecee2019-01-03 00:58:32 +01003481 u32 dst = insn->dst_reg, src = insn->src_reg;
Josef Bacik48461132016-09-28 10:54:32 -04003482 u8 opcode = BPF_OP(insn->code);
Daniel Borkmann979d63d2019-01-03 00:58:34 +01003483 int ret;
Josef Bacik48461132016-09-28 10:54:32 -04003484
Edward Creef1174f72017-08-07 15:26:19 +01003485 dst_reg = &regs[dst];
Josef Bacik48461132016-09-28 10:54:32 -04003486
Daniel Borkmann6f161012018-01-18 01:15:21 +01003487 if ((known && (smin_val != smax_val || umin_val != umax_val)) ||
3488 smin_val > smax_val || umin_val > umax_val) {
3489 /* Taint dst register if offset had invalid bounds derived from
3490 * e.g. dead branches.
3491 */
3492 __mark_reg_unknown(dst_reg);
3493 return 0;
Josef Bacik48461132016-09-28 10:54:32 -04003494 }
3495
Edward Creef1174f72017-08-07 15:26:19 +01003496 if (BPF_CLASS(insn->code) != BPF_ALU64) {
3497 /* 32-bit ALU ops on pointers produce (meaningless) scalars */
Alexei Starovoitov82abbf82017-12-18 20:15:20 -08003498 verbose(env,
3499 "R%d 32-bit pointer arithmetic prohibited\n",
3500 dst);
Edward Creef1174f72017-08-07 15:26:19 +01003501 return -EACCES;
3502 }
David S. Millerd1174412017-05-10 11:22:52 -07003503
Joe Stringeraad2eea2018-10-02 13:35:30 -07003504 switch (ptr_reg->type) {
3505 case PTR_TO_MAP_VALUE_OR_NULL:
3506 verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n",
3507 dst, reg_type_str[ptr_reg->type]);
Edward Creef1174f72017-08-07 15:26:19 +01003508 return -EACCES;
Joe Stringeraad2eea2018-10-02 13:35:30 -07003509 case CONST_PTR_TO_MAP:
3510 case PTR_TO_PACKET_END:
Joe Stringerc64b7982018-10-02 13:35:33 -07003511 case PTR_TO_SOCKET:
3512 case PTR_TO_SOCKET_OR_NULL:
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08003513 case PTR_TO_SOCK_COMMON:
3514 case PTR_TO_SOCK_COMMON_OR_NULL:
Martin KaFai Lau655a51e2019-02-09 23:22:24 -08003515 case PTR_TO_TCP_SOCK:
3516 case PTR_TO_TCP_SOCK_OR_NULL:
Joe Stringeraad2eea2018-10-02 13:35:30 -07003517 verbose(env, "R%d pointer arithmetic on %s prohibited\n",
3518 dst, reg_type_str[ptr_reg->type]);
Edward Creef1174f72017-08-07 15:26:19 +01003519 return -EACCES;
Daniel Borkmann9d7ecee2019-01-03 00:58:32 +01003520 case PTR_TO_MAP_VALUE:
3521 if (!env->allow_ptr_leaks && !known && (smin_val < 0) != (smax_val < 0)) {
3522 verbose(env, "R%d has unknown scalar with mixed signed bounds, pointer arithmetic with it prohibited for !root\n",
3523 off_reg == dst_reg ? dst : src);
3524 return -EACCES;
3525 }
3526 /* fall-through */
Joe Stringeraad2eea2018-10-02 13:35:30 -07003527 default:
3528 break;
Edward Creef1174f72017-08-07 15:26:19 +01003529 }
3530
3531 /* In case of 'scalar += pointer', dst_reg inherits pointer type and id.
3532 * The id may be overwritten later if we create a new variable offset.
Josef Bacik48461132016-09-28 10:54:32 -04003533 */
Edward Creef1174f72017-08-07 15:26:19 +01003534 dst_reg->type = ptr_reg->type;
3535 dst_reg->id = ptr_reg->id;
Josef Bacikf23cc642016-11-14 15:45:36 -05003536
Alexei Starovoitovbb7f0f92017-12-18 20:12:00 -08003537 if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) ||
3538 !check_reg_sane_offset(env, ptr_reg, ptr_reg->type))
3539 return -EINVAL;
3540
Josef Bacik48461132016-09-28 10:54:32 -04003541 switch (opcode) {
3542 case BPF_ADD:
Daniel Borkmann979d63d2019-01-03 00:58:34 +01003543 ret = sanitize_ptr_alu(env, insn, ptr_reg, dst_reg, smin_val < 0);
3544 if (ret < 0) {
3545 verbose(env, "R%d tried to add from different maps or paths\n", dst);
3546 return ret;
3547 }
Edward Creef1174f72017-08-07 15:26:19 +01003548 /* We can take a fixed offset as long as it doesn't overflow
3549 * the s32 'off' field
3550 */
Edward Creeb03c9f92017-08-07 15:26:36 +01003551 if (known && (ptr_reg->off + smin_val ==
3552 (s64)(s32)(ptr_reg->off + smin_val))) {
Edward Creef1174f72017-08-07 15:26:19 +01003553 /* pointer += K. Accumulate it into fixed offset */
Edward Creeb03c9f92017-08-07 15:26:36 +01003554 dst_reg->smin_value = smin_ptr;
3555 dst_reg->smax_value = smax_ptr;
3556 dst_reg->umin_value = umin_ptr;
3557 dst_reg->umax_value = umax_ptr;
Edward Creef1174f72017-08-07 15:26:19 +01003558 dst_reg->var_off = ptr_reg->var_off;
Edward Creeb03c9f92017-08-07 15:26:36 +01003559 dst_reg->off = ptr_reg->off + smin_val;
Daniel Borkmann09625902018-11-01 00:05:52 +01003560 dst_reg->raw = ptr_reg->raw;
Edward Creef1174f72017-08-07 15:26:19 +01003561 break;
3562 }
Edward Creef1174f72017-08-07 15:26:19 +01003563 /* A new variable offset is created. Note that off_reg->off
3564 * == 0, since it's a scalar.
3565 * dst_reg gets the pointer type and since some positive
3566 * integer value was added to the pointer, give it a new 'id'
3567 * if it's a PTR_TO_PACKET.
3568 * this creates a new 'base' pointer, off_reg (variable) gets
3569 * added into the variable offset, and we copy the fixed offset
3570 * from ptr_reg.
3571 */
Edward Creeb03c9f92017-08-07 15:26:36 +01003572 if (signed_add_overflows(smin_ptr, smin_val) ||
3573 signed_add_overflows(smax_ptr, smax_val)) {
3574 dst_reg->smin_value = S64_MIN;
3575 dst_reg->smax_value = S64_MAX;
3576 } else {
3577 dst_reg->smin_value = smin_ptr + smin_val;
3578 dst_reg->smax_value = smax_ptr + smax_val;
3579 }
3580 if (umin_ptr + umin_val < umin_ptr ||
3581 umax_ptr + umax_val < umax_ptr) {
3582 dst_reg->umin_value = 0;
3583 dst_reg->umax_value = U64_MAX;
3584 } else {
3585 dst_reg->umin_value = umin_ptr + umin_val;
3586 dst_reg->umax_value = umax_ptr + umax_val;
3587 }
Edward Creef1174f72017-08-07 15:26:19 +01003588 dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off);
3589 dst_reg->off = ptr_reg->off;
Daniel Borkmann09625902018-11-01 00:05:52 +01003590 dst_reg->raw = ptr_reg->raw;
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02003591 if (reg_is_pkt_pointer(ptr_reg)) {
Edward Creef1174f72017-08-07 15:26:19 +01003592 dst_reg->id = ++env->id_gen;
3593 /* something was added to pkt_ptr, set range to zero */
Daniel Borkmann09625902018-11-01 00:05:52 +01003594 dst_reg->raw = 0;
Edward Creef1174f72017-08-07 15:26:19 +01003595 }
Josef Bacik48461132016-09-28 10:54:32 -04003596 break;
3597 case BPF_SUB:
Daniel Borkmann979d63d2019-01-03 00:58:34 +01003598 ret = sanitize_ptr_alu(env, insn, ptr_reg, dst_reg, smin_val < 0);
3599 if (ret < 0) {
3600 verbose(env, "R%d tried to sub from different maps or paths\n", dst);
3601 return ret;
3602 }
Edward Creef1174f72017-08-07 15:26:19 +01003603 if (dst_reg == off_reg) {
3604 /* scalar -= pointer. Creates an unknown scalar */
Alexei Starovoitov82abbf82017-12-18 20:15:20 -08003605 verbose(env, "R%d tried to subtract pointer from scalar\n",
3606 dst);
Edward Creef1174f72017-08-07 15:26:19 +01003607 return -EACCES;
3608 }
3609 /* We don't allow subtraction from FP, because (according to
3610 * test_verifier.c test "invalid fp arithmetic", JITs might not
3611 * be able to deal with it.
Edward Cree93057062017-07-21 14:37:34 +01003612 */
Edward Creef1174f72017-08-07 15:26:19 +01003613 if (ptr_reg->type == PTR_TO_STACK) {
Alexei Starovoitov82abbf82017-12-18 20:15:20 -08003614 verbose(env, "R%d subtraction from stack pointer prohibited\n",
3615 dst);
Edward Creef1174f72017-08-07 15:26:19 +01003616 return -EACCES;
3617 }
Edward Creeb03c9f92017-08-07 15:26:36 +01003618 if (known && (ptr_reg->off - smin_val ==
3619 (s64)(s32)(ptr_reg->off - smin_val))) {
Edward Creef1174f72017-08-07 15:26:19 +01003620 /* pointer -= K. Subtract it from fixed offset */
Edward Creeb03c9f92017-08-07 15:26:36 +01003621 dst_reg->smin_value = smin_ptr;
3622 dst_reg->smax_value = smax_ptr;
3623 dst_reg->umin_value = umin_ptr;
3624 dst_reg->umax_value = umax_ptr;
Edward Creef1174f72017-08-07 15:26:19 +01003625 dst_reg->var_off = ptr_reg->var_off;
3626 dst_reg->id = ptr_reg->id;
Edward Creeb03c9f92017-08-07 15:26:36 +01003627 dst_reg->off = ptr_reg->off - smin_val;
Daniel Borkmann09625902018-11-01 00:05:52 +01003628 dst_reg->raw = ptr_reg->raw;
Edward Creef1174f72017-08-07 15:26:19 +01003629 break;
3630 }
Edward Creef1174f72017-08-07 15:26:19 +01003631 /* A new variable offset is created. If the subtrahend is known
3632 * nonnegative, then any reg->range we had before is still good.
3633 */
Edward Creeb03c9f92017-08-07 15:26:36 +01003634 if (signed_sub_overflows(smin_ptr, smax_val) ||
3635 signed_sub_overflows(smax_ptr, smin_val)) {
3636 /* Overflow possible, we know nothing */
3637 dst_reg->smin_value = S64_MIN;
3638 dst_reg->smax_value = S64_MAX;
3639 } else {
3640 dst_reg->smin_value = smin_ptr - smax_val;
3641 dst_reg->smax_value = smax_ptr - smin_val;
3642 }
3643 if (umin_ptr < umax_val) {
3644 /* Overflow possible, we know nothing */
3645 dst_reg->umin_value = 0;
3646 dst_reg->umax_value = U64_MAX;
3647 } else {
3648 /* Cannot overflow (as long as bounds are consistent) */
3649 dst_reg->umin_value = umin_ptr - umax_val;
3650 dst_reg->umax_value = umax_ptr - umin_val;
3651 }
Edward Creef1174f72017-08-07 15:26:19 +01003652 dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off);
3653 dst_reg->off = ptr_reg->off;
Daniel Borkmann09625902018-11-01 00:05:52 +01003654 dst_reg->raw = ptr_reg->raw;
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02003655 if (reg_is_pkt_pointer(ptr_reg)) {
Edward Creef1174f72017-08-07 15:26:19 +01003656 dst_reg->id = ++env->id_gen;
3657 /* something was added to pkt_ptr, set range to zero */
Edward Creeb03c9f92017-08-07 15:26:36 +01003658 if (smin_val < 0)
Daniel Borkmann09625902018-11-01 00:05:52 +01003659 dst_reg->raw = 0;
Edward Creef1174f72017-08-07 15:26:19 +01003660 }
3661 break;
3662 case BPF_AND:
3663 case BPF_OR:
3664 case BPF_XOR:
Alexei Starovoitov82abbf82017-12-18 20:15:20 -08003665 /* bitwise ops on pointers are troublesome, prohibit. */
3666 verbose(env, "R%d bitwise operator %s on pointer prohibited\n",
3667 dst, bpf_alu_string[opcode >> 4]);
Edward Creef1174f72017-08-07 15:26:19 +01003668 return -EACCES;
3669 default:
3670 /* other operators (e.g. MUL,LSH) produce non-pointer results */
Alexei Starovoitov82abbf82017-12-18 20:15:20 -08003671 verbose(env, "R%d pointer arithmetic with %s operator prohibited\n",
3672 dst, bpf_alu_string[opcode >> 4]);
Edward Creef1174f72017-08-07 15:26:19 +01003673 return -EACCES;
3674 }
3675
Alexei Starovoitovbb7f0f92017-12-18 20:12:00 -08003676 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type))
3677 return -EINVAL;
3678
Edward Creeb03c9f92017-08-07 15:26:36 +01003679 __update_reg_bounds(dst_reg);
3680 __reg_deduce_bounds(dst_reg);
3681 __reg_bound_offset(dst_reg);
Daniel Borkmann0d6303d2019-01-03 00:58:30 +01003682
3683 /* For unprivileged we require that resulting offset must be in bounds
3684 * in order to be able to sanitize access later on.
3685 */
Daniel Borkmanne4298d22019-01-03 00:58:31 +01003686 if (!env->allow_ptr_leaks) {
3687 if (dst_reg->type == PTR_TO_MAP_VALUE &&
3688 check_map_access(env, dst, dst_reg->off, 1, false)) {
3689 verbose(env, "R%d pointer arithmetic of map value goes out of range, "
3690 "prohibited for !root\n", dst);
3691 return -EACCES;
3692 } else if (dst_reg->type == PTR_TO_STACK &&
3693 check_stack_access(env, dst_reg, dst_reg->off +
3694 dst_reg->var_off.value, 1)) {
3695 verbose(env, "R%d stack pointer arithmetic goes out of range, "
3696 "prohibited for !root\n", dst);
3697 return -EACCES;
3698 }
Daniel Borkmann0d6303d2019-01-03 00:58:30 +01003699 }
3700
Edward Creef1174f72017-08-07 15:26:19 +01003701 return 0;
3702}
3703
Jann Horn468f6ea2017-12-18 20:11:56 -08003704/* WARNING: This function does calculations on 64-bit values, but the actual
3705 * execution may occur on 32-bit values. Therefore, things like bitshifts
3706 * need extra checks in the 32-bit case.
3707 */
Edward Creef1174f72017-08-07 15:26:19 +01003708static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env,
3709 struct bpf_insn *insn,
3710 struct bpf_reg_state *dst_reg,
3711 struct bpf_reg_state src_reg)
3712{
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07003713 struct bpf_reg_state *regs = cur_regs(env);
Edward Creef1174f72017-08-07 15:26:19 +01003714 u8 opcode = BPF_OP(insn->code);
3715 bool src_known, dst_known;
Edward Creeb03c9f92017-08-07 15:26:36 +01003716 s64 smin_val, smax_val;
3717 u64 umin_val, umax_val;
Jann Horn468f6ea2017-12-18 20:11:56 -08003718 u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32;
Daniel Borkmannd3bd7412019-01-06 00:54:37 +01003719 u32 dst = insn->dst_reg;
3720 int ret;
Edward Creef1174f72017-08-07 15:26:19 +01003721
Jann Hornb7992072018-10-05 18:17:59 +02003722 if (insn_bitness == 32) {
3723 /* Relevant for 32-bit RSH: Information can propagate towards
3724 * LSB, so it isn't sufficient to only truncate the output to
3725 * 32 bits.
3726 */
3727 coerce_reg_to_size(dst_reg, 4);
3728 coerce_reg_to_size(&src_reg, 4);
3729 }
3730
Edward Creeb03c9f92017-08-07 15:26:36 +01003731 smin_val = src_reg.smin_value;
3732 smax_val = src_reg.smax_value;
3733 umin_val = src_reg.umin_value;
3734 umax_val = src_reg.umax_value;
Edward Creef1174f72017-08-07 15:26:19 +01003735 src_known = tnum_is_const(src_reg.var_off);
3736 dst_known = tnum_is_const(dst_reg->var_off);
3737
Daniel Borkmann6f161012018-01-18 01:15:21 +01003738 if ((src_known && (smin_val != smax_val || umin_val != umax_val)) ||
3739 smin_val > smax_val || umin_val > umax_val) {
3740 /* Taint dst register if offset had invalid bounds derived from
3741 * e.g. dead branches.
3742 */
3743 __mark_reg_unknown(dst_reg);
3744 return 0;
3745 }
3746
Alexei Starovoitovbb7f0f92017-12-18 20:12:00 -08003747 if (!src_known &&
3748 opcode != BPF_ADD && opcode != BPF_SUB && opcode != BPF_AND) {
3749 __mark_reg_unknown(dst_reg);
3750 return 0;
3751 }
3752
Edward Creef1174f72017-08-07 15:26:19 +01003753 switch (opcode) {
3754 case BPF_ADD:
Daniel Borkmannd3bd7412019-01-06 00:54:37 +01003755 ret = sanitize_val_alu(env, insn);
3756 if (ret < 0) {
3757 verbose(env, "R%d tried to add from different pointers or scalars\n", dst);
3758 return ret;
3759 }
Edward Creeb03c9f92017-08-07 15:26:36 +01003760 if (signed_add_overflows(dst_reg->smin_value, smin_val) ||
3761 signed_add_overflows(dst_reg->smax_value, smax_val)) {
3762 dst_reg->smin_value = S64_MIN;
3763 dst_reg->smax_value = S64_MAX;
3764 } else {
3765 dst_reg->smin_value += smin_val;
3766 dst_reg->smax_value += smax_val;
3767 }
3768 if (dst_reg->umin_value + umin_val < umin_val ||
3769 dst_reg->umax_value + umax_val < umax_val) {
3770 dst_reg->umin_value = 0;
3771 dst_reg->umax_value = U64_MAX;
3772 } else {
3773 dst_reg->umin_value += umin_val;
3774 dst_reg->umax_value += umax_val;
3775 }
Edward Creef1174f72017-08-07 15:26:19 +01003776 dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off);
3777 break;
3778 case BPF_SUB:
Daniel Borkmannd3bd7412019-01-06 00:54:37 +01003779 ret = sanitize_val_alu(env, insn);
3780 if (ret < 0) {
3781 verbose(env, "R%d tried to sub from different pointers or scalars\n", dst);
3782 return ret;
3783 }
Edward Creeb03c9f92017-08-07 15:26:36 +01003784 if (signed_sub_overflows(dst_reg->smin_value, smax_val) ||
3785 signed_sub_overflows(dst_reg->smax_value, smin_val)) {
3786 /* Overflow possible, we know nothing */
3787 dst_reg->smin_value = S64_MIN;
3788 dst_reg->smax_value = S64_MAX;
3789 } else {
3790 dst_reg->smin_value -= smax_val;
3791 dst_reg->smax_value -= smin_val;
3792 }
3793 if (dst_reg->umin_value < umax_val) {
3794 /* Overflow possible, we know nothing */
3795 dst_reg->umin_value = 0;
3796 dst_reg->umax_value = U64_MAX;
3797 } else {
3798 /* Cannot overflow (as long as bounds are consistent) */
3799 dst_reg->umin_value -= umax_val;
3800 dst_reg->umax_value -= umin_val;
3801 }
Edward Creef1174f72017-08-07 15:26:19 +01003802 dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off);
Josef Bacik48461132016-09-28 10:54:32 -04003803 break;
3804 case BPF_MUL:
Edward Creeb03c9f92017-08-07 15:26:36 +01003805 dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off);
3806 if (smin_val < 0 || dst_reg->smin_value < 0) {
Edward Creef1174f72017-08-07 15:26:19 +01003807 /* Ain't nobody got time to multiply that sign */
Edward Creeb03c9f92017-08-07 15:26:36 +01003808 __mark_reg_unbounded(dst_reg);
3809 __update_reg_bounds(dst_reg);
Edward Creef1174f72017-08-07 15:26:19 +01003810 break;
3811 }
Edward Creeb03c9f92017-08-07 15:26:36 +01003812 /* Both values are positive, so we can work with unsigned and
3813 * copy the result to signed (unless it exceeds S64_MAX).
Edward Creef1174f72017-08-07 15:26:19 +01003814 */
Edward Creeb03c9f92017-08-07 15:26:36 +01003815 if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) {
3816 /* Potential overflow, we know nothing */
3817 __mark_reg_unbounded(dst_reg);
3818 /* (except what we can learn from the var_off) */
3819 __update_reg_bounds(dst_reg);
3820 break;
3821 }
3822 dst_reg->umin_value *= umin_val;
3823 dst_reg->umax_value *= umax_val;
3824 if (dst_reg->umax_value > S64_MAX) {
3825 /* Overflow possible, we know nothing */
3826 dst_reg->smin_value = S64_MIN;
3827 dst_reg->smax_value = S64_MAX;
3828 } else {
3829 dst_reg->smin_value = dst_reg->umin_value;
3830 dst_reg->smax_value = dst_reg->umax_value;
3831 }
Josef Bacik48461132016-09-28 10:54:32 -04003832 break;
3833 case BPF_AND:
Edward Creef1174f72017-08-07 15:26:19 +01003834 if (src_known && dst_known) {
Edward Creeb03c9f92017-08-07 15:26:36 +01003835 __mark_reg_known(dst_reg, dst_reg->var_off.value &
3836 src_reg.var_off.value);
Edward Creef1174f72017-08-07 15:26:19 +01003837 break;
3838 }
Edward Creeb03c9f92017-08-07 15:26:36 +01003839 /* We get our minimum from the var_off, since that's inherently
3840 * bitwise. Our maximum is the minimum of the operands' maxima.
Josef Bacikf23cc642016-11-14 15:45:36 -05003841 */
Edward Creef1174f72017-08-07 15:26:19 +01003842 dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off);
Edward Creeb03c9f92017-08-07 15:26:36 +01003843 dst_reg->umin_value = dst_reg->var_off.value;
3844 dst_reg->umax_value = min(dst_reg->umax_value, umax_val);
3845 if (dst_reg->smin_value < 0 || smin_val < 0) {
3846 /* Lose signed bounds when ANDing negative numbers,
3847 * ain't nobody got time for that.
3848 */
3849 dst_reg->smin_value = S64_MIN;
3850 dst_reg->smax_value = S64_MAX;
3851 } else {
3852 /* ANDing two positives gives a positive, so safe to
3853 * cast result into s64.
3854 */
3855 dst_reg->smin_value = dst_reg->umin_value;
3856 dst_reg->smax_value = dst_reg->umax_value;
3857 }
3858 /* We may learn something more from the var_off */
3859 __update_reg_bounds(dst_reg);
Edward Creef1174f72017-08-07 15:26:19 +01003860 break;
3861 case BPF_OR:
3862 if (src_known && dst_known) {
Edward Creeb03c9f92017-08-07 15:26:36 +01003863 __mark_reg_known(dst_reg, dst_reg->var_off.value |
3864 src_reg.var_off.value);
Edward Creef1174f72017-08-07 15:26:19 +01003865 break;
3866 }
Edward Creeb03c9f92017-08-07 15:26:36 +01003867 /* We get our maximum from the var_off, and our minimum is the
3868 * maximum of the operands' minima
Edward Creef1174f72017-08-07 15:26:19 +01003869 */
3870 dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off);
Edward Creeb03c9f92017-08-07 15:26:36 +01003871 dst_reg->umin_value = max(dst_reg->umin_value, umin_val);
3872 dst_reg->umax_value = dst_reg->var_off.value |
3873 dst_reg->var_off.mask;
3874 if (dst_reg->smin_value < 0 || smin_val < 0) {
3875 /* Lose signed bounds when ORing negative numbers,
3876 * ain't nobody got time for that.
3877 */
3878 dst_reg->smin_value = S64_MIN;
3879 dst_reg->smax_value = S64_MAX;
Edward Creef1174f72017-08-07 15:26:19 +01003880 } else {
Edward Creeb03c9f92017-08-07 15:26:36 +01003881 /* ORing two positives gives a positive, so safe to
3882 * cast result into s64.
3883 */
3884 dst_reg->smin_value = dst_reg->umin_value;
3885 dst_reg->smax_value = dst_reg->umax_value;
Edward Creef1174f72017-08-07 15:26:19 +01003886 }
Edward Creeb03c9f92017-08-07 15:26:36 +01003887 /* We may learn something more from the var_off */
3888 __update_reg_bounds(dst_reg);
Josef Bacik48461132016-09-28 10:54:32 -04003889 break;
3890 case BPF_LSH:
Jann Horn468f6ea2017-12-18 20:11:56 -08003891 if (umax_val >= insn_bitness) {
3892 /* Shifts greater than 31 or 63 are undefined.
3893 * This includes shifts by a negative number.
Edward Creeb03c9f92017-08-07 15:26:36 +01003894 */
Jakub Kicinski61bd5212017-10-09 10:30:11 -07003895 mark_reg_unknown(env, regs, insn->dst_reg);
Edward Creef1174f72017-08-07 15:26:19 +01003896 break;
3897 }
Edward Creeb03c9f92017-08-07 15:26:36 +01003898 /* We lose all sign bit information (except what we can pick
3899 * up from var_off)
Josef Bacik48461132016-09-28 10:54:32 -04003900 */
Edward Creeb03c9f92017-08-07 15:26:36 +01003901 dst_reg->smin_value = S64_MIN;
3902 dst_reg->smax_value = S64_MAX;
3903 /* If we might shift our top bit out, then we know nothing */
3904 if (dst_reg->umax_value > 1ULL << (63 - umax_val)) {
3905 dst_reg->umin_value = 0;
3906 dst_reg->umax_value = U64_MAX;
David S. Millerd1174412017-05-10 11:22:52 -07003907 } else {
Edward Creeb03c9f92017-08-07 15:26:36 +01003908 dst_reg->umin_value <<= umin_val;
3909 dst_reg->umax_value <<= umax_val;
David S. Millerd1174412017-05-10 11:22:52 -07003910 }
Yonghong Songafbe1a52018-04-28 22:28:10 -07003911 dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val);
Edward Creeb03c9f92017-08-07 15:26:36 +01003912 /* We may learn something more from the var_off */
3913 __update_reg_bounds(dst_reg);
Josef Bacik48461132016-09-28 10:54:32 -04003914 break;
3915 case BPF_RSH:
Jann Horn468f6ea2017-12-18 20:11:56 -08003916 if (umax_val >= insn_bitness) {
3917 /* Shifts greater than 31 or 63 are undefined.
3918 * This includes shifts by a negative number.
Edward Creeb03c9f92017-08-07 15:26:36 +01003919 */
Jakub Kicinski61bd5212017-10-09 10:30:11 -07003920 mark_reg_unknown(env, regs, insn->dst_reg);
Edward Creef1174f72017-08-07 15:26:19 +01003921 break;
3922 }
Edward Cree4374f252017-12-18 20:11:53 -08003923 /* BPF_RSH is an unsigned shift. If the value in dst_reg might
3924 * be negative, then either:
3925 * 1) src_reg might be zero, so the sign bit of the result is
3926 * unknown, so we lose our signed bounds
3927 * 2) it's known negative, thus the unsigned bounds capture the
3928 * signed bounds
3929 * 3) the signed bounds cross zero, so they tell us nothing
3930 * about the result
3931 * If the value in dst_reg is known nonnegative, then again the
3932 * unsigned bounts capture the signed bounds.
3933 * Thus, in all cases it suffices to blow away our signed bounds
3934 * and rely on inferring new ones from the unsigned bounds and
3935 * var_off of the result.
3936 */
3937 dst_reg->smin_value = S64_MIN;
3938 dst_reg->smax_value = S64_MAX;
Yonghong Songafbe1a52018-04-28 22:28:10 -07003939 dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val);
Edward Creeb03c9f92017-08-07 15:26:36 +01003940 dst_reg->umin_value >>= umax_val;
3941 dst_reg->umax_value >>= umin_val;
3942 /* We may learn something more from the var_off */
3943 __update_reg_bounds(dst_reg);
Josef Bacik48461132016-09-28 10:54:32 -04003944 break;
Yonghong Song9cbe1f5a2018-04-28 22:28:11 -07003945 case BPF_ARSH:
3946 if (umax_val >= insn_bitness) {
3947 /* Shifts greater than 31 or 63 are undefined.
3948 * This includes shifts by a negative number.
3949 */
3950 mark_reg_unknown(env, regs, insn->dst_reg);
3951 break;
3952 }
3953
3954 /* Upon reaching here, src_known is true and
3955 * umax_val is equal to umin_val.
3956 */
3957 dst_reg->smin_value >>= umin_val;
3958 dst_reg->smax_value >>= umin_val;
3959 dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val);
3960
3961 /* blow away the dst_reg umin_value/umax_value and rely on
3962 * dst_reg var_off to refine the result.
3963 */
3964 dst_reg->umin_value = 0;
3965 dst_reg->umax_value = U64_MAX;
3966 __update_reg_bounds(dst_reg);
3967 break;
Josef Bacik48461132016-09-28 10:54:32 -04003968 default:
Jakub Kicinski61bd5212017-10-09 10:30:11 -07003969 mark_reg_unknown(env, regs, insn->dst_reg);
Josef Bacik48461132016-09-28 10:54:32 -04003970 break;
3971 }
3972
Jann Horn468f6ea2017-12-18 20:11:56 -08003973 if (BPF_CLASS(insn->code) != BPF_ALU64) {
3974 /* 32-bit ALU ops are (32,32)->32 */
3975 coerce_reg_to_size(dst_reg, 4);
Jann Horn468f6ea2017-12-18 20:11:56 -08003976 }
3977
Edward Creeb03c9f92017-08-07 15:26:36 +01003978 __reg_deduce_bounds(dst_reg);
3979 __reg_bound_offset(dst_reg);
Edward Creef1174f72017-08-07 15:26:19 +01003980 return 0;
3981}
3982
3983/* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max
3984 * and var_off.
3985 */
3986static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,
3987 struct bpf_insn *insn)
3988{
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08003989 struct bpf_verifier_state *vstate = env->cur_state;
3990 struct bpf_func_state *state = vstate->frame[vstate->curframe];
3991 struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg;
Edward Creef1174f72017-08-07 15:26:19 +01003992 struct bpf_reg_state *ptr_reg = NULL, off_reg = {0};
3993 u8 opcode = BPF_OP(insn->code);
Edward Creef1174f72017-08-07 15:26:19 +01003994
3995 dst_reg = &regs[insn->dst_reg];
Edward Creef1174f72017-08-07 15:26:19 +01003996 src_reg = NULL;
3997 if (dst_reg->type != SCALAR_VALUE)
3998 ptr_reg = dst_reg;
3999 if (BPF_SRC(insn->code) == BPF_X) {
4000 src_reg = &regs[insn->src_reg];
Edward Creef1174f72017-08-07 15:26:19 +01004001 if (src_reg->type != SCALAR_VALUE) {
4002 if (dst_reg->type != SCALAR_VALUE) {
4003 /* Combining two pointers by any ALU op yields
Alexei Starovoitov82abbf82017-12-18 20:15:20 -08004004 * an arbitrary scalar. Disallow all math except
4005 * pointer subtraction
Edward Creef1174f72017-08-07 15:26:19 +01004006 */
Alexei Starovoitovdd066822018-09-12 14:06:10 -07004007 if (opcode == BPF_SUB && env->allow_ptr_leaks) {
Alexei Starovoitov82abbf82017-12-18 20:15:20 -08004008 mark_reg_unknown(env, regs, insn->dst_reg);
4009 return 0;
Edward Creef1174f72017-08-07 15:26:19 +01004010 }
Alexei Starovoitov82abbf82017-12-18 20:15:20 -08004011 verbose(env, "R%d pointer %s pointer prohibited\n",
4012 insn->dst_reg,
4013 bpf_alu_string[opcode >> 4]);
4014 return -EACCES;
Edward Creef1174f72017-08-07 15:26:19 +01004015 } else {
4016 /* scalar += pointer
4017 * This is legal, but we have to reverse our
4018 * src/dest handling in computing the range
4019 */
Alexei Starovoitov82abbf82017-12-18 20:15:20 -08004020 return adjust_ptr_min_max_vals(env, insn,
4021 src_reg, dst_reg);
Edward Creef1174f72017-08-07 15:26:19 +01004022 }
4023 } else if (ptr_reg) {
4024 /* pointer += scalar */
Alexei Starovoitov82abbf82017-12-18 20:15:20 -08004025 return adjust_ptr_min_max_vals(env, insn,
4026 dst_reg, src_reg);
Edward Creef1174f72017-08-07 15:26:19 +01004027 }
4028 } else {
4029 /* Pretend the src is a reg with a known value, since we only
4030 * need to be able to read from this state.
4031 */
4032 off_reg.type = SCALAR_VALUE;
Edward Creeb03c9f92017-08-07 15:26:36 +01004033 __mark_reg_known(&off_reg, insn->imm);
Edward Creef1174f72017-08-07 15:26:19 +01004034 src_reg = &off_reg;
Alexei Starovoitov82abbf82017-12-18 20:15:20 -08004035 if (ptr_reg) /* pointer += K */
4036 return adjust_ptr_min_max_vals(env, insn,
4037 ptr_reg, src_reg);
Edward Creef1174f72017-08-07 15:26:19 +01004038 }
4039
4040 /* Got here implies adding two SCALAR_VALUEs */
4041 if (WARN_ON_ONCE(ptr_reg)) {
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004042 print_verifier_state(env, state);
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004043 verbose(env, "verifier internal error: unexpected ptr_reg\n");
Edward Creef1174f72017-08-07 15:26:19 +01004044 return -EINVAL;
4045 }
4046 if (WARN_ON(!src_reg)) {
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004047 print_verifier_state(env, state);
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004048 verbose(env, "verifier internal error: no src_reg\n");
Edward Creef1174f72017-08-07 15:26:19 +01004049 return -EINVAL;
4050 }
4051 return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg);
Josef Bacik48461132016-09-28 10:54:32 -04004052}
4053
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004054/* check validity of 32-bit and 64-bit arithmetic operations */
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01004055static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004056{
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07004057 struct bpf_reg_state *regs = cur_regs(env);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004058 u8 opcode = BPF_OP(insn->code);
4059 int err;
4060
4061 if (opcode == BPF_END || opcode == BPF_NEG) {
4062 if (opcode == BPF_NEG) {
4063 if (BPF_SRC(insn->code) != 0 ||
4064 insn->src_reg != BPF_REG_0 ||
4065 insn->off != 0 || insn->imm != 0) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004066 verbose(env, "BPF_NEG uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004067 return -EINVAL;
4068 }
4069 } else {
4070 if (insn->src_reg != BPF_REG_0 || insn->off != 0 ||
Edward Creee67b8a62017-09-15 14:37:38 +01004071 (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) ||
4072 BPF_CLASS(insn->code) == BPF_ALU64) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004073 verbose(env, "BPF_END uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004074 return -EINVAL;
4075 }
4076 }
4077
4078 /* check src operand */
Edward Creedc503a82017-08-15 20:34:35 +01004079 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004080 if (err)
4081 return err;
4082
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07004083 if (is_pointer_value(env, insn->dst_reg)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004084 verbose(env, "R%d pointer arithmetic prohibited\n",
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07004085 insn->dst_reg);
4086 return -EACCES;
4087 }
4088
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004089 /* check dest operand */
Edward Creedc503a82017-08-15 20:34:35 +01004090 err = check_reg_arg(env, insn->dst_reg, DST_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004091 if (err)
4092 return err;
4093
4094 } else if (opcode == BPF_MOV) {
4095
4096 if (BPF_SRC(insn->code) == BPF_X) {
4097 if (insn->imm != 0 || insn->off != 0) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004098 verbose(env, "BPF_MOV uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004099 return -EINVAL;
4100 }
4101
4102 /* check src operand */
Edward Creedc503a82017-08-15 20:34:35 +01004103 err = check_reg_arg(env, insn->src_reg, SRC_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004104 if (err)
4105 return err;
4106 } else {
4107 if (insn->src_reg != BPF_REG_0 || insn->off != 0) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004108 verbose(env, "BPF_MOV uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004109 return -EINVAL;
4110 }
4111 }
4112
Arthur Fabrefbeb1602018-07-31 18:17:22 +01004113 /* check dest operand, mark as required later */
4114 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004115 if (err)
4116 return err;
4117
4118 if (BPF_SRC(insn->code) == BPF_X) {
Jiong Wange434b8c2018-12-07 12:16:18 -05004119 struct bpf_reg_state *src_reg = regs + insn->src_reg;
4120 struct bpf_reg_state *dst_reg = regs + insn->dst_reg;
4121
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004122 if (BPF_CLASS(insn->code) == BPF_ALU64) {
4123 /* case: R1 = R2
4124 * copy register state to dest reg
4125 */
Jiong Wange434b8c2018-12-07 12:16:18 -05004126 *dst_reg = *src_reg;
4127 dst_reg->live |= REG_LIVE_WRITTEN;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004128 } else {
Edward Creef1174f72017-08-07 15:26:19 +01004129 /* R1 = (u32) R2 */
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07004130 if (is_pointer_value(env, insn->src_reg)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004131 verbose(env,
4132 "R%d partial copy of pointer\n",
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07004133 insn->src_reg);
4134 return -EACCES;
Jiong Wange434b8c2018-12-07 12:16:18 -05004135 } else if (src_reg->type == SCALAR_VALUE) {
4136 *dst_reg = *src_reg;
4137 dst_reg->live |= REG_LIVE_WRITTEN;
4138 } else {
4139 mark_reg_unknown(env, regs,
4140 insn->dst_reg);
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07004141 }
Jiong Wange434b8c2018-12-07 12:16:18 -05004142 coerce_reg_to_size(dst_reg, 4);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004143 }
4144 } else {
4145 /* case: R = imm
4146 * remember the value we stored into this reg
4147 */
Arthur Fabrefbeb1602018-07-31 18:17:22 +01004148 /* clear any state __mark_reg_known doesn't set */
4149 mark_reg_unknown(env, regs, insn->dst_reg);
Edward Creef1174f72017-08-07 15:26:19 +01004150 regs[insn->dst_reg].type = SCALAR_VALUE;
Jann Horn95a762e2017-12-18 20:11:54 -08004151 if (BPF_CLASS(insn->code) == BPF_ALU64) {
4152 __mark_reg_known(regs + insn->dst_reg,
4153 insn->imm);
4154 } else {
4155 __mark_reg_known(regs + insn->dst_reg,
4156 (u32)insn->imm);
4157 }
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004158 }
4159
4160 } else if (opcode > BPF_END) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004161 verbose(env, "invalid BPF_ALU opcode %x\n", opcode);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004162 return -EINVAL;
4163
4164 } else { /* all other ALU ops: and, sub, xor, add, ... */
4165
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004166 if (BPF_SRC(insn->code) == BPF_X) {
4167 if (insn->imm != 0 || insn->off != 0) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004168 verbose(env, "BPF_ALU uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004169 return -EINVAL;
4170 }
4171 /* check src1 operand */
Edward Creedc503a82017-08-15 20:34:35 +01004172 err = check_reg_arg(env, insn->src_reg, SRC_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004173 if (err)
4174 return err;
4175 } else {
4176 if (insn->src_reg != BPF_REG_0 || insn->off != 0) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004177 verbose(env, "BPF_ALU uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004178 return -EINVAL;
4179 }
4180 }
4181
4182 /* check src2 operand */
Edward Creedc503a82017-08-15 20:34:35 +01004183 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004184 if (err)
4185 return err;
4186
4187 if ((opcode == BPF_MOD || opcode == BPF_DIV) &&
4188 BPF_SRC(insn->code) == BPF_K && insn->imm == 0) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004189 verbose(env, "div by zero\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004190 return -EINVAL;
4191 }
4192
Rabin Vincent229394e82016-01-12 20:17:08 +01004193 if ((opcode == BPF_LSH || opcode == BPF_RSH ||
4194 opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) {
4195 int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32;
4196
4197 if (insn->imm < 0 || insn->imm >= size) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004198 verbose(env, "invalid shift %d\n", insn->imm);
Rabin Vincent229394e82016-01-12 20:17:08 +01004199 return -EINVAL;
4200 }
4201 }
4202
Alexei Starovoitov1a0dc1a2016-05-05 19:49:09 -07004203 /* check dest operand */
Edward Creedc503a82017-08-15 20:34:35 +01004204 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
Alexei Starovoitov1a0dc1a2016-05-05 19:49:09 -07004205 if (err)
4206 return err;
4207
Edward Creef1174f72017-08-07 15:26:19 +01004208 return adjust_reg_min_max_vals(env, insn);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004209 }
4210
4211 return 0;
4212}
4213
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004214static void find_good_pkt_pointers(struct bpf_verifier_state *vstate,
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02004215 struct bpf_reg_state *dst_reg,
David S. Millerf8ddadc2017-10-22 13:36:53 +01004216 enum bpf_reg_type type,
Daniel Borkmannfb2a3112017-10-21 02:34:21 +02004217 bool range_right_open)
Alexei Starovoitov969bf052016-05-05 19:49:10 -07004218{
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004219 struct bpf_func_state *state = vstate->frame[vstate->curframe];
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01004220 struct bpf_reg_state *regs = state->regs, *reg;
Daniel Borkmannfb2a3112017-10-21 02:34:21 +02004221 u16 new_range;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004222 int i, j;
Daniel Borkmann2d2be8c2016-09-08 01:03:42 +02004223
Daniel Borkmannfb2a3112017-10-21 02:34:21 +02004224 if (dst_reg->off < 0 ||
4225 (dst_reg->off == 0 && range_right_open))
Edward Creef1174f72017-08-07 15:26:19 +01004226 /* This doesn't give us any range */
4227 return;
4228
Edward Creeb03c9f92017-08-07 15:26:36 +01004229 if (dst_reg->umax_value > MAX_PACKET_OFF ||
4230 dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF)
Edward Creef1174f72017-08-07 15:26:19 +01004231 /* Risk of overflow. For instance, ptr + (1<<63) may be less
4232 * than pkt_end, but that's because it's also less than pkt.
4233 */
4234 return;
4235
Daniel Borkmannfb2a3112017-10-21 02:34:21 +02004236 new_range = dst_reg->off;
4237 if (range_right_open)
4238 new_range--;
4239
4240 /* Examples for register markings:
Daniel Borkmann2d2be8c2016-09-08 01:03:42 +02004241 *
Daniel Borkmannfb2a3112017-10-21 02:34:21 +02004242 * pkt_data in dst register:
Daniel Borkmann2d2be8c2016-09-08 01:03:42 +02004243 *
4244 * r2 = r3;
4245 * r2 += 8;
4246 * if (r2 > pkt_end) goto <handle exception>
4247 * <access okay>
4248 *
Daniel Borkmannb4e432f2017-08-10 01:40:02 +02004249 * r2 = r3;
4250 * r2 += 8;
4251 * if (r2 < pkt_end) goto <access okay>
4252 * <handle exception>
4253 *
Daniel Borkmann2d2be8c2016-09-08 01:03:42 +02004254 * Where:
4255 * r2 == dst_reg, pkt_end == src_reg
4256 * r2=pkt(id=n,off=8,r=0)
4257 * r3=pkt(id=n,off=0,r=0)
4258 *
Daniel Borkmannfb2a3112017-10-21 02:34:21 +02004259 * pkt_data in src register:
Daniel Borkmann2d2be8c2016-09-08 01:03:42 +02004260 *
4261 * r2 = r3;
4262 * r2 += 8;
4263 * if (pkt_end >= r2) goto <access okay>
4264 * <handle exception>
4265 *
Daniel Borkmannb4e432f2017-08-10 01:40:02 +02004266 * r2 = r3;
4267 * r2 += 8;
4268 * if (pkt_end <= r2) goto <handle exception>
4269 * <access okay>
4270 *
Daniel Borkmann2d2be8c2016-09-08 01:03:42 +02004271 * Where:
4272 * pkt_end == dst_reg, r2 == src_reg
4273 * r2=pkt(id=n,off=8,r=0)
4274 * r3=pkt(id=n,off=0,r=0)
4275 *
4276 * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8)
Daniel Borkmannfb2a3112017-10-21 02:34:21 +02004277 * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8)
4278 * and [r3, r3 + 8-1) respectively is safe to access depending on
4279 * the check.
Alexei Starovoitov969bf052016-05-05 19:49:10 -07004280 */
Daniel Borkmann2d2be8c2016-09-08 01:03:42 +02004281
Edward Creef1174f72017-08-07 15:26:19 +01004282 /* If our ids match, then we must have the same max_value. And we
4283 * don't care about the other reg's fixed offset, since if it's too big
4284 * the range won't allow anything.
4285 * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16.
4286 */
Alexei Starovoitov969bf052016-05-05 19:49:10 -07004287 for (i = 0; i < MAX_BPF_REG; i++)
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02004288 if (regs[i].type == type && regs[i].id == dst_reg->id)
Alexei Starovoitovb1977682017-03-24 15:57:33 -07004289 /* keep the maximum range already checked */
Daniel Borkmannfb2a3112017-10-21 02:34:21 +02004290 regs[i].range = max(regs[i].range, new_range);
Alexei Starovoitov969bf052016-05-05 19:49:10 -07004291
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004292 for (j = 0; j <= vstate->curframe; j++) {
4293 state = vstate->frame[j];
Joe Stringerf3709f62018-10-02 13:35:29 -07004294 bpf_for_each_spilled_reg(i, state, reg) {
4295 if (!reg)
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004296 continue;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004297 if (reg->type == type && reg->id == dst_reg->id)
4298 reg->range = max(reg->range, new_range);
4299 }
Alexei Starovoitov969bf052016-05-05 19:49:10 -07004300 }
4301}
4302
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004303/* compute branch direction of the expression "if (reg opcode val) goto target;"
4304 * and return:
4305 * 1 - branch will be taken and "goto target" will be executed
4306 * 0 - branch will not be taken and fall-through to next insn
4307 * -1 - unknown. Example: "if (reg < 5)" is unknown when register value range [0,10]
4308 */
Jiong Wang092ed092019-01-26 12:26:01 -05004309static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode,
4310 bool is_jmp32)
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004311{
Jiong Wang092ed092019-01-26 12:26:01 -05004312 struct bpf_reg_state reg_lo;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004313 s64 sval;
4314
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004315 if (__is_pointer_value(false, reg))
4316 return -1;
4317
Jiong Wang092ed092019-01-26 12:26:01 -05004318 if (is_jmp32) {
4319 reg_lo = *reg;
4320 reg = &reg_lo;
4321 /* For JMP32, only low 32 bits are compared, coerce_reg_to_size
4322 * could truncate high bits and update umin/umax according to
4323 * information of low bits.
4324 */
4325 coerce_reg_to_size(reg, 4);
4326 /* smin/smax need special handling. For example, after coerce,
4327 * if smin_value is 0x00000000ffffffffLL, the value is -1 when
4328 * used as operand to JMP32. It is a negative number from s32's
4329 * point of view, while it is a positive number when seen as
4330 * s64. The smin/smax are kept as s64, therefore, when used with
4331 * JMP32, they need to be transformed into s32, then sign
4332 * extended back to s64.
4333 *
4334 * Also, smin/smax were copied from umin/umax. If umin/umax has
4335 * different sign bit, then min/max relationship doesn't
4336 * maintain after casting into s32, for this case, set smin/smax
4337 * to safest range.
4338 */
4339 if ((reg->umax_value ^ reg->umin_value) &
4340 (1ULL << 31)) {
4341 reg->smin_value = S32_MIN;
4342 reg->smax_value = S32_MAX;
4343 }
4344 reg->smin_value = (s64)(s32)reg->smin_value;
4345 reg->smax_value = (s64)(s32)reg->smax_value;
4346
4347 val = (u32)val;
4348 sval = (s64)(s32)val;
4349 } else {
4350 sval = (s64)val;
4351 }
Jiong Wanga72dafa2019-01-26 12:26:00 -05004352
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004353 switch (opcode) {
4354 case BPF_JEQ:
4355 if (tnum_is_const(reg->var_off))
4356 return !!tnum_equals_const(reg->var_off, val);
4357 break;
4358 case BPF_JNE:
4359 if (tnum_is_const(reg->var_off))
4360 return !tnum_equals_const(reg->var_off, val);
4361 break;
Jakub Kicinski960ea052018-12-19 22:13:04 -08004362 case BPF_JSET:
4363 if ((~reg->var_off.mask & reg->var_off.value) & val)
4364 return 1;
4365 if (!((reg->var_off.mask | reg->var_off.value) & val))
4366 return 0;
4367 break;
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004368 case BPF_JGT:
4369 if (reg->umin_value > val)
4370 return 1;
4371 else if (reg->umax_value <= val)
4372 return 0;
4373 break;
4374 case BPF_JSGT:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004375 if (reg->smin_value > sval)
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004376 return 1;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004377 else if (reg->smax_value < sval)
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004378 return 0;
4379 break;
4380 case BPF_JLT:
4381 if (reg->umax_value < val)
4382 return 1;
4383 else if (reg->umin_value >= val)
4384 return 0;
4385 break;
4386 case BPF_JSLT:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004387 if (reg->smax_value < sval)
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004388 return 1;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004389 else if (reg->smin_value >= sval)
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004390 return 0;
4391 break;
4392 case BPF_JGE:
4393 if (reg->umin_value >= val)
4394 return 1;
4395 else if (reg->umax_value < val)
4396 return 0;
4397 break;
4398 case BPF_JSGE:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004399 if (reg->smin_value >= sval)
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004400 return 1;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004401 else if (reg->smax_value < sval)
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004402 return 0;
4403 break;
4404 case BPF_JLE:
4405 if (reg->umax_value <= val)
4406 return 1;
4407 else if (reg->umin_value > val)
4408 return 0;
4409 break;
4410 case BPF_JSLE:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004411 if (reg->smax_value <= sval)
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004412 return 1;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004413 else if (reg->smin_value > sval)
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004414 return 0;
4415 break;
4416 }
4417
4418 return -1;
4419}
4420
Jiong Wang092ed092019-01-26 12:26:01 -05004421/* Generate min value of the high 32-bit from TNUM info. */
4422static u64 gen_hi_min(struct tnum var)
4423{
4424 return var.value & ~0xffffffffULL;
4425}
4426
4427/* Generate max value of the high 32-bit from TNUM info. */
4428static u64 gen_hi_max(struct tnum var)
4429{
4430 return (var.value | var.mask) & ~0xffffffffULL;
4431}
4432
4433/* Return true if VAL is compared with a s64 sign extended from s32, and they
4434 * are with the same signedness.
4435 */
4436static bool cmp_val_with_extended_s64(s64 sval, struct bpf_reg_state *reg)
4437{
4438 return ((s32)sval >= 0 &&
4439 reg->smin_value >= 0 && reg->smax_value <= S32_MAX) ||
4440 ((s32)sval < 0 &&
4441 reg->smax_value <= 0 && reg->smin_value >= S32_MIN);
4442}
4443
Josef Bacik48461132016-09-28 10:54:32 -04004444/* Adjusts the register min/max values in the case that the dst_reg is the
4445 * variable register that we are working on, and src_reg is a constant or we're
4446 * simply doing a BPF_K check.
Edward Creef1174f72017-08-07 15:26:19 +01004447 * In JEQ/JNE cases we also adjust the var_off values.
Josef Bacik48461132016-09-28 10:54:32 -04004448 */
4449static void reg_set_min_max(struct bpf_reg_state *true_reg,
4450 struct bpf_reg_state *false_reg, u64 val,
Jiong Wang092ed092019-01-26 12:26:01 -05004451 u8 opcode, bool is_jmp32)
Josef Bacik48461132016-09-28 10:54:32 -04004452{
Jiong Wanga72dafa2019-01-26 12:26:00 -05004453 s64 sval;
4454
Edward Creef1174f72017-08-07 15:26:19 +01004455 /* If the dst_reg is a pointer, we can't learn anything about its
4456 * variable offset from the compare (unless src_reg were a pointer into
4457 * the same object, but we don't bother with that.
4458 * Since false_reg and true_reg have the same type by construction, we
4459 * only need to check one of them for pointerness.
4460 */
4461 if (__is_pointer_value(false, false_reg))
4462 return;
Daniel Borkmann4cabc5b2017-07-21 00:00:21 +02004463
Jiong Wang092ed092019-01-26 12:26:01 -05004464 val = is_jmp32 ? (u32)val : val;
4465 sval = is_jmp32 ? (s64)(s32)val : (s64)val;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004466
Josef Bacik48461132016-09-28 10:54:32 -04004467 switch (opcode) {
4468 case BPF_JEQ:
Josef Bacik48461132016-09-28 10:54:32 -04004469 case BPF_JNE:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004470 {
4471 struct bpf_reg_state *reg =
4472 opcode == BPF_JEQ ? true_reg : false_reg;
4473
4474 /* For BPF_JEQ, if this is false we know nothing Jon Snow, but
4475 * if it is true we know the value for sure. Likewise for
4476 * BPF_JNE.
Josef Bacik48461132016-09-28 10:54:32 -04004477 */
Jiong Wang092ed092019-01-26 12:26:01 -05004478 if (is_jmp32) {
4479 u64 old_v = reg->var_off.value;
4480 u64 hi_mask = ~0xffffffffULL;
4481
4482 reg->var_off.value = (old_v & hi_mask) | val;
4483 reg->var_off.mask &= hi_mask;
4484 } else {
4485 __mark_reg_known(reg, val);
4486 }
Josef Bacik48461132016-09-28 10:54:32 -04004487 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004488 }
Jakub Kicinski960ea052018-12-19 22:13:04 -08004489 case BPF_JSET:
4490 false_reg->var_off = tnum_and(false_reg->var_off,
4491 tnum_const(~val));
4492 if (is_power_of_2(val))
4493 true_reg->var_off = tnum_or(true_reg->var_off,
4494 tnum_const(val));
4495 break;
Josef Bacik48461132016-09-28 10:54:32 -04004496 case BPF_JGE:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004497 case BPF_JGT:
4498 {
4499 u64 false_umax = opcode == BPF_JGT ? val : val - 1;
4500 u64 true_umin = opcode == BPF_JGT ? val + 1 : val;
4501
Jiong Wang092ed092019-01-26 12:26:01 -05004502 if (is_jmp32) {
4503 false_umax += gen_hi_max(false_reg->var_off);
4504 true_umin += gen_hi_min(true_reg->var_off);
4505 }
Jiong Wanga72dafa2019-01-26 12:26:00 -05004506 false_reg->umax_value = min(false_reg->umax_value, false_umax);
4507 true_reg->umin_value = max(true_reg->umin_value, true_umin);
Edward Creeb03c9f92017-08-07 15:26:36 +01004508 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004509 }
Josef Bacik48461132016-09-28 10:54:32 -04004510 case BPF_JSGE:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004511 case BPF_JSGT:
4512 {
4513 s64 false_smax = opcode == BPF_JSGT ? sval : sval - 1;
4514 s64 true_smin = opcode == BPF_JSGT ? sval + 1 : sval;
4515
Jiong Wang092ed092019-01-26 12:26:01 -05004516 /* If the full s64 was not sign-extended from s32 then don't
4517 * deduct further info.
4518 */
4519 if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg))
4520 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004521 false_reg->smax_value = min(false_reg->smax_value, false_smax);
4522 true_reg->smin_value = max(true_reg->smin_value, true_smin);
Josef Bacik48461132016-09-28 10:54:32 -04004523 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004524 }
Daniel Borkmannb4e432f2017-08-10 01:40:02 +02004525 case BPF_JLE:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004526 case BPF_JLT:
4527 {
4528 u64 false_umin = opcode == BPF_JLT ? val : val + 1;
4529 u64 true_umax = opcode == BPF_JLT ? val - 1 : val;
4530
Jiong Wang092ed092019-01-26 12:26:01 -05004531 if (is_jmp32) {
4532 false_umin += gen_hi_min(false_reg->var_off);
4533 true_umax += gen_hi_max(true_reg->var_off);
4534 }
Jiong Wanga72dafa2019-01-26 12:26:00 -05004535 false_reg->umin_value = max(false_reg->umin_value, false_umin);
4536 true_reg->umax_value = min(true_reg->umax_value, true_umax);
Daniel Borkmannb4e432f2017-08-10 01:40:02 +02004537 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004538 }
Daniel Borkmannb4e432f2017-08-10 01:40:02 +02004539 case BPF_JSLE:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004540 case BPF_JSLT:
4541 {
4542 s64 false_smin = opcode == BPF_JSLT ? sval : sval + 1;
4543 s64 true_smax = opcode == BPF_JSLT ? sval - 1 : sval;
4544
Jiong Wang092ed092019-01-26 12:26:01 -05004545 if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg))
4546 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004547 false_reg->smin_value = max(false_reg->smin_value, false_smin);
4548 true_reg->smax_value = min(true_reg->smax_value, true_smax);
Daniel Borkmannb4e432f2017-08-10 01:40:02 +02004549 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004550 }
Josef Bacik48461132016-09-28 10:54:32 -04004551 default:
4552 break;
4553 }
4554
Edward Creeb03c9f92017-08-07 15:26:36 +01004555 __reg_deduce_bounds(false_reg);
4556 __reg_deduce_bounds(true_reg);
4557 /* We might have learned some bits from the bounds. */
4558 __reg_bound_offset(false_reg);
4559 __reg_bound_offset(true_reg);
4560 /* Intersecting with the old var_off might have improved our bounds
4561 * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc),
4562 * then new var_off is (0; 0x7f...fc) which improves our umax.
4563 */
4564 __update_reg_bounds(false_reg);
4565 __update_reg_bounds(true_reg);
Josef Bacik48461132016-09-28 10:54:32 -04004566}
4567
Edward Creef1174f72017-08-07 15:26:19 +01004568/* Same as above, but for the case that dst_reg holds a constant and src_reg is
4569 * the variable reg.
Josef Bacik48461132016-09-28 10:54:32 -04004570 */
4571static void reg_set_min_max_inv(struct bpf_reg_state *true_reg,
4572 struct bpf_reg_state *false_reg, u64 val,
Jiong Wang092ed092019-01-26 12:26:01 -05004573 u8 opcode, bool is_jmp32)
Josef Bacik48461132016-09-28 10:54:32 -04004574{
Jiong Wanga72dafa2019-01-26 12:26:00 -05004575 s64 sval;
4576
Edward Creef1174f72017-08-07 15:26:19 +01004577 if (__is_pointer_value(false, false_reg))
4578 return;
Daniel Borkmann4cabc5b2017-07-21 00:00:21 +02004579
Jiong Wang092ed092019-01-26 12:26:01 -05004580 val = is_jmp32 ? (u32)val : val;
4581 sval = is_jmp32 ? (s64)(s32)val : (s64)val;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004582
Josef Bacik48461132016-09-28 10:54:32 -04004583 switch (opcode) {
4584 case BPF_JEQ:
Josef Bacik48461132016-09-28 10:54:32 -04004585 case BPF_JNE:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004586 {
4587 struct bpf_reg_state *reg =
4588 opcode == BPF_JEQ ? true_reg : false_reg;
4589
Jiong Wang092ed092019-01-26 12:26:01 -05004590 if (is_jmp32) {
4591 u64 old_v = reg->var_off.value;
4592 u64 hi_mask = ~0xffffffffULL;
4593
4594 reg->var_off.value = (old_v & hi_mask) | val;
4595 reg->var_off.mask &= hi_mask;
4596 } else {
4597 __mark_reg_known(reg, val);
4598 }
Josef Bacik48461132016-09-28 10:54:32 -04004599 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004600 }
Jakub Kicinski960ea052018-12-19 22:13:04 -08004601 case BPF_JSET:
4602 false_reg->var_off = tnum_and(false_reg->var_off,
4603 tnum_const(~val));
4604 if (is_power_of_2(val))
4605 true_reg->var_off = tnum_or(true_reg->var_off,
4606 tnum_const(val));
4607 break;
Josef Bacik48461132016-09-28 10:54:32 -04004608 case BPF_JGE:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004609 case BPF_JGT:
4610 {
4611 u64 false_umin = opcode == BPF_JGT ? val : val + 1;
4612 u64 true_umax = opcode == BPF_JGT ? val - 1 : val;
4613
Jiong Wang092ed092019-01-26 12:26:01 -05004614 if (is_jmp32) {
4615 false_umin += gen_hi_min(false_reg->var_off);
4616 true_umax += gen_hi_max(true_reg->var_off);
4617 }
Jiong Wanga72dafa2019-01-26 12:26:00 -05004618 false_reg->umin_value = max(false_reg->umin_value, false_umin);
4619 true_reg->umax_value = min(true_reg->umax_value, true_umax);
Edward Creeb03c9f92017-08-07 15:26:36 +01004620 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004621 }
Josef Bacik48461132016-09-28 10:54:32 -04004622 case BPF_JSGE:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004623 case BPF_JSGT:
4624 {
4625 s64 false_smin = opcode == BPF_JSGT ? sval : sval + 1;
4626 s64 true_smax = opcode == BPF_JSGT ? sval - 1 : sval;
4627
Jiong Wang092ed092019-01-26 12:26:01 -05004628 if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg))
4629 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004630 false_reg->smin_value = max(false_reg->smin_value, false_smin);
4631 true_reg->smax_value = min(true_reg->smax_value, true_smax);
Josef Bacik48461132016-09-28 10:54:32 -04004632 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004633 }
Daniel Borkmannb4e432f2017-08-10 01:40:02 +02004634 case BPF_JLE:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004635 case BPF_JLT:
4636 {
4637 u64 false_umax = opcode == BPF_JLT ? val : val - 1;
4638 u64 true_umin = opcode == BPF_JLT ? val + 1 : val;
4639
Jiong Wang092ed092019-01-26 12:26:01 -05004640 if (is_jmp32) {
4641 false_umax += gen_hi_max(false_reg->var_off);
4642 true_umin += gen_hi_min(true_reg->var_off);
4643 }
Jiong Wanga72dafa2019-01-26 12:26:00 -05004644 false_reg->umax_value = min(false_reg->umax_value, false_umax);
4645 true_reg->umin_value = max(true_reg->umin_value, true_umin);
Daniel Borkmannb4e432f2017-08-10 01:40:02 +02004646 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004647 }
Daniel Borkmannb4e432f2017-08-10 01:40:02 +02004648 case BPF_JSLE:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004649 case BPF_JSLT:
4650 {
4651 s64 false_smax = opcode == BPF_JSLT ? sval : sval - 1;
4652 s64 true_smin = opcode == BPF_JSLT ? sval + 1 : sval;
4653
Jiong Wang092ed092019-01-26 12:26:01 -05004654 if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg))
4655 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004656 false_reg->smax_value = min(false_reg->smax_value, false_smax);
4657 true_reg->smin_value = max(true_reg->smin_value, true_smin);
Daniel Borkmannb4e432f2017-08-10 01:40:02 +02004658 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004659 }
Josef Bacik48461132016-09-28 10:54:32 -04004660 default:
4661 break;
4662 }
4663
Edward Creeb03c9f92017-08-07 15:26:36 +01004664 __reg_deduce_bounds(false_reg);
4665 __reg_deduce_bounds(true_reg);
4666 /* We might have learned some bits from the bounds. */
4667 __reg_bound_offset(false_reg);
4668 __reg_bound_offset(true_reg);
4669 /* Intersecting with the old var_off might have improved our bounds
4670 * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc),
4671 * then new var_off is (0; 0x7f...fc) which improves our umax.
4672 */
4673 __update_reg_bounds(false_reg);
4674 __update_reg_bounds(true_reg);
Edward Creef1174f72017-08-07 15:26:19 +01004675}
4676
4677/* Regs are known to be equal, so intersect their min/max/var_off */
4678static void __reg_combine_min_max(struct bpf_reg_state *src_reg,
4679 struct bpf_reg_state *dst_reg)
4680{
Edward Creeb03c9f92017-08-07 15:26:36 +01004681 src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value,
4682 dst_reg->umin_value);
4683 src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value,
4684 dst_reg->umax_value);
4685 src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value,
4686 dst_reg->smin_value);
4687 src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value,
4688 dst_reg->smax_value);
Edward Creef1174f72017-08-07 15:26:19 +01004689 src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off,
4690 dst_reg->var_off);
Edward Creeb03c9f92017-08-07 15:26:36 +01004691 /* We might have learned new bounds from the var_off. */
4692 __update_reg_bounds(src_reg);
4693 __update_reg_bounds(dst_reg);
4694 /* We might have learned something about the sign bit. */
4695 __reg_deduce_bounds(src_reg);
4696 __reg_deduce_bounds(dst_reg);
4697 /* We might have learned some bits from the bounds. */
4698 __reg_bound_offset(src_reg);
4699 __reg_bound_offset(dst_reg);
4700 /* Intersecting with the old var_off might have improved our bounds
4701 * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc),
4702 * then new var_off is (0; 0x7f...fc) which improves our umax.
4703 */
4704 __update_reg_bounds(src_reg);
4705 __update_reg_bounds(dst_reg);
Edward Creef1174f72017-08-07 15:26:19 +01004706}
4707
4708static void reg_combine_min_max(struct bpf_reg_state *true_src,
4709 struct bpf_reg_state *true_dst,
4710 struct bpf_reg_state *false_src,
4711 struct bpf_reg_state *false_dst,
4712 u8 opcode)
4713{
4714 switch (opcode) {
4715 case BPF_JEQ:
4716 __reg_combine_min_max(true_src, true_dst);
4717 break;
4718 case BPF_JNE:
4719 __reg_combine_min_max(false_src, false_dst);
Edward Creeb03c9f92017-08-07 15:26:36 +01004720 break;
Daniel Borkmann4cabc5b2017-07-21 00:00:21 +02004721 }
Josef Bacik48461132016-09-28 10:54:32 -04004722}
4723
Joe Stringerfd978bf72018-10-02 13:35:35 -07004724static void mark_ptr_or_null_reg(struct bpf_func_state *state,
4725 struct bpf_reg_state *reg, u32 id,
Joe Stringer840b9612018-10-02 13:35:32 -07004726 bool is_null)
Thomas Graf57a09bf2016-10-18 19:51:19 +02004727{
Joe Stringer840b9612018-10-02 13:35:32 -07004728 if (reg_type_may_be_null(reg->type) && reg->id == id) {
Edward Creef1174f72017-08-07 15:26:19 +01004729 /* Old offset (both fixed and variable parts) should
4730 * have been known-zero, because we don't allow pointer
4731 * arithmetic on pointers that might be NULL.
4732 */
Edward Creeb03c9f92017-08-07 15:26:36 +01004733 if (WARN_ON_ONCE(reg->smin_value || reg->smax_value ||
4734 !tnum_equals_const(reg->var_off, 0) ||
Edward Creef1174f72017-08-07 15:26:19 +01004735 reg->off)) {
Edward Creeb03c9f92017-08-07 15:26:36 +01004736 __mark_reg_known_zero(reg);
4737 reg->off = 0;
Edward Creef1174f72017-08-07 15:26:19 +01004738 }
4739 if (is_null) {
4740 reg->type = SCALAR_VALUE;
Joe Stringer840b9612018-10-02 13:35:32 -07004741 } else if (reg->type == PTR_TO_MAP_VALUE_OR_NULL) {
4742 if (reg->map_ptr->inner_map_meta) {
4743 reg->type = CONST_PTR_TO_MAP;
4744 reg->map_ptr = reg->map_ptr->inner_map_meta;
4745 } else {
4746 reg->type = PTR_TO_MAP_VALUE;
4747 }
Joe Stringerc64b7982018-10-02 13:35:33 -07004748 } else if (reg->type == PTR_TO_SOCKET_OR_NULL) {
4749 reg->type = PTR_TO_SOCKET;
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08004750 } else if (reg->type == PTR_TO_SOCK_COMMON_OR_NULL) {
4751 reg->type = PTR_TO_SOCK_COMMON;
Martin KaFai Lau655a51e2019-02-09 23:22:24 -08004752 } else if (reg->type == PTR_TO_TCP_SOCK_OR_NULL) {
4753 reg->type = PTR_TO_TCP_SOCK;
Martin KaFai Lau56f668d2017-03-22 10:00:33 -07004754 }
Martin KaFai Lau1b986582019-03-12 10:23:02 -07004755 if (is_null) {
4756 /* We don't need id and ref_obj_id from this point
4757 * onwards anymore, thus we should better reset it,
4758 * so that state pruning has chances to take effect.
4759 */
4760 reg->id = 0;
4761 reg->ref_obj_id = 0;
4762 } else if (!reg_may_point_to_spin_lock(reg)) {
4763 /* For not-NULL ptr, reg->ref_obj_id will be reset
4764 * in release_reg_references().
4765 *
4766 * reg->id is still used by spin_lock ptr. Other
4767 * than spin_lock ptr type, reg->id can be reset.
Joe Stringerfd978bf72018-10-02 13:35:35 -07004768 */
4769 reg->id = 0;
4770 }
Thomas Graf57a09bf2016-10-18 19:51:19 +02004771 }
4772}
4773
4774/* The logic is similar to find_good_pkt_pointers(), both could eventually
4775 * be folded together at some point.
4776 */
Joe Stringer840b9612018-10-02 13:35:32 -07004777static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno,
4778 bool is_null)
Thomas Graf57a09bf2016-10-18 19:51:19 +02004779{
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004780 struct bpf_func_state *state = vstate->frame[vstate->curframe];
Joe Stringerf3709f62018-10-02 13:35:29 -07004781 struct bpf_reg_state *reg, *regs = state->regs;
Martin KaFai Lau1b986582019-03-12 10:23:02 -07004782 u32 ref_obj_id = regs[regno].ref_obj_id;
Daniel Borkmanna08dd0d2016-12-15 01:30:06 +01004783 u32 id = regs[regno].id;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004784 int i, j;
Thomas Graf57a09bf2016-10-18 19:51:19 +02004785
Martin KaFai Lau1b986582019-03-12 10:23:02 -07004786 if (ref_obj_id && ref_obj_id == id && is_null)
4787 /* regs[regno] is in the " == NULL" branch.
4788 * No one could have freed the reference state before
4789 * doing the NULL check.
4790 */
4791 WARN_ON_ONCE(release_reference_state(state, id));
Joe Stringerfd978bf72018-10-02 13:35:35 -07004792
Thomas Graf57a09bf2016-10-18 19:51:19 +02004793 for (i = 0; i < MAX_BPF_REG; i++)
Joe Stringerfd978bf72018-10-02 13:35:35 -07004794 mark_ptr_or_null_reg(state, &regs[i], id, is_null);
Thomas Graf57a09bf2016-10-18 19:51:19 +02004795
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004796 for (j = 0; j <= vstate->curframe; j++) {
4797 state = vstate->frame[j];
Joe Stringerf3709f62018-10-02 13:35:29 -07004798 bpf_for_each_spilled_reg(i, state, reg) {
4799 if (!reg)
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004800 continue;
Joe Stringerfd978bf72018-10-02 13:35:35 -07004801 mark_ptr_or_null_reg(state, reg, id, is_null);
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004802 }
Thomas Graf57a09bf2016-10-18 19:51:19 +02004803 }
4804}
4805
Daniel Borkmann5beca082017-11-01 23:58:10 +01004806static bool try_match_pkt_pointers(const struct bpf_insn *insn,
4807 struct bpf_reg_state *dst_reg,
4808 struct bpf_reg_state *src_reg,
4809 struct bpf_verifier_state *this_branch,
4810 struct bpf_verifier_state *other_branch)
4811{
4812 if (BPF_SRC(insn->code) != BPF_X)
4813 return false;
4814
Jiong Wang092ed092019-01-26 12:26:01 -05004815 /* Pointers are always 64-bit. */
4816 if (BPF_CLASS(insn->code) == BPF_JMP32)
4817 return false;
4818
Daniel Borkmann5beca082017-11-01 23:58:10 +01004819 switch (BPF_OP(insn->code)) {
4820 case BPF_JGT:
4821 if ((dst_reg->type == PTR_TO_PACKET &&
4822 src_reg->type == PTR_TO_PACKET_END) ||
4823 (dst_reg->type == PTR_TO_PACKET_META &&
4824 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
4825 /* pkt_data' > pkt_end, pkt_meta' > pkt_data */
4826 find_good_pkt_pointers(this_branch, dst_reg,
4827 dst_reg->type, false);
4828 } else if ((dst_reg->type == PTR_TO_PACKET_END &&
4829 src_reg->type == PTR_TO_PACKET) ||
4830 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
4831 src_reg->type == PTR_TO_PACKET_META)) {
4832 /* pkt_end > pkt_data', pkt_data > pkt_meta' */
4833 find_good_pkt_pointers(other_branch, src_reg,
4834 src_reg->type, true);
4835 } else {
4836 return false;
4837 }
4838 break;
4839 case BPF_JLT:
4840 if ((dst_reg->type == PTR_TO_PACKET &&
4841 src_reg->type == PTR_TO_PACKET_END) ||
4842 (dst_reg->type == PTR_TO_PACKET_META &&
4843 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
4844 /* pkt_data' < pkt_end, pkt_meta' < pkt_data */
4845 find_good_pkt_pointers(other_branch, dst_reg,
4846 dst_reg->type, true);
4847 } else if ((dst_reg->type == PTR_TO_PACKET_END &&
4848 src_reg->type == PTR_TO_PACKET) ||
4849 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
4850 src_reg->type == PTR_TO_PACKET_META)) {
4851 /* pkt_end < pkt_data', pkt_data > pkt_meta' */
4852 find_good_pkt_pointers(this_branch, src_reg,
4853 src_reg->type, false);
4854 } else {
4855 return false;
4856 }
4857 break;
4858 case BPF_JGE:
4859 if ((dst_reg->type == PTR_TO_PACKET &&
4860 src_reg->type == PTR_TO_PACKET_END) ||
4861 (dst_reg->type == PTR_TO_PACKET_META &&
4862 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
4863 /* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */
4864 find_good_pkt_pointers(this_branch, dst_reg,
4865 dst_reg->type, true);
4866 } else if ((dst_reg->type == PTR_TO_PACKET_END &&
4867 src_reg->type == PTR_TO_PACKET) ||
4868 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
4869 src_reg->type == PTR_TO_PACKET_META)) {
4870 /* pkt_end >= pkt_data', pkt_data >= pkt_meta' */
4871 find_good_pkt_pointers(other_branch, src_reg,
4872 src_reg->type, false);
4873 } else {
4874 return false;
4875 }
4876 break;
4877 case BPF_JLE:
4878 if ((dst_reg->type == PTR_TO_PACKET &&
4879 src_reg->type == PTR_TO_PACKET_END) ||
4880 (dst_reg->type == PTR_TO_PACKET_META &&
4881 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
4882 /* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */
4883 find_good_pkt_pointers(other_branch, dst_reg,
4884 dst_reg->type, false);
4885 } else if ((dst_reg->type == PTR_TO_PACKET_END &&
4886 src_reg->type == PTR_TO_PACKET) ||
4887 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
4888 src_reg->type == PTR_TO_PACKET_META)) {
4889 /* pkt_end <= pkt_data', pkt_data <= pkt_meta' */
4890 find_good_pkt_pointers(this_branch, src_reg,
4891 src_reg->type, true);
4892 } else {
4893 return false;
4894 }
4895 break;
4896 default:
4897 return false;
4898 }
4899
4900 return true;
4901}
4902
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01004903static int check_cond_jmp_op(struct bpf_verifier_env *env,
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004904 struct bpf_insn *insn, int *insn_idx)
4905{
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004906 struct bpf_verifier_state *this_branch = env->cur_state;
4907 struct bpf_verifier_state *other_branch;
4908 struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs;
4909 struct bpf_reg_state *dst_reg, *other_branch_regs;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004910 u8 opcode = BPF_OP(insn->code);
Jiong Wang092ed092019-01-26 12:26:01 -05004911 bool is_jmp32;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004912 int err;
4913
Jiong Wang092ed092019-01-26 12:26:01 -05004914 /* Only conditional jumps are expected to reach here. */
4915 if (opcode == BPF_JA || opcode > BPF_JSLE) {
4916 verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004917 return -EINVAL;
4918 }
4919
4920 if (BPF_SRC(insn->code) == BPF_X) {
4921 if (insn->imm != 0) {
Jiong Wang092ed092019-01-26 12:26:01 -05004922 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004923 return -EINVAL;
4924 }
4925
4926 /* check src1 operand */
Edward Creedc503a82017-08-15 20:34:35 +01004927 err = check_reg_arg(env, insn->src_reg, SRC_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004928 if (err)
4929 return err;
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07004930
4931 if (is_pointer_value(env, insn->src_reg)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004932 verbose(env, "R%d pointer comparison prohibited\n",
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07004933 insn->src_reg);
4934 return -EACCES;
4935 }
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004936 } else {
4937 if (insn->src_reg != BPF_REG_0) {
Jiong Wang092ed092019-01-26 12:26:01 -05004938 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004939 return -EINVAL;
4940 }
4941 }
4942
4943 /* check src2 operand */
Edward Creedc503a82017-08-15 20:34:35 +01004944 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004945 if (err)
4946 return err;
4947
Alexei Starovoitov1a0dc1a2016-05-05 19:49:09 -07004948 dst_reg = &regs[insn->dst_reg];
Jiong Wang092ed092019-01-26 12:26:01 -05004949 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
Alexei Starovoitov1a0dc1a2016-05-05 19:49:09 -07004950
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004951 if (BPF_SRC(insn->code) == BPF_K) {
Jiong Wang092ed092019-01-26 12:26:01 -05004952 int pred = is_branch_taken(dst_reg, insn->imm, opcode,
4953 is_jmp32);
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004954
4955 if (pred == 1) {
4956 /* only follow the goto, ignore fall-through */
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004957 *insn_idx += insn->off;
4958 return 0;
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004959 } else if (pred == 0) {
4960 /* only follow fall-through branch, since
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004961 * that's where the program will go
4962 */
4963 return 0;
4964 }
4965 }
4966
Daniel Borkmann979d63d2019-01-03 00:58:34 +01004967 other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx,
4968 false);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004969 if (!other_branch)
4970 return -EFAULT;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004971 other_branch_regs = other_branch->frame[other_branch->curframe]->regs;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004972
Josef Bacik48461132016-09-28 10:54:32 -04004973 /* detect if we are comparing against a constant value so we can adjust
4974 * our min/max values for our dst register.
Edward Creef1174f72017-08-07 15:26:19 +01004975 * this is only legit if both are scalars (or pointers to the same
4976 * object, I suppose, but we don't support that right now), because
4977 * otherwise the different base pointers mean the offsets aren't
4978 * comparable.
Josef Bacik48461132016-09-28 10:54:32 -04004979 */
4980 if (BPF_SRC(insn->code) == BPF_X) {
Jiong Wang092ed092019-01-26 12:26:01 -05004981 struct bpf_reg_state *src_reg = &regs[insn->src_reg];
4982 struct bpf_reg_state lo_reg0 = *dst_reg;
4983 struct bpf_reg_state lo_reg1 = *src_reg;
4984 struct bpf_reg_state *src_lo, *dst_lo;
4985
4986 dst_lo = &lo_reg0;
4987 src_lo = &lo_reg1;
4988 coerce_reg_to_size(dst_lo, 4);
4989 coerce_reg_to_size(src_lo, 4);
4990
Edward Creef1174f72017-08-07 15:26:19 +01004991 if (dst_reg->type == SCALAR_VALUE &&
Jiong Wang092ed092019-01-26 12:26:01 -05004992 src_reg->type == SCALAR_VALUE) {
4993 if (tnum_is_const(src_reg->var_off) ||
4994 (is_jmp32 && tnum_is_const(src_lo->var_off)))
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004995 reg_set_min_max(&other_branch_regs[insn->dst_reg],
Jiong Wang092ed092019-01-26 12:26:01 -05004996 dst_reg,
4997 is_jmp32
4998 ? src_lo->var_off.value
4999 : src_reg->var_off.value,
5000 opcode, is_jmp32);
5001 else if (tnum_is_const(dst_reg->var_off) ||
5002 (is_jmp32 && tnum_is_const(dst_lo->var_off)))
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08005003 reg_set_min_max_inv(&other_branch_regs[insn->src_reg],
Jiong Wang092ed092019-01-26 12:26:01 -05005004 src_reg,
5005 is_jmp32
5006 ? dst_lo->var_off.value
5007 : dst_reg->var_off.value,
5008 opcode, is_jmp32);
5009 else if (!is_jmp32 &&
5010 (opcode == BPF_JEQ || opcode == BPF_JNE))
Edward Creef1174f72017-08-07 15:26:19 +01005011 /* Comparing for equality, we can combine knowledge */
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08005012 reg_combine_min_max(&other_branch_regs[insn->src_reg],
5013 &other_branch_regs[insn->dst_reg],
Jiong Wang092ed092019-01-26 12:26:01 -05005014 src_reg, dst_reg, opcode);
Edward Creef1174f72017-08-07 15:26:19 +01005015 }
5016 } else if (dst_reg->type == SCALAR_VALUE) {
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08005017 reg_set_min_max(&other_branch_regs[insn->dst_reg],
Jiong Wang092ed092019-01-26 12:26:01 -05005018 dst_reg, insn->imm, opcode, is_jmp32);
Josef Bacik48461132016-09-28 10:54:32 -04005019 }
5020
Jiong Wang092ed092019-01-26 12:26:01 -05005021 /* detect if R == 0 where R is returned from bpf_map_lookup_elem().
5022 * NOTE: these optimizations below are related with pointer comparison
5023 * which will never be JMP32.
5024 */
5025 if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K &&
Alexei Starovoitov1a0dc1a2016-05-05 19:49:09 -07005026 insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) &&
Joe Stringer840b9612018-10-02 13:35:32 -07005027 reg_type_may_be_null(dst_reg->type)) {
5028 /* Mark all identical registers in each branch as either
Thomas Graf57a09bf2016-10-18 19:51:19 +02005029 * safe or unknown depending R == 0 or R != 0 conditional.
5030 */
Joe Stringer840b9612018-10-02 13:35:32 -07005031 mark_ptr_or_null_regs(this_branch, insn->dst_reg,
5032 opcode == BPF_JNE);
5033 mark_ptr_or_null_regs(other_branch, insn->dst_reg,
5034 opcode == BPF_JEQ);
Daniel Borkmann5beca082017-11-01 23:58:10 +01005035 } else if (!try_match_pkt_pointers(insn, dst_reg, &regs[insn->src_reg],
5036 this_branch, other_branch) &&
5037 is_pointer_value(env, insn->dst_reg)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005038 verbose(env, "R%d pointer comparison prohibited\n",
5039 insn->dst_reg);
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07005040 return -EACCES;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07005041 }
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07005042 if (env->log.level & BPF_LOG_LEVEL)
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08005043 print_verifier_state(env, this_branch->frame[this_branch->curframe]);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07005044 return 0;
5045}
5046
Alexei Starovoitov0246e642014-09-26 00:17:04 -07005047/* return the map pointer stored inside BPF_LD_IMM64 instruction */
5048static struct bpf_map *ld_imm64_to_map_ptr(struct bpf_insn *insn)
5049{
5050 u64 imm64 = ((u64) (u32) insn[0].imm) | ((u64) (u32) insn[1].imm) << 32;
5051
5052 return (struct bpf_map *) (unsigned long) imm64;
5053}
5054
Alexei Starovoitov17a52672014-09-26 00:17:06 -07005055/* verify BPF_LD_IMM64 instruction */
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01005056static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)
Alexei Starovoitov17a52672014-09-26 00:17:06 -07005057{
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07005058 struct bpf_reg_state *regs = cur_regs(env);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07005059 int err;
5060
5061 if (BPF_SIZE(insn->code) != BPF_DW) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005062 verbose(env, "invalid BPF_LD_IMM insn\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07005063 return -EINVAL;
5064 }
5065 if (insn->off != 0) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005066 verbose(env, "BPF_LD_IMM64 uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07005067 return -EINVAL;
5068 }
5069
Edward Creedc503a82017-08-15 20:34:35 +01005070 err = check_reg_arg(env, insn->dst_reg, DST_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07005071 if (err)
5072 return err;
5073
Jakub Kicinski6b173872016-09-21 11:43:59 +01005074 if (insn->src_reg == 0) {
Jakub Kicinski6b173872016-09-21 11:43:59 +01005075 u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm;
5076
Edward Creef1174f72017-08-07 15:26:19 +01005077 regs[insn->dst_reg].type = SCALAR_VALUE;
Edward Creeb03c9f92017-08-07 15:26:36 +01005078 __mark_reg_known(&regs[insn->dst_reg], imm);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07005079 return 0;
Jakub Kicinski6b173872016-09-21 11:43:59 +01005080 }
Alexei Starovoitov17a52672014-09-26 00:17:06 -07005081
5082 /* replace_map_fd_with_map_ptr() should have caught bad ld_imm64 */
5083 BUG_ON(insn->src_reg != BPF_PSEUDO_MAP_FD);
5084
5085 regs[insn->dst_reg].type = CONST_PTR_TO_MAP;
5086 regs[insn->dst_reg].map_ptr = ld_imm64_to_map_ptr(insn);
5087 return 0;
5088}
5089
Daniel Borkmann96be4322015-03-01 12:31:46 +01005090static bool may_access_skb(enum bpf_prog_type type)
5091{
5092 switch (type) {
5093 case BPF_PROG_TYPE_SOCKET_FILTER:
5094 case BPF_PROG_TYPE_SCHED_CLS:
Daniel Borkmann94caee8c2015-03-20 15:11:11 +01005095 case BPF_PROG_TYPE_SCHED_ACT:
Daniel Borkmann96be4322015-03-01 12:31:46 +01005096 return true;
5097 default:
5098 return false;
5099 }
5100}
5101
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005102/* verify safety of LD_ABS|LD_IND instructions:
5103 * - they can only appear in the programs where ctx == skb
5104 * - since they are wrappers of function calls, they scratch R1-R5 registers,
5105 * preserve R6-R9, and store return value into R0
5106 *
5107 * Implicit input:
5108 * ctx == skb == R6 == CTX
5109 *
5110 * Explicit input:
5111 * SRC == any register
5112 * IMM == 32-bit immediate
5113 *
5114 * Output:
5115 * R0 - 8/16/32-bit skb data converted to cpu endianness
5116 */
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01005117static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005118{
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07005119 struct bpf_reg_state *regs = cur_regs(env);
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005120 u8 mode = BPF_MODE(insn->code);
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005121 int i, err;
5122
Daniel Borkmann24701ec2015-03-01 12:31:47 +01005123 if (!may_access_skb(env->prog->type)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005124 verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n");
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005125 return -EINVAL;
5126 }
5127
Daniel Borkmanne0cea7c2018-05-04 01:08:14 +02005128 if (!env->ops->gen_ld_abs) {
5129 verbose(env, "bpf verifier is misconfigured\n");
5130 return -EINVAL;
5131 }
5132
Jiong Wangf910cef2018-05-02 16:17:17 -04005133 if (env->subprog_cnt > 1) {
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08005134 /* when program has LD_ABS insn JITs and interpreter assume
5135 * that r1 == ctx == skb which is not the case for callees
5136 * that can have arbitrary arguments. It's problematic
5137 * for main prog as well since JITs would need to analyze
5138 * all functions in order to make proper register save/restore
5139 * decisions in the main prog. Hence disallow LD_ABS with calls
5140 */
5141 verbose(env, "BPF_LD_[ABS|IND] instructions cannot be mixed with bpf-to-bpf calls\n");
5142 return -EINVAL;
5143 }
5144
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005145 if (insn->dst_reg != BPF_REG_0 || insn->off != 0 ||
Alexei Starovoitovd82bccc2016-04-12 10:26:19 -07005146 BPF_SIZE(insn->code) == BPF_DW ||
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005147 (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005148 verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n");
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005149 return -EINVAL;
5150 }
5151
5152 /* check whether implicit source operand (register R6) is readable */
Edward Creedc503a82017-08-15 20:34:35 +01005153 err = check_reg_arg(env, BPF_REG_6, SRC_OP);
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005154 if (err)
5155 return err;
5156
Joe Stringerfd978bf72018-10-02 13:35:35 -07005157 /* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as
5158 * gen_ld_abs() may terminate the program at runtime, leading to
5159 * reference leak.
5160 */
5161 err = check_reference_leak(env);
5162 if (err) {
5163 verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n");
5164 return err;
5165 }
5166
Alexei Starovoitovd83525c2019-01-31 15:40:04 -08005167 if (env->cur_state->active_spin_lock) {
5168 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n");
5169 return -EINVAL;
5170 }
5171
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005172 if (regs[BPF_REG_6].type != PTR_TO_CTX) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005173 verbose(env,
5174 "at the time of BPF_LD_ABS|IND R6 != pointer to skb\n");
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005175 return -EINVAL;
5176 }
5177
5178 if (mode == BPF_IND) {
5179 /* check explicit source operand */
Edward Creedc503a82017-08-15 20:34:35 +01005180 err = check_reg_arg(env, insn->src_reg, SRC_OP);
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005181 if (err)
5182 return err;
5183 }
5184
5185 /* reset caller saved regs to unreadable */
Edward Creedc503a82017-08-15 20:34:35 +01005186 for (i = 0; i < CALLER_SAVED_REGS; i++) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005187 mark_reg_not_init(env, regs, caller_saved[i]);
Edward Creedc503a82017-08-15 20:34:35 +01005188 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
5189 }
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005190
5191 /* mark destination R0 register as readable, since it contains
Edward Creedc503a82017-08-15 20:34:35 +01005192 * the value fetched from the packet.
5193 * Already marked as written above.
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005194 */
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005195 mark_reg_unknown(env, regs, BPF_REG_0);
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005196 return 0;
5197}
5198
Alexei Starovoitov390ee7e2017-10-02 22:50:23 -07005199static int check_return_code(struct bpf_verifier_env *env)
5200{
5201 struct bpf_reg_state *reg;
5202 struct tnum range = tnum_range(0, 1);
5203
5204 switch (env->prog->type) {
5205 case BPF_PROG_TYPE_CGROUP_SKB:
5206 case BPF_PROG_TYPE_CGROUP_SOCK:
Andrey Ignatov4fbac772018-03-30 15:08:02 -07005207 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
Alexei Starovoitov390ee7e2017-10-02 22:50:23 -07005208 case BPF_PROG_TYPE_SOCK_OPS:
Roman Gushchinebc614f2017-11-05 08:15:32 -05005209 case BPF_PROG_TYPE_CGROUP_DEVICE:
Alexei Starovoitov390ee7e2017-10-02 22:50:23 -07005210 break;
5211 default:
5212 return 0;
5213 }
5214
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07005215 reg = cur_regs(env) + BPF_REG_0;
Alexei Starovoitov390ee7e2017-10-02 22:50:23 -07005216 if (reg->type != SCALAR_VALUE) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005217 verbose(env, "At program exit the register R0 is not a known value (%s)\n",
Alexei Starovoitov390ee7e2017-10-02 22:50:23 -07005218 reg_type_str[reg->type]);
5219 return -EINVAL;
5220 }
5221
5222 if (!tnum_in(range, reg->var_off)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005223 verbose(env, "At program exit the register R0 ");
Alexei Starovoitov390ee7e2017-10-02 22:50:23 -07005224 if (!tnum_is_unknown(reg->var_off)) {
5225 char tn_buf[48];
5226
5227 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005228 verbose(env, "has value %s", tn_buf);
Alexei Starovoitov390ee7e2017-10-02 22:50:23 -07005229 } else {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005230 verbose(env, "has unknown scalar value");
Alexei Starovoitov390ee7e2017-10-02 22:50:23 -07005231 }
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005232 verbose(env, " should have been 0 or 1\n");
Alexei Starovoitov390ee7e2017-10-02 22:50:23 -07005233 return -EINVAL;
5234 }
5235 return 0;
5236}
5237
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005238/* non-recursive DFS pseudo code
5239 * 1 procedure DFS-iterative(G,v):
5240 * 2 label v as discovered
5241 * 3 let S be a stack
5242 * 4 S.push(v)
5243 * 5 while S is not empty
5244 * 6 t <- S.pop()
5245 * 7 if t is what we're looking for:
5246 * 8 return t
5247 * 9 for all edges e in G.adjacentEdges(t) do
5248 * 10 if edge e is already labelled
5249 * 11 continue with the next edge
5250 * 12 w <- G.adjacentVertex(t,e)
5251 * 13 if vertex w is not discovered and not explored
5252 * 14 label e as tree-edge
5253 * 15 label w as discovered
5254 * 16 S.push(w)
5255 * 17 continue at 5
5256 * 18 else if vertex w is discovered
5257 * 19 label e as back-edge
5258 * 20 else
5259 * 21 // vertex w is explored
5260 * 22 label e as forward- or cross-edge
5261 * 23 label t as explored
5262 * 24 S.pop()
5263 *
5264 * convention:
5265 * 0x10 - discovered
5266 * 0x11 - discovered and fall-through edge labelled
5267 * 0x12 - discovered and fall-through and branch edges labelled
5268 * 0x20 - explored
5269 */
5270
5271enum {
5272 DISCOVERED = 0x10,
5273 EXPLORED = 0x20,
5274 FALLTHROUGH = 1,
5275 BRANCH = 2,
5276};
5277
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01005278#define STATE_LIST_MARK ((struct bpf_verifier_state_list *) -1L)
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07005279
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005280static int *insn_stack; /* stack of insns to process */
5281static int cur_stack; /* current stack index */
5282static int *insn_state;
5283
5284/* t, w, e - match pseudo-code above:
5285 * t - index of current instruction
5286 * w - next instruction
5287 * e - edge
5288 */
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01005289static int push_insn(int t, int w, int e, struct bpf_verifier_env *env)
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005290{
5291 if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH))
5292 return 0;
5293
5294 if (e == BRANCH && insn_state[t] >= (DISCOVERED | BRANCH))
5295 return 0;
5296
5297 if (w < 0 || w >= env->prog->len) {
Martin KaFai Laud9762e82018-12-13 10:41:48 -08005298 verbose_linfo(env, t, "%d: ", t);
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005299 verbose(env, "jump out of range from insn %d to %d\n", t, w);
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005300 return -EINVAL;
5301 }
5302
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07005303 if (e == BRANCH)
5304 /* mark branch target for state pruning */
5305 env->explored_states[w] = STATE_LIST_MARK;
5306
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005307 if (insn_state[w] == 0) {
5308 /* tree-edge */
5309 insn_state[t] = DISCOVERED | e;
5310 insn_state[w] = DISCOVERED;
5311 if (cur_stack >= env->prog->len)
5312 return -E2BIG;
5313 insn_stack[cur_stack++] = w;
5314 return 1;
5315 } else if ((insn_state[w] & 0xF0) == DISCOVERED) {
Martin KaFai Laud9762e82018-12-13 10:41:48 -08005316 verbose_linfo(env, t, "%d: ", t);
5317 verbose_linfo(env, w, "%d: ", w);
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005318 verbose(env, "back-edge from insn %d to %d\n", t, w);
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005319 return -EINVAL;
5320 } else if (insn_state[w] == EXPLORED) {
5321 /* forward- or cross-edge */
5322 insn_state[t] = DISCOVERED | e;
5323 } else {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005324 verbose(env, "insn state internal bug\n");
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005325 return -EFAULT;
5326 }
5327 return 0;
5328}
5329
5330/* non-recursive depth-first-search to detect loops in BPF program
5331 * loop == back-edge in directed graph
5332 */
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01005333static int check_cfg(struct bpf_verifier_env *env)
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005334{
5335 struct bpf_insn *insns = env->prog->insnsi;
5336 int insn_cnt = env->prog->len;
5337 int ret = 0;
5338 int i, t;
5339
Alexei Starovoitov71dde682019-04-01 21:27:43 -07005340 insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005341 if (!insn_state)
5342 return -ENOMEM;
5343
Alexei Starovoitov71dde682019-04-01 21:27:43 -07005344 insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005345 if (!insn_stack) {
Alexei Starovoitov71dde682019-04-01 21:27:43 -07005346 kvfree(insn_state);
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005347 return -ENOMEM;
5348 }
5349
5350 insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */
5351 insn_stack[0] = 0; /* 0 is the first instruction */
5352 cur_stack = 1;
5353
5354peek_stack:
5355 if (cur_stack == 0)
5356 goto check_state;
5357 t = insn_stack[cur_stack - 1];
5358
Jiong Wang092ed092019-01-26 12:26:01 -05005359 if (BPF_CLASS(insns[t].code) == BPF_JMP ||
5360 BPF_CLASS(insns[t].code) == BPF_JMP32) {
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005361 u8 opcode = BPF_OP(insns[t].code);
5362
5363 if (opcode == BPF_EXIT) {
5364 goto mark_explored;
5365 } else if (opcode == BPF_CALL) {
5366 ret = push_insn(t, t + 1, FALLTHROUGH, env);
5367 if (ret == 1)
5368 goto peek_stack;
5369 else if (ret < 0)
5370 goto err_free;
Daniel Borkmann07016152016-04-05 22:33:17 +02005371 if (t + 1 < insn_cnt)
5372 env->explored_states[t + 1] = STATE_LIST_MARK;
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08005373 if (insns[t].src_reg == BPF_PSEUDO_CALL) {
5374 env->explored_states[t] = STATE_LIST_MARK;
5375 ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env);
5376 if (ret == 1)
5377 goto peek_stack;
5378 else if (ret < 0)
5379 goto err_free;
5380 }
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005381 } else if (opcode == BPF_JA) {
5382 if (BPF_SRC(insns[t].code) != BPF_K) {
5383 ret = -EINVAL;
5384 goto err_free;
5385 }
5386 /* unconditional jump with single edge */
5387 ret = push_insn(t, t + insns[t].off + 1,
5388 FALLTHROUGH, env);
5389 if (ret == 1)
5390 goto peek_stack;
5391 else if (ret < 0)
5392 goto err_free;
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07005393 /* tell verifier to check for equivalent states
5394 * after every call and jump
5395 */
Alexei Starovoitovc3de6312015-04-14 15:57:13 -07005396 if (t + 1 < insn_cnt)
5397 env->explored_states[t + 1] = STATE_LIST_MARK;
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005398 } else {
5399 /* conditional jump with two edges */
Daniel Borkmann3c2ce602017-05-18 03:00:06 +02005400 env->explored_states[t] = STATE_LIST_MARK;
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005401 ret = push_insn(t, t + 1, FALLTHROUGH, env);
5402 if (ret == 1)
5403 goto peek_stack;
5404 else if (ret < 0)
5405 goto err_free;
5406
5407 ret = push_insn(t, t + insns[t].off + 1, BRANCH, env);
5408 if (ret == 1)
5409 goto peek_stack;
5410 else if (ret < 0)
5411 goto err_free;
5412 }
5413 } else {
5414 /* all other non-branch instructions with single
5415 * fall-through edge
5416 */
5417 ret = push_insn(t, t + 1, FALLTHROUGH, env);
5418 if (ret == 1)
5419 goto peek_stack;
5420 else if (ret < 0)
5421 goto err_free;
5422 }
5423
5424mark_explored:
5425 insn_state[t] = EXPLORED;
5426 if (cur_stack-- <= 0) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005427 verbose(env, "pop stack internal bug\n");
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005428 ret = -EFAULT;
5429 goto err_free;
5430 }
5431 goto peek_stack;
5432
5433check_state:
5434 for (i = 0; i < insn_cnt; i++) {
5435 if (insn_state[i] != EXPLORED) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005436 verbose(env, "unreachable insn %d\n", i);
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005437 ret = -EINVAL;
5438 goto err_free;
5439 }
5440 }
5441 ret = 0; /* cfg looks good */
5442
5443err_free:
Alexei Starovoitov71dde682019-04-01 21:27:43 -07005444 kvfree(insn_state);
5445 kvfree(insn_stack);
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005446 return ret;
5447}
5448
Yonghong Song838e9692018-11-19 15:29:11 -08005449/* The minimum supported BTF func info size */
5450#define MIN_BPF_FUNCINFO_SIZE 8
5451#define MAX_FUNCINFO_REC_SIZE 252
5452
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005453static int check_btf_func(struct bpf_verifier_env *env,
5454 const union bpf_attr *attr,
5455 union bpf_attr __user *uattr)
Yonghong Song838e9692018-11-19 15:29:11 -08005456{
Peter Oskolkovd0b28182019-01-16 10:43:01 -08005457 u32 i, nfuncs, urec_size, min_size;
Yonghong Song838e9692018-11-19 15:29:11 -08005458 u32 krec_size = sizeof(struct bpf_func_info);
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005459 struct bpf_func_info *krecord;
Yonghong Song838e9692018-11-19 15:29:11 -08005460 const struct btf_type *type;
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005461 struct bpf_prog *prog;
5462 const struct btf *btf;
Yonghong Song838e9692018-11-19 15:29:11 -08005463 void __user *urecord;
Peter Oskolkovd0b28182019-01-16 10:43:01 -08005464 u32 prev_offset = 0;
Yonghong Song838e9692018-11-19 15:29:11 -08005465 int ret = 0;
5466
5467 nfuncs = attr->func_info_cnt;
5468 if (!nfuncs)
5469 return 0;
5470
5471 if (nfuncs != env->subprog_cnt) {
5472 verbose(env, "number of funcs in func_info doesn't match number of subprogs\n");
5473 return -EINVAL;
5474 }
5475
5476 urec_size = attr->func_info_rec_size;
5477 if (urec_size < MIN_BPF_FUNCINFO_SIZE ||
5478 urec_size > MAX_FUNCINFO_REC_SIZE ||
5479 urec_size % sizeof(u32)) {
5480 verbose(env, "invalid func info rec size %u\n", urec_size);
5481 return -EINVAL;
5482 }
5483
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005484 prog = env->prog;
5485 btf = prog->aux->btf;
Yonghong Song838e9692018-11-19 15:29:11 -08005486
5487 urecord = u64_to_user_ptr(attr->func_info);
5488 min_size = min_t(u32, krec_size, urec_size);
5489
Yonghong Songba64e7d2018-11-24 23:20:44 -08005490 krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN);
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005491 if (!krecord)
5492 return -ENOMEM;
Yonghong Songba64e7d2018-11-24 23:20:44 -08005493
Yonghong Song838e9692018-11-19 15:29:11 -08005494 for (i = 0; i < nfuncs; i++) {
5495 ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size);
5496 if (ret) {
5497 if (ret == -E2BIG) {
5498 verbose(env, "nonzero tailing record in func info");
5499 /* set the size kernel expects so loader can zero
5500 * out the rest of the record.
5501 */
5502 if (put_user(min_size, &uattr->func_info_rec_size))
5503 ret = -EFAULT;
5504 }
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005505 goto err_free;
Yonghong Song838e9692018-11-19 15:29:11 -08005506 }
5507
Yonghong Songba64e7d2018-11-24 23:20:44 -08005508 if (copy_from_user(&krecord[i], urecord, min_size)) {
Yonghong Song838e9692018-11-19 15:29:11 -08005509 ret = -EFAULT;
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005510 goto err_free;
Yonghong Song838e9692018-11-19 15:29:11 -08005511 }
5512
Martin KaFai Laud30d42e2018-12-05 17:35:44 -08005513 /* check insn_off */
Yonghong Song838e9692018-11-19 15:29:11 -08005514 if (i == 0) {
Martin KaFai Laud30d42e2018-12-05 17:35:44 -08005515 if (krecord[i].insn_off) {
Yonghong Song838e9692018-11-19 15:29:11 -08005516 verbose(env,
Martin KaFai Laud30d42e2018-12-05 17:35:44 -08005517 "nonzero insn_off %u for the first func info record",
5518 krecord[i].insn_off);
Yonghong Song838e9692018-11-19 15:29:11 -08005519 ret = -EINVAL;
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005520 goto err_free;
Yonghong Song838e9692018-11-19 15:29:11 -08005521 }
Martin KaFai Laud30d42e2018-12-05 17:35:44 -08005522 } else if (krecord[i].insn_off <= prev_offset) {
Yonghong Song838e9692018-11-19 15:29:11 -08005523 verbose(env,
5524 "same or smaller insn offset (%u) than previous func info record (%u)",
Martin KaFai Laud30d42e2018-12-05 17:35:44 -08005525 krecord[i].insn_off, prev_offset);
Yonghong Song838e9692018-11-19 15:29:11 -08005526 ret = -EINVAL;
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005527 goto err_free;
Yonghong Song838e9692018-11-19 15:29:11 -08005528 }
5529
Martin KaFai Laud30d42e2018-12-05 17:35:44 -08005530 if (env->subprog_info[i].start != krecord[i].insn_off) {
Yonghong Song838e9692018-11-19 15:29:11 -08005531 verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");
5532 ret = -EINVAL;
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005533 goto err_free;
Yonghong Song838e9692018-11-19 15:29:11 -08005534 }
5535
5536 /* check type_id */
Yonghong Songba64e7d2018-11-24 23:20:44 -08005537 type = btf_type_by_id(btf, krecord[i].type_id);
Yonghong Song838e9692018-11-19 15:29:11 -08005538 if (!type || BTF_INFO_KIND(type->info) != BTF_KIND_FUNC) {
5539 verbose(env, "invalid type id %d in func info",
Yonghong Songba64e7d2018-11-24 23:20:44 -08005540 krecord[i].type_id);
Yonghong Song838e9692018-11-19 15:29:11 -08005541 ret = -EINVAL;
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005542 goto err_free;
Yonghong Song838e9692018-11-19 15:29:11 -08005543 }
5544
Martin KaFai Laud30d42e2018-12-05 17:35:44 -08005545 prev_offset = krecord[i].insn_off;
Yonghong Song838e9692018-11-19 15:29:11 -08005546 urecord += urec_size;
5547 }
5548
Yonghong Songba64e7d2018-11-24 23:20:44 -08005549 prog->aux->func_info = krecord;
5550 prog->aux->func_info_cnt = nfuncs;
Yonghong Song838e9692018-11-19 15:29:11 -08005551 return 0;
5552
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005553err_free:
Yonghong Songba64e7d2018-11-24 23:20:44 -08005554 kvfree(krecord);
Yonghong Song838e9692018-11-19 15:29:11 -08005555 return ret;
5556}
5557
Yonghong Songba64e7d2018-11-24 23:20:44 -08005558static void adjust_btf_func(struct bpf_verifier_env *env)
5559{
5560 int i;
5561
5562 if (!env->prog->aux->func_info)
5563 return;
5564
5565 for (i = 0; i < env->subprog_cnt; i++)
Martin KaFai Laud30d42e2018-12-05 17:35:44 -08005566 env->prog->aux->func_info[i].insn_off = env->subprog_info[i].start;
Yonghong Songba64e7d2018-11-24 23:20:44 -08005567}
5568
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005569#define MIN_BPF_LINEINFO_SIZE (offsetof(struct bpf_line_info, line_col) + \
5570 sizeof(((struct bpf_line_info *)(0))->line_col))
5571#define MAX_LINEINFO_REC_SIZE MAX_FUNCINFO_REC_SIZE
5572
5573static int check_btf_line(struct bpf_verifier_env *env,
5574 const union bpf_attr *attr,
5575 union bpf_attr __user *uattr)
5576{
5577 u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0;
5578 struct bpf_subprog_info *sub;
5579 struct bpf_line_info *linfo;
5580 struct bpf_prog *prog;
5581 const struct btf *btf;
5582 void __user *ulinfo;
5583 int err;
5584
5585 nr_linfo = attr->line_info_cnt;
5586 if (!nr_linfo)
5587 return 0;
5588
5589 rec_size = attr->line_info_rec_size;
5590 if (rec_size < MIN_BPF_LINEINFO_SIZE ||
5591 rec_size > MAX_LINEINFO_REC_SIZE ||
5592 rec_size & (sizeof(u32) - 1))
5593 return -EINVAL;
5594
5595 /* Need to zero it in case the userspace may
5596 * pass in a smaller bpf_line_info object.
5597 */
5598 linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info),
5599 GFP_KERNEL | __GFP_NOWARN);
5600 if (!linfo)
5601 return -ENOMEM;
5602
5603 prog = env->prog;
5604 btf = prog->aux->btf;
5605
5606 s = 0;
5607 sub = env->subprog_info;
5608 ulinfo = u64_to_user_ptr(attr->line_info);
5609 expected_size = sizeof(struct bpf_line_info);
5610 ncopy = min_t(u32, expected_size, rec_size);
5611 for (i = 0; i < nr_linfo; i++) {
5612 err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size);
5613 if (err) {
5614 if (err == -E2BIG) {
5615 verbose(env, "nonzero tailing record in line_info");
5616 if (put_user(expected_size,
5617 &uattr->line_info_rec_size))
5618 err = -EFAULT;
5619 }
5620 goto err_free;
5621 }
5622
5623 if (copy_from_user(&linfo[i], ulinfo, ncopy)) {
5624 err = -EFAULT;
5625 goto err_free;
5626 }
5627
5628 /*
5629 * Check insn_off to ensure
5630 * 1) strictly increasing AND
5631 * 2) bounded by prog->len
5632 *
5633 * The linfo[0].insn_off == 0 check logically falls into
5634 * the later "missing bpf_line_info for func..." case
5635 * because the first linfo[0].insn_off must be the
5636 * first sub also and the first sub must have
5637 * subprog_info[0].start == 0.
5638 */
5639 if ((i && linfo[i].insn_off <= prev_offset) ||
5640 linfo[i].insn_off >= prog->len) {
5641 verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n",
5642 i, linfo[i].insn_off, prev_offset,
5643 prog->len);
5644 err = -EINVAL;
5645 goto err_free;
5646 }
5647
Martin KaFai Laufdbaa0b2018-12-19 13:01:01 -08005648 if (!prog->insnsi[linfo[i].insn_off].code) {
5649 verbose(env,
5650 "Invalid insn code at line_info[%u].insn_off\n",
5651 i);
5652 err = -EINVAL;
5653 goto err_free;
5654 }
5655
Martin KaFai Lau23127b32018-12-13 10:41:46 -08005656 if (!btf_name_by_offset(btf, linfo[i].line_off) ||
5657 !btf_name_by_offset(btf, linfo[i].file_name_off)) {
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005658 verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i);
5659 err = -EINVAL;
5660 goto err_free;
5661 }
5662
5663 if (s != env->subprog_cnt) {
5664 if (linfo[i].insn_off == sub[s].start) {
5665 sub[s].linfo_idx = i;
5666 s++;
5667 } else if (sub[s].start < linfo[i].insn_off) {
5668 verbose(env, "missing bpf_line_info for func#%u\n", s);
5669 err = -EINVAL;
5670 goto err_free;
5671 }
5672 }
5673
5674 prev_offset = linfo[i].insn_off;
5675 ulinfo += rec_size;
5676 }
5677
5678 if (s != env->subprog_cnt) {
5679 verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n",
5680 env->subprog_cnt - s, s);
5681 err = -EINVAL;
5682 goto err_free;
5683 }
5684
5685 prog->aux->linfo = linfo;
5686 prog->aux->nr_linfo = nr_linfo;
5687
5688 return 0;
5689
5690err_free:
5691 kvfree(linfo);
5692 return err;
5693}
5694
5695static int check_btf_info(struct bpf_verifier_env *env,
5696 const union bpf_attr *attr,
5697 union bpf_attr __user *uattr)
5698{
5699 struct btf *btf;
5700 int err;
5701
5702 if (!attr->func_info_cnt && !attr->line_info_cnt)
5703 return 0;
5704
5705 btf = btf_get_by_fd(attr->prog_btf_fd);
5706 if (IS_ERR(btf))
5707 return PTR_ERR(btf);
5708 env->prog->aux->btf = btf;
5709
5710 err = check_btf_func(env, attr, uattr);
5711 if (err)
5712 return err;
5713
5714 err = check_btf_line(env, attr, uattr);
5715 if (err)
5716 return err;
5717
5718 return 0;
Alexei Starovoitov0246e642014-09-26 00:17:04 -07005719}
5720
Edward Creef1174f72017-08-07 15:26:19 +01005721/* check %cur's range satisfies %old's */
5722static bool range_within(struct bpf_reg_state *old,
5723 struct bpf_reg_state *cur)
Alexei Starovoitov969bf052016-05-05 19:49:10 -07005724{
Edward Creeb03c9f92017-08-07 15:26:36 +01005725 return old->umin_value <= cur->umin_value &&
5726 old->umax_value >= cur->umax_value &&
5727 old->smin_value <= cur->smin_value &&
5728 old->smax_value >= cur->smax_value;
Edward Creef1174f72017-08-07 15:26:19 +01005729}
5730
5731/* Maximum number of register states that can exist at once */
5732#define ID_MAP_SIZE (MAX_BPF_REG + MAX_BPF_STACK / BPF_REG_SIZE)
5733struct idpair {
5734 u32 old;
5735 u32 cur;
5736};
5737
5738/* If in the old state two registers had the same id, then they need to have
5739 * the same id in the new state as well. But that id could be different from
5740 * the old state, so we need to track the mapping from old to new ids.
5741 * Once we have seen that, say, a reg with old id 5 had new id 9, any subsequent
5742 * regs with old id 5 must also have new id 9 for the new state to be safe. But
5743 * regs with a different old id could still have new id 9, we don't care about
5744 * that.
5745 * So we look through our idmap to see if this old id has been seen before. If
5746 * so, we require the new id to match; otherwise, we add the id pair to the map.
5747 */
5748static bool check_ids(u32 old_id, u32 cur_id, struct idpair *idmap)
5749{
5750 unsigned int i;
5751
5752 for (i = 0; i < ID_MAP_SIZE; i++) {
5753 if (!idmap[i].old) {
5754 /* Reached an empty slot; haven't seen this id before */
5755 idmap[i].old = old_id;
5756 idmap[i].cur = cur_id;
5757 return true;
5758 }
5759 if (idmap[i].old == old_id)
5760 return idmap[i].cur == cur_id;
5761 }
5762 /* We ran out of idmap slots, which should be impossible */
5763 WARN_ON_ONCE(1);
5764 return false;
5765}
5766
Alexei Starovoitov9242b5f2018-12-13 11:42:34 -08005767static void clean_func_state(struct bpf_verifier_env *env,
5768 struct bpf_func_state *st)
5769{
5770 enum bpf_reg_liveness live;
5771 int i, j;
5772
5773 for (i = 0; i < BPF_REG_FP; i++) {
5774 live = st->regs[i].live;
5775 /* liveness must not touch this register anymore */
5776 st->regs[i].live |= REG_LIVE_DONE;
5777 if (!(live & REG_LIVE_READ))
5778 /* since the register is unused, clear its state
5779 * to make further comparison simpler
5780 */
5781 __mark_reg_not_init(&st->regs[i]);
5782 }
5783
5784 for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) {
5785 live = st->stack[i].spilled_ptr.live;
5786 /* liveness must not touch this stack slot anymore */
5787 st->stack[i].spilled_ptr.live |= REG_LIVE_DONE;
5788 if (!(live & REG_LIVE_READ)) {
5789 __mark_reg_not_init(&st->stack[i].spilled_ptr);
5790 for (j = 0; j < BPF_REG_SIZE; j++)
5791 st->stack[i].slot_type[j] = STACK_INVALID;
5792 }
5793 }
5794}
5795
5796static void clean_verifier_state(struct bpf_verifier_env *env,
5797 struct bpf_verifier_state *st)
5798{
5799 int i;
5800
5801 if (st->frame[0]->regs[0].live & REG_LIVE_DONE)
5802 /* all regs in this state in all frames were already marked */
5803 return;
5804
5805 for (i = 0; i <= st->curframe; i++)
5806 clean_func_state(env, st->frame[i]);
5807}
5808
5809/* the parentage chains form a tree.
5810 * the verifier states are added to state lists at given insn and
5811 * pushed into state stack for future exploration.
5812 * when the verifier reaches bpf_exit insn some of the verifer states
5813 * stored in the state lists have their final liveness state already,
5814 * but a lot of states will get revised from liveness point of view when
5815 * the verifier explores other branches.
5816 * Example:
5817 * 1: r0 = 1
5818 * 2: if r1 == 100 goto pc+1
5819 * 3: r0 = 2
5820 * 4: exit
5821 * when the verifier reaches exit insn the register r0 in the state list of
5822 * insn 2 will be seen as !REG_LIVE_READ. Then the verifier pops the other_branch
5823 * of insn 2 and goes exploring further. At the insn 4 it will walk the
5824 * parentage chain from insn 4 into insn 2 and will mark r0 as REG_LIVE_READ.
5825 *
5826 * Since the verifier pushes the branch states as it sees them while exploring
5827 * the program the condition of walking the branch instruction for the second
5828 * time means that all states below this branch were already explored and
5829 * their final liveness markes are already propagated.
5830 * Hence when the verifier completes the search of state list in is_state_visited()
5831 * we can call this clean_live_states() function to mark all liveness states
5832 * as REG_LIVE_DONE to indicate that 'parent' pointers of 'struct bpf_reg_state'
5833 * will not be used.
5834 * This function also clears the registers and stack for states that !READ
5835 * to simplify state merging.
5836 *
5837 * Important note here that walking the same branch instruction in the callee
5838 * doesn't meant that the states are DONE. The verifier has to compare
5839 * the callsites
5840 */
5841static void clean_live_states(struct bpf_verifier_env *env, int insn,
5842 struct bpf_verifier_state *cur)
5843{
5844 struct bpf_verifier_state_list *sl;
5845 int i;
5846
5847 sl = env->explored_states[insn];
5848 if (!sl)
5849 return;
5850
5851 while (sl != STATE_LIST_MARK) {
5852 if (sl->state.curframe != cur->curframe)
5853 goto next;
5854 for (i = 0; i <= cur->curframe; i++)
5855 if (sl->state.frame[i]->callsite != cur->frame[i]->callsite)
5856 goto next;
5857 clean_verifier_state(env, &sl->state);
5858next:
5859 sl = sl->next;
5860 }
5861}
5862
Edward Creef1174f72017-08-07 15:26:19 +01005863/* Returns true if (rold safe implies rcur safe) */
Edward Cree1b688a12017-08-23 15:10:50 +01005864static bool regsafe(struct bpf_reg_state *rold, struct bpf_reg_state *rcur,
5865 struct idpair *idmap)
Edward Creef1174f72017-08-07 15:26:19 +01005866{
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08005867 bool equal;
5868
Edward Creedc503a82017-08-15 20:34:35 +01005869 if (!(rold->live & REG_LIVE_READ))
5870 /* explored state didn't use this */
5871 return true;
5872
Edward Cree679c7822018-08-22 20:02:19 +01005873 equal = memcmp(rold, rcur, offsetof(struct bpf_reg_state, parent)) == 0;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08005874
5875 if (rold->type == PTR_TO_STACK)
5876 /* two stack pointers are equal only if they're pointing to
5877 * the same stack frame, since fp-8 in foo != fp-8 in bar
5878 */
5879 return equal && rold->frameno == rcur->frameno;
5880
5881 if (equal)
Edward Creef1174f72017-08-07 15:26:19 +01005882 return true;
5883
5884 if (rold->type == NOT_INIT)
5885 /* explored state can't have used this */
5886 return true;
5887 if (rcur->type == NOT_INIT)
Alexei Starovoitov969bf052016-05-05 19:49:10 -07005888 return false;
Edward Creef1174f72017-08-07 15:26:19 +01005889 switch (rold->type) {
5890 case SCALAR_VALUE:
5891 if (rcur->type == SCALAR_VALUE) {
5892 /* new val must satisfy old val knowledge */
5893 return range_within(rold, rcur) &&
5894 tnum_in(rold->var_off, rcur->var_off);
5895 } else {
Jann Horn179d1c52017-12-18 20:11:59 -08005896 /* We're trying to use a pointer in place of a scalar.
5897 * Even if the scalar was unbounded, this could lead to
5898 * pointer leaks because scalars are allowed to leak
5899 * while pointers are not. We could make this safe in
5900 * special cases if root is calling us, but it's
5901 * probably not worth the hassle.
Edward Creef1174f72017-08-07 15:26:19 +01005902 */
Jann Horn179d1c52017-12-18 20:11:59 -08005903 return false;
Edward Creef1174f72017-08-07 15:26:19 +01005904 }
5905 case PTR_TO_MAP_VALUE:
Edward Cree1b688a12017-08-23 15:10:50 +01005906 /* If the new min/max/var_off satisfy the old ones and
5907 * everything else matches, we are OK.
Alexei Starovoitovd83525c2019-01-31 15:40:04 -08005908 * 'id' is not compared, since it's only used for maps with
5909 * bpf_spin_lock inside map element and in such cases if
5910 * the rest of the prog is valid for one map element then
5911 * it's valid for all map elements regardless of the key
5912 * used in bpf_map_lookup()
Edward Cree1b688a12017-08-23 15:10:50 +01005913 */
5914 return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 &&
5915 range_within(rold, rcur) &&
5916 tnum_in(rold->var_off, rcur->var_off);
Edward Creef1174f72017-08-07 15:26:19 +01005917 case PTR_TO_MAP_VALUE_OR_NULL:
5918 /* a PTR_TO_MAP_VALUE could be safe to use as a
5919 * PTR_TO_MAP_VALUE_OR_NULL into the same map.
5920 * However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL-
5921 * checked, doing so could have affected others with the same
5922 * id, and we can't check for that because we lost the id when
5923 * we converted to a PTR_TO_MAP_VALUE.
5924 */
5925 if (rcur->type != PTR_TO_MAP_VALUE_OR_NULL)
5926 return false;
5927 if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)))
5928 return false;
5929 /* Check our ids match any regs they're supposed to */
5930 return check_ids(rold->id, rcur->id, idmap);
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02005931 case PTR_TO_PACKET_META:
Edward Creef1174f72017-08-07 15:26:19 +01005932 case PTR_TO_PACKET:
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02005933 if (rcur->type != rold->type)
Edward Creef1174f72017-08-07 15:26:19 +01005934 return false;
5935 /* We must have at least as much range as the old ptr
5936 * did, so that any accesses which were safe before are
5937 * still safe. This is true even if old range < old off,
5938 * since someone could have accessed through (ptr - k), or
5939 * even done ptr -= k in a register, to get a safe access.
5940 */
5941 if (rold->range > rcur->range)
5942 return false;
5943 /* If the offsets don't match, we can't trust our alignment;
5944 * nor can we be sure that we won't fall out of range.
5945 */
5946 if (rold->off != rcur->off)
5947 return false;
5948 /* id relations must be preserved */
5949 if (rold->id && !check_ids(rold->id, rcur->id, idmap))
5950 return false;
5951 /* new val must satisfy old val knowledge */
5952 return range_within(rold, rcur) &&
5953 tnum_in(rold->var_off, rcur->var_off);
5954 case PTR_TO_CTX:
5955 case CONST_PTR_TO_MAP:
Edward Creef1174f72017-08-07 15:26:19 +01005956 case PTR_TO_PACKET_END:
Petar Penkovd58e4682018-09-14 07:46:18 -07005957 case PTR_TO_FLOW_KEYS:
Joe Stringerc64b7982018-10-02 13:35:33 -07005958 case PTR_TO_SOCKET:
5959 case PTR_TO_SOCKET_OR_NULL:
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08005960 case PTR_TO_SOCK_COMMON:
5961 case PTR_TO_SOCK_COMMON_OR_NULL:
Martin KaFai Lau655a51e2019-02-09 23:22:24 -08005962 case PTR_TO_TCP_SOCK:
5963 case PTR_TO_TCP_SOCK_OR_NULL:
Edward Creef1174f72017-08-07 15:26:19 +01005964 /* Only valid matches are exact, which memcmp() above
5965 * would have accepted
5966 */
5967 default:
5968 /* Don't know what's going on, just say it's not safe */
5969 return false;
5970 }
Alexei Starovoitov969bf052016-05-05 19:49:10 -07005971
Edward Creef1174f72017-08-07 15:26:19 +01005972 /* Shouldn't get here; if we do, say it's not safe */
5973 WARN_ON_ONCE(1);
Alexei Starovoitov969bf052016-05-05 19:49:10 -07005974 return false;
5975}
5976
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08005977static bool stacksafe(struct bpf_func_state *old,
5978 struct bpf_func_state *cur,
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07005979 struct idpair *idmap)
5980{
5981 int i, spi;
5982
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07005983 /* walk slots of the explored stack and ignore any additional
5984 * slots in the current stack, since explored(safe) state
5985 * didn't use them
5986 */
5987 for (i = 0; i < old->allocated_stack; i++) {
5988 spi = i / BPF_REG_SIZE;
5989
Alexei Starovoitovb2339202018-12-13 11:42:31 -08005990 if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ)) {
5991 i += BPF_REG_SIZE - 1;
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08005992 /* explored state didn't use this */
Gianluca Borellofd05e572017-12-23 10:09:55 +00005993 continue;
Alexei Starovoitovb2339202018-12-13 11:42:31 -08005994 }
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08005995
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07005996 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID)
5997 continue;
Alexei Starovoitov19e2dbb2018-12-13 11:42:33 -08005998
5999 /* explored stack has more populated slots than current stack
6000 * and these slots were used
6001 */
6002 if (i >= cur->allocated_stack)
6003 return false;
6004
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08006005 /* if old state was safe with misc data in the stack
6006 * it will be safe with zero-initialized stack.
6007 * The opposite is not true
6008 */
6009 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC &&
6010 cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO)
6011 continue;
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07006012 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] !=
6013 cur->stack[spi].slot_type[i % BPF_REG_SIZE])
6014 /* Ex: old explored (safe) state has STACK_SPILL in
6015 * this stack slot, but current has has STACK_MISC ->
6016 * this verifier states are not equivalent,
6017 * return false to continue verification of this path
6018 */
6019 return false;
6020 if (i % BPF_REG_SIZE)
6021 continue;
6022 if (old->stack[spi].slot_type[0] != STACK_SPILL)
6023 continue;
6024 if (!regsafe(&old->stack[spi].spilled_ptr,
6025 &cur->stack[spi].spilled_ptr,
6026 idmap))
6027 /* when explored and current stack slot are both storing
6028 * spilled registers, check that stored pointers types
6029 * are the same as well.
6030 * Ex: explored safe path could have stored
6031 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -8}
6032 * but current path has stored:
6033 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -16}
6034 * such verifier states are not equivalent.
6035 * return false to continue verification of this path
6036 */
6037 return false;
6038 }
6039 return true;
6040}
6041
Joe Stringerfd978bf72018-10-02 13:35:35 -07006042static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur)
6043{
6044 if (old->acquired_refs != cur->acquired_refs)
6045 return false;
6046 return !memcmp(old->refs, cur->refs,
6047 sizeof(*old->refs) * old->acquired_refs);
6048}
6049
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006050/* compare two verifier states
6051 *
6052 * all states stored in state_list are known to be valid, since
6053 * verifier reached 'bpf_exit' instruction through them
6054 *
6055 * this function is called when verifier exploring different branches of
6056 * execution popped from the state stack. If it sees an old state that has
6057 * more strict register state and more strict stack state then this execution
6058 * branch doesn't need to be explored further, since verifier already
6059 * concluded that more strict state leads to valid finish.
6060 *
6061 * Therefore two states are equivalent if register state is more conservative
6062 * and explored stack state is more conservative than the current one.
6063 * Example:
6064 * explored current
6065 * (slot1=INV slot2=MISC) == (slot1=MISC slot2=MISC)
6066 * (slot1=MISC slot2=MISC) != (slot1=INV slot2=MISC)
6067 *
6068 * In other words if current stack state (one being explored) has more
6069 * valid slots than old one that already passed validation, it means
6070 * the verifier can stop exploring and conclude that current state is valid too
6071 *
6072 * Similarly with registers. If explored state has register type as invalid
6073 * whereas register type in current state is meaningful, it means that
6074 * the current state will reach 'bpf_exit' instruction safely
6075 */
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006076static bool func_states_equal(struct bpf_func_state *old,
6077 struct bpf_func_state *cur)
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006078{
Edward Creef1174f72017-08-07 15:26:19 +01006079 struct idpair *idmap;
6080 bool ret = false;
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006081 int i;
6082
Edward Creef1174f72017-08-07 15:26:19 +01006083 idmap = kcalloc(ID_MAP_SIZE, sizeof(struct idpair), GFP_KERNEL);
6084 /* If we failed to allocate the idmap, just say it's not safe */
6085 if (!idmap)
Alexei Starovoitov1a0dc1a2016-05-05 19:49:09 -07006086 return false;
Edward Creef1174f72017-08-07 15:26:19 +01006087
6088 for (i = 0; i < MAX_BPF_REG; i++) {
Edward Cree1b688a12017-08-23 15:10:50 +01006089 if (!regsafe(&old->regs[i], &cur->regs[i], idmap))
Edward Creef1174f72017-08-07 15:26:19 +01006090 goto out_free;
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006091 }
6092
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07006093 if (!stacksafe(old, cur, idmap))
6094 goto out_free;
Joe Stringerfd978bf72018-10-02 13:35:35 -07006095
6096 if (!refsafe(old, cur))
6097 goto out_free;
Edward Creef1174f72017-08-07 15:26:19 +01006098 ret = true;
6099out_free:
6100 kfree(idmap);
6101 return ret;
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006102}
6103
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006104static bool states_equal(struct bpf_verifier_env *env,
6105 struct bpf_verifier_state *old,
6106 struct bpf_verifier_state *cur)
Edward Creedc503a82017-08-15 20:34:35 +01006107{
Edward Creedc503a82017-08-15 20:34:35 +01006108 int i;
6109
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006110 if (old->curframe != cur->curframe)
6111 return false;
6112
Daniel Borkmann979d63d2019-01-03 00:58:34 +01006113 /* Verification state from speculative execution simulation
6114 * must never prune a non-speculative execution one.
6115 */
6116 if (old->speculative && !cur->speculative)
6117 return false;
6118
Alexei Starovoitovd83525c2019-01-31 15:40:04 -08006119 if (old->active_spin_lock != cur->active_spin_lock)
6120 return false;
6121
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006122 /* for states to be equal callsites have to be the same
6123 * and all frame states need to be equivalent
6124 */
6125 for (i = 0; i <= old->curframe; i++) {
6126 if (old->frame[i]->callsite != cur->frame[i]->callsite)
6127 return false;
6128 if (!func_states_equal(old->frame[i], cur->frame[i]))
6129 return false;
6130 }
6131 return true;
6132}
6133
6134/* A write screens off any subsequent reads; but write marks come from the
6135 * straight-line code between a state and its parent. When we arrive at an
6136 * equivalent state (jump target or such) we didn't arrive by the straight-line
6137 * code, so read marks in the state must propagate to the parent regardless
6138 * of the state's write marks. That's what 'parent == state->parent' comparison
Edward Cree679c7822018-08-22 20:02:19 +01006139 * in mark_reg_read() is for.
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006140 */
6141static int propagate_liveness(struct bpf_verifier_env *env,
6142 const struct bpf_verifier_state *vstate,
6143 struct bpf_verifier_state *vparent)
6144{
6145 int i, frame, err = 0;
6146 struct bpf_func_state *state, *parent;
6147
6148 if (vparent->curframe != vstate->curframe) {
6149 WARN(1, "propagate_live: parent frame %d current frame %d\n",
6150 vparent->curframe, vstate->curframe);
6151 return -EFAULT;
6152 }
Edward Creedc503a82017-08-15 20:34:35 +01006153 /* Propagate read liveness of registers... */
6154 BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG);
Jakub Kicinski83d16312019-03-21 14:34:36 -07006155 for (frame = 0; frame <= vstate->curframe; frame++) {
6156 /* We don't need to worry about FP liveness, it's read-only */
6157 for (i = frame < vstate->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) {
6158 if (vparent->frame[frame]->regs[i].live & REG_LIVE_READ)
6159 continue;
6160 if (vstate->frame[frame]->regs[i].live & REG_LIVE_READ) {
6161 err = mark_reg_read(env, &vstate->frame[frame]->regs[i],
6162 &vparent->frame[frame]->regs[i]);
6163 if (err)
6164 return err;
6165 }
Edward Creedc503a82017-08-15 20:34:35 +01006166 }
6167 }
Edward Creedc503a82017-08-15 20:34:35 +01006168
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006169 /* ... and stack slots */
6170 for (frame = 0; frame <= vstate->curframe; frame++) {
6171 state = vstate->frame[frame];
6172 parent = vparent->frame[frame];
6173 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE &&
6174 i < parent->allocated_stack / BPF_REG_SIZE; i++) {
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006175 if (parent->stack[i].spilled_ptr.live & REG_LIVE_READ)
6176 continue;
6177 if (state->stack[i].spilled_ptr.live & REG_LIVE_READ)
Edward Cree679c7822018-08-22 20:02:19 +01006178 mark_reg_read(env, &state->stack[i].spilled_ptr,
6179 &parent->stack[i].spilled_ptr);
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006180 }
Edward Creedc503a82017-08-15 20:34:35 +01006181 }
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006182 return err;
Edward Creedc503a82017-08-15 20:34:35 +01006183}
6184
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01006185static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006186{
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01006187 struct bpf_verifier_state_list *new_sl;
Alexei Starovoitov9f4686c2019-04-01 21:27:41 -07006188 struct bpf_verifier_state_list *sl, **pprev;
Edward Cree679c7822018-08-22 20:02:19 +01006189 struct bpf_verifier_state *cur = env->cur_state, *new;
Alexei Starovoitovceefbc92018-12-03 22:46:06 -08006190 int i, j, err, states_cnt = 0;
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006191
Alexei Starovoitov9f4686c2019-04-01 21:27:41 -07006192 pprev = &env->explored_states[insn_idx];
6193 sl = *pprev;
6194
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006195 if (!sl)
6196 /* this 'insn_idx' instruction wasn't marked, so we will not
6197 * be doing state search here
6198 */
6199 return 0;
6200
Alexei Starovoitov9242b5f2018-12-13 11:42:34 -08006201 clean_live_states(env, insn_idx, cur);
6202
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006203 while (sl != STATE_LIST_MARK) {
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07006204 if (states_equal(env, &sl->state, cur)) {
Alexei Starovoitov9f4686c2019-04-01 21:27:41 -07006205 sl->hit_cnt++;
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006206 /* reached equivalent register/stack state,
Edward Creedc503a82017-08-15 20:34:35 +01006207 * prune the search.
6208 * Registers read by the continuation are read by us.
Edward Cree8e9cd9c2017-08-23 15:11:21 +01006209 * If we have any write marks in env->cur_state, they
6210 * will prevent corresponding reads in the continuation
6211 * from reaching our parent (an explored_state). Our
6212 * own state will get the read marks recorded, but
6213 * they'll be immediately forgotten as we're pruning
6214 * this state and will pop a new one.
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006215 */
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006216 err = propagate_liveness(env, &sl->state, cur);
6217 if (err)
6218 return err;
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006219 return 1;
Edward Creedc503a82017-08-15 20:34:35 +01006220 }
Alexei Starovoitovceefbc92018-12-03 22:46:06 -08006221 states_cnt++;
Alexei Starovoitov9f4686c2019-04-01 21:27:41 -07006222 sl->miss_cnt++;
6223 /* heuristic to determine whether this state is beneficial
6224 * to keep checking from state equivalence point of view.
6225 * Higher numbers increase max_states_per_insn and verification time,
6226 * but do not meaningfully decrease insn_processed.
6227 */
6228 if (sl->miss_cnt > sl->hit_cnt * 3 + 3) {
6229 /* the state is unlikely to be useful. Remove it to
6230 * speed up verification
6231 */
6232 *pprev = sl->next;
6233 if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE) {
6234 free_verifier_state(&sl->state, false);
6235 kfree(sl);
6236 env->peak_states--;
6237 } else {
6238 /* cannot free this state, since parentage chain may
6239 * walk it later. Add it for free_list instead to
6240 * be freed at the end of verification
6241 */
6242 sl->next = env->free_list;
6243 env->free_list = sl;
6244 }
6245 sl = *pprev;
6246 continue;
6247 }
6248 pprev = &sl->next;
6249 sl = *pprev;
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006250 }
6251
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07006252 if (env->max_states_per_insn < states_cnt)
6253 env->max_states_per_insn = states_cnt;
6254
Alexei Starovoitovceefbc92018-12-03 22:46:06 -08006255 if (!env->allow_ptr_leaks && states_cnt > BPF_COMPLEXITY_LIMIT_STATES)
6256 return 0;
6257
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006258 /* there were no equivalent states, remember current one.
6259 * technically the current state is not proven to be safe yet,
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006260 * but it will either reach outer most bpf_exit (which means it's safe)
6261 * or it will be rejected. Since there are no loops, we won't be
6262 * seeing this tuple (frame[0].callsite, frame[1].callsite, .. insn_idx)
6263 * again on the way to bpf_exit
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006264 */
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07006265 new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL);
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006266 if (!new_sl)
6267 return -ENOMEM;
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07006268 env->total_states++;
6269 env->peak_states++;
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006270
6271 /* add new state to the head of linked list */
Edward Cree679c7822018-08-22 20:02:19 +01006272 new = &new_sl->state;
6273 err = copy_verifier_state(new, cur);
Alexei Starovoitov1969db42017-11-01 00:08:04 -07006274 if (err) {
Edward Cree679c7822018-08-22 20:02:19 +01006275 free_verifier_state(new, false);
Alexei Starovoitov1969db42017-11-01 00:08:04 -07006276 kfree(new_sl);
6277 return err;
6278 }
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006279 new_sl->next = env->explored_states[insn_idx];
6280 env->explored_states[insn_idx] = new_sl;
Jakub Kicinski7640ead2018-12-12 16:29:07 -08006281 /* connect new state to parentage chain. Current frame needs all
6282 * registers connected. Only r6 - r9 of the callers are alive (pushed
6283 * to the stack implicitly by JITs) so in callers' frames connect just
6284 * r6 - r9 as an optimization. Callers will have r1 - r5 connected to
6285 * the state of the call instruction (with WRITTEN set), and r0 comes
6286 * from callee with its full parentage chain, anyway.
6287 */
6288 for (j = 0; j <= cur->curframe; j++)
6289 for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++)
6290 cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i];
Edward Cree8e9cd9c2017-08-23 15:11:21 +01006291 /* clear write marks in current state: the writes we did are not writes
6292 * our child did, so they don't screen off its reads from us.
6293 * (There are no read marks in current state, because reads always mark
6294 * their parent and current state never has children yet. Only
6295 * explored_states can get read marks.)
6296 */
Edward Creedc503a82017-08-15 20:34:35 +01006297 for (i = 0; i < BPF_REG_FP; i++)
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006298 cur->frame[cur->curframe]->regs[i].live = REG_LIVE_NONE;
6299
6300 /* all stack frames are accessible from callee, clear them all */
6301 for (j = 0; j <= cur->curframe; j++) {
6302 struct bpf_func_state *frame = cur->frame[j];
Edward Cree679c7822018-08-22 20:02:19 +01006303 struct bpf_func_state *newframe = new->frame[j];
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006304
Edward Cree679c7822018-08-22 20:02:19 +01006305 for (i = 0; i < frame->allocated_stack / BPF_REG_SIZE; i++) {
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08006306 frame->stack[i].spilled_ptr.live = REG_LIVE_NONE;
Edward Cree679c7822018-08-22 20:02:19 +01006307 frame->stack[i].spilled_ptr.parent =
6308 &newframe->stack[i].spilled_ptr;
6309 }
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006310 }
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006311 return 0;
6312}
6313
Joe Stringerc64b7982018-10-02 13:35:33 -07006314/* Return true if it's OK to have the same insn return a different type. */
6315static bool reg_type_mismatch_ok(enum bpf_reg_type type)
6316{
6317 switch (type) {
6318 case PTR_TO_CTX:
6319 case PTR_TO_SOCKET:
6320 case PTR_TO_SOCKET_OR_NULL:
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08006321 case PTR_TO_SOCK_COMMON:
6322 case PTR_TO_SOCK_COMMON_OR_NULL:
Martin KaFai Lau655a51e2019-02-09 23:22:24 -08006323 case PTR_TO_TCP_SOCK:
6324 case PTR_TO_TCP_SOCK_OR_NULL:
Joe Stringerc64b7982018-10-02 13:35:33 -07006325 return false;
6326 default:
6327 return true;
6328 }
6329}
6330
6331/* If an instruction was previously used with particular pointer types, then we
6332 * need to be careful to avoid cases such as the below, where it may be ok
6333 * for one branch accessing the pointer, but not ok for the other branch:
6334 *
6335 * R1 = sock_ptr
6336 * goto X;
6337 * ...
6338 * R1 = some_other_valid_ptr;
6339 * goto X;
6340 * ...
6341 * R2 = *(u32 *)(R1 + 0);
6342 */
6343static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev)
6344{
6345 return src != prev && (!reg_type_mismatch_ok(src) ||
6346 !reg_type_mismatch_ok(prev));
6347}
6348
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01006349static int do_check(struct bpf_verifier_env *env)
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006350{
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07006351 struct bpf_verifier_state *state;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006352 struct bpf_insn *insns = env->prog->insnsi;
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07006353 struct bpf_reg_state *regs;
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07006354 int insn_cnt = env->prog->len;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006355 bool do_print_state = false;
6356
Martin KaFai Laud9762e82018-12-13 10:41:48 -08006357 env->prev_linfo = NULL;
6358
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07006359 state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL);
6360 if (!state)
6361 return -ENOMEM;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006362 state->curframe = 0;
Daniel Borkmann979d63d2019-01-03 00:58:34 +01006363 state->speculative = false;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006364 state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL);
6365 if (!state->frame[0]) {
6366 kfree(state);
6367 return -ENOMEM;
6368 }
6369 env->cur_state = state;
6370 init_func_state(env, state->frame[0],
6371 BPF_MAIN_FUNC /* callsite */,
6372 0 /* frameno */,
6373 0 /* subprogno, zero == main subprog */);
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006374
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006375 for (;;) {
6376 struct bpf_insn *insn;
6377 u8 class;
6378 int err;
6379
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006380 if (env->insn_idx >= insn_cnt) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006381 verbose(env, "invalid insn idx %d insn_cnt %d\n",
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006382 env->insn_idx, insn_cnt);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006383 return -EFAULT;
6384 }
6385
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006386 insn = &insns[env->insn_idx];
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006387 class = BPF_CLASS(insn->code);
6388
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07006389 if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006390 verbose(env,
6391 "BPF program is too large. Processed %d insn\n",
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07006392 env->insn_processed);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006393 return -E2BIG;
6394 }
6395
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006396 err = is_state_visited(env, env->insn_idx);
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006397 if (err < 0)
6398 return err;
6399 if (err == 1) {
6400 /* found equivalent state, can prune the search */
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07006401 if (env->log.level & BPF_LOG_LEVEL) {
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006402 if (do_print_state)
Daniel Borkmann979d63d2019-01-03 00:58:34 +01006403 verbose(env, "\nfrom %d to %d%s: safe\n",
6404 env->prev_insn_idx, env->insn_idx,
6405 env->cur_state->speculative ?
6406 " (speculative execution)" : "");
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006407 else
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006408 verbose(env, "%d: safe\n", env->insn_idx);
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006409 }
6410 goto process_bpf_exit;
6411 }
6412
Alexei Starovoitovc3494802018-12-03 22:46:04 -08006413 if (signal_pending(current))
6414 return -EAGAIN;
6415
Daniel Borkmann3c2ce602017-05-18 03:00:06 +02006416 if (need_resched())
6417 cond_resched();
6418
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07006419 if (env->log.level & BPF_LOG_LEVEL2 ||
6420 (env->log.level & BPF_LOG_LEVEL && do_print_state)) {
6421 if (env->log.level & BPF_LOG_LEVEL2)
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006422 verbose(env, "%d:", env->insn_idx);
David S. Millerc5fc9692017-05-10 11:25:17 -07006423 else
Daniel Borkmann979d63d2019-01-03 00:58:34 +01006424 verbose(env, "\nfrom %d to %d%s:",
6425 env->prev_insn_idx, env->insn_idx,
6426 env->cur_state->speculative ?
6427 " (speculative execution)" : "");
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006428 print_verifier_state(env, state->frame[state->curframe]);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006429 do_print_state = false;
6430 }
6431
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07006432 if (env->log.level & BPF_LOG_LEVEL) {
Daniel Borkmann7105e822017-12-20 13:42:57 +01006433 const struct bpf_insn_cbs cbs = {
6434 .cb_print = verbose,
Jiri Olsaabe08842018-03-23 11:41:28 +01006435 .private_data = env,
Daniel Borkmann7105e822017-12-20 13:42:57 +01006436 };
6437
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006438 verbose_linfo(env, env->insn_idx, "; ");
6439 verbose(env, "%d: ", env->insn_idx);
Jiri Olsaabe08842018-03-23 11:41:28 +01006440 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006441 }
6442
Jakub Kicinskicae19272017-12-27 18:39:05 -08006443 if (bpf_prog_is_dev_bound(env->prog->aux)) {
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006444 err = bpf_prog_offload_verify_insn(env, env->insn_idx,
6445 env->prev_insn_idx);
Jakub Kicinskicae19272017-12-27 18:39:05 -08006446 if (err)
6447 return err;
6448 }
Jakub Kicinski13a27df2016-09-21 11:43:58 +01006449
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07006450 regs = cur_regs(env);
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006451 env->insn_aux_data[env->insn_idx].seen = true;
Joe Stringerfd978bf72018-10-02 13:35:35 -07006452
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006453 if (class == BPF_ALU || class == BPF_ALU64) {
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07006454 err = check_alu_op(env, insn);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006455 if (err)
6456 return err;
6457
6458 } else if (class == BPF_LDX) {
Jakub Kicinski3df126f2016-09-21 11:43:56 +01006459 enum bpf_reg_type *prev_src_type, src_reg_type;
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07006460
6461 /* check for reserved fields is already done */
6462
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006463 /* check src operand */
Edward Creedc503a82017-08-15 20:34:35 +01006464 err = check_reg_arg(env, insn->src_reg, SRC_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006465 if (err)
6466 return err;
6467
Edward Creedc503a82017-08-15 20:34:35 +01006468 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006469 if (err)
6470 return err;
6471
Alexei Starovoitov725f9dc2015-04-15 16:19:33 -07006472 src_reg_type = regs[insn->src_reg].type;
6473
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006474 /* check that memory (src_reg + off) is readable,
6475 * the state of dst_reg will be updated by this func
6476 */
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006477 err = check_mem_access(env, env->insn_idx, insn->src_reg,
6478 insn->off, BPF_SIZE(insn->code),
6479 BPF_READ, insn->dst_reg, false);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006480 if (err)
6481 return err;
6482
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006483 prev_src_type = &env->insn_aux_data[env->insn_idx].ptr_type;
Jakub Kicinski3df126f2016-09-21 11:43:56 +01006484
6485 if (*prev_src_type == NOT_INIT) {
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07006486 /* saw a valid insn
6487 * dst_reg = *(u32 *)(src_reg + off)
Jakub Kicinski3df126f2016-09-21 11:43:56 +01006488 * save type to validate intersecting paths
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07006489 */
Jakub Kicinski3df126f2016-09-21 11:43:56 +01006490 *prev_src_type = src_reg_type;
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07006491
Joe Stringerc64b7982018-10-02 13:35:33 -07006492 } else if (reg_type_mismatch(src_reg_type, *prev_src_type)) {
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07006493 /* ABuser program is trying to use the same insn
6494 * dst_reg = *(u32*) (src_reg + off)
6495 * with different pointer types:
6496 * src_reg == ctx in one branch and
6497 * src_reg == stack|map in some other branch.
6498 * Reject it.
6499 */
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006500 verbose(env, "same insn cannot be used with different pointers\n");
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07006501 return -EINVAL;
6502 }
6503
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006504 } else if (class == BPF_STX) {
Jakub Kicinski3df126f2016-09-21 11:43:56 +01006505 enum bpf_reg_type *prev_dst_type, dst_reg_type;
Alexei Starovoitovd691f9e2015-06-04 10:11:54 -07006506
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006507 if (BPF_MODE(insn->code) == BPF_XADD) {
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006508 err = check_xadd(env, env->insn_idx, insn);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006509 if (err)
6510 return err;
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006511 env->insn_idx++;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006512 continue;
6513 }
6514
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006515 /* check src1 operand */
Edward Creedc503a82017-08-15 20:34:35 +01006516 err = check_reg_arg(env, insn->src_reg, SRC_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006517 if (err)
6518 return err;
6519 /* check src2 operand */
Edward Creedc503a82017-08-15 20:34:35 +01006520 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006521 if (err)
6522 return err;
6523
Alexei Starovoitovd691f9e2015-06-04 10:11:54 -07006524 dst_reg_type = regs[insn->dst_reg].type;
6525
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006526 /* check that memory (dst_reg + off) is writeable */
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006527 err = check_mem_access(env, env->insn_idx, insn->dst_reg,
6528 insn->off, BPF_SIZE(insn->code),
6529 BPF_WRITE, insn->src_reg, false);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006530 if (err)
6531 return err;
6532
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006533 prev_dst_type = &env->insn_aux_data[env->insn_idx].ptr_type;
Jakub Kicinski3df126f2016-09-21 11:43:56 +01006534
6535 if (*prev_dst_type == NOT_INIT) {
6536 *prev_dst_type = dst_reg_type;
Joe Stringerc64b7982018-10-02 13:35:33 -07006537 } else if (reg_type_mismatch(dst_reg_type, *prev_dst_type)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006538 verbose(env, "same insn cannot be used with different pointers\n");
Alexei Starovoitovd691f9e2015-06-04 10:11:54 -07006539 return -EINVAL;
6540 }
6541
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006542 } else if (class == BPF_ST) {
6543 if (BPF_MODE(insn->code) != BPF_MEM ||
6544 insn->src_reg != BPF_REG_0) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006545 verbose(env, "BPF_ST uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006546 return -EINVAL;
6547 }
6548 /* check src operand */
Edward Creedc503a82017-08-15 20:34:35 +01006549 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006550 if (err)
6551 return err;
6552
Daniel Borkmannf37a8cb2018-01-16 23:30:10 +01006553 if (is_ctx_reg(env, insn->dst_reg)) {
Joe Stringer9d2be442018-10-02 13:35:31 -07006554 verbose(env, "BPF_ST stores into R%d %s is not allowed\n",
Daniel Borkmann2a159c62018-10-21 02:09:24 +02006555 insn->dst_reg,
6556 reg_type_str[reg_state(env, insn->dst_reg)->type]);
Daniel Borkmannf37a8cb2018-01-16 23:30:10 +01006557 return -EACCES;
6558 }
6559
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006560 /* check that memory (dst_reg + off) is writeable */
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006561 err = check_mem_access(env, env->insn_idx, insn->dst_reg,
6562 insn->off, BPF_SIZE(insn->code),
6563 BPF_WRITE, -1, false);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006564 if (err)
6565 return err;
6566
Jiong Wang092ed092019-01-26 12:26:01 -05006567 } else if (class == BPF_JMP || class == BPF_JMP32) {
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006568 u8 opcode = BPF_OP(insn->code);
6569
6570 if (opcode == BPF_CALL) {
6571 if (BPF_SRC(insn->code) != BPF_K ||
6572 insn->off != 0 ||
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006573 (insn->src_reg != BPF_REG_0 &&
6574 insn->src_reg != BPF_PSEUDO_CALL) ||
Jiong Wang092ed092019-01-26 12:26:01 -05006575 insn->dst_reg != BPF_REG_0 ||
6576 class == BPF_JMP32) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006577 verbose(env, "BPF_CALL uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006578 return -EINVAL;
6579 }
6580
Alexei Starovoitovd83525c2019-01-31 15:40:04 -08006581 if (env->cur_state->active_spin_lock &&
6582 (insn->src_reg == BPF_PSEUDO_CALL ||
6583 insn->imm != BPF_FUNC_spin_unlock)) {
6584 verbose(env, "function calls are not allowed while holding a lock\n");
6585 return -EINVAL;
6586 }
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006587 if (insn->src_reg == BPF_PSEUDO_CALL)
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006588 err = check_func_call(env, insn, &env->insn_idx);
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006589 else
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006590 err = check_helper_call(env, insn->imm, env->insn_idx);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006591 if (err)
6592 return err;
6593
6594 } else if (opcode == BPF_JA) {
6595 if (BPF_SRC(insn->code) != BPF_K ||
6596 insn->imm != 0 ||
6597 insn->src_reg != BPF_REG_0 ||
Jiong Wang092ed092019-01-26 12:26:01 -05006598 insn->dst_reg != BPF_REG_0 ||
6599 class == BPF_JMP32) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006600 verbose(env, "BPF_JA uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006601 return -EINVAL;
6602 }
6603
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006604 env->insn_idx += insn->off + 1;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006605 continue;
6606
6607 } else if (opcode == BPF_EXIT) {
6608 if (BPF_SRC(insn->code) != BPF_K ||
6609 insn->imm != 0 ||
6610 insn->src_reg != BPF_REG_0 ||
Jiong Wang092ed092019-01-26 12:26:01 -05006611 insn->dst_reg != BPF_REG_0 ||
6612 class == BPF_JMP32) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006613 verbose(env, "BPF_EXIT uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006614 return -EINVAL;
6615 }
6616
Alexei Starovoitovd83525c2019-01-31 15:40:04 -08006617 if (env->cur_state->active_spin_lock) {
6618 verbose(env, "bpf_spin_unlock is missing\n");
6619 return -EINVAL;
6620 }
6621
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006622 if (state->curframe) {
6623 /* exit from nested function */
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006624 env->prev_insn_idx = env->insn_idx;
6625 err = prepare_func_exit(env, &env->insn_idx);
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006626 if (err)
6627 return err;
6628 do_print_state = true;
6629 continue;
6630 }
6631
Joe Stringerfd978bf72018-10-02 13:35:35 -07006632 err = check_reference_leak(env);
6633 if (err)
6634 return err;
6635
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006636 /* eBPF calling convetion is such that R0 is used
6637 * to return the value from eBPF program.
6638 * Make sure that it's readable at this time
6639 * of bpf_exit, which means that program wrote
6640 * something into it earlier
6641 */
Edward Creedc503a82017-08-15 20:34:35 +01006642 err = check_reg_arg(env, BPF_REG_0, SRC_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006643 if (err)
6644 return err;
6645
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07006646 if (is_pointer_value(env, BPF_REG_0)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006647 verbose(env, "R0 leaks addr as return value\n");
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07006648 return -EACCES;
6649 }
6650
Alexei Starovoitov390ee7e2017-10-02 22:50:23 -07006651 err = check_return_code(env);
6652 if (err)
6653 return err;
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006654process_bpf_exit:
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006655 err = pop_stack(env, &env->prev_insn_idx,
6656 &env->insn_idx);
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07006657 if (err < 0) {
6658 if (err != -ENOENT)
6659 return err;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006660 break;
6661 } else {
6662 do_print_state = true;
6663 continue;
6664 }
6665 } else {
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006666 err = check_cond_jmp_op(env, insn, &env->insn_idx);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006667 if (err)
6668 return err;
6669 }
6670 } else if (class == BPF_LD) {
6671 u8 mode = BPF_MODE(insn->code);
6672
6673 if (mode == BPF_ABS || mode == BPF_IND) {
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08006674 err = check_ld_abs(env, insn);
6675 if (err)
6676 return err;
6677
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006678 } else if (mode == BPF_IMM) {
6679 err = check_ld_imm(env, insn);
6680 if (err)
6681 return err;
6682
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006683 env->insn_idx++;
6684 env->insn_aux_data[env->insn_idx].seen = true;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006685 } else {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006686 verbose(env, "invalid BPF_LD mode\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006687 return -EINVAL;
6688 }
6689 } else {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006690 verbose(env, "unknown insn class %d\n", class);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006691 return -EINVAL;
6692 }
6693
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006694 env->insn_idx++;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006695 }
6696
Jiong Wang9c8105b2018-05-02 16:17:18 -04006697 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006698 return 0;
6699}
6700
Martin KaFai Lau56f668d2017-03-22 10:00:33 -07006701static int check_map_prealloc(struct bpf_map *map)
6702{
6703 return (map->map_type != BPF_MAP_TYPE_HASH &&
Martin KaFai Laubcc6b1b2017-03-22 10:00:34 -07006704 map->map_type != BPF_MAP_TYPE_PERCPU_HASH &&
6705 map->map_type != BPF_MAP_TYPE_HASH_OF_MAPS) ||
Martin KaFai Lau56f668d2017-03-22 10:00:33 -07006706 !(map->map_flags & BPF_F_NO_PREALLOC);
6707}
6708
Alexei Starovoitovd83525c2019-01-31 15:40:04 -08006709static bool is_tracing_prog_type(enum bpf_prog_type type)
6710{
6711 switch (type) {
6712 case BPF_PROG_TYPE_KPROBE:
6713 case BPF_PROG_TYPE_TRACEPOINT:
6714 case BPF_PROG_TYPE_PERF_EVENT:
6715 case BPF_PROG_TYPE_RAW_TRACEPOINT:
6716 return true;
6717 default:
6718 return false;
6719 }
6720}
6721
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006722static int check_map_prog_compatibility(struct bpf_verifier_env *env,
6723 struct bpf_map *map,
Alexei Starovoitovfdc15d32016-09-01 18:37:23 -07006724 struct bpf_prog *prog)
6725
6726{
Martin KaFai Lau56f668d2017-03-22 10:00:33 -07006727 /* Make sure that BPF_PROG_TYPE_PERF_EVENT programs only use
6728 * preallocated hash maps, since doing memory allocation
6729 * in overflow_handler can crash depending on where nmi got
6730 * triggered.
6731 */
6732 if (prog->type == BPF_PROG_TYPE_PERF_EVENT) {
6733 if (!check_map_prealloc(map)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006734 verbose(env, "perf_event programs can only use preallocated hash map\n");
Martin KaFai Lau56f668d2017-03-22 10:00:33 -07006735 return -EINVAL;
6736 }
6737 if (map->inner_map_meta &&
6738 !check_map_prealloc(map->inner_map_meta)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006739 verbose(env, "perf_event programs can only use preallocated inner hash map\n");
Martin KaFai Lau56f668d2017-03-22 10:00:33 -07006740 return -EINVAL;
6741 }
Alexei Starovoitovfdc15d32016-09-01 18:37:23 -07006742 }
Jakub Kicinskia3884572018-01-11 20:29:09 -08006743
Alexei Starovoitovd83525c2019-01-31 15:40:04 -08006744 if ((is_tracing_prog_type(prog->type) ||
6745 prog->type == BPF_PROG_TYPE_SOCKET_FILTER) &&
6746 map_value_has_spin_lock(map)) {
6747 verbose(env, "tracing progs cannot use bpf_spin_lock yet\n");
6748 return -EINVAL;
6749 }
6750
Jakub Kicinskia3884572018-01-11 20:29:09 -08006751 if ((bpf_prog_is_dev_bound(prog->aux) || bpf_map_is_dev_bound(map)) &&
Jakub Kicinski09728262018-07-17 10:53:23 -07006752 !bpf_offload_prog_map_match(prog, map)) {
Jakub Kicinskia3884572018-01-11 20:29:09 -08006753 verbose(env, "offload device mismatch between prog and map\n");
6754 return -EINVAL;
6755 }
6756
Alexei Starovoitovfdc15d32016-09-01 18:37:23 -07006757 return 0;
6758}
6759
Roman Gushchinb741f162018-09-28 14:45:43 +00006760static bool bpf_map_is_cgroup_storage(struct bpf_map *map)
6761{
6762 return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE ||
6763 map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE);
6764}
6765
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006766/* look for pseudo eBPF instructions that access map FDs and
6767 * replace them with actual map pointers
6768 */
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01006769static int replace_map_fd_with_map_ptr(struct bpf_verifier_env *env)
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006770{
6771 struct bpf_insn *insn = env->prog->insnsi;
6772 int insn_cnt = env->prog->len;
Alexei Starovoitovfdc15d32016-09-01 18:37:23 -07006773 int i, j, err;
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006774
Daniel Borkmannf1f77142017-01-13 23:38:15 +01006775 err = bpf_prog_calc_tag(env->prog);
Daniel Borkmannaafe6ae2016-12-18 01:52:57 +01006776 if (err)
6777 return err;
6778
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006779 for (i = 0; i < insn_cnt; i++, insn++) {
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07006780 if (BPF_CLASS(insn->code) == BPF_LDX &&
Alexei Starovoitovd691f9e2015-06-04 10:11:54 -07006781 (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006782 verbose(env, "BPF_LDX uses reserved fields\n");
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07006783 return -EINVAL;
6784 }
6785
Alexei Starovoitovd691f9e2015-06-04 10:11:54 -07006786 if (BPF_CLASS(insn->code) == BPF_STX &&
6787 ((BPF_MODE(insn->code) != BPF_MEM &&
6788 BPF_MODE(insn->code) != BPF_XADD) || insn->imm != 0)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006789 verbose(env, "BPF_STX uses reserved fields\n");
Alexei Starovoitovd691f9e2015-06-04 10:11:54 -07006790 return -EINVAL;
6791 }
6792
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006793 if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) {
6794 struct bpf_map *map;
6795 struct fd f;
6796
6797 if (i == insn_cnt - 1 || insn[1].code != 0 ||
6798 insn[1].dst_reg != 0 || insn[1].src_reg != 0 ||
6799 insn[1].off != 0) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006800 verbose(env, "invalid bpf_ld_imm64 insn\n");
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006801 return -EINVAL;
6802 }
6803
6804 if (insn->src_reg == 0)
6805 /* valid generic load 64-bit imm */
6806 goto next_insn;
6807
Daniel Borkmann20182392019-03-04 21:08:53 +01006808 if (insn[0].src_reg != BPF_PSEUDO_MAP_FD ||
6809 insn[1].imm != 0) {
6810 verbose(env, "unrecognized bpf_ld_imm64 insn\n");
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006811 return -EINVAL;
6812 }
6813
Daniel Borkmann20182392019-03-04 21:08:53 +01006814 f = fdget(insn[0].imm);
Daniel Borkmannc2101292015-10-29 14:58:07 +01006815 map = __bpf_map_get(f);
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006816 if (IS_ERR(map)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006817 verbose(env, "fd %d is not pointing to valid bpf_map\n",
Daniel Borkmann20182392019-03-04 21:08:53 +01006818 insn[0].imm);
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006819 return PTR_ERR(map);
6820 }
6821
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006822 err = check_map_prog_compatibility(env, map, env->prog);
Alexei Starovoitovfdc15d32016-09-01 18:37:23 -07006823 if (err) {
6824 fdput(f);
6825 return err;
6826 }
6827
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006828 /* store map pointer inside BPF_LD_IMM64 instruction */
6829 insn[0].imm = (u32) (unsigned long) map;
6830 insn[1].imm = ((u64) (unsigned long) map) >> 32;
6831
6832 /* check whether we recorded this map already */
6833 for (j = 0; j < env->used_map_cnt; j++)
6834 if (env->used_maps[j] == map) {
6835 fdput(f);
6836 goto next_insn;
6837 }
6838
6839 if (env->used_map_cnt >= MAX_USED_MAPS) {
6840 fdput(f);
6841 return -E2BIG;
6842 }
6843
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006844 /* hold the map. If the program is rejected by verifier,
6845 * the map will be released by release_maps() or it
6846 * will be used by the valid program until it's unloaded
Jakub Kicinskiab7f5bf2018-05-03 18:37:17 -07006847 * and all maps are released in free_used_maps()
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006848 */
Alexei Starovoitov92117d82016-04-27 18:56:20 -07006849 map = bpf_map_inc(map, false);
6850 if (IS_ERR(map)) {
6851 fdput(f);
6852 return PTR_ERR(map);
6853 }
6854 env->used_maps[env->used_map_cnt++] = map;
6855
Roman Gushchinb741f162018-09-28 14:45:43 +00006856 if (bpf_map_is_cgroup_storage(map) &&
Roman Gushchinde9cbba2018-08-02 14:27:18 -07006857 bpf_cgroup_storage_assign(env->prog, map)) {
Roman Gushchinb741f162018-09-28 14:45:43 +00006858 verbose(env, "only one cgroup storage of each type is allowed\n");
Roman Gushchinde9cbba2018-08-02 14:27:18 -07006859 fdput(f);
6860 return -EBUSY;
6861 }
6862
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006863 fdput(f);
6864next_insn:
6865 insn++;
6866 i++;
Daniel Borkmann5e581da2018-01-26 23:33:38 +01006867 continue;
6868 }
6869
6870 /* Basic sanity check before we invest more work here. */
6871 if (!bpf_opcode_in_insntable(insn->code)) {
6872 verbose(env, "unknown opcode %02x\n", insn->code);
6873 return -EINVAL;
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006874 }
6875 }
6876
6877 /* now all pseudo BPF_LD_IMM64 instructions load valid
6878 * 'struct bpf_map *' into a register instead of user map_fd.
6879 * These pointers will be used later by verifier to validate map access.
6880 */
6881 return 0;
6882}
6883
6884/* drop refcnt of maps used by the rejected program */
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01006885static void release_maps(struct bpf_verifier_env *env)
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006886{
Roman Gushchin8bad74f2018-09-28 14:45:36 +00006887 enum bpf_cgroup_storage_type stype;
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006888 int i;
6889
Roman Gushchin8bad74f2018-09-28 14:45:36 +00006890 for_each_cgroup_storage_type(stype) {
6891 if (!env->prog->aux->cgroup_storage[stype])
6892 continue;
Roman Gushchinde9cbba2018-08-02 14:27:18 -07006893 bpf_cgroup_storage_release(env->prog,
Roman Gushchin8bad74f2018-09-28 14:45:36 +00006894 env->prog->aux->cgroup_storage[stype]);
6895 }
Roman Gushchinde9cbba2018-08-02 14:27:18 -07006896
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006897 for (i = 0; i < env->used_map_cnt; i++)
6898 bpf_map_put(env->used_maps[i]);
6899}
6900
6901/* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01006902static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env)
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006903{
6904 struct bpf_insn *insn = env->prog->insnsi;
6905 int insn_cnt = env->prog->len;
6906 int i;
6907
6908 for (i = 0; i < insn_cnt; i++, insn++)
6909 if (insn->code == (BPF_LD | BPF_IMM | BPF_DW))
6910 insn->src_reg = 0;
6911}
6912
Alexei Starovoitov80419022017-03-15 18:26:41 -07006913/* single env->prog->insni[off] instruction was replaced with the range
6914 * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying
6915 * [0, off) and [off, end) to new locations, so the patched range stays zero
6916 */
6917static int adjust_insn_aux_data(struct bpf_verifier_env *env, u32 prog_len,
6918 u32 off, u32 cnt)
6919{
6920 struct bpf_insn_aux_data *new_data, *old_data = env->insn_aux_data;
Alexei Starovoitovc1311872017-11-22 16:42:05 -08006921 int i;
Alexei Starovoitov80419022017-03-15 18:26:41 -07006922
6923 if (cnt == 1)
6924 return 0;
Kees Cookfad953c2018-06-12 14:27:37 -07006925 new_data = vzalloc(array_size(prog_len,
6926 sizeof(struct bpf_insn_aux_data)));
Alexei Starovoitov80419022017-03-15 18:26:41 -07006927 if (!new_data)
6928 return -ENOMEM;
6929 memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off);
6930 memcpy(new_data + off + cnt - 1, old_data + off,
6931 sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1));
Alexei Starovoitovc1311872017-11-22 16:42:05 -08006932 for (i = off; i < off + cnt - 1; i++)
6933 new_data[i].seen = true;
Alexei Starovoitov80419022017-03-15 18:26:41 -07006934 env->insn_aux_data = new_data;
6935 vfree(old_data);
6936 return 0;
6937}
6938
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08006939static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)
6940{
6941 int i;
6942
6943 if (len == 1)
6944 return;
Jiong Wang4cb3d992018-05-02 16:17:19 -04006945 /* NOTE: fake 'exit' subprog should be updated as well. */
6946 for (i = 0; i <= env->subprog_cnt; i++) {
Edward Creeafd59422018-11-16 12:00:07 +00006947 if (env->subprog_info[i].start <= off)
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08006948 continue;
Jiong Wang9c8105b2018-05-02 16:17:18 -04006949 env->subprog_info[i].start += len - 1;
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08006950 }
6951}
6952
Alexei Starovoitov80419022017-03-15 18:26:41 -07006953static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
6954 const struct bpf_insn *patch, u32 len)
6955{
6956 struct bpf_prog *new_prog;
6957
6958 new_prog = bpf_patch_insn_single(env->prog, off, patch, len);
Alexei Starovoitov4f733792019-04-01 21:27:44 -07006959 if (IS_ERR(new_prog)) {
6960 if (PTR_ERR(new_prog) == -ERANGE)
6961 verbose(env,
6962 "insn %d cannot be patched due to 16-bit range\n",
6963 env->insn_aux_data[off].orig_idx);
Alexei Starovoitov80419022017-03-15 18:26:41 -07006964 return NULL;
Alexei Starovoitov4f733792019-04-01 21:27:44 -07006965 }
Alexei Starovoitov80419022017-03-15 18:26:41 -07006966 if (adjust_insn_aux_data(env, new_prog->len, off, len))
6967 return NULL;
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08006968 adjust_subprog_starts(env, off, len);
Alexei Starovoitov80419022017-03-15 18:26:41 -07006969 return new_prog;
6970}
6971
Jakub Kicinski52875a02019-01-22 22:45:20 -08006972static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env,
6973 u32 off, u32 cnt)
6974{
6975 int i, j;
6976
6977 /* find first prog starting at or after off (first to remove) */
6978 for (i = 0; i < env->subprog_cnt; i++)
6979 if (env->subprog_info[i].start >= off)
6980 break;
6981 /* find first prog starting at or after off + cnt (first to stay) */
6982 for (j = i; j < env->subprog_cnt; j++)
6983 if (env->subprog_info[j].start >= off + cnt)
6984 break;
6985 /* if j doesn't start exactly at off + cnt, we are just removing
6986 * the front of previous prog
6987 */
6988 if (env->subprog_info[j].start != off + cnt)
6989 j--;
6990
6991 if (j > i) {
6992 struct bpf_prog_aux *aux = env->prog->aux;
6993 int move;
6994
6995 /* move fake 'exit' subprog as well */
6996 move = env->subprog_cnt + 1 - j;
6997
6998 memmove(env->subprog_info + i,
6999 env->subprog_info + j,
7000 sizeof(*env->subprog_info) * move);
7001 env->subprog_cnt -= j - i;
7002
7003 /* remove func_info */
7004 if (aux->func_info) {
7005 move = aux->func_info_cnt - j;
7006
7007 memmove(aux->func_info + i,
7008 aux->func_info + j,
7009 sizeof(*aux->func_info) * move);
7010 aux->func_info_cnt -= j - i;
7011 /* func_info->insn_off is set after all code rewrites,
7012 * in adjust_btf_func() - no need to adjust
7013 */
7014 }
7015 } else {
7016 /* convert i from "first prog to remove" to "first to adjust" */
7017 if (env->subprog_info[i].start == off)
7018 i++;
7019 }
7020
7021 /* update fake 'exit' subprog as well */
7022 for (; i <= env->subprog_cnt; i++)
7023 env->subprog_info[i].start -= cnt;
7024
7025 return 0;
7026}
7027
7028static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off,
7029 u32 cnt)
7030{
7031 struct bpf_prog *prog = env->prog;
7032 u32 i, l_off, l_cnt, nr_linfo;
7033 struct bpf_line_info *linfo;
7034
7035 nr_linfo = prog->aux->nr_linfo;
7036 if (!nr_linfo)
7037 return 0;
7038
7039 linfo = prog->aux->linfo;
7040
7041 /* find first line info to remove, count lines to be removed */
7042 for (i = 0; i < nr_linfo; i++)
7043 if (linfo[i].insn_off >= off)
7044 break;
7045
7046 l_off = i;
7047 l_cnt = 0;
7048 for (; i < nr_linfo; i++)
7049 if (linfo[i].insn_off < off + cnt)
7050 l_cnt++;
7051 else
7052 break;
7053
7054 /* First live insn doesn't match first live linfo, it needs to "inherit"
7055 * last removed linfo. prog is already modified, so prog->len == off
7056 * means no live instructions after (tail of the program was removed).
7057 */
7058 if (prog->len != off && l_cnt &&
7059 (i == nr_linfo || linfo[i].insn_off != off + cnt)) {
7060 l_cnt--;
7061 linfo[--i].insn_off = off + cnt;
7062 }
7063
7064 /* remove the line info which refer to the removed instructions */
7065 if (l_cnt) {
7066 memmove(linfo + l_off, linfo + i,
7067 sizeof(*linfo) * (nr_linfo - i));
7068
7069 prog->aux->nr_linfo -= l_cnt;
7070 nr_linfo = prog->aux->nr_linfo;
7071 }
7072
7073 /* pull all linfo[i].insn_off >= off + cnt in by cnt */
7074 for (i = l_off; i < nr_linfo; i++)
7075 linfo[i].insn_off -= cnt;
7076
7077 /* fix up all subprogs (incl. 'exit') which start >= off */
7078 for (i = 0; i <= env->subprog_cnt; i++)
7079 if (env->subprog_info[i].linfo_idx > l_off) {
7080 /* program may have started in the removed region but
7081 * may not be fully removed
7082 */
7083 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt)
7084 env->subprog_info[i].linfo_idx -= l_cnt;
7085 else
7086 env->subprog_info[i].linfo_idx = l_off;
7087 }
7088
7089 return 0;
7090}
7091
7092static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)
7093{
7094 struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
7095 unsigned int orig_prog_len = env->prog->len;
7096 int err;
7097
Jakub Kicinski08ca90a2019-01-22 22:45:24 -08007098 if (bpf_prog_is_dev_bound(env->prog->aux))
7099 bpf_prog_offload_remove_insns(env, off, cnt);
7100
Jakub Kicinski52875a02019-01-22 22:45:20 -08007101 err = bpf_remove_insns(env->prog, off, cnt);
7102 if (err)
7103 return err;
7104
7105 err = adjust_subprog_starts_after_remove(env, off, cnt);
7106 if (err)
7107 return err;
7108
7109 err = bpf_adj_linfo_after_remove(env, off, cnt);
7110 if (err)
7111 return err;
7112
7113 memmove(aux_data + off, aux_data + off + cnt,
7114 sizeof(*aux_data) * (orig_prog_len - off - cnt));
7115
7116 return 0;
7117}
7118
Daniel Borkmann2a5418a2018-01-26 23:33:37 +01007119/* The verifier does more data flow analysis than llvm and will not
7120 * explore branches that are dead at run time. Malicious programs can
7121 * have dead code too. Therefore replace all dead at-run-time code
7122 * with 'ja -1'.
7123 *
7124 * Just nops are not optimal, e.g. if they would sit at the end of the
7125 * program and through another bug we would manage to jump there, then
7126 * we'd execute beyond program memory otherwise. Returning exception
7127 * code also wouldn't work since we can have subprogs where the dead
7128 * code could be located.
Alexei Starovoitovc1311872017-11-22 16:42:05 -08007129 */
7130static void sanitize_dead_code(struct bpf_verifier_env *env)
7131{
7132 struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
Daniel Borkmann2a5418a2018-01-26 23:33:37 +01007133 struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1);
Alexei Starovoitovc1311872017-11-22 16:42:05 -08007134 struct bpf_insn *insn = env->prog->insnsi;
7135 const int insn_cnt = env->prog->len;
7136 int i;
7137
7138 for (i = 0; i < insn_cnt; i++) {
7139 if (aux_data[i].seen)
7140 continue;
Daniel Borkmann2a5418a2018-01-26 23:33:37 +01007141 memcpy(insn + i, &trap, sizeof(trap));
Alexei Starovoitovc1311872017-11-22 16:42:05 -08007142 }
7143}
7144
Jakub Kicinskie2ae4ca2019-01-22 22:45:19 -08007145static bool insn_is_cond_jump(u8 code)
7146{
7147 u8 op;
7148
Jiong Wang092ed092019-01-26 12:26:01 -05007149 if (BPF_CLASS(code) == BPF_JMP32)
7150 return true;
7151
Jakub Kicinskie2ae4ca2019-01-22 22:45:19 -08007152 if (BPF_CLASS(code) != BPF_JMP)
7153 return false;
7154
7155 op = BPF_OP(code);
7156 return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL;
7157}
7158
7159static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env)
7160{
7161 struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
7162 struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0);
7163 struct bpf_insn *insn = env->prog->insnsi;
7164 const int insn_cnt = env->prog->len;
7165 int i;
7166
7167 for (i = 0; i < insn_cnt; i++, insn++) {
7168 if (!insn_is_cond_jump(insn->code))
7169 continue;
7170
7171 if (!aux_data[i + 1].seen)
7172 ja.off = insn->off;
7173 else if (!aux_data[i + 1 + insn->off].seen)
7174 ja.off = 0;
7175 else
7176 continue;
7177
Jakub Kicinski08ca90a2019-01-22 22:45:24 -08007178 if (bpf_prog_is_dev_bound(env->prog->aux))
7179 bpf_prog_offload_replace_insn(env, i, &ja);
7180
Jakub Kicinskie2ae4ca2019-01-22 22:45:19 -08007181 memcpy(insn, &ja, sizeof(ja));
7182 }
7183}
7184
Jakub Kicinski52875a02019-01-22 22:45:20 -08007185static int opt_remove_dead_code(struct bpf_verifier_env *env)
7186{
7187 struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
7188 int insn_cnt = env->prog->len;
7189 int i, err;
7190
7191 for (i = 0; i < insn_cnt; i++) {
7192 int j;
7193
7194 j = 0;
7195 while (i + j < insn_cnt && !aux_data[i + j].seen)
7196 j++;
7197 if (!j)
7198 continue;
7199
7200 err = verifier_remove_insns(env, i, j);
7201 if (err)
7202 return err;
7203 insn_cnt = env->prog->len;
7204 }
7205
7206 return 0;
7207}
7208
Jakub Kicinskia1b14ab2019-01-22 22:45:21 -08007209static int opt_remove_nops(struct bpf_verifier_env *env)
7210{
7211 const struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0);
7212 struct bpf_insn *insn = env->prog->insnsi;
7213 int insn_cnt = env->prog->len;
7214 int i, err;
7215
7216 for (i = 0; i < insn_cnt; i++) {
7217 if (memcmp(&insn[i], &ja, sizeof(ja)))
7218 continue;
7219
7220 err = verifier_remove_insns(env, i, 1);
7221 if (err)
7222 return err;
7223 insn_cnt--;
7224 i--;
7225 }
7226
7227 return 0;
7228}
7229
Joe Stringerc64b7982018-10-02 13:35:33 -07007230/* convert load instructions that access fields of a context type into a
7231 * sequence of instructions that access fields of the underlying structure:
7232 * struct __sk_buff -> struct sk_buff
7233 * struct bpf_sock_ops -> struct sock
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007234 */
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01007235static int convert_ctx_accesses(struct bpf_verifier_env *env)
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007236{
Jakub Kicinski00176a32017-10-16 16:40:54 -07007237 const struct bpf_verifier_ops *ops = env->ops;
Daniel Borkmannf96da092017-07-02 02:13:27 +02007238 int i, cnt, size, ctx_field_size, delta = 0;
Jakub Kicinski3df126f2016-09-21 11:43:56 +01007239 const int insn_cnt = env->prog->len;
Daniel Borkmann36bbef52016-09-20 00:26:13 +02007240 struct bpf_insn insn_buf[16], *insn;
Andrey Ignatov46f53a62018-11-10 22:15:13 -08007241 u32 target_size, size_default, off;
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007242 struct bpf_prog *new_prog;
Alexei Starovoitovd691f9e2015-06-04 10:11:54 -07007243 enum bpf_access_type type;
Daniel Borkmannf96da092017-07-02 02:13:27 +02007244 bool is_narrower_load;
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007245
Daniel Borkmannb09928b2018-10-24 22:05:49 +02007246 if (ops->gen_prologue || env->seen_direct_write) {
7247 if (!ops->gen_prologue) {
7248 verbose(env, "bpf verifier is misconfigured\n");
7249 return -EINVAL;
7250 }
Daniel Borkmann36bbef52016-09-20 00:26:13 +02007251 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write,
7252 env->prog);
7253 if (cnt >= ARRAY_SIZE(insn_buf)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07007254 verbose(env, "bpf verifier is misconfigured\n");
Daniel Borkmann36bbef52016-09-20 00:26:13 +02007255 return -EINVAL;
7256 } else if (cnt) {
Alexei Starovoitov80419022017-03-15 18:26:41 -07007257 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);
Daniel Borkmann36bbef52016-09-20 00:26:13 +02007258 if (!new_prog)
7259 return -ENOMEM;
Alexei Starovoitov80419022017-03-15 18:26:41 -07007260
Daniel Borkmann36bbef52016-09-20 00:26:13 +02007261 env->prog = new_prog;
Jakub Kicinski3df126f2016-09-21 11:43:56 +01007262 delta += cnt - 1;
Daniel Borkmann36bbef52016-09-20 00:26:13 +02007263 }
7264 }
7265
Joe Stringerc64b7982018-10-02 13:35:33 -07007266 if (bpf_prog_is_dev_bound(env->prog->aux))
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007267 return 0;
7268
Jakub Kicinski3df126f2016-09-21 11:43:56 +01007269 insn = env->prog->insnsi + delta;
Daniel Borkmann36bbef52016-09-20 00:26:13 +02007270
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007271 for (i = 0; i < insn_cnt; i++, insn++) {
Joe Stringerc64b7982018-10-02 13:35:33 -07007272 bpf_convert_ctx_access_t convert_ctx_access;
7273
Daniel Borkmann62c79892017-01-12 11:51:33 +01007274 if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) ||
7275 insn->code == (BPF_LDX | BPF_MEM | BPF_H) ||
7276 insn->code == (BPF_LDX | BPF_MEM | BPF_W) ||
Alexei Starovoitovea2e7ce2016-09-01 18:37:21 -07007277 insn->code == (BPF_LDX | BPF_MEM | BPF_DW))
Alexei Starovoitovd691f9e2015-06-04 10:11:54 -07007278 type = BPF_READ;
Daniel Borkmann62c79892017-01-12 11:51:33 +01007279 else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) ||
7280 insn->code == (BPF_STX | BPF_MEM | BPF_H) ||
7281 insn->code == (BPF_STX | BPF_MEM | BPF_W) ||
Alexei Starovoitovea2e7ce2016-09-01 18:37:21 -07007282 insn->code == (BPF_STX | BPF_MEM | BPF_DW))
Alexei Starovoitovd691f9e2015-06-04 10:11:54 -07007283 type = BPF_WRITE;
7284 else
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007285 continue;
7286
Alexei Starovoitovaf86ca42018-05-15 09:27:05 -07007287 if (type == BPF_WRITE &&
7288 env->insn_aux_data[i + delta].sanitize_stack_off) {
7289 struct bpf_insn patch[] = {
7290 /* Sanitize suspicious stack slot with zero.
7291 * There are no memory dependencies for this store,
7292 * since it's only using frame pointer and immediate
7293 * constant of zero
7294 */
7295 BPF_ST_MEM(BPF_DW, BPF_REG_FP,
7296 env->insn_aux_data[i + delta].sanitize_stack_off,
7297 0),
7298 /* the original STX instruction will immediately
7299 * overwrite the same stack slot with appropriate value
7300 */
7301 *insn,
7302 };
7303
7304 cnt = ARRAY_SIZE(patch);
7305 new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt);
7306 if (!new_prog)
7307 return -ENOMEM;
7308
7309 delta += cnt - 1;
7310 env->prog = new_prog;
7311 insn = new_prog->insnsi + i + delta;
7312 continue;
7313 }
7314
Joe Stringerc64b7982018-10-02 13:35:33 -07007315 switch (env->insn_aux_data[i + delta].ptr_type) {
7316 case PTR_TO_CTX:
7317 if (!ops->convert_ctx_access)
7318 continue;
7319 convert_ctx_access = ops->convert_ctx_access;
7320 break;
7321 case PTR_TO_SOCKET:
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08007322 case PTR_TO_SOCK_COMMON:
Joe Stringerc64b7982018-10-02 13:35:33 -07007323 convert_ctx_access = bpf_sock_convert_ctx_access;
7324 break;
Martin KaFai Lau655a51e2019-02-09 23:22:24 -08007325 case PTR_TO_TCP_SOCK:
7326 convert_ctx_access = bpf_tcp_sock_convert_ctx_access;
7327 break;
Joe Stringerc64b7982018-10-02 13:35:33 -07007328 default:
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007329 continue;
Joe Stringerc64b7982018-10-02 13:35:33 -07007330 }
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007331
Yonghong Song31fd8582017-06-13 15:52:13 -07007332 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size;
Daniel Borkmannf96da092017-07-02 02:13:27 +02007333 size = BPF_LDST_BYTES(insn);
Yonghong Song31fd8582017-06-13 15:52:13 -07007334
7335 /* If the read access is a narrower load of the field,
7336 * convert to a 4/8-byte load, to minimum program type specific
7337 * convert_ctx_access changes. If conversion is successful,
7338 * we will apply proper mask to the result.
7339 */
Daniel Borkmannf96da092017-07-02 02:13:27 +02007340 is_narrower_load = size < ctx_field_size;
Andrey Ignatov46f53a62018-11-10 22:15:13 -08007341 size_default = bpf_ctx_off_adjust_machine(ctx_field_size);
7342 off = insn->off;
Yonghong Song31fd8582017-06-13 15:52:13 -07007343 if (is_narrower_load) {
Daniel Borkmannf96da092017-07-02 02:13:27 +02007344 u8 size_code;
Yonghong Song31fd8582017-06-13 15:52:13 -07007345
Daniel Borkmannf96da092017-07-02 02:13:27 +02007346 if (type == BPF_WRITE) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07007347 verbose(env, "bpf verifier narrow ctx access misconfigured\n");
Daniel Borkmannf96da092017-07-02 02:13:27 +02007348 return -EINVAL;
7349 }
7350
7351 size_code = BPF_H;
Yonghong Song31fd8582017-06-13 15:52:13 -07007352 if (ctx_field_size == 4)
7353 size_code = BPF_W;
7354 else if (ctx_field_size == 8)
7355 size_code = BPF_DW;
Daniel Borkmannf96da092017-07-02 02:13:27 +02007356
Daniel Borkmannbc231052018-06-02 23:06:39 +02007357 insn->off = off & ~(size_default - 1);
Yonghong Song31fd8582017-06-13 15:52:13 -07007358 insn->code = BPF_LDX | BPF_MEM | size_code;
7359 }
Daniel Borkmannf96da092017-07-02 02:13:27 +02007360
7361 target_size = 0;
Joe Stringerc64b7982018-10-02 13:35:33 -07007362 cnt = convert_ctx_access(type, insn, insn_buf, env->prog,
7363 &target_size);
Daniel Borkmannf96da092017-07-02 02:13:27 +02007364 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf) ||
7365 (ctx_field_size && !target_size)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07007366 verbose(env, "bpf verifier is misconfigured\n");
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007367 return -EINVAL;
7368 }
Daniel Borkmannf96da092017-07-02 02:13:27 +02007369
7370 if (is_narrower_load && size < target_size) {
Andrey Ignatov46f53a62018-11-10 22:15:13 -08007371 u8 shift = (off & (size_default - 1)) * 8;
7372
7373 if (ctx_field_size <= 4) {
7374 if (shift)
7375 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH,
7376 insn->dst_reg,
7377 shift);
Yonghong Song31fd8582017-06-13 15:52:13 -07007378 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg,
Daniel Borkmannf96da092017-07-02 02:13:27 +02007379 (1 << size * 8) - 1);
Andrey Ignatov46f53a62018-11-10 22:15:13 -08007380 } else {
7381 if (shift)
7382 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH,
7383 insn->dst_reg,
7384 shift);
Yonghong Song31fd8582017-06-13 15:52:13 -07007385 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_AND, insn->dst_reg,
Daniel Borkmannf96da092017-07-02 02:13:27 +02007386 (1 << size * 8) - 1);
Andrey Ignatov46f53a62018-11-10 22:15:13 -08007387 }
Yonghong Song31fd8582017-06-13 15:52:13 -07007388 }
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007389
Alexei Starovoitov80419022017-03-15 18:26:41 -07007390 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007391 if (!new_prog)
7392 return -ENOMEM;
7393
Jakub Kicinski3df126f2016-09-21 11:43:56 +01007394 delta += cnt - 1;
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007395
7396 /* keep walking new program and skip insns we just inserted */
7397 env->prog = new_prog;
Jakub Kicinski3df126f2016-09-21 11:43:56 +01007398 insn = new_prog->insnsi + i + delta;
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007399 }
7400
7401 return 0;
7402}
7403
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007404static int jit_subprogs(struct bpf_verifier_env *env)
7405{
7406 struct bpf_prog *prog = env->prog, **func, *tmp;
7407 int i, j, subprog_start, subprog_end = 0, len, subprog;
Daniel Borkmann7105e822017-12-20 13:42:57 +01007408 struct bpf_insn *insn;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007409 void *old_bpf_func;
Martin KaFai Lauc454a462018-12-07 16:42:25 -08007410 int err;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007411
Jiong Wangf910cef2018-05-02 16:17:17 -04007412 if (env->subprog_cnt <= 1)
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007413 return 0;
7414
Daniel Borkmann7105e822017-12-20 13:42:57 +01007415 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007416 if (insn->code != (BPF_JMP | BPF_CALL) ||
7417 insn->src_reg != BPF_PSEUDO_CALL)
7418 continue;
Daniel Borkmannc7a89782018-07-12 21:44:28 +02007419 /* Upon error here we cannot fall back to interpreter but
7420 * need a hard reject of the program. Thus -EFAULT is
7421 * propagated in any case.
7422 */
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007423 subprog = find_subprog(env, i + insn->imm + 1);
7424 if (subprog < 0) {
7425 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
7426 i + insn->imm + 1);
7427 return -EFAULT;
7428 }
7429 /* temporarily remember subprog id inside insn instead of
7430 * aux_data, since next loop will split up all insns into funcs
7431 */
Jiong Wangf910cef2018-05-02 16:17:17 -04007432 insn->off = subprog;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007433 /* remember original imm in case JIT fails and fallback
7434 * to interpreter will be needed
7435 */
7436 env->insn_aux_data[i].call_imm = insn->imm;
7437 /* point imm to __bpf_call_base+1 from JITs point of view */
7438 insn->imm = 1;
7439 }
7440
Martin KaFai Lauc454a462018-12-07 16:42:25 -08007441 err = bpf_prog_alloc_jited_linfo(prog);
7442 if (err)
7443 goto out_undo_insn;
7444
7445 err = -ENOMEM;
Kees Cook6396bb22018-06-12 14:03:40 -07007446 func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL);
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007447 if (!func)
Daniel Borkmannc7a89782018-07-12 21:44:28 +02007448 goto out_undo_insn;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007449
Jiong Wangf910cef2018-05-02 16:17:17 -04007450 for (i = 0; i < env->subprog_cnt; i++) {
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007451 subprog_start = subprog_end;
Jiong Wang4cb3d992018-05-02 16:17:19 -04007452 subprog_end = env->subprog_info[i + 1].start;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007453
7454 len = subprog_end - subprog_start;
Alexei Starovoitov492ecee2019-02-25 14:28:39 -08007455 /* BPF_PROG_RUN doesn't call subprogs directly,
7456 * hence main prog stats include the runtime of subprogs.
7457 * subprogs don't have IDs and not reachable via prog_get_next_id
7458 * func[i]->aux->stats will never be accessed and stays NULL
7459 */
7460 func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER);
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007461 if (!func[i])
7462 goto out_free;
7463 memcpy(func[i]->insnsi, &prog->insnsi[subprog_start],
7464 len * sizeof(struct bpf_insn));
Daniel Borkmann4f74d802017-12-20 13:42:56 +01007465 func[i]->type = prog->type;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007466 func[i]->len = len;
Daniel Borkmann4f74d802017-12-20 13:42:56 +01007467 if (bpf_prog_calc_tag(func[i]))
7468 goto out_free;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007469 func[i]->is_func = 1;
Yonghong Songba64e7d2018-11-24 23:20:44 -08007470 func[i]->aux->func_idx = i;
7471 /* the btf and func_info will be freed only at prog->aux */
7472 func[i]->aux->btf = prog->aux->btf;
7473 func[i]->aux->func_info = prog->aux->func_info;
7474
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007475 /* Use bpf_prog_F_tag to indicate functions in stack traces.
7476 * Long term would need debug info to populate names
7477 */
7478 func[i]->aux->name[0] = 'F';
Jiong Wang9c8105b2018-05-02 16:17:18 -04007479 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007480 func[i]->jit_requested = 1;
Martin KaFai Lauc454a462018-12-07 16:42:25 -08007481 func[i]->aux->linfo = prog->aux->linfo;
7482 func[i]->aux->nr_linfo = prog->aux->nr_linfo;
7483 func[i]->aux->jited_linfo = prog->aux->jited_linfo;
7484 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007485 func[i] = bpf_int_jit_compile(func[i]);
7486 if (!func[i]->jited) {
7487 err = -ENOTSUPP;
7488 goto out_free;
7489 }
7490 cond_resched();
7491 }
7492 /* at this point all bpf functions were successfully JITed
7493 * now populate all bpf_calls with correct addresses and
7494 * run last pass of JIT
7495 */
Jiong Wangf910cef2018-05-02 16:17:17 -04007496 for (i = 0; i < env->subprog_cnt; i++) {
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007497 insn = func[i]->insnsi;
7498 for (j = 0; j < func[i]->len; j++, insn++) {
7499 if (insn->code != (BPF_JMP | BPF_CALL) ||
7500 insn->src_reg != BPF_PSEUDO_CALL)
7501 continue;
7502 subprog = insn->off;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007503 insn->imm = (u64 (*)(u64, u64, u64, u64, u64))
7504 func[subprog]->bpf_func -
7505 __bpf_call_base;
7506 }
Sandipan Das2162fed2018-05-24 12:26:45 +05307507
7508 /* we use the aux data to keep a list of the start addresses
7509 * of the JITed images for each function in the program
7510 *
7511 * for some architectures, such as powerpc64, the imm field
7512 * might not be large enough to hold the offset of the start
7513 * address of the callee's JITed image from __bpf_call_base
7514 *
7515 * in such cases, we can lookup the start address of a callee
7516 * by using its subprog id, available from the off field of
7517 * the call instruction, as an index for this list
7518 */
7519 func[i]->aux->func = func;
7520 func[i]->aux->func_cnt = env->subprog_cnt;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007521 }
Jiong Wangf910cef2018-05-02 16:17:17 -04007522 for (i = 0; i < env->subprog_cnt; i++) {
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007523 old_bpf_func = func[i]->bpf_func;
7524 tmp = bpf_int_jit_compile(func[i]);
7525 if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) {
7526 verbose(env, "JIT doesn't support bpf-to-bpf calls\n");
Daniel Borkmannc7a89782018-07-12 21:44:28 +02007527 err = -ENOTSUPP;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007528 goto out_free;
7529 }
7530 cond_resched();
7531 }
7532
7533 /* finally lock prog and jit images for all functions and
7534 * populate kallsysm
7535 */
Jiong Wangf910cef2018-05-02 16:17:17 -04007536 for (i = 0; i < env->subprog_cnt; i++) {
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007537 bpf_prog_lock_ro(func[i]);
7538 bpf_prog_kallsyms_add(func[i]);
7539 }
Daniel Borkmann7105e822017-12-20 13:42:57 +01007540
7541 /* Last step: make now unused interpreter insns from main
7542 * prog consistent for later dump requests, so they can
7543 * later look the same as if they were interpreted only.
7544 */
7545 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
Daniel Borkmann7105e822017-12-20 13:42:57 +01007546 if (insn->code != (BPF_JMP | BPF_CALL) ||
7547 insn->src_reg != BPF_PSEUDO_CALL)
7548 continue;
7549 insn->off = env->insn_aux_data[i].call_imm;
7550 subprog = find_subprog(env, i + insn->off + 1);
Sandipan Dasdbecd732018-05-24 12:26:48 +05307551 insn->imm = subprog;
Daniel Borkmann7105e822017-12-20 13:42:57 +01007552 }
7553
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007554 prog->jited = 1;
7555 prog->bpf_func = func[0]->bpf_func;
7556 prog->aux->func = func;
Jiong Wangf910cef2018-05-02 16:17:17 -04007557 prog->aux->func_cnt = env->subprog_cnt;
Martin KaFai Lauc454a462018-12-07 16:42:25 -08007558 bpf_prog_free_unused_jited_linfo(prog);
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007559 return 0;
7560out_free:
Jiong Wangf910cef2018-05-02 16:17:17 -04007561 for (i = 0; i < env->subprog_cnt; i++)
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007562 if (func[i])
7563 bpf_jit_free(func[i]);
7564 kfree(func);
Daniel Borkmannc7a89782018-07-12 21:44:28 +02007565out_undo_insn:
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007566 /* cleanup main prog to be interpreted */
7567 prog->jit_requested = 0;
7568 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
7569 if (insn->code != (BPF_JMP | BPF_CALL) ||
7570 insn->src_reg != BPF_PSEUDO_CALL)
7571 continue;
7572 insn->off = 0;
7573 insn->imm = env->insn_aux_data[i].call_imm;
7574 }
Martin KaFai Lauc454a462018-12-07 16:42:25 -08007575 bpf_prog_free_jited_linfo(prog);
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007576 return err;
7577}
7578
Alexei Starovoitov1ea47e02017-12-14 17:55:13 -08007579static int fixup_call_args(struct bpf_verifier_env *env)
7580{
David S. Miller19d28fb2018-01-11 21:27:54 -05007581#ifndef CONFIG_BPF_JIT_ALWAYS_ON
Alexei Starovoitov1ea47e02017-12-14 17:55:13 -08007582 struct bpf_prog *prog = env->prog;
7583 struct bpf_insn *insn = prog->insnsi;
7584 int i, depth;
David S. Miller19d28fb2018-01-11 21:27:54 -05007585#endif
Quentin Monnete4052d02018-10-07 12:56:58 +01007586 int err = 0;
Alexei Starovoitov1ea47e02017-12-14 17:55:13 -08007587
Quentin Monnete4052d02018-10-07 12:56:58 +01007588 if (env->prog->jit_requested &&
7589 !bpf_prog_is_dev_bound(env->prog->aux)) {
David S. Miller19d28fb2018-01-11 21:27:54 -05007590 err = jit_subprogs(env);
7591 if (err == 0)
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007592 return 0;
Daniel Borkmannc7a89782018-07-12 21:44:28 +02007593 if (err == -EFAULT)
7594 return err;
David S. Miller19d28fb2018-01-11 21:27:54 -05007595 }
7596#ifndef CONFIG_BPF_JIT_ALWAYS_ON
Alexei Starovoitov1ea47e02017-12-14 17:55:13 -08007597 for (i = 0; i < prog->len; i++, insn++) {
7598 if (insn->code != (BPF_JMP | BPF_CALL) ||
7599 insn->src_reg != BPF_PSEUDO_CALL)
7600 continue;
7601 depth = get_callee_stack_depth(env, insn, i);
7602 if (depth < 0)
7603 return depth;
7604 bpf_patch_call_args(insn, depth);
7605 }
David S. Miller19d28fb2018-01-11 21:27:54 -05007606 err = 0;
7607#endif
7608 return err;
Alexei Starovoitov1ea47e02017-12-14 17:55:13 -08007609}
7610
Alexei Starovoitov79741b32017-03-15 18:26:40 -07007611/* fixup insn->imm field of bpf_call instructions
Alexei Starovoitov81ed18a2017-03-15 18:26:42 -07007612 * and inline eligible helpers as explicit sequence of BPF instructions
Alexei Starovoitove245c5c62017-03-15 18:26:39 -07007613 *
7614 * this function is called after eBPF program passed verification
7615 */
Alexei Starovoitov79741b32017-03-15 18:26:40 -07007616static int fixup_bpf_calls(struct bpf_verifier_env *env)
Alexei Starovoitove245c5c62017-03-15 18:26:39 -07007617{
Alexei Starovoitov79741b32017-03-15 18:26:40 -07007618 struct bpf_prog *prog = env->prog;
7619 struct bpf_insn *insn = prog->insnsi;
Alexei Starovoitove245c5c62017-03-15 18:26:39 -07007620 const struct bpf_func_proto *fn;
Alexei Starovoitov79741b32017-03-15 18:26:40 -07007621 const int insn_cnt = prog->len;
Daniel Borkmann09772d92018-06-02 23:06:35 +02007622 const struct bpf_map_ops *ops;
Daniel Borkmannc93552c2018-05-24 02:32:53 +02007623 struct bpf_insn_aux_data *aux;
Alexei Starovoitov81ed18a2017-03-15 18:26:42 -07007624 struct bpf_insn insn_buf[16];
7625 struct bpf_prog *new_prog;
7626 struct bpf_map *map_ptr;
7627 int i, cnt, delta = 0;
Alexei Starovoitove245c5c62017-03-15 18:26:39 -07007628
Alexei Starovoitov79741b32017-03-15 18:26:40 -07007629 for (i = 0; i < insn_cnt; i++, insn++) {
Daniel Borkmannf6b1b3b2018-01-26 23:33:39 +01007630 if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) ||
7631 insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) ||
7632 insn->code == (BPF_ALU | BPF_MOD | BPF_X) ||
Alexei Starovoitov68fda452018-01-12 18:59:52 -08007633 insn->code == (BPF_ALU | BPF_DIV | BPF_X)) {
Daniel Borkmannf6b1b3b2018-01-26 23:33:39 +01007634 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;
7635 struct bpf_insn mask_and_div[] = {
7636 BPF_MOV32_REG(insn->src_reg, insn->src_reg),
7637 /* Rx div 0 -> 0 */
7638 BPF_JMP_IMM(BPF_JNE, insn->src_reg, 0, 2),
7639 BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg),
7640 BPF_JMP_IMM(BPF_JA, 0, 0, 1),
7641 *insn,
7642 };
7643 struct bpf_insn mask_and_mod[] = {
7644 BPF_MOV32_REG(insn->src_reg, insn->src_reg),
7645 /* Rx mod 0 -> Rx */
7646 BPF_JMP_IMM(BPF_JEQ, insn->src_reg, 0, 1),
7647 *insn,
7648 };
7649 struct bpf_insn *patchlet;
7650
7651 if (insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) ||
7652 insn->code == (BPF_ALU | BPF_DIV | BPF_X)) {
7653 patchlet = mask_and_div + (is64 ? 1 : 0);
7654 cnt = ARRAY_SIZE(mask_and_div) - (is64 ? 1 : 0);
7655 } else {
7656 patchlet = mask_and_mod + (is64 ? 1 : 0);
7657 cnt = ARRAY_SIZE(mask_and_mod) - (is64 ? 1 : 0);
7658 }
7659
7660 new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt);
Alexei Starovoitov68fda452018-01-12 18:59:52 -08007661 if (!new_prog)
7662 return -ENOMEM;
7663
7664 delta += cnt - 1;
7665 env->prog = prog = new_prog;
7666 insn = new_prog->insnsi + i + delta;
7667 continue;
7668 }
7669
Daniel Borkmanne0cea7c2018-05-04 01:08:14 +02007670 if (BPF_CLASS(insn->code) == BPF_LD &&
7671 (BPF_MODE(insn->code) == BPF_ABS ||
7672 BPF_MODE(insn->code) == BPF_IND)) {
7673 cnt = env->ops->gen_ld_abs(insn, insn_buf);
7674 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) {
7675 verbose(env, "bpf verifier is misconfigured\n");
7676 return -EINVAL;
7677 }
7678
7679 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
7680 if (!new_prog)
7681 return -ENOMEM;
7682
7683 delta += cnt - 1;
7684 env->prog = prog = new_prog;
7685 insn = new_prog->insnsi + i + delta;
7686 continue;
7687 }
7688
Daniel Borkmann979d63d2019-01-03 00:58:34 +01007689 if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) ||
7690 insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) {
7691 const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X;
7692 const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X;
7693 struct bpf_insn insn_buf[16];
7694 struct bpf_insn *patch = &insn_buf[0];
7695 bool issrc, isneg;
7696 u32 off_reg;
7697
7698 aux = &env->insn_aux_data[i + delta];
Daniel Borkmann3612af72019-03-01 22:05:29 +01007699 if (!aux->alu_state ||
7700 aux->alu_state == BPF_ALU_NON_POINTER)
Daniel Borkmann979d63d2019-01-03 00:58:34 +01007701 continue;
7702
7703 isneg = aux->alu_state & BPF_ALU_NEG_VALUE;
7704 issrc = (aux->alu_state & BPF_ALU_SANITIZE) ==
7705 BPF_ALU_SANITIZE_SRC;
7706
7707 off_reg = issrc ? insn->src_reg : insn->dst_reg;
7708 if (isneg)
7709 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
7710 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit - 1);
7711 *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg);
7712 *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg);
7713 *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0);
7714 *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63);
7715 if (issrc) {
7716 *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX,
7717 off_reg);
7718 insn->src_reg = BPF_REG_AX;
7719 } else {
7720 *patch++ = BPF_ALU64_REG(BPF_AND, off_reg,
7721 BPF_REG_AX);
7722 }
7723 if (isneg)
7724 insn->code = insn->code == code_add ?
7725 code_sub : code_add;
7726 *patch++ = *insn;
7727 if (issrc && isneg)
7728 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
7729 cnt = patch - insn_buf;
7730
7731 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
7732 if (!new_prog)
7733 return -ENOMEM;
7734
7735 delta += cnt - 1;
7736 env->prog = prog = new_prog;
7737 insn = new_prog->insnsi + i + delta;
7738 continue;
7739 }
7740
Alexei Starovoitov79741b32017-03-15 18:26:40 -07007741 if (insn->code != (BPF_JMP | BPF_CALL))
7742 continue;
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08007743 if (insn->src_reg == BPF_PSEUDO_CALL)
7744 continue;
Alexei Starovoitove245c5c62017-03-15 18:26:39 -07007745
Alexei Starovoitov79741b32017-03-15 18:26:40 -07007746 if (insn->imm == BPF_FUNC_get_route_realm)
7747 prog->dst_needed = 1;
7748 if (insn->imm == BPF_FUNC_get_prandom_u32)
7749 bpf_user_rnd_init_once();
Josef Bacik9802d862017-12-11 11:36:48 -05007750 if (insn->imm == BPF_FUNC_override_return)
7751 prog->kprobe_override = 1;
Alexei Starovoitov79741b32017-03-15 18:26:40 -07007752 if (insn->imm == BPF_FUNC_tail_call) {
David S. Miller7b9f6da2017-04-20 10:35:33 -04007753 /* If we tail call into other programs, we
7754 * cannot make any assumptions since they can
7755 * be replaced dynamically during runtime in
7756 * the program array.
7757 */
7758 prog->cb_access = 1;
Alexei Starovoitov80a58d02017-05-30 13:31:30 -07007759 env->prog->aux->stack_depth = MAX_BPF_STACK;
Jiong Wange6478152018-11-08 04:08:42 -05007760 env->prog->aux->max_pkt_offset = MAX_PACKET_OFF;
David S. Miller7b9f6da2017-04-20 10:35:33 -04007761
Alexei Starovoitov79741b32017-03-15 18:26:40 -07007762 /* mark bpf_tail_call as different opcode to avoid
7763 * conditional branch in the interpeter for every normal
7764 * call and to prevent accidental JITing by JIT compiler
7765 * that doesn't support bpf_tail_call yet
Alexei Starovoitove245c5c62017-03-15 18:26:39 -07007766 */
Alexei Starovoitov79741b32017-03-15 18:26:40 -07007767 insn->imm = 0;
Alexei Starovoitov71189fa2017-05-30 13:31:27 -07007768 insn->code = BPF_JMP | BPF_TAIL_CALL;
Alexei Starovoitovb2157392018-01-07 17:33:02 -08007769
Daniel Borkmannc93552c2018-05-24 02:32:53 +02007770 aux = &env->insn_aux_data[i + delta];
7771 if (!bpf_map_ptr_unpriv(aux))
7772 continue;
7773
Alexei Starovoitovb2157392018-01-07 17:33:02 -08007774 /* instead of changing every JIT dealing with tail_call
7775 * emit two extra insns:
7776 * if (index >= max_entries) goto out;
7777 * index &= array->index_mask;
7778 * to avoid out-of-bounds cpu speculation
7779 */
Daniel Borkmannc93552c2018-05-24 02:32:53 +02007780 if (bpf_map_ptr_poisoned(aux)) {
Colin Ian King40950342018-01-10 09:20:54 +00007781 verbose(env, "tail_call abusing map_ptr\n");
Alexei Starovoitovb2157392018-01-07 17:33:02 -08007782 return -EINVAL;
7783 }
Daniel Borkmannc93552c2018-05-24 02:32:53 +02007784
7785 map_ptr = BPF_MAP_PTR(aux->map_state);
Alexei Starovoitovb2157392018-01-07 17:33:02 -08007786 insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3,
7787 map_ptr->max_entries, 2);
7788 insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3,
7789 container_of(map_ptr,
7790 struct bpf_array,
7791 map)->index_mask);
7792 insn_buf[2] = *insn;
7793 cnt = 3;
7794 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
7795 if (!new_prog)
7796 return -ENOMEM;
7797
7798 delta += cnt - 1;
7799 env->prog = prog = new_prog;
7800 insn = new_prog->insnsi + i + delta;
Alexei Starovoitov79741b32017-03-15 18:26:40 -07007801 continue;
Alexei Starovoitove245c5c62017-03-15 18:26:39 -07007802 }
Alexei Starovoitove245c5c62017-03-15 18:26:39 -07007803
Daniel Borkmann89c63072017-08-19 03:12:45 +02007804 /* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup
Daniel Borkmann09772d92018-06-02 23:06:35 +02007805 * and other inlining handlers are currently limited to 64 bit
7806 * only.
Daniel Borkmann89c63072017-08-19 03:12:45 +02007807 */
Alexei Starovoitov60b58afc2017-12-14 17:55:14 -08007808 if (prog->jit_requested && BITS_PER_LONG == 64 &&
Daniel Borkmann09772d92018-06-02 23:06:35 +02007809 (insn->imm == BPF_FUNC_map_lookup_elem ||
7810 insn->imm == BPF_FUNC_map_update_elem ||
Daniel Borkmann84430d42018-10-21 02:09:27 +02007811 insn->imm == BPF_FUNC_map_delete_elem ||
7812 insn->imm == BPF_FUNC_map_push_elem ||
7813 insn->imm == BPF_FUNC_map_pop_elem ||
7814 insn->imm == BPF_FUNC_map_peek_elem)) {
Daniel Borkmannc93552c2018-05-24 02:32:53 +02007815 aux = &env->insn_aux_data[i + delta];
7816 if (bpf_map_ptr_poisoned(aux))
7817 goto patch_call_imm;
7818
7819 map_ptr = BPF_MAP_PTR(aux->map_state);
Daniel Borkmann09772d92018-06-02 23:06:35 +02007820 ops = map_ptr->ops;
7821 if (insn->imm == BPF_FUNC_map_lookup_elem &&
7822 ops->map_gen_lookup) {
7823 cnt = ops->map_gen_lookup(map_ptr, insn_buf);
7824 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) {
7825 verbose(env, "bpf verifier is misconfigured\n");
7826 return -EINVAL;
7827 }
Alexei Starovoitov81ed18a2017-03-15 18:26:42 -07007828
Daniel Borkmann09772d92018-06-02 23:06:35 +02007829 new_prog = bpf_patch_insn_data(env, i + delta,
7830 insn_buf, cnt);
7831 if (!new_prog)
7832 return -ENOMEM;
7833
7834 delta += cnt - 1;
7835 env->prog = prog = new_prog;
7836 insn = new_prog->insnsi + i + delta;
7837 continue;
Alexei Starovoitov81ed18a2017-03-15 18:26:42 -07007838 }
7839
Daniel Borkmann09772d92018-06-02 23:06:35 +02007840 BUILD_BUG_ON(!__same_type(ops->map_lookup_elem,
7841 (void *(*)(struct bpf_map *map, void *key))NULL));
7842 BUILD_BUG_ON(!__same_type(ops->map_delete_elem,
7843 (int (*)(struct bpf_map *map, void *key))NULL));
7844 BUILD_BUG_ON(!__same_type(ops->map_update_elem,
7845 (int (*)(struct bpf_map *map, void *key, void *value,
7846 u64 flags))NULL));
Daniel Borkmann84430d42018-10-21 02:09:27 +02007847 BUILD_BUG_ON(!__same_type(ops->map_push_elem,
7848 (int (*)(struct bpf_map *map, void *value,
7849 u64 flags))NULL));
7850 BUILD_BUG_ON(!__same_type(ops->map_pop_elem,
7851 (int (*)(struct bpf_map *map, void *value))NULL));
7852 BUILD_BUG_ON(!__same_type(ops->map_peek_elem,
7853 (int (*)(struct bpf_map *map, void *value))NULL));
7854
Daniel Borkmann09772d92018-06-02 23:06:35 +02007855 switch (insn->imm) {
7856 case BPF_FUNC_map_lookup_elem:
7857 insn->imm = BPF_CAST_CALL(ops->map_lookup_elem) -
7858 __bpf_call_base;
7859 continue;
7860 case BPF_FUNC_map_update_elem:
7861 insn->imm = BPF_CAST_CALL(ops->map_update_elem) -
7862 __bpf_call_base;
7863 continue;
7864 case BPF_FUNC_map_delete_elem:
7865 insn->imm = BPF_CAST_CALL(ops->map_delete_elem) -
7866 __bpf_call_base;
7867 continue;
Daniel Borkmann84430d42018-10-21 02:09:27 +02007868 case BPF_FUNC_map_push_elem:
7869 insn->imm = BPF_CAST_CALL(ops->map_push_elem) -
7870 __bpf_call_base;
7871 continue;
7872 case BPF_FUNC_map_pop_elem:
7873 insn->imm = BPF_CAST_CALL(ops->map_pop_elem) -
7874 __bpf_call_base;
7875 continue;
7876 case BPF_FUNC_map_peek_elem:
7877 insn->imm = BPF_CAST_CALL(ops->map_peek_elem) -
7878 __bpf_call_base;
7879 continue;
Daniel Borkmann09772d92018-06-02 23:06:35 +02007880 }
Alexei Starovoitov81ed18a2017-03-15 18:26:42 -07007881
Daniel Borkmann09772d92018-06-02 23:06:35 +02007882 goto patch_call_imm;
Alexei Starovoitov81ed18a2017-03-15 18:26:42 -07007883 }
7884
7885patch_call_imm:
Andrey Ignatov5e43f892018-03-30 15:08:00 -07007886 fn = env->ops->get_func_proto(insn->imm, env->prog);
Alexei Starovoitov79741b32017-03-15 18:26:40 -07007887 /* all functions that have prototype and verifier allowed
7888 * programs to call them, must be real in-kernel functions
7889 */
7890 if (!fn->func) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07007891 verbose(env,
7892 "kernel subsystem misconfigured func %s#%d\n",
Alexei Starovoitov79741b32017-03-15 18:26:40 -07007893 func_id_name(insn->imm), insn->imm);
7894 return -EFAULT;
7895 }
7896 insn->imm = fn->func - __bpf_call_base;
7897 }
7898
7899 return 0;
7900}
Alexei Starovoitove245c5c62017-03-15 18:26:39 -07007901
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01007902static void free_states(struct bpf_verifier_env *env)
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07007903{
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01007904 struct bpf_verifier_state_list *sl, *sln;
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07007905 int i;
7906
Alexei Starovoitov9f4686c2019-04-01 21:27:41 -07007907 sl = env->free_list;
7908 while (sl) {
7909 sln = sl->next;
7910 free_verifier_state(&sl->state, false);
7911 kfree(sl);
7912 sl = sln;
7913 }
7914
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07007915 if (!env->explored_states)
7916 return;
7917
7918 for (i = 0; i < env->prog->len; i++) {
7919 sl = env->explored_states[i];
7920
7921 if (sl)
7922 while (sl != STATE_LIST_MARK) {
7923 sln = sl->next;
Alexei Starovoitov1969db42017-11-01 00:08:04 -07007924 free_verifier_state(&sl->state, false);
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07007925 kfree(sl);
7926 sl = sln;
7927 }
7928 }
7929
Alexei Starovoitov71dde682019-04-01 21:27:43 -07007930 kvfree(env->explored_states);
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07007931}
7932
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07007933static void print_verification_stats(struct bpf_verifier_env *env)
7934{
7935 int i;
7936
7937 if (env->log.level & BPF_LOG_STATS) {
7938 verbose(env, "verification time %lld usec\n",
7939 div_u64(env->verification_time, 1000));
7940 verbose(env, "stack depth ");
7941 for (i = 0; i < env->subprog_cnt; i++) {
7942 u32 depth = env->subprog_info[i].stack_depth;
7943
7944 verbose(env, "%d", depth);
7945 if (i + 1 < env->subprog_cnt)
7946 verbose(env, "+");
7947 }
7948 verbose(env, "\n");
7949 }
7950 verbose(env, "processed %d insns (limit %d) max_states_per_insn %d "
7951 "total_states %d peak_states %d mark_read %d\n",
7952 env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS,
7953 env->max_states_per_insn, env->total_states,
7954 env->peak_states, env->longest_mark_read_walk);
7955}
7956
Yonghong Song838e9692018-11-19 15:29:11 -08007957int bpf_check(struct bpf_prog **prog, union bpf_attr *attr,
7958 union bpf_attr __user *uattr)
Alexei Starovoitov51580e72014-09-26 00:17:02 -07007959{
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07007960 u64 start_time = ktime_get_ns();
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01007961 struct bpf_verifier_env *env;
Martin KaFai Laub9193c12018-03-24 11:44:22 -07007962 struct bpf_verifier_log *log;
Jakub Kicinski9e4c24e2019-01-22 22:45:23 -08007963 int i, len, ret = -EINVAL;
Jakub Kicinskie2ae4ca2019-01-22 22:45:19 -08007964 bool is_priv;
Alexei Starovoitov51580e72014-09-26 00:17:02 -07007965
Arnd Bergmanneba0c922017-11-02 12:05:52 +01007966 /* no program is valid */
7967 if (ARRAY_SIZE(bpf_verifier_ops) == 0)
7968 return -EINVAL;
7969
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01007970 /* 'struct bpf_verifier_env' can be global, but since it's not small,
Alexei Starovoitovcbd35702014-09-26 00:17:03 -07007971 * allocate/free it every time bpf_check() is called
7972 */
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01007973 env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL);
Alexei Starovoitovcbd35702014-09-26 00:17:03 -07007974 if (!env)
7975 return -ENOMEM;
Jakub Kicinski61bd5212017-10-09 10:30:11 -07007976 log = &env->log;
Alexei Starovoitovcbd35702014-09-26 00:17:03 -07007977
Jakub Kicinski9e4c24e2019-01-22 22:45:23 -08007978 len = (*prog)->len;
Kees Cookfad953c2018-06-12 14:27:37 -07007979 env->insn_aux_data =
Jakub Kicinski9e4c24e2019-01-22 22:45:23 -08007980 vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len));
Jakub Kicinski3df126f2016-09-21 11:43:56 +01007981 ret = -ENOMEM;
7982 if (!env->insn_aux_data)
7983 goto err_free_env;
Jakub Kicinski9e4c24e2019-01-22 22:45:23 -08007984 for (i = 0; i < len; i++)
7985 env->insn_aux_data[i].orig_idx = i;
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007986 env->prog = *prog;
Jakub Kicinski00176a32017-10-16 16:40:54 -07007987 env->ops = bpf_verifier_ops[env->prog->type];
Alexei Starovoitov0246e642014-09-26 00:17:04 -07007988
Alexei Starovoitovcbd35702014-09-26 00:17:03 -07007989 /* grab the mutex to protect few globals used by verifier */
7990 mutex_lock(&bpf_verifier_lock);
7991
7992 if (attr->log_level || attr->log_buf || attr->log_size) {
7993 /* user requested verbose verifier output
7994 * and supplied buffer to store the verification trace
7995 */
Jakub Kicinskie7bf8242017-10-09 10:30:10 -07007996 log->level = attr->log_level;
7997 log->ubuf = (char __user *) (unsigned long) attr->log_buf;
7998 log->len_total = attr->log_size;
Alexei Starovoitovcbd35702014-09-26 00:17:03 -07007999
8000 ret = -EINVAL;
Jakub Kicinskie7bf8242017-10-09 10:30:10 -07008001 /* log attributes have to be sane */
Alexei Starovoitov7a9f5c62019-04-01 21:27:46 -07008002 if (log->len_total < 128 || log->len_total > UINT_MAX >> 2 ||
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07008003 !log->level || !log->ubuf || log->level & ~BPF_LOG_MASK)
Jakub Kicinski3df126f2016-09-21 11:43:56 +01008004 goto err_unlock;
Alexei Starovoitovcbd35702014-09-26 00:17:03 -07008005 }
Daniel Borkmann1ad2f582017-05-25 01:05:05 +02008006
8007 env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT);
8008 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
David S. Millere07b98d2017-05-10 11:38:07 -07008009 env->strict_alignment = true;
David Millere9ee9ef2018-11-30 21:08:14 -08008010 if (attr->prog_flags & BPF_F_ANY_ALIGNMENT)
8011 env->strict_alignment = false;
Alexei Starovoitovcbd35702014-09-26 00:17:03 -07008012
Jakub Kicinskie2ae4ca2019-01-22 22:45:19 -08008013 is_priv = capable(CAP_SYS_ADMIN);
8014 env->allow_ptr_leaks = is_priv;
8015
Alexei Starovoitov0246e642014-09-26 00:17:04 -07008016 ret = replace_map_fd_with_map_ptr(env);
8017 if (ret < 0)
8018 goto skip_full_check;
8019
Jakub Kicinskif4e3ec02018-05-03 18:37:11 -07008020 if (bpf_prog_is_dev_bound(env->prog->aux)) {
Quentin Monneta40a2632018-11-09 13:03:31 +00008021 ret = bpf_prog_offload_verifier_prep(env->prog);
Jakub Kicinskif4e3ec02018-05-03 18:37:11 -07008022 if (ret)
8023 goto skip_full_check;
8024 }
8025
Alexei Starovoitov71dde682019-04-01 21:27:43 -07008026 env->explored_states = kvcalloc(env->prog->len,
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01008027 sizeof(struct bpf_verifier_state_list *),
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07008028 GFP_USER);
8029 ret = -ENOMEM;
8030 if (!env->explored_states)
8031 goto skip_full_check;
8032
Martin KaFai Laud9762e82018-12-13 10:41:48 -08008033 ret = check_subprogs(env);
Alexei Starovoitov475fb782014-09-26 00:17:05 -07008034 if (ret < 0)
8035 goto skip_full_check;
8036
Martin KaFai Lauc454a462018-12-07 16:42:25 -08008037 ret = check_btf_info(env, attr, uattr);
Yonghong Song838e9692018-11-19 15:29:11 -08008038 if (ret < 0)
8039 goto skip_full_check;
8040
Martin KaFai Laud9762e82018-12-13 10:41:48 -08008041 ret = check_cfg(env);
8042 if (ret < 0)
8043 goto skip_full_check;
8044
Alexei Starovoitov17a52672014-09-26 00:17:06 -07008045 ret = do_check(env);
Craig Gallek8c01c4f2017-11-02 11:18:01 -04008046 if (env->cur_state) {
8047 free_verifier_state(env->cur_state, true);
8048 env->cur_state = NULL;
8049 }
Alexei Starovoitovcbd35702014-09-26 00:17:03 -07008050
Quentin Monnetc941ce92018-10-07 12:56:47 +01008051 if (ret == 0 && bpf_prog_is_dev_bound(env->prog->aux))
8052 ret = bpf_prog_offload_finalize(env);
8053
Alexei Starovoitov0246e642014-09-26 00:17:04 -07008054skip_full_check:
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07008055 while (!pop_stack(env, NULL, NULL));
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07008056 free_states(env);
Alexei Starovoitov0246e642014-09-26 00:17:04 -07008057
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07008058 if (ret == 0)
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08008059 ret = check_max_stack_depth(env);
8060
Jakub Kicinski9b38c402018-12-19 22:13:06 -08008061 /* instruction rewrites happen after this point */
Jakub Kicinskie2ae4ca2019-01-22 22:45:19 -08008062 if (is_priv) {
8063 if (ret == 0)
8064 opt_hard_wire_dead_code_branches(env);
Jakub Kicinski52875a02019-01-22 22:45:20 -08008065 if (ret == 0)
8066 ret = opt_remove_dead_code(env);
Jakub Kicinskia1b14ab2019-01-22 22:45:21 -08008067 if (ret == 0)
8068 ret = opt_remove_nops(env);
Jakub Kicinski52875a02019-01-22 22:45:20 -08008069 } else {
8070 if (ret == 0)
8071 sanitize_dead_code(env);
Jakub Kicinskie2ae4ca2019-01-22 22:45:19 -08008072 }
8073
Jakub Kicinski9b38c402018-12-19 22:13:06 -08008074 if (ret == 0)
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07008075 /* program is valid, convert *(u32*)(ctx + off) accesses */
8076 ret = convert_ctx_accesses(env);
8077
Alexei Starovoitove245c5c62017-03-15 18:26:39 -07008078 if (ret == 0)
Alexei Starovoitov79741b32017-03-15 18:26:40 -07008079 ret = fixup_bpf_calls(env);
Alexei Starovoitove245c5c62017-03-15 18:26:39 -07008080
Alexei Starovoitov1ea47e02017-12-14 17:55:13 -08008081 if (ret == 0)
8082 ret = fixup_call_args(env);
8083
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07008084 env->verification_time = ktime_get_ns() - start_time;
8085 print_verification_stats(env);
8086
Jakub Kicinskia2a7d572017-10-09 10:30:15 -07008087 if (log->level && bpf_verifier_log_full(log))
Alexei Starovoitovcbd35702014-09-26 00:17:03 -07008088 ret = -ENOSPC;
Jakub Kicinskia2a7d572017-10-09 10:30:15 -07008089 if (log->level && !log->ubuf) {
Alexei Starovoitovcbd35702014-09-26 00:17:03 -07008090 ret = -EFAULT;
Jakub Kicinskia2a7d572017-10-09 10:30:15 -07008091 goto err_release_maps;
Alexei Starovoitovcbd35702014-09-26 00:17:03 -07008092 }
8093
Alexei Starovoitov0246e642014-09-26 00:17:04 -07008094 if (ret == 0 && env->used_map_cnt) {
8095 /* if program passed verifier, update used_maps in bpf_prog_info */
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07008096 env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt,
8097 sizeof(env->used_maps[0]),
8098 GFP_KERNEL);
Alexei Starovoitov0246e642014-09-26 00:17:04 -07008099
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07008100 if (!env->prog->aux->used_maps) {
Alexei Starovoitov0246e642014-09-26 00:17:04 -07008101 ret = -ENOMEM;
Jakub Kicinskia2a7d572017-10-09 10:30:15 -07008102 goto err_release_maps;
Alexei Starovoitov0246e642014-09-26 00:17:04 -07008103 }
8104
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07008105 memcpy(env->prog->aux->used_maps, env->used_maps,
Alexei Starovoitov0246e642014-09-26 00:17:04 -07008106 sizeof(env->used_maps[0]) * env->used_map_cnt);
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07008107 env->prog->aux->used_map_cnt = env->used_map_cnt;
Alexei Starovoitov0246e642014-09-26 00:17:04 -07008108
8109 /* program is valid. Convert pseudo bpf_ld_imm64 into generic
8110 * bpf_ld_imm64 instructions
8111 */
8112 convert_pseudo_ld_imm64(env);
8113 }
Alexei Starovoitovcbd35702014-09-26 00:17:03 -07008114
Yonghong Songba64e7d2018-11-24 23:20:44 -08008115 if (ret == 0)
8116 adjust_btf_func(env);
8117
Jakub Kicinskia2a7d572017-10-09 10:30:15 -07008118err_release_maps:
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07008119 if (!env->prog->aux->used_maps)
Alexei Starovoitov0246e642014-09-26 00:17:04 -07008120 /* if we didn't copy map pointers into bpf_prog_info, release
Jakub Kicinskiab7f5bf2018-05-03 18:37:17 -07008121 * them now. Otherwise free_used_maps() will release them.
Alexei Starovoitov0246e642014-09-26 00:17:04 -07008122 */
8123 release_maps(env);
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07008124 *prog = env->prog;
Jakub Kicinski3df126f2016-09-21 11:43:56 +01008125err_unlock:
Alexei Starovoitovcbd35702014-09-26 00:17:03 -07008126 mutex_unlock(&bpf_verifier_lock);
Jakub Kicinski3df126f2016-09-21 11:43:56 +01008127 vfree(env->insn_aux_data);
8128err_free_env:
8129 kfree(env);
Alexei Starovoitov51580e72014-09-26 00:17:02 -07008130 return ret;
8131}