blob: f2d600199e66edd2229e88f0d06e0ab00674cc1c [file] [log] [blame]
Alexei Starovoitov51580e72014-09-26 00:17:02 -07001/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
Alexei Starovoitov969bf052016-05-05 19:49:10 -07002 * Copyright (c) 2016 Facebook
Joe Stringerfd978bf72018-10-02 13:35:35 -07003 * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io
Alexei Starovoitov51580e72014-09-26 00:17:02 -07004 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of version 2 of the GNU General Public
7 * License as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License for more details.
13 */
Yonghong Song838e9692018-11-19 15:29:11 -080014#include <uapi/linux/btf.h>
Alexei Starovoitov51580e72014-09-26 00:17:02 -070015#include <linux/kernel.h>
16#include <linux/types.h>
17#include <linux/slab.h>
18#include <linux/bpf.h>
Yonghong Song838e9692018-11-19 15:29:11 -080019#include <linux/btf.h>
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +010020#include <linux/bpf_verifier.h>
Alexei Starovoitov51580e72014-09-26 00:17:02 -070021#include <linux/filter.h>
22#include <net/netlink.h>
23#include <linux/file.h>
24#include <linux/vmalloc.h>
Thomas Grafebb676d2016-10-27 11:23:51 +020025#include <linux/stringify.h>
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -080026#include <linux/bsearch.h>
27#include <linux/sort.h>
Yonghong Songc195651e2018-04-28 22:28:08 -070028#include <linux/perf_event.h>
Martin KaFai Laud9762e82018-12-13 10:41:48 -080029#include <linux/ctype.h>
Alexei Starovoitov51580e72014-09-26 00:17:02 -070030
Jakub Kicinskif4ac7e02017-10-09 10:30:12 -070031#include "disasm.h"
32
Jakub Kicinski00176a32017-10-16 16:40:54 -070033static const struct bpf_verifier_ops * const bpf_verifier_ops[] = {
34#define BPF_PROG_TYPE(_id, _name) \
35 [_id] = & _name ## _verifier_ops,
36#define BPF_MAP_TYPE(_id, _ops)
37#include <linux/bpf_types.h>
38#undef BPF_PROG_TYPE
39#undef BPF_MAP_TYPE
40};
41
Alexei Starovoitov51580e72014-09-26 00:17:02 -070042/* bpf_check() is a static code analyzer that walks eBPF program
43 * instruction by instruction and updates register/stack state.
44 * All paths of conditional branches are analyzed until 'bpf_exit' insn.
45 *
46 * The first pass is depth-first-search to check that the program is a DAG.
47 * It rejects the following programs:
48 * - larger than BPF_MAXINSNS insns
49 * - if loop is present (detected via back-edge)
50 * - unreachable insns exist (shouldn't be a forest. program = one function)
51 * - out of bounds or malformed jumps
52 * The second pass is all possible path descent from the 1st insn.
53 * Since it's analyzing all pathes through the program, the length of the
Gary Lineba38a92017-03-01 16:25:51 +080054 * analysis is limited to 64k insn, which may be hit even if total number of
Alexei Starovoitov51580e72014-09-26 00:17:02 -070055 * insn is less then 4K, but there are too many branches that change stack/regs.
56 * Number of 'branches to be analyzed' is limited to 1k
57 *
58 * On entry to each instruction, each register has a type, and the instruction
59 * changes the types of the registers depending on instruction semantics.
60 * If instruction is BPF_MOV64_REG(BPF_REG_1, BPF_REG_5), then type of R5 is
61 * copied to R1.
62 *
63 * All registers are 64-bit.
64 * R0 - return register
65 * R1-R5 argument passing registers
66 * R6-R9 callee saved registers
67 * R10 - frame pointer read-only
68 *
69 * At the start of BPF program the register R1 contains a pointer to bpf_context
70 * and has type PTR_TO_CTX.
71 *
72 * Verifier tracks arithmetic operations on pointers in case:
73 * BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
74 * BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -20),
75 * 1st insn copies R10 (which has FRAME_PTR) type into R1
76 * and 2nd arithmetic instruction is pattern matched to recognize
77 * that it wants to construct a pointer to some element within stack.
78 * So after 2nd insn, the register R1 has type PTR_TO_STACK
79 * (and -20 constant is saved for further stack bounds checking).
80 * Meaning that this reg is a pointer to stack plus known immediate constant.
81 *
Edward Creef1174f72017-08-07 15:26:19 +010082 * Most of the time the registers have SCALAR_VALUE type, which
Alexei Starovoitov51580e72014-09-26 00:17:02 -070083 * means the register has some value, but it's not a valid pointer.
Edward Creef1174f72017-08-07 15:26:19 +010084 * (like pointer plus pointer becomes SCALAR_VALUE type)
Alexei Starovoitov51580e72014-09-26 00:17:02 -070085 *
86 * When verifier sees load or store instructions the type of base register
Joe Stringerc64b7982018-10-02 13:35:33 -070087 * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are
88 * four pointer types recognized by check_mem_access() function.
Alexei Starovoitov51580e72014-09-26 00:17:02 -070089 *
90 * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value'
91 * and the range of [ptr, ptr + map's value_size) is accessible.
92 *
93 * registers used to pass values to function calls are checked against
94 * function argument constraints.
95 *
96 * ARG_PTR_TO_MAP_KEY is one of such argument constraints.
97 * It means that the register type passed to this function must be
98 * PTR_TO_STACK and it will be used inside the function as
99 * 'pointer to map element key'
100 *
101 * For example the argument constraints for bpf_map_lookup_elem():
102 * .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
103 * .arg1_type = ARG_CONST_MAP_PTR,
104 * .arg2_type = ARG_PTR_TO_MAP_KEY,
105 *
106 * ret_type says that this function returns 'pointer to map elem value or null'
107 * function expects 1st argument to be a const pointer to 'struct bpf_map' and
108 * 2nd argument should be a pointer to stack, which will be used inside
109 * the helper function as a pointer to map element key.
110 *
111 * On the kernel side the helper function looks like:
112 * u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
113 * {
114 * struct bpf_map *map = (struct bpf_map *) (unsigned long) r1;
115 * void *key = (void *) (unsigned long) r2;
116 * void *value;
117 *
118 * here kernel can access 'key' and 'map' pointers safely, knowing that
119 * [key, key + map->key_size) bytes are valid and were initialized on
120 * the stack of eBPF program.
121 * }
122 *
123 * Corresponding eBPF program may look like:
124 * BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), // after this insn R2 type is FRAME_PTR
125 * BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), // after this insn R2 type is PTR_TO_STACK
126 * BPF_LD_MAP_FD(BPF_REG_1, map_fd), // after this insn R1 type is CONST_PTR_TO_MAP
127 * BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem),
128 * here verifier looks at prototype of map_lookup_elem() and sees:
129 * .arg1_type == ARG_CONST_MAP_PTR and R1->type == CONST_PTR_TO_MAP, which is ok,
130 * Now verifier knows that this map has key of R1->map_ptr->key_size bytes
131 *
132 * Then .arg2_type == ARG_PTR_TO_MAP_KEY and R2->type == PTR_TO_STACK, ok so far,
133 * Now verifier checks that [R2, R2 + map's key_size) are within stack limits
134 * and were initialized prior to this call.
135 * If it's ok, then verifier allows this BPF_CALL insn and looks at
136 * .ret_type which is RET_PTR_TO_MAP_VALUE_OR_NULL, so it sets
137 * R0->type = PTR_TO_MAP_VALUE_OR_NULL which means bpf_map_lookup_elem() function
138 * returns ether pointer to map value or NULL.
139 *
140 * When type PTR_TO_MAP_VALUE_OR_NULL passes through 'if (reg != 0) goto +off'
141 * insn, the register holding that pointer in the true branch changes state to
142 * PTR_TO_MAP_VALUE and the same register changes state to CONST_IMM in the false
143 * branch. See check_cond_jmp_op().
144 *
145 * After the call R0 is set to return type of the function and registers R1-R5
146 * are set to NOT_INIT to indicate that they are no longer readable.
Joe Stringerfd978bf72018-10-02 13:35:35 -0700147 *
148 * The following reference types represent a potential reference to a kernel
149 * resource which, after first being allocated, must be checked and freed by
150 * the BPF program:
151 * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET
152 *
153 * When the verifier sees a helper call return a reference type, it allocates a
154 * pointer id for the reference and stores it in the current function state.
155 * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into
156 * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type
157 * passes through a NULL-check conditional. For the branch wherein the state is
158 * changed to CONST_IMM, the verifier releases the reference.
Joe Stringer6acc9b42018-10-02 13:35:36 -0700159 *
160 * For each helper function that allocates a reference, such as
161 * bpf_sk_lookup_tcp(), there is a corresponding release function, such as
162 * bpf_sk_release(). When a reference type passes into the release function,
163 * the verifier also releases the reference. If any unchecked or unreleased
164 * reference remains at the end of the program, the verifier rejects it.
Alexei Starovoitov51580e72014-09-26 00:17:02 -0700165 */
166
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700167/* verifier_state + insn_idx are pushed to stack when branch is encountered */
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +0100168struct bpf_verifier_stack_elem {
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700169 /* verifer state is 'st'
170 * before processing instruction 'insn_idx'
171 * and after processing instruction 'prev_insn_idx'
172 */
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +0100173 struct bpf_verifier_state st;
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700174 int insn_idx;
175 int prev_insn_idx;
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +0100176 struct bpf_verifier_stack_elem *next;
Alexei Starovoitovcbd35702014-09-26 00:17:03 -0700177};
178
Daniel Borkmann07016152016-04-05 22:33:17 +0200179#define BPF_COMPLEXITY_LIMIT_STACK 1024
Alexei Starovoitovceefbc92018-12-03 22:46:06 -0800180#define BPF_COMPLEXITY_LIMIT_STATES 64
Daniel Borkmann07016152016-04-05 22:33:17 +0200181
Daniel Borkmannc93552c2018-05-24 02:32:53 +0200182#define BPF_MAP_PTR_UNPRIV 1UL
183#define BPF_MAP_PTR_POISON ((void *)((0xeB9FUL << 1) + \
184 POISON_POINTER_DELTA))
185#define BPF_MAP_PTR(X) ((struct bpf_map *)((X) & ~BPF_MAP_PTR_UNPRIV))
186
187static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux)
188{
189 return BPF_MAP_PTR(aux->map_state) == BPF_MAP_PTR_POISON;
190}
191
192static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux)
193{
194 return aux->map_state & BPF_MAP_PTR_UNPRIV;
195}
196
197static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux,
198 const struct bpf_map *map, bool unpriv)
199{
200 BUILD_BUG_ON((unsigned long)BPF_MAP_PTR_POISON & BPF_MAP_PTR_UNPRIV);
201 unpriv |= bpf_map_ptr_unpriv(aux);
202 aux->map_state = (unsigned long)map |
203 (unpriv ? BPF_MAP_PTR_UNPRIV : 0UL);
204}
Martin KaFai Laufad73a12017-03-22 10:00:32 -0700205
Daniel Borkmann33ff9822016-04-13 00:10:50 +0200206struct bpf_call_arg_meta {
207 struct bpf_map *map_ptr;
Daniel Borkmann435faee12016-04-13 00:10:51 +0200208 bool raw_mode;
Daniel Borkmann36bbef52016-09-20 00:26:13 +0200209 bool pkt_access;
Daniel Borkmann435faee12016-04-13 00:10:51 +0200210 int regno;
211 int access_size;
Yonghong Song849fa502018-04-28 22:28:09 -0700212 s64 msize_smax_value;
213 u64 msize_umax_value;
Martin KaFai Lau1b986582019-03-12 10:23:02 -0700214 int ref_obj_id;
Alexei Starovoitovd83525c2019-01-31 15:40:04 -0800215 int func_id;
Daniel Borkmann33ff9822016-04-13 00:10:50 +0200216};
217
Alexei Starovoitovcbd35702014-09-26 00:17:03 -0700218static DEFINE_MUTEX(bpf_verifier_lock);
219
Martin KaFai Laud9762e82018-12-13 10:41:48 -0800220static const struct bpf_line_info *
221find_linfo(const struct bpf_verifier_env *env, u32 insn_off)
222{
223 const struct bpf_line_info *linfo;
224 const struct bpf_prog *prog;
225 u32 i, nr_linfo;
226
227 prog = env->prog;
228 nr_linfo = prog->aux->nr_linfo;
229
230 if (!nr_linfo || insn_off >= prog->len)
231 return NULL;
232
233 linfo = prog->aux->linfo;
234 for (i = 1; i < nr_linfo; i++)
235 if (insn_off < linfo[i].insn_off)
236 break;
237
238 return &linfo[i - 1];
239}
240
Martin KaFai Lau77d2e052018-03-24 11:44:23 -0700241void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt,
242 va_list args)
Alexei Starovoitovcbd35702014-09-26 00:17:03 -0700243{
Jakub Kicinskia2a7d572017-10-09 10:30:15 -0700244 unsigned int n;
Alexei Starovoitovcbd35702014-09-26 00:17:03 -0700245
Jakub Kicinskia2a7d572017-10-09 10:30:15 -0700246 n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args);
Jakub Kicinskia2a7d572017-10-09 10:30:15 -0700247
248 WARN_ONCE(n >= BPF_VERIFIER_TMP_LOG_SIZE - 1,
249 "verifier log line truncated - local buffer too short\n");
250
251 n = min(log->len_total - log->len_used - 1, n);
252 log->kbuf[n] = '\0';
253
254 if (!copy_to_user(log->ubuf + log->len_used, log->kbuf, n + 1))
255 log->len_used += n;
256 else
257 log->ubuf = NULL;
Alexei Starovoitovcbd35702014-09-26 00:17:03 -0700258}
Jiri Olsaabe08842018-03-23 11:41:28 +0100259
260/* log_level controls verbosity level of eBPF verifier.
261 * bpf_verifier_log_write() is used to dump the verification trace to the log,
262 * so the user can figure out what's wrong with the program
Quentin Monnet430e68d2018-01-10 12:26:06 +0000263 */
Jiri Olsaabe08842018-03-23 11:41:28 +0100264__printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env,
265 const char *fmt, ...)
266{
267 va_list args;
268
Martin KaFai Lau77d2e052018-03-24 11:44:23 -0700269 if (!bpf_verifier_log_needed(&env->log))
270 return;
271
Jiri Olsaabe08842018-03-23 11:41:28 +0100272 va_start(args, fmt);
Martin KaFai Lau77d2e052018-03-24 11:44:23 -0700273 bpf_verifier_vlog(&env->log, fmt, args);
Jiri Olsaabe08842018-03-23 11:41:28 +0100274 va_end(args);
275}
276EXPORT_SYMBOL_GPL(bpf_verifier_log_write);
277
278__printf(2, 3) static void verbose(void *private_data, const char *fmt, ...)
279{
Martin KaFai Lau77d2e052018-03-24 11:44:23 -0700280 struct bpf_verifier_env *env = private_data;
Jiri Olsaabe08842018-03-23 11:41:28 +0100281 va_list args;
282
Martin KaFai Lau77d2e052018-03-24 11:44:23 -0700283 if (!bpf_verifier_log_needed(&env->log))
284 return;
285
Jiri Olsaabe08842018-03-23 11:41:28 +0100286 va_start(args, fmt);
Martin KaFai Lau77d2e052018-03-24 11:44:23 -0700287 bpf_verifier_vlog(&env->log, fmt, args);
Jiri Olsaabe08842018-03-23 11:41:28 +0100288 va_end(args);
289}
Alexei Starovoitovcbd35702014-09-26 00:17:03 -0700290
Martin KaFai Laud9762e82018-12-13 10:41:48 -0800291static const char *ltrim(const char *s)
292{
293 while (isspace(*s))
294 s++;
295
296 return s;
297}
298
299__printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env,
300 u32 insn_off,
301 const char *prefix_fmt, ...)
302{
303 const struct bpf_line_info *linfo;
304
305 if (!bpf_verifier_log_needed(&env->log))
306 return;
307
308 linfo = find_linfo(env, insn_off);
309 if (!linfo || linfo == env->prev_linfo)
310 return;
311
312 if (prefix_fmt) {
313 va_list args;
314
315 va_start(args, prefix_fmt);
316 bpf_verifier_vlog(&env->log, prefix_fmt, args);
317 va_end(args);
318 }
319
320 verbose(env, "%s\n",
321 ltrim(btf_name_by_offset(env->prog->aux->btf,
322 linfo->line_off)));
323
324 env->prev_linfo = linfo;
325}
326
Daniel Borkmannde8f3a82017-09-25 02:25:51 +0200327static bool type_is_pkt_pointer(enum bpf_reg_type type)
328{
329 return type == PTR_TO_PACKET ||
330 type == PTR_TO_PACKET_META;
331}
332
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -0800333static bool type_is_sk_pointer(enum bpf_reg_type type)
334{
335 return type == PTR_TO_SOCKET ||
Martin KaFai Lau655a51e2019-02-09 23:22:24 -0800336 type == PTR_TO_SOCK_COMMON ||
337 type == PTR_TO_TCP_SOCK;
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -0800338}
339
Joe Stringer840b9612018-10-02 13:35:32 -0700340static bool reg_type_may_be_null(enum bpf_reg_type type)
341{
Joe Stringerfd978bf72018-10-02 13:35:35 -0700342 return type == PTR_TO_MAP_VALUE_OR_NULL ||
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -0800343 type == PTR_TO_SOCKET_OR_NULL ||
Martin KaFai Lau655a51e2019-02-09 23:22:24 -0800344 type == PTR_TO_SOCK_COMMON_OR_NULL ||
345 type == PTR_TO_TCP_SOCK_OR_NULL;
Joe Stringerfd978bf72018-10-02 13:35:35 -0700346}
347
Alexei Starovoitovd83525c2019-01-31 15:40:04 -0800348static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg)
349{
350 return reg->type == PTR_TO_MAP_VALUE &&
351 map_value_has_spin_lock(reg->map_ptr);
352}
353
Martin KaFai Laucba368c2019-03-18 10:37:13 -0700354static bool reg_type_may_be_refcounted_or_null(enum bpf_reg_type type)
355{
356 return type == PTR_TO_SOCKET ||
357 type == PTR_TO_SOCKET_OR_NULL ||
358 type == PTR_TO_TCP_SOCK ||
359 type == PTR_TO_TCP_SOCK_OR_NULL;
360}
361
Martin KaFai Lau1b986582019-03-12 10:23:02 -0700362static bool arg_type_may_be_refcounted(enum bpf_arg_type type)
Joe Stringerfd978bf72018-10-02 13:35:35 -0700363{
Martin KaFai Lau1b986582019-03-12 10:23:02 -0700364 return type == ARG_PTR_TO_SOCK_COMMON;
Joe Stringerfd978bf72018-10-02 13:35:35 -0700365}
366
367/* Determine whether the function releases some resources allocated by another
368 * function call. The first reference type argument will be assumed to be
369 * released by release_reference().
370 */
371static bool is_release_function(enum bpf_func_id func_id)
372{
Joe Stringer6acc9b42018-10-02 13:35:36 -0700373 return func_id == BPF_FUNC_sk_release;
Joe Stringer840b9612018-10-02 13:35:32 -0700374}
375
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -0800376static bool is_acquire_function(enum bpf_func_id func_id)
377{
378 return func_id == BPF_FUNC_sk_lookup_tcp ||
Lorenz Baueredbf8c02019-03-22 09:54:01 +0800379 func_id == BPF_FUNC_sk_lookup_udp ||
380 func_id == BPF_FUNC_skc_lookup_tcp;
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -0800381}
382
Martin KaFai Lau1b986582019-03-12 10:23:02 -0700383static bool is_ptr_cast_function(enum bpf_func_id func_id)
384{
385 return func_id == BPF_FUNC_tcp_sock ||
386 func_id == BPF_FUNC_sk_fullsock;
387}
388
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700389/* string representation of 'enum bpf_reg_type' */
390static const char * const reg_type_str[] = {
391 [NOT_INIT] = "?",
Edward Creef1174f72017-08-07 15:26:19 +0100392 [SCALAR_VALUE] = "inv",
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700393 [PTR_TO_CTX] = "ctx",
394 [CONST_PTR_TO_MAP] = "map_ptr",
395 [PTR_TO_MAP_VALUE] = "map_value",
396 [PTR_TO_MAP_VALUE_OR_NULL] = "map_value_or_null",
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700397 [PTR_TO_STACK] = "fp",
Alexei Starovoitov969bf052016-05-05 19:49:10 -0700398 [PTR_TO_PACKET] = "pkt",
Daniel Borkmannde8f3a82017-09-25 02:25:51 +0200399 [PTR_TO_PACKET_META] = "pkt_meta",
Alexei Starovoitov969bf052016-05-05 19:49:10 -0700400 [PTR_TO_PACKET_END] = "pkt_end",
Petar Penkovd58e4682018-09-14 07:46:18 -0700401 [PTR_TO_FLOW_KEYS] = "flow_keys",
Joe Stringerc64b7982018-10-02 13:35:33 -0700402 [PTR_TO_SOCKET] = "sock",
403 [PTR_TO_SOCKET_OR_NULL] = "sock_or_null",
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -0800404 [PTR_TO_SOCK_COMMON] = "sock_common",
405 [PTR_TO_SOCK_COMMON_OR_NULL] = "sock_common_or_null",
Martin KaFai Lau655a51e2019-02-09 23:22:24 -0800406 [PTR_TO_TCP_SOCK] = "tcp_sock",
407 [PTR_TO_TCP_SOCK_OR_NULL] = "tcp_sock_or_null",
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700408};
409
Edward Cree8efea212018-08-22 20:02:44 +0100410static char slot_type_char[] = {
411 [STACK_INVALID] = '?',
412 [STACK_SPILL] = 'r',
413 [STACK_MISC] = 'm',
414 [STACK_ZERO] = '0',
415};
416
Alexei Starovoitov4e920242017-11-30 21:31:36 -0800417static void print_liveness(struct bpf_verifier_env *env,
418 enum bpf_reg_liveness live)
419{
Alexei Starovoitov9242b5f2018-12-13 11:42:34 -0800420 if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE))
Alexei Starovoitov4e920242017-11-30 21:31:36 -0800421 verbose(env, "_");
422 if (live & REG_LIVE_READ)
423 verbose(env, "r");
424 if (live & REG_LIVE_WRITTEN)
425 verbose(env, "w");
Alexei Starovoitov9242b5f2018-12-13 11:42:34 -0800426 if (live & REG_LIVE_DONE)
427 verbose(env, "D");
Alexei Starovoitov4e920242017-11-30 21:31:36 -0800428}
429
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800430static struct bpf_func_state *func(struct bpf_verifier_env *env,
431 const struct bpf_reg_state *reg)
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700432{
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800433 struct bpf_verifier_state *cur = env->cur_state;
434
435 return cur->frame[reg->frameno];
436}
437
438static void print_verifier_state(struct bpf_verifier_env *env,
439 const struct bpf_func_state *state)
440{
441 const struct bpf_reg_state *reg;
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700442 enum bpf_reg_type t;
443 int i;
444
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800445 if (state->frameno)
446 verbose(env, " frame%d:", state->frameno);
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700447 for (i = 0; i < MAX_BPF_REG; i++) {
Alexei Starovoitov1a0dc1a2016-05-05 19:49:09 -0700448 reg = &state->regs[i];
449 t = reg->type;
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700450 if (t == NOT_INIT)
451 continue;
Alexei Starovoitov4e920242017-11-30 21:31:36 -0800452 verbose(env, " R%d", i);
453 print_liveness(env, reg->live);
454 verbose(env, "=%s", reg_type_str[t]);
Edward Creef1174f72017-08-07 15:26:19 +0100455 if ((t == SCALAR_VALUE || t == PTR_TO_STACK) &&
456 tnum_is_const(reg->var_off)) {
457 /* reg->off should be 0 for SCALAR_VALUE */
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700458 verbose(env, "%lld", reg->var_off.value + reg->off);
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800459 if (t == PTR_TO_STACK)
460 verbose(env, ",call_%d", func(env, reg)->callsite);
Edward Creef1174f72017-08-07 15:26:19 +0100461 } else {
Martin KaFai Laucba368c2019-03-18 10:37:13 -0700462 verbose(env, "(id=%d", reg->id);
463 if (reg_type_may_be_refcounted_or_null(t))
464 verbose(env, ",ref_obj_id=%d", reg->ref_obj_id);
Edward Creef1174f72017-08-07 15:26:19 +0100465 if (t != SCALAR_VALUE)
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700466 verbose(env, ",off=%d", reg->off);
Daniel Borkmannde8f3a82017-09-25 02:25:51 +0200467 if (type_is_pkt_pointer(t))
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700468 verbose(env, ",r=%d", reg->range);
Edward Creef1174f72017-08-07 15:26:19 +0100469 else if (t == CONST_PTR_TO_MAP ||
470 t == PTR_TO_MAP_VALUE ||
471 t == PTR_TO_MAP_VALUE_OR_NULL)
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700472 verbose(env, ",ks=%d,vs=%d",
Edward Creef1174f72017-08-07 15:26:19 +0100473 reg->map_ptr->key_size,
474 reg->map_ptr->value_size);
Edward Cree7d1238f2017-08-07 15:26:56 +0100475 if (tnum_is_const(reg->var_off)) {
476 /* Typically an immediate SCALAR_VALUE, but
477 * could be a pointer whose offset is too big
478 * for reg->off
479 */
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700480 verbose(env, ",imm=%llx", reg->var_off.value);
Edward Cree7d1238f2017-08-07 15:26:56 +0100481 } else {
482 if (reg->smin_value != reg->umin_value &&
483 reg->smin_value != S64_MIN)
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700484 verbose(env, ",smin_value=%lld",
Edward Cree7d1238f2017-08-07 15:26:56 +0100485 (long long)reg->smin_value);
486 if (reg->smax_value != reg->umax_value &&
487 reg->smax_value != S64_MAX)
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700488 verbose(env, ",smax_value=%lld",
Edward Cree7d1238f2017-08-07 15:26:56 +0100489 (long long)reg->smax_value);
490 if (reg->umin_value != 0)
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700491 verbose(env, ",umin_value=%llu",
Edward Cree7d1238f2017-08-07 15:26:56 +0100492 (unsigned long long)reg->umin_value);
493 if (reg->umax_value != U64_MAX)
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700494 verbose(env, ",umax_value=%llu",
Edward Cree7d1238f2017-08-07 15:26:56 +0100495 (unsigned long long)reg->umax_value);
496 if (!tnum_is_unknown(reg->var_off)) {
497 char tn_buf[48];
Edward Creef1174f72017-08-07 15:26:19 +0100498
Edward Cree7d1238f2017-08-07 15:26:56 +0100499 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700500 verbose(env, ",var_off=%s", tn_buf);
Edward Cree7d1238f2017-08-07 15:26:56 +0100501 }
Edward Creef1174f72017-08-07 15:26:19 +0100502 }
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700503 verbose(env, ")");
Edward Creef1174f72017-08-07 15:26:19 +0100504 }
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700505 }
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700506 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
Edward Cree8efea212018-08-22 20:02:44 +0100507 char types_buf[BPF_REG_SIZE + 1];
508 bool valid = false;
509 int j;
510
511 for (j = 0; j < BPF_REG_SIZE; j++) {
512 if (state->stack[i].slot_type[j] != STACK_INVALID)
513 valid = true;
514 types_buf[j] = slot_type_char[
515 state->stack[i].slot_type[j]];
516 }
517 types_buf[BPF_REG_SIZE] = 0;
518 if (!valid)
519 continue;
520 verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
521 print_liveness(env, state->stack[i].spilled_ptr.live);
522 if (state->stack[i].slot_type[0] == STACK_SPILL)
Alexei Starovoitov4e920242017-11-30 21:31:36 -0800523 verbose(env, "=%s",
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700524 reg_type_str[state->stack[i].spilled_ptr.type]);
Edward Cree8efea212018-08-22 20:02:44 +0100525 else
526 verbose(env, "=%s", types_buf);
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700527 }
Joe Stringerfd978bf72018-10-02 13:35:35 -0700528 if (state->acquired_refs && state->refs[0].id) {
529 verbose(env, " refs=%d", state->refs[0].id);
530 for (i = 1; i < state->acquired_refs; i++)
531 if (state->refs[i].id)
532 verbose(env, ",%d", state->refs[i].id);
533 }
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700534 verbose(env, "\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700535}
536
Joe Stringer84dbf352018-10-02 13:35:34 -0700537#define COPY_STATE_FN(NAME, COUNT, FIELD, SIZE) \
538static int copy_##NAME##_state(struct bpf_func_state *dst, \
539 const struct bpf_func_state *src) \
540{ \
541 if (!src->FIELD) \
542 return 0; \
543 if (WARN_ON_ONCE(dst->COUNT < src->COUNT)) { \
544 /* internal bug, make state invalid to reject the program */ \
545 memset(dst, 0, sizeof(*dst)); \
546 return -EFAULT; \
547 } \
548 memcpy(dst->FIELD, src->FIELD, \
549 sizeof(*src->FIELD) * (src->COUNT / SIZE)); \
550 return 0; \
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700551}
Joe Stringerfd978bf72018-10-02 13:35:35 -0700552/* copy_reference_state() */
553COPY_STATE_FN(reference, acquired_refs, refs, 1)
Joe Stringer84dbf352018-10-02 13:35:34 -0700554/* copy_stack_state() */
555COPY_STATE_FN(stack, allocated_stack, stack, BPF_REG_SIZE)
556#undef COPY_STATE_FN
557
558#define REALLOC_STATE_FN(NAME, COUNT, FIELD, SIZE) \
559static int realloc_##NAME##_state(struct bpf_func_state *state, int size, \
560 bool copy_old) \
561{ \
562 u32 old_size = state->COUNT; \
563 struct bpf_##NAME##_state *new_##FIELD; \
564 int slot = size / SIZE; \
565 \
566 if (size <= old_size || !size) { \
567 if (copy_old) \
568 return 0; \
569 state->COUNT = slot * SIZE; \
570 if (!size && old_size) { \
571 kfree(state->FIELD); \
572 state->FIELD = NULL; \
573 } \
574 return 0; \
575 } \
576 new_##FIELD = kmalloc_array(slot, sizeof(struct bpf_##NAME##_state), \
577 GFP_KERNEL); \
578 if (!new_##FIELD) \
579 return -ENOMEM; \
580 if (copy_old) { \
581 if (state->FIELD) \
582 memcpy(new_##FIELD, state->FIELD, \
583 sizeof(*new_##FIELD) * (old_size / SIZE)); \
584 memset(new_##FIELD + old_size / SIZE, 0, \
585 sizeof(*new_##FIELD) * (size - old_size) / SIZE); \
586 } \
587 state->COUNT = slot * SIZE; \
588 kfree(state->FIELD); \
589 state->FIELD = new_##FIELD; \
590 return 0; \
591}
Joe Stringerfd978bf72018-10-02 13:35:35 -0700592/* realloc_reference_state() */
593REALLOC_STATE_FN(reference, acquired_refs, refs, 1)
Joe Stringer84dbf352018-10-02 13:35:34 -0700594/* realloc_stack_state() */
595REALLOC_STATE_FN(stack, allocated_stack, stack, BPF_REG_SIZE)
596#undef REALLOC_STATE_FN
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700597
598/* do_check() starts with zero-sized stack in struct bpf_verifier_state to
599 * make it consume minimal amount of memory. check_stack_write() access from
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800600 * the program calls into realloc_func_state() to grow the stack size.
Joe Stringer84dbf352018-10-02 13:35:34 -0700601 * Note there is a non-zero 'parent' pointer inside bpf_verifier_state
602 * which realloc_stack_state() copies over. It points to previous
603 * bpf_verifier_state which is never reallocated.
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700604 */
Joe Stringerfd978bf72018-10-02 13:35:35 -0700605static int realloc_func_state(struct bpf_func_state *state, int stack_size,
606 int refs_size, bool copy_old)
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700607{
Joe Stringerfd978bf72018-10-02 13:35:35 -0700608 int err = realloc_reference_state(state, refs_size, copy_old);
609 if (err)
610 return err;
611 return realloc_stack_state(state, stack_size, copy_old);
612}
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700613
Joe Stringerfd978bf72018-10-02 13:35:35 -0700614/* Acquire a pointer id from the env and update the state->refs to include
615 * this new pointer reference.
616 * On success, returns a valid pointer id to associate with the register
617 * On failure, returns a negative errno.
618 */
619static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx)
620{
621 struct bpf_func_state *state = cur_func(env);
622 int new_ofs = state->acquired_refs;
623 int id, err;
624
625 err = realloc_reference_state(state, state->acquired_refs + 1, true);
626 if (err)
627 return err;
628 id = ++env->id_gen;
629 state->refs[new_ofs].id = id;
630 state->refs[new_ofs].insn_idx = insn_idx;
631
632 return id;
633}
634
635/* release function corresponding to acquire_reference_state(). Idempotent. */
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -0800636static int release_reference_state(struct bpf_func_state *state, int ptr_id)
Joe Stringerfd978bf72018-10-02 13:35:35 -0700637{
638 int i, last_idx;
639
Joe Stringerfd978bf72018-10-02 13:35:35 -0700640 last_idx = state->acquired_refs - 1;
641 for (i = 0; i < state->acquired_refs; i++) {
642 if (state->refs[i].id == ptr_id) {
643 if (last_idx && i != last_idx)
644 memcpy(&state->refs[i], &state->refs[last_idx],
645 sizeof(*state->refs));
646 memset(&state->refs[last_idx], 0, sizeof(*state->refs));
647 state->acquired_refs--;
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700648 return 0;
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700649 }
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700650 }
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -0800651 return -EINVAL;
Joe Stringerfd978bf72018-10-02 13:35:35 -0700652}
653
654static int transfer_reference_state(struct bpf_func_state *dst,
655 struct bpf_func_state *src)
656{
657 int err = realloc_reference_state(dst, src->acquired_refs, false);
658 if (err)
659 return err;
660 err = copy_reference_state(dst, src);
661 if (err)
662 return err;
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700663 return 0;
664}
665
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800666static void free_func_state(struct bpf_func_state *state)
667{
Alexei Starovoitov58963512018-01-08 07:51:17 -0800668 if (!state)
669 return;
Joe Stringerfd978bf72018-10-02 13:35:35 -0700670 kfree(state->refs);
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800671 kfree(state->stack);
672 kfree(state);
673}
674
Alexei Starovoitov1969db42017-11-01 00:08:04 -0700675static void free_verifier_state(struct bpf_verifier_state *state,
676 bool free_self)
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700677{
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800678 int i;
679
680 for (i = 0; i <= state->curframe; i++) {
681 free_func_state(state->frame[i]);
682 state->frame[i] = NULL;
683 }
Alexei Starovoitov1969db42017-11-01 00:08:04 -0700684 if (free_self)
685 kfree(state);
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700686}
687
688/* copy verifier state from src to dst growing dst stack space
689 * when necessary to accommodate larger src stack
690 */
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800691static int copy_func_state(struct bpf_func_state *dst,
692 const struct bpf_func_state *src)
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700693{
694 int err;
695
Joe Stringerfd978bf72018-10-02 13:35:35 -0700696 err = realloc_func_state(dst, src->allocated_stack, src->acquired_refs,
697 false);
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700698 if (err)
699 return err;
Joe Stringerfd978bf72018-10-02 13:35:35 -0700700 memcpy(dst, src, offsetof(struct bpf_func_state, acquired_refs));
701 err = copy_reference_state(dst, src);
702 if (err)
703 return err;
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700704 return copy_stack_state(dst, src);
705}
706
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800707static int copy_verifier_state(struct bpf_verifier_state *dst_state,
708 const struct bpf_verifier_state *src)
709{
710 struct bpf_func_state *dst;
711 int i, err;
712
713 /* if dst has more stack frames then src frame, free them */
714 for (i = src->curframe + 1; i <= dst_state->curframe; i++) {
715 free_func_state(dst_state->frame[i]);
716 dst_state->frame[i] = NULL;
717 }
Daniel Borkmann979d63d2019-01-03 00:58:34 +0100718 dst_state->speculative = src->speculative;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800719 dst_state->curframe = src->curframe;
Alexei Starovoitovd83525c2019-01-31 15:40:04 -0800720 dst_state->active_spin_lock = src->active_spin_lock;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800721 for (i = 0; i <= src->curframe; i++) {
722 dst = dst_state->frame[i];
723 if (!dst) {
724 dst = kzalloc(sizeof(*dst), GFP_KERNEL);
725 if (!dst)
726 return -ENOMEM;
727 dst_state->frame[i] = dst;
728 }
729 err = copy_func_state(dst, src->frame[i]);
730 if (err)
731 return err;
732 }
733 return 0;
734}
735
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700736static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx,
737 int *insn_idx)
738{
739 struct bpf_verifier_state *cur = env->cur_state;
740 struct bpf_verifier_stack_elem *elem, *head = env->head;
741 int err;
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700742
743 if (env->head == NULL)
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700744 return -ENOENT;
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700745
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700746 if (cur) {
747 err = copy_verifier_state(cur, &head->st);
748 if (err)
749 return err;
750 }
751 if (insn_idx)
752 *insn_idx = head->insn_idx;
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700753 if (prev_insn_idx)
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700754 *prev_insn_idx = head->prev_insn_idx;
755 elem = head->next;
Alexei Starovoitov1969db42017-11-01 00:08:04 -0700756 free_verifier_state(&head->st, false);
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700757 kfree(head);
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700758 env->head = elem;
759 env->stack_size--;
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700760 return 0;
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700761}
762
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +0100763static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env,
Daniel Borkmann979d63d2019-01-03 00:58:34 +0100764 int insn_idx, int prev_insn_idx,
765 bool speculative)
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700766{
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700767 struct bpf_verifier_state *cur = env->cur_state;
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +0100768 struct bpf_verifier_stack_elem *elem;
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700769 int err;
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700770
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700771 elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL);
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700772 if (!elem)
773 goto err;
774
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700775 elem->insn_idx = insn_idx;
776 elem->prev_insn_idx = prev_insn_idx;
777 elem->next = env->head;
778 env->head = elem;
779 env->stack_size++;
Alexei Starovoitov1969db42017-11-01 00:08:04 -0700780 err = copy_verifier_state(&elem->st, cur);
781 if (err)
782 goto err;
Daniel Borkmann979d63d2019-01-03 00:58:34 +0100783 elem->st.speculative |= speculative;
Daniel Borkmann07016152016-04-05 22:33:17 +0200784 if (env->stack_size > BPF_COMPLEXITY_LIMIT_STACK) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700785 verbose(env, "BPF program is too complex\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700786 goto err;
787 }
788 return &elem->st;
789err:
Alexei Starovoitov58963512018-01-08 07:51:17 -0800790 free_verifier_state(env->cur_state, true);
791 env->cur_state = NULL;
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700792 /* pop all elements and return */
Alexei Starovoitov638f5b92017-10-31 18:16:05 -0700793 while (!pop_stack(env, NULL, NULL));
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700794 return NULL;
795}
796
797#define CALLER_SAVED_REGS 6
798static const int caller_saved[CALLER_SAVED_REGS] = {
799 BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5
800};
801
Edward Creef1174f72017-08-07 15:26:19 +0100802static void __mark_reg_not_init(struct bpf_reg_state *reg);
803
Edward Creeb03c9f92017-08-07 15:26:36 +0100804/* Mark the unknown part of a register (variable offset or scalar value) as
805 * known to have the value @imm.
806 */
807static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm)
808{
Alexei Starovoitova9c676b2018-09-04 19:13:44 -0700809 /* Clear id, off, and union(map_ptr, range) */
810 memset(((u8 *)reg) + sizeof(reg->type), 0,
811 offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type));
Edward Creeb03c9f92017-08-07 15:26:36 +0100812 reg->var_off = tnum_const(imm);
813 reg->smin_value = (s64)imm;
814 reg->smax_value = (s64)imm;
815 reg->umin_value = imm;
816 reg->umax_value = imm;
817}
818
Edward Creef1174f72017-08-07 15:26:19 +0100819/* Mark the 'variable offset' part of a register as zero. This should be
820 * used only on registers holding a pointer type.
821 */
822static void __mark_reg_known_zero(struct bpf_reg_state *reg)
823{
Edward Creeb03c9f92017-08-07 15:26:36 +0100824 __mark_reg_known(reg, 0);
Edward Creef1174f72017-08-07 15:26:19 +0100825}
826
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -0800827static void __mark_reg_const_zero(struct bpf_reg_state *reg)
828{
829 __mark_reg_known(reg, 0);
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -0800830 reg->type = SCALAR_VALUE;
831}
832
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700833static void mark_reg_known_zero(struct bpf_verifier_env *env,
834 struct bpf_reg_state *regs, u32 regno)
Edward Creef1174f72017-08-07 15:26:19 +0100835{
836 if (WARN_ON(regno >= MAX_BPF_REG)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700837 verbose(env, "mark_reg_known_zero(regs, %u)\n", regno);
Edward Creef1174f72017-08-07 15:26:19 +0100838 /* Something bad happened, let's kill all regs */
839 for (regno = 0; regno < MAX_BPF_REG; regno++)
840 __mark_reg_not_init(regs + regno);
841 return;
842 }
843 __mark_reg_known_zero(regs + regno);
844}
845
Daniel Borkmannde8f3a82017-09-25 02:25:51 +0200846static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)
847{
848 return type_is_pkt_pointer(reg->type);
849}
850
851static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg)
852{
853 return reg_is_pkt_pointer(reg) ||
854 reg->type == PTR_TO_PACKET_END;
855}
856
857/* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */
858static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg,
859 enum bpf_reg_type which)
860{
861 /* The register can already have a range from prior markings.
862 * This is fine as long as it hasn't been advanced from its
863 * origin.
864 */
865 return reg->type == which &&
866 reg->id == 0 &&
867 reg->off == 0 &&
868 tnum_equals_const(reg->var_off, 0);
869}
870
Edward Creeb03c9f92017-08-07 15:26:36 +0100871/* Attempts to improve min/max values based on var_off information */
872static void __update_reg_bounds(struct bpf_reg_state *reg)
873{
874 /* min signed is max(sign bit) | min(other bits) */
875 reg->smin_value = max_t(s64, reg->smin_value,
876 reg->var_off.value | (reg->var_off.mask & S64_MIN));
877 /* max signed is min(sign bit) | max(other bits) */
878 reg->smax_value = min_t(s64, reg->smax_value,
879 reg->var_off.value | (reg->var_off.mask & S64_MAX));
880 reg->umin_value = max(reg->umin_value, reg->var_off.value);
881 reg->umax_value = min(reg->umax_value,
882 reg->var_off.value | reg->var_off.mask);
883}
884
885/* Uses signed min/max values to inform unsigned, and vice-versa */
886static void __reg_deduce_bounds(struct bpf_reg_state *reg)
887{
888 /* Learn sign from signed bounds.
889 * If we cannot cross the sign boundary, then signed and unsigned bounds
890 * are the same, so combine. This works even in the negative case, e.g.
891 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff.
892 */
893 if (reg->smin_value >= 0 || reg->smax_value < 0) {
894 reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value,
895 reg->umin_value);
896 reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value,
897 reg->umax_value);
898 return;
899 }
900 /* Learn sign from unsigned bounds. Signed bounds cross the sign
901 * boundary, so we must be careful.
902 */
903 if ((s64)reg->umax_value >= 0) {
904 /* Positive. We can't learn anything from the smin, but smax
905 * is positive, hence safe.
906 */
907 reg->smin_value = reg->umin_value;
908 reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value,
909 reg->umax_value);
910 } else if ((s64)reg->umin_value < 0) {
911 /* Negative. We can't learn anything from the smax, but smin
912 * is negative, hence safe.
913 */
914 reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value,
915 reg->umin_value);
916 reg->smax_value = reg->umax_value;
917 }
918}
919
920/* Attempts to improve var_off based on unsigned min/max information */
921static void __reg_bound_offset(struct bpf_reg_state *reg)
922{
923 reg->var_off = tnum_intersect(reg->var_off,
924 tnum_range(reg->umin_value,
925 reg->umax_value));
926}
927
928/* Reset the min/max bounds of a register */
929static void __mark_reg_unbounded(struct bpf_reg_state *reg)
930{
931 reg->smin_value = S64_MIN;
932 reg->smax_value = S64_MAX;
933 reg->umin_value = 0;
934 reg->umax_value = U64_MAX;
935}
936
Edward Creef1174f72017-08-07 15:26:19 +0100937/* Mark a register as having a completely unknown (scalar) value. */
938static void __mark_reg_unknown(struct bpf_reg_state *reg)
939{
Alexei Starovoitova9c676b2018-09-04 19:13:44 -0700940 /*
941 * Clear type, id, off, and union(map_ptr, range) and
942 * padding between 'type' and union
943 */
944 memset(reg, 0, offsetof(struct bpf_reg_state, var_off));
Edward Creef1174f72017-08-07 15:26:19 +0100945 reg->type = SCALAR_VALUE;
Edward Creef1174f72017-08-07 15:26:19 +0100946 reg->var_off = tnum_unknown;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800947 reg->frameno = 0;
Edward Creeb03c9f92017-08-07 15:26:36 +0100948 __mark_reg_unbounded(reg);
Edward Creef1174f72017-08-07 15:26:19 +0100949}
950
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700951static void mark_reg_unknown(struct bpf_verifier_env *env,
952 struct bpf_reg_state *regs, u32 regno)
Edward Creef1174f72017-08-07 15:26:19 +0100953{
954 if (WARN_ON(regno >= MAX_BPF_REG)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700955 verbose(env, "mark_reg_unknown(regs, %u)\n", regno);
Alexei Starovoitov19ceb412017-11-30 21:31:37 -0800956 /* Something bad happened, let's kill all regs except FP */
957 for (regno = 0; regno < BPF_REG_FP; regno++)
Edward Creef1174f72017-08-07 15:26:19 +0100958 __mark_reg_not_init(regs + regno);
959 return;
960 }
961 __mark_reg_unknown(regs + regno);
962}
963
964static void __mark_reg_not_init(struct bpf_reg_state *reg)
965{
966 __mark_reg_unknown(reg);
967 reg->type = NOT_INIT;
968}
969
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700970static void mark_reg_not_init(struct bpf_verifier_env *env,
971 struct bpf_reg_state *regs, u32 regno)
Daniel Borkmanna9789ef2017-05-25 01:05:06 +0200972{
Edward Creef1174f72017-08-07 15:26:19 +0100973 if (WARN_ON(regno >= MAX_BPF_REG)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700974 verbose(env, "mark_reg_not_init(regs, %u)\n", regno);
Alexei Starovoitov19ceb412017-11-30 21:31:37 -0800975 /* Something bad happened, let's kill all regs except FP */
976 for (regno = 0; regno < BPF_REG_FP; regno++)
Edward Creef1174f72017-08-07 15:26:19 +0100977 __mark_reg_not_init(regs + regno);
978 return;
979 }
980 __mark_reg_not_init(regs + regno);
Daniel Borkmanna9789ef2017-05-25 01:05:06 +0200981}
982
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700983static void init_reg_state(struct bpf_verifier_env *env,
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800984 struct bpf_func_state *state)
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700985{
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800986 struct bpf_reg_state *regs = state->regs;
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700987 int i;
988
Edward Creedc503a82017-08-15 20:34:35 +0100989 for (i = 0; i < MAX_BPF_REG; i++) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700990 mark_reg_not_init(env, regs, i);
Edward Creedc503a82017-08-15 20:34:35 +0100991 regs[i].live = REG_LIVE_NONE;
Edward Cree679c7822018-08-22 20:02:19 +0100992 regs[i].parent = NULL;
Edward Creedc503a82017-08-15 20:34:35 +0100993 }
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700994
995 /* frame pointer */
Edward Creef1174f72017-08-07 15:26:19 +0100996 regs[BPF_REG_FP].type = PTR_TO_STACK;
Jakub Kicinski61bd5212017-10-09 10:30:11 -0700997 mark_reg_known_zero(env, regs, BPF_REG_FP);
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -0800998 regs[BPF_REG_FP].frameno = state->frameno;
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700999
1000 /* 1st arg to a function */
1001 regs[BPF_REG_1].type = PTR_TO_CTX;
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001002 mark_reg_known_zero(env, regs, BPF_REG_1);
Daniel Borkmann6760bf22016-12-18 01:52:59 +01001003}
1004
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001005#define BPF_MAIN_FUNC (-1)
1006static void init_func_state(struct bpf_verifier_env *env,
1007 struct bpf_func_state *state,
1008 int callsite, int frameno, int subprogno)
1009{
1010 state->callsite = callsite;
1011 state->frameno = frameno;
1012 state->subprogno = subprogno;
1013 init_reg_state(env, state);
1014}
1015
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001016enum reg_arg_type {
1017 SRC_OP, /* register is used as source operand */
1018 DST_OP, /* register is used as destination operand */
1019 DST_OP_NO_MARK /* same as above, check only, don't mark */
1020};
1021
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001022static int cmp_subprogs(const void *a, const void *b)
1023{
Jiong Wang9c8105b2018-05-02 16:17:18 -04001024 return ((struct bpf_subprog_info *)a)->start -
1025 ((struct bpf_subprog_info *)b)->start;
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001026}
1027
1028static int find_subprog(struct bpf_verifier_env *env, int off)
1029{
Jiong Wang9c8105b2018-05-02 16:17:18 -04001030 struct bpf_subprog_info *p;
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001031
Jiong Wang9c8105b2018-05-02 16:17:18 -04001032 p = bsearch(&off, env->subprog_info, env->subprog_cnt,
1033 sizeof(env->subprog_info[0]), cmp_subprogs);
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001034 if (!p)
1035 return -ENOENT;
Jiong Wang9c8105b2018-05-02 16:17:18 -04001036 return p - env->subprog_info;
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001037
1038}
1039
1040static int add_subprog(struct bpf_verifier_env *env, int off)
1041{
1042 int insn_cnt = env->prog->len;
1043 int ret;
1044
1045 if (off >= insn_cnt || off < 0) {
1046 verbose(env, "call to invalid destination\n");
1047 return -EINVAL;
1048 }
1049 ret = find_subprog(env, off);
1050 if (ret >= 0)
1051 return 0;
Jiong Wang4cb3d992018-05-02 16:17:19 -04001052 if (env->subprog_cnt >= BPF_MAX_SUBPROGS) {
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001053 verbose(env, "too many subprograms\n");
1054 return -E2BIG;
1055 }
Jiong Wang9c8105b2018-05-02 16:17:18 -04001056 env->subprog_info[env->subprog_cnt++].start = off;
1057 sort(env->subprog_info, env->subprog_cnt,
1058 sizeof(env->subprog_info[0]), cmp_subprogs, NULL);
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001059 return 0;
1060}
1061
1062static int check_subprogs(struct bpf_verifier_env *env)
1063{
1064 int i, ret, subprog_start, subprog_end, off, cur_subprog = 0;
Jiong Wang9c8105b2018-05-02 16:17:18 -04001065 struct bpf_subprog_info *subprog = env->subprog_info;
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001066 struct bpf_insn *insn = env->prog->insnsi;
1067 int insn_cnt = env->prog->len;
1068
Jiong Wangf910cef2018-05-02 16:17:17 -04001069 /* Add entry function. */
1070 ret = add_subprog(env, 0);
1071 if (ret < 0)
1072 return ret;
1073
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001074 /* determine subprog starts. The end is one before the next starts */
1075 for (i = 0; i < insn_cnt; i++) {
1076 if (insn[i].code != (BPF_JMP | BPF_CALL))
1077 continue;
1078 if (insn[i].src_reg != BPF_PSEUDO_CALL)
1079 continue;
1080 if (!env->allow_ptr_leaks) {
1081 verbose(env, "function calls to other bpf functions are allowed for root only\n");
1082 return -EPERM;
1083 }
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001084 ret = add_subprog(env, i + insn[i].imm + 1);
1085 if (ret < 0)
1086 return ret;
1087 }
1088
Jiong Wang4cb3d992018-05-02 16:17:19 -04001089 /* Add a fake 'exit' subprog which could simplify subprog iteration
1090 * logic. 'subprog_cnt' should not be increased.
1091 */
1092 subprog[env->subprog_cnt].start = insn_cnt;
1093
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07001094 if (env->log.level & BPF_LOG_LEVEL2)
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001095 for (i = 0; i < env->subprog_cnt; i++)
Jiong Wang9c8105b2018-05-02 16:17:18 -04001096 verbose(env, "func#%d @%d\n", i, subprog[i].start);
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001097
1098 /* now check that all jumps are within the same subprog */
Jiong Wang4cb3d992018-05-02 16:17:19 -04001099 subprog_start = subprog[cur_subprog].start;
1100 subprog_end = subprog[cur_subprog + 1].start;
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001101 for (i = 0; i < insn_cnt; i++) {
1102 u8 code = insn[i].code;
1103
Jiong Wang092ed092019-01-26 12:26:01 -05001104 if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32)
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001105 goto next;
1106 if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL)
1107 goto next;
1108 off = i + insn[i].off + 1;
1109 if (off < subprog_start || off >= subprog_end) {
1110 verbose(env, "jump out of range from insn %d to %d\n", i, off);
1111 return -EINVAL;
1112 }
1113next:
1114 if (i == subprog_end - 1) {
1115 /* to avoid fall-through from one subprog into another
1116 * the last insn of the subprog should be either exit
1117 * or unconditional jump back
1118 */
1119 if (code != (BPF_JMP | BPF_EXIT) &&
1120 code != (BPF_JMP | BPF_JA)) {
1121 verbose(env, "last insn is not an exit or jmp\n");
1122 return -EINVAL;
1123 }
1124 subprog_start = subprog_end;
Jiong Wang4cb3d992018-05-02 16:17:19 -04001125 cur_subprog++;
1126 if (cur_subprog < env->subprog_cnt)
Jiong Wang9c8105b2018-05-02 16:17:18 -04001127 subprog_end = subprog[cur_subprog + 1].start;
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08001128 }
1129 }
1130 return 0;
1131}
1132
Edward Cree679c7822018-08-22 20:02:19 +01001133/* Parentage chain of this register (or stack slot) should take care of all
1134 * issues like callee-saved registers, stack slot allocation time, etc.
1135 */
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001136static int mark_reg_read(struct bpf_verifier_env *env,
Edward Cree679c7822018-08-22 20:02:19 +01001137 const struct bpf_reg_state *state,
1138 struct bpf_reg_state *parent)
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001139{
1140 bool writes = parent == state->parent; /* Observe write marks */
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07001141 int cnt = 0;
Edward Creedc503a82017-08-15 20:34:35 +01001142
1143 while (parent) {
1144 /* if read wasn't screened by an earlier write ... */
Edward Cree679c7822018-08-22 20:02:19 +01001145 if (writes && state->live & REG_LIVE_WRITTEN)
Edward Creedc503a82017-08-15 20:34:35 +01001146 break;
Alexei Starovoitov9242b5f2018-12-13 11:42:34 -08001147 if (parent->live & REG_LIVE_DONE) {
1148 verbose(env, "verifier BUG type %s var_off %lld off %d\n",
1149 reg_type_str[parent->type],
1150 parent->var_off.value, parent->off);
1151 return -EFAULT;
1152 }
Alexei Starovoitov25af32d2019-04-01 21:27:42 -07001153 if (parent->live & REG_LIVE_READ)
1154 /* The parentage chain never changes and
1155 * this parent was already marked as LIVE_READ.
1156 * There is no need to keep walking the chain again and
1157 * keep re-marking all parents as LIVE_READ.
1158 * This case happens when the same register is read
1159 * multiple times without writes into it in-between.
1160 */
1161 break;
Edward Creedc503a82017-08-15 20:34:35 +01001162 /* ... then we depend on parent's value */
Edward Cree679c7822018-08-22 20:02:19 +01001163 parent->live |= REG_LIVE_READ;
Edward Creedc503a82017-08-15 20:34:35 +01001164 state = parent;
1165 parent = state->parent;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001166 writes = true;
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07001167 cnt++;
Edward Creedc503a82017-08-15 20:34:35 +01001168 }
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07001169
1170 if (env->longest_mark_read_walk < cnt)
1171 env->longest_mark_read_walk = cnt;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001172 return 0;
Edward Creedc503a82017-08-15 20:34:35 +01001173}
1174
1175static int check_reg_arg(struct bpf_verifier_env *env, u32 regno,
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001176 enum reg_arg_type t)
1177{
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001178 struct bpf_verifier_state *vstate = env->cur_state;
1179 struct bpf_func_state *state = vstate->frame[vstate->curframe];
1180 struct bpf_reg_state *regs = state->regs;
Edward Creedc503a82017-08-15 20:34:35 +01001181
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001182 if (regno >= MAX_BPF_REG) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001183 verbose(env, "R%d is invalid\n", regno);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001184 return -EINVAL;
1185 }
1186
1187 if (t == SRC_OP) {
1188 /* check whether register used as source operand can be read */
1189 if (regs[regno].type == NOT_INIT) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001190 verbose(env, "R%d !read_ok\n", regno);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001191 return -EACCES;
1192 }
Edward Cree679c7822018-08-22 20:02:19 +01001193 /* We don't need to worry about FP liveness because it's read-only */
1194 if (regno != BPF_REG_FP)
1195 return mark_reg_read(env, &regs[regno],
1196 regs[regno].parent);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001197 } else {
1198 /* check whether register used as dest operand can be written to */
1199 if (regno == BPF_REG_FP) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001200 verbose(env, "frame pointer is read only\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001201 return -EACCES;
1202 }
Edward Creedc503a82017-08-15 20:34:35 +01001203 regs[regno].live |= REG_LIVE_WRITTEN;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001204 if (t == DST_OP)
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001205 mark_reg_unknown(env, regs, regno);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001206 }
1207 return 0;
1208}
1209
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07001210static bool is_spillable_regtype(enum bpf_reg_type type)
1211{
1212 switch (type) {
1213 case PTR_TO_MAP_VALUE:
1214 case PTR_TO_MAP_VALUE_OR_NULL:
1215 case PTR_TO_STACK:
1216 case PTR_TO_CTX:
Alexei Starovoitov969bf052016-05-05 19:49:10 -07001217 case PTR_TO_PACKET:
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02001218 case PTR_TO_PACKET_META:
Alexei Starovoitov969bf052016-05-05 19:49:10 -07001219 case PTR_TO_PACKET_END:
Petar Penkovd58e4682018-09-14 07:46:18 -07001220 case PTR_TO_FLOW_KEYS:
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07001221 case CONST_PTR_TO_MAP:
Joe Stringerc64b7982018-10-02 13:35:33 -07001222 case PTR_TO_SOCKET:
1223 case PTR_TO_SOCKET_OR_NULL:
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08001224 case PTR_TO_SOCK_COMMON:
1225 case PTR_TO_SOCK_COMMON_OR_NULL:
Martin KaFai Lau655a51e2019-02-09 23:22:24 -08001226 case PTR_TO_TCP_SOCK:
1227 case PTR_TO_TCP_SOCK_OR_NULL:
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07001228 return true;
1229 default:
1230 return false;
1231 }
1232}
1233
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08001234/* Does this register contain a constant zero? */
1235static bool register_is_null(struct bpf_reg_state *reg)
1236{
1237 return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0);
1238}
1239
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001240/* check_stack_read/write functions track spill/fill of registers,
1241 * stack boundary and alignment are checked in check_mem_access()
1242 */
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001243static int check_stack_write(struct bpf_verifier_env *env,
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001244 struct bpf_func_state *state, /* func where register points to */
Alexei Starovoitovaf86ca42018-05-15 09:27:05 -07001245 int off, int size, int value_regno, int insn_idx)
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001246{
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001247 struct bpf_func_state *cur; /* state of the current function */
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001248 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001249 enum bpf_reg_type type;
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001250
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001251 err = realloc_func_state(state, round_up(slot + 1, BPF_REG_SIZE),
Joe Stringerfd978bf72018-10-02 13:35:35 -07001252 state->acquired_refs, true);
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001253 if (err)
1254 return err;
Alexei Starovoitov9c3997602014-10-28 15:11:41 -07001255 /* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0,
1256 * so it's aligned access and [off, off + size) are within stack limits
1257 */
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001258 if (!env->allow_ptr_leaks &&
1259 state->stack[spi].slot_type[0] == STACK_SPILL &&
1260 size != BPF_REG_SIZE) {
1261 verbose(env, "attempt to corrupt spilled pointer on stack\n");
1262 return -EACCES;
1263 }
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001264
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001265 cur = env->cur_state->frame[env->cur_state->curframe];
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001266 if (value_regno >= 0 &&
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001267 is_spillable_regtype((type = cur->regs[value_regno].type))) {
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001268
1269 /* register containing pointer is being spilled into stack */
Alexei Starovoitov9c3997602014-10-28 15:11:41 -07001270 if (size != BPF_REG_SIZE) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001271 verbose(env, "invalid size of register spill\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001272 return -EACCES;
1273 }
1274
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001275 if (state != cur && type == PTR_TO_STACK) {
1276 verbose(env, "cannot spill pointers to stack into stack frame of the caller\n");
1277 return -EINVAL;
1278 }
1279
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001280 /* save register state */
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001281 state->stack[spi].spilled_ptr = cur->regs[value_regno];
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001282 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001283
Alexei Starovoitovaf86ca42018-05-15 09:27:05 -07001284 for (i = 0; i < BPF_REG_SIZE; i++) {
1285 if (state->stack[spi].slot_type[i] == STACK_MISC &&
1286 !env->allow_ptr_leaks) {
1287 int *poff = &env->insn_aux_data[insn_idx].sanitize_stack_off;
1288 int soff = (-spi - 1) * BPF_REG_SIZE;
1289
1290 /* detected reuse of integer stack slot with a pointer
1291 * which means either llvm is reusing stack slot or
1292 * an attacker is trying to exploit CVE-2018-3639
1293 * (speculative store bypass)
1294 * Have to sanitize that slot with preemptive
1295 * store of zero.
1296 */
1297 if (*poff && *poff != soff) {
1298 /* disallow programs where single insn stores
1299 * into two different stack slots, since verifier
1300 * cannot sanitize them
1301 */
1302 verbose(env,
1303 "insn %d cannot access two stack slots fp%d and fp%d",
1304 insn_idx, *poff, soff);
1305 return -EINVAL;
1306 }
1307 *poff = soff;
1308 }
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001309 state->stack[spi].slot_type[i] = STACK_SPILL;
Alexei Starovoitovaf86ca42018-05-15 09:27:05 -07001310 }
Alexei Starovoitov9c3997602014-10-28 15:11:41 -07001311 } else {
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08001312 u8 type = STACK_MISC;
1313
Edward Cree679c7822018-08-22 20:02:19 +01001314 /* regular write of data into stack destroys any spilled ptr */
1315 state->stack[spi].spilled_ptr.type = NOT_INIT;
Jiong Wang0bae2d42018-12-15 03:34:40 -05001316 /* Mark slots as STACK_MISC if they belonged to spilled ptr. */
1317 if (state->stack[spi].slot_type[0] == STACK_SPILL)
1318 for (i = 0; i < BPF_REG_SIZE; i++)
1319 state->stack[spi].slot_type[i] = STACK_MISC;
Alexei Starovoitov9c3997602014-10-28 15:11:41 -07001320
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08001321 /* only mark the slot as written if all 8 bytes were written
1322 * otherwise read propagation may incorrectly stop too soon
1323 * when stack slots are partially written.
1324 * This heuristic means that read propagation will be
1325 * conservative, since it will add reg_live_read marks
1326 * to stack slots all the way to first state when programs
1327 * writes+reads less than 8 bytes
1328 */
1329 if (size == BPF_REG_SIZE)
1330 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
1331
1332 /* when we zero initialize stack slots mark them as such */
1333 if (value_regno >= 0 &&
1334 register_is_null(&cur->regs[value_regno]))
1335 type = STACK_ZERO;
1336
Jiong Wang0bae2d42018-12-15 03:34:40 -05001337 /* Mark slots affected by this stack write. */
Alexei Starovoitov9c3997602014-10-28 15:11:41 -07001338 for (i = 0; i < size; i++)
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001339 state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] =
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08001340 type;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001341 }
1342 return 0;
1343}
1344
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001345static int check_stack_read(struct bpf_verifier_env *env,
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001346 struct bpf_func_state *reg_state /* func where register points to */,
1347 int off, int size, int value_regno)
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001348{
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001349 struct bpf_verifier_state *vstate = env->cur_state;
1350 struct bpf_func_state *state = vstate->frame[vstate->curframe];
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001351 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE;
1352 u8 *stype;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001353
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001354 if (reg_state->allocated_stack <= slot) {
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001355 verbose(env, "invalid read from stack off %d+0 size %d\n",
1356 off, size);
1357 return -EACCES;
1358 }
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001359 stype = reg_state->stack[spi].slot_type;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001360
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001361 if (stype[0] == STACK_SPILL) {
Alexei Starovoitov9c3997602014-10-28 15:11:41 -07001362 if (size != BPF_REG_SIZE) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001363 verbose(env, "invalid size of register spill\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001364 return -EACCES;
1365 }
Alexei Starovoitov9c3997602014-10-28 15:11:41 -07001366 for (i = 1; i < BPF_REG_SIZE; i++) {
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001367 if (stype[(slot - i) % BPF_REG_SIZE] != STACK_SPILL) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001368 verbose(env, "corrupted spill memory\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001369 return -EACCES;
1370 }
1371 }
1372
Edward Creedc503a82017-08-15 20:34:35 +01001373 if (value_regno >= 0) {
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001374 /* restore register state from stack */
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001375 state->regs[value_regno] = reg_state->stack[spi].spilled_ptr;
Alexei Starovoitov2f18f622017-11-30 21:31:38 -08001376 /* mark reg as written since spilled pointer state likely
1377 * has its liveness marks cleared by is_state_visited()
1378 * which resets stack/reg liveness for state transitions
1379 */
1380 state->regs[value_regno].live |= REG_LIVE_WRITTEN;
Edward Creedc503a82017-08-15 20:34:35 +01001381 }
Edward Cree679c7822018-08-22 20:02:19 +01001382 mark_reg_read(env, &reg_state->stack[spi].spilled_ptr,
1383 reg_state->stack[spi].spilled_ptr.parent);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001384 return 0;
1385 } else {
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08001386 int zeros = 0;
1387
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001388 for (i = 0; i < size; i++) {
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08001389 if (stype[(slot - i) % BPF_REG_SIZE] == STACK_MISC)
1390 continue;
1391 if (stype[(slot - i) % BPF_REG_SIZE] == STACK_ZERO) {
1392 zeros++;
1393 continue;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001394 }
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08001395 verbose(env, "invalid read from stack off %d+%d size %d\n",
1396 off, i, size);
1397 return -EACCES;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001398 }
Edward Cree679c7822018-08-22 20:02:19 +01001399 mark_reg_read(env, &reg_state->stack[spi].spilled_ptr,
1400 reg_state->stack[spi].spilled_ptr.parent);
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08001401 if (value_regno >= 0) {
1402 if (zeros == size) {
1403 /* any size read into register is zero extended,
1404 * so the whole register == const_zero
1405 */
1406 __mark_reg_const_zero(&state->regs[value_regno]);
1407 } else {
1408 /* have read misc data from the stack */
1409 mark_reg_unknown(env, state->regs, value_regno);
1410 }
1411 state->regs[value_regno].live |= REG_LIVE_WRITTEN;
1412 }
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001413 return 0;
1414 }
1415}
1416
Daniel Borkmanne4298d22019-01-03 00:58:31 +01001417static int check_stack_access(struct bpf_verifier_env *env,
1418 const struct bpf_reg_state *reg,
1419 int off, int size)
1420{
1421 /* Stack accesses must be at a fixed offset, so that we
1422 * can determine what type of data were returned. See
1423 * check_stack_read().
1424 */
1425 if (!tnum_is_const(reg->var_off)) {
1426 char tn_buf[48];
1427
1428 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
1429 verbose(env, "variable stack access var_off=%s off=%d size=%d",
1430 tn_buf, off, size);
1431 return -EACCES;
1432 }
1433
1434 if (off >= 0 || off < -MAX_BPF_STACK) {
1435 verbose(env, "invalid stack off=%d size=%d\n", off, size);
1436 return -EACCES;
1437 }
1438
1439 return 0;
1440}
1441
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001442/* check read/write into map element returned by bpf_map_lookup_elem() */
Edward Creef1174f72017-08-07 15:26:19 +01001443static int __check_map_access(struct bpf_verifier_env *env, u32 regno, int off,
Yonghong Song9fd29c02017-11-12 14:49:09 -08001444 int size, bool zero_size_allowed)
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001445{
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001446 struct bpf_reg_state *regs = cur_regs(env);
1447 struct bpf_map *map = regs[regno].map_ptr;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001448
Yonghong Song9fd29c02017-11-12 14:49:09 -08001449 if (off < 0 || size < 0 || (size == 0 && !zero_size_allowed) ||
1450 off + size > map->value_size) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001451 verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n",
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001452 map->value_size, off, size);
1453 return -EACCES;
1454 }
1455 return 0;
1456}
1457
Edward Creef1174f72017-08-07 15:26:19 +01001458/* check read/write into a map element with possible variable offset */
1459static int check_map_access(struct bpf_verifier_env *env, u32 regno,
Yonghong Song9fd29c02017-11-12 14:49:09 -08001460 int off, int size, bool zero_size_allowed)
Gianluca Borellodbcfe5f2017-01-09 10:19:46 -08001461{
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001462 struct bpf_verifier_state *vstate = env->cur_state;
1463 struct bpf_func_state *state = vstate->frame[vstate->curframe];
Gianluca Borellodbcfe5f2017-01-09 10:19:46 -08001464 struct bpf_reg_state *reg = &state->regs[regno];
1465 int err;
1466
Edward Creef1174f72017-08-07 15:26:19 +01001467 /* We may have adjusted the register to this map value, so we
1468 * need to try adding each of min_value and max_value to off
1469 * to make sure our theoretical access will be safe.
Gianluca Borellodbcfe5f2017-01-09 10:19:46 -08001470 */
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07001471 if (env->log.level & BPF_LOG_LEVEL)
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001472 print_verifier_state(env, state);
Daniel Borkmannb7137c42019-01-03 00:58:33 +01001473
Gianluca Borellodbcfe5f2017-01-09 10:19:46 -08001474 /* The minimum value is only important with signed
1475 * comparisons where we can't assume the floor of a
1476 * value is 0. If we are using signed variables for our
1477 * index'es we need to make sure that whatever we use
1478 * will have a set floor within our range.
1479 */
Daniel Borkmannb7137c42019-01-03 00:58:33 +01001480 if (reg->smin_value < 0 &&
1481 (reg->smin_value == S64_MIN ||
1482 (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) ||
1483 reg->smin_value + off < 0)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001484 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
Gianluca Borellodbcfe5f2017-01-09 10:19:46 -08001485 regno);
1486 return -EACCES;
1487 }
Yonghong Song9fd29c02017-11-12 14:49:09 -08001488 err = __check_map_access(env, regno, reg->smin_value + off, size,
1489 zero_size_allowed);
Gianluca Borellodbcfe5f2017-01-09 10:19:46 -08001490 if (err) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001491 verbose(env, "R%d min value is outside of the array range\n",
1492 regno);
Gianluca Borellodbcfe5f2017-01-09 10:19:46 -08001493 return err;
1494 }
1495
Edward Creeb03c9f92017-08-07 15:26:36 +01001496 /* If we haven't set a max value then we need to bail since we can't be
1497 * sure we won't do bad things.
1498 * If reg->umax_value + off could overflow, treat that as unbounded too.
Gianluca Borellodbcfe5f2017-01-09 10:19:46 -08001499 */
Edward Creeb03c9f92017-08-07 15:26:36 +01001500 if (reg->umax_value >= BPF_MAX_VAR_OFF) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001501 verbose(env, "R%d unbounded memory access, make sure to bounds check any array access into a map\n",
Gianluca Borellodbcfe5f2017-01-09 10:19:46 -08001502 regno);
1503 return -EACCES;
1504 }
Yonghong Song9fd29c02017-11-12 14:49:09 -08001505 err = __check_map_access(env, regno, reg->umax_value + off, size,
1506 zero_size_allowed);
Edward Creef1174f72017-08-07 15:26:19 +01001507 if (err)
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001508 verbose(env, "R%d max value is outside of the array range\n",
1509 regno);
Alexei Starovoitovd83525c2019-01-31 15:40:04 -08001510
1511 if (map_value_has_spin_lock(reg->map_ptr)) {
1512 u32 lock = reg->map_ptr->spin_lock_off;
1513
1514 /* if any part of struct bpf_spin_lock can be touched by
1515 * load/store reject this program.
1516 * To check that [x1, x2) overlaps with [y1, y2)
1517 * it is sufficient to check x1 < y2 && y1 < x2.
1518 */
1519 if (reg->smin_value + off < lock + sizeof(struct bpf_spin_lock) &&
1520 lock < reg->umax_value + off + size) {
1521 verbose(env, "bpf_spin_lock cannot be accessed directly by load/store\n");
1522 return -EACCES;
1523 }
1524 }
Edward Creef1174f72017-08-07 15:26:19 +01001525 return err;
Gianluca Borellodbcfe5f2017-01-09 10:19:46 -08001526}
1527
Alexei Starovoitov969bf052016-05-05 19:49:10 -07001528#define MAX_PACKET_OFF 0xffff
1529
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01001530static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,
Thomas Graf3a0af8f2016-11-30 17:10:10 +01001531 const struct bpf_call_arg_meta *meta,
1532 enum bpf_access_type t)
Brenden Blanco4acf6c02016-07-19 12:16:56 -07001533{
Daniel Borkmann36bbef52016-09-20 00:26:13 +02001534 switch (env->prog->type) {
Daniel Borkmann5d66fa72018-10-24 22:05:45 +02001535 /* Program types only with direct read access go here! */
Thomas Graf3a0af8f2016-11-30 17:10:10 +01001536 case BPF_PROG_TYPE_LWT_IN:
1537 case BPF_PROG_TYPE_LWT_OUT:
Mathieu Xhonneux004d4b22018-05-20 14:58:16 +01001538 case BPF_PROG_TYPE_LWT_SEG6LOCAL:
Martin KaFai Lau2dbb9b92018-08-08 01:01:25 -07001539 case BPF_PROG_TYPE_SK_REUSEPORT:
Daniel Borkmann5d66fa72018-10-24 22:05:45 +02001540 case BPF_PROG_TYPE_FLOW_DISSECTOR:
Daniel Borkmannd5563d32018-10-24 22:05:46 +02001541 case BPF_PROG_TYPE_CGROUP_SKB:
Thomas Graf3a0af8f2016-11-30 17:10:10 +01001542 if (t == BPF_WRITE)
1543 return false;
Alexander Alemayhu7e57fbb2017-02-14 00:02:35 +01001544 /* fallthrough */
Daniel Borkmann5d66fa72018-10-24 22:05:45 +02001545
1546 /* Program types with direct read + write access go here! */
Daniel Borkmann36bbef52016-09-20 00:26:13 +02001547 case BPF_PROG_TYPE_SCHED_CLS:
1548 case BPF_PROG_TYPE_SCHED_ACT:
Brenden Blanco4acf6c02016-07-19 12:16:56 -07001549 case BPF_PROG_TYPE_XDP:
Thomas Graf3a0af8f2016-11-30 17:10:10 +01001550 case BPF_PROG_TYPE_LWT_XMIT:
John Fastabend8a31db52017-08-15 22:33:09 -07001551 case BPF_PROG_TYPE_SK_SKB:
John Fastabend4f738ad2018-03-18 12:57:10 -07001552 case BPF_PROG_TYPE_SK_MSG:
Daniel Borkmann36bbef52016-09-20 00:26:13 +02001553 if (meta)
1554 return meta->pkt_access;
1555
1556 env->seen_direct_write = true;
Brenden Blanco4acf6c02016-07-19 12:16:56 -07001557 return true;
1558 default:
1559 return false;
1560 }
1561}
1562
Edward Creef1174f72017-08-07 15:26:19 +01001563static int __check_packet_access(struct bpf_verifier_env *env, u32 regno,
Yonghong Song9fd29c02017-11-12 14:49:09 -08001564 int off, int size, bool zero_size_allowed)
Alexei Starovoitov969bf052016-05-05 19:49:10 -07001565{
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001566 struct bpf_reg_state *regs = cur_regs(env);
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01001567 struct bpf_reg_state *reg = &regs[regno];
Alexei Starovoitov969bf052016-05-05 19:49:10 -07001568
Yonghong Song9fd29c02017-11-12 14:49:09 -08001569 if (off < 0 || size < 0 || (size == 0 && !zero_size_allowed) ||
1570 (u64)off + size > reg->range) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001571 verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n",
Alexei Starovoitovd91b28e2016-05-19 18:17:13 -07001572 off, size, regno, reg->id, reg->off, reg->range);
Alexei Starovoitov969bf052016-05-05 19:49:10 -07001573 return -EACCES;
1574 }
1575 return 0;
1576}
1577
Edward Creef1174f72017-08-07 15:26:19 +01001578static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off,
Yonghong Song9fd29c02017-11-12 14:49:09 -08001579 int size, bool zero_size_allowed)
Edward Creef1174f72017-08-07 15:26:19 +01001580{
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07001581 struct bpf_reg_state *regs = cur_regs(env);
Edward Creef1174f72017-08-07 15:26:19 +01001582 struct bpf_reg_state *reg = &regs[regno];
1583 int err;
1584
1585 /* We may have added a variable offset to the packet pointer; but any
1586 * reg->range we have comes after that. We are only checking the fixed
1587 * offset.
1588 */
1589
1590 /* We don't allow negative numbers, because we aren't tracking enough
1591 * detail to prove they're safe.
1592 */
Edward Creeb03c9f92017-08-07 15:26:36 +01001593 if (reg->smin_value < 0) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001594 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
Edward Creef1174f72017-08-07 15:26:19 +01001595 regno);
1596 return -EACCES;
1597 }
Yonghong Song9fd29c02017-11-12 14:49:09 -08001598 err = __check_packet_access(env, regno, off, size, zero_size_allowed);
Edward Creef1174f72017-08-07 15:26:19 +01001599 if (err) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001600 verbose(env, "R%d offset is outside of the packet\n", regno);
Edward Creef1174f72017-08-07 15:26:19 +01001601 return err;
1602 }
Jiong Wange6478152018-11-08 04:08:42 -05001603
1604 /* __check_packet_access has made sure "off + size - 1" is within u16.
1605 * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff,
1606 * otherwise find_good_pkt_pointers would have refused to set range info
1607 * that __check_packet_access would have rejected this pkt access.
1608 * Therefore, "off + reg->umax_value + size - 1" won't overflow u32.
1609 */
1610 env->prog->aux->max_pkt_offset =
1611 max_t(u32, env->prog->aux->max_pkt_offset,
1612 off + reg->umax_value + size - 1);
1613
Edward Creef1174f72017-08-07 15:26:19 +01001614 return err;
1615}
1616
1617/* check access to 'struct bpf_context' fields. Supports fixed offsets only */
Yonghong Song31fd8582017-06-13 15:52:13 -07001618static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size,
Alexei Starovoitov19de99f2016-06-15 18:25:38 -07001619 enum bpf_access_type t, enum bpf_reg_type *reg_type)
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001620{
Daniel Borkmannf96da092017-07-02 02:13:27 +02001621 struct bpf_insn_access_aux info = {
1622 .reg_type = *reg_type,
1623 };
Yonghong Song31fd8582017-06-13 15:52:13 -07001624
Jakub Kicinski4f9218a2017-10-16 16:40:55 -07001625 if (env->ops->is_valid_access &&
Andrey Ignatov5e43f892018-03-30 15:08:00 -07001626 env->ops->is_valid_access(off, size, t, env->prog, &info)) {
Daniel Borkmannf96da092017-07-02 02:13:27 +02001627 /* A non zero info.ctx_field_size indicates that this field is a
1628 * candidate for later verifier transformation to load the whole
1629 * field and then apply a mask when accessed with a narrower
1630 * access than actual ctx access size. A zero info.ctx_field_size
1631 * will only allow for whole field access and rejects any other
1632 * type of narrower access.
Yonghong Song31fd8582017-06-13 15:52:13 -07001633 */
Yonghong Song23994632017-06-22 15:07:39 -07001634 *reg_type = info.reg_type;
Yonghong Song31fd8582017-06-13 15:52:13 -07001635
Jakub Kicinski4f9218a2017-10-16 16:40:55 -07001636 env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size;
Alexei Starovoitov32bbe002016-04-06 18:43:28 -07001637 /* remember the offset of last byte accessed in ctx */
1638 if (env->prog->aux->max_ctx_offset < off + size)
1639 env->prog->aux->max_ctx_offset = off + size;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001640 return 0;
Alexei Starovoitov32bbe002016-04-06 18:43:28 -07001641 }
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001642
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001643 verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001644 return -EACCES;
1645}
1646
Petar Penkovd58e4682018-09-14 07:46:18 -07001647static int check_flow_keys_access(struct bpf_verifier_env *env, int off,
1648 int size)
1649{
1650 if (size < 0 || off < 0 ||
1651 (u64)off + size > sizeof(struct bpf_flow_keys)) {
1652 verbose(env, "invalid access to flow keys off=%d size=%d\n",
1653 off, size);
1654 return -EACCES;
1655 }
1656 return 0;
1657}
1658
Martin KaFai Lau5f456642019-02-08 22:25:54 -08001659static int check_sock_access(struct bpf_verifier_env *env, int insn_idx,
1660 u32 regno, int off, int size,
1661 enum bpf_access_type t)
Joe Stringerc64b7982018-10-02 13:35:33 -07001662{
1663 struct bpf_reg_state *regs = cur_regs(env);
1664 struct bpf_reg_state *reg = &regs[regno];
Martin KaFai Lau5f456642019-02-08 22:25:54 -08001665 struct bpf_insn_access_aux info = {};
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08001666 bool valid;
Joe Stringerc64b7982018-10-02 13:35:33 -07001667
1668 if (reg->smin_value < 0) {
1669 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
1670 regno);
1671 return -EACCES;
1672 }
1673
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08001674 switch (reg->type) {
1675 case PTR_TO_SOCK_COMMON:
1676 valid = bpf_sock_common_is_valid_access(off, size, t, &info);
1677 break;
1678 case PTR_TO_SOCKET:
1679 valid = bpf_sock_is_valid_access(off, size, t, &info);
1680 break;
Martin KaFai Lau655a51e2019-02-09 23:22:24 -08001681 case PTR_TO_TCP_SOCK:
1682 valid = bpf_tcp_sock_is_valid_access(off, size, t, &info);
1683 break;
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08001684 default:
1685 valid = false;
Joe Stringerc64b7982018-10-02 13:35:33 -07001686 }
1687
Martin KaFai Lau5f456642019-02-08 22:25:54 -08001688
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08001689 if (valid) {
1690 env->insn_aux_data[insn_idx].ctx_field_size =
1691 info.ctx_field_size;
1692 return 0;
1693 }
1694
1695 verbose(env, "R%d invalid %s access off=%d size=%d\n",
1696 regno, reg_type_str[reg->type], off, size);
1697
1698 return -EACCES;
Joe Stringerc64b7982018-10-02 13:35:33 -07001699}
1700
Daniel Borkmann4cabc5b2017-07-21 00:00:21 +02001701static bool __is_pointer_value(bool allow_ptr_leaks,
1702 const struct bpf_reg_state *reg)
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07001703{
Daniel Borkmann4cabc5b2017-07-21 00:00:21 +02001704 if (allow_ptr_leaks)
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07001705 return false;
1706
Edward Creef1174f72017-08-07 15:26:19 +01001707 return reg->type != SCALAR_VALUE;
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07001708}
1709
Daniel Borkmann2a159c62018-10-21 02:09:24 +02001710static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)
1711{
1712 return cur_regs(env) + regno;
1713}
1714
Daniel Borkmann4cabc5b2017-07-21 00:00:21 +02001715static bool is_pointer_value(struct bpf_verifier_env *env, int regno)
1716{
Daniel Borkmann2a159c62018-10-21 02:09:24 +02001717 return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno));
Daniel Borkmann4cabc5b2017-07-21 00:00:21 +02001718}
1719
Daniel Borkmannf37a8cb2018-01-16 23:30:10 +01001720static bool is_ctx_reg(struct bpf_verifier_env *env, int regno)
1721{
Daniel Borkmann2a159c62018-10-21 02:09:24 +02001722 const struct bpf_reg_state *reg = reg_state(env, regno);
Daniel Borkmannf37a8cb2018-01-16 23:30:10 +01001723
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08001724 return reg->type == PTR_TO_CTX;
1725}
1726
1727static bool is_sk_reg(struct bpf_verifier_env *env, int regno)
1728{
1729 const struct bpf_reg_state *reg = reg_state(env, regno);
1730
1731 return type_is_sk_pointer(reg->type);
Daniel Borkmannf37a8cb2018-01-16 23:30:10 +01001732}
1733
Daniel Borkmannca369602018-02-23 22:29:05 +01001734static bool is_pkt_reg(struct bpf_verifier_env *env, int regno)
1735{
Daniel Borkmann2a159c62018-10-21 02:09:24 +02001736 const struct bpf_reg_state *reg = reg_state(env, regno);
Daniel Borkmannca369602018-02-23 22:29:05 +01001737
1738 return type_is_pkt_pointer(reg->type);
1739}
1740
Daniel Borkmann4b5defd2018-10-21 02:09:25 +02001741static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno)
1742{
1743 const struct bpf_reg_state *reg = reg_state(env, regno);
1744
1745 /* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */
1746 return reg->type == PTR_TO_FLOW_KEYS;
1747}
1748
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001749static int check_pkt_ptr_alignment(struct bpf_verifier_env *env,
1750 const struct bpf_reg_state *reg,
David S. Millerd1174412017-05-10 11:22:52 -07001751 int off, int size, bool strict)
Alexei Starovoitov969bf052016-05-05 19:49:10 -07001752{
Edward Creef1174f72017-08-07 15:26:19 +01001753 struct tnum reg_off;
David S. Millere07b98d2017-05-10 11:38:07 -07001754 int ip_align;
David S. Millerd1174412017-05-10 11:22:52 -07001755
1756 /* Byte size accesses are always allowed. */
1757 if (!strict || size == 1)
1758 return 0;
1759
David S. Millere4eda882017-05-22 12:27:07 -04001760 /* For platforms that do not have a Kconfig enabling
1761 * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of
1762 * NET_IP_ALIGN is universally set to '2'. And on platforms
1763 * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get
1764 * to this code only in strict mode where we want to emulate
1765 * the NET_IP_ALIGN==2 checking. Therefore use an
1766 * unconditional IP align value of '2'.
David S. Millere07b98d2017-05-10 11:38:07 -07001767 */
David S. Millere4eda882017-05-22 12:27:07 -04001768 ip_align = 2;
Edward Creef1174f72017-08-07 15:26:19 +01001769
1770 reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off));
1771 if (!tnum_is_aligned(reg_off, size)) {
1772 char tn_buf[48];
1773
1774 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001775 verbose(env,
1776 "misaligned packet access off %d+%s+%d+%d size %d\n",
Edward Creef1174f72017-08-07 15:26:19 +01001777 ip_align, tn_buf, reg->off, off, size);
Alexei Starovoitov969bf052016-05-05 19:49:10 -07001778 return -EACCES;
1779 }
Daniel Borkmann79adffc2017-03-31 02:24:03 +02001780
Alexei Starovoitov969bf052016-05-05 19:49:10 -07001781 return 0;
1782}
1783
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001784static int check_generic_ptr_alignment(struct bpf_verifier_env *env,
1785 const struct bpf_reg_state *reg,
Edward Creef1174f72017-08-07 15:26:19 +01001786 const char *pointer_desc,
1787 int off, int size, bool strict)
Daniel Borkmann79adffc2017-03-31 02:24:03 +02001788{
Edward Creef1174f72017-08-07 15:26:19 +01001789 struct tnum reg_off;
1790
1791 /* Byte size accesses are always allowed. */
1792 if (!strict || size == 1)
1793 return 0;
1794
1795 reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off));
1796 if (!tnum_is_aligned(reg_off, size)) {
1797 char tn_buf[48];
1798
1799 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001800 verbose(env, "misaligned %saccess off %s+%d+%d size %d\n",
Edward Creef1174f72017-08-07 15:26:19 +01001801 pointer_desc, tn_buf, reg->off, off, size);
Daniel Borkmann79adffc2017-03-31 02:24:03 +02001802 return -EACCES;
1803 }
1804
1805 return 0;
1806}
1807
David S. Millere07b98d2017-05-10 11:38:07 -07001808static int check_ptr_alignment(struct bpf_verifier_env *env,
Daniel Borkmannca369602018-02-23 22:29:05 +01001809 const struct bpf_reg_state *reg, int off,
1810 int size, bool strict_alignment_once)
Daniel Borkmann79adffc2017-03-31 02:24:03 +02001811{
Daniel Borkmannca369602018-02-23 22:29:05 +01001812 bool strict = env->strict_alignment || strict_alignment_once;
Edward Creef1174f72017-08-07 15:26:19 +01001813 const char *pointer_desc = "";
David S. Millerd1174412017-05-10 11:22:52 -07001814
Daniel Borkmann79adffc2017-03-31 02:24:03 +02001815 switch (reg->type) {
1816 case PTR_TO_PACKET:
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02001817 case PTR_TO_PACKET_META:
1818 /* Special case, because of NET_IP_ALIGN. Given metadata sits
1819 * right in front, treat it the very same way.
1820 */
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001821 return check_pkt_ptr_alignment(env, reg, off, size, strict);
Petar Penkovd58e4682018-09-14 07:46:18 -07001822 case PTR_TO_FLOW_KEYS:
1823 pointer_desc = "flow keys ";
1824 break;
Edward Creef1174f72017-08-07 15:26:19 +01001825 case PTR_TO_MAP_VALUE:
1826 pointer_desc = "value ";
1827 break;
1828 case PTR_TO_CTX:
1829 pointer_desc = "context ";
1830 break;
1831 case PTR_TO_STACK:
1832 pointer_desc = "stack ";
Jann Horna5ec6ae2017-12-18 20:11:58 -08001833 /* The stack spill tracking logic in check_stack_write()
1834 * and check_stack_read() relies on stack accesses being
1835 * aligned.
1836 */
1837 strict = true;
Edward Creef1174f72017-08-07 15:26:19 +01001838 break;
Joe Stringerc64b7982018-10-02 13:35:33 -07001839 case PTR_TO_SOCKET:
1840 pointer_desc = "sock ";
1841 break;
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08001842 case PTR_TO_SOCK_COMMON:
1843 pointer_desc = "sock_common ";
1844 break;
Martin KaFai Lau655a51e2019-02-09 23:22:24 -08001845 case PTR_TO_TCP_SOCK:
1846 pointer_desc = "tcp_sock ";
1847 break;
Daniel Borkmann79adffc2017-03-31 02:24:03 +02001848 default:
Edward Creef1174f72017-08-07 15:26:19 +01001849 break;
Daniel Borkmann79adffc2017-03-31 02:24:03 +02001850 }
Jakub Kicinski61bd5212017-10-09 10:30:11 -07001851 return check_generic_ptr_alignment(env, reg, pointer_desc, off, size,
1852 strict);
Daniel Borkmann79adffc2017-03-31 02:24:03 +02001853}
1854
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001855static int update_stack_depth(struct bpf_verifier_env *env,
1856 const struct bpf_func_state *func,
1857 int off)
1858{
Jiong Wang9c8105b2018-05-02 16:17:18 -04001859 u16 stack = env->subprog_info[func->subprogno].stack_depth;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001860
1861 if (stack >= -off)
1862 return 0;
1863
1864 /* update known max for given subprogram */
Jiong Wang9c8105b2018-05-02 16:17:18 -04001865 env->subprog_info[func->subprogno].stack_depth = -off;
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001866 return 0;
1867}
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001868
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001869/* starting from main bpf function walk all instructions of the function
1870 * and recursively walk all callees that given function can call.
1871 * Ignore jump and exit insns.
1872 * Since recursion is prevented by check_cfg() this algorithm
1873 * only needs a local stack of MAX_CALL_FRAMES to remember callsites
1874 */
1875static int check_max_stack_depth(struct bpf_verifier_env *env)
1876{
Jiong Wang9c8105b2018-05-02 16:17:18 -04001877 int depth = 0, frame = 0, idx = 0, i = 0, subprog_end;
1878 struct bpf_subprog_info *subprog = env->subprog_info;
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001879 struct bpf_insn *insn = env->prog->insnsi;
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001880 int ret_insn[MAX_CALL_FRAMES];
1881 int ret_prog[MAX_CALL_FRAMES];
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001882
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001883process_func:
1884 /* round up to 32-bytes, since this is granularity
1885 * of interpreter stack size
1886 */
Jiong Wang9c8105b2018-05-02 16:17:18 -04001887 depth += round_up(max_t(u32, subprog[idx].stack_depth, 1), 32);
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001888 if (depth > MAX_BPF_STACK) {
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001889 verbose(env, "combined stack size of %d calls is %d. Too large\n",
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001890 frame + 1, depth);
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001891 return -EACCES;
1892 }
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001893continue_func:
Jiong Wang4cb3d992018-05-02 16:17:19 -04001894 subprog_end = subprog[idx + 1].start;
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001895 for (; i < subprog_end; i++) {
1896 if (insn[i].code != (BPF_JMP | BPF_CALL))
1897 continue;
1898 if (insn[i].src_reg != BPF_PSEUDO_CALL)
1899 continue;
1900 /* remember insn and function to return to */
1901 ret_insn[frame] = i + 1;
Jiong Wang9c8105b2018-05-02 16:17:18 -04001902 ret_prog[frame] = idx;
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001903
1904 /* find the callee */
1905 i = i + insn[i].imm + 1;
Jiong Wang9c8105b2018-05-02 16:17:18 -04001906 idx = find_subprog(env, i);
1907 if (idx < 0) {
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001908 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
1909 i);
1910 return -EFAULT;
1911 }
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001912 frame++;
1913 if (frame >= MAX_CALL_FRAMES) {
1914 WARN_ONCE(1, "verifier bug. Call stack is too deep\n");
1915 return -EFAULT;
1916 }
1917 goto process_func;
1918 }
1919 /* end of for() loop means the last insn of the 'subprog'
1920 * was reached. Doesn't matter whether it was JA or EXIT
1921 */
1922 if (frame == 0)
1923 return 0;
Jiong Wang9c8105b2018-05-02 16:17:18 -04001924 depth -= round_up(max_t(u32, subprog[idx].stack_depth, 1), 32);
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001925 frame--;
1926 i = ret_insn[frame];
Jiong Wang9c8105b2018-05-02 16:17:18 -04001927 idx = ret_prog[frame];
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08001928 goto continue_func;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08001929}
1930
David S. Miller19d28fb2018-01-11 21:27:54 -05001931#ifndef CONFIG_BPF_JIT_ALWAYS_ON
Alexei Starovoitov1ea47e02017-12-14 17:55:13 -08001932static int get_callee_stack_depth(struct bpf_verifier_env *env,
1933 const struct bpf_insn *insn, int idx)
1934{
1935 int start = idx + insn->imm + 1, subprog;
1936
1937 subprog = find_subprog(env, start);
1938 if (subprog < 0) {
1939 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
1940 start);
1941 return -EFAULT;
1942 }
Jiong Wang9c8105b2018-05-02 16:17:18 -04001943 return env->subprog_info[subprog].stack_depth;
Alexei Starovoitov1ea47e02017-12-14 17:55:13 -08001944}
David S. Miller19d28fb2018-01-11 21:27:54 -05001945#endif
Alexei Starovoitov1ea47e02017-12-14 17:55:13 -08001946
Daniel Borkmann58990d12018-06-07 17:40:03 +02001947static int check_ctx_reg(struct bpf_verifier_env *env,
1948 const struct bpf_reg_state *reg, int regno)
1949{
1950 /* Access to ctx or passing it to a helper is only allowed in
1951 * its original, unmodified form.
1952 */
1953
1954 if (reg->off) {
1955 verbose(env, "dereference of modified ctx ptr R%d off=%d disallowed\n",
1956 regno, reg->off);
1957 return -EACCES;
1958 }
1959
1960 if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
1961 char tn_buf[48];
1962
1963 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
1964 verbose(env, "variable ctx access var_off=%s disallowed\n", tn_buf);
1965 return -EACCES;
1966 }
1967
1968 return 0;
1969}
1970
Jann Horn0c17d1d2017-12-18 20:11:55 -08001971/* truncate register to smaller size (in bytes)
1972 * must be called with size < BPF_REG_SIZE
1973 */
1974static void coerce_reg_to_size(struct bpf_reg_state *reg, int size)
1975{
1976 u64 mask;
1977
1978 /* clear high bits in bit representation */
1979 reg->var_off = tnum_cast(reg->var_off, size);
1980
1981 /* fix arithmetic bounds */
1982 mask = ((u64)1 << (size * 8)) - 1;
1983 if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) {
1984 reg->umin_value &= mask;
1985 reg->umax_value &= mask;
1986 } else {
1987 reg->umin_value = 0;
1988 reg->umax_value = mask;
1989 }
1990 reg->smin_value = reg->umin_value;
1991 reg->smax_value = reg->umax_value;
1992}
1993
Alexei Starovoitov17a52672014-09-26 00:17:06 -07001994/* check whether memory at (regno + off) is accessible for t = (read | write)
1995 * if t==write, value_regno is a register which value is stored into memory
1996 * if t==read, value_regno is a register which will receive the value from memory
1997 * if t==write && value_regno==-1, some unknown value is stored into memory
1998 * if t==read && value_regno==-1, don't care what we read from memory
1999 */
Daniel Borkmannca369602018-02-23 22:29:05 +01002000static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno,
2001 int off, int bpf_size, enum bpf_access_type t,
2002 int value_regno, bool strict_alignment_once)
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002003{
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07002004 struct bpf_reg_state *regs = cur_regs(env);
2005 struct bpf_reg_state *reg = regs + regno;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002006 struct bpf_func_state *state;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002007 int size, err = 0;
2008
2009 size = bpf_size_to_bytes(bpf_size);
2010 if (size < 0)
2011 return size;
2012
Edward Creef1174f72017-08-07 15:26:19 +01002013 /* alignment checks will add in reg->off themselves */
Daniel Borkmannca369602018-02-23 22:29:05 +01002014 err = check_ptr_alignment(env, reg, off, size, strict_alignment_once);
Alexei Starovoitov969bf052016-05-05 19:49:10 -07002015 if (err)
2016 return err;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002017
Edward Creef1174f72017-08-07 15:26:19 +01002018 /* for access checks, reg->off is just part of off */
2019 off += reg->off;
2020
2021 if (reg->type == PTR_TO_MAP_VALUE) {
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07002022 if (t == BPF_WRITE && value_regno >= 0 &&
2023 is_pointer_value(env, value_regno)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002024 verbose(env, "R%d leaks addr into map\n", value_regno);
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07002025 return -EACCES;
2026 }
Josef Bacik48461132016-09-28 10:54:32 -04002027
Yonghong Song9fd29c02017-11-12 14:49:09 -08002028 err = check_map_access(env, regno, off, size, false);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002029 if (!err && t == BPF_READ && value_regno >= 0)
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07002030 mark_reg_unknown(env, regs, value_regno);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002031
Alexei Starovoitov1a0dc1a2016-05-05 19:49:09 -07002032 } else if (reg->type == PTR_TO_CTX) {
Edward Creef1174f72017-08-07 15:26:19 +01002033 enum bpf_reg_type reg_type = SCALAR_VALUE;
Alexei Starovoitov19de99f2016-06-15 18:25:38 -07002034
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07002035 if (t == BPF_WRITE && value_regno >= 0 &&
2036 is_pointer_value(env, value_regno)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002037 verbose(env, "R%d leaks addr into ctx\n", value_regno);
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07002038 return -EACCES;
2039 }
Edward Creef1174f72017-08-07 15:26:19 +01002040
Daniel Borkmann58990d12018-06-07 17:40:03 +02002041 err = check_ctx_reg(env, reg, regno);
2042 if (err < 0)
2043 return err;
2044
Yonghong Song31fd8582017-06-13 15:52:13 -07002045 err = check_ctx_access(env, insn_idx, off, size, t, &reg_type);
Alexei Starovoitov969bf052016-05-05 19:49:10 -07002046 if (!err && t == BPF_READ && value_regno >= 0) {
Edward Creef1174f72017-08-07 15:26:19 +01002047 /* ctx access returns either a scalar, or a
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02002048 * PTR_TO_PACKET[_META,_END]. In the latter
2049 * case, we know the offset is zero.
Edward Creef1174f72017-08-07 15:26:19 +01002050 */
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08002051 if (reg_type == SCALAR_VALUE) {
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07002052 mark_reg_unknown(env, regs, value_regno);
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08002053 } else {
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07002054 mark_reg_known_zero(env, regs,
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002055 value_regno);
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08002056 if (reg_type_may_be_null(reg_type))
2057 regs[value_regno].id = ++env->id_gen;
2058 }
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07002059 regs[value_regno].type = reg_type;
Alexei Starovoitov969bf052016-05-05 19:49:10 -07002060 }
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002061
Edward Creef1174f72017-08-07 15:26:19 +01002062 } else if (reg->type == PTR_TO_STACK) {
Edward Creef1174f72017-08-07 15:26:19 +01002063 off += reg->var_off.value;
Daniel Borkmanne4298d22019-01-03 00:58:31 +01002064 err = check_stack_access(env, reg, off, size);
2065 if (err)
2066 return err;
Alexei Starovoitov87266792017-05-30 13:31:29 -07002067
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002068 state = func(env, reg);
2069 err = update_stack_depth(env, state, off);
2070 if (err)
2071 return err;
Alexei Starovoitov87266792017-05-30 13:31:29 -07002072
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07002073 if (t == BPF_WRITE)
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002074 err = check_stack_write(env, state, off, size,
Alexei Starovoitovaf86ca42018-05-15 09:27:05 -07002075 value_regno, insn_idx);
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07002076 else
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002077 err = check_stack_read(env, state, off, size,
2078 value_regno);
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02002079 } else if (reg_is_pkt_pointer(reg)) {
Thomas Graf3a0af8f2016-11-30 17:10:10 +01002080 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002081 verbose(env, "cannot write into packet\n");
Alexei Starovoitov969bf052016-05-05 19:49:10 -07002082 return -EACCES;
2083 }
Brenden Blanco4acf6c02016-07-19 12:16:56 -07002084 if (t == BPF_WRITE && value_regno >= 0 &&
2085 is_pointer_value(env, value_regno)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002086 verbose(env, "R%d leaks addr into packet\n",
2087 value_regno);
Brenden Blanco4acf6c02016-07-19 12:16:56 -07002088 return -EACCES;
2089 }
Yonghong Song9fd29c02017-11-12 14:49:09 -08002090 err = check_packet_access(env, regno, off, size, false);
Alexei Starovoitov969bf052016-05-05 19:49:10 -07002091 if (!err && t == BPF_READ && value_regno >= 0)
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07002092 mark_reg_unknown(env, regs, value_regno);
Petar Penkovd58e4682018-09-14 07:46:18 -07002093 } else if (reg->type == PTR_TO_FLOW_KEYS) {
2094 if (t == BPF_WRITE && value_regno >= 0 &&
2095 is_pointer_value(env, value_regno)) {
2096 verbose(env, "R%d leaks addr into flow keys\n",
2097 value_regno);
2098 return -EACCES;
2099 }
2100
2101 err = check_flow_keys_access(env, off, size);
2102 if (!err && t == BPF_READ && value_regno >= 0)
2103 mark_reg_unknown(env, regs, value_regno);
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08002104 } else if (type_is_sk_pointer(reg->type)) {
Joe Stringerc64b7982018-10-02 13:35:33 -07002105 if (t == BPF_WRITE) {
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08002106 verbose(env, "R%d cannot write into %s\n",
2107 regno, reg_type_str[reg->type]);
Joe Stringerc64b7982018-10-02 13:35:33 -07002108 return -EACCES;
2109 }
Martin KaFai Lau5f456642019-02-08 22:25:54 -08002110 err = check_sock_access(env, insn_idx, regno, off, size, t);
Joe Stringerc64b7982018-10-02 13:35:33 -07002111 if (!err && value_regno >= 0)
2112 mark_reg_unknown(env, regs, value_regno);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002113 } else {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002114 verbose(env, "R%d invalid mem access '%s'\n", regno,
2115 reg_type_str[reg->type]);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002116 return -EACCES;
2117 }
Alexei Starovoitov969bf052016-05-05 19:49:10 -07002118
Edward Creef1174f72017-08-07 15:26:19 +01002119 if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ &&
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07002120 regs[value_regno].type == SCALAR_VALUE) {
Edward Creef1174f72017-08-07 15:26:19 +01002121 /* b/h/w load zero-extends, mark upper bits as known 0 */
Jann Horn0c17d1d2017-12-18 20:11:55 -08002122 coerce_reg_to_size(&regs[value_regno], size);
Alexei Starovoitov969bf052016-05-05 19:49:10 -07002123 }
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002124 return err;
2125}
2126
Yonghong Song31fd8582017-06-13 15:52:13 -07002127static int check_xadd(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn)
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002128{
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002129 int err;
2130
2131 if ((BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) ||
2132 insn->imm != 0) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002133 verbose(env, "BPF_XADD uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002134 return -EINVAL;
2135 }
2136
2137 /* check src1 operand */
Edward Creedc503a82017-08-15 20:34:35 +01002138 err = check_reg_arg(env, insn->src_reg, SRC_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002139 if (err)
2140 return err;
2141
2142 /* check src2 operand */
Edward Creedc503a82017-08-15 20:34:35 +01002143 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002144 if (err)
2145 return err;
2146
Daniel Borkmann6bdf6ab2017-06-29 03:04:59 +02002147 if (is_pointer_value(env, insn->src_reg)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002148 verbose(env, "R%d leaks addr into mem\n", insn->src_reg);
Daniel Borkmann6bdf6ab2017-06-29 03:04:59 +02002149 return -EACCES;
2150 }
2151
Daniel Borkmannca369602018-02-23 22:29:05 +01002152 if (is_ctx_reg(env, insn->dst_reg) ||
Daniel Borkmann4b5defd2018-10-21 02:09:25 +02002153 is_pkt_reg(env, insn->dst_reg) ||
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08002154 is_flow_key_reg(env, insn->dst_reg) ||
2155 is_sk_reg(env, insn->dst_reg)) {
Daniel Borkmannca369602018-02-23 22:29:05 +01002156 verbose(env, "BPF_XADD stores into R%d %s is not allowed\n",
Daniel Borkmann2a159c62018-10-21 02:09:24 +02002157 insn->dst_reg,
2158 reg_type_str[reg_state(env, insn->dst_reg)->type]);
Daniel Borkmannf37a8cb2018-01-16 23:30:10 +01002159 return -EACCES;
2160 }
2161
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002162 /* check whether atomic_add can read the memory */
Yonghong Song31fd8582017-06-13 15:52:13 -07002163 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
Daniel Borkmannca369602018-02-23 22:29:05 +01002164 BPF_SIZE(insn->code), BPF_READ, -1, true);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002165 if (err)
2166 return err;
2167
2168 /* check whether atomic_add can write into the same memory */
Yonghong Song31fd8582017-06-13 15:52:13 -07002169 return check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
Daniel Borkmannca369602018-02-23 22:29:05 +01002170 BPF_SIZE(insn->code), BPF_WRITE, -1, true);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002171}
2172
Andrey Ignatov2011fcc2019-03-28 18:01:57 -07002173static int __check_stack_boundary(struct bpf_verifier_env *env, u32 regno,
2174 int off, int access_size,
2175 bool zero_size_allowed)
2176{
2177 struct bpf_reg_state *reg = reg_state(env, regno);
2178
2179 if (off >= 0 || off < -MAX_BPF_STACK || off + access_size > 0 ||
2180 access_size < 0 || (access_size == 0 && !zero_size_allowed)) {
2181 if (tnum_is_const(reg->var_off)) {
2182 verbose(env, "invalid stack type R%d off=%d access_size=%d\n",
2183 regno, off, access_size);
2184 } else {
2185 char tn_buf[48];
2186
2187 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
2188 verbose(env, "invalid stack type R%d var_off=%s access_size=%d\n",
2189 regno, tn_buf, access_size);
2190 }
2191 return -EACCES;
2192 }
2193 return 0;
2194}
2195
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002196/* when register 'regno' is passed into function that will read 'access_size'
2197 * bytes from that pointer, make sure that it's within stack boundary
Edward Creef1174f72017-08-07 15:26:19 +01002198 * and all elements of stack are initialized.
2199 * Unlike most pointer bounds-checking functions, this one doesn't take an
2200 * 'off' argument, so it has to add in reg->off itself.
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002201 */
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01002202static int check_stack_boundary(struct bpf_verifier_env *env, int regno,
Daniel Borkmann435faee12016-04-13 00:10:51 +02002203 int access_size, bool zero_size_allowed,
2204 struct bpf_call_arg_meta *meta)
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002205{
Daniel Borkmann2a159c62018-10-21 02:09:24 +02002206 struct bpf_reg_state *reg = reg_state(env, regno);
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002207 struct bpf_func_state *state = func(env, reg);
Andrey Ignatov2011fcc2019-03-28 18:01:57 -07002208 int err, min_off, max_off, i, slot, spi;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002209
Alexei Starovoitov914cb782017-11-30 21:31:40 -08002210 if (reg->type != PTR_TO_STACK) {
Edward Creef1174f72017-08-07 15:26:19 +01002211 /* Allow zero-byte read from NULL, regardless of pointer type */
Daniel Borkmann8e2fe1d92016-02-19 23:05:22 +01002212 if (zero_size_allowed && access_size == 0 &&
Alexei Starovoitov914cb782017-11-30 21:31:40 -08002213 register_is_null(reg))
Daniel Borkmann8e2fe1d92016-02-19 23:05:22 +01002214 return 0;
2215
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002216 verbose(env, "R%d type=%s expected=%s\n", regno,
Alexei Starovoitov914cb782017-11-30 21:31:40 -08002217 reg_type_str[reg->type],
Daniel Borkmann8e2fe1d92016-02-19 23:05:22 +01002218 reg_type_str[PTR_TO_STACK]);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002219 return -EACCES;
Daniel Borkmann8e2fe1d92016-02-19 23:05:22 +01002220 }
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002221
Andrey Ignatov2011fcc2019-03-28 18:01:57 -07002222 if (tnum_is_const(reg->var_off)) {
2223 min_off = max_off = reg->var_off.value + reg->off;
2224 err = __check_stack_boundary(env, regno, min_off, access_size,
2225 zero_size_allowed);
2226 if (err)
2227 return err;
2228 } else {
Andrey Ignatov088ec262019-04-03 23:22:39 -07002229 /* Variable offset is prohibited for unprivileged mode for
2230 * simplicity since it requires corresponding support in
2231 * Spectre masking for stack ALU.
2232 * See also retrieve_ptr_limit().
2233 */
2234 if (!env->allow_ptr_leaks) {
2235 char tn_buf[48];
2236
2237 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
2238 verbose(env, "R%d indirect variable offset stack access prohibited for !root, var_off=%s\n",
2239 regno, tn_buf);
2240 return -EACCES;
2241 }
Andrey Ignatovf2bcd052019-04-03 23:22:37 -07002242 /* Only initialized buffer on stack is allowed to be accessed
2243 * with variable offset. With uninitialized buffer it's hard to
2244 * guarantee that whole memory is marked as initialized on
2245 * helper return since specific bounds are unknown what may
2246 * cause uninitialized stack leaking.
2247 */
2248 if (meta && meta->raw_mode)
2249 meta = NULL;
2250
Andrey Ignatov107c26a72019-04-03 23:22:41 -07002251 if (reg->smax_value >= BPF_MAX_VAR_OFF ||
2252 reg->smax_value <= -BPF_MAX_VAR_OFF) {
2253 verbose(env, "R%d unbounded indirect variable offset stack access\n",
2254 regno);
2255 return -EACCES;
2256 }
Andrey Ignatov2011fcc2019-03-28 18:01:57 -07002257 min_off = reg->smin_value + reg->off;
Andrey Ignatov107c26a72019-04-03 23:22:41 -07002258 max_off = reg->smax_value + reg->off;
Andrey Ignatov2011fcc2019-03-28 18:01:57 -07002259 err = __check_stack_boundary(env, regno, min_off, access_size,
2260 zero_size_allowed);
Andrey Ignatov107c26a72019-04-03 23:22:41 -07002261 if (err) {
2262 verbose(env, "R%d min value is outside of stack bound\n",
2263 regno);
Andrey Ignatov2011fcc2019-03-28 18:01:57 -07002264 return err;
Andrey Ignatov107c26a72019-04-03 23:22:41 -07002265 }
Andrey Ignatov2011fcc2019-03-28 18:01:57 -07002266 err = __check_stack_boundary(env, regno, max_off, access_size,
2267 zero_size_allowed);
Andrey Ignatov107c26a72019-04-03 23:22:41 -07002268 if (err) {
2269 verbose(env, "R%d max value is outside of stack bound\n",
2270 regno);
Andrey Ignatov2011fcc2019-03-28 18:01:57 -07002271 return err;
Andrey Ignatov107c26a72019-04-03 23:22:41 -07002272 }
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002273 }
2274
Daniel Borkmann435faee12016-04-13 00:10:51 +02002275 if (meta && meta->raw_mode) {
2276 meta->access_size = access_size;
2277 meta->regno = regno;
2278 return 0;
2279 }
2280
Andrey Ignatov2011fcc2019-03-28 18:01:57 -07002281 for (i = min_off; i < max_off + access_size; i++) {
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08002282 u8 *stype;
2283
Andrey Ignatov2011fcc2019-03-28 18:01:57 -07002284 slot = -i - 1;
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07002285 spi = slot / BPF_REG_SIZE;
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08002286 if (state->allocated_stack <= slot)
2287 goto err;
2288 stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
2289 if (*stype == STACK_MISC)
2290 goto mark;
2291 if (*stype == STACK_ZERO) {
2292 /* helper can write anything into the stack */
2293 *stype = STACK_MISC;
2294 goto mark;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002295 }
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08002296err:
Andrey Ignatov2011fcc2019-03-28 18:01:57 -07002297 if (tnum_is_const(reg->var_off)) {
2298 verbose(env, "invalid indirect read from stack off %d+%d size %d\n",
2299 min_off, i - min_off, access_size);
2300 } else {
2301 char tn_buf[48];
2302
2303 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
2304 verbose(env, "invalid indirect read from stack var_off %s+%d size %d\n",
2305 tn_buf, i - min_off, access_size);
2306 }
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08002307 return -EACCES;
2308mark:
2309 /* reading any byte out of 8-byte 'spill_slot' will cause
2310 * the whole slot to be marked as 'read'
2311 */
Edward Cree679c7822018-08-22 20:02:19 +01002312 mark_reg_read(env, &state->stack[spi].spilled_ptr,
2313 state->stack[spi].spilled_ptr.parent);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002314 }
Andrey Ignatov2011fcc2019-03-28 18:01:57 -07002315 return update_stack_depth(env, state, min_off);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002316}
2317
Gianluca Borello06c1c042017-01-09 10:19:49 -08002318static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,
2319 int access_size, bool zero_size_allowed,
2320 struct bpf_call_arg_meta *meta)
2321{
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07002322 struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
Gianluca Borello06c1c042017-01-09 10:19:49 -08002323
Edward Creef1174f72017-08-07 15:26:19 +01002324 switch (reg->type) {
Gianluca Borello06c1c042017-01-09 10:19:49 -08002325 case PTR_TO_PACKET:
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02002326 case PTR_TO_PACKET_META:
Yonghong Song9fd29c02017-11-12 14:49:09 -08002327 return check_packet_access(env, regno, reg->off, access_size,
2328 zero_size_allowed);
Gianluca Borello06c1c042017-01-09 10:19:49 -08002329 case PTR_TO_MAP_VALUE:
Yonghong Song9fd29c02017-11-12 14:49:09 -08002330 return check_map_access(env, regno, reg->off, access_size,
2331 zero_size_allowed);
Edward Creef1174f72017-08-07 15:26:19 +01002332 default: /* scalar_value|ptr_to_stack or invalid ptr */
Gianluca Borello06c1c042017-01-09 10:19:49 -08002333 return check_stack_boundary(env, regno, access_size,
2334 zero_size_allowed, meta);
2335 }
2336}
2337
Alexei Starovoitovd83525c2019-01-31 15:40:04 -08002338/* Implementation details:
2339 * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL
2340 * Two bpf_map_lookups (even with the same key) will have different reg->id.
2341 * For traditional PTR_TO_MAP_VALUE the verifier clears reg->id after
2342 * value_or_null->value transition, since the verifier only cares about
2343 * the range of access to valid map value pointer and doesn't care about actual
2344 * address of the map element.
2345 * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps
2346 * reg->id > 0 after value_or_null->value transition. By doing so
2347 * two bpf_map_lookups will be considered two different pointers that
2348 * point to different bpf_spin_locks.
2349 * The verifier allows taking only one bpf_spin_lock at a time to avoid
2350 * dead-locks.
2351 * Since only one bpf_spin_lock is allowed the checks are simpler than
2352 * reg_is_refcounted() logic. The verifier needs to remember only
2353 * one spin_lock instead of array of acquired_refs.
2354 * cur_state->active_spin_lock remembers which map value element got locked
2355 * and clears it after bpf_spin_unlock.
2356 */
2357static int process_spin_lock(struct bpf_verifier_env *env, int regno,
2358 bool is_lock)
2359{
2360 struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
2361 struct bpf_verifier_state *cur = env->cur_state;
2362 bool is_const = tnum_is_const(reg->var_off);
2363 struct bpf_map *map = reg->map_ptr;
2364 u64 val = reg->var_off.value;
2365
2366 if (reg->type != PTR_TO_MAP_VALUE) {
2367 verbose(env, "R%d is not a pointer to map_value\n", regno);
2368 return -EINVAL;
2369 }
2370 if (!is_const) {
2371 verbose(env,
2372 "R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n",
2373 regno);
2374 return -EINVAL;
2375 }
2376 if (!map->btf) {
2377 verbose(env,
2378 "map '%s' has to have BTF in order to use bpf_spin_lock\n",
2379 map->name);
2380 return -EINVAL;
2381 }
2382 if (!map_value_has_spin_lock(map)) {
2383 if (map->spin_lock_off == -E2BIG)
2384 verbose(env,
2385 "map '%s' has more than one 'struct bpf_spin_lock'\n",
2386 map->name);
2387 else if (map->spin_lock_off == -ENOENT)
2388 verbose(env,
2389 "map '%s' doesn't have 'struct bpf_spin_lock'\n",
2390 map->name);
2391 else
2392 verbose(env,
2393 "map '%s' is not a struct type or bpf_spin_lock is mangled\n",
2394 map->name);
2395 return -EINVAL;
2396 }
2397 if (map->spin_lock_off != val + reg->off) {
2398 verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock'\n",
2399 val + reg->off);
2400 return -EINVAL;
2401 }
2402 if (is_lock) {
2403 if (cur->active_spin_lock) {
2404 verbose(env,
2405 "Locking two bpf_spin_locks are not allowed\n");
2406 return -EINVAL;
2407 }
2408 cur->active_spin_lock = reg->id;
2409 } else {
2410 if (!cur->active_spin_lock) {
2411 verbose(env, "bpf_spin_unlock without taking a lock\n");
2412 return -EINVAL;
2413 }
2414 if (cur->active_spin_lock != reg->id) {
2415 verbose(env, "bpf_spin_unlock of different lock\n");
2416 return -EINVAL;
2417 }
2418 cur->active_spin_lock = 0;
2419 }
2420 return 0;
2421}
2422
Daniel Borkmann90133412018-01-20 01:24:29 +01002423static bool arg_type_is_mem_ptr(enum bpf_arg_type type)
2424{
2425 return type == ARG_PTR_TO_MEM ||
2426 type == ARG_PTR_TO_MEM_OR_NULL ||
2427 type == ARG_PTR_TO_UNINIT_MEM;
2428}
2429
2430static bool arg_type_is_mem_size(enum bpf_arg_type type)
2431{
2432 return type == ARG_CONST_SIZE ||
2433 type == ARG_CONST_SIZE_OR_ZERO;
2434}
2435
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01002436static int check_func_arg(struct bpf_verifier_env *env, u32 regno,
Daniel Borkmann33ff9822016-04-13 00:10:50 +02002437 enum bpf_arg_type arg_type,
2438 struct bpf_call_arg_meta *meta)
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002439{
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07002440 struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
Alexei Starovoitov6841de82016-08-11 18:17:16 -07002441 enum bpf_reg_type expected_type, type = reg->type;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002442 int err = 0;
2443
Daniel Borkmann80f1d682015-03-12 17:21:42 +01002444 if (arg_type == ARG_DONTCARE)
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002445 return 0;
2446
Edward Creedc503a82017-08-15 20:34:35 +01002447 err = check_reg_arg(env, regno, SRC_OP);
2448 if (err)
2449 return err;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002450
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07002451 if (arg_type == ARG_ANYTHING) {
2452 if (is_pointer_value(env, regno)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002453 verbose(env, "R%d leaks addr into helper function\n",
2454 regno);
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07002455 return -EACCES;
2456 }
Daniel Borkmann80f1d682015-03-12 17:21:42 +01002457 return 0;
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07002458 }
Daniel Borkmann80f1d682015-03-12 17:21:42 +01002459
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02002460 if (type_is_pkt_pointer(type) &&
Thomas Graf3a0af8f2016-11-30 17:10:10 +01002461 !may_access_direct_pkt_data(env, meta, BPF_READ)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002462 verbose(env, "helper access to the packet is not allowed\n");
Alexei Starovoitov6841de82016-08-11 18:17:16 -07002463 return -EACCES;
2464 }
2465
Daniel Borkmann8e2fe1d92016-02-19 23:05:22 +01002466 if (arg_type == ARG_PTR_TO_MAP_KEY ||
Mauricio Vasquez B2ea864c2018-10-18 15:16:20 +02002467 arg_type == ARG_PTR_TO_MAP_VALUE ||
2468 arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) {
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002469 expected_type = PTR_TO_STACK;
Paul Chaignond71962f2018-04-24 15:07:54 +02002470 if (!type_is_pkt_pointer(type) && type != PTR_TO_MAP_VALUE &&
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02002471 type != expected_type)
Alexei Starovoitov6841de82016-08-11 18:17:16 -07002472 goto err_type;
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -08002473 } else if (arg_type == ARG_CONST_SIZE ||
2474 arg_type == ARG_CONST_SIZE_OR_ZERO) {
Edward Creef1174f72017-08-07 15:26:19 +01002475 expected_type = SCALAR_VALUE;
2476 if (type != expected_type)
Alexei Starovoitov6841de82016-08-11 18:17:16 -07002477 goto err_type;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002478 } else if (arg_type == ARG_CONST_MAP_PTR) {
2479 expected_type = CONST_PTR_TO_MAP;
Alexei Starovoitov6841de82016-08-11 18:17:16 -07002480 if (type != expected_type)
2481 goto err_type;
Alexei Starovoitov608cd712015-03-26 19:53:57 -07002482 } else if (arg_type == ARG_PTR_TO_CTX) {
2483 expected_type = PTR_TO_CTX;
Alexei Starovoitov6841de82016-08-11 18:17:16 -07002484 if (type != expected_type)
2485 goto err_type;
Daniel Borkmann58990d12018-06-07 17:40:03 +02002486 err = check_ctx_reg(env, reg, regno);
2487 if (err < 0)
2488 return err;
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08002489 } else if (arg_type == ARG_PTR_TO_SOCK_COMMON) {
2490 expected_type = PTR_TO_SOCK_COMMON;
2491 /* Any sk pointer can be ARG_PTR_TO_SOCK_COMMON */
2492 if (!type_is_sk_pointer(type))
2493 goto err_type;
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002494 if (reg->ref_obj_id) {
2495 if (meta->ref_obj_id) {
2496 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n",
2497 regno, reg->ref_obj_id,
2498 meta->ref_obj_id);
2499 return -EFAULT;
2500 }
2501 meta->ref_obj_id = reg->ref_obj_id;
Joe Stringerfd978bf72018-10-02 13:35:35 -07002502 }
Alexei Starovoitovd83525c2019-01-31 15:40:04 -08002503 } else if (arg_type == ARG_PTR_TO_SPIN_LOCK) {
2504 if (meta->func_id == BPF_FUNC_spin_lock) {
2505 if (process_spin_lock(env, regno, true))
2506 return -EACCES;
2507 } else if (meta->func_id == BPF_FUNC_spin_unlock) {
2508 if (process_spin_lock(env, regno, false))
2509 return -EACCES;
2510 } else {
2511 verbose(env, "verifier internal error\n");
2512 return -EFAULT;
2513 }
Daniel Borkmann90133412018-01-20 01:24:29 +01002514 } else if (arg_type_is_mem_ptr(arg_type)) {
Daniel Borkmann8e2fe1d92016-02-19 23:05:22 +01002515 expected_type = PTR_TO_STACK;
2516 /* One exception here. In case function allows for NULL to be
Edward Creef1174f72017-08-07 15:26:19 +01002517 * passed in as argument, it's a SCALAR_VALUE type. Final test
Daniel Borkmann8e2fe1d92016-02-19 23:05:22 +01002518 * happens during stack boundary checking.
2519 */
Alexei Starovoitov914cb782017-11-30 21:31:40 -08002520 if (register_is_null(reg) &&
Gianluca Borellodb1ac492017-11-22 18:32:53 +00002521 arg_type == ARG_PTR_TO_MEM_OR_NULL)
Alexei Starovoitov6841de82016-08-11 18:17:16 -07002522 /* final test in check_stack_boundary() */;
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02002523 else if (!type_is_pkt_pointer(type) &&
2524 type != PTR_TO_MAP_VALUE &&
Edward Creef1174f72017-08-07 15:26:19 +01002525 type != expected_type)
Alexei Starovoitov6841de82016-08-11 18:17:16 -07002526 goto err_type;
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -08002527 meta->raw_mode = arg_type == ARG_PTR_TO_UNINIT_MEM;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002528 } else {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002529 verbose(env, "unsupported arg_type %d\n", arg_type);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002530 return -EFAULT;
2531 }
2532
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002533 if (arg_type == ARG_CONST_MAP_PTR) {
2534 /* bpf_map_xxx(map_ptr) call: remember that map_ptr */
Daniel Borkmann33ff9822016-04-13 00:10:50 +02002535 meta->map_ptr = reg->map_ptr;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002536 } else if (arg_type == ARG_PTR_TO_MAP_KEY) {
2537 /* bpf_map_xxx(..., map_ptr, ..., key) call:
2538 * check that [key, key + map->key_size) are within
2539 * stack limits and initialized
2540 */
Daniel Borkmann33ff9822016-04-13 00:10:50 +02002541 if (!meta->map_ptr) {
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002542 /* in function declaration map_ptr must come before
2543 * map_key, so that it's verified and known before
2544 * we have to check map_key here. Otherwise it means
2545 * that kernel subsystem misconfigured verifier
2546 */
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002547 verbose(env, "invalid map_ptr to access map->key\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002548 return -EACCES;
2549 }
Paul Chaignond71962f2018-04-24 15:07:54 +02002550 err = check_helper_mem_access(env, regno,
2551 meta->map_ptr->key_size, false,
2552 NULL);
Mauricio Vasquez B2ea864c2018-10-18 15:16:20 +02002553 } else if (arg_type == ARG_PTR_TO_MAP_VALUE ||
2554 arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) {
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002555 /* bpf_map_xxx(..., map_ptr, ..., value) call:
2556 * check [value, value + map->value_size) validity
2557 */
Daniel Borkmann33ff9822016-04-13 00:10:50 +02002558 if (!meta->map_ptr) {
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002559 /* kernel subsystem misconfigured verifier */
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002560 verbose(env, "invalid map_ptr to access map->value\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002561 return -EACCES;
2562 }
Mauricio Vasquez B2ea864c2018-10-18 15:16:20 +02002563 meta->raw_mode = (arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE);
Paul Chaignond71962f2018-04-24 15:07:54 +02002564 err = check_helper_mem_access(env, regno,
2565 meta->map_ptr->value_size, false,
Mauricio Vasquez B2ea864c2018-10-18 15:16:20 +02002566 meta);
Daniel Borkmann90133412018-01-20 01:24:29 +01002567 } else if (arg_type_is_mem_size(arg_type)) {
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -08002568 bool zero_size_allowed = (arg_type == ARG_CONST_SIZE_OR_ZERO);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002569
Yonghong Song849fa502018-04-28 22:28:09 -07002570 /* remember the mem_size which may be used later
2571 * to refine return values.
2572 */
2573 meta->msize_smax_value = reg->smax_value;
2574 meta->msize_umax_value = reg->umax_value;
2575
Edward Creef1174f72017-08-07 15:26:19 +01002576 /* The register is SCALAR_VALUE; the access check
2577 * happens using its boundaries.
Gianluca Borello06c1c042017-01-09 10:19:49 -08002578 */
Edward Creef1174f72017-08-07 15:26:19 +01002579 if (!tnum_is_const(reg->var_off))
Gianluca Borello06c1c042017-01-09 10:19:49 -08002580 /* For unprivileged variable accesses, disable raw
2581 * mode so that the program is required to
2582 * initialize all the memory that the helper could
2583 * just partially fill up.
2584 */
2585 meta = NULL;
2586
Edward Creeb03c9f92017-08-07 15:26:36 +01002587 if (reg->smin_value < 0) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002588 verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n",
Edward Creef1174f72017-08-07 15:26:19 +01002589 regno);
2590 return -EACCES;
2591 }
Gianluca Borello06c1c042017-01-09 10:19:49 -08002592
Edward Creeb03c9f92017-08-07 15:26:36 +01002593 if (reg->umin_value == 0) {
Edward Creef1174f72017-08-07 15:26:19 +01002594 err = check_helper_mem_access(env, regno - 1, 0,
2595 zero_size_allowed,
2596 meta);
Gianluca Borello06c1c042017-01-09 10:19:49 -08002597 if (err)
2598 return err;
Gianluca Borello06c1c042017-01-09 10:19:49 -08002599 }
Edward Creef1174f72017-08-07 15:26:19 +01002600
Edward Creeb03c9f92017-08-07 15:26:36 +01002601 if (reg->umax_value >= BPF_MAX_VAR_SIZ) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002602 verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n",
Edward Creef1174f72017-08-07 15:26:19 +01002603 regno);
2604 return -EACCES;
2605 }
2606 err = check_helper_mem_access(env, regno - 1,
Edward Creeb03c9f92017-08-07 15:26:36 +01002607 reg->umax_value,
Edward Creef1174f72017-08-07 15:26:19 +01002608 zero_size_allowed, meta);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002609 }
2610
2611 return err;
Alexei Starovoitov6841de82016-08-11 18:17:16 -07002612err_type:
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002613 verbose(env, "R%d type=%s expected=%s\n", regno,
Alexei Starovoitov6841de82016-08-11 18:17:16 -07002614 reg_type_str[type], reg_type_str[expected_type]);
2615 return -EACCES;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07002616}
2617
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002618static int check_map_func_compatibility(struct bpf_verifier_env *env,
2619 struct bpf_map *map, int func_id)
Kaixu Xia35578d72015-08-06 07:02:35 +00002620{
Kaixu Xia35578d72015-08-06 07:02:35 +00002621 if (!map)
2622 return 0;
2623
Alexei Starovoitov6aff67c2016-04-27 18:56:21 -07002624 /* We need a two way check, first is from map perspective ... */
2625 switch (map->map_type) {
2626 case BPF_MAP_TYPE_PROG_ARRAY:
2627 if (func_id != BPF_FUNC_tail_call)
2628 goto error;
2629 break;
2630 case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
2631 if (func_id != BPF_FUNC_perf_event_read &&
Yonghong Song908432c2017-10-05 09:19:20 -07002632 func_id != BPF_FUNC_perf_event_output &&
2633 func_id != BPF_FUNC_perf_event_read_value)
Alexei Starovoitov6aff67c2016-04-27 18:56:21 -07002634 goto error;
2635 break;
2636 case BPF_MAP_TYPE_STACK_TRACE:
2637 if (func_id != BPF_FUNC_get_stackid)
2638 goto error;
2639 break;
Martin KaFai Lau4ed8ec52016-06-30 10:28:43 -07002640 case BPF_MAP_TYPE_CGROUP_ARRAY:
David S. Miller60747ef2016-08-18 01:17:32 -04002641 if (func_id != BPF_FUNC_skb_under_cgroup &&
Sargun Dhillon60d20f92016-08-12 08:56:52 -07002642 func_id != BPF_FUNC_current_task_under_cgroup)
Martin KaFai Lau4a482f32016-06-30 10:28:44 -07002643 goto error;
2644 break;
Roman Gushchincd339432018-08-02 14:27:24 -07002645 case BPF_MAP_TYPE_CGROUP_STORAGE:
Roman Gushchinb741f162018-09-28 14:45:43 +00002646 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE:
Roman Gushchincd339432018-08-02 14:27:24 -07002647 if (func_id != BPF_FUNC_get_local_storage)
2648 goto error;
2649 break;
John Fastabend546ac1f2017-07-17 09:28:56 -07002650 /* devmap returns a pointer to a live net_device ifindex that we cannot
2651 * allow to be modified from bpf side. So do not allow lookup elements
2652 * for now.
2653 */
2654 case BPF_MAP_TYPE_DEVMAP:
John Fastabend2ddf71e2017-07-17 09:30:02 -07002655 if (func_id != BPF_FUNC_redirect_map)
John Fastabend546ac1f2017-07-17 09:28:56 -07002656 goto error;
2657 break;
Björn Töpelfbfc504a2018-05-02 13:01:28 +02002658 /* Restrict bpf side of cpumap and xskmap, open when use-cases
2659 * appear.
2660 */
Jesper Dangaard Brouer6710e112017-10-16 12:19:28 +02002661 case BPF_MAP_TYPE_CPUMAP:
Björn Töpelfbfc504a2018-05-02 13:01:28 +02002662 case BPF_MAP_TYPE_XSKMAP:
Jesper Dangaard Brouer6710e112017-10-16 12:19:28 +02002663 if (func_id != BPF_FUNC_redirect_map)
2664 goto error;
2665 break;
Martin KaFai Lau56f668d2017-03-22 10:00:33 -07002666 case BPF_MAP_TYPE_ARRAY_OF_MAPS:
Martin KaFai Laubcc6b1b2017-03-22 10:00:34 -07002667 case BPF_MAP_TYPE_HASH_OF_MAPS:
Martin KaFai Lau56f668d2017-03-22 10:00:33 -07002668 if (func_id != BPF_FUNC_map_lookup_elem)
2669 goto error;
Martin KaFai Lau16a43622017-08-17 18:14:43 -07002670 break;
John Fastabend174a79f2017-08-15 22:32:47 -07002671 case BPF_MAP_TYPE_SOCKMAP:
2672 if (func_id != BPF_FUNC_sk_redirect_map &&
2673 func_id != BPF_FUNC_sock_map_update &&
John Fastabend4f738ad2018-03-18 12:57:10 -07002674 func_id != BPF_FUNC_map_delete_elem &&
2675 func_id != BPF_FUNC_msg_redirect_map)
John Fastabend174a79f2017-08-15 22:32:47 -07002676 goto error;
2677 break;
John Fastabend81110382018-05-14 10:00:17 -07002678 case BPF_MAP_TYPE_SOCKHASH:
2679 if (func_id != BPF_FUNC_sk_redirect_hash &&
2680 func_id != BPF_FUNC_sock_hash_update &&
2681 func_id != BPF_FUNC_map_delete_elem &&
2682 func_id != BPF_FUNC_msg_redirect_hash)
2683 goto error;
2684 break;
Martin KaFai Lau2dbb9b92018-08-08 01:01:25 -07002685 case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY:
2686 if (func_id != BPF_FUNC_sk_select_reuseport)
2687 goto error;
2688 break;
Mauricio Vasquez Bf1a2e442018-10-18 15:16:25 +02002689 case BPF_MAP_TYPE_QUEUE:
2690 case BPF_MAP_TYPE_STACK:
2691 if (func_id != BPF_FUNC_map_peek_elem &&
2692 func_id != BPF_FUNC_map_pop_elem &&
2693 func_id != BPF_FUNC_map_push_elem)
2694 goto error;
2695 break;
Alexei Starovoitov6aff67c2016-04-27 18:56:21 -07002696 default:
2697 break;
2698 }
2699
2700 /* ... and second from the function itself. */
2701 switch (func_id) {
2702 case BPF_FUNC_tail_call:
2703 if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY)
2704 goto error;
Jiong Wangf910cef2018-05-02 16:17:17 -04002705 if (env->subprog_cnt > 1) {
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002706 verbose(env, "tail_calls are not allowed in programs with bpf-to-bpf calls\n");
2707 return -EINVAL;
2708 }
Alexei Starovoitov6aff67c2016-04-27 18:56:21 -07002709 break;
2710 case BPF_FUNC_perf_event_read:
2711 case BPF_FUNC_perf_event_output:
Yonghong Song908432c2017-10-05 09:19:20 -07002712 case BPF_FUNC_perf_event_read_value:
Alexei Starovoitov6aff67c2016-04-27 18:56:21 -07002713 if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY)
2714 goto error;
2715 break;
2716 case BPF_FUNC_get_stackid:
2717 if (map->map_type != BPF_MAP_TYPE_STACK_TRACE)
2718 goto error;
2719 break;
Sargun Dhillon60d20f92016-08-12 08:56:52 -07002720 case BPF_FUNC_current_task_under_cgroup:
Daniel Borkmann747ea552016-08-12 22:17:17 +02002721 case BPF_FUNC_skb_under_cgroup:
Martin KaFai Lau4a482f32016-06-30 10:28:44 -07002722 if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY)
2723 goto error;
2724 break;
John Fastabend97f91a72017-07-17 09:29:18 -07002725 case BPF_FUNC_redirect_map:
Jesper Dangaard Brouer9c270af2017-10-16 12:19:34 +02002726 if (map->map_type != BPF_MAP_TYPE_DEVMAP &&
Björn Töpelfbfc504a2018-05-02 13:01:28 +02002727 map->map_type != BPF_MAP_TYPE_CPUMAP &&
2728 map->map_type != BPF_MAP_TYPE_XSKMAP)
John Fastabend97f91a72017-07-17 09:29:18 -07002729 goto error;
2730 break;
John Fastabend174a79f2017-08-15 22:32:47 -07002731 case BPF_FUNC_sk_redirect_map:
John Fastabend4f738ad2018-03-18 12:57:10 -07002732 case BPF_FUNC_msg_redirect_map:
John Fastabend81110382018-05-14 10:00:17 -07002733 case BPF_FUNC_sock_map_update:
John Fastabend174a79f2017-08-15 22:32:47 -07002734 if (map->map_type != BPF_MAP_TYPE_SOCKMAP)
2735 goto error;
2736 break;
John Fastabend81110382018-05-14 10:00:17 -07002737 case BPF_FUNC_sk_redirect_hash:
2738 case BPF_FUNC_msg_redirect_hash:
2739 case BPF_FUNC_sock_hash_update:
2740 if (map->map_type != BPF_MAP_TYPE_SOCKHASH)
John Fastabend174a79f2017-08-15 22:32:47 -07002741 goto error;
2742 break;
Roman Gushchincd339432018-08-02 14:27:24 -07002743 case BPF_FUNC_get_local_storage:
Roman Gushchinb741f162018-09-28 14:45:43 +00002744 if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
2745 map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
Roman Gushchincd339432018-08-02 14:27:24 -07002746 goto error;
2747 break;
Martin KaFai Lau2dbb9b92018-08-08 01:01:25 -07002748 case BPF_FUNC_sk_select_reuseport:
2749 if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY)
2750 goto error;
2751 break;
Mauricio Vasquez Bf1a2e442018-10-18 15:16:25 +02002752 case BPF_FUNC_map_peek_elem:
2753 case BPF_FUNC_map_pop_elem:
2754 case BPF_FUNC_map_push_elem:
2755 if (map->map_type != BPF_MAP_TYPE_QUEUE &&
2756 map->map_type != BPF_MAP_TYPE_STACK)
2757 goto error;
2758 break;
Alexei Starovoitov6aff67c2016-04-27 18:56:21 -07002759 default:
2760 break;
Kaixu Xia35578d72015-08-06 07:02:35 +00002761 }
2762
2763 return 0;
Alexei Starovoitov6aff67c2016-04-27 18:56:21 -07002764error:
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002765 verbose(env, "cannot pass map_type %d into func %s#%d\n",
Thomas Grafebb676d2016-10-27 11:23:51 +02002766 map->map_type, func_id_name(func_id), func_id);
Alexei Starovoitov6aff67c2016-04-27 18:56:21 -07002767 return -EINVAL;
Kaixu Xia35578d72015-08-06 07:02:35 +00002768}
2769
Daniel Borkmann90133412018-01-20 01:24:29 +01002770static bool check_raw_mode_ok(const struct bpf_func_proto *fn)
Daniel Borkmann435faee12016-04-13 00:10:51 +02002771{
2772 int count = 0;
2773
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -08002774 if (fn->arg1_type == ARG_PTR_TO_UNINIT_MEM)
Daniel Borkmann435faee12016-04-13 00:10:51 +02002775 count++;
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -08002776 if (fn->arg2_type == ARG_PTR_TO_UNINIT_MEM)
Daniel Borkmann435faee12016-04-13 00:10:51 +02002777 count++;
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -08002778 if (fn->arg3_type == ARG_PTR_TO_UNINIT_MEM)
Daniel Borkmann435faee12016-04-13 00:10:51 +02002779 count++;
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -08002780 if (fn->arg4_type == ARG_PTR_TO_UNINIT_MEM)
Daniel Borkmann435faee12016-04-13 00:10:51 +02002781 count++;
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -08002782 if (fn->arg5_type == ARG_PTR_TO_UNINIT_MEM)
Daniel Borkmann435faee12016-04-13 00:10:51 +02002783 count++;
2784
Daniel Borkmann90133412018-01-20 01:24:29 +01002785 /* We only support one arg being in raw mode at the moment,
2786 * which is sufficient for the helper functions we have
2787 * right now.
2788 */
2789 return count <= 1;
2790}
2791
2792static bool check_args_pair_invalid(enum bpf_arg_type arg_curr,
2793 enum bpf_arg_type arg_next)
2794{
2795 return (arg_type_is_mem_ptr(arg_curr) &&
2796 !arg_type_is_mem_size(arg_next)) ||
2797 (!arg_type_is_mem_ptr(arg_curr) &&
2798 arg_type_is_mem_size(arg_next));
2799}
2800
2801static bool check_arg_pair_ok(const struct bpf_func_proto *fn)
2802{
2803 /* bpf_xxx(..., buf, len) call will access 'len'
2804 * bytes from memory 'buf'. Both arg types need
2805 * to be paired, so make sure there's no buggy
2806 * helper function specification.
2807 */
2808 if (arg_type_is_mem_size(fn->arg1_type) ||
2809 arg_type_is_mem_ptr(fn->arg5_type) ||
2810 check_args_pair_invalid(fn->arg1_type, fn->arg2_type) ||
2811 check_args_pair_invalid(fn->arg2_type, fn->arg3_type) ||
2812 check_args_pair_invalid(fn->arg3_type, fn->arg4_type) ||
2813 check_args_pair_invalid(fn->arg4_type, fn->arg5_type))
2814 return false;
2815
2816 return true;
2817}
2818
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002819static bool check_refcount_ok(const struct bpf_func_proto *fn, int func_id)
Joe Stringerfd978bf72018-10-02 13:35:35 -07002820{
2821 int count = 0;
2822
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002823 if (arg_type_may_be_refcounted(fn->arg1_type))
Joe Stringerfd978bf72018-10-02 13:35:35 -07002824 count++;
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002825 if (arg_type_may_be_refcounted(fn->arg2_type))
Joe Stringerfd978bf72018-10-02 13:35:35 -07002826 count++;
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002827 if (arg_type_may_be_refcounted(fn->arg3_type))
Joe Stringerfd978bf72018-10-02 13:35:35 -07002828 count++;
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002829 if (arg_type_may_be_refcounted(fn->arg4_type))
Joe Stringerfd978bf72018-10-02 13:35:35 -07002830 count++;
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002831 if (arg_type_may_be_refcounted(fn->arg5_type))
Joe Stringerfd978bf72018-10-02 13:35:35 -07002832 count++;
2833
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002834 /* A reference acquiring function cannot acquire
2835 * another refcounted ptr.
2836 */
2837 if (is_acquire_function(func_id) && count)
2838 return false;
2839
Joe Stringerfd978bf72018-10-02 13:35:35 -07002840 /* We only support one arg being unreferenced at the moment,
2841 * which is sufficient for the helper functions we have right now.
2842 */
2843 return count <= 1;
2844}
2845
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002846static int check_func_proto(const struct bpf_func_proto *fn, int func_id)
Daniel Borkmann90133412018-01-20 01:24:29 +01002847{
2848 return check_raw_mode_ok(fn) &&
Joe Stringerfd978bf72018-10-02 13:35:35 -07002849 check_arg_pair_ok(fn) &&
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002850 check_refcount_ok(fn, func_id) ? 0 : -EINVAL;
Daniel Borkmann435faee12016-04-13 00:10:51 +02002851}
2852
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02002853/* Packet data might have moved, any old PTR_TO_PACKET[_META,_END]
2854 * are now invalid, so turn them into unknown SCALAR_VALUE.
Edward Creef1174f72017-08-07 15:26:19 +01002855 */
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002856static void __clear_all_pkt_pointers(struct bpf_verifier_env *env,
2857 struct bpf_func_state *state)
Alexei Starovoitov969bf052016-05-05 19:49:10 -07002858{
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01002859 struct bpf_reg_state *regs = state->regs, *reg;
Alexei Starovoitov969bf052016-05-05 19:49:10 -07002860 int i;
2861
2862 for (i = 0; i < MAX_BPF_REG; i++)
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02002863 if (reg_is_pkt_pointer_any(&regs[i]))
Jakub Kicinski61bd5212017-10-09 10:30:11 -07002864 mark_reg_unknown(env, regs, i);
Alexei Starovoitov969bf052016-05-05 19:49:10 -07002865
Joe Stringerf3709f62018-10-02 13:35:29 -07002866 bpf_for_each_spilled_reg(i, state, reg) {
2867 if (!reg)
Alexei Starovoitov969bf052016-05-05 19:49:10 -07002868 continue;
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02002869 if (reg_is_pkt_pointer_any(reg))
2870 __mark_reg_unknown(reg);
Alexei Starovoitov969bf052016-05-05 19:49:10 -07002871 }
2872}
2873
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002874static void clear_all_pkt_pointers(struct bpf_verifier_env *env)
2875{
2876 struct bpf_verifier_state *vstate = env->cur_state;
2877 int i;
2878
2879 for (i = 0; i <= vstate->curframe; i++)
2880 __clear_all_pkt_pointers(env, vstate->frame[i]);
2881}
2882
Joe Stringerfd978bf72018-10-02 13:35:35 -07002883static void release_reg_references(struct bpf_verifier_env *env,
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002884 struct bpf_func_state *state,
2885 int ref_obj_id)
Joe Stringerfd978bf72018-10-02 13:35:35 -07002886{
2887 struct bpf_reg_state *regs = state->regs, *reg;
2888 int i;
2889
2890 for (i = 0; i < MAX_BPF_REG; i++)
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002891 if (regs[i].ref_obj_id == ref_obj_id)
Joe Stringerfd978bf72018-10-02 13:35:35 -07002892 mark_reg_unknown(env, regs, i);
2893
2894 bpf_for_each_spilled_reg(i, state, reg) {
2895 if (!reg)
2896 continue;
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002897 if (reg->ref_obj_id == ref_obj_id)
Joe Stringerfd978bf72018-10-02 13:35:35 -07002898 __mark_reg_unknown(reg);
2899 }
2900}
2901
2902/* The pointer with the specified id has released its reference to kernel
2903 * resources. Identify all copies of the same pointer and clear the reference.
2904 */
2905static int release_reference(struct bpf_verifier_env *env,
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002906 int ref_obj_id)
Joe Stringerfd978bf72018-10-02 13:35:35 -07002907{
2908 struct bpf_verifier_state *vstate = env->cur_state;
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002909 int err;
Joe Stringerfd978bf72018-10-02 13:35:35 -07002910 int i;
2911
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002912 err = release_reference_state(cur_func(env), ref_obj_id);
2913 if (err)
2914 return err;
Joe Stringerfd978bf72018-10-02 13:35:35 -07002915
Martin KaFai Lau1b986582019-03-12 10:23:02 -07002916 for (i = 0; i <= vstate->curframe; i++)
2917 release_reg_references(env, vstate->frame[i], ref_obj_id);
2918
2919 return 0;
Joe Stringerfd978bf72018-10-02 13:35:35 -07002920}
2921
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002922static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
2923 int *insn_idx)
2924{
2925 struct bpf_verifier_state *state = env->cur_state;
2926 struct bpf_func_state *caller, *callee;
Joe Stringerfd978bf72018-10-02 13:35:35 -07002927 int i, err, subprog, target_insn;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002928
Alexei Starovoitovaada9ce2017-12-25 13:15:42 -08002929 if (state->curframe + 1 >= MAX_CALL_FRAMES) {
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002930 verbose(env, "the call stack of %d frames is too deep\n",
Alexei Starovoitovaada9ce2017-12-25 13:15:42 -08002931 state->curframe + 2);
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002932 return -E2BIG;
2933 }
2934
2935 target_insn = *insn_idx + insn->imm;
2936 subprog = find_subprog(env, target_insn + 1);
2937 if (subprog < 0) {
2938 verbose(env, "verifier bug. No program starts at insn %d\n",
2939 target_insn + 1);
2940 return -EFAULT;
2941 }
2942
2943 caller = state->frame[state->curframe];
2944 if (state->frame[state->curframe + 1]) {
2945 verbose(env, "verifier bug. Frame %d already allocated\n",
2946 state->curframe + 1);
2947 return -EFAULT;
2948 }
2949
2950 callee = kzalloc(sizeof(*callee), GFP_KERNEL);
2951 if (!callee)
2952 return -ENOMEM;
2953 state->frame[state->curframe + 1] = callee;
2954
2955 /* callee cannot access r0, r6 - r9 for reading and has to write
2956 * into its own stack before reading from it.
2957 * callee can read/write into caller's stack
2958 */
2959 init_func_state(env, callee,
2960 /* remember the callsite, it will be used by bpf_exit */
2961 *insn_idx /* callsite */,
2962 state->curframe + 1 /* frameno within this callchain */,
Jiong Wangf910cef2018-05-02 16:17:17 -04002963 subprog /* subprog number within this prog */);
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002964
Joe Stringerfd978bf72018-10-02 13:35:35 -07002965 /* Transfer references to the callee */
2966 err = transfer_reference_state(callee, caller);
2967 if (err)
2968 return err;
2969
Edward Cree679c7822018-08-22 20:02:19 +01002970 /* copy r1 - r5 args that callee can access. The copy includes parent
2971 * pointers, which connects us up to the liveness chain
2972 */
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002973 for (i = BPF_REG_1; i <= BPF_REG_5; i++)
2974 callee->regs[i] = caller->regs[i];
2975
Edward Cree679c7822018-08-22 20:02:19 +01002976 /* after the call registers r0 - r5 were scratched */
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002977 for (i = 0; i < CALLER_SAVED_REGS; i++) {
2978 mark_reg_not_init(env, caller->regs, caller_saved[i]);
2979 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
2980 }
2981
2982 /* only increment it after check_reg_arg() finished */
2983 state->curframe++;
2984
2985 /* and go analyze first insn of the callee */
2986 *insn_idx = target_insn;
2987
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07002988 if (env->log.level & BPF_LOG_LEVEL) {
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08002989 verbose(env, "caller:\n");
2990 print_verifier_state(env, caller);
2991 verbose(env, "callee:\n");
2992 print_verifier_state(env, callee);
2993 }
2994 return 0;
2995}
2996
2997static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
2998{
2999 struct bpf_verifier_state *state = env->cur_state;
3000 struct bpf_func_state *caller, *callee;
3001 struct bpf_reg_state *r0;
Joe Stringerfd978bf72018-10-02 13:35:35 -07003002 int err;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08003003
3004 callee = state->frame[state->curframe];
3005 r0 = &callee->regs[BPF_REG_0];
3006 if (r0->type == PTR_TO_STACK) {
3007 /* technically it's ok to return caller's stack pointer
3008 * (or caller's caller's pointer) back to the caller,
3009 * since these pointers are valid. Only current stack
3010 * pointer will be invalid as soon as function exits,
3011 * but let's be conservative
3012 */
3013 verbose(env, "cannot return stack pointer to the caller\n");
3014 return -EINVAL;
3015 }
3016
3017 state->curframe--;
3018 caller = state->frame[state->curframe];
3019 /* return to the caller whatever r0 had in the callee */
3020 caller->regs[BPF_REG_0] = *r0;
3021
Joe Stringerfd978bf72018-10-02 13:35:35 -07003022 /* Transfer references to the caller */
3023 err = transfer_reference_state(caller, callee);
3024 if (err)
3025 return err;
3026
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08003027 *insn_idx = callee->callsite + 1;
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07003028 if (env->log.level & BPF_LOG_LEVEL) {
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08003029 verbose(env, "returning from callee:\n");
3030 print_verifier_state(env, callee);
3031 verbose(env, "to caller at %d:\n", *insn_idx);
3032 print_verifier_state(env, caller);
3033 }
3034 /* clear everything in the callee */
3035 free_func_state(callee);
3036 state->frame[state->curframe + 1] = NULL;
3037 return 0;
3038}
3039
Yonghong Song849fa502018-04-28 22:28:09 -07003040static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type,
3041 int func_id,
3042 struct bpf_call_arg_meta *meta)
3043{
3044 struct bpf_reg_state *ret_reg = &regs[BPF_REG_0];
3045
3046 if (ret_type != RET_INTEGER ||
3047 (func_id != BPF_FUNC_get_stack &&
3048 func_id != BPF_FUNC_probe_read_str))
3049 return;
3050
3051 ret_reg->smax_value = meta->msize_smax_value;
3052 ret_reg->umax_value = meta->msize_umax_value;
3053 __reg_deduce_bounds(ret_reg);
3054 __reg_bound_offset(ret_reg);
3055}
3056
Daniel Borkmannc93552c2018-05-24 02:32:53 +02003057static int
3058record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
3059 int func_id, int insn_idx)
3060{
3061 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
3062
3063 if (func_id != BPF_FUNC_tail_call &&
Daniel Borkmann09772d92018-06-02 23:06:35 +02003064 func_id != BPF_FUNC_map_lookup_elem &&
3065 func_id != BPF_FUNC_map_update_elem &&
Mauricio Vasquez Bf1a2e442018-10-18 15:16:25 +02003066 func_id != BPF_FUNC_map_delete_elem &&
3067 func_id != BPF_FUNC_map_push_elem &&
3068 func_id != BPF_FUNC_map_pop_elem &&
3069 func_id != BPF_FUNC_map_peek_elem)
Daniel Borkmannc93552c2018-05-24 02:32:53 +02003070 return 0;
Daniel Borkmann09772d92018-06-02 23:06:35 +02003071
Daniel Borkmannc93552c2018-05-24 02:32:53 +02003072 if (meta->map_ptr == NULL) {
3073 verbose(env, "kernel subsystem misconfigured verifier\n");
3074 return -EINVAL;
3075 }
3076
3077 if (!BPF_MAP_PTR(aux->map_state))
3078 bpf_map_ptr_store(aux, meta->map_ptr,
3079 meta->map_ptr->unpriv_array);
3080 else if (BPF_MAP_PTR(aux->map_state) != meta->map_ptr)
3081 bpf_map_ptr_store(aux, BPF_MAP_PTR_POISON,
3082 meta->map_ptr->unpriv_array);
3083 return 0;
3084}
3085
Joe Stringerfd978bf72018-10-02 13:35:35 -07003086static int check_reference_leak(struct bpf_verifier_env *env)
3087{
3088 struct bpf_func_state *state = cur_func(env);
3089 int i;
3090
3091 for (i = 0; i < state->acquired_refs; i++) {
3092 verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",
3093 state->refs[i].id, state->refs[i].insn_idx);
3094 }
3095 return state->acquired_refs ? -EINVAL : 0;
3096}
3097
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08003098static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn_idx)
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003099{
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003100 const struct bpf_func_proto *fn = NULL;
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07003101 struct bpf_reg_state *regs;
Daniel Borkmann33ff9822016-04-13 00:10:50 +02003102 struct bpf_call_arg_meta meta;
Alexei Starovoitov969bf052016-05-05 19:49:10 -07003103 bool changes_data;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003104 int i, err;
3105
3106 /* find function prototype */
3107 if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07003108 verbose(env, "invalid func %s#%d\n", func_id_name(func_id),
3109 func_id);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003110 return -EINVAL;
3111 }
3112
Jakub Kicinski00176a32017-10-16 16:40:54 -07003113 if (env->ops->get_func_proto)
Andrey Ignatov5e43f892018-03-30 15:08:00 -07003114 fn = env->ops->get_func_proto(func_id, env->prog);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003115 if (!fn) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07003116 verbose(env, "unknown func %s#%d\n", func_id_name(func_id),
3117 func_id);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003118 return -EINVAL;
3119 }
3120
3121 /* eBPF programs must be GPL compatible to use GPL-ed functions */
Daniel Borkmann24701ec2015-03-01 12:31:47 +01003122 if (!env->prog->gpl_compatible && fn->gpl_only) {
Daniel Borkmann3fe28672018-06-02 23:06:33 +02003123 verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003124 return -EINVAL;
3125 }
3126
Daniel Borkmann04514d12017-12-14 21:07:25 +01003127 /* With LD_ABS/IND some JITs save/restore skb from r1. */
Martin KaFai Lau17bedab2016-12-07 15:53:11 -08003128 changes_data = bpf_helper_changes_pkt_data(fn->func);
Daniel Borkmann04514d12017-12-14 21:07:25 +01003129 if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) {
3130 verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n",
3131 func_id_name(func_id), func_id);
3132 return -EINVAL;
3133 }
Alexei Starovoitov969bf052016-05-05 19:49:10 -07003134
Daniel Borkmann33ff9822016-04-13 00:10:50 +02003135 memset(&meta, 0, sizeof(meta));
Daniel Borkmann36bbef52016-09-20 00:26:13 +02003136 meta.pkt_access = fn->pkt_access;
Daniel Borkmann33ff9822016-04-13 00:10:50 +02003137
Martin KaFai Lau1b986582019-03-12 10:23:02 -07003138 err = check_func_proto(fn, func_id);
Daniel Borkmann435faee12016-04-13 00:10:51 +02003139 if (err) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07003140 verbose(env, "kernel subsystem misconfigured func %s#%d\n",
Thomas Grafebb676d2016-10-27 11:23:51 +02003141 func_id_name(func_id), func_id);
Daniel Borkmann435faee12016-04-13 00:10:51 +02003142 return err;
3143 }
3144
Alexei Starovoitovd83525c2019-01-31 15:40:04 -08003145 meta.func_id = func_id;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003146 /* check args */
Daniel Borkmann33ff9822016-04-13 00:10:50 +02003147 err = check_func_arg(env, BPF_REG_1, fn->arg1_type, &meta);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003148 if (err)
3149 return err;
Daniel Borkmann33ff9822016-04-13 00:10:50 +02003150 err = check_func_arg(env, BPF_REG_2, fn->arg2_type, &meta);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003151 if (err)
3152 return err;
Daniel Borkmann33ff9822016-04-13 00:10:50 +02003153 err = check_func_arg(env, BPF_REG_3, fn->arg3_type, &meta);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003154 if (err)
3155 return err;
Daniel Borkmann33ff9822016-04-13 00:10:50 +02003156 err = check_func_arg(env, BPF_REG_4, fn->arg4_type, &meta);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003157 if (err)
3158 return err;
Daniel Borkmann33ff9822016-04-13 00:10:50 +02003159 err = check_func_arg(env, BPF_REG_5, fn->arg5_type, &meta);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003160 if (err)
3161 return err;
3162
Daniel Borkmannc93552c2018-05-24 02:32:53 +02003163 err = record_func_map(env, &meta, func_id, insn_idx);
3164 if (err)
3165 return err;
3166
Daniel Borkmann435faee12016-04-13 00:10:51 +02003167 /* Mark slots with STACK_MISC in case of raw mode, stack offset
3168 * is inferred from register state.
3169 */
3170 for (i = 0; i < meta.access_size; i++) {
Daniel Borkmannca369602018-02-23 22:29:05 +01003171 err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B,
3172 BPF_WRITE, -1, false);
Daniel Borkmann435faee12016-04-13 00:10:51 +02003173 if (err)
3174 return err;
3175 }
3176
Joe Stringerfd978bf72018-10-02 13:35:35 -07003177 if (func_id == BPF_FUNC_tail_call) {
3178 err = check_reference_leak(env);
3179 if (err) {
3180 verbose(env, "tail_call would lead to reference leak\n");
3181 return err;
3182 }
3183 } else if (is_release_function(func_id)) {
Martin KaFai Lau1b986582019-03-12 10:23:02 -07003184 err = release_reference(env, meta.ref_obj_id);
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08003185 if (err) {
3186 verbose(env, "func %s#%d reference has not been acquired before\n",
3187 func_id_name(func_id), func_id);
Joe Stringerfd978bf72018-10-02 13:35:35 -07003188 return err;
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08003189 }
Joe Stringerfd978bf72018-10-02 13:35:35 -07003190 }
3191
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07003192 regs = cur_regs(env);
Roman Gushchincd339432018-08-02 14:27:24 -07003193
3194 /* check that flags argument in get_local_storage(map, flags) is 0,
3195 * this is required because get_local_storage() can't return an error.
3196 */
3197 if (func_id == BPF_FUNC_get_local_storage &&
3198 !register_is_null(&regs[BPF_REG_2])) {
3199 verbose(env, "get_local_storage() doesn't support non-zero flags\n");
3200 return -EINVAL;
3201 }
3202
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003203 /* reset caller saved regs */
Edward Creedc503a82017-08-15 20:34:35 +01003204 for (i = 0; i < CALLER_SAVED_REGS; i++) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07003205 mark_reg_not_init(env, regs, caller_saved[i]);
Edward Creedc503a82017-08-15 20:34:35 +01003206 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
3207 }
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003208
Edward Creedc503a82017-08-15 20:34:35 +01003209 /* update return register (already marked as written above) */
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003210 if (fn->ret_type == RET_INTEGER) {
Edward Creef1174f72017-08-07 15:26:19 +01003211 /* sets type to SCALAR_VALUE */
Jakub Kicinski61bd5212017-10-09 10:30:11 -07003212 mark_reg_unknown(env, regs, BPF_REG_0);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003213 } else if (fn->ret_type == RET_VOID) {
3214 regs[BPF_REG_0].type = NOT_INIT;
Roman Gushchin3e6a4b32018-08-02 14:27:22 -07003215 } else if (fn->ret_type == RET_PTR_TO_MAP_VALUE_OR_NULL ||
3216 fn->ret_type == RET_PTR_TO_MAP_VALUE) {
Edward Creef1174f72017-08-07 15:26:19 +01003217 /* There is no offset yet applied, variable or fixed */
Jakub Kicinski61bd5212017-10-09 10:30:11 -07003218 mark_reg_known_zero(env, regs, BPF_REG_0);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003219 /* remember map_ptr, so that check_map_access()
3220 * can check 'value_size' boundary of memory access
3221 * to map element returned from bpf_map_lookup_elem()
3222 */
Daniel Borkmann33ff9822016-04-13 00:10:50 +02003223 if (meta.map_ptr == NULL) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07003224 verbose(env,
3225 "kernel subsystem misconfigured verifier\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003226 return -EINVAL;
3227 }
Daniel Borkmann33ff9822016-04-13 00:10:50 +02003228 regs[BPF_REG_0].map_ptr = meta.map_ptr;
Daniel Borkmann4d31f302018-11-01 00:05:53 +01003229 if (fn->ret_type == RET_PTR_TO_MAP_VALUE) {
3230 regs[BPF_REG_0].type = PTR_TO_MAP_VALUE;
Alexei Starovoitove16d2f12019-01-31 15:40:05 -08003231 if (map_value_has_spin_lock(meta.map_ptr))
3232 regs[BPF_REG_0].id = ++env->id_gen;
Daniel Borkmann4d31f302018-11-01 00:05:53 +01003233 } else {
3234 regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL;
3235 regs[BPF_REG_0].id = ++env->id_gen;
3236 }
Joe Stringerc64b7982018-10-02 13:35:33 -07003237 } else if (fn->ret_type == RET_PTR_TO_SOCKET_OR_NULL) {
3238 mark_reg_known_zero(env, regs, BPF_REG_0);
3239 regs[BPF_REG_0].type = PTR_TO_SOCKET_OR_NULL;
Lorenz Bauer0f3adc22019-03-22 09:53:59 +08003240 regs[BPF_REG_0].id = ++env->id_gen;
Lorenz Bauer85a51f82019-03-22 09:54:00 +08003241 } else if (fn->ret_type == RET_PTR_TO_SOCK_COMMON_OR_NULL) {
3242 mark_reg_known_zero(env, regs, BPF_REG_0);
3243 regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON_OR_NULL;
3244 regs[BPF_REG_0].id = ++env->id_gen;
Martin KaFai Lau655a51e2019-02-09 23:22:24 -08003245 } else if (fn->ret_type == RET_PTR_TO_TCP_SOCK_OR_NULL) {
3246 mark_reg_known_zero(env, regs, BPF_REG_0);
3247 regs[BPF_REG_0].type = PTR_TO_TCP_SOCK_OR_NULL;
3248 regs[BPF_REG_0].id = ++env->id_gen;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003249 } else {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07003250 verbose(env, "unknown return type %d of func %s#%d\n",
Thomas Grafebb676d2016-10-27 11:23:51 +02003251 fn->ret_type, func_id_name(func_id), func_id);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07003252 return -EINVAL;
3253 }
Alexei Starovoitov04fd61ab2015-05-19 16:59:03 -07003254
Lorenz Bauer0f3adc22019-03-22 09:53:59 +08003255 if (is_ptr_cast_function(func_id)) {
Martin KaFai Lau1b986582019-03-12 10:23:02 -07003256 /* For release_reference() */
3257 regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id;
Lorenz Bauer0f3adc22019-03-22 09:53:59 +08003258 } else if (is_acquire_function(func_id)) {
3259 int id = acquire_reference_state(env, insn_idx);
3260
3261 if (id < 0)
3262 return id;
3263 /* For mark_ptr_or_null_reg() */
3264 regs[BPF_REG_0].id = id;
3265 /* For release_reference() */
3266 regs[BPF_REG_0].ref_obj_id = id;
3267 }
Martin KaFai Lau1b986582019-03-12 10:23:02 -07003268
Yonghong Song849fa502018-04-28 22:28:09 -07003269 do_refine_retval_range(regs, fn->ret_type, func_id, &meta);
3270
Jakub Kicinski61bd5212017-10-09 10:30:11 -07003271 err = check_map_func_compatibility(env, meta.map_ptr, func_id);
Kaixu Xia35578d72015-08-06 07:02:35 +00003272 if (err)
3273 return err;
Alexei Starovoitov04fd61ab2015-05-19 16:59:03 -07003274
Yonghong Songc195651e2018-04-28 22:28:08 -07003275 if (func_id == BPF_FUNC_get_stack && !env->prog->has_callchain_buf) {
3276 const char *err_str;
3277
3278#ifdef CONFIG_PERF_EVENTS
3279 err = get_callchain_buffers(sysctl_perf_event_max_stack);
3280 err_str = "cannot get callchain buffer for func %s#%d\n";
3281#else
3282 err = -ENOTSUPP;
3283 err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n";
3284#endif
3285 if (err) {
3286 verbose(env, err_str, func_id_name(func_id), func_id);
3287 return err;
3288 }
3289
3290 env->prog->has_callchain_buf = true;
3291 }
3292
Alexei Starovoitov969bf052016-05-05 19:49:10 -07003293 if (changes_data)
3294 clear_all_pkt_pointers(env);
3295 return 0;
3296}
3297
Edward Creeb03c9f92017-08-07 15:26:36 +01003298static bool signed_add_overflows(s64 a, s64 b)
3299{
3300 /* Do the add in u64, where overflow is well-defined */
3301 s64 res = (s64)((u64)a + (u64)b);
3302
3303 if (b < 0)
3304 return res > a;
3305 return res < a;
3306}
3307
3308static bool signed_sub_overflows(s64 a, s64 b)
3309{
3310 /* Do the sub in u64, where overflow is well-defined */
3311 s64 res = (s64)((u64)a - (u64)b);
3312
3313 if (b < 0)
3314 return res < a;
3315 return res > a;
David S. Millerd1174412017-05-10 11:22:52 -07003316}
3317
Alexei Starovoitovbb7f0f92017-12-18 20:12:00 -08003318static bool check_reg_sane_offset(struct bpf_verifier_env *env,
3319 const struct bpf_reg_state *reg,
3320 enum bpf_reg_type type)
3321{
3322 bool known = tnum_is_const(reg->var_off);
3323 s64 val = reg->var_off.value;
3324 s64 smin = reg->smin_value;
3325
3326 if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) {
3327 verbose(env, "math between %s pointer and %lld is not allowed\n",
3328 reg_type_str[type], val);
3329 return false;
3330 }
3331
3332 if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) {
3333 verbose(env, "%s pointer offset %d is not allowed\n",
3334 reg_type_str[type], reg->off);
3335 return false;
3336 }
3337
3338 if (smin == S64_MIN) {
3339 verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n",
3340 reg_type_str[type]);
3341 return false;
3342 }
3343
3344 if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) {
3345 verbose(env, "value %lld makes %s pointer be out of bounds\n",
3346 smin, reg_type_str[type]);
3347 return false;
3348 }
3349
3350 return true;
3351}
3352
Daniel Borkmann979d63d2019-01-03 00:58:34 +01003353static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env)
3354{
3355 return &env->insn_aux_data[env->insn_idx];
3356}
3357
3358static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg,
3359 u32 *ptr_limit, u8 opcode, bool off_is_neg)
3360{
3361 bool mask_to_left = (opcode == BPF_ADD && off_is_neg) ||
3362 (opcode == BPF_SUB && !off_is_neg);
3363 u32 off;
3364
3365 switch (ptr_reg->type) {
3366 case PTR_TO_STACK:
Andrey Ignatov088ec262019-04-03 23:22:39 -07003367 /* Indirect variable offset stack access is prohibited in
3368 * unprivileged mode so it's not handled here.
3369 */
Daniel Borkmann979d63d2019-01-03 00:58:34 +01003370 off = ptr_reg->off + ptr_reg->var_off.value;
3371 if (mask_to_left)
3372 *ptr_limit = MAX_BPF_STACK + off;
3373 else
3374 *ptr_limit = -off;
3375 return 0;
3376 case PTR_TO_MAP_VALUE:
3377 if (mask_to_left) {
3378 *ptr_limit = ptr_reg->umax_value + ptr_reg->off;
3379 } else {
3380 off = ptr_reg->smin_value + ptr_reg->off;
3381 *ptr_limit = ptr_reg->map_ptr->value_size - off;
3382 }
3383 return 0;
3384 default:
3385 return -EINVAL;
3386 }
3387}
3388
Daniel Borkmannd3bd7412019-01-06 00:54:37 +01003389static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env,
3390 const struct bpf_insn *insn)
3391{
3392 return env->allow_ptr_leaks || BPF_SRC(insn->code) == BPF_K;
3393}
3394
3395static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux,
3396 u32 alu_state, u32 alu_limit)
3397{
3398 /* If we arrived here from different branches with different
3399 * state or limits to sanitize, then this won't work.
3400 */
3401 if (aux->alu_state &&
3402 (aux->alu_state != alu_state ||
3403 aux->alu_limit != alu_limit))
3404 return -EACCES;
3405
3406 /* Corresponding fixup done in fixup_bpf_calls(). */
3407 aux->alu_state = alu_state;
3408 aux->alu_limit = alu_limit;
3409 return 0;
3410}
3411
3412static int sanitize_val_alu(struct bpf_verifier_env *env,
3413 struct bpf_insn *insn)
3414{
3415 struct bpf_insn_aux_data *aux = cur_aux(env);
3416
3417 if (can_skip_alu_sanitation(env, insn))
3418 return 0;
3419
3420 return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0);
3421}
3422
Daniel Borkmann979d63d2019-01-03 00:58:34 +01003423static int sanitize_ptr_alu(struct bpf_verifier_env *env,
3424 struct bpf_insn *insn,
3425 const struct bpf_reg_state *ptr_reg,
3426 struct bpf_reg_state *dst_reg,
3427 bool off_is_neg)
3428{
3429 struct bpf_verifier_state *vstate = env->cur_state;
3430 struct bpf_insn_aux_data *aux = cur_aux(env);
3431 bool ptr_is_dst_reg = ptr_reg == dst_reg;
3432 u8 opcode = BPF_OP(insn->code);
3433 u32 alu_state, alu_limit;
3434 struct bpf_reg_state tmp;
3435 bool ret;
3436
Daniel Borkmannd3bd7412019-01-06 00:54:37 +01003437 if (can_skip_alu_sanitation(env, insn))
Daniel Borkmann979d63d2019-01-03 00:58:34 +01003438 return 0;
3439
3440 /* We already marked aux for masking from non-speculative
3441 * paths, thus we got here in the first place. We only care
3442 * to explore bad access from here.
3443 */
3444 if (vstate->speculative)
3445 goto do_sim;
3446
3447 alu_state = off_is_neg ? BPF_ALU_NEG_VALUE : 0;
3448 alu_state |= ptr_is_dst_reg ?
3449 BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST;
3450
3451 if (retrieve_ptr_limit(ptr_reg, &alu_limit, opcode, off_is_neg))
3452 return 0;
Daniel Borkmannd3bd7412019-01-06 00:54:37 +01003453 if (update_alu_sanitation_state(aux, alu_state, alu_limit))
Daniel Borkmann979d63d2019-01-03 00:58:34 +01003454 return -EACCES;
Daniel Borkmann979d63d2019-01-03 00:58:34 +01003455do_sim:
3456 /* Simulate and find potential out-of-bounds access under
3457 * speculative execution from truncation as a result of
3458 * masking when off was not within expected range. If off
3459 * sits in dst, then we temporarily need to move ptr there
3460 * to simulate dst (== 0) +/-= ptr. Needed, for example,
3461 * for cases where we use K-based arithmetic in one direction
3462 * and truncated reg-based in the other in order to explore
3463 * bad access.
3464 */
3465 if (!ptr_is_dst_reg) {
3466 tmp = *dst_reg;
3467 *dst_reg = *ptr_reg;
3468 }
3469 ret = push_stack(env, env->insn_idx + 1, env->insn_idx, true);
Xu Yu08032782019-03-21 18:00:35 +08003470 if (!ptr_is_dst_reg && ret)
Daniel Borkmann979d63d2019-01-03 00:58:34 +01003471 *dst_reg = tmp;
3472 return !ret ? -EFAULT : 0;
3473}
3474
Edward Creef1174f72017-08-07 15:26:19 +01003475/* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off.
Edward Creef1174f72017-08-07 15:26:19 +01003476 * Caller should also handle BPF_MOV case separately.
3477 * If we return -EACCES, caller may want to try again treating pointer as a
3478 * scalar. So we only emit a diagnostic if !env->allow_ptr_leaks.
3479 */
3480static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
3481 struct bpf_insn *insn,
3482 const struct bpf_reg_state *ptr_reg,
3483 const struct bpf_reg_state *off_reg)
Josef Bacik48461132016-09-28 10:54:32 -04003484{
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08003485 struct bpf_verifier_state *vstate = env->cur_state;
3486 struct bpf_func_state *state = vstate->frame[vstate->curframe];
3487 struct bpf_reg_state *regs = state->regs, *dst_reg;
Edward Creef1174f72017-08-07 15:26:19 +01003488 bool known = tnum_is_const(off_reg->var_off);
Edward Creeb03c9f92017-08-07 15:26:36 +01003489 s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value,
3490 smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value;
3491 u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value,
3492 umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value;
Daniel Borkmann9d7ecee2019-01-03 00:58:32 +01003493 u32 dst = insn->dst_reg, src = insn->src_reg;
Josef Bacik48461132016-09-28 10:54:32 -04003494 u8 opcode = BPF_OP(insn->code);
Daniel Borkmann979d63d2019-01-03 00:58:34 +01003495 int ret;
Josef Bacik48461132016-09-28 10:54:32 -04003496
Edward Creef1174f72017-08-07 15:26:19 +01003497 dst_reg = &regs[dst];
Josef Bacik48461132016-09-28 10:54:32 -04003498
Daniel Borkmann6f161012018-01-18 01:15:21 +01003499 if ((known && (smin_val != smax_val || umin_val != umax_val)) ||
3500 smin_val > smax_val || umin_val > umax_val) {
3501 /* Taint dst register if offset had invalid bounds derived from
3502 * e.g. dead branches.
3503 */
3504 __mark_reg_unknown(dst_reg);
3505 return 0;
Josef Bacik48461132016-09-28 10:54:32 -04003506 }
3507
Edward Creef1174f72017-08-07 15:26:19 +01003508 if (BPF_CLASS(insn->code) != BPF_ALU64) {
3509 /* 32-bit ALU ops on pointers produce (meaningless) scalars */
Alexei Starovoitov82abbf82017-12-18 20:15:20 -08003510 verbose(env,
3511 "R%d 32-bit pointer arithmetic prohibited\n",
3512 dst);
Edward Creef1174f72017-08-07 15:26:19 +01003513 return -EACCES;
3514 }
David S. Millerd1174412017-05-10 11:22:52 -07003515
Joe Stringeraad2eea2018-10-02 13:35:30 -07003516 switch (ptr_reg->type) {
3517 case PTR_TO_MAP_VALUE_OR_NULL:
3518 verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n",
3519 dst, reg_type_str[ptr_reg->type]);
Edward Creef1174f72017-08-07 15:26:19 +01003520 return -EACCES;
Joe Stringeraad2eea2018-10-02 13:35:30 -07003521 case CONST_PTR_TO_MAP:
3522 case PTR_TO_PACKET_END:
Joe Stringerc64b7982018-10-02 13:35:33 -07003523 case PTR_TO_SOCKET:
3524 case PTR_TO_SOCKET_OR_NULL:
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08003525 case PTR_TO_SOCK_COMMON:
3526 case PTR_TO_SOCK_COMMON_OR_NULL:
Martin KaFai Lau655a51e2019-02-09 23:22:24 -08003527 case PTR_TO_TCP_SOCK:
3528 case PTR_TO_TCP_SOCK_OR_NULL:
Joe Stringeraad2eea2018-10-02 13:35:30 -07003529 verbose(env, "R%d pointer arithmetic on %s prohibited\n",
3530 dst, reg_type_str[ptr_reg->type]);
Edward Creef1174f72017-08-07 15:26:19 +01003531 return -EACCES;
Daniel Borkmann9d7ecee2019-01-03 00:58:32 +01003532 case PTR_TO_MAP_VALUE:
3533 if (!env->allow_ptr_leaks && !known && (smin_val < 0) != (smax_val < 0)) {
3534 verbose(env, "R%d has unknown scalar with mixed signed bounds, pointer arithmetic with it prohibited for !root\n",
3535 off_reg == dst_reg ? dst : src);
3536 return -EACCES;
3537 }
3538 /* fall-through */
Joe Stringeraad2eea2018-10-02 13:35:30 -07003539 default:
3540 break;
Edward Creef1174f72017-08-07 15:26:19 +01003541 }
3542
3543 /* In case of 'scalar += pointer', dst_reg inherits pointer type and id.
3544 * The id may be overwritten later if we create a new variable offset.
Josef Bacik48461132016-09-28 10:54:32 -04003545 */
Edward Creef1174f72017-08-07 15:26:19 +01003546 dst_reg->type = ptr_reg->type;
3547 dst_reg->id = ptr_reg->id;
Josef Bacikf23cc642016-11-14 15:45:36 -05003548
Alexei Starovoitovbb7f0f92017-12-18 20:12:00 -08003549 if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) ||
3550 !check_reg_sane_offset(env, ptr_reg, ptr_reg->type))
3551 return -EINVAL;
3552
Josef Bacik48461132016-09-28 10:54:32 -04003553 switch (opcode) {
3554 case BPF_ADD:
Daniel Borkmann979d63d2019-01-03 00:58:34 +01003555 ret = sanitize_ptr_alu(env, insn, ptr_reg, dst_reg, smin_val < 0);
3556 if (ret < 0) {
3557 verbose(env, "R%d tried to add from different maps or paths\n", dst);
3558 return ret;
3559 }
Edward Creef1174f72017-08-07 15:26:19 +01003560 /* We can take a fixed offset as long as it doesn't overflow
3561 * the s32 'off' field
3562 */
Edward Creeb03c9f92017-08-07 15:26:36 +01003563 if (known && (ptr_reg->off + smin_val ==
3564 (s64)(s32)(ptr_reg->off + smin_val))) {
Edward Creef1174f72017-08-07 15:26:19 +01003565 /* pointer += K. Accumulate it into fixed offset */
Edward Creeb03c9f92017-08-07 15:26:36 +01003566 dst_reg->smin_value = smin_ptr;
3567 dst_reg->smax_value = smax_ptr;
3568 dst_reg->umin_value = umin_ptr;
3569 dst_reg->umax_value = umax_ptr;
Edward Creef1174f72017-08-07 15:26:19 +01003570 dst_reg->var_off = ptr_reg->var_off;
Edward Creeb03c9f92017-08-07 15:26:36 +01003571 dst_reg->off = ptr_reg->off + smin_val;
Daniel Borkmann09625902018-11-01 00:05:52 +01003572 dst_reg->raw = ptr_reg->raw;
Edward Creef1174f72017-08-07 15:26:19 +01003573 break;
3574 }
Edward Creef1174f72017-08-07 15:26:19 +01003575 /* A new variable offset is created. Note that off_reg->off
3576 * == 0, since it's a scalar.
3577 * dst_reg gets the pointer type and since some positive
3578 * integer value was added to the pointer, give it a new 'id'
3579 * if it's a PTR_TO_PACKET.
3580 * this creates a new 'base' pointer, off_reg (variable) gets
3581 * added into the variable offset, and we copy the fixed offset
3582 * from ptr_reg.
3583 */
Edward Creeb03c9f92017-08-07 15:26:36 +01003584 if (signed_add_overflows(smin_ptr, smin_val) ||
3585 signed_add_overflows(smax_ptr, smax_val)) {
3586 dst_reg->smin_value = S64_MIN;
3587 dst_reg->smax_value = S64_MAX;
3588 } else {
3589 dst_reg->smin_value = smin_ptr + smin_val;
3590 dst_reg->smax_value = smax_ptr + smax_val;
3591 }
3592 if (umin_ptr + umin_val < umin_ptr ||
3593 umax_ptr + umax_val < umax_ptr) {
3594 dst_reg->umin_value = 0;
3595 dst_reg->umax_value = U64_MAX;
3596 } else {
3597 dst_reg->umin_value = umin_ptr + umin_val;
3598 dst_reg->umax_value = umax_ptr + umax_val;
3599 }
Edward Creef1174f72017-08-07 15:26:19 +01003600 dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off);
3601 dst_reg->off = ptr_reg->off;
Daniel Borkmann09625902018-11-01 00:05:52 +01003602 dst_reg->raw = ptr_reg->raw;
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02003603 if (reg_is_pkt_pointer(ptr_reg)) {
Edward Creef1174f72017-08-07 15:26:19 +01003604 dst_reg->id = ++env->id_gen;
3605 /* something was added to pkt_ptr, set range to zero */
Daniel Borkmann09625902018-11-01 00:05:52 +01003606 dst_reg->raw = 0;
Edward Creef1174f72017-08-07 15:26:19 +01003607 }
Josef Bacik48461132016-09-28 10:54:32 -04003608 break;
3609 case BPF_SUB:
Daniel Borkmann979d63d2019-01-03 00:58:34 +01003610 ret = sanitize_ptr_alu(env, insn, ptr_reg, dst_reg, smin_val < 0);
3611 if (ret < 0) {
3612 verbose(env, "R%d tried to sub from different maps or paths\n", dst);
3613 return ret;
3614 }
Edward Creef1174f72017-08-07 15:26:19 +01003615 if (dst_reg == off_reg) {
3616 /* scalar -= pointer. Creates an unknown scalar */
Alexei Starovoitov82abbf82017-12-18 20:15:20 -08003617 verbose(env, "R%d tried to subtract pointer from scalar\n",
3618 dst);
Edward Creef1174f72017-08-07 15:26:19 +01003619 return -EACCES;
3620 }
3621 /* We don't allow subtraction from FP, because (according to
3622 * test_verifier.c test "invalid fp arithmetic", JITs might not
3623 * be able to deal with it.
Edward Cree93057062017-07-21 14:37:34 +01003624 */
Edward Creef1174f72017-08-07 15:26:19 +01003625 if (ptr_reg->type == PTR_TO_STACK) {
Alexei Starovoitov82abbf82017-12-18 20:15:20 -08003626 verbose(env, "R%d subtraction from stack pointer prohibited\n",
3627 dst);
Edward Creef1174f72017-08-07 15:26:19 +01003628 return -EACCES;
3629 }
Edward Creeb03c9f92017-08-07 15:26:36 +01003630 if (known && (ptr_reg->off - smin_val ==
3631 (s64)(s32)(ptr_reg->off - smin_val))) {
Edward Creef1174f72017-08-07 15:26:19 +01003632 /* pointer -= K. Subtract it from fixed offset */
Edward Creeb03c9f92017-08-07 15:26:36 +01003633 dst_reg->smin_value = smin_ptr;
3634 dst_reg->smax_value = smax_ptr;
3635 dst_reg->umin_value = umin_ptr;
3636 dst_reg->umax_value = umax_ptr;
Edward Creef1174f72017-08-07 15:26:19 +01003637 dst_reg->var_off = ptr_reg->var_off;
3638 dst_reg->id = ptr_reg->id;
Edward Creeb03c9f92017-08-07 15:26:36 +01003639 dst_reg->off = ptr_reg->off - smin_val;
Daniel Borkmann09625902018-11-01 00:05:52 +01003640 dst_reg->raw = ptr_reg->raw;
Edward Creef1174f72017-08-07 15:26:19 +01003641 break;
3642 }
Edward Creef1174f72017-08-07 15:26:19 +01003643 /* A new variable offset is created. If the subtrahend is known
3644 * nonnegative, then any reg->range we had before is still good.
3645 */
Edward Creeb03c9f92017-08-07 15:26:36 +01003646 if (signed_sub_overflows(smin_ptr, smax_val) ||
3647 signed_sub_overflows(smax_ptr, smin_val)) {
3648 /* Overflow possible, we know nothing */
3649 dst_reg->smin_value = S64_MIN;
3650 dst_reg->smax_value = S64_MAX;
3651 } else {
3652 dst_reg->smin_value = smin_ptr - smax_val;
3653 dst_reg->smax_value = smax_ptr - smin_val;
3654 }
3655 if (umin_ptr < umax_val) {
3656 /* Overflow possible, we know nothing */
3657 dst_reg->umin_value = 0;
3658 dst_reg->umax_value = U64_MAX;
3659 } else {
3660 /* Cannot overflow (as long as bounds are consistent) */
3661 dst_reg->umin_value = umin_ptr - umax_val;
3662 dst_reg->umax_value = umax_ptr - umin_val;
3663 }
Edward Creef1174f72017-08-07 15:26:19 +01003664 dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off);
3665 dst_reg->off = ptr_reg->off;
Daniel Borkmann09625902018-11-01 00:05:52 +01003666 dst_reg->raw = ptr_reg->raw;
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02003667 if (reg_is_pkt_pointer(ptr_reg)) {
Edward Creef1174f72017-08-07 15:26:19 +01003668 dst_reg->id = ++env->id_gen;
3669 /* something was added to pkt_ptr, set range to zero */
Edward Creeb03c9f92017-08-07 15:26:36 +01003670 if (smin_val < 0)
Daniel Borkmann09625902018-11-01 00:05:52 +01003671 dst_reg->raw = 0;
Edward Creef1174f72017-08-07 15:26:19 +01003672 }
3673 break;
3674 case BPF_AND:
3675 case BPF_OR:
3676 case BPF_XOR:
Alexei Starovoitov82abbf82017-12-18 20:15:20 -08003677 /* bitwise ops on pointers are troublesome, prohibit. */
3678 verbose(env, "R%d bitwise operator %s on pointer prohibited\n",
3679 dst, bpf_alu_string[opcode >> 4]);
Edward Creef1174f72017-08-07 15:26:19 +01003680 return -EACCES;
3681 default:
3682 /* other operators (e.g. MUL,LSH) produce non-pointer results */
Alexei Starovoitov82abbf82017-12-18 20:15:20 -08003683 verbose(env, "R%d pointer arithmetic with %s operator prohibited\n",
3684 dst, bpf_alu_string[opcode >> 4]);
Edward Creef1174f72017-08-07 15:26:19 +01003685 return -EACCES;
3686 }
3687
Alexei Starovoitovbb7f0f92017-12-18 20:12:00 -08003688 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type))
3689 return -EINVAL;
3690
Edward Creeb03c9f92017-08-07 15:26:36 +01003691 __update_reg_bounds(dst_reg);
3692 __reg_deduce_bounds(dst_reg);
3693 __reg_bound_offset(dst_reg);
Daniel Borkmann0d6303d2019-01-03 00:58:30 +01003694
3695 /* For unprivileged we require that resulting offset must be in bounds
3696 * in order to be able to sanitize access later on.
3697 */
Daniel Borkmanne4298d22019-01-03 00:58:31 +01003698 if (!env->allow_ptr_leaks) {
3699 if (dst_reg->type == PTR_TO_MAP_VALUE &&
3700 check_map_access(env, dst, dst_reg->off, 1, false)) {
3701 verbose(env, "R%d pointer arithmetic of map value goes out of range, "
3702 "prohibited for !root\n", dst);
3703 return -EACCES;
3704 } else if (dst_reg->type == PTR_TO_STACK &&
3705 check_stack_access(env, dst_reg, dst_reg->off +
3706 dst_reg->var_off.value, 1)) {
3707 verbose(env, "R%d stack pointer arithmetic goes out of range, "
3708 "prohibited for !root\n", dst);
3709 return -EACCES;
3710 }
Daniel Borkmann0d6303d2019-01-03 00:58:30 +01003711 }
3712
Edward Creef1174f72017-08-07 15:26:19 +01003713 return 0;
3714}
3715
Jann Horn468f6ea2017-12-18 20:11:56 -08003716/* WARNING: This function does calculations on 64-bit values, but the actual
3717 * execution may occur on 32-bit values. Therefore, things like bitshifts
3718 * need extra checks in the 32-bit case.
3719 */
Edward Creef1174f72017-08-07 15:26:19 +01003720static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env,
3721 struct bpf_insn *insn,
3722 struct bpf_reg_state *dst_reg,
3723 struct bpf_reg_state src_reg)
3724{
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07003725 struct bpf_reg_state *regs = cur_regs(env);
Edward Creef1174f72017-08-07 15:26:19 +01003726 u8 opcode = BPF_OP(insn->code);
3727 bool src_known, dst_known;
Edward Creeb03c9f92017-08-07 15:26:36 +01003728 s64 smin_val, smax_val;
3729 u64 umin_val, umax_val;
Jann Horn468f6ea2017-12-18 20:11:56 -08003730 u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32;
Daniel Borkmannd3bd7412019-01-06 00:54:37 +01003731 u32 dst = insn->dst_reg;
3732 int ret;
Edward Creef1174f72017-08-07 15:26:19 +01003733
Jann Hornb7992072018-10-05 18:17:59 +02003734 if (insn_bitness == 32) {
3735 /* Relevant for 32-bit RSH: Information can propagate towards
3736 * LSB, so it isn't sufficient to only truncate the output to
3737 * 32 bits.
3738 */
3739 coerce_reg_to_size(dst_reg, 4);
3740 coerce_reg_to_size(&src_reg, 4);
3741 }
3742
Edward Creeb03c9f92017-08-07 15:26:36 +01003743 smin_val = src_reg.smin_value;
3744 smax_val = src_reg.smax_value;
3745 umin_val = src_reg.umin_value;
3746 umax_val = src_reg.umax_value;
Edward Creef1174f72017-08-07 15:26:19 +01003747 src_known = tnum_is_const(src_reg.var_off);
3748 dst_known = tnum_is_const(dst_reg->var_off);
3749
Daniel Borkmann6f161012018-01-18 01:15:21 +01003750 if ((src_known && (smin_val != smax_val || umin_val != umax_val)) ||
3751 smin_val > smax_val || umin_val > umax_val) {
3752 /* Taint dst register if offset had invalid bounds derived from
3753 * e.g. dead branches.
3754 */
3755 __mark_reg_unknown(dst_reg);
3756 return 0;
3757 }
3758
Alexei Starovoitovbb7f0f92017-12-18 20:12:00 -08003759 if (!src_known &&
3760 opcode != BPF_ADD && opcode != BPF_SUB && opcode != BPF_AND) {
3761 __mark_reg_unknown(dst_reg);
3762 return 0;
3763 }
3764
Edward Creef1174f72017-08-07 15:26:19 +01003765 switch (opcode) {
3766 case BPF_ADD:
Daniel Borkmannd3bd7412019-01-06 00:54:37 +01003767 ret = sanitize_val_alu(env, insn);
3768 if (ret < 0) {
3769 verbose(env, "R%d tried to add from different pointers or scalars\n", dst);
3770 return ret;
3771 }
Edward Creeb03c9f92017-08-07 15:26:36 +01003772 if (signed_add_overflows(dst_reg->smin_value, smin_val) ||
3773 signed_add_overflows(dst_reg->smax_value, smax_val)) {
3774 dst_reg->smin_value = S64_MIN;
3775 dst_reg->smax_value = S64_MAX;
3776 } else {
3777 dst_reg->smin_value += smin_val;
3778 dst_reg->smax_value += smax_val;
3779 }
3780 if (dst_reg->umin_value + umin_val < umin_val ||
3781 dst_reg->umax_value + umax_val < umax_val) {
3782 dst_reg->umin_value = 0;
3783 dst_reg->umax_value = U64_MAX;
3784 } else {
3785 dst_reg->umin_value += umin_val;
3786 dst_reg->umax_value += umax_val;
3787 }
Edward Creef1174f72017-08-07 15:26:19 +01003788 dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off);
3789 break;
3790 case BPF_SUB:
Daniel Borkmannd3bd7412019-01-06 00:54:37 +01003791 ret = sanitize_val_alu(env, insn);
3792 if (ret < 0) {
3793 verbose(env, "R%d tried to sub from different pointers or scalars\n", dst);
3794 return ret;
3795 }
Edward Creeb03c9f92017-08-07 15:26:36 +01003796 if (signed_sub_overflows(dst_reg->smin_value, smax_val) ||
3797 signed_sub_overflows(dst_reg->smax_value, smin_val)) {
3798 /* Overflow possible, we know nothing */
3799 dst_reg->smin_value = S64_MIN;
3800 dst_reg->smax_value = S64_MAX;
3801 } else {
3802 dst_reg->smin_value -= smax_val;
3803 dst_reg->smax_value -= smin_val;
3804 }
3805 if (dst_reg->umin_value < umax_val) {
3806 /* Overflow possible, we know nothing */
3807 dst_reg->umin_value = 0;
3808 dst_reg->umax_value = U64_MAX;
3809 } else {
3810 /* Cannot overflow (as long as bounds are consistent) */
3811 dst_reg->umin_value -= umax_val;
3812 dst_reg->umax_value -= umin_val;
3813 }
Edward Creef1174f72017-08-07 15:26:19 +01003814 dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off);
Josef Bacik48461132016-09-28 10:54:32 -04003815 break;
3816 case BPF_MUL:
Edward Creeb03c9f92017-08-07 15:26:36 +01003817 dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off);
3818 if (smin_val < 0 || dst_reg->smin_value < 0) {
Edward Creef1174f72017-08-07 15:26:19 +01003819 /* Ain't nobody got time to multiply that sign */
Edward Creeb03c9f92017-08-07 15:26:36 +01003820 __mark_reg_unbounded(dst_reg);
3821 __update_reg_bounds(dst_reg);
Edward Creef1174f72017-08-07 15:26:19 +01003822 break;
3823 }
Edward Creeb03c9f92017-08-07 15:26:36 +01003824 /* Both values are positive, so we can work with unsigned and
3825 * copy the result to signed (unless it exceeds S64_MAX).
Edward Creef1174f72017-08-07 15:26:19 +01003826 */
Edward Creeb03c9f92017-08-07 15:26:36 +01003827 if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) {
3828 /* Potential overflow, we know nothing */
3829 __mark_reg_unbounded(dst_reg);
3830 /* (except what we can learn from the var_off) */
3831 __update_reg_bounds(dst_reg);
3832 break;
3833 }
3834 dst_reg->umin_value *= umin_val;
3835 dst_reg->umax_value *= umax_val;
3836 if (dst_reg->umax_value > S64_MAX) {
3837 /* Overflow possible, we know nothing */
3838 dst_reg->smin_value = S64_MIN;
3839 dst_reg->smax_value = S64_MAX;
3840 } else {
3841 dst_reg->smin_value = dst_reg->umin_value;
3842 dst_reg->smax_value = dst_reg->umax_value;
3843 }
Josef Bacik48461132016-09-28 10:54:32 -04003844 break;
3845 case BPF_AND:
Edward Creef1174f72017-08-07 15:26:19 +01003846 if (src_known && dst_known) {
Edward Creeb03c9f92017-08-07 15:26:36 +01003847 __mark_reg_known(dst_reg, dst_reg->var_off.value &
3848 src_reg.var_off.value);
Edward Creef1174f72017-08-07 15:26:19 +01003849 break;
3850 }
Edward Creeb03c9f92017-08-07 15:26:36 +01003851 /* We get our minimum from the var_off, since that's inherently
3852 * bitwise. Our maximum is the minimum of the operands' maxima.
Josef Bacikf23cc642016-11-14 15:45:36 -05003853 */
Edward Creef1174f72017-08-07 15:26:19 +01003854 dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off);
Edward Creeb03c9f92017-08-07 15:26:36 +01003855 dst_reg->umin_value = dst_reg->var_off.value;
3856 dst_reg->umax_value = min(dst_reg->umax_value, umax_val);
3857 if (dst_reg->smin_value < 0 || smin_val < 0) {
3858 /* Lose signed bounds when ANDing negative numbers,
3859 * ain't nobody got time for that.
3860 */
3861 dst_reg->smin_value = S64_MIN;
3862 dst_reg->smax_value = S64_MAX;
3863 } else {
3864 /* ANDing two positives gives a positive, so safe to
3865 * cast result into s64.
3866 */
3867 dst_reg->smin_value = dst_reg->umin_value;
3868 dst_reg->smax_value = dst_reg->umax_value;
3869 }
3870 /* We may learn something more from the var_off */
3871 __update_reg_bounds(dst_reg);
Edward Creef1174f72017-08-07 15:26:19 +01003872 break;
3873 case BPF_OR:
3874 if (src_known && dst_known) {
Edward Creeb03c9f92017-08-07 15:26:36 +01003875 __mark_reg_known(dst_reg, dst_reg->var_off.value |
3876 src_reg.var_off.value);
Edward Creef1174f72017-08-07 15:26:19 +01003877 break;
3878 }
Edward Creeb03c9f92017-08-07 15:26:36 +01003879 /* We get our maximum from the var_off, and our minimum is the
3880 * maximum of the operands' minima
Edward Creef1174f72017-08-07 15:26:19 +01003881 */
3882 dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off);
Edward Creeb03c9f92017-08-07 15:26:36 +01003883 dst_reg->umin_value = max(dst_reg->umin_value, umin_val);
3884 dst_reg->umax_value = dst_reg->var_off.value |
3885 dst_reg->var_off.mask;
3886 if (dst_reg->smin_value < 0 || smin_val < 0) {
3887 /* Lose signed bounds when ORing negative numbers,
3888 * ain't nobody got time for that.
3889 */
3890 dst_reg->smin_value = S64_MIN;
3891 dst_reg->smax_value = S64_MAX;
Edward Creef1174f72017-08-07 15:26:19 +01003892 } else {
Edward Creeb03c9f92017-08-07 15:26:36 +01003893 /* ORing two positives gives a positive, so safe to
3894 * cast result into s64.
3895 */
3896 dst_reg->smin_value = dst_reg->umin_value;
3897 dst_reg->smax_value = dst_reg->umax_value;
Edward Creef1174f72017-08-07 15:26:19 +01003898 }
Edward Creeb03c9f92017-08-07 15:26:36 +01003899 /* We may learn something more from the var_off */
3900 __update_reg_bounds(dst_reg);
Josef Bacik48461132016-09-28 10:54:32 -04003901 break;
3902 case BPF_LSH:
Jann Horn468f6ea2017-12-18 20:11:56 -08003903 if (umax_val >= insn_bitness) {
3904 /* Shifts greater than 31 or 63 are undefined.
3905 * This includes shifts by a negative number.
Edward Creeb03c9f92017-08-07 15:26:36 +01003906 */
Jakub Kicinski61bd5212017-10-09 10:30:11 -07003907 mark_reg_unknown(env, regs, insn->dst_reg);
Edward Creef1174f72017-08-07 15:26:19 +01003908 break;
3909 }
Edward Creeb03c9f92017-08-07 15:26:36 +01003910 /* We lose all sign bit information (except what we can pick
3911 * up from var_off)
Josef Bacik48461132016-09-28 10:54:32 -04003912 */
Edward Creeb03c9f92017-08-07 15:26:36 +01003913 dst_reg->smin_value = S64_MIN;
3914 dst_reg->smax_value = S64_MAX;
3915 /* If we might shift our top bit out, then we know nothing */
3916 if (dst_reg->umax_value > 1ULL << (63 - umax_val)) {
3917 dst_reg->umin_value = 0;
3918 dst_reg->umax_value = U64_MAX;
David S. Millerd1174412017-05-10 11:22:52 -07003919 } else {
Edward Creeb03c9f92017-08-07 15:26:36 +01003920 dst_reg->umin_value <<= umin_val;
3921 dst_reg->umax_value <<= umax_val;
David S. Millerd1174412017-05-10 11:22:52 -07003922 }
Yonghong Songafbe1a52018-04-28 22:28:10 -07003923 dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val);
Edward Creeb03c9f92017-08-07 15:26:36 +01003924 /* We may learn something more from the var_off */
3925 __update_reg_bounds(dst_reg);
Josef Bacik48461132016-09-28 10:54:32 -04003926 break;
3927 case BPF_RSH:
Jann Horn468f6ea2017-12-18 20:11:56 -08003928 if (umax_val >= insn_bitness) {
3929 /* Shifts greater than 31 or 63 are undefined.
3930 * This includes shifts by a negative number.
Edward Creeb03c9f92017-08-07 15:26:36 +01003931 */
Jakub Kicinski61bd5212017-10-09 10:30:11 -07003932 mark_reg_unknown(env, regs, insn->dst_reg);
Edward Creef1174f72017-08-07 15:26:19 +01003933 break;
3934 }
Edward Cree4374f252017-12-18 20:11:53 -08003935 /* BPF_RSH is an unsigned shift. If the value in dst_reg might
3936 * be negative, then either:
3937 * 1) src_reg might be zero, so the sign bit of the result is
3938 * unknown, so we lose our signed bounds
3939 * 2) it's known negative, thus the unsigned bounds capture the
3940 * signed bounds
3941 * 3) the signed bounds cross zero, so they tell us nothing
3942 * about the result
3943 * If the value in dst_reg is known nonnegative, then again the
3944 * unsigned bounts capture the signed bounds.
3945 * Thus, in all cases it suffices to blow away our signed bounds
3946 * and rely on inferring new ones from the unsigned bounds and
3947 * var_off of the result.
3948 */
3949 dst_reg->smin_value = S64_MIN;
3950 dst_reg->smax_value = S64_MAX;
Yonghong Songafbe1a52018-04-28 22:28:10 -07003951 dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val);
Edward Creeb03c9f92017-08-07 15:26:36 +01003952 dst_reg->umin_value >>= umax_val;
3953 dst_reg->umax_value >>= umin_val;
3954 /* We may learn something more from the var_off */
3955 __update_reg_bounds(dst_reg);
Josef Bacik48461132016-09-28 10:54:32 -04003956 break;
Yonghong Song9cbe1f5a2018-04-28 22:28:11 -07003957 case BPF_ARSH:
3958 if (umax_val >= insn_bitness) {
3959 /* Shifts greater than 31 or 63 are undefined.
3960 * This includes shifts by a negative number.
3961 */
3962 mark_reg_unknown(env, regs, insn->dst_reg);
3963 break;
3964 }
3965
3966 /* Upon reaching here, src_known is true and
3967 * umax_val is equal to umin_val.
3968 */
3969 dst_reg->smin_value >>= umin_val;
3970 dst_reg->smax_value >>= umin_val;
3971 dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val);
3972
3973 /* blow away the dst_reg umin_value/umax_value and rely on
3974 * dst_reg var_off to refine the result.
3975 */
3976 dst_reg->umin_value = 0;
3977 dst_reg->umax_value = U64_MAX;
3978 __update_reg_bounds(dst_reg);
3979 break;
Josef Bacik48461132016-09-28 10:54:32 -04003980 default:
Jakub Kicinski61bd5212017-10-09 10:30:11 -07003981 mark_reg_unknown(env, regs, insn->dst_reg);
Josef Bacik48461132016-09-28 10:54:32 -04003982 break;
3983 }
3984
Jann Horn468f6ea2017-12-18 20:11:56 -08003985 if (BPF_CLASS(insn->code) != BPF_ALU64) {
3986 /* 32-bit ALU ops are (32,32)->32 */
3987 coerce_reg_to_size(dst_reg, 4);
Jann Horn468f6ea2017-12-18 20:11:56 -08003988 }
3989
Edward Creeb03c9f92017-08-07 15:26:36 +01003990 __reg_deduce_bounds(dst_reg);
3991 __reg_bound_offset(dst_reg);
Edward Creef1174f72017-08-07 15:26:19 +01003992 return 0;
3993}
3994
3995/* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max
3996 * and var_off.
3997 */
3998static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,
3999 struct bpf_insn *insn)
4000{
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004001 struct bpf_verifier_state *vstate = env->cur_state;
4002 struct bpf_func_state *state = vstate->frame[vstate->curframe];
4003 struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg;
Edward Creef1174f72017-08-07 15:26:19 +01004004 struct bpf_reg_state *ptr_reg = NULL, off_reg = {0};
4005 u8 opcode = BPF_OP(insn->code);
Edward Creef1174f72017-08-07 15:26:19 +01004006
4007 dst_reg = &regs[insn->dst_reg];
Edward Creef1174f72017-08-07 15:26:19 +01004008 src_reg = NULL;
4009 if (dst_reg->type != SCALAR_VALUE)
4010 ptr_reg = dst_reg;
4011 if (BPF_SRC(insn->code) == BPF_X) {
4012 src_reg = &regs[insn->src_reg];
Edward Creef1174f72017-08-07 15:26:19 +01004013 if (src_reg->type != SCALAR_VALUE) {
4014 if (dst_reg->type != SCALAR_VALUE) {
4015 /* Combining two pointers by any ALU op yields
Alexei Starovoitov82abbf82017-12-18 20:15:20 -08004016 * an arbitrary scalar. Disallow all math except
4017 * pointer subtraction
Edward Creef1174f72017-08-07 15:26:19 +01004018 */
Alexei Starovoitovdd066822018-09-12 14:06:10 -07004019 if (opcode == BPF_SUB && env->allow_ptr_leaks) {
Alexei Starovoitov82abbf82017-12-18 20:15:20 -08004020 mark_reg_unknown(env, regs, insn->dst_reg);
4021 return 0;
Edward Creef1174f72017-08-07 15:26:19 +01004022 }
Alexei Starovoitov82abbf82017-12-18 20:15:20 -08004023 verbose(env, "R%d pointer %s pointer prohibited\n",
4024 insn->dst_reg,
4025 bpf_alu_string[opcode >> 4]);
4026 return -EACCES;
Edward Creef1174f72017-08-07 15:26:19 +01004027 } else {
4028 /* scalar += pointer
4029 * This is legal, but we have to reverse our
4030 * src/dest handling in computing the range
4031 */
Alexei Starovoitov82abbf82017-12-18 20:15:20 -08004032 return adjust_ptr_min_max_vals(env, insn,
4033 src_reg, dst_reg);
Edward Creef1174f72017-08-07 15:26:19 +01004034 }
4035 } else if (ptr_reg) {
4036 /* pointer += scalar */
Alexei Starovoitov82abbf82017-12-18 20:15:20 -08004037 return adjust_ptr_min_max_vals(env, insn,
4038 dst_reg, src_reg);
Edward Creef1174f72017-08-07 15:26:19 +01004039 }
4040 } else {
4041 /* Pretend the src is a reg with a known value, since we only
4042 * need to be able to read from this state.
4043 */
4044 off_reg.type = SCALAR_VALUE;
Edward Creeb03c9f92017-08-07 15:26:36 +01004045 __mark_reg_known(&off_reg, insn->imm);
Edward Creef1174f72017-08-07 15:26:19 +01004046 src_reg = &off_reg;
Alexei Starovoitov82abbf82017-12-18 20:15:20 -08004047 if (ptr_reg) /* pointer += K */
4048 return adjust_ptr_min_max_vals(env, insn,
4049 ptr_reg, src_reg);
Edward Creef1174f72017-08-07 15:26:19 +01004050 }
4051
4052 /* Got here implies adding two SCALAR_VALUEs */
4053 if (WARN_ON_ONCE(ptr_reg)) {
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004054 print_verifier_state(env, state);
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004055 verbose(env, "verifier internal error: unexpected ptr_reg\n");
Edward Creef1174f72017-08-07 15:26:19 +01004056 return -EINVAL;
4057 }
4058 if (WARN_ON(!src_reg)) {
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004059 print_verifier_state(env, state);
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004060 verbose(env, "verifier internal error: no src_reg\n");
Edward Creef1174f72017-08-07 15:26:19 +01004061 return -EINVAL;
4062 }
4063 return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg);
Josef Bacik48461132016-09-28 10:54:32 -04004064}
4065
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004066/* check validity of 32-bit and 64-bit arithmetic operations */
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01004067static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004068{
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07004069 struct bpf_reg_state *regs = cur_regs(env);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004070 u8 opcode = BPF_OP(insn->code);
4071 int err;
4072
4073 if (opcode == BPF_END || opcode == BPF_NEG) {
4074 if (opcode == BPF_NEG) {
4075 if (BPF_SRC(insn->code) != 0 ||
4076 insn->src_reg != BPF_REG_0 ||
4077 insn->off != 0 || insn->imm != 0) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004078 verbose(env, "BPF_NEG uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004079 return -EINVAL;
4080 }
4081 } else {
4082 if (insn->src_reg != BPF_REG_0 || insn->off != 0 ||
Edward Creee67b8a62017-09-15 14:37:38 +01004083 (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) ||
4084 BPF_CLASS(insn->code) == BPF_ALU64) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004085 verbose(env, "BPF_END uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004086 return -EINVAL;
4087 }
4088 }
4089
4090 /* check src operand */
Edward Creedc503a82017-08-15 20:34:35 +01004091 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004092 if (err)
4093 return err;
4094
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07004095 if (is_pointer_value(env, insn->dst_reg)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004096 verbose(env, "R%d pointer arithmetic prohibited\n",
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07004097 insn->dst_reg);
4098 return -EACCES;
4099 }
4100
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004101 /* check dest operand */
Edward Creedc503a82017-08-15 20:34:35 +01004102 err = check_reg_arg(env, insn->dst_reg, DST_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004103 if (err)
4104 return err;
4105
4106 } else if (opcode == BPF_MOV) {
4107
4108 if (BPF_SRC(insn->code) == BPF_X) {
4109 if (insn->imm != 0 || insn->off != 0) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004110 verbose(env, "BPF_MOV uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004111 return -EINVAL;
4112 }
4113
4114 /* check src operand */
Edward Creedc503a82017-08-15 20:34:35 +01004115 err = check_reg_arg(env, insn->src_reg, SRC_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004116 if (err)
4117 return err;
4118 } else {
4119 if (insn->src_reg != BPF_REG_0 || insn->off != 0) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004120 verbose(env, "BPF_MOV uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004121 return -EINVAL;
4122 }
4123 }
4124
Arthur Fabrefbeb1602018-07-31 18:17:22 +01004125 /* check dest operand, mark as required later */
4126 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004127 if (err)
4128 return err;
4129
4130 if (BPF_SRC(insn->code) == BPF_X) {
Jiong Wange434b8c2018-12-07 12:16:18 -05004131 struct bpf_reg_state *src_reg = regs + insn->src_reg;
4132 struct bpf_reg_state *dst_reg = regs + insn->dst_reg;
4133
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004134 if (BPF_CLASS(insn->code) == BPF_ALU64) {
4135 /* case: R1 = R2
4136 * copy register state to dest reg
4137 */
Jiong Wange434b8c2018-12-07 12:16:18 -05004138 *dst_reg = *src_reg;
4139 dst_reg->live |= REG_LIVE_WRITTEN;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004140 } else {
Edward Creef1174f72017-08-07 15:26:19 +01004141 /* R1 = (u32) R2 */
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07004142 if (is_pointer_value(env, insn->src_reg)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004143 verbose(env,
4144 "R%d partial copy of pointer\n",
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07004145 insn->src_reg);
4146 return -EACCES;
Jiong Wange434b8c2018-12-07 12:16:18 -05004147 } else if (src_reg->type == SCALAR_VALUE) {
4148 *dst_reg = *src_reg;
4149 dst_reg->live |= REG_LIVE_WRITTEN;
4150 } else {
4151 mark_reg_unknown(env, regs,
4152 insn->dst_reg);
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07004153 }
Jiong Wange434b8c2018-12-07 12:16:18 -05004154 coerce_reg_to_size(dst_reg, 4);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004155 }
4156 } else {
4157 /* case: R = imm
4158 * remember the value we stored into this reg
4159 */
Arthur Fabrefbeb1602018-07-31 18:17:22 +01004160 /* clear any state __mark_reg_known doesn't set */
4161 mark_reg_unknown(env, regs, insn->dst_reg);
Edward Creef1174f72017-08-07 15:26:19 +01004162 regs[insn->dst_reg].type = SCALAR_VALUE;
Jann Horn95a762e2017-12-18 20:11:54 -08004163 if (BPF_CLASS(insn->code) == BPF_ALU64) {
4164 __mark_reg_known(regs + insn->dst_reg,
4165 insn->imm);
4166 } else {
4167 __mark_reg_known(regs + insn->dst_reg,
4168 (u32)insn->imm);
4169 }
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004170 }
4171
4172 } else if (opcode > BPF_END) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004173 verbose(env, "invalid BPF_ALU opcode %x\n", opcode);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004174 return -EINVAL;
4175
4176 } else { /* all other ALU ops: and, sub, xor, add, ... */
4177
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004178 if (BPF_SRC(insn->code) == BPF_X) {
4179 if (insn->imm != 0 || insn->off != 0) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004180 verbose(env, "BPF_ALU uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004181 return -EINVAL;
4182 }
4183 /* check src1 operand */
Edward Creedc503a82017-08-15 20:34:35 +01004184 err = check_reg_arg(env, insn->src_reg, SRC_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004185 if (err)
4186 return err;
4187 } else {
4188 if (insn->src_reg != BPF_REG_0 || insn->off != 0) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004189 verbose(env, "BPF_ALU uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004190 return -EINVAL;
4191 }
4192 }
4193
4194 /* check src2 operand */
Edward Creedc503a82017-08-15 20:34:35 +01004195 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004196 if (err)
4197 return err;
4198
4199 if ((opcode == BPF_MOD || opcode == BPF_DIV) &&
4200 BPF_SRC(insn->code) == BPF_K && insn->imm == 0) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004201 verbose(env, "div by zero\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004202 return -EINVAL;
4203 }
4204
Rabin Vincent229394e82016-01-12 20:17:08 +01004205 if ((opcode == BPF_LSH || opcode == BPF_RSH ||
4206 opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) {
4207 int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32;
4208
4209 if (insn->imm < 0 || insn->imm >= size) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004210 verbose(env, "invalid shift %d\n", insn->imm);
Rabin Vincent229394e82016-01-12 20:17:08 +01004211 return -EINVAL;
4212 }
4213 }
4214
Alexei Starovoitov1a0dc1a2016-05-05 19:49:09 -07004215 /* check dest operand */
Edward Creedc503a82017-08-15 20:34:35 +01004216 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
Alexei Starovoitov1a0dc1a2016-05-05 19:49:09 -07004217 if (err)
4218 return err;
4219
Edward Creef1174f72017-08-07 15:26:19 +01004220 return adjust_reg_min_max_vals(env, insn);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004221 }
4222
4223 return 0;
4224}
4225
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004226static void find_good_pkt_pointers(struct bpf_verifier_state *vstate,
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02004227 struct bpf_reg_state *dst_reg,
David S. Millerf8ddadc2017-10-22 13:36:53 +01004228 enum bpf_reg_type type,
Daniel Borkmannfb2a3112017-10-21 02:34:21 +02004229 bool range_right_open)
Alexei Starovoitov969bf052016-05-05 19:49:10 -07004230{
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004231 struct bpf_func_state *state = vstate->frame[vstate->curframe];
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01004232 struct bpf_reg_state *regs = state->regs, *reg;
Daniel Borkmannfb2a3112017-10-21 02:34:21 +02004233 u16 new_range;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004234 int i, j;
Daniel Borkmann2d2be8c2016-09-08 01:03:42 +02004235
Daniel Borkmannfb2a3112017-10-21 02:34:21 +02004236 if (dst_reg->off < 0 ||
4237 (dst_reg->off == 0 && range_right_open))
Edward Creef1174f72017-08-07 15:26:19 +01004238 /* This doesn't give us any range */
4239 return;
4240
Edward Creeb03c9f92017-08-07 15:26:36 +01004241 if (dst_reg->umax_value > MAX_PACKET_OFF ||
4242 dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF)
Edward Creef1174f72017-08-07 15:26:19 +01004243 /* Risk of overflow. For instance, ptr + (1<<63) may be less
4244 * than pkt_end, but that's because it's also less than pkt.
4245 */
4246 return;
4247
Daniel Borkmannfb2a3112017-10-21 02:34:21 +02004248 new_range = dst_reg->off;
4249 if (range_right_open)
4250 new_range--;
4251
4252 /* Examples for register markings:
Daniel Borkmann2d2be8c2016-09-08 01:03:42 +02004253 *
Daniel Borkmannfb2a3112017-10-21 02:34:21 +02004254 * pkt_data in dst register:
Daniel Borkmann2d2be8c2016-09-08 01:03:42 +02004255 *
4256 * r2 = r3;
4257 * r2 += 8;
4258 * if (r2 > pkt_end) goto <handle exception>
4259 * <access okay>
4260 *
Daniel Borkmannb4e432f2017-08-10 01:40:02 +02004261 * r2 = r3;
4262 * r2 += 8;
4263 * if (r2 < pkt_end) goto <access okay>
4264 * <handle exception>
4265 *
Daniel Borkmann2d2be8c2016-09-08 01:03:42 +02004266 * Where:
4267 * r2 == dst_reg, pkt_end == src_reg
4268 * r2=pkt(id=n,off=8,r=0)
4269 * r3=pkt(id=n,off=0,r=0)
4270 *
Daniel Borkmannfb2a3112017-10-21 02:34:21 +02004271 * pkt_data in src register:
Daniel Borkmann2d2be8c2016-09-08 01:03:42 +02004272 *
4273 * r2 = r3;
4274 * r2 += 8;
4275 * if (pkt_end >= r2) goto <access okay>
4276 * <handle exception>
4277 *
Daniel Borkmannb4e432f2017-08-10 01:40:02 +02004278 * r2 = r3;
4279 * r2 += 8;
4280 * if (pkt_end <= r2) goto <handle exception>
4281 * <access okay>
4282 *
Daniel Borkmann2d2be8c2016-09-08 01:03:42 +02004283 * Where:
4284 * pkt_end == dst_reg, r2 == src_reg
4285 * r2=pkt(id=n,off=8,r=0)
4286 * r3=pkt(id=n,off=0,r=0)
4287 *
4288 * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8)
Daniel Borkmannfb2a3112017-10-21 02:34:21 +02004289 * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8)
4290 * and [r3, r3 + 8-1) respectively is safe to access depending on
4291 * the check.
Alexei Starovoitov969bf052016-05-05 19:49:10 -07004292 */
Daniel Borkmann2d2be8c2016-09-08 01:03:42 +02004293
Edward Creef1174f72017-08-07 15:26:19 +01004294 /* If our ids match, then we must have the same max_value. And we
4295 * don't care about the other reg's fixed offset, since if it's too big
4296 * the range won't allow anything.
4297 * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16.
4298 */
Alexei Starovoitov969bf052016-05-05 19:49:10 -07004299 for (i = 0; i < MAX_BPF_REG; i++)
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02004300 if (regs[i].type == type && regs[i].id == dst_reg->id)
Alexei Starovoitovb1977682017-03-24 15:57:33 -07004301 /* keep the maximum range already checked */
Daniel Borkmannfb2a3112017-10-21 02:34:21 +02004302 regs[i].range = max(regs[i].range, new_range);
Alexei Starovoitov969bf052016-05-05 19:49:10 -07004303
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004304 for (j = 0; j <= vstate->curframe; j++) {
4305 state = vstate->frame[j];
Joe Stringerf3709f62018-10-02 13:35:29 -07004306 bpf_for_each_spilled_reg(i, state, reg) {
4307 if (!reg)
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004308 continue;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004309 if (reg->type == type && reg->id == dst_reg->id)
4310 reg->range = max(reg->range, new_range);
4311 }
Alexei Starovoitov969bf052016-05-05 19:49:10 -07004312 }
4313}
4314
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004315/* compute branch direction of the expression "if (reg opcode val) goto target;"
4316 * and return:
4317 * 1 - branch will be taken and "goto target" will be executed
4318 * 0 - branch will not be taken and fall-through to next insn
4319 * -1 - unknown. Example: "if (reg < 5)" is unknown when register value range [0,10]
4320 */
Jiong Wang092ed092019-01-26 12:26:01 -05004321static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode,
4322 bool is_jmp32)
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004323{
Jiong Wang092ed092019-01-26 12:26:01 -05004324 struct bpf_reg_state reg_lo;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004325 s64 sval;
4326
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004327 if (__is_pointer_value(false, reg))
4328 return -1;
4329
Jiong Wang092ed092019-01-26 12:26:01 -05004330 if (is_jmp32) {
4331 reg_lo = *reg;
4332 reg = &reg_lo;
4333 /* For JMP32, only low 32 bits are compared, coerce_reg_to_size
4334 * could truncate high bits and update umin/umax according to
4335 * information of low bits.
4336 */
4337 coerce_reg_to_size(reg, 4);
4338 /* smin/smax need special handling. For example, after coerce,
4339 * if smin_value is 0x00000000ffffffffLL, the value is -1 when
4340 * used as operand to JMP32. It is a negative number from s32's
4341 * point of view, while it is a positive number when seen as
4342 * s64. The smin/smax are kept as s64, therefore, when used with
4343 * JMP32, they need to be transformed into s32, then sign
4344 * extended back to s64.
4345 *
4346 * Also, smin/smax were copied from umin/umax. If umin/umax has
4347 * different sign bit, then min/max relationship doesn't
4348 * maintain after casting into s32, for this case, set smin/smax
4349 * to safest range.
4350 */
4351 if ((reg->umax_value ^ reg->umin_value) &
4352 (1ULL << 31)) {
4353 reg->smin_value = S32_MIN;
4354 reg->smax_value = S32_MAX;
4355 }
4356 reg->smin_value = (s64)(s32)reg->smin_value;
4357 reg->smax_value = (s64)(s32)reg->smax_value;
4358
4359 val = (u32)val;
4360 sval = (s64)(s32)val;
4361 } else {
4362 sval = (s64)val;
4363 }
Jiong Wanga72dafa2019-01-26 12:26:00 -05004364
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004365 switch (opcode) {
4366 case BPF_JEQ:
4367 if (tnum_is_const(reg->var_off))
4368 return !!tnum_equals_const(reg->var_off, val);
4369 break;
4370 case BPF_JNE:
4371 if (tnum_is_const(reg->var_off))
4372 return !tnum_equals_const(reg->var_off, val);
4373 break;
Jakub Kicinski960ea052018-12-19 22:13:04 -08004374 case BPF_JSET:
4375 if ((~reg->var_off.mask & reg->var_off.value) & val)
4376 return 1;
4377 if (!((reg->var_off.mask | reg->var_off.value) & val))
4378 return 0;
4379 break;
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004380 case BPF_JGT:
4381 if (reg->umin_value > val)
4382 return 1;
4383 else if (reg->umax_value <= val)
4384 return 0;
4385 break;
4386 case BPF_JSGT:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004387 if (reg->smin_value > sval)
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004388 return 1;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004389 else if (reg->smax_value < sval)
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004390 return 0;
4391 break;
4392 case BPF_JLT:
4393 if (reg->umax_value < val)
4394 return 1;
4395 else if (reg->umin_value >= val)
4396 return 0;
4397 break;
4398 case BPF_JSLT:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004399 if (reg->smax_value < sval)
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004400 return 1;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004401 else if (reg->smin_value >= sval)
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004402 return 0;
4403 break;
4404 case BPF_JGE:
4405 if (reg->umin_value >= val)
4406 return 1;
4407 else if (reg->umax_value < val)
4408 return 0;
4409 break;
4410 case BPF_JSGE:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004411 if (reg->smin_value >= sval)
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004412 return 1;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004413 else if (reg->smax_value < sval)
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004414 return 0;
4415 break;
4416 case BPF_JLE:
4417 if (reg->umax_value <= val)
4418 return 1;
4419 else if (reg->umin_value > val)
4420 return 0;
4421 break;
4422 case BPF_JSLE:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004423 if (reg->smax_value <= sval)
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004424 return 1;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004425 else if (reg->smin_value > sval)
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004426 return 0;
4427 break;
4428 }
4429
4430 return -1;
4431}
4432
Jiong Wang092ed092019-01-26 12:26:01 -05004433/* Generate min value of the high 32-bit from TNUM info. */
4434static u64 gen_hi_min(struct tnum var)
4435{
4436 return var.value & ~0xffffffffULL;
4437}
4438
4439/* Generate max value of the high 32-bit from TNUM info. */
4440static u64 gen_hi_max(struct tnum var)
4441{
4442 return (var.value | var.mask) & ~0xffffffffULL;
4443}
4444
4445/* Return true if VAL is compared with a s64 sign extended from s32, and they
4446 * are with the same signedness.
4447 */
4448static bool cmp_val_with_extended_s64(s64 sval, struct bpf_reg_state *reg)
4449{
4450 return ((s32)sval >= 0 &&
4451 reg->smin_value >= 0 && reg->smax_value <= S32_MAX) ||
4452 ((s32)sval < 0 &&
4453 reg->smax_value <= 0 && reg->smin_value >= S32_MIN);
4454}
4455
Josef Bacik48461132016-09-28 10:54:32 -04004456/* Adjusts the register min/max values in the case that the dst_reg is the
4457 * variable register that we are working on, and src_reg is a constant or we're
4458 * simply doing a BPF_K check.
Edward Creef1174f72017-08-07 15:26:19 +01004459 * In JEQ/JNE cases we also adjust the var_off values.
Josef Bacik48461132016-09-28 10:54:32 -04004460 */
4461static void reg_set_min_max(struct bpf_reg_state *true_reg,
4462 struct bpf_reg_state *false_reg, u64 val,
Jiong Wang092ed092019-01-26 12:26:01 -05004463 u8 opcode, bool is_jmp32)
Josef Bacik48461132016-09-28 10:54:32 -04004464{
Jiong Wanga72dafa2019-01-26 12:26:00 -05004465 s64 sval;
4466
Edward Creef1174f72017-08-07 15:26:19 +01004467 /* If the dst_reg is a pointer, we can't learn anything about its
4468 * variable offset from the compare (unless src_reg were a pointer into
4469 * the same object, but we don't bother with that.
4470 * Since false_reg and true_reg have the same type by construction, we
4471 * only need to check one of them for pointerness.
4472 */
4473 if (__is_pointer_value(false, false_reg))
4474 return;
Daniel Borkmann4cabc5b2017-07-21 00:00:21 +02004475
Jiong Wang092ed092019-01-26 12:26:01 -05004476 val = is_jmp32 ? (u32)val : val;
4477 sval = is_jmp32 ? (s64)(s32)val : (s64)val;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004478
Josef Bacik48461132016-09-28 10:54:32 -04004479 switch (opcode) {
4480 case BPF_JEQ:
Josef Bacik48461132016-09-28 10:54:32 -04004481 case BPF_JNE:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004482 {
4483 struct bpf_reg_state *reg =
4484 opcode == BPF_JEQ ? true_reg : false_reg;
4485
4486 /* For BPF_JEQ, if this is false we know nothing Jon Snow, but
4487 * if it is true we know the value for sure. Likewise for
4488 * BPF_JNE.
Josef Bacik48461132016-09-28 10:54:32 -04004489 */
Jiong Wang092ed092019-01-26 12:26:01 -05004490 if (is_jmp32) {
4491 u64 old_v = reg->var_off.value;
4492 u64 hi_mask = ~0xffffffffULL;
4493
4494 reg->var_off.value = (old_v & hi_mask) | val;
4495 reg->var_off.mask &= hi_mask;
4496 } else {
4497 __mark_reg_known(reg, val);
4498 }
Josef Bacik48461132016-09-28 10:54:32 -04004499 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004500 }
Jakub Kicinski960ea052018-12-19 22:13:04 -08004501 case BPF_JSET:
4502 false_reg->var_off = tnum_and(false_reg->var_off,
4503 tnum_const(~val));
4504 if (is_power_of_2(val))
4505 true_reg->var_off = tnum_or(true_reg->var_off,
4506 tnum_const(val));
4507 break;
Josef Bacik48461132016-09-28 10:54:32 -04004508 case BPF_JGE:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004509 case BPF_JGT:
4510 {
4511 u64 false_umax = opcode == BPF_JGT ? val : val - 1;
4512 u64 true_umin = opcode == BPF_JGT ? val + 1 : val;
4513
Jiong Wang092ed092019-01-26 12:26:01 -05004514 if (is_jmp32) {
4515 false_umax += gen_hi_max(false_reg->var_off);
4516 true_umin += gen_hi_min(true_reg->var_off);
4517 }
Jiong Wanga72dafa2019-01-26 12:26:00 -05004518 false_reg->umax_value = min(false_reg->umax_value, false_umax);
4519 true_reg->umin_value = max(true_reg->umin_value, true_umin);
Edward Creeb03c9f92017-08-07 15:26:36 +01004520 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004521 }
Josef Bacik48461132016-09-28 10:54:32 -04004522 case BPF_JSGE:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004523 case BPF_JSGT:
4524 {
4525 s64 false_smax = opcode == BPF_JSGT ? sval : sval - 1;
4526 s64 true_smin = opcode == BPF_JSGT ? sval + 1 : sval;
4527
Jiong Wang092ed092019-01-26 12:26:01 -05004528 /* If the full s64 was not sign-extended from s32 then don't
4529 * deduct further info.
4530 */
4531 if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg))
4532 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004533 false_reg->smax_value = min(false_reg->smax_value, false_smax);
4534 true_reg->smin_value = max(true_reg->smin_value, true_smin);
Josef Bacik48461132016-09-28 10:54:32 -04004535 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004536 }
Daniel Borkmannb4e432f2017-08-10 01:40:02 +02004537 case BPF_JLE:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004538 case BPF_JLT:
4539 {
4540 u64 false_umin = opcode == BPF_JLT ? val : val + 1;
4541 u64 true_umax = opcode == BPF_JLT ? val - 1 : val;
4542
Jiong Wang092ed092019-01-26 12:26:01 -05004543 if (is_jmp32) {
4544 false_umin += gen_hi_min(false_reg->var_off);
4545 true_umax += gen_hi_max(true_reg->var_off);
4546 }
Jiong Wanga72dafa2019-01-26 12:26:00 -05004547 false_reg->umin_value = max(false_reg->umin_value, false_umin);
4548 true_reg->umax_value = min(true_reg->umax_value, true_umax);
Daniel Borkmannb4e432f2017-08-10 01:40:02 +02004549 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004550 }
Daniel Borkmannb4e432f2017-08-10 01:40:02 +02004551 case BPF_JSLE:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004552 case BPF_JSLT:
4553 {
4554 s64 false_smin = opcode == BPF_JSLT ? sval : sval + 1;
4555 s64 true_smax = opcode == BPF_JSLT ? sval - 1 : sval;
4556
Jiong Wang092ed092019-01-26 12:26:01 -05004557 if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg))
4558 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004559 false_reg->smin_value = max(false_reg->smin_value, false_smin);
4560 true_reg->smax_value = min(true_reg->smax_value, true_smax);
Daniel Borkmannb4e432f2017-08-10 01:40:02 +02004561 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004562 }
Josef Bacik48461132016-09-28 10:54:32 -04004563 default:
4564 break;
4565 }
4566
Edward Creeb03c9f92017-08-07 15:26:36 +01004567 __reg_deduce_bounds(false_reg);
4568 __reg_deduce_bounds(true_reg);
4569 /* We might have learned some bits from the bounds. */
4570 __reg_bound_offset(false_reg);
4571 __reg_bound_offset(true_reg);
4572 /* Intersecting with the old var_off might have improved our bounds
4573 * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc),
4574 * then new var_off is (0; 0x7f...fc) which improves our umax.
4575 */
4576 __update_reg_bounds(false_reg);
4577 __update_reg_bounds(true_reg);
Josef Bacik48461132016-09-28 10:54:32 -04004578}
4579
Edward Creef1174f72017-08-07 15:26:19 +01004580/* Same as above, but for the case that dst_reg holds a constant and src_reg is
4581 * the variable reg.
Josef Bacik48461132016-09-28 10:54:32 -04004582 */
4583static void reg_set_min_max_inv(struct bpf_reg_state *true_reg,
4584 struct bpf_reg_state *false_reg, u64 val,
Jiong Wang092ed092019-01-26 12:26:01 -05004585 u8 opcode, bool is_jmp32)
Josef Bacik48461132016-09-28 10:54:32 -04004586{
Jiong Wanga72dafa2019-01-26 12:26:00 -05004587 s64 sval;
4588
Edward Creef1174f72017-08-07 15:26:19 +01004589 if (__is_pointer_value(false, false_reg))
4590 return;
Daniel Borkmann4cabc5b2017-07-21 00:00:21 +02004591
Jiong Wang092ed092019-01-26 12:26:01 -05004592 val = is_jmp32 ? (u32)val : val;
4593 sval = is_jmp32 ? (s64)(s32)val : (s64)val;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004594
Josef Bacik48461132016-09-28 10:54:32 -04004595 switch (opcode) {
4596 case BPF_JEQ:
Josef Bacik48461132016-09-28 10:54:32 -04004597 case BPF_JNE:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004598 {
4599 struct bpf_reg_state *reg =
4600 opcode == BPF_JEQ ? true_reg : false_reg;
4601
Jiong Wang092ed092019-01-26 12:26:01 -05004602 if (is_jmp32) {
4603 u64 old_v = reg->var_off.value;
4604 u64 hi_mask = ~0xffffffffULL;
4605
4606 reg->var_off.value = (old_v & hi_mask) | val;
4607 reg->var_off.mask &= hi_mask;
4608 } else {
4609 __mark_reg_known(reg, val);
4610 }
Josef Bacik48461132016-09-28 10:54:32 -04004611 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004612 }
Jakub Kicinski960ea052018-12-19 22:13:04 -08004613 case BPF_JSET:
4614 false_reg->var_off = tnum_and(false_reg->var_off,
4615 tnum_const(~val));
4616 if (is_power_of_2(val))
4617 true_reg->var_off = tnum_or(true_reg->var_off,
4618 tnum_const(val));
4619 break;
Josef Bacik48461132016-09-28 10:54:32 -04004620 case BPF_JGE:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004621 case BPF_JGT:
4622 {
4623 u64 false_umin = opcode == BPF_JGT ? val : val + 1;
4624 u64 true_umax = opcode == BPF_JGT ? val - 1 : val;
4625
Jiong Wang092ed092019-01-26 12:26:01 -05004626 if (is_jmp32) {
4627 false_umin += gen_hi_min(false_reg->var_off);
4628 true_umax += gen_hi_max(true_reg->var_off);
4629 }
Jiong Wanga72dafa2019-01-26 12:26:00 -05004630 false_reg->umin_value = max(false_reg->umin_value, false_umin);
4631 true_reg->umax_value = min(true_reg->umax_value, true_umax);
Edward Creeb03c9f92017-08-07 15:26:36 +01004632 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004633 }
Josef Bacik48461132016-09-28 10:54:32 -04004634 case BPF_JSGE:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004635 case BPF_JSGT:
4636 {
4637 s64 false_smin = opcode == BPF_JSGT ? sval : sval + 1;
4638 s64 true_smax = opcode == BPF_JSGT ? sval - 1 : sval;
4639
Jiong Wang092ed092019-01-26 12:26:01 -05004640 if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg))
4641 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004642 false_reg->smin_value = max(false_reg->smin_value, false_smin);
4643 true_reg->smax_value = min(true_reg->smax_value, true_smax);
Josef Bacik48461132016-09-28 10:54:32 -04004644 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004645 }
Daniel Borkmannb4e432f2017-08-10 01:40:02 +02004646 case BPF_JLE:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004647 case BPF_JLT:
4648 {
4649 u64 false_umax = opcode == BPF_JLT ? val : val - 1;
4650 u64 true_umin = opcode == BPF_JLT ? val + 1 : val;
4651
Jiong Wang092ed092019-01-26 12:26:01 -05004652 if (is_jmp32) {
4653 false_umax += gen_hi_max(false_reg->var_off);
4654 true_umin += gen_hi_min(true_reg->var_off);
4655 }
Jiong Wanga72dafa2019-01-26 12:26:00 -05004656 false_reg->umax_value = min(false_reg->umax_value, false_umax);
4657 true_reg->umin_value = max(true_reg->umin_value, true_umin);
Daniel Borkmannb4e432f2017-08-10 01:40:02 +02004658 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004659 }
Daniel Borkmannb4e432f2017-08-10 01:40:02 +02004660 case BPF_JSLE:
Jiong Wanga72dafa2019-01-26 12:26:00 -05004661 case BPF_JSLT:
4662 {
4663 s64 false_smax = opcode == BPF_JSLT ? sval : sval - 1;
4664 s64 true_smin = opcode == BPF_JSLT ? sval + 1 : sval;
4665
Jiong Wang092ed092019-01-26 12:26:01 -05004666 if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg))
4667 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004668 false_reg->smax_value = min(false_reg->smax_value, false_smax);
4669 true_reg->smin_value = max(true_reg->smin_value, true_smin);
Daniel Borkmannb4e432f2017-08-10 01:40:02 +02004670 break;
Jiong Wanga72dafa2019-01-26 12:26:00 -05004671 }
Josef Bacik48461132016-09-28 10:54:32 -04004672 default:
4673 break;
4674 }
4675
Edward Creeb03c9f92017-08-07 15:26:36 +01004676 __reg_deduce_bounds(false_reg);
4677 __reg_deduce_bounds(true_reg);
4678 /* We might have learned some bits from the bounds. */
4679 __reg_bound_offset(false_reg);
4680 __reg_bound_offset(true_reg);
4681 /* Intersecting with the old var_off might have improved our bounds
4682 * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc),
4683 * then new var_off is (0; 0x7f...fc) which improves our umax.
4684 */
4685 __update_reg_bounds(false_reg);
4686 __update_reg_bounds(true_reg);
Edward Creef1174f72017-08-07 15:26:19 +01004687}
4688
4689/* Regs are known to be equal, so intersect their min/max/var_off */
4690static void __reg_combine_min_max(struct bpf_reg_state *src_reg,
4691 struct bpf_reg_state *dst_reg)
4692{
Edward Creeb03c9f92017-08-07 15:26:36 +01004693 src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value,
4694 dst_reg->umin_value);
4695 src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value,
4696 dst_reg->umax_value);
4697 src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value,
4698 dst_reg->smin_value);
4699 src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value,
4700 dst_reg->smax_value);
Edward Creef1174f72017-08-07 15:26:19 +01004701 src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off,
4702 dst_reg->var_off);
Edward Creeb03c9f92017-08-07 15:26:36 +01004703 /* We might have learned new bounds from the var_off. */
4704 __update_reg_bounds(src_reg);
4705 __update_reg_bounds(dst_reg);
4706 /* We might have learned something about the sign bit. */
4707 __reg_deduce_bounds(src_reg);
4708 __reg_deduce_bounds(dst_reg);
4709 /* We might have learned some bits from the bounds. */
4710 __reg_bound_offset(src_reg);
4711 __reg_bound_offset(dst_reg);
4712 /* Intersecting with the old var_off might have improved our bounds
4713 * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc),
4714 * then new var_off is (0; 0x7f...fc) which improves our umax.
4715 */
4716 __update_reg_bounds(src_reg);
4717 __update_reg_bounds(dst_reg);
Edward Creef1174f72017-08-07 15:26:19 +01004718}
4719
4720static void reg_combine_min_max(struct bpf_reg_state *true_src,
4721 struct bpf_reg_state *true_dst,
4722 struct bpf_reg_state *false_src,
4723 struct bpf_reg_state *false_dst,
4724 u8 opcode)
4725{
4726 switch (opcode) {
4727 case BPF_JEQ:
4728 __reg_combine_min_max(true_src, true_dst);
4729 break;
4730 case BPF_JNE:
4731 __reg_combine_min_max(false_src, false_dst);
Edward Creeb03c9f92017-08-07 15:26:36 +01004732 break;
Daniel Borkmann4cabc5b2017-07-21 00:00:21 +02004733 }
Josef Bacik48461132016-09-28 10:54:32 -04004734}
4735
Joe Stringerfd978bf72018-10-02 13:35:35 -07004736static void mark_ptr_or_null_reg(struct bpf_func_state *state,
4737 struct bpf_reg_state *reg, u32 id,
Joe Stringer840b9612018-10-02 13:35:32 -07004738 bool is_null)
Thomas Graf57a09bf2016-10-18 19:51:19 +02004739{
Joe Stringer840b9612018-10-02 13:35:32 -07004740 if (reg_type_may_be_null(reg->type) && reg->id == id) {
Edward Creef1174f72017-08-07 15:26:19 +01004741 /* Old offset (both fixed and variable parts) should
4742 * have been known-zero, because we don't allow pointer
4743 * arithmetic on pointers that might be NULL.
4744 */
Edward Creeb03c9f92017-08-07 15:26:36 +01004745 if (WARN_ON_ONCE(reg->smin_value || reg->smax_value ||
4746 !tnum_equals_const(reg->var_off, 0) ||
Edward Creef1174f72017-08-07 15:26:19 +01004747 reg->off)) {
Edward Creeb03c9f92017-08-07 15:26:36 +01004748 __mark_reg_known_zero(reg);
4749 reg->off = 0;
Edward Creef1174f72017-08-07 15:26:19 +01004750 }
4751 if (is_null) {
4752 reg->type = SCALAR_VALUE;
Joe Stringer840b9612018-10-02 13:35:32 -07004753 } else if (reg->type == PTR_TO_MAP_VALUE_OR_NULL) {
4754 if (reg->map_ptr->inner_map_meta) {
4755 reg->type = CONST_PTR_TO_MAP;
4756 reg->map_ptr = reg->map_ptr->inner_map_meta;
4757 } else {
4758 reg->type = PTR_TO_MAP_VALUE;
4759 }
Joe Stringerc64b7982018-10-02 13:35:33 -07004760 } else if (reg->type == PTR_TO_SOCKET_OR_NULL) {
4761 reg->type = PTR_TO_SOCKET;
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08004762 } else if (reg->type == PTR_TO_SOCK_COMMON_OR_NULL) {
4763 reg->type = PTR_TO_SOCK_COMMON;
Martin KaFai Lau655a51e2019-02-09 23:22:24 -08004764 } else if (reg->type == PTR_TO_TCP_SOCK_OR_NULL) {
4765 reg->type = PTR_TO_TCP_SOCK;
Martin KaFai Lau56f668d2017-03-22 10:00:33 -07004766 }
Martin KaFai Lau1b986582019-03-12 10:23:02 -07004767 if (is_null) {
4768 /* We don't need id and ref_obj_id from this point
4769 * onwards anymore, thus we should better reset it,
4770 * so that state pruning has chances to take effect.
4771 */
4772 reg->id = 0;
4773 reg->ref_obj_id = 0;
4774 } else if (!reg_may_point_to_spin_lock(reg)) {
4775 /* For not-NULL ptr, reg->ref_obj_id will be reset
4776 * in release_reg_references().
4777 *
4778 * reg->id is still used by spin_lock ptr. Other
4779 * than spin_lock ptr type, reg->id can be reset.
Joe Stringerfd978bf72018-10-02 13:35:35 -07004780 */
4781 reg->id = 0;
4782 }
Thomas Graf57a09bf2016-10-18 19:51:19 +02004783 }
4784}
4785
4786/* The logic is similar to find_good_pkt_pointers(), both could eventually
4787 * be folded together at some point.
4788 */
Joe Stringer840b9612018-10-02 13:35:32 -07004789static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno,
4790 bool is_null)
Thomas Graf57a09bf2016-10-18 19:51:19 +02004791{
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004792 struct bpf_func_state *state = vstate->frame[vstate->curframe];
Joe Stringerf3709f62018-10-02 13:35:29 -07004793 struct bpf_reg_state *reg, *regs = state->regs;
Martin KaFai Lau1b986582019-03-12 10:23:02 -07004794 u32 ref_obj_id = regs[regno].ref_obj_id;
Daniel Borkmanna08dd0d2016-12-15 01:30:06 +01004795 u32 id = regs[regno].id;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004796 int i, j;
Thomas Graf57a09bf2016-10-18 19:51:19 +02004797
Martin KaFai Lau1b986582019-03-12 10:23:02 -07004798 if (ref_obj_id && ref_obj_id == id && is_null)
4799 /* regs[regno] is in the " == NULL" branch.
4800 * No one could have freed the reference state before
4801 * doing the NULL check.
4802 */
4803 WARN_ON_ONCE(release_reference_state(state, id));
Joe Stringerfd978bf72018-10-02 13:35:35 -07004804
Thomas Graf57a09bf2016-10-18 19:51:19 +02004805 for (i = 0; i < MAX_BPF_REG; i++)
Joe Stringerfd978bf72018-10-02 13:35:35 -07004806 mark_ptr_or_null_reg(state, &regs[i], id, is_null);
Thomas Graf57a09bf2016-10-18 19:51:19 +02004807
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004808 for (j = 0; j <= vstate->curframe; j++) {
4809 state = vstate->frame[j];
Joe Stringerf3709f62018-10-02 13:35:29 -07004810 bpf_for_each_spilled_reg(i, state, reg) {
4811 if (!reg)
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004812 continue;
Joe Stringerfd978bf72018-10-02 13:35:35 -07004813 mark_ptr_or_null_reg(state, reg, id, is_null);
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004814 }
Thomas Graf57a09bf2016-10-18 19:51:19 +02004815 }
4816}
4817
Daniel Borkmann5beca082017-11-01 23:58:10 +01004818static bool try_match_pkt_pointers(const struct bpf_insn *insn,
4819 struct bpf_reg_state *dst_reg,
4820 struct bpf_reg_state *src_reg,
4821 struct bpf_verifier_state *this_branch,
4822 struct bpf_verifier_state *other_branch)
4823{
4824 if (BPF_SRC(insn->code) != BPF_X)
4825 return false;
4826
Jiong Wang092ed092019-01-26 12:26:01 -05004827 /* Pointers are always 64-bit. */
4828 if (BPF_CLASS(insn->code) == BPF_JMP32)
4829 return false;
4830
Daniel Borkmann5beca082017-11-01 23:58:10 +01004831 switch (BPF_OP(insn->code)) {
4832 case BPF_JGT:
4833 if ((dst_reg->type == PTR_TO_PACKET &&
4834 src_reg->type == PTR_TO_PACKET_END) ||
4835 (dst_reg->type == PTR_TO_PACKET_META &&
4836 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
4837 /* pkt_data' > pkt_end, pkt_meta' > pkt_data */
4838 find_good_pkt_pointers(this_branch, dst_reg,
4839 dst_reg->type, false);
4840 } else if ((dst_reg->type == PTR_TO_PACKET_END &&
4841 src_reg->type == PTR_TO_PACKET) ||
4842 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
4843 src_reg->type == PTR_TO_PACKET_META)) {
4844 /* pkt_end > pkt_data', pkt_data > pkt_meta' */
4845 find_good_pkt_pointers(other_branch, src_reg,
4846 src_reg->type, true);
4847 } else {
4848 return false;
4849 }
4850 break;
4851 case BPF_JLT:
4852 if ((dst_reg->type == PTR_TO_PACKET &&
4853 src_reg->type == PTR_TO_PACKET_END) ||
4854 (dst_reg->type == PTR_TO_PACKET_META &&
4855 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
4856 /* pkt_data' < pkt_end, pkt_meta' < pkt_data */
4857 find_good_pkt_pointers(other_branch, dst_reg,
4858 dst_reg->type, true);
4859 } else if ((dst_reg->type == PTR_TO_PACKET_END &&
4860 src_reg->type == PTR_TO_PACKET) ||
4861 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
4862 src_reg->type == PTR_TO_PACKET_META)) {
4863 /* pkt_end < pkt_data', pkt_data > pkt_meta' */
4864 find_good_pkt_pointers(this_branch, src_reg,
4865 src_reg->type, false);
4866 } else {
4867 return false;
4868 }
4869 break;
4870 case BPF_JGE:
4871 if ((dst_reg->type == PTR_TO_PACKET &&
4872 src_reg->type == PTR_TO_PACKET_END) ||
4873 (dst_reg->type == PTR_TO_PACKET_META &&
4874 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
4875 /* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */
4876 find_good_pkt_pointers(this_branch, dst_reg,
4877 dst_reg->type, true);
4878 } else if ((dst_reg->type == PTR_TO_PACKET_END &&
4879 src_reg->type == PTR_TO_PACKET) ||
4880 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
4881 src_reg->type == PTR_TO_PACKET_META)) {
4882 /* pkt_end >= pkt_data', pkt_data >= pkt_meta' */
4883 find_good_pkt_pointers(other_branch, src_reg,
4884 src_reg->type, false);
4885 } else {
4886 return false;
4887 }
4888 break;
4889 case BPF_JLE:
4890 if ((dst_reg->type == PTR_TO_PACKET &&
4891 src_reg->type == PTR_TO_PACKET_END) ||
4892 (dst_reg->type == PTR_TO_PACKET_META &&
4893 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
4894 /* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */
4895 find_good_pkt_pointers(other_branch, dst_reg,
4896 dst_reg->type, false);
4897 } else if ((dst_reg->type == PTR_TO_PACKET_END &&
4898 src_reg->type == PTR_TO_PACKET) ||
4899 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
4900 src_reg->type == PTR_TO_PACKET_META)) {
4901 /* pkt_end <= pkt_data', pkt_data <= pkt_meta' */
4902 find_good_pkt_pointers(this_branch, src_reg,
4903 src_reg->type, true);
4904 } else {
4905 return false;
4906 }
4907 break;
4908 default:
4909 return false;
4910 }
4911
4912 return true;
4913}
4914
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01004915static int check_cond_jmp_op(struct bpf_verifier_env *env,
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004916 struct bpf_insn *insn, int *insn_idx)
4917{
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004918 struct bpf_verifier_state *this_branch = env->cur_state;
4919 struct bpf_verifier_state *other_branch;
4920 struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs;
4921 struct bpf_reg_state *dst_reg, *other_branch_regs;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004922 u8 opcode = BPF_OP(insn->code);
Jiong Wang092ed092019-01-26 12:26:01 -05004923 bool is_jmp32;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004924 int err;
4925
Jiong Wang092ed092019-01-26 12:26:01 -05004926 /* Only conditional jumps are expected to reach here. */
4927 if (opcode == BPF_JA || opcode > BPF_JSLE) {
4928 verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004929 return -EINVAL;
4930 }
4931
4932 if (BPF_SRC(insn->code) == BPF_X) {
4933 if (insn->imm != 0) {
Jiong Wang092ed092019-01-26 12:26:01 -05004934 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004935 return -EINVAL;
4936 }
4937
4938 /* check src1 operand */
Edward Creedc503a82017-08-15 20:34:35 +01004939 err = check_reg_arg(env, insn->src_reg, SRC_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004940 if (err)
4941 return err;
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07004942
4943 if (is_pointer_value(env, insn->src_reg)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07004944 verbose(env, "R%d pointer comparison prohibited\n",
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07004945 insn->src_reg);
4946 return -EACCES;
4947 }
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004948 } else {
4949 if (insn->src_reg != BPF_REG_0) {
Jiong Wang092ed092019-01-26 12:26:01 -05004950 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004951 return -EINVAL;
4952 }
4953 }
4954
4955 /* check src2 operand */
Edward Creedc503a82017-08-15 20:34:35 +01004956 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004957 if (err)
4958 return err;
4959
Alexei Starovoitov1a0dc1a2016-05-05 19:49:09 -07004960 dst_reg = &regs[insn->dst_reg];
Jiong Wang092ed092019-01-26 12:26:01 -05004961 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
Alexei Starovoitov1a0dc1a2016-05-05 19:49:09 -07004962
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004963 if (BPF_SRC(insn->code) == BPF_K) {
Jiong Wang092ed092019-01-26 12:26:01 -05004964 int pred = is_branch_taken(dst_reg, insn->imm, opcode,
4965 is_jmp32);
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004966
4967 if (pred == 1) {
4968 /* only follow the goto, ignore fall-through */
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004969 *insn_idx += insn->off;
4970 return 0;
Alexei Starovoitov4f7b3e82018-12-03 22:46:05 -08004971 } else if (pred == 0) {
4972 /* only follow fall-through branch, since
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004973 * that's where the program will go
4974 */
4975 return 0;
4976 }
4977 }
4978
Daniel Borkmann979d63d2019-01-03 00:58:34 +01004979 other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx,
4980 false);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004981 if (!other_branch)
4982 return -EFAULT;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08004983 other_branch_regs = other_branch->frame[other_branch->curframe]->regs;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07004984
Josef Bacik48461132016-09-28 10:54:32 -04004985 /* detect if we are comparing against a constant value so we can adjust
4986 * our min/max values for our dst register.
Edward Creef1174f72017-08-07 15:26:19 +01004987 * this is only legit if both are scalars (or pointers to the same
4988 * object, I suppose, but we don't support that right now), because
4989 * otherwise the different base pointers mean the offsets aren't
4990 * comparable.
Josef Bacik48461132016-09-28 10:54:32 -04004991 */
4992 if (BPF_SRC(insn->code) == BPF_X) {
Jiong Wang092ed092019-01-26 12:26:01 -05004993 struct bpf_reg_state *src_reg = &regs[insn->src_reg];
4994 struct bpf_reg_state lo_reg0 = *dst_reg;
4995 struct bpf_reg_state lo_reg1 = *src_reg;
4996 struct bpf_reg_state *src_lo, *dst_lo;
4997
4998 dst_lo = &lo_reg0;
4999 src_lo = &lo_reg1;
5000 coerce_reg_to_size(dst_lo, 4);
5001 coerce_reg_to_size(src_lo, 4);
5002
Edward Creef1174f72017-08-07 15:26:19 +01005003 if (dst_reg->type == SCALAR_VALUE &&
Jiong Wang092ed092019-01-26 12:26:01 -05005004 src_reg->type == SCALAR_VALUE) {
5005 if (tnum_is_const(src_reg->var_off) ||
5006 (is_jmp32 && tnum_is_const(src_lo->var_off)))
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08005007 reg_set_min_max(&other_branch_regs[insn->dst_reg],
Jiong Wang092ed092019-01-26 12:26:01 -05005008 dst_reg,
5009 is_jmp32
5010 ? src_lo->var_off.value
5011 : src_reg->var_off.value,
5012 opcode, is_jmp32);
5013 else if (tnum_is_const(dst_reg->var_off) ||
5014 (is_jmp32 && tnum_is_const(dst_lo->var_off)))
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08005015 reg_set_min_max_inv(&other_branch_regs[insn->src_reg],
Jiong Wang092ed092019-01-26 12:26:01 -05005016 src_reg,
5017 is_jmp32
5018 ? dst_lo->var_off.value
5019 : dst_reg->var_off.value,
5020 opcode, is_jmp32);
5021 else if (!is_jmp32 &&
5022 (opcode == BPF_JEQ || opcode == BPF_JNE))
Edward Creef1174f72017-08-07 15:26:19 +01005023 /* Comparing for equality, we can combine knowledge */
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08005024 reg_combine_min_max(&other_branch_regs[insn->src_reg],
5025 &other_branch_regs[insn->dst_reg],
Jiong Wang092ed092019-01-26 12:26:01 -05005026 src_reg, dst_reg, opcode);
Edward Creef1174f72017-08-07 15:26:19 +01005027 }
5028 } else if (dst_reg->type == SCALAR_VALUE) {
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08005029 reg_set_min_max(&other_branch_regs[insn->dst_reg],
Jiong Wang092ed092019-01-26 12:26:01 -05005030 dst_reg, insn->imm, opcode, is_jmp32);
Josef Bacik48461132016-09-28 10:54:32 -04005031 }
5032
Jiong Wang092ed092019-01-26 12:26:01 -05005033 /* detect if R == 0 where R is returned from bpf_map_lookup_elem().
5034 * NOTE: these optimizations below are related with pointer comparison
5035 * which will never be JMP32.
5036 */
5037 if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K &&
Alexei Starovoitov1a0dc1a2016-05-05 19:49:09 -07005038 insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) &&
Joe Stringer840b9612018-10-02 13:35:32 -07005039 reg_type_may_be_null(dst_reg->type)) {
5040 /* Mark all identical registers in each branch as either
Thomas Graf57a09bf2016-10-18 19:51:19 +02005041 * safe or unknown depending R == 0 or R != 0 conditional.
5042 */
Joe Stringer840b9612018-10-02 13:35:32 -07005043 mark_ptr_or_null_regs(this_branch, insn->dst_reg,
5044 opcode == BPF_JNE);
5045 mark_ptr_or_null_regs(other_branch, insn->dst_reg,
5046 opcode == BPF_JEQ);
Daniel Borkmann5beca082017-11-01 23:58:10 +01005047 } else if (!try_match_pkt_pointers(insn, dst_reg, &regs[insn->src_reg],
5048 this_branch, other_branch) &&
5049 is_pointer_value(env, insn->dst_reg)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005050 verbose(env, "R%d pointer comparison prohibited\n",
5051 insn->dst_reg);
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07005052 return -EACCES;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07005053 }
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07005054 if (env->log.level & BPF_LOG_LEVEL)
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08005055 print_verifier_state(env, this_branch->frame[this_branch->curframe]);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07005056 return 0;
5057}
5058
Alexei Starovoitov0246e642014-09-26 00:17:04 -07005059/* return the map pointer stored inside BPF_LD_IMM64 instruction */
5060static struct bpf_map *ld_imm64_to_map_ptr(struct bpf_insn *insn)
5061{
5062 u64 imm64 = ((u64) (u32) insn[0].imm) | ((u64) (u32) insn[1].imm) << 32;
5063
5064 return (struct bpf_map *) (unsigned long) imm64;
5065}
5066
Alexei Starovoitov17a52672014-09-26 00:17:06 -07005067/* verify BPF_LD_IMM64 instruction */
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01005068static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)
Alexei Starovoitov17a52672014-09-26 00:17:06 -07005069{
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07005070 struct bpf_reg_state *regs = cur_regs(env);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07005071 int err;
5072
5073 if (BPF_SIZE(insn->code) != BPF_DW) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005074 verbose(env, "invalid BPF_LD_IMM insn\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07005075 return -EINVAL;
5076 }
5077 if (insn->off != 0) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005078 verbose(env, "BPF_LD_IMM64 uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07005079 return -EINVAL;
5080 }
5081
Edward Creedc503a82017-08-15 20:34:35 +01005082 err = check_reg_arg(env, insn->dst_reg, DST_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07005083 if (err)
5084 return err;
5085
Jakub Kicinski6b173872016-09-21 11:43:59 +01005086 if (insn->src_reg == 0) {
Jakub Kicinski6b173872016-09-21 11:43:59 +01005087 u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm;
5088
Edward Creef1174f72017-08-07 15:26:19 +01005089 regs[insn->dst_reg].type = SCALAR_VALUE;
Edward Creeb03c9f92017-08-07 15:26:36 +01005090 __mark_reg_known(&regs[insn->dst_reg], imm);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07005091 return 0;
Jakub Kicinski6b173872016-09-21 11:43:59 +01005092 }
Alexei Starovoitov17a52672014-09-26 00:17:06 -07005093
5094 /* replace_map_fd_with_map_ptr() should have caught bad ld_imm64 */
5095 BUG_ON(insn->src_reg != BPF_PSEUDO_MAP_FD);
5096
5097 regs[insn->dst_reg].type = CONST_PTR_TO_MAP;
5098 regs[insn->dst_reg].map_ptr = ld_imm64_to_map_ptr(insn);
5099 return 0;
5100}
5101
Daniel Borkmann96be4322015-03-01 12:31:46 +01005102static bool may_access_skb(enum bpf_prog_type type)
5103{
5104 switch (type) {
5105 case BPF_PROG_TYPE_SOCKET_FILTER:
5106 case BPF_PROG_TYPE_SCHED_CLS:
Daniel Borkmann94caee8c2015-03-20 15:11:11 +01005107 case BPF_PROG_TYPE_SCHED_ACT:
Daniel Borkmann96be4322015-03-01 12:31:46 +01005108 return true;
5109 default:
5110 return false;
5111 }
5112}
5113
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005114/* verify safety of LD_ABS|LD_IND instructions:
5115 * - they can only appear in the programs where ctx == skb
5116 * - since they are wrappers of function calls, they scratch R1-R5 registers,
5117 * preserve R6-R9, and store return value into R0
5118 *
5119 * Implicit input:
5120 * ctx == skb == R6 == CTX
5121 *
5122 * Explicit input:
5123 * SRC == any register
5124 * IMM == 32-bit immediate
5125 *
5126 * Output:
5127 * R0 - 8/16/32-bit skb data converted to cpu endianness
5128 */
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01005129static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005130{
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07005131 struct bpf_reg_state *regs = cur_regs(env);
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005132 u8 mode = BPF_MODE(insn->code);
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005133 int i, err;
5134
Daniel Borkmann24701ec2015-03-01 12:31:47 +01005135 if (!may_access_skb(env->prog->type)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005136 verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n");
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005137 return -EINVAL;
5138 }
5139
Daniel Borkmanne0cea7c2018-05-04 01:08:14 +02005140 if (!env->ops->gen_ld_abs) {
5141 verbose(env, "bpf verifier is misconfigured\n");
5142 return -EINVAL;
5143 }
5144
Jiong Wangf910cef2018-05-02 16:17:17 -04005145 if (env->subprog_cnt > 1) {
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08005146 /* when program has LD_ABS insn JITs and interpreter assume
5147 * that r1 == ctx == skb which is not the case for callees
5148 * that can have arbitrary arguments. It's problematic
5149 * for main prog as well since JITs would need to analyze
5150 * all functions in order to make proper register save/restore
5151 * decisions in the main prog. Hence disallow LD_ABS with calls
5152 */
5153 verbose(env, "BPF_LD_[ABS|IND] instructions cannot be mixed with bpf-to-bpf calls\n");
5154 return -EINVAL;
5155 }
5156
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005157 if (insn->dst_reg != BPF_REG_0 || insn->off != 0 ||
Alexei Starovoitovd82bccc2016-04-12 10:26:19 -07005158 BPF_SIZE(insn->code) == BPF_DW ||
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005159 (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005160 verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n");
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005161 return -EINVAL;
5162 }
5163
5164 /* check whether implicit source operand (register R6) is readable */
Edward Creedc503a82017-08-15 20:34:35 +01005165 err = check_reg_arg(env, BPF_REG_6, SRC_OP);
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005166 if (err)
5167 return err;
5168
Joe Stringerfd978bf72018-10-02 13:35:35 -07005169 /* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as
5170 * gen_ld_abs() may terminate the program at runtime, leading to
5171 * reference leak.
5172 */
5173 err = check_reference_leak(env);
5174 if (err) {
5175 verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n");
5176 return err;
5177 }
5178
Alexei Starovoitovd83525c2019-01-31 15:40:04 -08005179 if (env->cur_state->active_spin_lock) {
5180 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n");
5181 return -EINVAL;
5182 }
5183
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005184 if (regs[BPF_REG_6].type != PTR_TO_CTX) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005185 verbose(env,
5186 "at the time of BPF_LD_ABS|IND R6 != pointer to skb\n");
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005187 return -EINVAL;
5188 }
5189
5190 if (mode == BPF_IND) {
5191 /* check explicit source operand */
Edward Creedc503a82017-08-15 20:34:35 +01005192 err = check_reg_arg(env, insn->src_reg, SRC_OP);
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005193 if (err)
5194 return err;
5195 }
5196
5197 /* reset caller saved regs to unreadable */
Edward Creedc503a82017-08-15 20:34:35 +01005198 for (i = 0; i < CALLER_SAVED_REGS; i++) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005199 mark_reg_not_init(env, regs, caller_saved[i]);
Edward Creedc503a82017-08-15 20:34:35 +01005200 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
5201 }
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005202
5203 /* mark destination R0 register as readable, since it contains
Edward Creedc503a82017-08-15 20:34:35 +01005204 * the value fetched from the packet.
5205 * Already marked as written above.
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005206 */
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005207 mark_reg_unknown(env, regs, BPF_REG_0);
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08005208 return 0;
5209}
5210
Alexei Starovoitov390ee7e2017-10-02 22:50:23 -07005211static int check_return_code(struct bpf_verifier_env *env)
5212{
5213 struct bpf_reg_state *reg;
5214 struct tnum range = tnum_range(0, 1);
5215
5216 switch (env->prog->type) {
5217 case BPF_PROG_TYPE_CGROUP_SKB:
5218 case BPF_PROG_TYPE_CGROUP_SOCK:
Andrey Ignatov4fbac772018-03-30 15:08:02 -07005219 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
Alexei Starovoitov390ee7e2017-10-02 22:50:23 -07005220 case BPF_PROG_TYPE_SOCK_OPS:
Roman Gushchinebc614f2017-11-05 08:15:32 -05005221 case BPF_PROG_TYPE_CGROUP_DEVICE:
Alexei Starovoitov390ee7e2017-10-02 22:50:23 -07005222 break;
5223 default:
5224 return 0;
5225 }
5226
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07005227 reg = cur_regs(env) + BPF_REG_0;
Alexei Starovoitov390ee7e2017-10-02 22:50:23 -07005228 if (reg->type != SCALAR_VALUE) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005229 verbose(env, "At program exit the register R0 is not a known value (%s)\n",
Alexei Starovoitov390ee7e2017-10-02 22:50:23 -07005230 reg_type_str[reg->type]);
5231 return -EINVAL;
5232 }
5233
5234 if (!tnum_in(range, reg->var_off)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005235 verbose(env, "At program exit the register R0 ");
Alexei Starovoitov390ee7e2017-10-02 22:50:23 -07005236 if (!tnum_is_unknown(reg->var_off)) {
5237 char tn_buf[48];
5238
5239 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005240 verbose(env, "has value %s", tn_buf);
Alexei Starovoitov390ee7e2017-10-02 22:50:23 -07005241 } else {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005242 verbose(env, "has unknown scalar value");
Alexei Starovoitov390ee7e2017-10-02 22:50:23 -07005243 }
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005244 verbose(env, " should have been 0 or 1\n");
Alexei Starovoitov390ee7e2017-10-02 22:50:23 -07005245 return -EINVAL;
5246 }
5247 return 0;
5248}
5249
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005250/* non-recursive DFS pseudo code
5251 * 1 procedure DFS-iterative(G,v):
5252 * 2 label v as discovered
5253 * 3 let S be a stack
5254 * 4 S.push(v)
5255 * 5 while S is not empty
5256 * 6 t <- S.pop()
5257 * 7 if t is what we're looking for:
5258 * 8 return t
5259 * 9 for all edges e in G.adjacentEdges(t) do
5260 * 10 if edge e is already labelled
5261 * 11 continue with the next edge
5262 * 12 w <- G.adjacentVertex(t,e)
5263 * 13 if vertex w is not discovered and not explored
5264 * 14 label e as tree-edge
5265 * 15 label w as discovered
5266 * 16 S.push(w)
5267 * 17 continue at 5
5268 * 18 else if vertex w is discovered
5269 * 19 label e as back-edge
5270 * 20 else
5271 * 21 // vertex w is explored
5272 * 22 label e as forward- or cross-edge
5273 * 23 label t as explored
5274 * 24 S.pop()
5275 *
5276 * convention:
5277 * 0x10 - discovered
5278 * 0x11 - discovered and fall-through edge labelled
5279 * 0x12 - discovered and fall-through and branch edges labelled
5280 * 0x20 - explored
5281 */
5282
5283enum {
5284 DISCOVERED = 0x10,
5285 EXPLORED = 0x20,
5286 FALLTHROUGH = 1,
5287 BRANCH = 2,
5288};
5289
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01005290#define STATE_LIST_MARK ((struct bpf_verifier_state_list *) -1L)
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07005291
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005292static int *insn_stack; /* stack of insns to process */
5293static int cur_stack; /* current stack index */
5294static int *insn_state;
5295
5296/* t, w, e - match pseudo-code above:
5297 * t - index of current instruction
5298 * w - next instruction
5299 * e - edge
5300 */
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01005301static int push_insn(int t, int w, int e, struct bpf_verifier_env *env)
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005302{
5303 if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH))
5304 return 0;
5305
5306 if (e == BRANCH && insn_state[t] >= (DISCOVERED | BRANCH))
5307 return 0;
5308
5309 if (w < 0 || w >= env->prog->len) {
Martin KaFai Laud9762e82018-12-13 10:41:48 -08005310 verbose_linfo(env, t, "%d: ", t);
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005311 verbose(env, "jump out of range from insn %d to %d\n", t, w);
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005312 return -EINVAL;
5313 }
5314
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07005315 if (e == BRANCH)
5316 /* mark branch target for state pruning */
5317 env->explored_states[w] = STATE_LIST_MARK;
5318
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005319 if (insn_state[w] == 0) {
5320 /* tree-edge */
5321 insn_state[t] = DISCOVERED | e;
5322 insn_state[w] = DISCOVERED;
5323 if (cur_stack >= env->prog->len)
5324 return -E2BIG;
5325 insn_stack[cur_stack++] = w;
5326 return 1;
5327 } else if ((insn_state[w] & 0xF0) == DISCOVERED) {
Martin KaFai Laud9762e82018-12-13 10:41:48 -08005328 verbose_linfo(env, t, "%d: ", t);
5329 verbose_linfo(env, w, "%d: ", w);
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005330 verbose(env, "back-edge from insn %d to %d\n", t, w);
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005331 return -EINVAL;
5332 } else if (insn_state[w] == EXPLORED) {
5333 /* forward- or cross-edge */
5334 insn_state[t] = DISCOVERED | e;
5335 } else {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005336 verbose(env, "insn state internal bug\n");
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005337 return -EFAULT;
5338 }
5339 return 0;
5340}
5341
5342/* non-recursive depth-first-search to detect loops in BPF program
5343 * loop == back-edge in directed graph
5344 */
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01005345static int check_cfg(struct bpf_verifier_env *env)
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005346{
5347 struct bpf_insn *insns = env->prog->insnsi;
5348 int insn_cnt = env->prog->len;
5349 int ret = 0;
5350 int i, t;
5351
Alexei Starovoitov71dde682019-04-01 21:27:43 -07005352 insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005353 if (!insn_state)
5354 return -ENOMEM;
5355
Alexei Starovoitov71dde682019-04-01 21:27:43 -07005356 insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005357 if (!insn_stack) {
Alexei Starovoitov71dde682019-04-01 21:27:43 -07005358 kvfree(insn_state);
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005359 return -ENOMEM;
5360 }
5361
5362 insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */
5363 insn_stack[0] = 0; /* 0 is the first instruction */
5364 cur_stack = 1;
5365
5366peek_stack:
5367 if (cur_stack == 0)
5368 goto check_state;
5369 t = insn_stack[cur_stack - 1];
5370
Jiong Wang092ed092019-01-26 12:26:01 -05005371 if (BPF_CLASS(insns[t].code) == BPF_JMP ||
5372 BPF_CLASS(insns[t].code) == BPF_JMP32) {
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005373 u8 opcode = BPF_OP(insns[t].code);
5374
5375 if (opcode == BPF_EXIT) {
5376 goto mark_explored;
5377 } else if (opcode == BPF_CALL) {
5378 ret = push_insn(t, t + 1, FALLTHROUGH, env);
5379 if (ret == 1)
5380 goto peek_stack;
5381 else if (ret < 0)
5382 goto err_free;
Daniel Borkmann07016152016-04-05 22:33:17 +02005383 if (t + 1 < insn_cnt)
5384 env->explored_states[t + 1] = STATE_LIST_MARK;
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08005385 if (insns[t].src_reg == BPF_PSEUDO_CALL) {
5386 env->explored_states[t] = STATE_LIST_MARK;
5387 ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env);
5388 if (ret == 1)
5389 goto peek_stack;
5390 else if (ret < 0)
5391 goto err_free;
5392 }
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005393 } else if (opcode == BPF_JA) {
5394 if (BPF_SRC(insns[t].code) != BPF_K) {
5395 ret = -EINVAL;
5396 goto err_free;
5397 }
5398 /* unconditional jump with single edge */
5399 ret = push_insn(t, t + insns[t].off + 1,
5400 FALLTHROUGH, env);
5401 if (ret == 1)
5402 goto peek_stack;
5403 else if (ret < 0)
5404 goto err_free;
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07005405 /* tell verifier to check for equivalent states
5406 * after every call and jump
5407 */
Alexei Starovoitovc3de6312015-04-14 15:57:13 -07005408 if (t + 1 < insn_cnt)
5409 env->explored_states[t + 1] = STATE_LIST_MARK;
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005410 } else {
5411 /* conditional jump with two edges */
Daniel Borkmann3c2ce602017-05-18 03:00:06 +02005412 env->explored_states[t] = STATE_LIST_MARK;
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005413 ret = push_insn(t, t + 1, FALLTHROUGH, env);
5414 if (ret == 1)
5415 goto peek_stack;
5416 else if (ret < 0)
5417 goto err_free;
5418
5419 ret = push_insn(t, t + insns[t].off + 1, BRANCH, env);
5420 if (ret == 1)
5421 goto peek_stack;
5422 else if (ret < 0)
5423 goto err_free;
5424 }
5425 } else {
5426 /* all other non-branch instructions with single
5427 * fall-through edge
5428 */
5429 ret = push_insn(t, t + 1, FALLTHROUGH, env);
5430 if (ret == 1)
5431 goto peek_stack;
5432 else if (ret < 0)
5433 goto err_free;
5434 }
5435
5436mark_explored:
5437 insn_state[t] = EXPLORED;
5438 if (cur_stack-- <= 0) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005439 verbose(env, "pop stack internal bug\n");
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005440 ret = -EFAULT;
5441 goto err_free;
5442 }
5443 goto peek_stack;
5444
5445check_state:
5446 for (i = 0; i < insn_cnt; i++) {
5447 if (insn_state[i] != EXPLORED) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07005448 verbose(env, "unreachable insn %d\n", i);
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005449 ret = -EINVAL;
5450 goto err_free;
5451 }
5452 }
5453 ret = 0; /* cfg looks good */
5454
5455err_free:
Alexei Starovoitov71dde682019-04-01 21:27:43 -07005456 kvfree(insn_state);
5457 kvfree(insn_stack);
Alexei Starovoitov475fb782014-09-26 00:17:05 -07005458 return ret;
5459}
5460
Yonghong Song838e9692018-11-19 15:29:11 -08005461/* The minimum supported BTF func info size */
5462#define MIN_BPF_FUNCINFO_SIZE 8
5463#define MAX_FUNCINFO_REC_SIZE 252
5464
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005465static int check_btf_func(struct bpf_verifier_env *env,
5466 const union bpf_attr *attr,
5467 union bpf_attr __user *uattr)
Yonghong Song838e9692018-11-19 15:29:11 -08005468{
Peter Oskolkovd0b28182019-01-16 10:43:01 -08005469 u32 i, nfuncs, urec_size, min_size;
Yonghong Song838e9692018-11-19 15:29:11 -08005470 u32 krec_size = sizeof(struct bpf_func_info);
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005471 struct bpf_func_info *krecord;
Yonghong Song838e9692018-11-19 15:29:11 -08005472 const struct btf_type *type;
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005473 struct bpf_prog *prog;
5474 const struct btf *btf;
Yonghong Song838e9692018-11-19 15:29:11 -08005475 void __user *urecord;
Peter Oskolkovd0b28182019-01-16 10:43:01 -08005476 u32 prev_offset = 0;
Yonghong Song838e9692018-11-19 15:29:11 -08005477 int ret = 0;
5478
5479 nfuncs = attr->func_info_cnt;
5480 if (!nfuncs)
5481 return 0;
5482
5483 if (nfuncs != env->subprog_cnt) {
5484 verbose(env, "number of funcs in func_info doesn't match number of subprogs\n");
5485 return -EINVAL;
5486 }
5487
5488 urec_size = attr->func_info_rec_size;
5489 if (urec_size < MIN_BPF_FUNCINFO_SIZE ||
5490 urec_size > MAX_FUNCINFO_REC_SIZE ||
5491 urec_size % sizeof(u32)) {
5492 verbose(env, "invalid func info rec size %u\n", urec_size);
5493 return -EINVAL;
5494 }
5495
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005496 prog = env->prog;
5497 btf = prog->aux->btf;
Yonghong Song838e9692018-11-19 15:29:11 -08005498
5499 urecord = u64_to_user_ptr(attr->func_info);
5500 min_size = min_t(u32, krec_size, urec_size);
5501
Yonghong Songba64e7d2018-11-24 23:20:44 -08005502 krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN);
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005503 if (!krecord)
5504 return -ENOMEM;
Yonghong Songba64e7d2018-11-24 23:20:44 -08005505
Yonghong Song838e9692018-11-19 15:29:11 -08005506 for (i = 0; i < nfuncs; i++) {
5507 ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size);
5508 if (ret) {
5509 if (ret == -E2BIG) {
5510 verbose(env, "nonzero tailing record in func info");
5511 /* set the size kernel expects so loader can zero
5512 * out the rest of the record.
5513 */
5514 if (put_user(min_size, &uattr->func_info_rec_size))
5515 ret = -EFAULT;
5516 }
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005517 goto err_free;
Yonghong Song838e9692018-11-19 15:29:11 -08005518 }
5519
Yonghong Songba64e7d2018-11-24 23:20:44 -08005520 if (copy_from_user(&krecord[i], urecord, min_size)) {
Yonghong Song838e9692018-11-19 15:29:11 -08005521 ret = -EFAULT;
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005522 goto err_free;
Yonghong Song838e9692018-11-19 15:29:11 -08005523 }
5524
Martin KaFai Laud30d42e2018-12-05 17:35:44 -08005525 /* check insn_off */
Yonghong Song838e9692018-11-19 15:29:11 -08005526 if (i == 0) {
Martin KaFai Laud30d42e2018-12-05 17:35:44 -08005527 if (krecord[i].insn_off) {
Yonghong Song838e9692018-11-19 15:29:11 -08005528 verbose(env,
Martin KaFai Laud30d42e2018-12-05 17:35:44 -08005529 "nonzero insn_off %u for the first func info record",
5530 krecord[i].insn_off);
Yonghong Song838e9692018-11-19 15:29:11 -08005531 ret = -EINVAL;
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005532 goto err_free;
Yonghong Song838e9692018-11-19 15:29:11 -08005533 }
Martin KaFai Laud30d42e2018-12-05 17:35:44 -08005534 } else if (krecord[i].insn_off <= prev_offset) {
Yonghong Song838e9692018-11-19 15:29:11 -08005535 verbose(env,
5536 "same or smaller insn offset (%u) than previous func info record (%u)",
Martin KaFai Laud30d42e2018-12-05 17:35:44 -08005537 krecord[i].insn_off, prev_offset);
Yonghong Song838e9692018-11-19 15:29:11 -08005538 ret = -EINVAL;
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005539 goto err_free;
Yonghong Song838e9692018-11-19 15:29:11 -08005540 }
5541
Martin KaFai Laud30d42e2018-12-05 17:35:44 -08005542 if (env->subprog_info[i].start != krecord[i].insn_off) {
Yonghong Song838e9692018-11-19 15:29:11 -08005543 verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");
5544 ret = -EINVAL;
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005545 goto err_free;
Yonghong Song838e9692018-11-19 15:29:11 -08005546 }
5547
5548 /* check type_id */
Yonghong Songba64e7d2018-11-24 23:20:44 -08005549 type = btf_type_by_id(btf, krecord[i].type_id);
Yonghong Song838e9692018-11-19 15:29:11 -08005550 if (!type || BTF_INFO_KIND(type->info) != BTF_KIND_FUNC) {
5551 verbose(env, "invalid type id %d in func info",
Yonghong Songba64e7d2018-11-24 23:20:44 -08005552 krecord[i].type_id);
Yonghong Song838e9692018-11-19 15:29:11 -08005553 ret = -EINVAL;
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005554 goto err_free;
Yonghong Song838e9692018-11-19 15:29:11 -08005555 }
5556
Martin KaFai Laud30d42e2018-12-05 17:35:44 -08005557 prev_offset = krecord[i].insn_off;
Yonghong Song838e9692018-11-19 15:29:11 -08005558 urecord += urec_size;
5559 }
5560
Yonghong Songba64e7d2018-11-24 23:20:44 -08005561 prog->aux->func_info = krecord;
5562 prog->aux->func_info_cnt = nfuncs;
Yonghong Song838e9692018-11-19 15:29:11 -08005563 return 0;
5564
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005565err_free:
Yonghong Songba64e7d2018-11-24 23:20:44 -08005566 kvfree(krecord);
Yonghong Song838e9692018-11-19 15:29:11 -08005567 return ret;
5568}
5569
Yonghong Songba64e7d2018-11-24 23:20:44 -08005570static void adjust_btf_func(struct bpf_verifier_env *env)
5571{
5572 int i;
5573
5574 if (!env->prog->aux->func_info)
5575 return;
5576
5577 for (i = 0; i < env->subprog_cnt; i++)
Martin KaFai Laud30d42e2018-12-05 17:35:44 -08005578 env->prog->aux->func_info[i].insn_off = env->subprog_info[i].start;
Yonghong Songba64e7d2018-11-24 23:20:44 -08005579}
5580
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005581#define MIN_BPF_LINEINFO_SIZE (offsetof(struct bpf_line_info, line_col) + \
5582 sizeof(((struct bpf_line_info *)(0))->line_col))
5583#define MAX_LINEINFO_REC_SIZE MAX_FUNCINFO_REC_SIZE
5584
5585static int check_btf_line(struct bpf_verifier_env *env,
5586 const union bpf_attr *attr,
5587 union bpf_attr __user *uattr)
5588{
5589 u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0;
5590 struct bpf_subprog_info *sub;
5591 struct bpf_line_info *linfo;
5592 struct bpf_prog *prog;
5593 const struct btf *btf;
5594 void __user *ulinfo;
5595 int err;
5596
5597 nr_linfo = attr->line_info_cnt;
5598 if (!nr_linfo)
5599 return 0;
5600
5601 rec_size = attr->line_info_rec_size;
5602 if (rec_size < MIN_BPF_LINEINFO_SIZE ||
5603 rec_size > MAX_LINEINFO_REC_SIZE ||
5604 rec_size & (sizeof(u32) - 1))
5605 return -EINVAL;
5606
5607 /* Need to zero it in case the userspace may
5608 * pass in a smaller bpf_line_info object.
5609 */
5610 linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info),
5611 GFP_KERNEL | __GFP_NOWARN);
5612 if (!linfo)
5613 return -ENOMEM;
5614
5615 prog = env->prog;
5616 btf = prog->aux->btf;
5617
5618 s = 0;
5619 sub = env->subprog_info;
5620 ulinfo = u64_to_user_ptr(attr->line_info);
5621 expected_size = sizeof(struct bpf_line_info);
5622 ncopy = min_t(u32, expected_size, rec_size);
5623 for (i = 0; i < nr_linfo; i++) {
5624 err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size);
5625 if (err) {
5626 if (err == -E2BIG) {
5627 verbose(env, "nonzero tailing record in line_info");
5628 if (put_user(expected_size,
5629 &uattr->line_info_rec_size))
5630 err = -EFAULT;
5631 }
5632 goto err_free;
5633 }
5634
5635 if (copy_from_user(&linfo[i], ulinfo, ncopy)) {
5636 err = -EFAULT;
5637 goto err_free;
5638 }
5639
5640 /*
5641 * Check insn_off to ensure
5642 * 1) strictly increasing AND
5643 * 2) bounded by prog->len
5644 *
5645 * The linfo[0].insn_off == 0 check logically falls into
5646 * the later "missing bpf_line_info for func..." case
5647 * because the first linfo[0].insn_off must be the
5648 * first sub also and the first sub must have
5649 * subprog_info[0].start == 0.
5650 */
5651 if ((i && linfo[i].insn_off <= prev_offset) ||
5652 linfo[i].insn_off >= prog->len) {
5653 verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n",
5654 i, linfo[i].insn_off, prev_offset,
5655 prog->len);
5656 err = -EINVAL;
5657 goto err_free;
5658 }
5659
Martin KaFai Laufdbaa0b2018-12-19 13:01:01 -08005660 if (!prog->insnsi[linfo[i].insn_off].code) {
5661 verbose(env,
5662 "Invalid insn code at line_info[%u].insn_off\n",
5663 i);
5664 err = -EINVAL;
5665 goto err_free;
5666 }
5667
Martin KaFai Lau23127b32018-12-13 10:41:46 -08005668 if (!btf_name_by_offset(btf, linfo[i].line_off) ||
5669 !btf_name_by_offset(btf, linfo[i].file_name_off)) {
Martin KaFai Lauc454a462018-12-07 16:42:25 -08005670 verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i);
5671 err = -EINVAL;
5672 goto err_free;
5673 }
5674
5675 if (s != env->subprog_cnt) {
5676 if (linfo[i].insn_off == sub[s].start) {
5677 sub[s].linfo_idx = i;
5678 s++;
5679 } else if (sub[s].start < linfo[i].insn_off) {
5680 verbose(env, "missing bpf_line_info for func#%u\n", s);
5681 err = -EINVAL;
5682 goto err_free;
5683 }
5684 }
5685
5686 prev_offset = linfo[i].insn_off;
5687 ulinfo += rec_size;
5688 }
5689
5690 if (s != env->subprog_cnt) {
5691 verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n",
5692 env->subprog_cnt - s, s);
5693 err = -EINVAL;
5694 goto err_free;
5695 }
5696
5697 prog->aux->linfo = linfo;
5698 prog->aux->nr_linfo = nr_linfo;
5699
5700 return 0;
5701
5702err_free:
5703 kvfree(linfo);
5704 return err;
5705}
5706
5707static int check_btf_info(struct bpf_verifier_env *env,
5708 const union bpf_attr *attr,
5709 union bpf_attr __user *uattr)
5710{
5711 struct btf *btf;
5712 int err;
5713
5714 if (!attr->func_info_cnt && !attr->line_info_cnt)
5715 return 0;
5716
5717 btf = btf_get_by_fd(attr->prog_btf_fd);
5718 if (IS_ERR(btf))
5719 return PTR_ERR(btf);
5720 env->prog->aux->btf = btf;
5721
5722 err = check_btf_func(env, attr, uattr);
5723 if (err)
5724 return err;
5725
5726 err = check_btf_line(env, attr, uattr);
5727 if (err)
5728 return err;
5729
5730 return 0;
Alexei Starovoitov0246e642014-09-26 00:17:04 -07005731}
5732
Edward Creef1174f72017-08-07 15:26:19 +01005733/* check %cur's range satisfies %old's */
5734static bool range_within(struct bpf_reg_state *old,
5735 struct bpf_reg_state *cur)
Alexei Starovoitov969bf052016-05-05 19:49:10 -07005736{
Edward Creeb03c9f92017-08-07 15:26:36 +01005737 return old->umin_value <= cur->umin_value &&
5738 old->umax_value >= cur->umax_value &&
5739 old->smin_value <= cur->smin_value &&
5740 old->smax_value >= cur->smax_value;
Edward Creef1174f72017-08-07 15:26:19 +01005741}
5742
5743/* Maximum number of register states that can exist at once */
5744#define ID_MAP_SIZE (MAX_BPF_REG + MAX_BPF_STACK / BPF_REG_SIZE)
5745struct idpair {
5746 u32 old;
5747 u32 cur;
5748};
5749
5750/* If in the old state two registers had the same id, then they need to have
5751 * the same id in the new state as well. But that id could be different from
5752 * the old state, so we need to track the mapping from old to new ids.
5753 * Once we have seen that, say, a reg with old id 5 had new id 9, any subsequent
5754 * regs with old id 5 must also have new id 9 for the new state to be safe. But
5755 * regs with a different old id could still have new id 9, we don't care about
5756 * that.
5757 * So we look through our idmap to see if this old id has been seen before. If
5758 * so, we require the new id to match; otherwise, we add the id pair to the map.
5759 */
5760static bool check_ids(u32 old_id, u32 cur_id, struct idpair *idmap)
5761{
5762 unsigned int i;
5763
5764 for (i = 0; i < ID_MAP_SIZE; i++) {
5765 if (!idmap[i].old) {
5766 /* Reached an empty slot; haven't seen this id before */
5767 idmap[i].old = old_id;
5768 idmap[i].cur = cur_id;
5769 return true;
5770 }
5771 if (idmap[i].old == old_id)
5772 return idmap[i].cur == cur_id;
5773 }
5774 /* We ran out of idmap slots, which should be impossible */
5775 WARN_ON_ONCE(1);
5776 return false;
5777}
5778
Alexei Starovoitov9242b5f2018-12-13 11:42:34 -08005779static void clean_func_state(struct bpf_verifier_env *env,
5780 struct bpf_func_state *st)
5781{
5782 enum bpf_reg_liveness live;
5783 int i, j;
5784
5785 for (i = 0; i < BPF_REG_FP; i++) {
5786 live = st->regs[i].live;
5787 /* liveness must not touch this register anymore */
5788 st->regs[i].live |= REG_LIVE_DONE;
5789 if (!(live & REG_LIVE_READ))
5790 /* since the register is unused, clear its state
5791 * to make further comparison simpler
5792 */
5793 __mark_reg_not_init(&st->regs[i]);
5794 }
5795
5796 for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) {
5797 live = st->stack[i].spilled_ptr.live;
5798 /* liveness must not touch this stack slot anymore */
5799 st->stack[i].spilled_ptr.live |= REG_LIVE_DONE;
5800 if (!(live & REG_LIVE_READ)) {
5801 __mark_reg_not_init(&st->stack[i].spilled_ptr);
5802 for (j = 0; j < BPF_REG_SIZE; j++)
5803 st->stack[i].slot_type[j] = STACK_INVALID;
5804 }
5805 }
5806}
5807
5808static void clean_verifier_state(struct bpf_verifier_env *env,
5809 struct bpf_verifier_state *st)
5810{
5811 int i;
5812
5813 if (st->frame[0]->regs[0].live & REG_LIVE_DONE)
5814 /* all regs in this state in all frames were already marked */
5815 return;
5816
5817 for (i = 0; i <= st->curframe; i++)
5818 clean_func_state(env, st->frame[i]);
5819}
5820
5821/* the parentage chains form a tree.
5822 * the verifier states are added to state lists at given insn and
5823 * pushed into state stack for future exploration.
5824 * when the verifier reaches bpf_exit insn some of the verifer states
5825 * stored in the state lists have their final liveness state already,
5826 * but a lot of states will get revised from liveness point of view when
5827 * the verifier explores other branches.
5828 * Example:
5829 * 1: r0 = 1
5830 * 2: if r1 == 100 goto pc+1
5831 * 3: r0 = 2
5832 * 4: exit
5833 * when the verifier reaches exit insn the register r0 in the state list of
5834 * insn 2 will be seen as !REG_LIVE_READ. Then the verifier pops the other_branch
5835 * of insn 2 and goes exploring further. At the insn 4 it will walk the
5836 * parentage chain from insn 4 into insn 2 and will mark r0 as REG_LIVE_READ.
5837 *
5838 * Since the verifier pushes the branch states as it sees them while exploring
5839 * the program the condition of walking the branch instruction for the second
5840 * time means that all states below this branch were already explored and
5841 * their final liveness markes are already propagated.
5842 * Hence when the verifier completes the search of state list in is_state_visited()
5843 * we can call this clean_live_states() function to mark all liveness states
5844 * as REG_LIVE_DONE to indicate that 'parent' pointers of 'struct bpf_reg_state'
5845 * will not be used.
5846 * This function also clears the registers and stack for states that !READ
5847 * to simplify state merging.
5848 *
5849 * Important note here that walking the same branch instruction in the callee
5850 * doesn't meant that the states are DONE. The verifier has to compare
5851 * the callsites
5852 */
5853static void clean_live_states(struct bpf_verifier_env *env, int insn,
5854 struct bpf_verifier_state *cur)
5855{
5856 struct bpf_verifier_state_list *sl;
5857 int i;
5858
5859 sl = env->explored_states[insn];
5860 if (!sl)
5861 return;
5862
5863 while (sl != STATE_LIST_MARK) {
5864 if (sl->state.curframe != cur->curframe)
5865 goto next;
5866 for (i = 0; i <= cur->curframe; i++)
5867 if (sl->state.frame[i]->callsite != cur->frame[i]->callsite)
5868 goto next;
5869 clean_verifier_state(env, &sl->state);
5870next:
5871 sl = sl->next;
5872 }
5873}
5874
Edward Creef1174f72017-08-07 15:26:19 +01005875/* Returns true if (rold safe implies rcur safe) */
Edward Cree1b688a12017-08-23 15:10:50 +01005876static bool regsafe(struct bpf_reg_state *rold, struct bpf_reg_state *rcur,
5877 struct idpair *idmap)
Edward Creef1174f72017-08-07 15:26:19 +01005878{
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08005879 bool equal;
5880
Edward Creedc503a82017-08-15 20:34:35 +01005881 if (!(rold->live & REG_LIVE_READ))
5882 /* explored state didn't use this */
5883 return true;
5884
Edward Cree679c7822018-08-22 20:02:19 +01005885 equal = memcmp(rold, rcur, offsetof(struct bpf_reg_state, parent)) == 0;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08005886
5887 if (rold->type == PTR_TO_STACK)
5888 /* two stack pointers are equal only if they're pointing to
5889 * the same stack frame, since fp-8 in foo != fp-8 in bar
5890 */
5891 return equal && rold->frameno == rcur->frameno;
5892
5893 if (equal)
Edward Creef1174f72017-08-07 15:26:19 +01005894 return true;
5895
5896 if (rold->type == NOT_INIT)
5897 /* explored state can't have used this */
5898 return true;
5899 if (rcur->type == NOT_INIT)
Alexei Starovoitov969bf052016-05-05 19:49:10 -07005900 return false;
Edward Creef1174f72017-08-07 15:26:19 +01005901 switch (rold->type) {
5902 case SCALAR_VALUE:
5903 if (rcur->type == SCALAR_VALUE) {
5904 /* new val must satisfy old val knowledge */
5905 return range_within(rold, rcur) &&
5906 tnum_in(rold->var_off, rcur->var_off);
5907 } else {
Jann Horn179d1c52017-12-18 20:11:59 -08005908 /* We're trying to use a pointer in place of a scalar.
5909 * Even if the scalar was unbounded, this could lead to
5910 * pointer leaks because scalars are allowed to leak
5911 * while pointers are not. We could make this safe in
5912 * special cases if root is calling us, but it's
5913 * probably not worth the hassle.
Edward Creef1174f72017-08-07 15:26:19 +01005914 */
Jann Horn179d1c52017-12-18 20:11:59 -08005915 return false;
Edward Creef1174f72017-08-07 15:26:19 +01005916 }
5917 case PTR_TO_MAP_VALUE:
Edward Cree1b688a12017-08-23 15:10:50 +01005918 /* If the new min/max/var_off satisfy the old ones and
5919 * everything else matches, we are OK.
Alexei Starovoitovd83525c2019-01-31 15:40:04 -08005920 * 'id' is not compared, since it's only used for maps with
5921 * bpf_spin_lock inside map element and in such cases if
5922 * the rest of the prog is valid for one map element then
5923 * it's valid for all map elements regardless of the key
5924 * used in bpf_map_lookup()
Edward Cree1b688a12017-08-23 15:10:50 +01005925 */
5926 return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 &&
5927 range_within(rold, rcur) &&
5928 tnum_in(rold->var_off, rcur->var_off);
Edward Creef1174f72017-08-07 15:26:19 +01005929 case PTR_TO_MAP_VALUE_OR_NULL:
5930 /* a PTR_TO_MAP_VALUE could be safe to use as a
5931 * PTR_TO_MAP_VALUE_OR_NULL into the same map.
5932 * However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL-
5933 * checked, doing so could have affected others with the same
5934 * id, and we can't check for that because we lost the id when
5935 * we converted to a PTR_TO_MAP_VALUE.
5936 */
5937 if (rcur->type != PTR_TO_MAP_VALUE_OR_NULL)
5938 return false;
5939 if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)))
5940 return false;
5941 /* Check our ids match any regs they're supposed to */
5942 return check_ids(rold->id, rcur->id, idmap);
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02005943 case PTR_TO_PACKET_META:
Edward Creef1174f72017-08-07 15:26:19 +01005944 case PTR_TO_PACKET:
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02005945 if (rcur->type != rold->type)
Edward Creef1174f72017-08-07 15:26:19 +01005946 return false;
5947 /* We must have at least as much range as the old ptr
5948 * did, so that any accesses which were safe before are
5949 * still safe. This is true even if old range < old off,
5950 * since someone could have accessed through (ptr - k), or
5951 * even done ptr -= k in a register, to get a safe access.
5952 */
5953 if (rold->range > rcur->range)
5954 return false;
5955 /* If the offsets don't match, we can't trust our alignment;
5956 * nor can we be sure that we won't fall out of range.
5957 */
5958 if (rold->off != rcur->off)
5959 return false;
5960 /* id relations must be preserved */
5961 if (rold->id && !check_ids(rold->id, rcur->id, idmap))
5962 return false;
5963 /* new val must satisfy old val knowledge */
5964 return range_within(rold, rcur) &&
5965 tnum_in(rold->var_off, rcur->var_off);
5966 case PTR_TO_CTX:
5967 case CONST_PTR_TO_MAP:
Edward Creef1174f72017-08-07 15:26:19 +01005968 case PTR_TO_PACKET_END:
Petar Penkovd58e4682018-09-14 07:46:18 -07005969 case PTR_TO_FLOW_KEYS:
Joe Stringerc64b7982018-10-02 13:35:33 -07005970 case PTR_TO_SOCKET:
5971 case PTR_TO_SOCKET_OR_NULL:
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08005972 case PTR_TO_SOCK_COMMON:
5973 case PTR_TO_SOCK_COMMON_OR_NULL:
Martin KaFai Lau655a51e2019-02-09 23:22:24 -08005974 case PTR_TO_TCP_SOCK:
5975 case PTR_TO_TCP_SOCK_OR_NULL:
Edward Creef1174f72017-08-07 15:26:19 +01005976 /* Only valid matches are exact, which memcmp() above
5977 * would have accepted
5978 */
5979 default:
5980 /* Don't know what's going on, just say it's not safe */
5981 return false;
5982 }
Alexei Starovoitov969bf052016-05-05 19:49:10 -07005983
Edward Creef1174f72017-08-07 15:26:19 +01005984 /* Shouldn't get here; if we do, say it's not safe */
5985 WARN_ON_ONCE(1);
Alexei Starovoitov969bf052016-05-05 19:49:10 -07005986 return false;
5987}
5988
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08005989static bool stacksafe(struct bpf_func_state *old,
5990 struct bpf_func_state *cur,
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07005991 struct idpair *idmap)
5992{
5993 int i, spi;
5994
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07005995 /* walk slots of the explored stack and ignore any additional
5996 * slots in the current stack, since explored(safe) state
5997 * didn't use them
5998 */
5999 for (i = 0; i < old->allocated_stack; i++) {
6000 spi = i / BPF_REG_SIZE;
6001
Alexei Starovoitovb2339202018-12-13 11:42:31 -08006002 if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ)) {
6003 i += BPF_REG_SIZE - 1;
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08006004 /* explored state didn't use this */
Gianluca Borellofd05e572017-12-23 10:09:55 +00006005 continue;
Alexei Starovoitovb2339202018-12-13 11:42:31 -08006006 }
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08006007
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07006008 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID)
6009 continue;
Alexei Starovoitov19e2dbb2018-12-13 11:42:33 -08006010
6011 /* explored stack has more populated slots than current stack
6012 * and these slots were used
6013 */
6014 if (i >= cur->allocated_stack)
6015 return false;
6016
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08006017 /* if old state was safe with misc data in the stack
6018 * it will be safe with zero-initialized stack.
6019 * The opposite is not true
6020 */
6021 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC &&
6022 cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO)
6023 continue;
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07006024 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] !=
6025 cur->stack[spi].slot_type[i % BPF_REG_SIZE])
6026 /* Ex: old explored (safe) state has STACK_SPILL in
6027 * this stack slot, but current has has STACK_MISC ->
6028 * this verifier states are not equivalent,
6029 * return false to continue verification of this path
6030 */
6031 return false;
6032 if (i % BPF_REG_SIZE)
6033 continue;
6034 if (old->stack[spi].slot_type[0] != STACK_SPILL)
6035 continue;
6036 if (!regsafe(&old->stack[spi].spilled_ptr,
6037 &cur->stack[spi].spilled_ptr,
6038 idmap))
6039 /* when explored and current stack slot are both storing
6040 * spilled registers, check that stored pointers types
6041 * are the same as well.
6042 * Ex: explored safe path could have stored
6043 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -8}
6044 * but current path has stored:
6045 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -16}
6046 * such verifier states are not equivalent.
6047 * return false to continue verification of this path
6048 */
6049 return false;
6050 }
6051 return true;
6052}
6053
Joe Stringerfd978bf72018-10-02 13:35:35 -07006054static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur)
6055{
6056 if (old->acquired_refs != cur->acquired_refs)
6057 return false;
6058 return !memcmp(old->refs, cur->refs,
6059 sizeof(*old->refs) * old->acquired_refs);
6060}
6061
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006062/* compare two verifier states
6063 *
6064 * all states stored in state_list are known to be valid, since
6065 * verifier reached 'bpf_exit' instruction through them
6066 *
6067 * this function is called when verifier exploring different branches of
6068 * execution popped from the state stack. If it sees an old state that has
6069 * more strict register state and more strict stack state then this execution
6070 * branch doesn't need to be explored further, since verifier already
6071 * concluded that more strict state leads to valid finish.
6072 *
6073 * Therefore two states are equivalent if register state is more conservative
6074 * and explored stack state is more conservative than the current one.
6075 * Example:
6076 * explored current
6077 * (slot1=INV slot2=MISC) == (slot1=MISC slot2=MISC)
6078 * (slot1=MISC slot2=MISC) != (slot1=INV slot2=MISC)
6079 *
6080 * In other words if current stack state (one being explored) has more
6081 * valid slots than old one that already passed validation, it means
6082 * the verifier can stop exploring and conclude that current state is valid too
6083 *
6084 * Similarly with registers. If explored state has register type as invalid
6085 * whereas register type in current state is meaningful, it means that
6086 * the current state will reach 'bpf_exit' instruction safely
6087 */
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006088static bool func_states_equal(struct bpf_func_state *old,
6089 struct bpf_func_state *cur)
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006090{
Edward Creef1174f72017-08-07 15:26:19 +01006091 struct idpair *idmap;
6092 bool ret = false;
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006093 int i;
6094
Edward Creef1174f72017-08-07 15:26:19 +01006095 idmap = kcalloc(ID_MAP_SIZE, sizeof(struct idpair), GFP_KERNEL);
6096 /* If we failed to allocate the idmap, just say it's not safe */
6097 if (!idmap)
Alexei Starovoitov1a0dc1a2016-05-05 19:49:09 -07006098 return false;
Edward Creef1174f72017-08-07 15:26:19 +01006099
6100 for (i = 0; i < MAX_BPF_REG; i++) {
Edward Cree1b688a12017-08-23 15:10:50 +01006101 if (!regsafe(&old->regs[i], &cur->regs[i], idmap))
Edward Creef1174f72017-08-07 15:26:19 +01006102 goto out_free;
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006103 }
6104
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07006105 if (!stacksafe(old, cur, idmap))
6106 goto out_free;
Joe Stringerfd978bf72018-10-02 13:35:35 -07006107
6108 if (!refsafe(old, cur))
6109 goto out_free;
Edward Creef1174f72017-08-07 15:26:19 +01006110 ret = true;
6111out_free:
6112 kfree(idmap);
6113 return ret;
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006114}
6115
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006116static bool states_equal(struct bpf_verifier_env *env,
6117 struct bpf_verifier_state *old,
6118 struct bpf_verifier_state *cur)
Edward Creedc503a82017-08-15 20:34:35 +01006119{
Edward Creedc503a82017-08-15 20:34:35 +01006120 int i;
6121
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006122 if (old->curframe != cur->curframe)
6123 return false;
6124
Daniel Borkmann979d63d2019-01-03 00:58:34 +01006125 /* Verification state from speculative execution simulation
6126 * must never prune a non-speculative execution one.
6127 */
6128 if (old->speculative && !cur->speculative)
6129 return false;
6130
Alexei Starovoitovd83525c2019-01-31 15:40:04 -08006131 if (old->active_spin_lock != cur->active_spin_lock)
6132 return false;
6133
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006134 /* for states to be equal callsites have to be the same
6135 * and all frame states need to be equivalent
6136 */
6137 for (i = 0; i <= old->curframe; i++) {
6138 if (old->frame[i]->callsite != cur->frame[i]->callsite)
6139 return false;
6140 if (!func_states_equal(old->frame[i], cur->frame[i]))
6141 return false;
6142 }
6143 return true;
6144}
6145
6146/* A write screens off any subsequent reads; but write marks come from the
6147 * straight-line code between a state and its parent. When we arrive at an
6148 * equivalent state (jump target or such) we didn't arrive by the straight-line
6149 * code, so read marks in the state must propagate to the parent regardless
6150 * of the state's write marks. That's what 'parent == state->parent' comparison
Edward Cree679c7822018-08-22 20:02:19 +01006151 * in mark_reg_read() is for.
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006152 */
6153static int propagate_liveness(struct bpf_verifier_env *env,
6154 const struct bpf_verifier_state *vstate,
6155 struct bpf_verifier_state *vparent)
6156{
6157 int i, frame, err = 0;
6158 struct bpf_func_state *state, *parent;
6159
6160 if (vparent->curframe != vstate->curframe) {
6161 WARN(1, "propagate_live: parent frame %d current frame %d\n",
6162 vparent->curframe, vstate->curframe);
6163 return -EFAULT;
6164 }
Edward Creedc503a82017-08-15 20:34:35 +01006165 /* Propagate read liveness of registers... */
6166 BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG);
Jakub Kicinski83d16312019-03-21 14:34:36 -07006167 for (frame = 0; frame <= vstate->curframe; frame++) {
6168 /* We don't need to worry about FP liveness, it's read-only */
6169 for (i = frame < vstate->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) {
6170 if (vparent->frame[frame]->regs[i].live & REG_LIVE_READ)
6171 continue;
6172 if (vstate->frame[frame]->regs[i].live & REG_LIVE_READ) {
6173 err = mark_reg_read(env, &vstate->frame[frame]->regs[i],
6174 &vparent->frame[frame]->regs[i]);
6175 if (err)
6176 return err;
6177 }
Edward Creedc503a82017-08-15 20:34:35 +01006178 }
6179 }
Edward Creedc503a82017-08-15 20:34:35 +01006180
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006181 /* ... and stack slots */
6182 for (frame = 0; frame <= vstate->curframe; frame++) {
6183 state = vstate->frame[frame];
6184 parent = vparent->frame[frame];
6185 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE &&
6186 i < parent->allocated_stack / BPF_REG_SIZE; i++) {
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006187 if (parent->stack[i].spilled_ptr.live & REG_LIVE_READ)
6188 continue;
6189 if (state->stack[i].spilled_ptr.live & REG_LIVE_READ)
Edward Cree679c7822018-08-22 20:02:19 +01006190 mark_reg_read(env, &state->stack[i].spilled_ptr,
6191 &parent->stack[i].spilled_ptr);
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006192 }
Edward Creedc503a82017-08-15 20:34:35 +01006193 }
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006194 return err;
Edward Creedc503a82017-08-15 20:34:35 +01006195}
6196
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01006197static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006198{
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01006199 struct bpf_verifier_state_list *new_sl;
Alexei Starovoitov9f4686c2019-04-01 21:27:41 -07006200 struct bpf_verifier_state_list *sl, **pprev;
Edward Cree679c7822018-08-22 20:02:19 +01006201 struct bpf_verifier_state *cur = env->cur_state, *new;
Alexei Starovoitovceefbc92018-12-03 22:46:06 -08006202 int i, j, err, states_cnt = 0;
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006203
Alexei Starovoitov9f4686c2019-04-01 21:27:41 -07006204 pprev = &env->explored_states[insn_idx];
6205 sl = *pprev;
6206
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006207 if (!sl)
6208 /* this 'insn_idx' instruction wasn't marked, so we will not
6209 * be doing state search here
6210 */
6211 return 0;
6212
Alexei Starovoitov9242b5f2018-12-13 11:42:34 -08006213 clean_live_states(env, insn_idx, cur);
6214
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006215 while (sl != STATE_LIST_MARK) {
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07006216 if (states_equal(env, &sl->state, cur)) {
Alexei Starovoitov9f4686c2019-04-01 21:27:41 -07006217 sl->hit_cnt++;
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006218 /* reached equivalent register/stack state,
Edward Creedc503a82017-08-15 20:34:35 +01006219 * prune the search.
6220 * Registers read by the continuation are read by us.
Edward Cree8e9cd9c2017-08-23 15:11:21 +01006221 * If we have any write marks in env->cur_state, they
6222 * will prevent corresponding reads in the continuation
6223 * from reaching our parent (an explored_state). Our
6224 * own state will get the read marks recorded, but
6225 * they'll be immediately forgotten as we're pruning
6226 * this state and will pop a new one.
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006227 */
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006228 err = propagate_liveness(env, &sl->state, cur);
6229 if (err)
6230 return err;
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006231 return 1;
Edward Creedc503a82017-08-15 20:34:35 +01006232 }
Alexei Starovoitovceefbc92018-12-03 22:46:06 -08006233 states_cnt++;
Alexei Starovoitov9f4686c2019-04-01 21:27:41 -07006234 sl->miss_cnt++;
6235 /* heuristic to determine whether this state is beneficial
6236 * to keep checking from state equivalence point of view.
6237 * Higher numbers increase max_states_per_insn and verification time,
6238 * but do not meaningfully decrease insn_processed.
6239 */
6240 if (sl->miss_cnt > sl->hit_cnt * 3 + 3) {
6241 /* the state is unlikely to be useful. Remove it to
6242 * speed up verification
6243 */
6244 *pprev = sl->next;
6245 if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE) {
6246 free_verifier_state(&sl->state, false);
6247 kfree(sl);
6248 env->peak_states--;
6249 } else {
6250 /* cannot free this state, since parentage chain may
6251 * walk it later. Add it for free_list instead to
6252 * be freed at the end of verification
6253 */
6254 sl->next = env->free_list;
6255 env->free_list = sl;
6256 }
6257 sl = *pprev;
6258 continue;
6259 }
6260 pprev = &sl->next;
6261 sl = *pprev;
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006262 }
6263
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07006264 if (env->max_states_per_insn < states_cnt)
6265 env->max_states_per_insn = states_cnt;
6266
Alexei Starovoitovceefbc92018-12-03 22:46:06 -08006267 if (!env->allow_ptr_leaks && states_cnt > BPF_COMPLEXITY_LIMIT_STATES)
6268 return 0;
6269
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006270 /* there were no equivalent states, remember current one.
6271 * technically the current state is not proven to be safe yet,
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006272 * but it will either reach outer most bpf_exit (which means it's safe)
6273 * or it will be rejected. Since there are no loops, we won't be
6274 * seeing this tuple (frame[0].callsite, frame[1].callsite, .. insn_idx)
6275 * again on the way to bpf_exit
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006276 */
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07006277 new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL);
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006278 if (!new_sl)
6279 return -ENOMEM;
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07006280 env->total_states++;
6281 env->peak_states++;
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006282
6283 /* add new state to the head of linked list */
Edward Cree679c7822018-08-22 20:02:19 +01006284 new = &new_sl->state;
6285 err = copy_verifier_state(new, cur);
Alexei Starovoitov1969db42017-11-01 00:08:04 -07006286 if (err) {
Edward Cree679c7822018-08-22 20:02:19 +01006287 free_verifier_state(new, false);
Alexei Starovoitov1969db42017-11-01 00:08:04 -07006288 kfree(new_sl);
6289 return err;
6290 }
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006291 new_sl->next = env->explored_states[insn_idx];
6292 env->explored_states[insn_idx] = new_sl;
Jakub Kicinski7640ead2018-12-12 16:29:07 -08006293 /* connect new state to parentage chain. Current frame needs all
6294 * registers connected. Only r6 - r9 of the callers are alive (pushed
6295 * to the stack implicitly by JITs) so in callers' frames connect just
6296 * r6 - r9 as an optimization. Callers will have r1 - r5 connected to
6297 * the state of the call instruction (with WRITTEN set), and r0 comes
6298 * from callee with its full parentage chain, anyway.
6299 */
6300 for (j = 0; j <= cur->curframe; j++)
6301 for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++)
6302 cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i];
Edward Cree8e9cd9c2017-08-23 15:11:21 +01006303 /* clear write marks in current state: the writes we did are not writes
6304 * our child did, so they don't screen off its reads from us.
6305 * (There are no read marks in current state, because reads always mark
6306 * their parent and current state never has children yet. Only
6307 * explored_states can get read marks.)
6308 */
Edward Creedc503a82017-08-15 20:34:35 +01006309 for (i = 0; i < BPF_REG_FP; i++)
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006310 cur->frame[cur->curframe]->regs[i].live = REG_LIVE_NONE;
6311
6312 /* all stack frames are accessible from callee, clear them all */
6313 for (j = 0; j <= cur->curframe; j++) {
6314 struct bpf_func_state *frame = cur->frame[j];
Edward Cree679c7822018-08-22 20:02:19 +01006315 struct bpf_func_state *newframe = new->frame[j];
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006316
Edward Cree679c7822018-08-22 20:02:19 +01006317 for (i = 0; i < frame->allocated_stack / BPF_REG_SIZE; i++) {
Alexei Starovoitovcc2b14d2017-12-14 17:55:08 -08006318 frame->stack[i].spilled_ptr.live = REG_LIVE_NONE;
Edward Cree679c7822018-08-22 20:02:19 +01006319 frame->stack[i].spilled_ptr.parent =
6320 &newframe->stack[i].spilled_ptr;
6321 }
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006322 }
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006323 return 0;
6324}
6325
Joe Stringerc64b7982018-10-02 13:35:33 -07006326/* Return true if it's OK to have the same insn return a different type. */
6327static bool reg_type_mismatch_ok(enum bpf_reg_type type)
6328{
6329 switch (type) {
6330 case PTR_TO_CTX:
6331 case PTR_TO_SOCKET:
6332 case PTR_TO_SOCKET_OR_NULL:
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08006333 case PTR_TO_SOCK_COMMON:
6334 case PTR_TO_SOCK_COMMON_OR_NULL:
Martin KaFai Lau655a51e2019-02-09 23:22:24 -08006335 case PTR_TO_TCP_SOCK:
6336 case PTR_TO_TCP_SOCK_OR_NULL:
Joe Stringerc64b7982018-10-02 13:35:33 -07006337 return false;
6338 default:
6339 return true;
6340 }
6341}
6342
6343/* If an instruction was previously used with particular pointer types, then we
6344 * need to be careful to avoid cases such as the below, where it may be ok
6345 * for one branch accessing the pointer, but not ok for the other branch:
6346 *
6347 * R1 = sock_ptr
6348 * goto X;
6349 * ...
6350 * R1 = some_other_valid_ptr;
6351 * goto X;
6352 * ...
6353 * R2 = *(u32 *)(R1 + 0);
6354 */
6355static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev)
6356{
6357 return src != prev && (!reg_type_mismatch_ok(src) ||
6358 !reg_type_mismatch_ok(prev));
6359}
6360
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01006361static int do_check(struct bpf_verifier_env *env)
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006362{
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07006363 struct bpf_verifier_state *state;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006364 struct bpf_insn *insns = env->prog->insnsi;
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07006365 struct bpf_reg_state *regs;
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07006366 int insn_cnt = env->prog->len;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006367 bool do_print_state = false;
6368
Martin KaFai Laud9762e82018-12-13 10:41:48 -08006369 env->prev_linfo = NULL;
6370
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07006371 state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL);
6372 if (!state)
6373 return -ENOMEM;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006374 state->curframe = 0;
Daniel Borkmann979d63d2019-01-03 00:58:34 +01006375 state->speculative = false;
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006376 state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL);
6377 if (!state->frame[0]) {
6378 kfree(state);
6379 return -ENOMEM;
6380 }
6381 env->cur_state = state;
6382 init_func_state(env, state->frame[0],
6383 BPF_MAIN_FUNC /* callsite */,
6384 0 /* frameno */,
6385 0 /* subprogno, zero == main subprog */);
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006386
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006387 for (;;) {
6388 struct bpf_insn *insn;
6389 u8 class;
6390 int err;
6391
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006392 if (env->insn_idx >= insn_cnt) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006393 verbose(env, "invalid insn idx %d insn_cnt %d\n",
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006394 env->insn_idx, insn_cnt);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006395 return -EFAULT;
6396 }
6397
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006398 insn = &insns[env->insn_idx];
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006399 class = BPF_CLASS(insn->code);
6400
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07006401 if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006402 verbose(env,
6403 "BPF program is too large. Processed %d insn\n",
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07006404 env->insn_processed);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006405 return -E2BIG;
6406 }
6407
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006408 err = is_state_visited(env, env->insn_idx);
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006409 if (err < 0)
6410 return err;
6411 if (err == 1) {
6412 /* found equivalent state, can prune the search */
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07006413 if (env->log.level & BPF_LOG_LEVEL) {
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006414 if (do_print_state)
Daniel Borkmann979d63d2019-01-03 00:58:34 +01006415 verbose(env, "\nfrom %d to %d%s: safe\n",
6416 env->prev_insn_idx, env->insn_idx,
6417 env->cur_state->speculative ?
6418 " (speculative execution)" : "");
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006419 else
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006420 verbose(env, "%d: safe\n", env->insn_idx);
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006421 }
6422 goto process_bpf_exit;
6423 }
6424
Alexei Starovoitovc3494802018-12-03 22:46:04 -08006425 if (signal_pending(current))
6426 return -EAGAIN;
6427
Daniel Borkmann3c2ce602017-05-18 03:00:06 +02006428 if (need_resched())
6429 cond_resched();
6430
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07006431 if (env->log.level & BPF_LOG_LEVEL2 ||
6432 (env->log.level & BPF_LOG_LEVEL && do_print_state)) {
6433 if (env->log.level & BPF_LOG_LEVEL2)
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006434 verbose(env, "%d:", env->insn_idx);
David S. Millerc5fc9692017-05-10 11:25:17 -07006435 else
Daniel Borkmann979d63d2019-01-03 00:58:34 +01006436 verbose(env, "\nfrom %d to %d%s:",
6437 env->prev_insn_idx, env->insn_idx,
6438 env->cur_state->speculative ?
6439 " (speculative execution)" : "");
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006440 print_verifier_state(env, state->frame[state->curframe]);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006441 do_print_state = false;
6442 }
6443
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07006444 if (env->log.level & BPF_LOG_LEVEL) {
Daniel Borkmann7105e822017-12-20 13:42:57 +01006445 const struct bpf_insn_cbs cbs = {
6446 .cb_print = verbose,
Jiri Olsaabe08842018-03-23 11:41:28 +01006447 .private_data = env,
Daniel Borkmann7105e822017-12-20 13:42:57 +01006448 };
6449
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006450 verbose_linfo(env, env->insn_idx, "; ");
6451 verbose(env, "%d: ", env->insn_idx);
Jiri Olsaabe08842018-03-23 11:41:28 +01006452 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006453 }
6454
Jakub Kicinskicae19272017-12-27 18:39:05 -08006455 if (bpf_prog_is_dev_bound(env->prog->aux)) {
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006456 err = bpf_prog_offload_verify_insn(env, env->insn_idx,
6457 env->prev_insn_idx);
Jakub Kicinskicae19272017-12-27 18:39:05 -08006458 if (err)
6459 return err;
6460 }
Jakub Kicinski13a27df2016-09-21 11:43:58 +01006461
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07006462 regs = cur_regs(env);
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006463 env->insn_aux_data[env->insn_idx].seen = true;
Joe Stringerfd978bf72018-10-02 13:35:35 -07006464
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006465 if (class == BPF_ALU || class == BPF_ALU64) {
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07006466 err = check_alu_op(env, insn);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006467 if (err)
6468 return err;
6469
6470 } else if (class == BPF_LDX) {
Jakub Kicinski3df126f2016-09-21 11:43:56 +01006471 enum bpf_reg_type *prev_src_type, src_reg_type;
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07006472
6473 /* check for reserved fields is already done */
6474
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006475 /* check src operand */
Edward Creedc503a82017-08-15 20:34:35 +01006476 err = check_reg_arg(env, insn->src_reg, SRC_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006477 if (err)
6478 return err;
6479
Edward Creedc503a82017-08-15 20:34:35 +01006480 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006481 if (err)
6482 return err;
6483
Alexei Starovoitov725f9dc2015-04-15 16:19:33 -07006484 src_reg_type = regs[insn->src_reg].type;
6485
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006486 /* check that memory (src_reg + off) is readable,
6487 * the state of dst_reg will be updated by this func
6488 */
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006489 err = check_mem_access(env, env->insn_idx, insn->src_reg,
6490 insn->off, BPF_SIZE(insn->code),
6491 BPF_READ, insn->dst_reg, false);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006492 if (err)
6493 return err;
6494
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006495 prev_src_type = &env->insn_aux_data[env->insn_idx].ptr_type;
Jakub Kicinski3df126f2016-09-21 11:43:56 +01006496
6497 if (*prev_src_type == NOT_INIT) {
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07006498 /* saw a valid insn
6499 * dst_reg = *(u32 *)(src_reg + off)
Jakub Kicinski3df126f2016-09-21 11:43:56 +01006500 * save type to validate intersecting paths
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07006501 */
Jakub Kicinski3df126f2016-09-21 11:43:56 +01006502 *prev_src_type = src_reg_type;
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07006503
Joe Stringerc64b7982018-10-02 13:35:33 -07006504 } else if (reg_type_mismatch(src_reg_type, *prev_src_type)) {
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07006505 /* ABuser program is trying to use the same insn
6506 * dst_reg = *(u32*) (src_reg + off)
6507 * with different pointer types:
6508 * src_reg == ctx in one branch and
6509 * src_reg == stack|map in some other branch.
6510 * Reject it.
6511 */
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006512 verbose(env, "same insn cannot be used with different pointers\n");
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07006513 return -EINVAL;
6514 }
6515
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006516 } else if (class == BPF_STX) {
Jakub Kicinski3df126f2016-09-21 11:43:56 +01006517 enum bpf_reg_type *prev_dst_type, dst_reg_type;
Alexei Starovoitovd691f9e2015-06-04 10:11:54 -07006518
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006519 if (BPF_MODE(insn->code) == BPF_XADD) {
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006520 err = check_xadd(env, env->insn_idx, insn);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006521 if (err)
6522 return err;
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006523 env->insn_idx++;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006524 continue;
6525 }
6526
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006527 /* check src1 operand */
Edward Creedc503a82017-08-15 20:34:35 +01006528 err = check_reg_arg(env, insn->src_reg, SRC_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006529 if (err)
6530 return err;
6531 /* check src2 operand */
Edward Creedc503a82017-08-15 20:34:35 +01006532 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006533 if (err)
6534 return err;
6535
Alexei Starovoitovd691f9e2015-06-04 10:11:54 -07006536 dst_reg_type = regs[insn->dst_reg].type;
6537
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006538 /* check that memory (dst_reg + off) is writeable */
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006539 err = check_mem_access(env, env->insn_idx, insn->dst_reg,
6540 insn->off, BPF_SIZE(insn->code),
6541 BPF_WRITE, insn->src_reg, false);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006542 if (err)
6543 return err;
6544
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006545 prev_dst_type = &env->insn_aux_data[env->insn_idx].ptr_type;
Jakub Kicinski3df126f2016-09-21 11:43:56 +01006546
6547 if (*prev_dst_type == NOT_INIT) {
6548 *prev_dst_type = dst_reg_type;
Joe Stringerc64b7982018-10-02 13:35:33 -07006549 } else if (reg_type_mismatch(dst_reg_type, *prev_dst_type)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006550 verbose(env, "same insn cannot be used with different pointers\n");
Alexei Starovoitovd691f9e2015-06-04 10:11:54 -07006551 return -EINVAL;
6552 }
6553
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006554 } else if (class == BPF_ST) {
6555 if (BPF_MODE(insn->code) != BPF_MEM ||
6556 insn->src_reg != BPF_REG_0) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006557 verbose(env, "BPF_ST uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006558 return -EINVAL;
6559 }
6560 /* check src operand */
Edward Creedc503a82017-08-15 20:34:35 +01006561 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006562 if (err)
6563 return err;
6564
Daniel Borkmannf37a8cb2018-01-16 23:30:10 +01006565 if (is_ctx_reg(env, insn->dst_reg)) {
Joe Stringer9d2be442018-10-02 13:35:31 -07006566 verbose(env, "BPF_ST stores into R%d %s is not allowed\n",
Daniel Borkmann2a159c62018-10-21 02:09:24 +02006567 insn->dst_reg,
6568 reg_type_str[reg_state(env, insn->dst_reg)->type]);
Daniel Borkmannf37a8cb2018-01-16 23:30:10 +01006569 return -EACCES;
6570 }
6571
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006572 /* check that memory (dst_reg + off) is writeable */
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006573 err = check_mem_access(env, env->insn_idx, insn->dst_reg,
6574 insn->off, BPF_SIZE(insn->code),
6575 BPF_WRITE, -1, false);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006576 if (err)
6577 return err;
6578
Jiong Wang092ed092019-01-26 12:26:01 -05006579 } else if (class == BPF_JMP || class == BPF_JMP32) {
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006580 u8 opcode = BPF_OP(insn->code);
6581
6582 if (opcode == BPF_CALL) {
6583 if (BPF_SRC(insn->code) != BPF_K ||
6584 insn->off != 0 ||
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006585 (insn->src_reg != BPF_REG_0 &&
6586 insn->src_reg != BPF_PSEUDO_CALL) ||
Jiong Wang092ed092019-01-26 12:26:01 -05006587 insn->dst_reg != BPF_REG_0 ||
6588 class == BPF_JMP32) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006589 verbose(env, "BPF_CALL uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006590 return -EINVAL;
6591 }
6592
Alexei Starovoitovd83525c2019-01-31 15:40:04 -08006593 if (env->cur_state->active_spin_lock &&
6594 (insn->src_reg == BPF_PSEUDO_CALL ||
6595 insn->imm != BPF_FUNC_spin_unlock)) {
6596 verbose(env, "function calls are not allowed while holding a lock\n");
6597 return -EINVAL;
6598 }
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006599 if (insn->src_reg == BPF_PSEUDO_CALL)
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006600 err = check_func_call(env, insn, &env->insn_idx);
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006601 else
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006602 err = check_helper_call(env, insn->imm, env->insn_idx);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006603 if (err)
6604 return err;
6605
6606 } else if (opcode == BPF_JA) {
6607 if (BPF_SRC(insn->code) != BPF_K ||
6608 insn->imm != 0 ||
6609 insn->src_reg != BPF_REG_0 ||
Jiong Wang092ed092019-01-26 12:26:01 -05006610 insn->dst_reg != BPF_REG_0 ||
6611 class == BPF_JMP32) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006612 verbose(env, "BPF_JA uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006613 return -EINVAL;
6614 }
6615
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006616 env->insn_idx += insn->off + 1;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006617 continue;
6618
6619 } else if (opcode == BPF_EXIT) {
6620 if (BPF_SRC(insn->code) != BPF_K ||
6621 insn->imm != 0 ||
6622 insn->src_reg != BPF_REG_0 ||
Jiong Wang092ed092019-01-26 12:26:01 -05006623 insn->dst_reg != BPF_REG_0 ||
6624 class == BPF_JMP32) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006625 verbose(env, "BPF_EXIT uses reserved fields\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006626 return -EINVAL;
6627 }
6628
Alexei Starovoitovd83525c2019-01-31 15:40:04 -08006629 if (env->cur_state->active_spin_lock) {
6630 verbose(env, "bpf_spin_unlock is missing\n");
6631 return -EINVAL;
6632 }
6633
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006634 if (state->curframe) {
6635 /* exit from nested function */
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006636 env->prev_insn_idx = env->insn_idx;
6637 err = prepare_func_exit(env, &env->insn_idx);
Alexei Starovoitovf4d7e402017-12-14 17:55:06 -08006638 if (err)
6639 return err;
6640 do_print_state = true;
6641 continue;
6642 }
6643
Joe Stringerfd978bf72018-10-02 13:35:35 -07006644 err = check_reference_leak(env);
6645 if (err)
6646 return err;
6647
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006648 /* eBPF calling convetion is such that R0 is used
6649 * to return the value from eBPF program.
6650 * Make sure that it's readable at this time
6651 * of bpf_exit, which means that program wrote
6652 * something into it earlier
6653 */
Edward Creedc503a82017-08-15 20:34:35 +01006654 err = check_reg_arg(env, BPF_REG_0, SRC_OP);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006655 if (err)
6656 return err;
6657
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07006658 if (is_pointer_value(env, BPF_REG_0)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006659 verbose(env, "R0 leaks addr as return value\n");
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07006660 return -EACCES;
6661 }
6662
Alexei Starovoitov390ee7e2017-10-02 22:50:23 -07006663 err = check_return_code(env);
6664 if (err)
6665 return err;
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07006666process_bpf_exit:
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006667 err = pop_stack(env, &env->prev_insn_idx,
6668 &env->insn_idx);
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07006669 if (err < 0) {
6670 if (err != -ENOENT)
6671 return err;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006672 break;
6673 } else {
6674 do_print_state = true;
6675 continue;
6676 }
6677 } else {
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006678 err = check_cond_jmp_op(env, insn, &env->insn_idx);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006679 if (err)
6680 return err;
6681 }
6682 } else if (class == BPF_LD) {
6683 u8 mode = BPF_MODE(insn->code);
6684
6685 if (mode == BPF_ABS || mode == BPF_IND) {
Alexei Starovoitovddd872b2014-12-01 15:06:34 -08006686 err = check_ld_abs(env, insn);
6687 if (err)
6688 return err;
6689
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006690 } else if (mode == BPF_IMM) {
6691 err = check_ld_imm(env, insn);
6692 if (err)
6693 return err;
6694
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006695 env->insn_idx++;
6696 env->insn_aux_data[env->insn_idx].seen = true;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006697 } else {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006698 verbose(env, "invalid BPF_LD mode\n");
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006699 return -EINVAL;
6700 }
6701 } else {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006702 verbose(env, "unknown insn class %d\n", class);
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006703 return -EINVAL;
6704 }
6705
Daniel Borkmannc08435e2019-01-03 00:58:27 +01006706 env->insn_idx++;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006707 }
6708
Jiong Wang9c8105b2018-05-02 16:17:18 -04006709 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;
Alexei Starovoitov17a52672014-09-26 00:17:06 -07006710 return 0;
6711}
6712
Martin KaFai Lau56f668d2017-03-22 10:00:33 -07006713static int check_map_prealloc(struct bpf_map *map)
6714{
6715 return (map->map_type != BPF_MAP_TYPE_HASH &&
Martin KaFai Laubcc6b1b2017-03-22 10:00:34 -07006716 map->map_type != BPF_MAP_TYPE_PERCPU_HASH &&
6717 map->map_type != BPF_MAP_TYPE_HASH_OF_MAPS) ||
Martin KaFai Lau56f668d2017-03-22 10:00:33 -07006718 !(map->map_flags & BPF_F_NO_PREALLOC);
6719}
6720
Alexei Starovoitovd83525c2019-01-31 15:40:04 -08006721static bool is_tracing_prog_type(enum bpf_prog_type type)
6722{
6723 switch (type) {
6724 case BPF_PROG_TYPE_KPROBE:
6725 case BPF_PROG_TYPE_TRACEPOINT:
6726 case BPF_PROG_TYPE_PERF_EVENT:
6727 case BPF_PROG_TYPE_RAW_TRACEPOINT:
6728 return true;
6729 default:
6730 return false;
6731 }
6732}
6733
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006734static int check_map_prog_compatibility(struct bpf_verifier_env *env,
6735 struct bpf_map *map,
Alexei Starovoitovfdc15d32016-09-01 18:37:23 -07006736 struct bpf_prog *prog)
6737
6738{
Martin KaFai Lau56f668d2017-03-22 10:00:33 -07006739 /* Make sure that BPF_PROG_TYPE_PERF_EVENT programs only use
6740 * preallocated hash maps, since doing memory allocation
6741 * in overflow_handler can crash depending on where nmi got
6742 * triggered.
6743 */
6744 if (prog->type == BPF_PROG_TYPE_PERF_EVENT) {
6745 if (!check_map_prealloc(map)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006746 verbose(env, "perf_event programs can only use preallocated hash map\n");
Martin KaFai Lau56f668d2017-03-22 10:00:33 -07006747 return -EINVAL;
6748 }
6749 if (map->inner_map_meta &&
6750 !check_map_prealloc(map->inner_map_meta)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006751 verbose(env, "perf_event programs can only use preallocated inner hash map\n");
Martin KaFai Lau56f668d2017-03-22 10:00:33 -07006752 return -EINVAL;
6753 }
Alexei Starovoitovfdc15d32016-09-01 18:37:23 -07006754 }
Jakub Kicinskia3884572018-01-11 20:29:09 -08006755
Alexei Starovoitovd83525c2019-01-31 15:40:04 -08006756 if ((is_tracing_prog_type(prog->type) ||
6757 prog->type == BPF_PROG_TYPE_SOCKET_FILTER) &&
6758 map_value_has_spin_lock(map)) {
6759 verbose(env, "tracing progs cannot use bpf_spin_lock yet\n");
6760 return -EINVAL;
6761 }
6762
Jakub Kicinskia3884572018-01-11 20:29:09 -08006763 if ((bpf_prog_is_dev_bound(prog->aux) || bpf_map_is_dev_bound(map)) &&
Jakub Kicinski09728262018-07-17 10:53:23 -07006764 !bpf_offload_prog_map_match(prog, map)) {
Jakub Kicinskia3884572018-01-11 20:29:09 -08006765 verbose(env, "offload device mismatch between prog and map\n");
6766 return -EINVAL;
6767 }
6768
Alexei Starovoitovfdc15d32016-09-01 18:37:23 -07006769 return 0;
6770}
6771
Roman Gushchinb741f162018-09-28 14:45:43 +00006772static bool bpf_map_is_cgroup_storage(struct bpf_map *map)
6773{
6774 return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE ||
6775 map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE);
6776}
6777
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006778/* look for pseudo eBPF instructions that access map FDs and
6779 * replace them with actual map pointers
6780 */
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01006781static int replace_map_fd_with_map_ptr(struct bpf_verifier_env *env)
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006782{
6783 struct bpf_insn *insn = env->prog->insnsi;
6784 int insn_cnt = env->prog->len;
Alexei Starovoitovfdc15d32016-09-01 18:37:23 -07006785 int i, j, err;
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006786
Daniel Borkmannf1f77142017-01-13 23:38:15 +01006787 err = bpf_prog_calc_tag(env->prog);
Daniel Borkmannaafe6ae2016-12-18 01:52:57 +01006788 if (err)
6789 return err;
6790
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006791 for (i = 0; i < insn_cnt; i++, insn++) {
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07006792 if (BPF_CLASS(insn->code) == BPF_LDX &&
Alexei Starovoitovd691f9e2015-06-04 10:11:54 -07006793 (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006794 verbose(env, "BPF_LDX uses reserved fields\n");
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07006795 return -EINVAL;
6796 }
6797
Alexei Starovoitovd691f9e2015-06-04 10:11:54 -07006798 if (BPF_CLASS(insn->code) == BPF_STX &&
6799 ((BPF_MODE(insn->code) != BPF_MEM &&
6800 BPF_MODE(insn->code) != BPF_XADD) || insn->imm != 0)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006801 verbose(env, "BPF_STX uses reserved fields\n");
Alexei Starovoitovd691f9e2015-06-04 10:11:54 -07006802 return -EINVAL;
6803 }
6804
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006805 if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) {
6806 struct bpf_map *map;
6807 struct fd f;
6808
6809 if (i == insn_cnt - 1 || insn[1].code != 0 ||
6810 insn[1].dst_reg != 0 || insn[1].src_reg != 0 ||
6811 insn[1].off != 0) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006812 verbose(env, "invalid bpf_ld_imm64 insn\n");
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006813 return -EINVAL;
6814 }
6815
6816 if (insn->src_reg == 0)
6817 /* valid generic load 64-bit imm */
6818 goto next_insn;
6819
Daniel Borkmann20182392019-03-04 21:08:53 +01006820 if (insn[0].src_reg != BPF_PSEUDO_MAP_FD ||
6821 insn[1].imm != 0) {
6822 verbose(env, "unrecognized bpf_ld_imm64 insn\n");
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006823 return -EINVAL;
6824 }
6825
Daniel Borkmann20182392019-03-04 21:08:53 +01006826 f = fdget(insn[0].imm);
Daniel Borkmannc2101292015-10-29 14:58:07 +01006827 map = __bpf_map_get(f);
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006828 if (IS_ERR(map)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006829 verbose(env, "fd %d is not pointing to valid bpf_map\n",
Daniel Borkmann20182392019-03-04 21:08:53 +01006830 insn[0].imm);
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006831 return PTR_ERR(map);
6832 }
6833
Jakub Kicinski61bd5212017-10-09 10:30:11 -07006834 err = check_map_prog_compatibility(env, map, env->prog);
Alexei Starovoitovfdc15d32016-09-01 18:37:23 -07006835 if (err) {
6836 fdput(f);
6837 return err;
6838 }
6839
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006840 /* store map pointer inside BPF_LD_IMM64 instruction */
6841 insn[0].imm = (u32) (unsigned long) map;
6842 insn[1].imm = ((u64) (unsigned long) map) >> 32;
6843
6844 /* check whether we recorded this map already */
6845 for (j = 0; j < env->used_map_cnt; j++)
6846 if (env->used_maps[j] == map) {
6847 fdput(f);
6848 goto next_insn;
6849 }
6850
6851 if (env->used_map_cnt >= MAX_USED_MAPS) {
6852 fdput(f);
6853 return -E2BIG;
6854 }
6855
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006856 /* hold the map. If the program is rejected by verifier,
6857 * the map will be released by release_maps() or it
6858 * will be used by the valid program until it's unloaded
Jakub Kicinskiab7f5bf2018-05-03 18:37:17 -07006859 * and all maps are released in free_used_maps()
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006860 */
Alexei Starovoitov92117d82016-04-27 18:56:20 -07006861 map = bpf_map_inc(map, false);
6862 if (IS_ERR(map)) {
6863 fdput(f);
6864 return PTR_ERR(map);
6865 }
6866 env->used_maps[env->used_map_cnt++] = map;
6867
Roman Gushchinb741f162018-09-28 14:45:43 +00006868 if (bpf_map_is_cgroup_storage(map) &&
Roman Gushchinde9cbba2018-08-02 14:27:18 -07006869 bpf_cgroup_storage_assign(env->prog, map)) {
Roman Gushchinb741f162018-09-28 14:45:43 +00006870 verbose(env, "only one cgroup storage of each type is allowed\n");
Roman Gushchinde9cbba2018-08-02 14:27:18 -07006871 fdput(f);
6872 return -EBUSY;
6873 }
6874
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006875 fdput(f);
6876next_insn:
6877 insn++;
6878 i++;
Daniel Borkmann5e581da2018-01-26 23:33:38 +01006879 continue;
6880 }
6881
6882 /* Basic sanity check before we invest more work here. */
6883 if (!bpf_opcode_in_insntable(insn->code)) {
6884 verbose(env, "unknown opcode %02x\n", insn->code);
6885 return -EINVAL;
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006886 }
6887 }
6888
6889 /* now all pseudo BPF_LD_IMM64 instructions load valid
6890 * 'struct bpf_map *' into a register instead of user map_fd.
6891 * These pointers will be used later by verifier to validate map access.
6892 */
6893 return 0;
6894}
6895
6896/* drop refcnt of maps used by the rejected program */
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01006897static void release_maps(struct bpf_verifier_env *env)
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006898{
Roman Gushchin8bad74f2018-09-28 14:45:36 +00006899 enum bpf_cgroup_storage_type stype;
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006900 int i;
6901
Roman Gushchin8bad74f2018-09-28 14:45:36 +00006902 for_each_cgroup_storage_type(stype) {
6903 if (!env->prog->aux->cgroup_storage[stype])
6904 continue;
Roman Gushchinde9cbba2018-08-02 14:27:18 -07006905 bpf_cgroup_storage_release(env->prog,
Roman Gushchin8bad74f2018-09-28 14:45:36 +00006906 env->prog->aux->cgroup_storage[stype]);
6907 }
Roman Gushchinde9cbba2018-08-02 14:27:18 -07006908
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006909 for (i = 0; i < env->used_map_cnt; i++)
6910 bpf_map_put(env->used_maps[i]);
6911}
6912
6913/* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01006914static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env)
Alexei Starovoitov0246e642014-09-26 00:17:04 -07006915{
6916 struct bpf_insn *insn = env->prog->insnsi;
6917 int insn_cnt = env->prog->len;
6918 int i;
6919
6920 for (i = 0; i < insn_cnt; i++, insn++)
6921 if (insn->code == (BPF_LD | BPF_IMM | BPF_DW))
6922 insn->src_reg = 0;
6923}
6924
Alexei Starovoitov80419022017-03-15 18:26:41 -07006925/* single env->prog->insni[off] instruction was replaced with the range
6926 * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying
6927 * [0, off) and [off, end) to new locations, so the patched range stays zero
6928 */
6929static int adjust_insn_aux_data(struct bpf_verifier_env *env, u32 prog_len,
6930 u32 off, u32 cnt)
6931{
6932 struct bpf_insn_aux_data *new_data, *old_data = env->insn_aux_data;
Alexei Starovoitovc1311872017-11-22 16:42:05 -08006933 int i;
Alexei Starovoitov80419022017-03-15 18:26:41 -07006934
6935 if (cnt == 1)
6936 return 0;
Kees Cookfad953c2018-06-12 14:27:37 -07006937 new_data = vzalloc(array_size(prog_len,
6938 sizeof(struct bpf_insn_aux_data)));
Alexei Starovoitov80419022017-03-15 18:26:41 -07006939 if (!new_data)
6940 return -ENOMEM;
6941 memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off);
6942 memcpy(new_data + off + cnt - 1, old_data + off,
6943 sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1));
Alexei Starovoitovc1311872017-11-22 16:42:05 -08006944 for (i = off; i < off + cnt - 1; i++)
6945 new_data[i].seen = true;
Alexei Starovoitov80419022017-03-15 18:26:41 -07006946 env->insn_aux_data = new_data;
6947 vfree(old_data);
6948 return 0;
6949}
6950
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08006951static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)
6952{
6953 int i;
6954
6955 if (len == 1)
6956 return;
Jiong Wang4cb3d992018-05-02 16:17:19 -04006957 /* NOTE: fake 'exit' subprog should be updated as well. */
6958 for (i = 0; i <= env->subprog_cnt; i++) {
Edward Creeafd59422018-11-16 12:00:07 +00006959 if (env->subprog_info[i].start <= off)
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08006960 continue;
Jiong Wang9c8105b2018-05-02 16:17:18 -04006961 env->subprog_info[i].start += len - 1;
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08006962 }
6963}
6964
Alexei Starovoitov80419022017-03-15 18:26:41 -07006965static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
6966 const struct bpf_insn *patch, u32 len)
6967{
6968 struct bpf_prog *new_prog;
6969
6970 new_prog = bpf_patch_insn_single(env->prog, off, patch, len);
Alexei Starovoitov4f733792019-04-01 21:27:44 -07006971 if (IS_ERR(new_prog)) {
6972 if (PTR_ERR(new_prog) == -ERANGE)
6973 verbose(env,
6974 "insn %d cannot be patched due to 16-bit range\n",
6975 env->insn_aux_data[off].orig_idx);
Alexei Starovoitov80419022017-03-15 18:26:41 -07006976 return NULL;
Alexei Starovoitov4f733792019-04-01 21:27:44 -07006977 }
Alexei Starovoitov80419022017-03-15 18:26:41 -07006978 if (adjust_insn_aux_data(env, new_prog->len, off, len))
6979 return NULL;
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08006980 adjust_subprog_starts(env, off, len);
Alexei Starovoitov80419022017-03-15 18:26:41 -07006981 return new_prog;
6982}
6983
Jakub Kicinski52875a02019-01-22 22:45:20 -08006984static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env,
6985 u32 off, u32 cnt)
6986{
6987 int i, j;
6988
6989 /* find first prog starting at or after off (first to remove) */
6990 for (i = 0; i < env->subprog_cnt; i++)
6991 if (env->subprog_info[i].start >= off)
6992 break;
6993 /* find first prog starting at or after off + cnt (first to stay) */
6994 for (j = i; j < env->subprog_cnt; j++)
6995 if (env->subprog_info[j].start >= off + cnt)
6996 break;
6997 /* if j doesn't start exactly at off + cnt, we are just removing
6998 * the front of previous prog
6999 */
7000 if (env->subprog_info[j].start != off + cnt)
7001 j--;
7002
7003 if (j > i) {
7004 struct bpf_prog_aux *aux = env->prog->aux;
7005 int move;
7006
7007 /* move fake 'exit' subprog as well */
7008 move = env->subprog_cnt + 1 - j;
7009
7010 memmove(env->subprog_info + i,
7011 env->subprog_info + j,
7012 sizeof(*env->subprog_info) * move);
7013 env->subprog_cnt -= j - i;
7014
7015 /* remove func_info */
7016 if (aux->func_info) {
7017 move = aux->func_info_cnt - j;
7018
7019 memmove(aux->func_info + i,
7020 aux->func_info + j,
7021 sizeof(*aux->func_info) * move);
7022 aux->func_info_cnt -= j - i;
7023 /* func_info->insn_off is set after all code rewrites,
7024 * in adjust_btf_func() - no need to adjust
7025 */
7026 }
7027 } else {
7028 /* convert i from "first prog to remove" to "first to adjust" */
7029 if (env->subprog_info[i].start == off)
7030 i++;
7031 }
7032
7033 /* update fake 'exit' subprog as well */
7034 for (; i <= env->subprog_cnt; i++)
7035 env->subprog_info[i].start -= cnt;
7036
7037 return 0;
7038}
7039
7040static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off,
7041 u32 cnt)
7042{
7043 struct bpf_prog *prog = env->prog;
7044 u32 i, l_off, l_cnt, nr_linfo;
7045 struct bpf_line_info *linfo;
7046
7047 nr_linfo = prog->aux->nr_linfo;
7048 if (!nr_linfo)
7049 return 0;
7050
7051 linfo = prog->aux->linfo;
7052
7053 /* find first line info to remove, count lines to be removed */
7054 for (i = 0; i < nr_linfo; i++)
7055 if (linfo[i].insn_off >= off)
7056 break;
7057
7058 l_off = i;
7059 l_cnt = 0;
7060 for (; i < nr_linfo; i++)
7061 if (linfo[i].insn_off < off + cnt)
7062 l_cnt++;
7063 else
7064 break;
7065
7066 /* First live insn doesn't match first live linfo, it needs to "inherit"
7067 * last removed linfo. prog is already modified, so prog->len == off
7068 * means no live instructions after (tail of the program was removed).
7069 */
7070 if (prog->len != off && l_cnt &&
7071 (i == nr_linfo || linfo[i].insn_off != off + cnt)) {
7072 l_cnt--;
7073 linfo[--i].insn_off = off + cnt;
7074 }
7075
7076 /* remove the line info which refer to the removed instructions */
7077 if (l_cnt) {
7078 memmove(linfo + l_off, linfo + i,
7079 sizeof(*linfo) * (nr_linfo - i));
7080
7081 prog->aux->nr_linfo -= l_cnt;
7082 nr_linfo = prog->aux->nr_linfo;
7083 }
7084
7085 /* pull all linfo[i].insn_off >= off + cnt in by cnt */
7086 for (i = l_off; i < nr_linfo; i++)
7087 linfo[i].insn_off -= cnt;
7088
7089 /* fix up all subprogs (incl. 'exit') which start >= off */
7090 for (i = 0; i <= env->subprog_cnt; i++)
7091 if (env->subprog_info[i].linfo_idx > l_off) {
7092 /* program may have started in the removed region but
7093 * may not be fully removed
7094 */
7095 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt)
7096 env->subprog_info[i].linfo_idx -= l_cnt;
7097 else
7098 env->subprog_info[i].linfo_idx = l_off;
7099 }
7100
7101 return 0;
7102}
7103
7104static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)
7105{
7106 struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
7107 unsigned int orig_prog_len = env->prog->len;
7108 int err;
7109
Jakub Kicinski08ca90a2019-01-22 22:45:24 -08007110 if (bpf_prog_is_dev_bound(env->prog->aux))
7111 bpf_prog_offload_remove_insns(env, off, cnt);
7112
Jakub Kicinski52875a02019-01-22 22:45:20 -08007113 err = bpf_remove_insns(env->prog, off, cnt);
7114 if (err)
7115 return err;
7116
7117 err = adjust_subprog_starts_after_remove(env, off, cnt);
7118 if (err)
7119 return err;
7120
7121 err = bpf_adj_linfo_after_remove(env, off, cnt);
7122 if (err)
7123 return err;
7124
7125 memmove(aux_data + off, aux_data + off + cnt,
7126 sizeof(*aux_data) * (orig_prog_len - off - cnt));
7127
7128 return 0;
7129}
7130
Daniel Borkmann2a5418a2018-01-26 23:33:37 +01007131/* The verifier does more data flow analysis than llvm and will not
7132 * explore branches that are dead at run time. Malicious programs can
7133 * have dead code too. Therefore replace all dead at-run-time code
7134 * with 'ja -1'.
7135 *
7136 * Just nops are not optimal, e.g. if they would sit at the end of the
7137 * program and through another bug we would manage to jump there, then
7138 * we'd execute beyond program memory otherwise. Returning exception
7139 * code also wouldn't work since we can have subprogs where the dead
7140 * code could be located.
Alexei Starovoitovc1311872017-11-22 16:42:05 -08007141 */
7142static void sanitize_dead_code(struct bpf_verifier_env *env)
7143{
7144 struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
Daniel Borkmann2a5418a2018-01-26 23:33:37 +01007145 struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1);
Alexei Starovoitovc1311872017-11-22 16:42:05 -08007146 struct bpf_insn *insn = env->prog->insnsi;
7147 const int insn_cnt = env->prog->len;
7148 int i;
7149
7150 for (i = 0; i < insn_cnt; i++) {
7151 if (aux_data[i].seen)
7152 continue;
Daniel Borkmann2a5418a2018-01-26 23:33:37 +01007153 memcpy(insn + i, &trap, sizeof(trap));
Alexei Starovoitovc1311872017-11-22 16:42:05 -08007154 }
7155}
7156
Jakub Kicinskie2ae4ca2019-01-22 22:45:19 -08007157static bool insn_is_cond_jump(u8 code)
7158{
7159 u8 op;
7160
Jiong Wang092ed092019-01-26 12:26:01 -05007161 if (BPF_CLASS(code) == BPF_JMP32)
7162 return true;
7163
Jakub Kicinskie2ae4ca2019-01-22 22:45:19 -08007164 if (BPF_CLASS(code) != BPF_JMP)
7165 return false;
7166
7167 op = BPF_OP(code);
7168 return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL;
7169}
7170
7171static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env)
7172{
7173 struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
7174 struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0);
7175 struct bpf_insn *insn = env->prog->insnsi;
7176 const int insn_cnt = env->prog->len;
7177 int i;
7178
7179 for (i = 0; i < insn_cnt; i++, insn++) {
7180 if (!insn_is_cond_jump(insn->code))
7181 continue;
7182
7183 if (!aux_data[i + 1].seen)
7184 ja.off = insn->off;
7185 else if (!aux_data[i + 1 + insn->off].seen)
7186 ja.off = 0;
7187 else
7188 continue;
7189
Jakub Kicinski08ca90a2019-01-22 22:45:24 -08007190 if (bpf_prog_is_dev_bound(env->prog->aux))
7191 bpf_prog_offload_replace_insn(env, i, &ja);
7192
Jakub Kicinskie2ae4ca2019-01-22 22:45:19 -08007193 memcpy(insn, &ja, sizeof(ja));
7194 }
7195}
7196
Jakub Kicinski52875a02019-01-22 22:45:20 -08007197static int opt_remove_dead_code(struct bpf_verifier_env *env)
7198{
7199 struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
7200 int insn_cnt = env->prog->len;
7201 int i, err;
7202
7203 for (i = 0; i < insn_cnt; i++) {
7204 int j;
7205
7206 j = 0;
7207 while (i + j < insn_cnt && !aux_data[i + j].seen)
7208 j++;
7209 if (!j)
7210 continue;
7211
7212 err = verifier_remove_insns(env, i, j);
7213 if (err)
7214 return err;
7215 insn_cnt = env->prog->len;
7216 }
7217
7218 return 0;
7219}
7220
Jakub Kicinskia1b14ab2019-01-22 22:45:21 -08007221static int opt_remove_nops(struct bpf_verifier_env *env)
7222{
7223 const struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0);
7224 struct bpf_insn *insn = env->prog->insnsi;
7225 int insn_cnt = env->prog->len;
7226 int i, err;
7227
7228 for (i = 0; i < insn_cnt; i++) {
7229 if (memcmp(&insn[i], &ja, sizeof(ja)))
7230 continue;
7231
7232 err = verifier_remove_insns(env, i, 1);
7233 if (err)
7234 return err;
7235 insn_cnt--;
7236 i--;
7237 }
7238
7239 return 0;
7240}
7241
Joe Stringerc64b7982018-10-02 13:35:33 -07007242/* convert load instructions that access fields of a context type into a
7243 * sequence of instructions that access fields of the underlying structure:
7244 * struct __sk_buff -> struct sk_buff
7245 * struct bpf_sock_ops -> struct sock
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007246 */
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01007247static int convert_ctx_accesses(struct bpf_verifier_env *env)
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007248{
Jakub Kicinski00176a32017-10-16 16:40:54 -07007249 const struct bpf_verifier_ops *ops = env->ops;
Daniel Borkmannf96da092017-07-02 02:13:27 +02007250 int i, cnt, size, ctx_field_size, delta = 0;
Jakub Kicinski3df126f2016-09-21 11:43:56 +01007251 const int insn_cnt = env->prog->len;
Daniel Borkmann36bbef52016-09-20 00:26:13 +02007252 struct bpf_insn insn_buf[16], *insn;
Andrey Ignatov46f53a62018-11-10 22:15:13 -08007253 u32 target_size, size_default, off;
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007254 struct bpf_prog *new_prog;
Alexei Starovoitovd691f9e2015-06-04 10:11:54 -07007255 enum bpf_access_type type;
Daniel Borkmannf96da092017-07-02 02:13:27 +02007256 bool is_narrower_load;
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007257
Daniel Borkmannb09928b2018-10-24 22:05:49 +02007258 if (ops->gen_prologue || env->seen_direct_write) {
7259 if (!ops->gen_prologue) {
7260 verbose(env, "bpf verifier is misconfigured\n");
7261 return -EINVAL;
7262 }
Daniel Borkmann36bbef52016-09-20 00:26:13 +02007263 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write,
7264 env->prog);
7265 if (cnt >= ARRAY_SIZE(insn_buf)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07007266 verbose(env, "bpf verifier is misconfigured\n");
Daniel Borkmann36bbef52016-09-20 00:26:13 +02007267 return -EINVAL;
7268 } else if (cnt) {
Alexei Starovoitov80419022017-03-15 18:26:41 -07007269 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);
Daniel Borkmann36bbef52016-09-20 00:26:13 +02007270 if (!new_prog)
7271 return -ENOMEM;
Alexei Starovoitov80419022017-03-15 18:26:41 -07007272
Daniel Borkmann36bbef52016-09-20 00:26:13 +02007273 env->prog = new_prog;
Jakub Kicinski3df126f2016-09-21 11:43:56 +01007274 delta += cnt - 1;
Daniel Borkmann36bbef52016-09-20 00:26:13 +02007275 }
7276 }
7277
Joe Stringerc64b7982018-10-02 13:35:33 -07007278 if (bpf_prog_is_dev_bound(env->prog->aux))
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007279 return 0;
7280
Jakub Kicinski3df126f2016-09-21 11:43:56 +01007281 insn = env->prog->insnsi + delta;
Daniel Borkmann36bbef52016-09-20 00:26:13 +02007282
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007283 for (i = 0; i < insn_cnt; i++, insn++) {
Joe Stringerc64b7982018-10-02 13:35:33 -07007284 bpf_convert_ctx_access_t convert_ctx_access;
7285
Daniel Borkmann62c79892017-01-12 11:51:33 +01007286 if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) ||
7287 insn->code == (BPF_LDX | BPF_MEM | BPF_H) ||
7288 insn->code == (BPF_LDX | BPF_MEM | BPF_W) ||
Alexei Starovoitovea2e7ce2016-09-01 18:37:21 -07007289 insn->code == (BPF_LDX | BPF_MEM | BPF_DW))
Alexei Starovoitovd691f9e2015-06-04 10:11:54 -07007290 type = BPF_READ;
Daniel Borkmann62c79892017-01-12 11:51:33 +01007291 else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) ||
7292 insn->code == (BPF_STX | BPF_MEM | BPF_H) ||
7293 insn->code == (BPF_STX | BPF_MEM | BPF_W) ||
Alexei Starovoitovea2e7ce2016-09-01 18:37:21 -07007294 insn->code == (BPF_STX | BPF_MEM | BPF_DW))
Alexei Starovoitovd691f9e2015-06-04 10:11:54 -07007295 type = BPF_WRITE;
7296 else
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007297 continue;
7298
Alexei Starovoitovaf86ca42018-05-15 09:27:05 -07007299 if (type == BPF_WRITE &&
7300 env->insn_aux_data[i + delta].sanitize_stack_off) {
7301 struct bpf_insn patch[] = {
7302 /* Sanitize suspicious stack slot with zero.
7303 * There are no memory dependencies for this store,
7304 * since it's only using frame pointer and immediate
7305 * constant of zero
7306 */
7307 BPF_ST_MEM(BPF_DW, BPF_REG_FP,
7308 env->insn_aux_data[i + delta].sanitize_stack_off,
7309 0),
7310 /* the original STX instruction will immediately
7311 * overwrite the same stack slot with appropriate value
7312 */
7313 *insn,
7314 };
7315
7316 cnt = ARRAY_SIZE(patch);
7317 new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt);
7318 if (!new_prog)
7319 return -ENOMEM;
7320
7321 delta += cnt - 1;
7322 env->prog = new_prog;
7323 insn = new_prog->insnsi + i + delta;
7324 continue;
7325 }
7326
Joe Stringerc64b7982018-10-02 13:35:33 -07007327 switch (env->insn_aux_data[i + delta].ptr_type) {
7328 case PTR_TO_CTX:
7329 if (!ops->convert_ctx_access)
7330 continue;
7331 convert_ctx_access = ops->convert_ctx_access;
7332 break;
7333 case PTR_TO_SOCKET:
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08007334 case PTR_TO_SOCK_COMMON:
Joe Stringerc64b7982018-10-02 13:35:33 -07007335 convert_ctx_access = bpf_sock_convert_ctx_access;
7336 break;
Martin KaFai Lau655a51e2019-02-09 23:22:24 -08007337 case PTR_TO_TCP_SOCK:
7338 convert_ctx_access = bpf_tcp_sock_convert_ctx_access;
7339 break;
Joe Stringerc64b7982018-10-02 13:35:33 -07007340 default:
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007341 continue;
Joe Stringerc64b7982018-10-02 13:35:33 -07007342 }
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007343
Yonghong Song31fd8582017-06-13 15:52:13 -07007344 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size;
Daniel Borkmannf96da092017-07-02 02:13:27 +02007345 size = BPF_LDST_BYTES(insn);
Yonghong Song31fd8582017-06-13 15:52:13 -07007346
7347 /* If the read access is a narrower load of the field,
7348 * convert to a 4/8-byte load, to minimum program type specific
7349 * convert_ctx_access changes. If conversion is successful,
7350 * we will apply proper mask to the result.
7351 */
Daniel Borkmannf96da092017-07-02 02:13:27 +02007352 is_narrower_load = size < ctx_field_size;
Andrey Ignatov46f53a62018-11-10 22:15:13 -08007353 size_default = bpf_ctx_off_adjust_machine(ctx_field_size);
7354 off = insn->off;
Yonghong Song31fd8582017-06-13 15:52:13 -07007355 if (is_narrower_load) {
Daniel Borkmannf96da092017-07-02 02:13:27 +02007356 u8 size_code;
Yonghong Song31fd8582017-06-13 15:52:13 -07007357
Daniel Borkmannf96da092017-07-02 02:13:27 +02007358 if (type == BPF_WRITE) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07007359 verbose(env, "bpf verifier narrow ctx access misconfigured\n");
Daniel Borkmannf96da092017-07-02 02:13:27 +02007360 return -EINVAL;
7361 }
7362
7363 size_code = BPF_H;
Yonghong Song31fd8582017-06-13 15:52:13 -07007364 if (ctx_field_size == 4)
7365 size_code = BPF_W;
7366 else if (ctx_field_size == 8)
7367 size_code = BPF_DW;
Daniel Borkmannf96da092017-07-02 02:13:27 +02007368
Daniel Borkmannbc231052018-06-02 23:06:39 +02007369 insn->off = off & ~(size_default - 1);
Yonghong Song31fd8582017-06-13 15:52:13 -07007370 insn->code = BPF_LDX | BPF_MEM | size_code;
7371 }
Daniel Borkmannf96da092017-07-02 02:13:27 +02007372
7373 target_size = 0;
Joe Stringerc64b7982018-10-02 13:35:33 -07007374 cnt = convert_ctx_access(type, insn, insn_buf, env->prog,
7375 &target_size);
Daniel Borkmannf96da092017-07-02 02:13:27 +02007376 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf) ||
7377 (ctx_field_size && !target_size)) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07007378 verbose(env, "bpf verifier is misconfigured\n");
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007379 return -EINVAL;
7380 }
Daniel Borkmannf96da092017-07-02 02:13:27 +02007381
7382 if (is_narrower_load && size < target_size) {
Andrey Ignatov46f53a62018-11-10 22:15:13 -08007383 u8 shift = (off & (size_default - 1)) * 8;
7384
7385 if (ctx_field_size <= 4) {
7386 if (shift)
7387 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH,
7388 insn->dst_reg,
7389 shift);
Yonghong Song31fd8582017-06-13 15:52:13 -07007390 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg,
Daniel Borkmannf96da092017-07-02 02:13:27 +02007391 (1 << size * 8) - 1);
Andrey Ignatov46f53a62018-11-10 22:15:13 -08007392 } else {
7393 if (shift)
7394 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH,
7395 insn->dst_reg,
7396 shift);
Yonghong Song31fd8582017-06-13 15:52:13 -07007397 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_AND, insn->dst_reg,
Daniel Borkmannf96da092017-07-02 02:13:27 +02007398 (1 << size * 8) - 1);
Andrey Ignatov46f53a62018-11-10 22:15:13 -08007399 }
Yonghong Song31fd8582017-06-13 15:52:13 -07007400 }
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007401
Alexei Starovoitov80419022017-03-15 18:26:41 -07007402 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007403 if (!new_prog)
7404 return -ENOMEM;
7405
Jakub Kicinski3df126f2016-09-21 11:43:56 +01007406 delta += cnt - 1;
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007407
7408 /* keep walking new program and skip insns we just inserted */
7409 env->prog = new_prog;
Jakub Kicinski3df126f2016-09-21 11:43:56 +01007410 insn = new_prog->insnsi + i + delta;
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007411 }
7412
7413 return 0;
7414}
7415
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007416static int jit_subprogs(struct bpf_verifier_env *env)
7417{
7418 struct bpf_prog *prog = env->prog, **func, *tmp;
7419 int i, j, subprog_start, subprog_end = 0, len, subprog;
Daniel Borkmann7105e822017-12-20 13:42:57 +01007420 struct bpf_insn *insn;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007421 void *old_bpf_func;
Martin KaFai Lauc454a462018-12-07 16:42:25 -08007422 int err;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007423
Jiong Wangf910cef2018-05-02 16:17:17 -04007424 if (env->subprog_cnt <= 1)
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007425 return 0;
7426
Daniel Borkmann7105e822017-12-20 13:42:57 +01007427 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007428 if (insn->code != (BPF_JMP | BPF_CALL) ||
7429 insn->src_reg != BPF_PSEUDO_CALL)
7430 continue;
Daniel Borkmannc7a89782018-07-12 21:44:28 +02007431 /* Upon error here we cannot fall back to interpreter but
7432 * need a hard reject of the program. Thus -EFAULT is
7433 * propagated in any case.
7434 */
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007435 subprog = find_subprog(env, i + insn->imm + 1);
7436 if (subprog < 0) {
7437 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
7438 i + insn->imm + 1);
7439 return -EFAULT;
7440 }
7441 /* temporarily remember subprog id inside insn instead of
7442 * aux_data, since next loop will split up all insns into funcs
7443 */
Jiong Wangf910cef2018-05-02 16:17:17 -04007444 insn->off = subprog;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007445 /* remember original imm in case JIT fails and fallback
7446 * to interpreter will be needed
7447 */
7448 env->insn_aux_data[i].call_imm = insn->imm;
7449 /* point imm to __bpf_call_base+1 from JITs point of view */
7450 insn->imm = 1;
7451 }
7452
Martin KaFai Lauc454a462018-12-07 16:42:25 -08007453 err = bpf_prog_alloc_jited_linfo(prog);
7454 if (err)
7455 goto out_undo_insn;
7456
7457 err = -ENOMEM;
Kees Cook6396bb22018-06-12 14:03:40 -07007458 func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL);
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007459 if (!func)
Daniel Borkmannc7a89782018-07-12 21:44:28 +02007460 goto out_undo_insn;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007461
Jiong Wangf910cef2018-05-02 16:17:17 -04007462 for (i = 0; i < env->subprog_cnt; i++) {
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007463 subprog_start = subprog_end;
Jiong Wang4cb3d992018-05-02 16:17:19 -04007464 subprog_end = env->subprog_info[i + 1].start;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007465
7466 len = subprog_end - subprog_start;
Alexei Starovoitov492ecee2019-02-25 14:28:39 -08007467 /* BPF_PROG_RUN doesn't call subprogs directly,
7468 * hence main prog stats include the runtime of subprogs.
7469 * subprogs don't have IDs and not reachable via prog_get_next_id
7470 * func[i]->aux->stats will never be accessed and stays NULL
7471 */
7472 func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER);
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007473 if (!func[i])
7474 goto out_free;
7475 memcpy(func[i]->insnsi, &prog->insnsi[subprog_start],
7476 len * sizeof(struct bpf_insn));
Daniel Borkmann4f74d802017-12-20 13:42:56 +01007477 func[i]->type = prog->type;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007478 func[i]->len = len;
Daniel Borkmann4f74d802017-12-20 13:42:56 +01007479 if (bpf_prog_calc_tag(func[i]))
7480 goto out_free;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007481 func[i]->is_func = 1;
Yonghong Songba64e7d2018-11-24 23:20:44 -08007482 func[i]->aux->func_idx = i;
7483 /* the btf and func_info will be freed only at prog->aux */
7484 func[i]->aux->btf = prog->aux->btf;
7485 func[i]->aux->func_info = prog->aux->func_info;
7486
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007487 /* Use bpf_prog_F_tag to indicate functions in stack traces.
7488 * Long term would need debug info to populate names
7489 */
7490 func[i]->aux->name[0] = 'F';
Jiong Wang9c8105b2018-05-02 16:17:18 -04007491 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007492 func[i]->jit_requested = 1;
Martin KaFai Lauc454a462018-12-07 16:42:25 -08007493 func[i]->aux->linfo = prog->aux->linfo;
7494 func[i]->aux->nr_linfo = prog->aux->nr_linfo;
7495 func[i]->aux->jited_linfo = prog->aux->jited_linfo;
7496 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007497 func[i] = bpf_int_jit_compile(func[i]);
7498 if (!func[i]->jited) {
7499 err = -ENOTSUPP;
7500 goto out_free;
7501 }
7502 cond_resched();
7503 }
7504 /* at this point all bpf functions were successfully JITed
7505 * now populate all bpf_calls with correct addresses and
7506 * run last pass of JIT
7507 */
Jiong Wangf910cef2018-05-02 16:17:17 -04007508 for (i = 0; i < env->subprog_cnt; i++) {
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007509 insn = func[i]->insnsi;
7510 for (j = 0; j < func[i]->len; j++, insn++) {
7511 if (insn->code != (BPF_JMP | BPF_CALL) ||
7512 insn->src_reg != BPF_PSEUDO_CALL)
7513 continue;
7514 subprog = insn->off;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007515 insn->imm = (u64 (*)(u64, u64, u64, u64, u64))
7516 func[subprog]->bpf_func -
7517 __bpf_call_base;
7518 }
Sandipan Das2162fed2018-05-24 12:26:45 +05307519
7520 /* we use the aux data to keep a list of the start addresses
7521 * of the JITed images for each function in the program
7522 *
7523 * for some architectures, such as powerpc64, the imm field
7524 * might not be large enough to hold the offset of the start
7525 * address of the callee's JITed image from __bpf_call_base
7526 *
7527 * in such cases, we can lookup the start address of a callee
7528 * by using its subprog id, available from the off field of
7529 * the call instruction, as an index for this list
7530 */
7531 func[i]->aux->func = func;
7532 func[i]->aux->func_cnt = env->subprog_cnt;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007533 }
Jiong Wangf910cef2018-05-02 16:17:17 -04007534 for (i = 0; i < env->subprog_cnt; i++) {
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007535 old_bpf_func = func[i]->bpf_func;
7536 tmp = bpf_int_jit_compile(func[i]);
7537 if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) {
7538 verbose(env, "JIT doesn't support bpf-to-bpf calls\n");
Daniel Borkmannc7a89782018-07-12 21:44:28 +02007539 err = -ENOTSUPP;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007540 goto out_free;
7541 }
7542 cond_resched();
7543 }
7544
7545 /* finally lock prog and jit images for all functions and
7546 * populate kallsysm
7547 */
Jiong Wangf910cef2018-05-02 16:17:17 -04007548 for (i = 0; i < env->subprog_cnt; i++) {
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007549 bpf_prog_lock_ro(func[i]);
7550 bpf_prog_kallsyms_add(func[i]);
7551 }
Daniel Borkmann7105e822017-12-20 13:42:57 +01007552
7553 /* Last step: make now unused interpreter insns from main
7554 * prog consistent for later dump requests, so they can
7555 * later look the same as if they were interpreted only.
7556 */
7557 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
Daniel Borkmann7105e822017-12-20 13:42:57 +01007558 if (insn->code != (BPF_JMP | BPF_CALL) ||
7559 insn->src_reg != BPF_PSEUDO_CALL)
7560 continue;
7561 insn->off = env->insn_aux_data[i].call_imm;
7562 subprog = find_subprog(env, i + insn->off + 1);
Sandipan Dasdbecd732018-05-24 12:26:48 +05307563 insn->imm = subprog;
Daniel Borkmann7105e822017-12-20 13:42:57 +01007564 }
7565
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007566 prog->jited = 1;
7567 prog->bpf_func = func[0]->bpf_func;
7568 prog->aux->func = func;
Jiong Wangf910cef2018-05-02 16:17:17 -04007569 prog->aux->func_cnt = env->subprog_cnt;
Martin KaFai Lauc454a462018-12-07 16:42:25 -08007570 bpf_prog_free_unused_jited_linfo(prog);
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007571 return 0;
7572out_free:
Jiong Wangf910cef2018-05-02 16:17:17 -04007573 for (i = 0; i < env->subprog_cnt; i++)
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007574 if (func[i])
7575 bpf_jit_free(func[i]);
7576 kfree(func);
Daniel Borkmannc7a89782018-07-12 21:44:28 +02007577out_undo_insn:
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007578 /* cleanup main prog to be interpreted */
7579 prog->jit_requested = 0;
7580 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
7581 if (insn->code != (BPF_JMP | BPF_CALL) ||
7582 insn->src_reg != BPF_PSEUDO_CALL)
7583 continue;
7584 insn->off = 0;
7585 insn->imm = env->insn_aux_data[i].call_imm;
7586 }
Martin KaFai Lauc454a462018-12-07 16:42:25 -08007587 bpf_prog_free_jited_linfo(prog);
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007588 return err;
7589}
7590
Alexei Starovoitov1ea47e02017-12-14 17:55:13 -08007591static int fixup_call_args(struct bpf_verifier_env *env)
7592{
David S. Miller19d28fb2018-01-11 21:27:54 -05007593#ifndef CONFIG_BPF_JIT_ALWAYS_ON
Alexei Starovoitov1ea47e02017-12-14 17:55:13 -08007594 struct bpf_prog *prog = env->prog;
7595 struct bpf_insn *insn = prog->insnsi;
7596 int i, depth;
David S. Miller19d28fb2018-01-11 21:27:54 -05007597#endif
Quentin Monnete4052d02018-10-07 12:56:58 +01007598 int err = 0;
Alexei Starovoitov1ea47e02017-12-14 17:55:13 -08007599
Quentin Monnete4052d02018-10-07 12:56:58 +01007600 if (env->prog->jit_requested &&
7601 !bpf_prog_is_dev_bound(env->prog->aux)) {
David S. Miller19d28fb2018-01-11 21:27:54 -05007602 err = jit_subprogs(env);
7603 if (err == 0)
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -08007604 return 0;
Daniel Borkmannc7a89782018-07-12 21:44:28 +02007605 if (err == -EFAULT)
7606 return err;
David S. Miller19d28fb2018-01-11 21:27:54 -05007607 }
7608#ifndef CONFIG_BPF_JIT_ALWAYS_ON
Alexei Starovoitov1ea47e02017-12-14 17:55:13 -08007609 for (i = 0; i < prog->len; i++, insn++) {
7610 if (insn->code != (BPF_JMP | BPF_CALL) ||
7611 insn->src_reg != BPF_PSEUDO_CALL)
7612 continue;
7613 depth = get_callee_stack_depth(env, insn, i);
7614 if (depth < 0)
7615 return depth;
7616 bpf_patch_call_args(insn, depth);
7617 }
David S. Miller19d28fb2018-01-11 21:27:54 -05007618 err = 0;
7619#endif
7620 return err;
Alexei Starovoitov1ea47e02017-12-14 17:55:13 -08007621}
7622
Alexei Starovoitov79741b32017-03-15 18:26:40 -07007623/* fixup insn->imm field of bpf_call instructions
Alexei Starovoitov81ed18a2017-03-15 18:26:42 -07007624 * and inline eligible helpers as explicit sequence of BPF instructions
Alexei Starovoitove245c5c62017-03-15 18:26:39 -07007625 *
7626 * this function is called after eBPF program passed verification
7627 */
Alexei Starovoitov79741b32017-03-15 18:26:40 -07007628static int fixup_bpf_calls(struct bpf_verifier_env *env)
Alexei Starovoitove245c5c62017-03-15 18:26:39 -07007629{
Alexei Starovoitov79741b32017-03-15 18:26:40 -07007630 struct bpf_prog *prog = env->prog;
7631 struct bpf_insn *insn = prog->insnsi;
Alexei Starovoitove245c5c62017-03-15 18:26:39 -07007632 const struct bpf_func_proto *fn;
Alexei Starovoitov79741b32017-03-15 18:26:40 -07007633 const int insn_cnt = prog->len;
Daniel Borkmann09772d92018-06-02 23:06:35 +02007634 const struct bpf_map_ops *ops;
Daniel Borkmannc93552c2018-05-24 02:32:53 +02007635 struct bpf_insn_aux_data *aux;
Alexei Starovoitov81ed18a2017-03-15 18:26:42 -07007636 struct bpf_insn insn_buf[16];
7637 struct bpf_prog *new_prog;
7638 struct bpf_map *map_ptr;
7639 int i, cnt, delta = 0;
Alexei Starovoitove245c5c62017-03-15 18:26:39 -07007640
Alexei Starovoitov79741b32017-03-15 18:26:40 -07007641 for (i = 0; i < insn_cnt; i++, insn++) {
Daniel Borkmannf6b1b3b2018-01-26 23:33:39 +01007642 if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) ||
7643 insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) ||
7644 insn->code == (BPF_ALU | BPF_MOD | BPF_X) ||
Alexei Starovoitov68fda452018-01-12 18:59:52 -08007645 insn->code == (BPF_ALU | BPF_DIV | BPF_X)) {
Daniel Borkmannf6b1b3b2018-01-26 23:33:39 +01007646 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;
7647 struct bpf_insn mask_and_div[] = {
7648 BPF_MOV32_REG(insn->src_reg, insn->src_reg),
7649 /* Rx div 0 -> 0 */
7650 BPF_JMP_IMM(BPF_JNE, insn->src_reg, 0, 2),
7651 BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg),
7652 BPF_JMP_IMM(BPF_JA, 0, 0, 1),
7653 *insn,
7654 };
7655 struct bpf_insn mask_and_mod[] = {
7656 BPF_MOV32_REG(insn->src_reg, insn->src_reg),
7657 /* Rx mod 0 -> Rx */
7658 BPF_JMP_IMM(BPF_JEQ, insn->src_reg, 0, 1),
7659 *insn,
7660 };
7661 struct bpf_insn *patchlet;
7662
7663 if (insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) ||
7664 insn->code == (BPF_ALU | BPF_DIV | BPF_X)) {
7665 patchlet = mask_and_div + (is64 ? 1 : 0);
7666 cnt = ARRAY_SIZE(mask_and_div) - (is64 ? 1 : 0);
7667 } else {
7668 patchlet = mask_and_mod + (is64 ? 1 : 0);
7669 cnt = ARRAY_SIZE(mask_and_mod) - (is64 ? 1 : 0);
7670 }
7671
7672 new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt);
Alexei Starovoitov68fda452018-01-12 18:59:52 -08007673 if (!new_prog)
7674 return -ENOMEM;
7675
7676 delta += cnt - 1;
7677 env->prog = prog = new_prog;
7678 insn = new_prog->insnsi + i + delta;
7679 continue;
7680 }
7681
Daniel Borkmanne0cea7c2018-05-04 01:08:14 +02007682 if (BPF_CLASS(insn->code) == BPF_LD &&
7683 (BPF_MODE(insn->code) == BPF_ABS ||
7684 BPF_MODE(insn->code) == BPF_IND)) {
7685 cnt = env->ops->gen_ld_abs(insn, insn_buf);
7686 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) {
7687 verbose(env, "bpf verifier is misconfigured\n");
7688 return -EINVAL;
7689 }
7690
7691 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
7692 if (!new_prog)
7693 return -ENOMEM;
7694
7695 delta += cnt - 1;
7696 env->prog = prog = new_prog;
7697 insn = new_prog->insnsi + i + delta;
7698 continue;
7699 }
7700
Daniel Borkmann979d63d2019-01-03 00:58:34 +01007701 if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) ||
7702 insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) {
7703 const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X;
7704 const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X;
7705 struct bpf_insn insn_buf[16];
7706 struct bpf_insn *patch = &insn_buf[0];
7707 bool issrc, isneg;
7708 u32 off_reg;
7709
7710 aux = &env->insn_aux_data[i + delta];
Daniel Borkmann3612af72019-03-01 22:05:29 +01007711 if (!aux->alu_state ||
7712 aux->alu_state == BPF_ALU_NON_POINTER)
Daniel Borkmann979d63d2019-01-03 00:58:34 +01007713 continue;
7714
7715 isneg = aux->alu_state & BPF_ALU_NEG_VALUE;
7716 issrc = (aux->alu_state & BPF_ALU_SANITIZE) ==
7717 BPF_ALU_SANITIZE_SRC;
7718
7719 off_reg = issrc ? insn->src_reg : insn->dst_reg;
7720 if (isneg)
7721 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
7722 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit - 1);
7723 *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg);
7724 *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg);
7725 *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0);
7726 *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63);
7727 if (issrc) {
7728 *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX,
7729 off_reg);
7730 insn->src_reg = BPF_REG_AX;
7731 } else {
7732 *patch++ = BPF_ALU64_REG(BPF_AND, off_reg,
7733 BPF_REG_AX);
7734 }
7735 if (isneg)
7736 insn->code = insn->code == code_add ?
7737 code_sub : code_add;
7738 *patch++ = *insn;
7739 if (issrc && isneg)
7740 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
7741 cnt = patch - insn_buf;
7742
7743 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
7744 if (!new_prog)
7745 return -ENOMEM;
7746
7747 delta += cnt - 1;
7748 env->prog = prog = new_prog;
7749 insn = new_prog->insnsi + i + delta;
7750 continue;
7751 }
7752
Alexei Starovoitov79741b32017-03-15 18:26:40 -07007753 if (insn->code != (BPF_JMP | BPF_CALL))
7754 continue;
Alexei Starovoitovcc8b0b92017-12-14 17:55:05 -08007755 if (insn->src_reg == BPF_PSEUDO_CALL)
7756 continue;
Alexei Starovoitove245c5c62017-03-15 18:26:39 -07007757
Alexei Starovoitov79741b32017-03-15 18:26:40 -07007758 if (insn->imm == BPF_FUNC_get_route_realm)
7759 prog->dst_needed = 1;
7760 if (insn->imm == BPF_FUNC_get_prandom_u32)
7761 bpf_user_rnd_init_once();
Josef Bacik9802d862017-12-11 11:36:48 -05007762 if (insn->imm == BPF_FUNC_override_return)
7763 prog->kprobe_override = 1;
Alexei Starovoitov79741b32017-03-15 18:26:40 -07007764 if (insn->imm == BPF_FUNC_tail_call) {
David S. Miller7b9f6da2017-04-20 10:35:33 -04007765 /* If we tail call into other programs, we
7766 * cannot make any assumptions since they can
7767 * be replaced dynamically during runtime in
7768 * the program array.
7769 */
7770 prog->cb_access = 1;
Alexei Starovoitov80a58d02017-05-30 13:31:30 -07007771 env->prog->aux->stack_depth = MAX_BPF_STACK;
Jiong Wange6478152018-11-08 04:08:42 -05007772 env->prog->aux->max_pkt_offset = MAX_PACKET_OFF;
David S. Miller7b9f6da2017-04-20 10:35:33 -04007773
Alexei Starovoitov79741b32017-03-15 18:26:40 -07007774 /* mark bpf_tail_call as different opcode to avoid
7775 * conditional branch in the interpeter for every normal
7776 * call and to prevent accidental JITing by JIT compiler
7777 * that doesn't support bpf_tail_call yet
Alexei Starovoitove245c5c62017-03-15 18:26:39 -07007778 */
Alexei Starovoitov79741b32017-03-15 18:26:40 -07007779 insn->imm = 0;
Alexei Starovoitov71189fa2017-05-30 13:31:27 -07007780 insn->code = BPF_JMP | BPF_TAIL_CALL;
Alexei Starovoitovb2157392018-01-07 17:33:02 -08007781
Daniel Borkmannc93552c2018-05-24 02:32:53 +02007782 aux = &env->insn_aux_data[i + delta];
7783 if (!bpf_map_ptr_unpriv(aux))
7784 continue;
7785
Alexei Starovoitovb2157392018-01-07 17:33:02 -08007786 /* instead of changing every JIT dealing with tail_call
7787 * emit two extra insns:
7788 * if (index >= max_entries) goto out;
7789 * index &= array->index_mask;
7790 * to avoid out-of-bounds cpu speculation
7791 */
Daniel Borkmannc93552c2018-05-24 02:32:53 +02007792 if (bpf_map_ptr_poisoned(aux)) {
Colin Ian King40950342018-01-10 09:20:54 +00007793 verbose(env, "tail_call abusing map_ptr\n");
Alexei Starovoitovb2157392018-01-07 17:33:02 -08007794 return -EINVAL;
7795 }
Daniel Borkmannc93552c2018-05-24 02:32:53 +02007796
7797 map_ptr = BPF_MAP_PTR(aux->map_state);
Alexei Starovoitovb2157392018-01-07 17:33:02 -08007798 insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3,
7799 map_ptr->max_entries, 2);
7800 insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3,
7801 container_of(map_ptr,
7802 struct bpf_array,
7803 map)->index_mask);
7804 insn_buf[2] = *insn;
7805 cnt = 3;
7806 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
7807 if (!new_prog)
7808 return -ENOMEM;
7809
7810 delta += cnt - 1;
7811 env->prog = prog = new_prog;
7812 insn = new_prog->insnsi + i + delta;
Alexei Starovoitov79741b32017-03-15 18:26:40 -07007813 continue;
Alexei Starovoitove245c5c62017-03-15 18:26:39 -07007814 }
Alexei Starovoitove245c5c62017-03-15 18:26:39 -07007815
Daniel Borkmann89c63072017-08-19 03:12:45 +02007816 /* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup
Daniel Borkmann09772d92018-06-02 23:06:35 +02007817 * and other inlining handlers are currently limited to 64 bit
7818 * only.
Daniel Borkmann89c63072017-08-19 03:12:45 +02007819 */
Alexei Starovoitov60b58afc2017-12-14 17:55:14 -08007820 if (prog->jit_requested && BITS_PER_LONG == 64 &&
Daniel Borkmann09772d92018-06-02 23:06:35 +02007821 (insn->imm == BPF_FUNC_map_lookup_elem ||
7822 insn->imm == BPF_FUNC_map_update_elem ||
Daniel Borkmann84430d42018-10-21 02:09:27 +02007823 insn->imm == BPF_FUNC_map_delete_elem ||
7824 insn->imm == BPF_FUNC_map_push_elem ||
7825 insn->imm == BPF_FUNC_map_pop_elem ||
7826 insn->imm == BPF_FUNC_map_peek_elem)) {
Daniel Borkmannc93552c2018-05-24 02:32:53 +02007827 aux = &env->insn_aux_data[i + delta];
7828 if (bpf_map_ptr_poisoned(aux))
7829 goto patch_call_imm;
7830
7831 map_ptr = BPF_MAP_PTR(aux->map_state);
Daniel Borkmann09772d92018-06-02 23:06:35 +02007832 ops = map_ptr->ops;
7833 if (insn->imm == BPF_FUNC_map_lookup_elem &&
7834 ops->map_gen_lookup) {
7835 cnt = ops->map_gen_lookup(map_ptr, insn_buf);
7836 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) {
7837 verbose(env, "bpf verifier is misconfigured\n");
7838 return -EINVAL;
7839 }
Alexei Starovoitov81ed18a2017-03-15 18:26:42 -07007840
Daniel Borkmann09772d92018-06-02 23:06:35 +02007841 new_prog = bpf_patch_insn_data(env, i + delta,
7842 insn_buf, cnt);
7843 if (!new_prog)
7844 return -ENOMEM;
7845
7846 delta += cnt - 1;
7847 env->prog = prog = new_prog;
7848 insn = new_prog->insnsi + i + delta;
7849 continue;
Alexei Starovoitov81ed18a2017-03-15 18:26:42 -07007850 }
7851
Daniel Borkmann09772d92018-06-02 23:06:35 +02007852 BUILD_BUG_ON(!__same_type(ops->map_lookup_elem,
7853 (void *(*)(struct bpf_map *map, void *key))NULL));
7854 BUILD_BUG_ON(!__same_type(ops->map_delete_elem,
7855 (int (*)(struct bpf_map *map, void *key))NULL));
7856 BUILD_BUG_ON(!__same_type(ops->map_update_elem,
7857 (int (*)(struct bpf_map *map, void *key, void *value,
7858 u64 flags))NULL));
Daniel Borkmann84430d42018-10-21 02:09:27 +02007859 BUILD_BUG_ON(!__same_type(ops->map_push_elem,
7860 (int (*)(struct bpf_map *map, void *value,
7861 u64 flags))NULL));
7862 BUILD_BUG_ON(!__same_type(ops->map_pop_elem,
7863 (int (*)(struct bpf_map *map, void *value))NULL));
7864 BUILD_BUG_ON(!__same_type(ops->map_peek_elem,
7865 (int (*)(struct bpf_map *map, void *value))NULL));
7866
Daniel Borkmann09772d92018-06-02 23:06:35 +02007867 switch (insn->imm) {
7868 case BPF_FUNC_map_lookup_elem:
7869 insn->imm = BPF_CAST_CALL(ops->map_lookup_elem) -
7870 __bpf_call_base;
7871 continue;
7872 case BPF_FUNC_map_update_elem:
7873 insn->imm = BPF_CAST_CALL(ops->map_update_elem) -
7874 __bpf_call_base;
7875 continue;
7876 case BPF_FUNC_map_delete_elem:
7877 insn->imm = BPF_CAST_CALL(ops->map_delete_elem) -
7878 __bpf_call_base;
7879 continue;
Daniel Borkmann84430d42018-10-21 02:09:27 +02007880 case BPF_FUNC_map_push_elem:
7881 insn->imm = BPF_CAST_CALL(ops->map_push_elem) -
7882 __bpf_call_base;
7883 continue;
7884 case BPF_FUNC_map_pop_elem:
7885 insn->imm = BPF_CAST_CALL(ops->map_pop_elem) -
7886 __bpf_call_base;
7887 continue;
7888 case BPF_FUNC_map_peek_elem:
7889 insn->imm = BPF_CAST_CALL(ops->map_peek_elem) -
7890 __bpf_call_base;
7891 continue;
Daniel Borkmann09772d92018-06-02 23:06:35 +02007892 }
Alexei Starovoitov81ed18a2017-03-15 18:26:42 -07007893
Daniel Borkmann09772d92018-06-02 23:06:35 +02007894 goto patch_call_imm;
Alexei Starovoitov81ed18a2017-03-15 18:26:42 -07007895 }
7896
7897patch_call_imm:
Andrey Ignatov5e43f892018-03-30 15:08:00 -07007898 fn = env->ops->get_func_proto(insn->imm, env->prog);
Alexei Starovoitov79741b32017-03-15 18:26:40 -07007899 /* all functions that have prototype and verifier allowed
7900 * programs to call them, must be real in-kernel functions
7901 */
7902 if (!fn->func) {
Jakub Kicinski61bd5212017-10-09 10:30:11 -07007903 verbose(env,
7904 "kernel subsystem misconfigured func %s#%d\n",
Alexei Starovoitov79741b32017-03-15 18:26:40 -07007905 func_id_name(insn->imm), insn->imm);
7906 return -EFAULT;
7907 }
7908 insn->imm = fn->func - __bpf_call_base;
7909 }
7910
7911 return 0;
7912}
Alexei Starovoitove245c5c62017-03-15 18:26:39 -07007913
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01007914static void free_states(struct bpf_verifier_env *env)
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07007915{
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01007916 struct bpf_verifier_state_list *sl, *sln;
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07007917 int i;
7918
Alexei Starovoitov9f4686c2019-04-01 21:27:41 -07007919 sl = env->free_list;
7920 while (sl) {
7921 sln = sl->next;
7922 free_verifier_state(&sl->state, false);
7923 kfree(sl);
7924 sl = sln;
7925 }
7926
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07007927 if (!env->explored_states)
7928 return;
7929
7930 for (i = 0; i < env->prog->len; i++) {
7931 sl = env->explored_states[i];
7932
7933 if (sl)
7934 while (sl != STATE_LIST_MARK) {
7935 sln = sl->next;
Alexei Starovoitov1969db42017-11-01 00:08:04 -07007936 free_verifier_state(&sl->state, false);
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07007937 kfree(sl);
7938 sl = sln;
7939 }
7940 }
7941
Alexei Starovoitov71dde682019-04-01 21:27:43 -07007942 kvfree(env->explored_states);
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07007943}
7944
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07007945static void print_verification_stats(struct bpf_verifier_env *env)
7946{
7947 int i;
7948
7949 if (env->log.level & BPF_LOG_STATS) {
7950 verbose(env, "verification time %lld usec\n",
7951 div_u64(env->verification_time, 1000));
7952 verbose(env, "stack depth ");
7953 for (i = 0; i < env->subprog_cnt; i++) {
7954 u32 depth = env->subprog_info[i].stack_depth;
7955
7956 verbose(env, "%d", depth);
7957 if (i + 1 < env->subprog_cnt)
7958 verbose(env, "+");
7959 }
7960 verbose(env, "\n");
7961 }
7962 verbose(env, "processed %d insns (limit %d) max_states_per_insn %d "
7963 "total_states %d peak_states %d mark_read %d\n",
7964 env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS,
7965 env->max_states_per_insn, env->total_states,
7966 env->peak_states, env->longest_mark_read_walk);
7967}
7968
Yonghong Song838e9692018-11-19 15:29:11 -08007969int bpf_check(struct bpf_prog **prog, union bpf_attr *attr,
7970 union bpf_attr __user *uattr)
Alexei Starovoitov51580e72014-09-26 00:17:02 -07007971{
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07007972 u64 start_time = ktime_get_ns();
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01007973 struct bpf_verifier_env *env;
Martin KaFai Laub9193c12018-03-24 11:44:22 -07007974 struct bpf_verifier_log *log;
Jakub Kicinski9e4c24e2019-01-22 22:45:23 -08007975 int i, len, ret = -EINVAL;
Jakub Kicinskie2ae4ca2019-01-22 22:45:19 -08007976 bool is_priv;
Alexei Starovoitov51580e72014-09-26 00:17:02 -07007977
Arnd Bergmanneba0c922017-11-02 12:05:52 +01007978 /* no program is valid */
7979 if (ARRAY_SIZE(bpf_verifier_ops) == 0)
7980 return -EINVAL;
7981
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01007982 /* 'struct bpf_verifier_env' can be global, but since it's not small,
Alexei Starovoitovcbd35702014-09-26 00:17:03 -07007983 * allocate/free it every time bpf_check() is called
7984 */
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01007985 env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL);
Alexei Starovoitovcbd35702014-09-26 00:17:03 -07007986 if (!env)
7987 return -ENOMEM;
Jakub Kicinski61bd5212017-10-09 10:30:11 -07007988 log = &env->log;
Alexei Starovoitovcbd35702014-09-26 00:17:03 -07007989
Jakub Kicinski9e4c24e2019-01-22 22:45:23 -08007990 len = (*prog)->len;
Kees Cookfad953c2018-06-12 14:27:37 -07007991 env->insn_aux_data =
Jakub Kicinski9e4c24e2019-01-22 22:45:23 -08007992 vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len));
Jakub Kicinski3df126f2016-09-21 11:43:56 +01007993 ret = -ENOMEM;
7994 if (!env->insn_aux_data)
7995 goto err_free_env;
Jakub Kicinski9e4c24e2019-01-22 22:45:23 -08007996 for (i = 0; i < len; i++)
7997 env->insn_aux_data[i].orig_idx = i;
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07007998 env->prog = *prog;
Jakub Kicinski00176a32017-10-16 16:40:54 -07007999 env->ops = bpf_verifier_ops[env->prog->type];
Alexei Starovoitov0246e642014-09-26 00:17:04 -07008000
Alexei Starovoitovcbd35702014-09-26 00:17:03 -07008001 /* grab the mutex to protect few globals used by verifier */
8002 mutex_lock(&bpf_verifier_lock);
8003
8004 if (attr->log_level || attr->log_buf || attr->log_size) {
8005 /* user requested verbose verifier output
8006 * and supplied buffer to store the verification trace
8007 */
Jakub Kicinskie7bf8242017-10-09 10:30:10 -07008008 log->level = attr->log_level;
8009 log->ubuf = (char __user *) (unsigned long) attr->log_buf;
8010 log->len_total = attr->log_size;
Alexei Starovoitovcbd35702014-09-26 00:17:03 -07008011
8012 ret = -EINVAL;
Jakub Kicinskie7bf8242017-10-09 10:30:10 -07008013 /* log attributes have to be sane */
Alexei Starovoitov7a9f5c62019-04-01 21:27:46 -07008014 if (log->len_total < 128 || log->len_total > UINT_MAX >> 2 ||
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07008015 !log->level || !log->ubuf || log->level & ~BPF_LOG_MASK)
Jakub Kicinski3df126f2016-09-21 11:43:56 +01008016 goto err_unlock;
Alexei Starovoitovcbd35702014-09-26 00:17:03 -07008017 }
Daniel Borkmann1ad2f582017-05-25 01:05:05 +02008018
8019 env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT);
8020 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
David S. Millere07b98d2017-05-10 11:38:07 -07008021 env->strict_alignment = true;
David Millere9ee9ef2018-11-30 21:08:14 -08008022 if (attr->prog_flags & BPF_F_ANY_ALIGNMENT)
8023 env->strict_alignment = false;
Alexei Starovoitovcbd35702014-09-26 00:17:03 -07008024
Jakub Kicinskie2ae4ca2019-01-22 22:45:19 -08008025 is_priv = capable(CAP_SYS_ADMIN);
8026 env->allow_ptr_leaks = is_priv;
8027
Alexei Starovoitov0246e642014-09-26 00:17:04 -07008028 ret = replace_map_fd_with_map_ptr(env);
8029 if (ret < 0)
8030 goto skip_full_check;
8031
Jakub Kicinskif4e3ec02018-05-03 18:37:11 -07008032 if (bpf_prog_is_dev_bound(env->prog->aux)) {
Quentin Monneta40a2632018-11-09 13:03:31 +00008033 ret = bpf_prog_offload_verifier_prep(env->prog);
Jakub Kicinskif4e3ec02018-05-03 18:37:11 -07008034 if (ret)
8035 goto skip_full_check;
8036 }
8037
Alexei Starovoitov71dde682019-04-01 21:27:43 -07008038 env->explored_states = kvcalloc(env->prog->len,
Jakub Kicinski58e2af8b2016-09-21 11:43:57 +01008039 sizeof(struct bpf_verifier_state_list *),
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07008040 GFP_USER);
8041 ret = -ENOMEM;
8042 if (!env->explored_states)
8043 goto skip_full_check;
8044
Martin KaFai Laud9762e82018-12-13 10:41:48 -08008045 ret = check_subprogs(env);
Alexei Starovoitov475fb782014-09-26 00:17:05 -07008046 if (ret < 0)
8047 goto skip_full_check;
8048
Martin KaFai Lauc454a462018-12-07 16:42:25 -08008049 ret = check_btf_info(env, attr, uattr);
Yonghong Song838e9692018-11-19 15:29:11 -08008050 if (ret < 0)
8051 goto skip_full_check;
8052
Martin KaFai Laud9762e82018-12-13 10:41:48 -08008053 ret = check_cfg(env);
8054 if (ret < 0)
8055 goto skip_full_check;
8056
Alexei Starovoitov17a52672014-09-26 00:17:06 -07008057 ret = do_check(env);
Craig Gallek8c01c4f2017-11-02 11:18:01 -04008058 if (env->cur_state) {
8059 free_verifier_state(env->cur_state, true);
8060 env->cur_state = NULL;
8061 }
Alexei Starovoitovcbd35702014-09-26 00:17:03 -07008062
Quentin Monnetc941ce92018-10-07 12:56:47 +01008063 if (ret == 0 && bpf_prog_is_dev_bound(env->prog->aux))
8064 ret = bpf_prog_offload_finalize(env);
8065
Alexei Starovoitov0246e642014-09-26 00:17:04 -07008066skip_full_check:
Alexei Starovoitov638f5b92017-10-31 18:16:05 -07008067 while (!pop_stack(env, NULL, NULL));
Alexei Starovoitovf1bca822014-09-29 18:50:01 -07008068 free_states(env);
Alexei Starovoitov0246e642014-09-26 00:17:04 -07008069
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07008070 if (ret == 0)
Alexei Starovoitov70a87ff2017-12-25 13:15:40 -08008071 ret = check_max_stack_depth(env);
8072
Jakub Kicinski9b38c402018-12-19 22:13:06 -08008073 /* instruction rewrites happen after this point */
Jakub Kicinskie2ae4ca2019-01-22 22:45:19 -08008074 if (is_priv) {
8075 if (ret == 0)
8076 opt_hard_wire_dead_code_branches(env);
Jakub Kicinski52875a02019-01-22 22:45:20 -08008077 if (ret == 0)
8078 ret = opt_remove_dead_code(env);
Jakub Kicinskia1b14ab2019-01-22 22:45:21 -08008079 if (ret == 0)
8080 ret = opt_remove_nops(env);
Jakub Kicinski52875a02019-01-22 22:45:20 -08008081 } else {
8082 if (ret == 0)
8083 sanitize_dead_code(env);
Jakub Kicinskie2ae4ca2019-01-22 22:45:19 -08008084 }
8085
Jakub Kicinski9b38c402018-12-19 22:13:06 -08008086 if (ret == 0)
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07008087 /* program is valid, convert *(u32*)(ctx + off) accesses */
8088 ret = convert_ctx_accesses(env);
8089
Alexei Starovoitove245c5c62017-03-15 18:26:39 -07008090 if (ret == 0)
Alexei Starovoitov79741b32017-03-15 18:26:40 -07008091 ret = fixup_bpf_calls(env);
Alexei Starovoitove245c5c62017-03-15 18:26:39 -07008092
Alexei Starovoitov1ea47e02017-12-14 17:55:13 -08008093 if (ret == 0)
8094 ret = fixup_call_args(env);
8095
Alexei Starovoitov06ee7112019-04-01 21:27:40 -07008096 env->verification_time = ktime_get_ns() - start_time;
8097 print_verification_stats(env);
8098
Jakub Kicinskia2a7d572017-10-09 10:30:15 -07008099 if (log->level && bpf_verifier_log_full(log))
Alexei Starovoitovcbd35702014-09-26 00:17:03 -07008100 ret = -ENOSPC;
Jakub Kicinskia2a7d572017-10-09 10:30:15 -07008101 if (log->level && !log->ubuf) {
Alexei Starovoitovcbd35702014-09-26 00:17:03 -07008102 ret = -EFAULT;
Jakub Kicinskia2a7d572017-10-09 10:30:15 -07008103 goto err_release_maps;
Alexei Starovoitovcbd35702014-09-26 00:17:03 -07008104 }
8105
Alexei Starovoitov0246e642014-09-26 00:17:04 -07008106 if (ret == 0 && env->used_map_cnt) {
8107 /* if program passed verifier, update used_maps in bpf_prog_info */
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07008108 env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt,
8109 sizeof(env->used_maps[0]),
8110 GFP_KERNEL);
Alexei Starovoitov0246e642014-09-26 00:17:04 -07008111
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07008112 if (!env->prog->aux->used_maps) {
Alexei Starovoitov0246e642014-09-26 00:17:04 -07008113 ret = -ENOMEM;
Jakub Kicinskia2a7d572017-10-09 10:30:15 -07008114 goto err_release_maps;
Alexei Starovoitov0246e642014-09-26 00:17:04 -07008115 }
8116
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07008117 memcpy(env->prog->aux->used_maps, env->used_maps,
Alexei Starovoitov0246e642014-09-26 00:17:04 -07008118 sizeof(env->used_maps[0]) * env->used_map_cnt);
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07008119 env->prog->aux->used_map_cnt = env->used_map_cnt;
Alexei Starovoitov0246e642014-09-26 00:17:04 -07008120
8121 /* program is valid. Convert pseudo bpf_ld_imm64 into generic
8122 * bpf_ld_imm64 instructions
8123 */
8124 convert_pseudo_ld_imm64(env);
8125 }
Alexei Starovoitovcbd35702014-09-26 00:17:03 -07008126
Yonghong Songba64e7d2018-11-24 23:20:44 -08008127 if (ret == 0)
8128 adjust_btf_func(env);
8129
Jakub Kicinskia2a7d572017-10-09 10:30:15 -07008130err_release_maps:
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07008131 if (!env->prog->aux->used_maps)
Alexei Starovoitov0246e642014-09-26 00:17:04 -07008132 /* if we didn't copy map pointers into bpf_prog_info, release
Jakub Kicinskiab7f5bf2018-05-03 18:37:17 -07008133 * them now. Otherwise free_used_maps() will release them.
Alexei Starovoitov0246e642014-09-26 00:17:04 -07008134 */
8135 release_maps(env);
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -07008136 *prog = env->prog;
Jakub Kicinski3df126f2016-09-21 11:43:56 +01008137err_unlock:
Alexei Starovoitovcbd35702014-09-26 00:17:03 -07008138 mutex_unlock(&bpf_verifier_lock);
Jakub Kicinski3df126f2016-09-21 11:43:56 +01008139 vfree(env->insn_aux_data);
8140err_free_env:
8141 kfree(env);
Alexei Starovoitov51580e72014-09-26 00:17:02 -07008142 return ret;
8143}