Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 1 | /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 2 | * Copyright (c) 2016 Facebook |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 3 | * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 4 | * |
| 5 | * This program is free software; you can redistribute it and/or |
| 6 | * modify it under the terms of version 2 of the GNU General Public |
| 7 | * License as published by the Free Software Foundation. |
| 8 | * |
| 9 | * This program is distributed in the hope that it will be useful, but |
| 10 | * WITHOUT ANY WARRANTY; without even the implied warranty of |
| 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 12 | * General Public License for more details. |
| 13 | */ |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 14 | #include <uapi/linux/btf.h> |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 15 | #include <linux/kernel.h> |
| 16 | #include <linux/types.h> |
| 17 | #include <linux/slab.h> |
| 18 | #include <linux/bpf.h> |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 19 | #include <linux/btf.h> |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 20 | #include <linux/bpf_verifier.h> |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 21 | #include <linux/filter.h> |
| 22 | #include <net/netlink.h> |
| 23 | #include <linux/file.h> |
| 24 | #include <linux/vmalloc.h> |
Thomas Graf | ebb676d | 2016-10-27 11:23:51 +0200 | [diff] [blame] | 25 | #include <linux/stringify.h> |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 26 | #include <linux/bsearch.h> |
| 27 | #include <linux/sort.h> |
Yonghong Song | c195651e | 2018-04-28 22:28:08 -0700 | [diff] [blame] | 28 | #include <linux/perf_event.h> |
Martin KaFai Lau | d9762e8 | 2018-12-13 10:41:48 -0800 | [diff] [blame] | 29 | #include <linux/ctype.h> |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 30 | |
Jakub Kicinski | f4ac7e0 | 2017-10-09 10:30:12 -0700 | [diff] [blame] | 31 | #include "disasm.h" |
| 32 | |
Jakub Kicinski | 00176a3 | 2017-10-16 16:40:54 -0700 | [diff] [blame] | 33 | static const struct bpf_verifier_ops * const bpf_verifier_ops[] = { |
| 34 | #define BPF_PROG_TYPE(_id, _name) \ |
| 35 | [_id] = & _name ## _verifier_ops, |
| 36 | #define BPF_MAP_TYPE(_id, _ops) |
| 37 | #include <linux/bpf_types.h> |
| 38 | #undef BPF_PROG_TYPE |
| 39 | #undef BPF_MAP_TYPE |
| 40 | }; |
| 41 | |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 42 | /* bpf_check() is a static code analyzer that walks eBPF program |
| 43 | * instruction by instruction and updates register/stack state. |
| 44 | * All paths of conditional branches are analyzed until 'bpf_exit' insn. |
| 45 | * |
| 46 | * The first pass is depth-first-search to check that the program is a DAG. |
| 47 | * It rejects the following programs: |
| 48 | * - larger than BPF_MAXINSNS insns |
| 49 | * - if loop is present (detected via back-edge) |
| 50 | * - unreachable insns exist (shouldn't be a forest. program = one function) |
| 51 | * - out of bounds or malformed jumps |
| 52 | * The second pass is all possible path descent from the 1st insn. |
| 53 | * Since it's analyzing all pathes through the program, the length of the |
Gary Lin | eba38a9 | 2017-03-01 16:25:51 +0800 | [diff] [blame] | 54 | * analysis is limited to 64k insn, which may be hit even if total number of |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 55 | * insn is less then 4K, but there are too many branches that change stack/regs. |
| 56 | * Number of 'branches to be analyzed' is limited to 1k |
| 57 | * |
| 58 | * On entry to each instruction, each register has a type, and the instruction |
| 59 | * changes the types of the registers depending on instruction semantics. |
| 60 | * If instruction is BPF_MOV64_REG(BPF_REG_1, BPF_REG_5), then type of R5 is |
| 61 | * copied to R1. |
| 62 | * |
| 63 | * All registers are 64-bit. |
| 64 | * R0 - return register |
| 65 | * R1-R5 argument passing registers |
| 66 | * R6-R9 callee saved registers |
| 67 | * R10 - frame pointer read-only |
| 68 | * |
| 69 | * At the start of BPF program the register R1 contains a pointer to bpf_context |
| 70 | * and has type PTR_TO_CTX. |
| 71 | * |
| 72 | * Verifier tracks arithmetic operations on pointers in case: |
| 73 | * BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), |
| 74 | * BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -20), |
| 75 | * 1st insn copies R10 (which has FRAME_PTR) type into R1 |
| 76 | * and 2nd arithmetic instruction is pattern matched to recognize |
| 77 | * that it wants to construct a pointer to some element within stack. |
| 78 | * So after 2nd insn, the register R1 has type PTR_TO_STACK |
| 79 | * (and -20 constant is saved for further stack bounds checking). |
| 80 | * Meaning that this reg is a pointer to stack plus known immediate constant. |
| 81 | * |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 82 | * Most of the time the registers have SCALAR_VALUE type, which |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 83 | * means the register has some value, but it's not a valid pointer. |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 84 | * (like pointer plus pointer becomes SCALAR_VALUE type) |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 85 | * |
| 86 | * When verifier sees load or store instructions the type of base register |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 87 | * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are |
| 88 | * four pointer types recognized by check_mem_access() function. |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 89 | * |
| 90 | * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value' |
| 91 | * and the range of [ptr, ptr + map's value_size) is accessible. |
| 92 | * |
| 93 | * registers used to pass values to function calls are checked against |
| 94 | * function argument constraints. |
| 95 | * |
| 96 | * ARG_PTR_TO_MAP_KEY is one of such argument constraints. |
| 97 | * It means that the register type passed to this function must be |
| 98 | * PTR_TO_STACK and it will be used inside the function as |
| 99 | * 'pointer to map element key' |
| 100 | * |
| 101 | * For example the argument constraints for bpf_map_lookup_elem(): |
| 102 | * .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, |
| 103 | * .arg1_type = ARG_CONST_MAP_PTR, |
| 104 | * .arg2_type = ARG_PTR_TO_MAP_KEY, |
| 105 | * |
| 106 | * ret_type says that this function returns 'pointer to map elem value or null' |
| 107 | * function expects 1st argument to be a const pointer to 'struct bpf_map' and |
| 108 | * 2nd argument should be a pointer to stack, which will be used inside |
| 109 | * the helper function as a pointer to map element key. |
| 110 | * |
| 111 | * On the kernel side the helper function looks like: |
| 112 | * u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) |
| 113 | * { |
| 114 | * struct bpf_map *map = (struct bpf_map *) (unsigned long) r1; |
| 115 | * void *key = (void *) (unsigned long) r2; |
| 116 | * void *value; |
| 117 | * |
| 118 | * here kernel can access 'key' and 'map' pointers safely, knowing that |
| 119 | * [key, key + map->key_size) bytes are valid and were initialized on |
| 120 | * the stack of eBPF program. |
| 121 | * } |
| 122 | * |
| 123 | * Corresponding eBPF program may look like: |
| 124 | * BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), // after this insn R2 type is FRAME_PTR |
| 125 | * BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), // after this insn R2 type is PTR_TO_STACK |
| 126 | * BPF_LD_MAP_FD(BPF_REG_1, map_fd), // after this insn R1 type is CONST_PTR_TO_MAP |
| 127 | * BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem), |
| 128 | * here verifier looks at prototype of map_lookup_elem() and sees: |
| 129 | * .arg1_type == ARG_CONST_MAP_PTR and R1->type == CONST_PTR_TO_MAP, which is ok, |
| 130 | * Now verifier knows that this map has key of R1->map_ptr->key_size bytes |
| 131 | * |
| 132 | * Then .arg2_type == ARG_PTR_TO_MAP_KEY and R2->type == PTR_TO_STACK, ok so far, |
| 133 | * Now verifier checks that [R2, R2 + map's key_size) are within stack limits |
| 134 | * and were initialized prior to this call. |
| 135 | * If it's ok, then verifier allows this BPF_CALL insn and looks at |
| 136 | * .ret_type which is RET_PTR_TO_MAP_VALUE_OR_NULL, so it sets |
| 137 | * R0->type = PTR_TO_MAP_VALUE_OR_NULL which means bpf_map_lookup_elem() function |
| 138 | * returns ether pointer to map value or NULL. |
| 139 | * |
| 140 | * When type PTR_TO_MAP_VALUE_OR_NULL passes through 'if (reg != 0) goto +off' |
| 141 | * insn, the register holding that pointer in the true branch changes state to |
| 142 | * PTR_TO_MAP_VALUE and the same register changes state to CONST_IMM in the false |
| 143 | * branch. See check_cond_jmp_op(). |
| 144 | * |
| 145 | * After the call R0 is set to return type of the function and registers R1-R5 |
| 146 | * are set to NOT_INIT to indicate that they are no longer readable. |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 147 | * |
| 148 | * The following reference types represent a potential reference to a kernel |
| 149 | * resource which, after first being allocated, must be checked and freed by |
| 150 | * the BPF program: |
| 151 | * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET |
| 152 | * |
| 153 | * When the verifier sees a helper call return a reference type, it allocates a |
| 154 | * pointer id for the reference and stores it in the current function state. |
| 155 | * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into |
| 156 | * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type |
| 157 | * passes through a NULL-check conditional. For the branch wherein the state is |
| 158 | * changed to CONST_IMM, the verifier releases the reference. |
Joe Stringer | 6acc9b4 | 2018-10-02 13:35:36 -0700 | [diff] [blame] | 159 | * |
| 160 | * For each helper function that allocates a reference, such as |
| 161 | * bpf_sk_lookup_tcp(), there is a corresponding release function, such as |
| 162 | * bpf_sk_release(). When a reference type passes into the release function, |
| 163 | * the verifier also releases the reference. If any unchecked or unreleased |
| 164 | * reference remains at the end of the program, the verifier rejects it. |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 165 | */ |
| 166 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 167 | /* verifier_state + insn_idx are pushed to stack when branch is encountered */ |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 168 | struct bpf_verifier_stack_elem { |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 169 | /* verifer state is 'st' |
| 170 | * before processing instruction 'insn_idx' |
| 171 | * and after processing instruction 'prev_insn_idx' |
| 172 | */ |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 173 | struct bpf_verifier_state st; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 174 | int insn_idx; |
| 175 | int prev_insn_idx; |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 176 | struct bpf_verifier_stack_elem *next; |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 177 | }; |
| 178 | |
Daniel Borkmann | 0701615 | 2016-04-05 22:33:17 +0200 | [diff] [blame] | 179 | #define BPF_COMPLEXITY_LIMIT_STACK 1024 |
Alexei Starovoitov | ceefbc9 | 2018-12-03 22:46:06 -0800 | [diff] [blame] | 180 | #define BPF_COMPLEXITY_LIMIT_STATES 64 |
Daniel Borkmann | 0701615 | 2016-04-05 22:33:17 +0200 | [diff] [blame] | 181 | |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 182 | #define BPF_MAP_PTR_UNPRIV 1UL |
| 183 | #define BPF_MAP_PTR_POISON ((void *)((0xeB9FUL << 1) + \ |
| 184 | POISON_POINTER_DELTA)) |
| 185 | #define BPF_MAP_PTR(X) ((struct bpf_map *)((X) & ~BPF_MAP_PTR_UNPRIV)) |
| 186 | |
| 187 | static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux) |
| 188 | { |
| 189 | return BPF_MAP_PTR(aux->map_state) == BPF_MAP_PTR_POISON; |
| 190 | } |
| 191 | |
| 192 | static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux) |
| 193 | { |
| 194 | return aux->map_state & BPF_MAP_PTR_UNPRIV; |
| 195 | } |
| 196 | |
| 197 | static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux, |
| 198 | const struct bpf_map *map, bool unpriv) |
| 199 | { |
| 200 | BUILD_BUG_ON((unsigned long)BPF_MAP_PTR_POISON & BPF_MAP_PTR_UNPRIV); |
| 201 | unpriv |= bpf_map_ptr_unpriv(aux); |
| 202 | aux->map_state = (unsigned long)map | |
| 203 | (unpriv ? BPF_MAP_PTR_UNPRIV : 0UL); |
| 204 | } |
Martin KaFai Lau | fad73a1 | 2017-03-22 10:00:32 -0700 | [diff] [blame] | 205 | |
Daniel Borkmann | 33ff982 | 2016-04-13 00:10:50 +0200 | [diff] [blame] | 206 | struct bpf_call_arg_meta { |
| 207 | struct bpf_map *map_ptr; |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 208 | bool raw_mode; |
Daniel Borkmann | 36bbef5 | 2016-09-20 00:26:13 +0200 | [diff] [blame] | 209 | bool pkt_access; |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 210 | int regno; |
| 211 | int access_size; |
Yonghong Song | 849fa50 | 2018-04-28 22:28:09 -0700 | [diff] [blame] | 212 | s64 msize_smax_value; |
| 213 | u64 msize_umax_value; |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 214 | int ref_obj_id; |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 215 | int func_id; |
Daniel Borkmann | 33ff982 | 2016-04-13 00:10:50 +0200 | [diff] [blame] | 216 | }; |
| 217 | |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 218 | static DEFINE_MUTEX(bpf_verifier_lock); |
| 219 | |
Martin KaFai Lau | d9762e8 | 2018-12-13 10:41:48 -0800 | [diff] [blame] | 220 | static const struct bpf_line_info * |
| 221 | find_linfo(const struct bpf_verifier_env *env, u32 insn_off) |
| 222 | { |
| 223 | const struct bpf_line_info *linfo; |
| 224 | const struct bpf_prog *prog; |
| 225 | u32 i, nr_linfo; |
| 226 | |
| 227 | prog = env->prog; |
| 228 | nr_linfo = prog->aux->nr_linfo; |
| 229 | |
| 230 | if (!nr_linfo || insn_off >= prog->len) |
| 231 | return NULL; |
| 232 | |
| 233 | linfo = prog->aux->linfo; |
| 234 | for (i = 1; i < nr_linfo; i++) |
| 235 | if (insn_off < linfo[i].insn_off) |
| 236 | break; |
| 237 | |
| 238 | return &linfo[i - 1]; |
| 239 | } |
| 240 | |
Martin KaFai Lau | 77d2e05 | 2018-03-24 11:44:23 -0700 | [diff] [blame] | 241 | void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt, |
| 242 | va_list args) |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 243 | { |
Jakub Kicinski | a2a7d57 | 2017-10-09 10:30:15 -0700 | [diff] [blame] | 244 | unsigned int n; |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 245 | |
Jakub Kicinski | a2a7d57 | 2017-10-09 10:30:15 -0700 | [diff] [blame] | 246 | n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args); |
Jakub Kicinski | a2a7d57 | 2017-10-09 10:30:15 -0700 | [diff] [blame] | 247 | |
| 248 | WARN_ONCE(n >= BPF_VERIFIER_TMP_LOG_SIZE - 1, |
| 249 | "verifier log line truncated - local buffer too short\n"); |
| 250 | |
| 251 | n = min(log->len_total - log->len_used - 1, n); |
| 252 | log->kbuf[n] = '\0'; |
| 253 | |
| 254 | if (!copy_to_user(log->ubuf + log->len_used, log->kbuf, n + 1)) |
| 255 | log->len_used += n; |
| 256 | else |
| 257 | log->ubuf = NULL; |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 258 | } |
Jiri Olsa | abe0884 | 2018-03-23 11:41:28 +0100 | [diff] [blame] | 259 | |
| 260 | /* log_level controls verbosity level of eBPF verifier. |
| 261 | * bpf_verifier_log_write() is used to dump the verification trace to the log, |
| 262 | * so the user can figure out what's wrong with the program |
Quentin Monnet | 430e68d | 2018-01-10 12:26:06 +0000 | [diff] [blame] | 263 | */ |
Jiri Olsa | abe0884 | 2018-03-23 11:41:28 +0100 | [diff] [blame] | 264 | __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, |
| 265 | const char *fmt, ...) |
| 266 | { |
| 267 | va_list args; |
| 268 | |
Martin KaFai Lau | 77d2e05 | 2018-03-24 11:44:23 -0700 | [diff] [blame] | 269 | if (!bpf_verifier_log_needed(&env->log)) |
| 270 | return; |
| 271 | |
Jiri Olsa | abe0884 | 2018-03-23 11:41:28 +0100 | [diff] [blame] | 272 | va_start(args, fmt); |
Martin KaFai Lau | 77d2e05 | 2018-03-24 11:44:23 -0700 | [diff] [blame] | 273 | bpf_verifier_vlog(&env->log, fmt, args); |
Jiri Olsa | abe0884 | 2018-03-23 11:41:28 +0100 | [diff] [blame] | 274 | va_end(args); |
| 275 | } |
| 276 | EXPORT_SYMBOL_GPL(bpf_verifier_log_write); |
| 277 | |
| 278 | __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...) |
| 279 | { |
Martin KaFai Lau | 77d2e05 | 2018-03-24 11:44:23 -0700 | [diff] [blame] | 280 | struct bpf_verifier_env *env = private_data; |
Jiri Olsa | abe0884 | 2018-03-23 11:41:28 +0100 | [diff] [blame] | 281 | va_list args; |
| 282 | |
Martin KaFai Lau | 77d2e05 | 2018-03-24 11:44:23 -0700 | [diff] [blame] | 283 | if (!bpf_verifier_log_needed(&env->log)) |
| 284 | return; |
| 285 | |
Jiri Olsa | abe0884 | 2018-03-23 11:41:28 +0100 | [diff] [blame] | 286 | va_start(args, fmt); |
Martin KaFai Lau | 77d2e05 | 2018-03-24 11:44:23 -0700 | [diff] [blame] | 287 | bpf_verifier_vlog(&env->log, fmt, args); |
Jiri Olsa | abe0884 | 2018-03-23 11:41:28 +0100 | [diff] [blame] | 288 | va_end(args); |
| 289 | } |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 290 | |
Martin KaFai Lau | d9762e8 | 2018-12-13 10:41:48 -0800 | [diff] [blame] | 291 | static const char *ltrim(const char *s) |
| 292 | { |
| 293 | while (isspace(*s)) |
| 294 | s++; |
| 295 | |
| 296 | return s; |
| 297 | } |
| 298 | |
| 299 | __printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env, |
| 300 | u32 insn_off, |
| 301 | const char *prefix_fmt, ...) |
| 302 | { |
| 303 | const struct bpf_line_info *linfo; |
| 304 | |
| 305 | if (!bpf_verifier_log_needed(&env->log)) |
| 306 | return; |
| 307 | |
| 308 | linfo = find_linfo(env, insn_off); |
| 309 | if (!linfo || linfo == env->prev_linfo) |
| 310 | return; |
| 311 | |
| 312 | if (prefix_fmt) { |
| 313 | va_list args; |
| 314 | |
| 315 | va_start(args, prefix_fmt); |
| 316 | bpf_verifier_vlog(&env->log, prefix_fmt, args); |
| 317 | va_end(args); |
| 318 | } |
| 319 | |
| 320 | verbose(env, "%s\n", |
| 321 | ltrim(btf_name_by_offset(env->prog->aux->btf, |
| 322 | linfo->line_off))); |
| 323 | |
| 324 | env->prev_linfo = linfo; |
| 325 | } |
| 326 | |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 327 | static bool type_is_pkt_pointer(enum bpf_reg_type type) |
| 328 | { |
| 329 | return type == PTR_TO_PACKET || |
| 330 | type == PTR_TO_PACKET_META; |
| 331 | } |
| 332 | |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 333 | static bool type_is_sk_pointer(enum bpf_reg_type type) |
| 334 | { |
| 335 | return type == PTR_TO_SOCKET || |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 336 | type == PTR_TO_SOCK_COMMON || |
| 337 | type == PTR_TO_TCP_SOCK; |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 338 | } |
| 339 | |
Joe Stringer | 840b961 | 2018-10-02 13:35:32 -0700 | [diff] [blame] | 340 | static bool reg_type_may_be_null(enum bpf_reg_type type) |
| 341 | { |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 342 | return type == PTR_TO_MAP_VALUE_OR_NULL || |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 343 | type == PTR_TO_SOCKET_OR_NULL || |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 344 | type == PTR_TO_SOCK_COMMON_OR_NULL || |
| 345 | type == PTR_TO_TCP_SOCK_OR_NULL; |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 346 | } |
| 347 | |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 348 | static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) |
| 349 | { |
| 350 | return reg->type == PTR_TO_MAP_VALUE && |
| 351 | map_value_has_spin_lock(reg->map_ptr); |
| 352 | } |
| 353 | |
Martin KaFai Lau | cba368c | 2019-03-18 10:37:13 -0700 | [diff] [blame] | 354 | static bool reg_type_may_be_refcounted_or_null(enum bpf_reg_type type) |
| 355 | { |
| 356 | return type == PTR_TO_SOCKET || |
| 357 | type == PTR_TO_SOCKET_OR_NULL || |
| 358 | type == PTR_TO_TCP_SOCK || |
| 359 | type == PTR_TO_TCP_SOCK_OR_NULL; |
| 360 | } |
| 361 | |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 362 | static bool arg_type_may_be_refcounted(enum bpf_arg_type type) |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 363 | { |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 364 | return type == ARG_PTR_TO_SOCK_COMMON; |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 365 | } |
| 366 | |
| 367 | /* Determine whether the function releases some resources allocated by another |
| 368 | * function call. The first reference type argument will be assumed to be |
| 369 | * released by release_reference(). |
| 370 | */ |
| 371 | static bool is_release_function(enum bpf_func_id func_id) |
| 372 | { |
Joe Stringer | 6acc9b4 | 2018-10-02 13:35:36 -0700 | [diff] [blame] | 373 | return func_id == BPF_FUNC_sk_release; |
Joe Stringer | 840b961 | 2018-10-02 13:35:32 -0700 | [diff] [blame] | 374 | } |
| 375 | |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 376 | static bool is_acquire_function(enum bpf_func_id func_id) |
| 377 | { |
| 378 | return func_id == BPF_FUNC_sk_lookup_tcp || |
Lorenz Bauer | edbf8c0 | 2019-03-22 09:54:01 +0800 | [diff] [blame] | 379 | func_id == BPF_FUNC_sk_lookup_udp || |
| 380 | func_id == BPF_FUNC_skc_lookup_tcp; |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 381 | } |
| 382 | |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 383 | static bool is_ptr_cast_function(enum bpf_func_id func_id) |
| 384 | { |
| 385 | return func_id == BPF_FUNC_tcp_sock || |
| 386 | func_id == BPF_FUNC_sk_fullsock; |
| 387 | } |
| 388 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 389 | /* string representation of 'enum bpf_reg_type' */ |
| 390 | static const char * const reg_type_str[] = { |
| 391 | [NOT_INIT] = "?", |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 392 | [SCALAR_VALUE] = "inv", |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 393 | [PTR_TO_CTX] = "ctx", |
| 394 | [CONST_PTR_TO_MAP] = "map_ptr", |
| 395 | [PTR_TO_MAP_VALUE] = "map_value", |
| 396 | [PTR_TO_MAP_VALUE_OR_NULL] = "map_value_or_null", |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 397 | [PTR_TO_STACK] = "fp", |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 398 | [PTR_TO_PACKET] = "pkt", |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 399 | [PTR_TO_PACKET_META] = "pkt_meta", |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 400 | [PTR_TO_PACKET_END] = "pkt_end", |
Petar Penkov | d58e468 | 2018-09-14 07:46:18 -0700 | [diff] [blame] | 401 | [PTR_TO_FLOW_KEYS] = "flow_keys", |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 402 | [PTR_TO_SOCKET] = "sock", |
| 403 | [PTR_TO_SOCKET_OR_NULL] = "sock_or_null", |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 404 | [PTR_TO_SOCK_COMMON] = "sock_common", |
| 405 | [PTR_TO_SOCK_COMMON_OR_NULL] = "sock_common_or_null", |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 406 | [PTR_TO_TCP_SOCK] = "tcp_sock", |
| 407 | [PTR_TO_TCP_SOCK_OR_NULL] = "tcp_sock_or_null", |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 408 | }; |
| 409 | |
Edward Cree | 8efea21 | 2018-08-22 20:02:44 +0100 | [diff] [blame] | 410 | static char slot_type_char[] = { |
| 411 | [STACK_INVALID] = '?', |
| 412 | [STACK_SPILL] = 'r', |
| 413 | [STACK_MISC] = 'm', |
| 414 | [STACK_ZERO] = '0', |
| 415 | }; |
| 416 | |
Alexei Starovoitov | 4e92024 | 2017-11-30 21:31:36 -0800 | [diff] [blame] | 417 | static void print_liveness(struct bpf_verifier_env *env, |
| 418 | enum bpf_reg_liveness live) |
| 419 | { |
Alexei Starovoitov | 9242b5f | 2018-12-13 11:42:34 -0800 | [diff] [blame] | 420 | if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE)) |
Alexei Starovoitov | 4e92024 | 2017-11-30 21:31:36 -0800 | [diff] [blame] | 421 | verbose(env, "_"); |
| 422 | if (live & REG_LIVE_READ) |
| 423 | verbose(env, "r"); |
| 424 | if (live & REG_LIVE_WRITTEN) |
| 425 | verbose(env, "w"); |
Alexei Starovoitov | 9242b5f | 2018-12-13 11:42:34 -0800 | [diff] [blame] | 426 | if (live & REG_LIVE_DONE) |
| 427 | verbose(env, "D"); |
Alexei Starovoitov | 4e92024 | 2017-11-30 21:31:36 -0800 | [diff] [blame] | 428 | } |
| 429 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 430 | static struct bpf_func_state *func(struct bpf_verifier_env *env, |
| 431 | const struct bpf_reg_state *reg) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 432 | { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 433 | struct bpf_verifier_state *cur = env->cur_state; |
| 434 | |
| 435 | return cur->frame[reg->frameno]; |
| 436 | } |
| 437 | |
| 438 | static void print_verifier_state(struct bpf_verifier_env *env, |
| 439 | const struct bpf_func_state *state) |
| 440 | { |
| 441 | const struct bpf_reg_state *reg; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 442 | enum bpf_reg_type t; |
| 443 | int i; |
| 444 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 445 | if (state->frameno) |
| 446 | verbose(env, " frame%d:", state->frameno); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 447 | for (i = 0; i < MAX_BPF_REG; i++) { |
Alexei Starovoitov | 1a0dc1a | 2016-05-05 19:49:09 -0700 | [diff] [blame] | 448 | reg = &state->regs[i]; |
| 449 | t = reg->type; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 450 | if (t == NOT_INIT) |
| 451 | continue; |
Alexei Starovoitov | 4e92024 | 2017-11-30 21:31:36 -0800 | [diff] [blame] | 452 | verbose(env, " R%d", i); |
| 453 | print_liveness(env, reg->live); |
| 454 | verbose(env, "=%s", reg_type_str[t]); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 455 | if ((t == SCALAR_VALUE || t == PTR_TO_STACK) && |
| 456 | tnum_is_const(reg->var_off)) { |
| 457 | /* reg->off should be 0 for SCALAR_VALUE */ |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 458 | verbose(env, "%lld", reg->var_off.value + reg->off); |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 459 | if (t == PTR_TO_STACK) |
| 460 | verbose(env, ",call_%d", func(env, reg)->callsite); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 461 | } else { |
Martin KaFai Lau | cba368c | 2019-03-18 10:37:13 -0700 | [diff] [blame] | 462 | verbose(env, "(id=%d", reg->id); |
| 463 | if (reg_type_may_be_refcounted_or_null(t)) |
| 464 | verbose(env, ",ref_obj_id=%d", reg->ref_obj_id); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 465 | if (t != SCALAR_VALUE) |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 466 | verbose(env, ",off=%d", reg->off); |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 467 | if (type_is_pkt_pointer(t)) |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 468 | verbose(env, ",r=%d", reg->range); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 469 | else if (t == CONST_PTR_TO_MAP || |
| 470 | t == PTR_TO_MAP_VALUE || |
| 471 | t == PTR_TO_MAP_VALUE_OR_NULL) |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 472 | verbose(env, ",ks=%d,vs=%d", |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 473 | reg->map_ptr->key_size, |
| 474 | reg->map_ptr->value_size); |
Edward Cree | 7d1238f | 2017-08-07 15:26:56 +0100 | [diff] [blame] | 475 | if (tnum_is_const(reg->var_off)) { |
| 476 | /* Typically an immediate SCALAR_VALUE, but |
| 477 | * could be a pointer whose offset is too big |
| 478 | * for reg->off |
| 479 | */ |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 480 | verbose(env, ",imm=%llx", reg->var_off.value); |
Edward Cree | 7d1238f | 2017-08-07 15:26:56 +0100 | [diff] [blame] | 481 | } else { |
| 482 | if (reg->smin_value != reg->umin_value && |
| 483 | reg->smin_value != S64_MIN) |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 484 | verbose(env, ",smin_value=%lld", |
Edward Cree | 7d1238f | 2017-08-07 15:26:56 +0100 | [diff] [blame] | 485 | (long long)reg->smin_value); |
| 486 | if (reg->smax_value != reg->umax_value && |
| 487 | reg->smax_value != S64_MAX) |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 488 | verbose(env, ",smax_value=%lld", |
Edward Cree | 7d1238f | 2017-08-07 15:26:56 +0100 | [diff] [blame] | 489 | (long long)reg->smax_value); |
| 490 | if (reg->umin_value != 0) |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 491 | verbose(env, ",umin_value=%llu", |
Edward Cree | 7d1238f | 2017-08-07 15:26:56 +0100 | [diff] [blame] | 492 | (unsigned long long)reg->umin_value); |
| 493 | if (reg->umax_value != U64_MAX) |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 494 | verbose(env, ",umax_value=%llu", |
Edward Cree | 7d1238f | 2017-08-07 15:26:56 +0100 | [diff] [blame] | 495 | (unsigned long long)reg->umax_value); |
| 496 | if (!tnum_is_unknown(reg->var_off)) { |
| 497 | char tn_buf[48]; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 498 | |
Edward Cree | 7d1238f | 2017-08-07 15:26:56 +0100 | [diff] [blame] | 499 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 500 | verbose(env, ",var_off=%s", tn_buf); |
Edward Cree | 7d1238f | 2017-08-07 15:26:56 +0100 | [diff] [blame] | 501 | } |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 502 | } |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 503 | verbose(env, ")"); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 504 | } |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 505 | } |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 506 | for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { |
Edward Cree | 8efea21 | 2018-08-22 20:02:44 +0100 | [diff] [blame] | 507 | char types_buf[BPF_REG_SIZE + 1]; |
| 508 | bool valid = false; |
| 509 | int j; |
| 510 | |
| 511 | for (j = 0; j < BPF_REG_SIZE; j++) { |
| 512 | if (state->stack[i].slot_type[j] != STACK_INVALID) |
| 513 | valid = true; |
| 514 | types_buf[j] = slot_type_char[ |
| 515 | state->stack[i].slot_type[j]]; |
| 516 | } |
| 517 | types_buf[BPF_REG_SIZE] = 0; |
| 518 | if (!valid) |
| 519 | continue; |
| 520 | verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); |
| 521 | print_liveness(env, state->stack[i].spilled_ptr.live); |
| 522 | if (state->stack[i].slot_type[0] == STACK_SPILL) |
Alexei Starovoitov | 4e92024 | 2017-11-30 21:31:36 -0800 | [diff] [blame] | 523 | verbose(env, "=%s", |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 524 | reg_type_str[state->stack[i].spilled_ptr.type]); |
Edward Cree | 8efea21 | 2018-08-22 20:02:44 +0100 | [diff] [blame] | 525 | else |
| 526 | verbose(env, "=%s", types_buf); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 527 | } |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 528 | if (state->acquired_refs && state->refs[0].id) { |
| 529 | verbose(env, " refs=%d", state->refs[0].id); |
| 530 | for (i = 1; i < state->acquired_refs; i++) |
| 531 | if (state->refs[i].id) |
| 532 | verbose(env, ",%d", state->refs[i].id); |
| 533 | } |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 534 | verbose(env, "\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 535 | } |
| 536 | |
Joe Stringer | 84dbf35 | 2018-10-02 13:35:34 -0700 | [diff] [blame] | 537 | #define COPY_STATE_FN(NAME, COUNT, FIELD, SIZE) \ |
| 538 | static int copy_##NAME##_state(struct bpf_func_state *dst, \ |
| 539 | const struct bpf_func_state *src) \ |
| 540 | { \ |
| 541 | if (!src->FIELD) \ |
| 542 | return 0; \ |
| 543 | if (WARN_ON_ONCE(dst->COUNT < src->COUNT)) { \ |
| 544 | /* internal bug, make state invalid to reject the program */ \ |
| 545 | memset(dst, 0, sizeof(*dst)); \ |
| 546 | return -EFAULT; \ |
| 547 | } \ |
| 548 | memcpy(dst->FIELD, src->FIELD, \ |
| 549 | sizeof(*src->FIELD) * (src->COUNT / SIZE)); \ |
| 550 | return 0; \ |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 551 | } |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 552 | /* copy_reference_state() */ |
| 553 | COPY_STATE_FN(reference, acquired_refs, refs, 1) |
Joe Stringer | 84dbf35 | 2018-10-02 13:35:34 -0700 | [diff] [blame] | 554 | /* copy_stack_state() */ |
| 555 | COPY_STATE_FN(stack, allocated_stack, stack, BPF_REG_SIZE) |
| 556 | #undef COPY_STATE_FN |
| 557 | |
| 558 | #define REALLOC_STATE_FN(NAME, COUNT, FIELD, SIZE) \ |
| 559 | static int realloc_##NAME##_state(struct bpf_func_state *state, int size, \ |
| 560 | bool copy_old) \ |
| 561 | { \ |
| 562 | u32 old_size = state->COUNT; \ |
| 563 | struct bpf_##NAME##_state *new_##FIELD; \ |
| 564 | int slot = size / SIZE; \ |
| 565 | \ |
| 566 | if (size <= old_size || !size) { \ |
| 567 | if (copy_old) \ |
| 568 | return 0; \ |
| 569 | state->COUNT = slot * SIZE; \ |
| 570 | if (!size && old_size) { \ |
| 571 | kfree(state->FIELD); \ |
| 572 | state->FIELD = NULL; \ |
| 573 | } \ |
| 574 | return 0; \ |
| 575 | } \ |
| 576 | new_##FIELD = kmalloc_array(slot, sizeof(struct bpf_##NAME##_state), \ |
| 577 | GFP_KERNEL); \ |
| 578 | if (!new_##FIELD) \ |
| 579 | return -ENOMEM; \ |
| 580 | if (copy_old) { \ |
| 581 | if (state->FIELD) \ |
| 582 | memcpy(new_##FIELD, state->FIELD, \ |
| 583 | sizeof(*new_##FIELD) * (old_size / SIZE)); \ |
| 584 | memset(new_##FIELD + old_size / SIZE, 0, \ |
| 585 | sizeof(*new_##FIELD) * (size - old_size) / SIZE); \ |
| 586 | } \ |
| 587 | state->COUNT = slot * SIZE; \ |
| 588 | kfree(state->FIELD); \ |
| 589 | state->FIELD = new_##FIELD; \ |
| 590 | return 0; \ |
| 591 | } |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 592 | /* realloc_reference_state() */ |
| 593 | REALLOC_STATE_FN(reference, acquired_refs, refs, 1) |
Joe Stringer | 84dbf35 | 2018-10-02 13:35:34 -0700 | [diff] [blame] | 594 | /* realloc_stack_state() */ |
| 595 | REALLOC_STATE_FN(stack, allocated_stack, stack, BPF_REG_SIZE) |
| 596 | #undef REALLOC_STATE_FN |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 597 | |
| 598 | /* do_check() starts with zero-sized stack in struct bpf_verifier_state to |
| 599 | * make it consume minimal amount of memory. check_stack_write() access from |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 600 | * the program calls into realloc_func_state() to grow the stack size. |
Joe Stringer | 84dbf35 | 2018-10-02 13:35:34 -0700 | [diff] [blame] | 601 | * Note there is a non-zero 'parent' pointer inside bpf_verifier_state |
| 602 | * which realloc_stack_state() copies over. It points to previous |
| 603 | * bpf_verifier_state which is never reallocated. |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 604 | */ |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 605 | static int realloc_func_state(struct bpf_func_state *state, int stack_size, |
| 606 | int refs_size, bool copy_old) |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 607 | { |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 608 | int err = realloc_reference_state(state, refs_size, copy_old); |
| 609 | if (err) |
| 610 | return err; |
| 611 | return realloc_stack_state(state, stack_size, copy_old); |
| 612 | } |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 613 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 614 | /* Acquire a pointer id from the env and update the state->refs to include |
| 615 | * this new pointer reference. |
| 616 | * On success, returns a valid pointer id to associate with the register |
| 617 | * On failure, returns a negative errno. |
| 618 | */ |
| 619 | static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx) |
| 620 | { |
| 621 | struct bpf_func_state *state = cur_func(env); |
| 622 | int new_ofs = state->acquired_refs; |
| 623 | int id, err; |
| 624 | |
| 625 | err = realloc_reference_state(state, state->acquired_refs + 1, true); |
| 626 | if (err) |
| 627 | return err; |
| 628 | id = ++env->id_gen; |
| 629 | state->refs[new_ofs].id = id; |
| 630 | state->refs[new_ofs].insn_idx = insn_idx; |
| 631 | |
| 632 | return id; |
| 633 | } |
| 634 | |
| 635 | /* release function corresponding to acquire_reference_state(). Idempotent. */ |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 636 | static int release_reference_state(struct bpf_func_state *state, int ptr_id) |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 637 | { |
| 638 | int i, last_idx; |
| 639 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 640 | last_idx = state->acquired_refs - 1; |
| 641 | for (i = 0; i < state->acquired_refs; i++) { |
| 642 | if (state->refs[i].id == ptr_id) { |
| 643 | if (last_idx && i != last_idx) |
| 644 | memcpy(&state->refs[i], &state->refs[last_idx], |
| 645 | sizeof(*state->refs)); |
| 646 | memset(&state->refs[last_idx], 0, sizeof(*state->refs)); |
| 647 | state->acquired_refs--; |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 648 | return 0; |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 649 | } |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 650 | } |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 651 | return -EINVAL; |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 652 | } |
| 653 | |
| 654 | static int transfer_reference_state(struct bpf_func_state *dst, |
| 655 | struct bpf_func_state *src) |
| 656 | { |
| 657 | int err = realloc_reference_state(dst, src->acquired_refs, false); |
| 658 | if (err) |
| 659 | return err; |
| 660 | err = copy_reference_state(dst, src); |
| 661 | if (err) |
| 662 | return err; |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 663 | return 0; |
| 664 | } |
| 665 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 666 | static void free_func_state(struct bpf_func_state *state) |
| 667 | { |
Alexei Starovoitov | 5896351 | 2018-01-08 07:51:17 -0800 | [diff] [blame] | 668 | if (!state) |
| 669 | return; |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 670 | kfree(state->refs); |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 671 | kfree(state->stack); |
| 672 | kfree(state); |
| 673 | } |
| 674 | |
Alexei Starovoitov | 1969db4 | 2017-11-01 00:08:04 -0700 | [diff] [blame] | 675 | static void free_verifier_state(struct bpf_verifier_state *state, |
| 676 | bool free_self) |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 677 | { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 678 | int i; |
| 679 | |
| 680 | for (i = 0; i <= state->curframe; i++) { |
| 681 | free_func_state(state->frame[i]); |
| 682 | state->frame[i] = NULL; |
| 683 | } |
Alexei Starovoitov | 1969db4 | 2017-11-01 00:08:04 -0700 | [diff] [blame] | 684 | if (free_self) |
| 685 | kfree(state); |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 686 | } |
| 687 | |
| 688 | /* copy verifier state from src to dst growing dst stack space |
| 689 | * when necessary to accommodate larger src stack |
| 690 | */ |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 691 | static int copy_func_state(struct bpf_func_state *dst, |
| 692 | const struct bpf_func_state *src) |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 693 | { |
| 694 | int err; |
| 695 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 696 | err = realloc_func_state(dst, src->allocated_stack, src->acquired_refs, |
| 697 | false); |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 698 | if (err) |
| 699 | return err; |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 700 | memcpy(dst, src, offsetof(struct bpf_func_state, acquired_refs)); |
| 701 | err = copy_reference_state(dst, src); |
| 702 | if (err) |
| 703 | return err; |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 704 | return copy_stack_state(dst, src); |
| 705 | } |
| 706 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 707 | static int copy_verifier_state(struct bpf_verifier_state *dst_state, |
| 708 | const struct bpf_verifier_state *src) |
| 709 | { |
| 710 | struct bpf_func_state *dst; |
| 711 | int i, err; |
| 712 | |
| 713 | /* if dst has more stack frames then src frame, free them */ |
| 714 | for (i = src->curframe + 1; i <= dst_state->curframe; i++) { |
| 715 | free_func_state(dst_state->frame[i]); |
| 716 | dst_state->frame[i] = NULL; |
| 717 | } |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 718 | dst_state->speculative = src->speculative; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 719 | dst_state->curframe = src->curframe; |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 720 | dst_state->active_spin_lock = src->active_spin_lock; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 721 | for (i = 0; i <= src->curframe; i++) { |
| 722 | dst = dst_state->frame[i]; |
| 723 | if (!dst) { |
| 724 | dst = kzalloc(sizeof(*dst), GFP_KERNEL); |
| 725 | if (!dst) |
| 726 | return -ENOMEM; |
| 727 | dst_state->frame[i] = dst; |
| 728 | } |
| 729 | err = copy_func_state(dst, src->frame[i]); |
| 730 | if (err) |
| 731 | return err; |
| 732 | } |
| 733 | return 0; |
| 734 | } |
| 735 | |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 736 | static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, |
| 737 | int *insn_idx) |
| 738 | { |
| 739 | struct bpf_verifier_state *cur = env->cur_state; |
| 740 | struct bpf_verifier_stack_elem *elem, *head = env->head; |
| 741 | int err; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 742 | |
| 743 | if (env->head == NULL) |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 744 | return -ENOENT; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 745 | |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 746 | if (cur) { |
| 747 | err = copy_verifier_state(cur, &head->st); |
| 748 | if (err) |
| 749 | return err; |
| 750 | } |
| 751 | if (insn_idx) |
| 752 | *insn_idx = head->insn_idx; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 753 | if (prev_insn_idx) |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 754 | *prev_insn_idx = head->prev_insn_idx; |
| 755 | elem = head->next; |
Alexei Starovoitov | 1969db4 | 2017-11-01 00:08:04 -0700 | [diff] [blame] | 756 | free_verifier_state(&head->st, false); |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 757 | kfree(head); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 758 | env->head = elem; |
| 759 | env->stack_size--; |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 760 | return 0; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 761 | } |
| 762 | |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 763 | static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 764 | int insn_idx, int prev_insn_idx, |
| 765 | bool speculative) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 766 | { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 767 | struct bpf_verifier_state *cur = env->cur_state; |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 768 | struct bpf_verifier_stack_elem *elem; |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 769 | int err; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 770 | |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 771 | elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 772 | if (!elem) |
| 773 | goto err; |
| 774 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 775 | elem->insn_idx = insn_idx; |
| 776 | elem->prev_insn_idx = prev_insn_idx; |
| 777 | elem->next = env->head; |
| 778 | env->head = elem; |
| 779 | env->stack_size++; |
Alexei Starovoitov | 1969db4 | 2017-11-01 00:08:04 -0700 | [diff] [blame] | 780 | err = copy_verifier_state(&elem->st, cur); |
| 781 | if (err) |
| 782 | goto err; |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 783 | elem->st.speculative |= speculative; |
Daniel Borkmann | 0701615 | 2016-04-05 22:33:17 +0200 | [diff] [blame] | 784 | if (env->stack_size > BPF_COMPLEXITY_LIMIT_STACK) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 785 | verbose(env, "BPF program is too complex\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 786 | goto err; |
| 787 | } |
| 788 | return &elem->st; |
| 789 | err: |
Alexei Starovoitov | 5896351 | 2018-01-08 07:51:17 -0800 | [diff] [blame] | 790 | free_verifier_state(env->cur_state, true); |
| 791 | env->cur_state = NULL; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 792 | /* pop all elements and return */ |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 793 | while (!pop_stack(env, NULL, NULL)); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 794 | return NULL; |
| 795 | } |
| 796 | |
| 797 | #define CALLER_SAVED_REGS 6 |
| 798 | static const int caller_saved[CALLER_SAVED_REGS] = { |
| 799 | BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5 |
| 800 | }; |
| 801 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 802 | static void __mark_reg_not_init(struct bpf_reg_state *reg); |
| 803 | |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 804 | /* Mark the unknown part of a register (variable offset or scalar value) as |
| 805 | * known to have the value @imm. |
| 806 | */ |
| 807 | static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm) |
| 808 | { |
Alexei Starovoitov | a9c676b | 2018-09-04 19:13:44 -0700 | [diff] [blame] | 809 | /* Clear id, off, and union(map_ptr, range) */ |
| 810 | memset(((u8 *)reg) + sizeof(reg->type), 0, |
| 811 | offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type)); |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 812 | reg->var_off = tnum_const(imm); |
| 813 | reg->smin_value = (s64)imm; |
| 814 | reg->smax_value = (s64)imm; |
| 815 | reg->umin_value = imm; |
| 816 | reg->umax_value = imm; |
| 817 | } |
| 818 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 819 | /* Mark the 'variable offset' part of a register as zero. This should be |
| 820 | * used only on registers holding a pointer type. |
| 821 | */ |
| 822 | static void __mark_reg_known_zero(struct bpf_reg_state *reg) |
| 823 | { |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 824 | __mark_reg_known(reg, 0); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 825 | } |
| 826 | |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 827 | static void __mark_reg_const_zero(struct bpf_reg_state *reg) |
| 828 | { |
| 829 | __mark_reg_known(reg, 0); |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 830 | reg->type = SCALAR_VALUE; |
| 831 | } |
| 832 | |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 833 | static void mark_reg_known_zero(struct bpf_verifier_env *env, |
| 834 | struct bpf_reg_state *regs, u32 regno) |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 835 | { |
| 836 | if (WARN_ON(regno >= MAX_BPF_REG)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 837 | verbose(env, "mark_reg_known_zero(regs, %u)\n", regno); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 838 | /* Something bad happened, let's kill all regs */ |
| 839 | for (regno = 0; regno < MAX_BPF_REG; regno++) |
| 840 | __mark_reg_not_init(regs + regno); |
| 841 | return; |
| 842 | } |
| 843 | __mark_reg_known_zero(regs + regno); |
| 844 | } |
| 845 | |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 846 | static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg) |
| 847 | { |
| 848 | return type_is_pkt_pointer(reg->type); |
| 849 | } |
| 850 | |
| 851 | static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg) |
| 852 | { |
| 853 | return reg_is_pkt_pointer(reg) || |
| 854 | reg->type == PTR_TO_PACKET_END; |
| 855 | } |
| 856 | |
| 857 | /* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */ |
| 858 | static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg, |
| 859 | enum bpf_reg_type which) |
| 860 | { |
| 861 | /* The register can already have a range from prior markings. |
| 862 | * This is fine as long as it hasn't been advanced from its |
| 863 | * origin. |
| 864 | */ |
| 865 | return reg->type == which && |
| 866 | reg->id == 0 && |
| 867 | reg->off == 0 && |
| 868 | tnum_equals_const(reg->var_off, 0); |
| 869 | } |
| 870 | |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 871 | /* Attempts to improve min/max values based on var_off information */ |
| 872 | static void __update_reg_bounds(struct bpf_reg_state *reg) |
| 873 | { |
| 874 | /* min signed is max(sign bit) | min(other bits) */ |
| 875 | reg->smin_value = max_t(s64, reg->smin_value, |
| 876 | reg->var_off.value | (reg->var_off.mask & S64_MIN)); |
| 877 | /* max signed is min(sign bit) | max(other bits) */ |
| 878 | reg->smax_value = min_t(s64, reg->smax_value, |
| 879 | reg->var_off.value | (reg->var_off.mask & S64_MAX)); |
| 880 | reg->umin_value = max(reg->umin_value, reg->var_off.value); |
| 881 | reg->umax_value = min(reg->umax_value, |
| 882 | reg->var_off.value | reg->var_off.mask); |
| 883 | } |
| 884 | |
| 885 | /* Uses signed min/max values to inform unsigned, and vice-versa */ |
| 886 | static void __reg_deduce_bounds(struct bpf_reg_state *reg) |
| 887 | { |
| 888 | /* Learn sign from signed bounds. |
| 889 | * If we cannot cross the sign boundary, then signed and unsigned bounds |
| 890 | * are the same, so combine. This works even in the negative case, e.g. |
| 891 | * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. |
| 892 | */ |
| 893 | if (reg->smin_value >= 0 || reg->smax_value < 0) { |
| 894 | reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, |
| 895 | reg->umin_value); |
| 896 | reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, |
| 897 | reg->umax_value); |
| 898 | return; |
| 899 | } |
| 900 | /* Learn sign from unsigned bounds. Signed bounds cross the sign |
| 901 | * boundary, so we must be careful. |
| 902 | */ |
| 903 | if ((s64)reg->umax_value >= 0) { |
| 904 | /* Positive. We can't learn anything from the smin, but smax |
| 905 | * is positive, hence safe. |
| 906 | */ |
| 907 | reg->smin_value = reg->umin_value; |
| 908 | reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, |
| 909 | reg->umax_value); |
| 910 | } else if ((s64)reg->umin_value < 0) { |
| 911 | /* Negative. We can't learn anything from the smax, but smin |
| 912 | * is negative, hence safe. |
| 913 | */ |
| 914 | reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, |
| 915 | reg->umin_value); |
| 916 | reg->smax_value = reg->umax_value; |
| 917 | } |
| 918 | } |
| 919 | |
| 920 | /* Attempts to improve var_off based on unsigned min/max information */ |
| 921 | static void __reg_bound_offset(struct bpf_reg_state *reg) |
| 922 | { |
| 923 | reg->var_off = tnum_intersect(reg->var_off, |
| 924 | tnum_range(reg->umin_value, |
| 925 | reg->umax_value)); |
| 926 | } |
| 927 | |
| 928 | /* Reset the min/max bounds of a register */ |
| 929 | static void __mark_reg_unbounded(struct bpf_reg_state *reg) |
| 930 | { |
| 931 | reg->smin_value = S64_MIN; |
| 932 | reg->smax_value = S64_MAX; |
| 933 | reg->umin_value = 0; |
| 934 | reg->umax_value = U64_MAX; |
| 935 | } |
| 936 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 937 | /* Mark a register as having a completely unknown (scalar) value. */ |
| 938 | static void __mark_reg_unknown(struct bpf_reg_state *reg) |
| 939 | { |
Alexei Starovoitov | a9c676b | 2018-09-04 19:13:44 -0700 | [diff] [blame] | 940 | /* |
| 941 | * Clear type, id, off, and union(map_ptr, range) and |
| 942 | * padding between 'type' and union |
| 943 | */ |
| 944 | memset(reg, 0, offsetof(struct bpf_reg_state, var_off)); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 945 | reg->type = SCALAR_VALUE; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 946 | reg->var_off = tnum_unknown; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 947 | reg->frameno = 0; |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 948 | __mark_reg_unbounded(reg); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 949 | } |
| 950 | |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 951 | static void mark_reg_unknown(struct bpf_verifier_env *env, |
| 952 | struct bpf_reg_state *regs, u32 regno) |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 953 | { |
| 954 | if (WARN_ON(regno >= MAX_BPF_REG)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 955 | verbose(env, "mark_reg_unknown(regs, %u)\n", regno); |
Alexei Starovoitov | 19ceb41 | 2017-11-30 21:31:37 -0800 | [diff] [blame] | 956 | /* Something bad happened, let's kill all regs except FP */ |
| 957 | for (regno = 0; regno < BPF_REG_FP; regno++) |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 958 | __mark_reg_not_init(regs + regno); |
| 959 | return; |
| 960 | } |
| 961 | __mark_reg_unknown(regs + regno); |
| 962 | } |
| 963 | |
| 964 | static void __mark_reg_not_init(struct bpf_reg_state *reg) |
| 965 | { |
| 966 | __mark_reg_unknown(reg); |
| 967 | reg->type = NOT_INIT; |
| 968 | } |
| 969 | |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 970 | static void mark_reg_not_init(struct bpf_verifier_env *env, |
| 971 | struct bpf_reg_state *regs, u32 regno) |
Daniel Borkmann | a9789ef | 2017-05-25 01:05:06 +0200 | [diff] [blame] | 972 | { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 973 | if (WARN_ON(regno >= MAX_BPF_REG)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 974 | verbose(env, "mark_reg_not_init(regs, %u)\n", regno); |
Alexei Starovoitov | 19ceb41 | 2017-11-30 21:31:37 -0800 | [diff] [blame] | 975 | /* Something bad happened, let's kill all regs except FP */ |
| 976 | for (regno = 0; regno < BPF_REG_FP; regno++) |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 977 | __mark_reg_not_init(regs + regno); |
| 978 | return; |
| 979 | } |
| 980 | __mark_reg_not_init(regs + regno); |
Daniel Borkmann | a9789ef | 2017-05-25 01:05:06 +0200 | [diff] [blame] | 981 | } |
| 982 | |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 983 | static void init_reg_state(struct bpf_verifier_env *env, |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 984 | struct bpf_func_state *state) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 985 | { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 986 | struct bpf_reg_state *regs = state->regs; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 987 | int i; |
| 988 | |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 989 | for (i = 0; i < MAX_BPF_REG; i++) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 990 | mark_reg_not_init(env, regs, i); |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 991 | regs[i].live = REG_LIVE_NONE; |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 992 | regs[i].parent = NULL; |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 993 | } |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 994 | |
| 995 | /* frame pointer */ |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 996 | regs[BPF_REG_FP].type = PTR_TO_STACK; |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 997 | mark_reg_known_zero(env, regs, BPF_REG_FP); |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 998 | regs[BPF_REG_FP].frameno = state->frameno; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 999 | |
| 1000 | /* 1st arg to a function */ |
| 1001 | regs[BPF_REG_1].type = PTR_TO_CTX; |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1002 | mark_reg_known_zero(env, regs, BPF_REG_1); |
Daniel Borkmann | 6760bf2 | 2016-12-18 01:52:59 +0100 | [diff] [blame] | 1003 | } |
| 1004 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1005 | #define BPF_MAIN_FUNC (-1) |
| 1006 | static void init_func_state(struct bpf_verifier_env *env, |
| 1007 | struct bpf_func_state *state, |
| 1008 | int callsite, int frameno, int subprogno) |
| 1009 | { |
| 1010 | state->callsite = callsite; |
| 1011 | state->frameno = frameno; |
| 1012 | state->subprogno = subprogno; |
| 1013 | init_reg_state(env, state); |
| 1014 | } |
| 1015 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1016 | enum reg_arg_type { |
| 1017 | SRC_OP, /* register is used as source operand */ |
| 1018 | DST_OP, /* register is used as destination operand */ |
| 1019 | DST_OP_NO_MARK /* same as above, check only, don't mark */ |
| 1020 | }; |
| 1021 | |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1022 | static int cmp_subprogs(const void *a, const void *b) |
| 1023 | { |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 1024 | return ((struct bpf_subprog_info *)a)->start - |
| 1025 | ((struct bpf_subprog_info *)b)->start; |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1026 | } |
| 1027 | |
| 1028 | static int find_subprog(struct bpf_verifier_env *env, int off) |
| 1029 | { |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 1030 | struct bpf_subprog_info *p; |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1031 | |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 1032 | p = bsearch(&off, env->subprog_info, env->subprog_cnt, |
| 1033 | sizeof(env->subprog_info[0]), cmp_subprogs); |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1034 | if (!p) |
| 1035 | return -ENOENT; |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 1036 | return p - env->subprog_info; |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1037 | |
| 1038 | } |
| 1039 | |
| 1040 | static int add_subprog(struct bpf_verifier_env *env, int off) |
| 1041 | { |
| 1042 | int insn_cnt = env->prog->len; |
| 1043 | int ret; |
| 1044 | |
| 1045 | if (off >= insn_cnt || off < 0) { |
| 1046 | verbose(env, "call to invalid destination\n"); |
| 1047 | return -EINVAL; |
| 1048 | } |
| 1049 | ret = find_subprog(env, off); |
| 1050 | if (ret >= 0) |
| 1051 | return 0; |
Jiong Wang | 4cb3d99 | 2018-05-02 16:17:19 -0400 | [diff] [blame] | 1052 | if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1053 | verbose(env, "too many subprograms\n"); |
| 1054 | return -E2BIG; |
| 1055 | } |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 1056 | env->subprog_info[env->subprog_cnt++].start = off; |
| 1057 | sort(env->subprog_info, env->subprog_cnt, |
| 1058 | sizeof(env->subprog_info[0]), cmp_subprogs, NULL); |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1059 | return 0; |
| 1060 | } |
| 1061 | |
| 1062 | static int check_subprogs(struct bpf_verifier_env *env) |
| 1063 | { |
| 1064 | int i, ret, subprog_start, subprog_end, off, cur_subprog = 0; |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 1065 | struct bpf_subprog_info *subprog = env->subprog_info; |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1066 | struct bpf_insn *insn = env->prog->insnsi; |
| 1067 | int insn_cnt = env->prog->len; |
| 1068 | |
Jiong Wang | f910cef | 2018-05-02 16:17:17 -0400 | [diff] [blame] | 1069 | /* Add entry function. */ |
| 1070 | ret = add_subprog(env, 0); |
| 1071 | if (ret < 0) |
| 1072 | return ret; |
| 1073 | |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1074 | /* determine subprog starts. The end is one before the next starts */ |
| 1075 | for (i = 0; i < insn_cnt; i++) { |
| 1076 | if (insn[i].code != (BPF_JMP | BPF_CALL)) |
| 1077 | continue; |
| 1078 | if (insn[i].src_reg != BPF_PSEUDO_CALL) |
| 1079 | continue; |
| 1080 | if (!env->allow_ptr_leaks) { |
| 1081 | verbose(env, "function calls to other bpf functions are allowed for root only\n"); |
| 1082 | return -EPERM; |
| 1083 | } |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1084 | ret = add_subprog(env, i + insn[i].imm + 1); |
| 1085 | if (ret < 0) |
| 1086 | return ret; |
| 1087 | } |
| 1088 | |
Jiong Wang | 4cb3d99 | 2018-05-02 16:17:19 -0400 | [diff] [blame] | 1089 | /* Add a fake 'exit' subprog which could simplify subprog iteration |
| 1090 | * logic. 'subprog_cnt' should not be increased. |
| 1091 | */ |
| 1092 | subprog[env->subprog_cnt].start = insn_cnt; |
| 1093 | |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 1094 | if (env->log.level & BPF_LOG_LEVEL2) |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1095 | for (i = 0; i < env->subprog_cnt; i++) |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 1096 | verbose(env, "func#%d @%d\n", i, subprog[i].start); |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1097 | |
| 1098 | /* now check that all jumps are within the same subprog */ |
Jiong Wang | 4cb3d99 | 2018-05-02 16:17:19 -0400 | [diff] [blame] | 1099 | subprog_start = subprog[cur_subprog].start; |
| 1100 | subprog_end = subprog[cur_subprog + 1].start; |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1101 | for (i = 0; i < insn_cnt; i++) { |
| 1102 | u8 code = insn[i].code; |
| 1103 | |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 1104 | if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1105 | goto next; |
| 1106 | if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL) |
| 1107 | goto next; |
| 1108 | off = i + insn[i].off + 1; |
| 1109 | if (off < subprog_start || off >= subprog_end) { |
| 1110 | verbose(env, "jump out of range from insn %d to %d\n", i, off); |
| 1111 | return -EINVAL; |
| 1112 | } |
| 1113 | next: |
| 1114 | if (i == subprog_end - 1) { |
| 1115 | /* to avoid fall-through from one subprog into another |
| 1116 | * the last insn of the subprog should be either exit |
| 1117 | * or unconditional jump back |
| 1118 | */ |
| 1119 | if (code != (BPF_JMP | BPF_EXIT) && |
| 1120 | code != (BPF_JMP | BPF_JA)) { |
| 1121 | verbose(env, "last insn is not an exit or jmp\n"); |
| 1122 | return -EINVAL; |
| 1123 | } |
| 1124 | subprog_start = subprog_end; |
Jiong Wang | 4cb3d99 | 2018-05-02 16:17:19 -0400 | [diff] [blame] | 1125 | cur_subprog++; |
| 1126 | if (cur_subprog < env->subprog_cnt) |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 1127 | subprog_end = subprog[cur_subprog + 1].start; |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1128 | } |
| 1129 | } |
| 1130 | return 0; |
| 1131 | } |
| 1132 | |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 1133 | /* Parentage chain of this register (or stack slot) should take care of all |
| 1134 | * issues like callee-saved registers, stack slot allocation time, etc. |
| 1135 | */ |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1136 | static int mark_reg_read(struct bpf_verifier_env *env, |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 1137 | const struct bpf_reg_state *state, |
| 1138 | struct bpf_reg_state *parent) |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1139 | { |
| 1140 | bool writes = parent == state->parent; /* Observe write marks */ |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 1141 | int cnt = 0; |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 1142 | |
| 1143 | while (parent) { |
| 1144 | /* if read wasn't screened by an earlier write ... */ |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 1145 | if (writes && state->live & REG_LIVE_WRITTEN) |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 1146 | break; |
Alexei Starovoitov | 9242b5f | 2018-12-13 11:42:34 -0800 | [diff] [blame] | 1147 | if (parent->live & REG_LIVE_DONE) { |
| 1148 | verbose(env, "verifier BUG type %s var_off %lld off %d\n", |
| 1149 | reg_type_str[parent->type], |
| 1150 | parent->var_off.value, parent->off); |
| 1151 | return -EFAULT; |
| 1152 | } |
Alexei Starovoitov | 25af32d | 2019-04-01 21:27:42 -0700 | [diff] [blame] | 1153 | if (parent->live & REG_LIVE_READ) |
| 1154 | /* The parentage chain never changes and |
| 1155 | * this parent was already marked as LIVE_READ. |
| 1156 | * There is no need to keep walking the chain again and |
| 1157 | * keep re-marking all parents as LIVE_READ. |
| 1158 | * This case happens when the same register is read |
| 1159 | * multiple times without writes into it in-between. |
| 1160 | */ |
| 1161 | break; |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 1162 | /* ... then we depend on parent's value */ |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 1163 | parent->live |= REG_LIVE_READ; |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 1164 | state = parent; |
| 1165 | parent = state->parent; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1166 | writes = true; |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 1167 | cnt++; |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 1168 | } |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 1169 | |
| 1170 | if (env->longest_mark_read_walk < cnt) |
| 1171 | env->longest_mark_read_walk = cnt; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1172 | return 0; |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 1173 | } |
| 1174 | |
| 1175 | static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1176 | enum reg_arg_type t) |
| 1177 | { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1178 | struct bpf_verifier_state *vstate = env->cur_state; |
| 1179 | struct bpf_func_state *state = vstate->frame[vstate->curframe]; |
| 1180 | struct bpf_reg_state *regs = state->regs; |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 1181 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1182 | if (regno >= MAX_BPF_REG) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1183 | verbose(env, "R%d is invalid\n", regno); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1184 | return -EINVAL; |
| 1185 | } |
| 1186 | |
| 1187 | if (t == SRC_OP) { |
| 1188 | /* check whether register used as source operand can be read */ |
| 1189 | if (regs[regno].type == NOT_INIT) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1190 | verbose(env, "R%d !read_ok\n", regno); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1191 | return -EACCES; |
| 1192 | } |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 1193 | /* We don't need to worry about FP liveness because it's read-only */ |
| 1194 | if (regno != BPF_REG_FP) |
| 1195 | return mark_reg_read(env, ®s[regno], |
| 1196 | regs[regno].parent); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1197 | } else { |
| 1198 | /* check whether register used as dest operand can be written to */ |
| 1199 | if (regno == BPF_REG_FP) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1200 | verbose(env, "frame pointer is read only\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1201 | return -EACCES; |
| 1202 | } |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 1203 | regs[regno].live |= REG_LIVE_WRITTEN; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1204 | if (t == DST_OP) |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1205 | mark_reg_unknown(env, regs, regno); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1206 | } |
| 1207 | return 0; |
| 1208 | } |
| 1209 | |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 1210 | static bool is_spillable_regtype(enum bpf_reg_type type) |
| 1211 | { |
| 1212 | switch (type) { |
| 1213 | case PTR_TO_MAP_VALUE: |
| 1214 | case PTR_TO_MAP_VALUE_OR_NULL: |
| 1215 | case PTR_TO_STACK: |
| 1216 | case PTR_TO_CTX: |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 1217 | case PTR_TO_PACKET: |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 1218 | case PTR_TO_PACKET_META: |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 1219 | case PTR_TO_PACKET_END: |
Petar Penkov | d58e468 | 2018-09-14 07:46:18 -0700 | [diff] [blame] | 1220 | case PTR_TO_FLOW_KEYS: |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 1221 | case CONST_PTR_TO_MAP: |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 1222 | case PTR_TO_SOCKET: |
| 1223 | case PTR_TO_SOCKET_OR_NULL: |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 1224 | case PTR_TO_SOCK_COMMON: |
| 1225 | case PTR_TO_SOCK_COMMON_OR_NULL: |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 1226 | case PTR_TO_TCP_SOCK: |
| 1227 | case PTR_TO_TCP_SOCK_OR_NULL: |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 1228 | return true; |
| 1229 | default: |
| 1230 | return false; |
| 1231 | } |
| 1232 | } |
| 1233 | |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 1234 | /* Does this register contain a constant zero? */ |
| 1235 | static bool register_is_null(struct bpf_reg_state *reg) |
| 1236 | { |
| 1237 | return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0); |
| 1238 | } |
| 1239 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1240 | /* check_stack_read/write functions track spill/fill of registers, |
| 1241 | * stack boundary and alignment are checked in check_mem_access() |
| 1242 | */ |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1243 | static int check_stack_write(struct bpf_verifier_env *env, |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1244 | struct bpf_func_state *state, /* func where register points to */ |
Alexei Starovoitov | af86ca4 | 2018-05-15 09:27:05 -0700 | [diff] [blame] | 1245 | int off, int size, int value_regno, int insn_idx) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1246 | { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1247 | struct bpf_func_state *cur; /* state of the current function */ |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 1248 | int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1249 | enum bpf_reg_type type; |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 1250 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1251 | err = realloc_func_state(state, round_up(slot + 1, BPF_REG_SIZE), |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 1252 | state->acquired_refs, true); |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 1253 | if (err) |
| 1254 | return err; |
Alexei Starovoitov | 9c399760 | 2014-10-28 15:11:41 -0700 | [diff] [blame] | 1255 | /* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0, |
| 1256 | * so it's aligned access and [off, off + size) are within stack limits |
| 1257 | */ |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 1258 | if (!env->allow_ptr_leaks && |
| 1259 | state->stack[spi].slot_type[0] == STACK_SPILL && |
| 1260 | size != BPF_REG_SIZE) { |
| 1261 | verbose(env, "attempt to corrupt spilled pointer on stack\n"); |
| 1262 | return -EACCES; |
| 1263 | } |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1264 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1265 | cur = env->cur_state->frame[env->cur_state->curframe]; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1266 | if (value_regno >= 0 && |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1267 | is_spillable_regtype((type = cur->regs[value_regno].type))) { |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1268 | |
| 1269 | /* register containing pointer is being spilled into stack */ |
Alexei Starovoitov | 9c399760 | 2014-10-28 15:11:41 -0700 | [diff] [blame] | 1270 | if (size != BPF_REG_SIZE) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1271 | verbose(env, "invalid size of register spill\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1272 | return -EACCES; |
| 1273 | } |
| 1274 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1275 | if (state != cur && type == PTR_TO_STACK) { |
| 1276 | verbose(env, "cannot spill pointers to stack into stack frame of the caller\n"); |
| 1277 | return -EINVAL; |
| 1278 | } |
| 1279 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1280 | /* save register state */ |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1281 | state->stack[spi].spilled_ptr = cur->regs[value_regno]; |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 1282 | state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1283 | |
Alexei Starovoitov | af86ca4 | 2018-05-15 09:27:05 -0700 | [diff] [blame] | 1284 | for (i = 0; i < BPF_REG_SIZE; i++) { |
| 1285 | if (state->stack[spi].slot_type[i] == STACK_MISC && |
| 1286 | !env->allow_ptr_leaks) { |
| 1287 | int *poff = &env->insn_aux_data[insn_idx].sanitize_stack_off; |
| 1288 | int soff = (-spi - 1) * BPF_REG_SIZE; |
| 1289 | |
| 1290 | /* detected reuse of integer stack slot with a pointer |
| 1291 | * which means either llvm is reusing stack slot or |
| 1292 | * an attacker is trying to exploit CVE-2018-3639 |
| 1293 | * (speculative store bypass) |
| 1294 | * Have to sanitize that slot with preemptive |
| 1295 | * store of zero. |
| 1296 | */ |
| 1297 | if (*poff && *poff != soff) { |
| 1298 | /* disallow programs where single insn stores |
| 1299 | * into two different stack slots, since verifier |
| 1300 | * cannot sanitize them |
| 1301 | */ |
| 1302 | verbose(env, |
| 1303 | "insn %d cannot access two stack slots fp%d and fp%d", |
| 1304 | insn_idx, *poff, soff); |
| 1305 | return -EINVAL; |
| 1306 | } |
| 1307 | *poff = soff; |
| 1308 | } |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 1309 | state->stack[spi].slot_type[i] = STACK_SPILL; |
Alexei Starovoitov | af86ca4 | 2018-05-15 09:27:05 -0700 | [diff] [blame] | 1310 | } |
Alexei Starovoitov | 9c399760 | 2014-10-28 15:11:41 -0700 | [diff] [blame] | 1311 | } else { |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 1312 | u8 type = STACK_MISC; |
| 1313 | |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 1314 | /* regular write of data into stack destroys any spilled ptr */ |
| 1315 | state->stack[spi].spilled_ptr.type = NOT_INIT; |
Jiong Wang | 0bae2d4 | 2018-12-15 03:34:40 -0500 | [diff] [blame] | 1316 | /* Mark slots as STACK_MISC if they belonged to spilled ptr. */ |
| 1317 | if (state->stack[spi].slot_type[0] == STACK_SPILL) |
| 1318 | for (i = 0; i < BPF_REG_SIZE; i++) |
| 1319 | state->stack[spi].slot_type[i] = STACK_MISC; |
Alexei Starovoitov | 9c399760 | 2014-10-28 15:11:41 -0700 | [diff] [blame] | 1320 | |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 1321 | /* only mark the slot as written if all 8 bytes were written |
| 1322 | * otherwise read propagation may incorrectly stop too soon |
| 1323 | * when stack slots are partially written. |
| 1324 | * This heuristic means that read propagation will be |
| 1325 | * conservative, since it will add reg_live_read marks |
| 1326 | * to stack slots all the way to first state when programs |
| 1327 | * writes+reads less than 8 bytes |
| 1328 | */ |
| 1329 | if (size == BPF_REG_SIZE) |
| 1330 | state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; |
| 1331 | |
| 1332 | /* when we zero initialize stack slots mark them as such */ |
| 1333 | if (value_regno >= 0 && |
| 1334 | register_is_null(&cur->regs[value_regno])) |
| 1335 | type = STACK_ZERO; |
| 1336 | |
Jiong Wang | 0bae2d4 | 2018-12-15 03:34:40 -0500 | [diff] [blame] | 1337 | /* Mark slots affected by this stack write. */ |
Alexei Starovoitov | 9c399760 | 2014-10-28 15:11:41 -0700 | [diff] [blame] | 1338 | for (i = 0; i < size; i++) |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 1339 | state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 1340 | type; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1341 | } |
| 1342 | return 0; |
| 1343 | } |
| 1344 | |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1345 | static int check_stack_read(struct bpf_verifier_env *env, |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1346 | struct bpf_func_state *reg_state /* func where register points to */, |
| 1347 | int off, int size, int value_regno) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1348 | { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1349 | struct bpf_verifier_state *vstate = env->cur_state; |
| 1350 | struct bpf_func_state *state = vstate->frame[vstate->curframe]; |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 1351 | int i, slot = -off - 1, spi = slot / BPF_REG_SIZE; |
| 1352 | u8 *stype; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1353 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1354 | if (reg_state->allocated_stack <= slot) { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 1355 | verbose(env, "invalid read from stack off %d+0 size %d\n", |
| 1356 | off, size); |
| 1357 | return -EACCES; |
| 1358 | } |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1359 | stype = reg_state->stack[spi].slot_type; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1360 | |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 1361 | if (stype[0] == STACK_SPILL) { |
Alexei Starovoitov | 9c399760 | 2014-10-28 15:11:41 -0700 | [diff] [blame] | 1362 | if (size != BPF_REG_SIZE) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1363 | verbose(env, "invalid size of register spill\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1364 | return -EACCES; |
| 1365 | } |
Alexei Starovoitov | 9c399760 | 2014-10-28 15:11:41 -0700 | [diff] [blame] | 1366 | for (i = 1; i < BPF_REG_SIZE; i++) { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 1367 | if (stype[(slot - i) % BPF_REG_SIZE] != STACK_SPILL) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1368 | verbose(env, "corrupted spill memory\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1369 | return -EACCES; |
| 1370 | } |
| 1371 | } |
| 1372 | |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 1373 | if (value_regno >= 0) { |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1374 | /* restore register state from stack */ |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1375 | state->regs[value_regno] = reg_state->stack[spi].spilled_ptr; |
Alexei Starovoitov | 2f18f62 | 2017-11-30 21:31:38 -0800 | [diff] [blame] | 1376 | /* mark reg as written since spilled pointer state likely |
| 1377 | * has its liveness marks cleared by is_state_visited() |
| 1378 | * which resets stack/reg liveness for state transitions |
| 1379 | */ |
| 1380 | state->regs[value_regno].live |= REG_LIVE_WRITTEN; |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 1381 | } |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 1382 | mark_reg_read(env, ®_state->stack[spi].spilled_ptr, |
| 1383 | reg_state->stack[spi].spilled_ptr.parent); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1384 | return 0; |
| 1385 | } else { |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 1386 | int zeros = 0; |
| 1387 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1388 | for (i = 0; i < size; i++) { |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 1389 | if (stype[(slot - i) % BPF_REG_SIZE] == STACK_MISC) |
| 1390 | continue; |
| 1391 | if (stype[(slot - i) % BPF_REG_SIZE] == STACK_ZERO) { |
| 1392 | zeros++; |
| 1393 | continue; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1394 | } |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 1395 | verbose(env, "invalid read from stack off %d+%d size %d\n", |
| 1396 | off, i, size); |
| 1397 | return -EACCES; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1398 | } |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 1399 | mark_reg_read(env, ®_state->stack[spi].spilled_ptr, |
| 1400 | reg_state->stack[spi].spilled_ptr.parent); |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 1401 | if (value_regno >= 0) { |
| 1402 | if (zeros == size) { |
| 1403 | /* any size read into register is zero extended, |
| 1404 | * so the whole register == const_zero |
| 1405 | */ |
| 1406 | __mark_reg_const_zero(&state->regs[value_regno]); |
| 1407 | } else { |
| 1408 | /* have read misc data from the stack */ |
| 1409 | mark_reg_unknown(env, state->regs, value_regno); |
| 1410 | } |
| 1411 | state->regs[value_regno].live |= REG_LIVE_WRITTEN; |
| 1412 | } |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1413 | return 0; |
| 1414 | } |
| 1415 | } |
| 1416 | |
Daniel Borkmann | e4298d2 | 2019-01-03 00:58:31 +0100 | [diff] [blame] | 1417 | static int check_stack_access(struct bpf_verifier_env *env, |
| 1418 | const struct bpf_reg_state *reg, |
| 1419 | int off, int size) |
| 1420 | { |
| 1421 | /* Stack accesses must be at a fixed offset, so that we |
| 1422 | * can determine what type of data were returned. See |
| 1423 | * check_stack_read(). |
| 1424 | */ |
| 1425 | if (!tnum_is_const(reg->var_off)) { |
| 1426 | char tn_buf[48]; |
| 1427 | |
| 1428 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
| 1429 | verbose(env, "variable stack access var_off=%s off=%d size=%d", |
| 1430 | tn_buf, off, size); |
| 1431 | return -EACCES; |
| 1432 | } |
| 1433 | |
| 1434 | if (off >= 0 || off < -MAX_BPF_STACK) { |
| 1435 | verbose(env, "invalid stack off=%d size=%d\n", off, size); |
| 1436 | return -EACCES; |
| 1437 | } |
| 1438 | |
| 1439 | return 0; |
| 1440 | } |
| 1441 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1442 | /* check read/write into map element returned by bpf_map_lookup_elem() */ |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1443 | static int __check_map_access(struct bpf_verifier_env *env, u32 regno, int off, |
Yonghong Song | 9fd29c0 | 2017-11-12 14:49:09 -0800 | [diff] [blame] | 1444 | int size, bool zero_size_allowed) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1445 | { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 1446 | struct bpf_reg_state *regs = cur_regs(env); |
| 1447 | struct bpf_map *map = regs[regno].map_ptr; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1448 | |
Yonghong Song | 9fd29c0 | 2017-11-12 14:49:09 -0800 | [diff] [blame] | 1449 | if (off < 0 || size < 0 || (size == 0 && !zero_size_allowed) || |
| 1450 | off + size > map->value_size) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1451 | verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n", |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1452 | map->value_size, off, size); |
| 1453 | return -EACCES; |
| 1454 | } |
| 1455 | return 0; |
| 1456 | } |
| 1457 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1458 | /* check read/write into a map element with possible variable offset */ |
| 1459 | static int check_map_access(struct bpf_verifier_env *env, u32 regno, |
Yonghong Song | 9fd29c0 | 2017-11-12 14:49:09 -0800 | [diff] [blame] | 1460 | int off, int size, bool zero_size_allowed) |
Gianluca Borello | dbcfe5f | 2017-01-09 10:19:46 -0800 | [diff] [blame] | 1461 | { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1462 | struct bpf_verifier_state *vstate = env->cur_state; |
| 1463 | struct bpf_func_state *state = vstate->frame[vstate->curframe]; |
Gianluca Borello | dbcfe5f | 2017-01-09 10:19:46 -0800 | [diff] [blame] | 1464 | struct bpf_reg_state *reg = &state->regs[regno]; |
| 1465 | int err; |
| 1466 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1467 | /* We may have adjusted the register to this map value, so we |
| 1468 | * need to try adding each of min_value and max_value to off |
| 1469 | * to make sure our theoretical access will be safe. |
Gianluca Borello | dbcfe5f | 2017-01-09 10:19:46 -0800 | [diff] [blame] | 1470 | */ |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 1471 | if (env->log.level & BPF_LOG_LEVEL) |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1472 | print_verifier_state(env, state); |
Daniel Borkmann | b7137c4 | 2019-01-03 00:58:33 +0100 | [diff] [blame] | 1473 | |
Gianluca Borello | dbcfe5f | 2017-01-09 10:19:46 -0800 | [diff] [blame] | 1474 | /* The minimum value is only important with signed |
| 1475 | * comparisons where we can't assume the floor of a |
| 1476 | * value is 0. If we are using signed variables for our |
| 1477 | * index'es we need to make sure that whatever we use |
| 1478 | * will have a set floor within our range. |
| 1479 | */ |
Daniel Borkmann | b7137c4 | 2019-01-03 00:58:33 +0100 | [diff] [blame] | 1480 | if (reg->smin_value < 0 && |
| 1481 | (reg->smin_value == S64_MIN || |
| 1482 | (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) || |
| 1483 | reg->smin_value + off < 0)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1484 | verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", |
Gianluca Borello | dbcfe5f | 2017-01-09 10:19:46 -0800 | [diff] [blame] | 1485 | regno); |
| 1486 | return -EACCES; |
| 1487 | } |
Yonghong Song | 9fd29c0 | 2017-11-12 14:49:09 -0800 | [diff] [blame] | 1488 | err = __check_map_access(env, regno, reg->smin_value + off, size, |
| 1489 | zero_size_allowed); |
Gianluca Borello | dbcfe5f | 2017-01-09 10:19:46 -0800 | [diff] [blame] | 1490 | if (err) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1491 | verbose(env, "R%d min value is outside of the array range\n", |
| 1492 | regno); |
Gianluca Borello | dbcfe5f | 2017-01-09 10:19:46 -0800 | [diff] [blame] | 1493 | return err; |
| 1494 | } |
| 1495 | |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 1496 | /* If we haven't set a max value then we need to bail since we can't be |
| 1497 | * sure we won't do bad things. |
| 1498 | * If reg->umax_value + off could overflow, treat that as unbounded too. |
Gianluca Borello | dbcfe5f | 2017-01-09 10:19:46 -0800 | [diff] [blame] | 1499 | */ |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 1500 | if (reg->umax_value >= BPF_MAX_VAR_OFF) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1501 | verbose(env, "R%d unbounded memory access, make sure to bounds check any array access into a map\n", |
Gianluca Borello | dbcfe5f | 2017-01-09 10:19:46 -0800 | [diff] [blame] | 1502 | regno); |
| 1503 | return -EACCES; |
| 1504 | } |
Yonghong Song | 9fd29c0 | 2017-11-12 14:49:09 -0800 | [diff] [blame] | 1505 | err = __check_map_access(env, regno, reg->umax_value + off, size, |
| 1506 | zero_size_allowed); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1507 | if (err) |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1508 | verbose(env, "R%d max value is outside of the array range\n", |
| 1509 | regno); |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 1510 | |
| 1511 | if (map_value_has_spin_lock(reg->map_ptr)) { |
| 1512 | u32 lock = reg->map_ptr->spin_lock_off; |
| 1513 | |
| 1514 | /* if any part of struct bpf_spin_lock can be touched by |
| 1515 | * load/store reject this program. |
| 1516 | * To check that [x1, x2) overlaps with [y1, y2) |
| 1517 | * it is sufficient to check x1 < y2 && y1 < x2. |
| 1518 | */ |
| 1519 | if (reg->smin_value + off < lock + sizeof(struct bpf_spin_lock) && |
| 1520 | lock < reg->umax_value + off + size) { |
| 1521 | verbose(env, "bpf_spin_lock cannot be accessed directly by load/store\n"); |
| 1522 | return -EACCES; |
| 1523 | } |
| 1524 | } |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1525 | return err; |
Gianluca Borello | dbcfe5f | 2017-01-09 10:19:46 -0800 | [diff] [blame] | 1526 | } |
| 1527 | |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 1528 | #define MAX_PACKET_OFF 0xffff |
| 1529 | |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 1530 | static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, |
Thomas Graf | 3a0af8f | 2016-11-30 17:10:10 +0100 | [diff] [blame] | 1531 | const struct bpf_call_arg_meta *meta, |
| 1532 | enum bpf_access_type t) |
Brenden Blanco | 4acf6c0 | 2016-07-19 12:16:56 -0700 | [diff] [blame] | 1533 | { |
Daniel Borkmann | 36bbef5 | 2016-09-20 00:26:13 +0200 | [diff] [blame] | 1534 | switch (env->prog->type) { |
Daniel Borkmann | 5d66fa7 | 2018-10-24 22:05:45 +0200 | [diff] [blame] | 1535 | /* Program types only with direct read access go here! */ |
Thomas Graf | 3a0af8f | 2016-11-30 17:10:10 +0100 | [diff] [blame] | 1536 | case BPF_PROG_TYPE_LWT_IN: |
| 1537 | case BPF_PROG_TYPE_LWT_OUT: |
Mathieu Xhonneux | 004d4b2 | 2018-05-20 14:58:16 +0100 | [diff] [blame] | 1538 | case BPF_PROG_TYPE_LWT_SEG6LOCAL: |
Martin KaFai Lau | 2dbb9b9 | 2018-08-08 01:01:25 -0700 | [diff] [blame] | 1539 | case BPF_PROG_TYPE_SK_REUSEPORT: |
Daniel Borkmann | 5d66fa7 | 2018-10-24 22:05:45 +0200 | [diff] [blame] | 1540 | case BPF_PROG_TYPE_FLOW_DISSECTOR: |
Daniel Borkmann | d5563d3 | 2018-10-24 22:05:46 +0200 | [diff] [blame] | 1541 | case BPF_PROG_TYPE_CGROUP_SKB: |
Thomas Graf | 3a0af8f | 2016-11-30 17:10:10 +0100 | [diff] [blame] | 1542 | if (t == BPF_WRITE) |
| 1543 | return false; |
Alexander Alemayhu | 7e57fbb | 2017-02-14 00:02:35 +0100 | [diff] [blame] | 1544 | /* fallthrough */ |
Daniel Borkmann | 5d66fa7 | 2018-10-24 22:05:45 +0200 | [diff] [blame] | 1545 | |
| 1546 | /* Program types with direct read + write access go here! */ |
Daniel Borkmann | 36bbef5 | 2016-09-20 00:26:13 +0200 | [diff] [blame] | 1547 | case BPF_PROG_TYPE_SCHED_CLS: |
| 1548 | case BPF_PROG_TYPE_SCHED_ACT: |
Brenden Blanco | 4acf6c0 | 2016-07-19 12:16:56 -0700 | [diff] [blame] | 1549 | case BPF_PROG_TYPE_XDP: |
Thomas Graf | 3a0af8f | 2016-11-30 17:10:10 +0100 | [diff] [blame] | 1550 | case BPF_PROG_TYPE_LWT_XMIT: |
John Fastabend | 8a31db5 | 2017-08-15 22:33:09 -0700 | [diff] [blame] | 1551 | case BPF_PROG_TYPE_SK_SKB: |
John Fastabend | 4f738ad | 2018-03-18 12:57:10 -0700 | [diff] [blame] | 1552 | case BPF_PROG_TYPE_SK_MSG: |
Daniel Borkmann | 36bbef5 | 2016-09-20 00:26:13 +0200 | [diff] [blame] | 1553 | if (meta) |
| 1554 | return meta->pkt_access; |
| 1555 | |
| 1556 | env->seen_direct_write = true; |
Brenden Blanco | 4acf6c0 | 2016-07-19 12:16:56 -0700 | [diff] [blame] | 1557 | return true; |
| 1558 | default: |
| 1559 | return false; |
| 1560 | } |
| 1561 | } |
| 1562 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1563 | static int __check_packet_access(struct bpf_verifier_env *env, u32 regno, |
Yonghong Song | 9fd29c0 | 2017-11-12 14:49:09 -0800 | [diff] [blame] | 1564 | int off, int size, bool zero_size_allowed) |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 1565 | { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 1566 | struct bpf_reg_state *regs = cur_regs(env); |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 1567 | struct bpf_reg_state *reg = ®s[regno]; |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 1568 | |
Yonghong Song | 9fd29c0 | 2017-11-12 14:49:09 -0800 | [diff] [blame] | 1569 | if (off < 0 || size < 0 || (size == 0 && !zero_size_allowed) || |
| 1570 | (u64)off + size > reg->range) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1571 | verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n", |
Alexei Starovoitov | d91b28e | 2016-05-19 18:17:13 -0700 | [diff] [blame] | 1572 | off, size, regno, reg->id, reg->off, reg->range); |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 1573 | return -EACCES; |
| 1574 | } |
| 1575 | return 0; |
| 1576 | } |
| 1577 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1578 | static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, |
Yonghong Song | 9fd29c0 | 2017-11-12 14:49:09 -0800 | [diff] [blame] | 1579 | int size, bool zero_size_allowed) |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1580 | { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 1581 | struct bpf_reg_state *regs = cur_regs(env); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1582 | struct bpf_reg_state *reg = ®s[regno]; |
| 1583 | int err; |
| 1584 | |
| 1585 | /* We may have added a variable offset to the packet pointer; but any |
| 1586 | * reg->range we have comes after that. We are only checking the fixed |
| 1587 | * offset. |
| 1588 | */ |
| 1589 | |
| 1590 | /* We don't allow negative numbers, because we aren't tracking enough |
| 1591 | * detail to prove they're safe. |
| 1592 | */ |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 1593 | if (reg->smin_value < 0) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1594 | verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1595 | regno); |
| 1596 | return -EACCES; |
| 1597 | } |
Yonghong Song | 9fd29c0 | 2017-11-12 14:49:09 -0800 | [diff] [blame] | 1598 | err = __check_packet_access(env, regno, off, size, zero_size_allowed); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1599 | if (err) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1600 | verbose(env, "R%d offset is outside of the packet\n", regno); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1601 | return err; |
| 1602 | } |
Jiong Wang | e647815 | 2018-11-08 04:08:42 -0500 | [diff] [blame] | 1603 | |
| 1604 | /* __check_packet_access has made sure "off + size - 1" is within u16. |
| 1605 | * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff, |
| 1606 | * otherwise find_good_pkt_pointers would have refused to set range info |
| 1607 | * that __check_packet_access would have rejected this pkt access. |
| 1608 | * Therefore, "off + reg->umax_value + size - 1" won't overflow u32. |
| 1609 | */ |
| 1610 | env->prog->aux->max_pkt_offset = |
| 1611 | max_t(u32, env->prog->aux->max_pkt_offset, |
| 1612 | off + reg->umax_value + size - 1); |
| 1613 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1614 | return err; |
| 1615 | } |
| 1616 | |
| 1617 | /* check access to 'struct bpf_context' fields. Supports fixed offsets only */ |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 1618 | static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size, |
Alexei Starovoitov | 19de99f | 2016-06-15 18:25:38 -0700 | [diff] [blame] | 1619 | enum bpf_access_type t, enum bpf_reg_type *reg_type) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1620 | { |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 1621 | struct bpf_insn_access_aux info = { |
| 1622 | .reg_type = *reg_type, |
| 1623 | }; |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 1624 | |
Jakub Kicinski | 4f9218a | 2017-10-16 16:40:55 -0700 | [diff] [blame] | 1625 | if (env->ops->is_valid_access && |
Andrey Ignatov | 5e43f89 | 2018-03-30 15:08:00 -0700 | [diff] [blame] | 1626 | env->ops->is_valid_access(off, size, t, env->prog, &info)) { |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 1627 | /* A non zero info.ctx_field_size indicates that this field is a |
| 1628 | * candidate for later verifier transformation to load the whole |
| 1629 | * field and then apply a mask when accessed with a narrower |
| 1630 | * access than actual ctx access size. A zero info.ctx_field_size |
| 1631 | * will only allow for whole field access and rejects any other |
| 1632 | * type of narrower access. |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 1633 | */ |
Yonghong Song | 2399463 | 2017-06-22 15:07:39 -0700 | [diff] [blame] | 1634 | *reg_type = info.reg_type; |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 1635 | |
Jakub Kicinski | 4f9218a | 2017-10-16 16:40:55 -0700 | [diff] [blame] | 1636 | env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; |
Alexei Starovoitov | 32bbe00 | 2016-04-06 18:43:28 -0700 | [diff] [blame] | 1637 | /* remember the offset of last byte accessed in ctx */ |
| 1638 | if (env->prog->aux->max_ctx_offset < off + size) |
| 1639 | env->prog->aux->max_ctx_offset = off + size; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1640 | return 0; |
Alexei Starovoitov | 32bbe00 | 2016-04-06 18:43:28 -0700 | [diff] [blame] | 1641 | } |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1642 | |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1643 | verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1644 | return -EACCES; |
| 1645 | } |
| 1646 | |
Petar Penkov | d58e468 | 2018-09-14 07:46:18 -0700 | [diff] [blame] | 1647 | static int check_flow_keys_access(struct bpf_verifier_env *env, int off, |
| 1648 | int size) |
| 1649 | { |
| 1650 | if (size < 0 || off < 0 || |
| 1651 | (u64)off + size > sizeof(struct bpf_flow_keys)) { |
| 1652 | verbose(env, "invalid access to flow keys off=%d size=%d\n", |
| 1653 | off, size); |
| 1654 | return -EACCES; |
| 1655 | } |
| 1656 | return 0; |
| 1657 | } |
| 1658 | |
Martin KaFai Lau | 5f45664 | 2019-02-08 22:25:54 -0800 | [diff] [blame] | 1659 | static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, |
| 1660 | u32 regno, int off, int size, |
| 1661 | enum bpf_access_type t) |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 1662 | { |
| 1663 | struct bpf_reg_state *regs = cur_regs(env); |
| 1664 | struct bpf_reg_state *reg = ®s[regno]; |
Martin KaFai Lau | 5f45664 | 2019-02-08 22:25:54 -0800 | [diff] [blame] | 1665 | struct bpf_insn_access_aux info = {}; |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 1666 | bool valid; |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 1667 | |
| 1668 | if (reg->smin_value < 0) { |
| 1669 | verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", |
| 1670 | regno); |
| 1671 | return -EACCES; |
| 1672 | } |
| 1673 | |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 1674 | switch (reg->type) { |
| 1675 | case PTR_TO_SOCK_COMMON: |
| 1676 | valid = bpf_sock_common_is_valid_access(off, size, t, &info); |
| 1677 | break; |
| 1678 | case PTR_TO_SOCKET: |
| 1679 | valid = bpf_sock_is_valid_access(off, size, t, &info); |
| 1680 | break; |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 1681 | case PTR_TO_TCP_SOCK: |
| 1682 | valid = bpf_tcp_sock_is_valid_access(off, size, t, &info); |
| 1683 | break; |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 1684 | default: |
| 1685 | valid = false; |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 1686 | } |
| 1687 | |
Martin KaFai Lau | 5f45664 | 2019-02-08 22:25:54 -0800 | [diff] [blame] | 1688 | |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 1689 | if (valid) { |
| 1690 | env->insn_aux_data[insn_idx].ctx_field_size = |
| 1691 | info.ctx_field_size; |
| 1692 | return 0; |
| 1693 | } |
| 1694 | |
| 1695 | verbose(env, "R%d invalid %s access off=%d size=%d\n", |
| 1696 | regno, reg_type_str[reg->type], off, size); |
| 1697 | |
| 1698 | return -EACCES; |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 1699 | } |
| 1700 | |
Daniel Borkmann | 4cabc5b | 2017-07-21 00:00:21 +0200 | [diff] [blame] | 1701 | static bool __is_pointer_value(bool allow_ptr_leaks, |
| 1702 | const struct bpf_reg_state *reg) |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 1703 | { |
Daniel Borkmann | 4cabc5b | 2017-07-21 00:00:21 +0200 | [diff] [blame] | 1704 | if (allow_ptr_leaks) |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 1705 | return false; |
| 1706 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1707 | return reg->type != SCALAR_VALUE; |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 1708 | } |
| 1709 | |
Daniel Borkmann | 2a159c6 | 2018-10-21 02:09:24 +0200 | [diff] [blame] | 1710 | static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) |
| 1711 | { |
| 1712 | return cur_regs(env) + regno; |
| 1713 | } |
| 1714 | |
Daniel Borkmann | 4cabc5b | 2017-07-21 00:00:21 +0200 | [diff] [blame] | 1715 | static bool is_pointer_value(struct bpf_verifier_env *env, int regno) |
| 1716 | { |
Daniel Borkmann | 2a159c6 | 2018-10-21 02:09:24 +0200 | [diff] [blame] | 1717 | return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno)); |
Daniel Borkmann | 4cabc5b | 2017-07-21 00:00:21 +0200 | [diff] [blame] | 1718 | } |
| 1719 | |
Daniel Borkmann | f37a8cb | 2018-01-16 23:30:10 +0100 | [diff] [blame] | 1720 | static bool is_ctx_reg(struct bpf_verifier_env *env, int regno) |
| 1721 | { |
Daniel Borkmann | 2a159c6 | 2018-10-21 02:09:24 +0200 | [diff] [blame] | 1722 | const struct bpf_reg_state *reg = reg_state(env, regno); |
Daniel Borkmann | f37a8cb | 2018-01-16 23:30:10 +0100 | [diff] [blame] | 1723 | |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 1724 | return reg->type == PTR_TO_CTX; |
| 1725 | } |
| 1726 | |
| 1727 | static bool is_sk_reg(struct bpf_verifier_env *env, int regno) |
| 1728 | { |
| 1729 | const struct bpf_reg_state *reg = reg_state(env, regno); |
| 1730 | |
| 1731 | return type_is_sk_pointer(reg->type); |
Daniel Borkmann | f37a8cb | 2018-01-16 23:30:10 +0100 | [diff] [blame] | 1732 | } |
| 1733 | |
Daniel Borkmann | ca36960 | 2018-02-23 22:29:05 +0100 | [diff] [blame] | 1734 | static bool is_pkt_reg(struct bpf_verifier_env *env, int regno) |
| 1735 | { |
Daniel Borkmann | 2a159c6 | 2018-10-21 02:09:24 +0200 | [diff] [blame] | 1736 | const struct bpf_reg_state *reg = reg_state(env, regno); |
Daniel Borkmann | ca36960 | 2018-02-23 22:29:05 +0100 | [diff] [blame] | 1737 | |
| 1738 | return type_is_pkt_pointer(reg->type); |
| 1739 | } |
| 1740 | |
Daniel Borkmann | 4b5defd | 2018-10-21 02:09:25 +0200 | [diff] [blame] | 1741 | static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno) |
| 1742 | { |
| 1743 | const struct bpf_reg_state *reg = reg_state(env, regno); |
| 1744 | |
| 1745 | /* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */ |
| 1746 | return reg->type == PTR_TO_FLOW_KEYS; |
| 1747 | } |
| 1748 | |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1749 | static int check_pkt_ptr_alignment(struct bpf_verifier_env *env, |
| 1750 | const struct bpf_reg_state *reg, |
David S. Miller | d117441 | 2017-05-10 11:22:52 -0700 | [diff] [blame] | 1751 | int off, int size, bool strict) |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 1752 | { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1753 | struct tnum reg_off; |
David S. Miller | e07b98d | 2017-05-10 11:38:07 -0700 | [diff] [blame] | 1754 | int ip_align; |
David S. Miller | d117441 | 2017-05-10 11:22:52 -0700 | [diff] [blame] | 1755 | |
| 1756 | /* Byte size accesses are always allowed. */ |
| 1757 | if (!strict || size == 1) |
| 1758 | return 0; |
| 1759 | |
David S. Miller | e4eda88 | 2017-05-22 12:27:07 -0400 | [diff] [blame] | 1760 | /* For platforms that do not have a Kconfig enabling |
| 1761 | * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of |
| 1762 | * NET_IP_ALIGN is universally set to '2'. And on platforms |
| 1763 | * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get |
| 1764 | * to this code only in strict mode where we want to emulate |
| 1765 | * the NET_IP_ALIGN==2 checking. Therefore use an |
| 1766 | * unconditional IP align value of '2'. |
David S. Miller | e07b98d | 2017-05-10 11:38:07 -0700 | [diff] [blame] | 1767 | */ |
David S. Miller | e4eda88 | 2017-05-22 12:27:07 -0400 | [diff] [blame] | 1768 | ip_align = 2; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1769 | |
| 1770 | reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off)); |
| 1771 | if (!tnum_is_aligned(reg_off, size)) { |
| 1772 | char tn_buf[48]; |
| 1773 | |
| 1774 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1775 | verbose(env, |
| 1776 | "misaligned packet access off %d+%s+%d+%d size %d\n", |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1777 | ip_align, tn_buf, reg->off, off, size); |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 1778 | return -EACCES; |
| 1779 | } |
Daniel Borkmann | 79adffc | 2017-03-31 02:24:03 +0200 | [diff] [blame] | 1780 | |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 1781 | return 0; |
| 1782 | } |
| 1783 | |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1784 | static int check_generic_ptr_alignment(struct bpf_verifier_env *env, |
| 1785 | const struct bpf_reg_state *reg, |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1786 | const char *pointer_desc, |
| 1787 | int off, int size, bool strict) |
Daniel Borkmann | 79adffc | 2017-03-31 02:24:03 +0200 | [diff] [blame] | 1788 | { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1789 | struct tnum reg_off; |
| 1790 | |
| 1791 | /* Byte size accesses are always allowed. */ |
| 1792 | if (!strict || size == 1) |
| 1793 | return 0; |
| 1794 | |
| 1795 | reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off)); |
| 1796 | if (!tnum_is_aligned(reg_off, size)) { |
| 1797 | char tn_buf[48]; |
| 1798 | |
| 1799 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1800 | verbose(env, "misaligned %saccess off %s+%d+%d size %d\n", |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1801 | pointer_desc, tn_buf, reg->off, off, size); |
Daniel Borkmann | 79adffc | 2017-03-31 02:24:03 +0200 | [diff] [blame] | 1802 | return -EACCES; |
| 1803 | } |
| 1804 | |
| 1805 | return 0; |
| 1806 | } |
| 1807 | |
David S. Miller | e07b98d | 2017-05-10 11:38:07 -0700 | [diff] [blame] | 1808 | static int check_ptr_alignment(struct bpf_verifier_env *env, |
Daniel Borkmann | ca36960 | 2018-02-23 22:29:05 +0100 | [diff] [blame] | 1809 | const struct bpf_reg_state *reg, int off, |
| 1810 | int size, bool strict_alignment_once) |
Daniel Borkmann | 79adffc | 2017-03-31 02:24:03 +0200 | [diff] [blame] | 1811 | { |
Daniel Borkmann | ca36960 | 2018-02-23 22:29:05 +0100 | [diff] [blame] | 1812 | bool strict = env->strict_alignment || strict_alignment_once; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1813 | const char *pointer_desc = ""; |
David S. Miller | d117441 | 2017-05-10 11:22:52 -0700 | [diff] [blame] | 1814 | |
Daniel Borkmann | 79adffc | 2017-03-31 02:24:03 +0200 | [diff] [blame] | 1815 | switch (reg->type) { |
| 1816 | case PTR_TO_PACKET: |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 1817 | case PTR_TO_PACKET_META: |
| 1818 | /* Special case, because of NET_IP_ALIGN. Given metadata sits |
| 1819 | * right in front, treat it the very same way. |
| 1820 | */ |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1821 | return check_pkt_ptr_alignment(env, reg, off, size, strict); |
Petar Penkov | d58e468 | 2018-09-14 07:46:18 -0700 | [diff] [blame] | 1822 | case PTR_TO_FLOW_KEYS: |
| 1823 | pointer_desc = "flow keys "; |
| 1824 | break; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1825 | case PTR_TO_MAP_VALUE: |
| 1826 | pointer_desc = "value "; |
| 1827 | break; |
| 1828 | case PTR_TO_CTX: |
| 1829 | pointer_desc = "context "; |
| 1830 | break; |
| 1831 | case PTR_TO_STACK: |
| 1832 | pointer_desc = "stack "; |
Jann Horn | a5ec6ae | 2017-12-18 20:11:58 -0800 | [diff] [blame] | 1833 | /* The stack spill tracking logic in check_stack_write() |
| 1834 | * and check_stack_read() relies on stack accesses being |
| 1835 | * aligned. |
| 1836 | */ |
| 1837 | strict = true; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1838 | break; |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 1839 | case PTR_TO_SOCKET: |
| 1840 | pointer_desc = "sock "; |
| 1841 | break; |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 1842 | case PTR_TO_SOCK_COMMON: |
| 1843 | pointer_desc = "sock_common "; |
| 1844 | break; |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 1845 | case PTR_TO_TCP_SOCK: |
| 1846 | pointer_desc = "tcp_sock "; |
| 1847 | break; |
Daniel Borkmann | 79adffc | 2017-03-31 02:24:03 +0200 | [diff] [blame] | 1848 | default: |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1849 | break; |
Daniel Borkmann | 79adffc | 2017-03-31 02:24:03 +0200 | [diff] [blame] | 1850 | } |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1851 | return check_generic_ptr_alignment(env, reg, pointer_desc, off, size, |
| 1852 | strict); |
Daniel Borkmann | 79adffc | 2017-03-31 02:24:03 +0200 | [diff] [blame] | 1853 | } |
| 1854 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1855 | static int update_stack_depth(struct bpf_verifier_env *env, |
| 1856 | const struct bpf_func_state *func, |
| 1857 | int off) |
| 1858 | { |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 1859 | u16 stack = env->subprog_info[func->subprogno].stack_depth; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1860 | |
| 1861 | if (stack >= -off) |
| 1862 | return 0; |
| 1863 | |
| 1864 | /* update known max for given subprogram */ |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 1865 | env->subprog_info[func->subprogno].stack_depth = -off; |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 1866 | return 0; |
| 1867 | } |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1868 | |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 1869 | /* starting from main bpf function walk all instructions of the function |
| 1870 | * and recursively walk all callees that given function can call. |
| 1871 | * Ignore jump and exit insns. |
| 1872 | * Since recursion is prevented by check_cfg() this algorithm |
| 1873 | * only needs a local stack of MAX_CALL_FRAMES to remember callsites |
| 1874 | */ |
| 1875 | static int check_max_stack_depth(struct bpf_verifier_env *env) |
| 1876 | { |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 1877 | int depth = 0, frame = 0, idx = 0, i = 0, subprog_end; |
| 1878 | struct bpf_subprog_info *subprog = env->subprog_info; |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 1879 | struct bpf_insn *insn = env->prog->insnsi; |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 1880 | int ret_insn[MAX_CALL_FRAMES]; |
| 1881 | int ret_prog[MAX_CALL_FRAMES]; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1882 | |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 1883 | process_func: |
| 1884 | /* round up to 32-bytes, since this is granularity |
| 1885 | * of interpreter stack size |
| 1886 | */ |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 1887 | depth += round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 1888 | if (depth > MAX_BPF_STACK) { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1889 | verbose(env, "combined stack size of %d calls is %d. Too large\n", |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 1890 | frame + 1, depth); |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1891 | return -EACCES; |
| 1892 | } |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 1893 | continue_func: |
Jiong Wang | 4cb3d99 | 2018-05-02 16:17:19 -0400 | [diff] [blame] | 1894 | subprog_end = subprog[idx + 1].start; |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 1895 | for (; i < subprog_end; i++) { |
| 1896 | if (insn[i].code != (BPF_JMP | BPF_CALL)) |
| 1897 | continue; |
| 1898 | if (insn[i].src_reg != BPF_PSEUDO_CALL) |
| 1899 | continue; |
| 1900 | /* remember insn and function to return to */ |
| 1901 | ret_insn[frame] = i + 1; |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 1902 | ret_prog[frame] = idx; |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 1903 | |
| 1904 | /* find the callee */ |
| 1905 | i = i + insn[i].imm + 1; |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 1906 | idx = find_subprog(env, i); |
| 1907 | if (idx < 0) { |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 1908 | WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", |
| 1909 | i); |
| 1910 | return -EFAULT; |
| 1911 | } |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 1912 | frame++; |
| 1913 | if (frame >= MAX_CALL_FRAMES) { |
| 1914 | WARN_ONCE(1, "verifier bug. Call stack is too deep\n"); |
| 1915 | return -EFAULT; |
| 1916 | } |
| 1917 | goto process_func; |
| 1918 | } |
| 1919 | /* end of for() loop means the last insn of the 'subprog' |
| 1920 | * was reached. Doesn't matter whether it was JA or EXIT |
| 1921 | */ |
| 1922 | if (frame == 0) |
| 1923 | return 0; |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 1924 | depth -= round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 1925 | frame--; |
| 1926 | i = ret_insn[frame]; |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 1927 | idx = ret_prog[frame]; |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 1928 | goto continue_func; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1929 | } |
| 1930 | |
David S. Miller | 19d28fb | 2018-01-11 21:27:54 -0500 | [diff] [blame] | 1931 | #ifndef CONFIG_BPF_JIT_ALWAYS_ON |
Alexei Starovoitov | 1ea47e0 | 2017-12-14 17:55:13 -0800 | [diff] [blame] | 1932 | static int get_callee_stack_depth(struct bpf_verifier_env *env, |
| 1933 | const struct bpf_insn *insn, int idx) |
| 1934 | { |
| 1935 | int start = idx + insn->imm + 1, subprog; |
| 1936 | |
| 1937 | subprog = find_subprog(env, start); |
| 1938 | if (subprog < 0) { |
| 1939 | WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", |
| 1940 | start); |
| 1941 | return -EFAULT; |
| 1942 | } |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 1943 | return env->subprog_info[subprog].stack_depth; |
Alexei Starovoitov | 1ea47e0 | 2017-12-14 17:55:13 -0800 | [diff] [blame] | 1944 | } |
David S. Miller | 19d28fb | 2018-01-11 21:27:54 -0500 | [diff] [blame] | 1945 | #endif |
Alexei Starovoitov | 1ea47e0 | 2017-12-14 17:55:13 -0800 | [diff] [blame] | 1946 | |
Daniel Borkmann | 58990d1 | 2018-06-07 17:40:03 +0200 | [diff] [blame] | 1947 | static int check_ctx_reg(struct bpf_verifier_env *env, |
| 1948 | const struct bpf_reg_state *reg, int regno) |
| 1949 | { |
| 1950 | /* Access to ctx or passing it to a helper is only allowed in |
| 1951 | * its original, unmodified form. |
| 1952 | */ |
| 1953 | |
| 1954 | if (reg->off) { |
| 1955 | verbose(env, "dereference of modified ctx ptr R%d off=%d disallowed\n", |
| 1956 | regno, reg->off); |
| 1957 | return -EACCES; |
| 1958 | } |
| 1959 | |
| 1960 | if (!tnum_is_const(reg->var_off) || reg->var_off.value) { |
| 1961 | char tn_buf[48]; |
| 1962 | |
| 1963 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
| 1964 | verbose(env, "variable ctx access var_off=%s disallowed\n", tn_buf); |
| 1965 | return -EACCES; |
| 1966 | } |
| 1967 | |
| 1968 | return 0; |
| 1969 | } |
| 1970 | |
Jann Horn | 0c17d1d | 2017-12-18 20:11:55 -0800 | [diff] [blame] | 1971 | /* truncate register to smaller size (in bytes) |
| 1972 | * must be called with size < BPF_REG_SIZE |
| 1973 | */ |
| 1974 | static void coerce_reg_to_size(struct bpf_reg_state *reg, int size) |
| 1975 | { |
| 1976 | u64 mask; |
| 1977 | |
| 1978 | /* clear high bits in bit representation */ |
| 1979 | reg->var_off = tnum_cast(reg->var_off, size); |
| 1980 | |
| 1981 | /* fix arithmetic bounds */ |
| 1982 | mask = ((u64)1 << (size * 8)) - 1; |
| 1983 | if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) { |
| 1984 | reg->umin_value &= mask; |
| 1985 | reg->umax_value &= mask; |
| 1986 | } else { |
| 1987 | reg->umin_value = 0; |
| 1988 | reg->umax_value = mask; |
| 1989 | } |
| 1990 | reg->smin_value = reg->umin_value; |
| 1991 | reg->smax_value = reg->umax_value; |
| 1992 | } |
| 1993 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1994 | /* check whether memory at (regno + off) is accessible for t = (read | write) |
| 1995 | * if t==write, value_regno is a register which value is stored into memory |
| 1996 | * if t==read, value_regno is a register which will receive the value from memory |
| 1997 | * if t==write && value_regno==-1, some unknown value is stored into memory |
| 1998 | * if t==read && value_regno==-1, don't care what we read from memory |
| 1999 | */ |
Daniel Borkmann | ca36960 | 2018-02-23 22:29:05 +0100 | [diff] [blame] | 2000 | static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, |
| 2001 | int off, int bpf_size, enum bpf_access_type t, |
| 2002 | int value_regno, bool strict_alignment_once) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2003 | { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 2004 | struct bpf_reg_state *regs = cur_regs(env); |
| 2005 | struct bpf_reg_state *reg = regs + regno; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 2006 | struct bpf_func_state *state; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2007 | int size, err = 0; |
| 2008 | |
| 2009 | size = bpf_size_to_bytes(bpf_size); |
| 2010 | if (size < 0) |
| 2011 | return size; |
| 2012 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 2013 | /* alignment checks will add in reg->off themselves */ |
Daniel Borkmann | ca36960 | 2018-02-23 22:29:05 +0100 | [diff] [blame] | 2014 | err = check_ptr_alignment(env, reg, off, size, strict_alignment_once); |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 2015 | if (err) |
| 2016 | return err; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2017 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 2018 | /* for access checks, reg->off is just part of off */ |
| 2019 | off += reg->off; |
| 2020 | |
| 2021 | if (reg->type == PTR_TO_MAP_VALUE) { |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 2022 | if (t == BPF_WRITE && value_regno >= 0 && |
| 2023 | is_pointer_value(env, value_regno)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2024 | verbose(env, "R%d leaks addr into map\n", value_regno); |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 2025 | return -EACCES; |
| 2026 | } |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 2027 | |
Yonghong Song | 9fd29c0 | 2017-11-12 14:49:09 -0800 | [diff] [blame] | 2028 | err = check_map_access(env, regno, off, size, false); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2029 | if (!err && t == BPF_READ && value_regno >= 0) |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 2030 | mark_reg_unknown(env, regs, value_regno); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2031 | |
Alexei Starovoitov | 1a0dc1a | 2016-05-05 19:49:09 -0700 | [diff] [blame] | 2032 | } else if (reg->type == PTR_TO_CTX) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 2033 | enum bpf_reg_type reg_type = SCALAR_VALUE; |
Alexei Starovoitov | 19de99f | 2016-06-15 18:25:38 -0700 | [diff] [blame] | 2034 | |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 2035 | if (t == BPF_WRITE && value_regno >= 0 && |
| 2036 | is_pointer_value(env, value_regno)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2037 | verbose(env, "R%d leaks addr into ctx\n", value_regno); |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 2038 | return -EACCES; |
| 2039 | } |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 2040 | |
Daniel Borkmann | 58990d1 | 2018-06-07 17:40:03 +0200 | [diff] [blame] | 2041 | err = check_ctx_reg(env, reg, regno); |
| 2042 | if (err < 0) |
| 2043 | return err; |
| 2044 | |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 2045 | err = check_ctx_access(env, insn_idx, off, size, t, ®_type); |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 2046 | if (!err && t == BPF_READ && value_regno >= 0) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 2047 | /* ctx access returns either a scalar, or a |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 2048 | * PTR_TO_PACKET[_META,_END]. In the latter |
| 2049 | * case, we know the offset is zero. |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 2050 | */ |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 2051 | if (reg_type == SCALAR_VALUE) { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 2052 | mark_reg_unknown(env, regs, value_regno); |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 2053 | } else { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 2054 | mark_reg_known_zero(env, regs, |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2055 | value_regno); |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 2056 | if (reg_type_may_be_null(reg_type)) |
| 2057 | regs[value_regno].id = ++env->id_gen; |
| 2058 | } |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 2059 | regs[value_regno].type = reg_type; |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 2060 | } |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2061 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 2062 | } else if (reg->type == PTR_TO_STACK) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 2063 | off += reg->var_off.value; |
Daniel Borkmann | e4298d2 | 2019-01-03 00:58:31 +0100 | [diff] [blame] | 2064 | err = check_stack_access(env, reg, off, size); |
| 2065 | if (err) |
| 2066 | return err; |
Alexei Starovoitov | 8726679 | 2017-05-30 13:31:29 -0700 | [diff] [blame] | 2067 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 2068 | state = func(env, reg); |
| 2069 | err = update_stack_depth(env, state, off); |
| 2070 | if (err) |
| 2071 | return err; |
Alexei Starovoitov | 8726679 | 2017-05-30 13:31:29 -0700 | [diff] [blame] | 2072 | |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 2073 | if (t == BPF_WRITE) |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2074 | err = check_stack_write(env, state, off, size, |
Alexei Starovoitov | af86ca4 | 2018-05-15 09:27:05 -0700 | [diff] [blame] | 2075 | value_regno, insn_idx); |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 2076 | else |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2077 | err = check_stack_read(env, state, off, size, |
| 2078 | value_regno); |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 2079 | } else if (reg_is_pkt_pointer(reg)) { |
Thomas Graf | 3a0af8f | 2016-11-30 17:10:10 +0100 | [diff] [blame] | 2080 | if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2081 | verbose(env, "cannot write into packet\n"); |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 2082 | return -EACCES; |
| 2083 | } |
Brenden Blanco | 4acf6c0 | 2016-07-19 12:16:56 -0700 | [diff] [blame] | 2084 | if (t == BPF_WRITE && value_regno >= 0 && |
| 2085 | is_pointer_value(env, value_regno)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2086 | verbose(env, "R%d leaks addr into packet\n", |
| 2087 | value_regno); |
Brenden Blanco | 4acf6c0 | 2016-07-19 12:16:56 -0700 | [diff] [blame] | 2088 | return -EACCES; |
| 2089 | } |
Yonghong Song | 9fd29c0 | 2017-11-12 14:49:09 -0800 | [diff] [blame] | 2090 | err = check_packet_access(env, regno, off, size, false); |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 2091 | if (!err && t == BPF_READ && value_regno >= 0) |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 2092 | mark_reg_unknown(env, regs, value_regno); |
Petar Penkov | d58e468 | 2018-09-14 07:46:18 -0700 | [diff] [blame] | 2093 | } else if (reg->type == PTR_TO_FLOW_KEYS) { |
| 2094 | if (t == BPF_WRITE && value_regno >= 0 && |
| 2095 | is_pointer_value(env, value_regno)) { |
| 2096 | verbose(env, "R%d leaks addr into flow keys\n", |
| 2097 | value_regno); |
| 2098 | return -EACCES; |
| 2099 | } |
| 2100 | |
| 2101 | err = check_flow_keys_access(env, off, size); |
| 2102 | if (!err && t == BPF_READ && value_regno >= 0) |
| 2103 | mark_reg_unknown(env, regs, value_regno); |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 2104 | } else if (type_is_sk_pointer(reg->type)) { |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 2105 | if (t == BPF_WRITE) { |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 2106 | verbose(env, "R%d cannot write into %s\n", |
| 2107 | regno, reg_type_str[reg->type]); |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 2108 | return -EACCES; |
| 2109 | } |
Martin KaFai Lau | 5f45664 | 2019-02-08 22:25:54 -0800 | [diff] [blame] | 2110 | err = check_sock_access(env, insn_idx, regno, off, size, t); |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 2111 | if (!err && value_regno >= 0) |
| 2112 | mark_reg_unknown(env, regs, value_regno); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2113 | } else { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2114 | verbose(env, "R%d invalid mem access '%s'\n", regno, |
| 2115 | reg_type_str[reg->type]); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2116 | return -EACCES; |
| 2117 | } |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 2118 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 2119 | if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ && |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 2120 | regs[value_regno].type == SCALAR_VALUE) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 2121 | /* b/h/w load zero-extends, mark upper bits as known 0 */ |
Jann Horn | 0c17d1d | 2017-12-18 20:11:55 -0800 | [diff] [blame] | 2122 | coerce_reg_to_size(®s[value_regno], size); |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 2123 | } |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2124 | return err; |
| 2125 | } |
| 2126 | |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 2127 | static int check_xadd(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2128 | { |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2129 | int err; |
| 2130 | |
| 2131 | if ((BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) || |
| 2132 | insn->imm != 0) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2133 | verbose(env, "BPF_XADD uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2134 | return -EINVAL; |
| 2135 | } |
| 2136 | |
| 2137 | /* check src1 operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 2138 | err = check_reg_arg(env, insn->src_reg, SRC_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2139 | if (err) |
| 2140 | return err; |
| 2141 | |
| 2142 | /* check src2 operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 2143 | err = check_reg_arg(env, insn->dst_reg, SRC_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2144 | if (err) |
| 2145 | return err; |
| 2146 | |
Daniel Borkmann | 6bdf6ab | 2017-06-29 03:04:59 +0200 | [diff] [blame] | 2147 | if (is_pointer_value(env, insn->src_reg)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2148 | verbose(env, "R%d leaks addr into mem\n", insn->src_reg); |
Daniel Borkmann | 6bdf6ab | 2017-06-29 03:04:59 +0200 | [diff] [blame] | 2149 | return -EACCES; |
| 2150 | } |
| 2151 | |
Daniel Borkmann | ca36960 | 2018-02-23 22:29:05 +0100 | [diff] [blame] | 2152 | if (is_ctx_reg(env, insn->dst_reg) || |
Daniel Borkmann | 4b5defd | 2018-10-21 02:09:25 +0200 | [diff] [blame] | 2153 | is_pkt_reg(env, insn->dst_reg) || |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 2154 | is_flow_key_reg(env, insn->dst_reg) || |
| 2155 | is_sk_reg(env, insn->dst_reg)) { |
Daniel Borkmann | ca36960 | 2018-02-23 22:29:05 +0100 | [diff] [blame] | 2156 | verbose(env, "BPF_XADD stores into R%d %s is not allowed\n", |
Daniel Borkmann | 2a159c6 | 2018-10-21 02:09:24 +0200 | [diff] [blame] | 2157 | insn->dst_reg, |
| 2158 | reg_type_str[reg_state(env, insn->dst_reg)->type]); |
Daniel Borkmann | f37a8cb | 2018-01-16 23:30:10 +0100 | [diff] [blame] | 2159 | return -EACCES; |
| 2160 | } |
| 2161 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2162 | /* check whether atomic_add can read the memory */ |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 2163 | err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, |
Daniel Borkmann | ca36960 | 2018-02-23 22:29:05 +0100 | [diff] [blame] | 2164 | BPF_SIZE(insn->code), BPF_READ, -1, true); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2165 | if (err) |
| 2166 | return err; |
| 2167 | |
| 2168 | /* check whether atomic_add can write into the same memory */ |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 2169 | return check_mem_access(env, insn_idx, insn->dst_reg, insn->off, |
Daniel Borkmann | ca36960 | 2018-02-23 22:29:05 +0100 | [diff] [blame] | 2170 | BPF_SIZE(insn->code), BPF_WRITE, -1, true); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2171 | } |
| 2172 | |
Andrey Ignatov | 2011fcc | 2019-03-28 18:01:57 -0700 | [diff] [blame] | 2173 | static int __check_stack_boundary(struct bpf_verifier_env *env, u32 regno, |
| 2174 | int off, int access_size, |
| 2175 | bool zero_size_allowed) |
| 2176 | { |
| 2177 | struct bpf_reg_state *reg = reg_state(env, regno); |
| 2178 | |
| 2179 | if (off >= 0 || off < -MAX_BPF_STACK || off + access_size > 0 || |
| 2180 | access_size < 0 || (access_size == 0 && !zero_size_allowed)) { |
| 2181 | if (tnum_is_const(reg->var_off)) { |
| 2182 | verbose(env, "invalid stack type R%d off=%d access_size=%d\n", |
| 2183 | regno, off, access_size); |
| 2184 | } else { |
| 2185 | char tn_buf[48]; |
| 2186 | |
| 2187 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
| 2188 | verbose(env, "invalid stack type R%d var_off=%s access_size=%d\n", |
| 2189 | regno, tn_buf, access_size); |
| 2190 | } |
| 2191 | return -EACCES; |
| 2192 | } |
| 2193 | return 0; |
| 2194 | } |
| 2195 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2196 | /* when register 'regno' is passed into function that will read 'access_size' |
| 2197 | * bytes from that pointer, make sure that it's within stack boundary |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 2198 | * and all elements of stack are initialized. |
| 2199 | * Unlike most pointer bounds-checking functions, this one doesn't take an |
| 2200 | * 'off' argument, so it has to add in reg->off itself. |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2201 | */ |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 2202 | static int check_stack_boundary(struct bpf_verifier_env *env, int regno, |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 2203 | int access_size, bool zero_size_allowed, |
| 2204 | struct bpf_call_arg_meta *meta) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2205 | { |
Daniel Borkmann | 2a159c6 | 2018-10-21 02:09:24 +0200 | [diff] [blame] | 2206 | struct bpf_reg_state *reg = reg_state(env, regno); |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 2207 | struct bpf_func_state *state = func(env, reg); |
Andrey Ignatov | 2011fcc | 2019-03-28 18:01:57 -0700 | [diff] [blame] | 2208 | int err, min_off, max_off, i, slot, spi; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2209 | |
Alexei Starovoitov | 914cb78 | 2017-11-30 21:31:40 -0800 | [diff] [blame] | 2210 | if (reg->type != PTR_TO_STACK) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 2211 | /* Allow zero-byte read from NULL, regardless of pointer type */ |
Daniel Borkmann | 8e2fe1d9 | 2016-02-19 23:05:22 +0100 | [diff] [blame] | 2212 | if (zero_size_allowed && access_size == 0 && |
Alexei Starovoitov | 914cb78 | 2017-11-30 21:31:40 -0800 | [diff] [blame] | 2213 | register_is_null(reg)) |
Daniel Borkmann | 8e2fe1d9 | 2016-02-19 23:05:22 +0100 | [diff] [blame] | 2214 | return 0; |
| 2215 | |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2216 | verbose(env, "R%d type=%s expected=%s\n", regno, |
Alexei Starovoitov | 914cb78 | 2017-11-30 21:31:40 -0800 | [diff] [blame] | 2217 | reg_type_str[reg->type], |
Daniel Borkmann | 8e2fe1d9 | 2016-02-19 23:05:22 +0100 | [diff] [blame] | 2218 | reg_type_str[PTR_TO_STACK]); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2219 | return -EACCES; |
Daniel Borkmann | 8e2fe1d9 | 2016-02-19 23:05:22 +0100 | [diff] [blame] | 2220 | } |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2221 | |
Andrey Ignatov | 2011fcc | 2019-03-28 18:01:57 -0700 | [diff] [blame] | 2222 | if (tnum_is_const(reg->var_off)) { |
| 2223 | min_off = max_off = reg->var_off.value + reg->off; |
| 2224 | err = __check_stack_boundary(env, regno, min_off, access_size, |
| 2225 | zero_size_allowed); |
| 2226 | if (err) |
| 2227 | return err; |
| 2228 | } else { |
Andrey Ignatov | 088ec26 | 2019-04-03 23:22:39 -0700 | [diff] [blame] | 2229 | /* Variable offset is prohibited for unprivileged mode for |
| 2230 | * simplicity since it requires corresponding support in |
| 2231 | * Spectre masking for stack ALU. |
| 2232 | * See also retrieve_ptr_limit(). |
| 2233 | */ |
| 2234 | if (!env->allow_ptr_leaks) { |
| 2235 | char tn_buf[48]; |
| 2236 | |
| 2237 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
| 2238 | verbose(env, "R%d indirect variable offset stack access prohibited for !root, var_off=%s\n", |
| 2239 | regno, tn_buf); |
| 2240 | return -EACCES; |
| 2241 | } |
Andrey Ignatov | f2bcd05 | 2019-04-03 23:22:37 -0700 | [diff] [blame] | 2242 | /* Only initialized buffer on stack is allowed to be accessed |
| 2243 | * with variable offset. With uninitialized buffer it's hard to |
| 2244 | * guarantee that whole memory is marked as initialized on |
| 2245 | * helper return since specific bounds are unknown what may |
| 2246 | * cause uninitialized stack leaking. |
| 2247 | */ |
| 2248 | if (meta && meta->raw_mode) |
| 2249 | meta = NULL; |
| 2250 | |
Andrey Ignatov | 2011fcc | 2019-03-28 18:01:57 -0700 | [diff] [blame] | 2251 | min_off = reg->smin_value + reg->off; |
| 2252 | max_off = reg->umax_value + reg->off; |
| 2253 | err = __check_stack_boundary(env, regno, min_off, access_size, |
| 2254 | zero_size_allowed); |
| 2255 | if (err) |
| 2256 | return err; |
| 2257 | err = __check_stack_boundary(env, regno, max_off, access_size, |
| 2258 | zero_size_allowed); |
| 2259 | if (err) |
| 2260 | return err; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2261 | } |
| 2262 | |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 2263 | if (meta && meta->raw_mode) { |
| 2264 | meta->access_size = access_size; |
| 2265 | meta->regno = regno; |
| 2266 | return 0; |
| 2267 | } |
| 2268 | |
Andrey Ignatov | 2011fcc | 2019-03-28 18:01:57 -0700 | [diff] [blame] | 2269 | for (i = min_off; i < max_off + access_size; i++) { |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 2270 | u8 *stype; |
| 2271 | |
Andrey Ignatov | 2011fcc | 2019-03-28 18:01:57 -0700 | [diff] [blame] | 2272 | slot = -i - 1; |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 2273 | spi = slot / BPF_REG_SIZE; |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 2274 | if (state->allocated_stack <= slot) |
| 2275 | goto err; |
| 2276 | stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; |
| 2277 | if (*stype == STACK_MISC) |
| 2278 | goto mark; |
| 2279 | if (*stype == STACK_ZERO) { |
| 2280 | /* helper can write anything into the stack */ |
| 2281 | *stype = STACK_MISC; |
| 2282 | goto mark; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2283 | } |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 2284 | err: |
Andrey Ignatov | 2011fcc | 2019-03-28 18:01:57 -0700 | [diff] [blame] | 2285 | if (tnum_is_const(reg->var_off)) { |
| 2286 | verbose(env, "invalid indirect read from stack off %d+%d size %d\n", |
| 2287 | min_off, i - min_off, access_size); |
| 2288 | } else { |
| 2289 | char tn_buf[48]; |
| 2290 | |
| 2291 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
| 2292 | verbose(env, "invalid indirect read from stack var_off %s+%d size %d\n", |
| 2293 | tn_buf, i - min_off, access_size); |
| 2294 | } |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 2295 | return -EACCES; |
| 2296 | mark: |
| 2297 | /* reading any byte out of 8-byte 'spill_slot' will cause |
| 2298 | * the whole slot to be marked as 'read' |
| 2299 | */ |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 2300 | mark_reg_read(env, &state->stack[spi].spilled_ptr, |
| 2301 | state->stack[spi].spilled_ptr.parent); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2302 | } |
Andrey Ignatov | 2011fcc | 2019-03-28 18:01:57 -0700 | [diff] [blame] | 2303 | return update_stack_depth(env, state, min_off); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2304 | } |
| 2305 | |
Gianluca Borello | 06c1c04 | 2017-01-09 10:19:49 -0800 | [diff] [blame] | 2306 | static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, |
| 2307 | int access_size, bool zero_size_allowed, |
| 2308 | struct bpf_call_arg_meta *meta) |
| 2309 | { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 2310 | struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; |
Gianluca Borello | 06c1c04 | 2017-01-09 10:19:49 -0800 | [diff] [blame] | 2311 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 2312 | switch (reg->type) { |
Gianluca Borello | 06c1c04 | 2017-01-09 10:19:49 -0800 | [diff] [blame] | 2313 | case PTR_TO_PACKET: |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 2314 | case PTR_TO_PACKET_META: |
Yonghong Song | 9fd29c0 | 2017-11-12 14:49:09 -0800 | [diff] [blame] | 2315 | return check_packet_access(env, regno, reg->off, access_size, |
| 2316 | zero_size_allowed); |
Gianluca Borello | 06c1c04 | 2017-01-09 10:19:49 -0800 | [diff] [blame] | 2317 | case PTR_TO_MAP_VALUE: |
Yonghong Song | 9fd29c0 | 2017-11-12 14:49:09 -0800 | [diff] [blame] | 2318 | return check_map_access(env, regno, reg->off, access_size, |
| 2319 | zero_size_allowed); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 2320 | default: /* scalar_value|ptr_to_stack or invalid ptr */ |
Gianluca Borello | 06c1c04 | 2017-01-09 10:19:49 -0800 | [diff] [blame] | 2321 | return check_stack_boundary(env, regno, access_size, |
| 2322 | zero_size_allowed, meta); |
| 2323 | } |
| 2324 | } |
| 2325 | |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 2326 | /* Implementation details: |
| 2327 | * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL |
| 2328 | * Two bpf_map_lookups (even with the same key) will have different reg->id. |
| 2329 | * For traditional PTR_TO_MAP_VALUE the verifier clears reg->id after |
| 2330 | * value_or_null->value transition, since the verifier only cares about |
| 2331 | * the range of access to valid map value pointer and doesn't care about actual |
| 2332 | * address of the map element. |
| 2333 | * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps |
| 2334 | * reg->id > 0 after value_or_null->value transition. By doing so |
| 2335 | * two bpf_map_lookups will be considered two different pointers that |
| 2336 | * point to different bpf_spin_locks. |
| 2337 | * The verifier allows taking only one bpf_spin_lock at a time to avoid |
| 2338 | * dead-locks. |
| 2339 | * Since only one bpf_spin_lock is allowed the checks are simpler than |
| 2340 | * reg_is_refcounted() logic. The verifier needs to remember only |
| 2341 | * one spin_lock instead of array of acquired_refs. |
| 2342 | * cur_state->active_spin_lock remembers which map value element got locked |
| 2343 | * and clears it after bpf_spin_unlock. |
| 2344 | */ |
| 2345 | static int process_spin_lock(struct bpf_verifier_env *env, int regno, |
| 2346 | bool is_lock) |
| 2347 | { |
| 2348 | struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; |
| 2349 | struct bpf_verifier_state *cur = env->cur_state; |
| 2350 | bool is_const = tnum_is_const(reg->var_off); |
| 2351 | struct bpf_map *map = reg->map_ptr; |
| 2352 | u64 val = reg->var_off.value; |
| 2353 | |
| 2354 | if (reg->type != PTR_TO_MAP_VALUE) { |
| 2355 | verbose(env, "R%d is not a pointer to map_value\n", regno); |
| 2356 | return -EINVAL; |
| 2357 | } |
| 2358 | if (!is_const) { |
| 2359 | verbose(env, |
| 2360 | "R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n", |
| 2361 | regno); |
| 2362 | return -EINVAL; |
| 2363 | } |
| 2364 | if (!map->btf) { |
| 2365 | verbose(env, |
| 2366 | "map '%s' has to have BTF in order to use bpf_spin_lock\n", |
| 2367 | map->name); |
| 2368 | return -EINVAL; |
| 2369 | } |
| 2370 | if (!map_value_has_spin_lock(map)) { |
| 2371 | if (map->spin_lock_off == -E2BIG) |
| 2372 | verbose(env, |
| 2373 | "map '%s' has more than one 'struct bpf_spin_lock'\n", |
| 2374 | map->name); |
| 2375 | else if (map->spin_lock_off == -ENOENT) |
| 2376 | verbose(env, |
| 2377 | "map '%s' doesn't have 'struct bpf_spin_lock'\n", |
| 2378 | map->name); |
| 2379 | else |
| 2380 | verbose(env, |
| 2381 | "map '%s' is not a struct type or bpf_spin_lock is mangled\n", |
| 2382 | map->name); |
| 2383 | return -EINVAL; |
| 2384 | } |
| 2385 | if (map->spin_lock_off != val + reg->off) { |
| 2386 | verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock'\n", |
| 2387 | val + reg->off); |
| 2388 | return -EINVAL; |
| 2389 | } |
| 2390 | if (is_lock) { |
| 2391 | if (cur->active_spin_lock) { |
| 2392 | verbose(env, |
| 2393 | "Locking two bpf_spin_locks are not allowed\n"); |
| 2394 | return -EINVAL; |
| 2395 | } |
| 2396 | cur->active_spin_lock = reg->id; |
| 2397 | } else { |
| 2398 | if (!cur->active_spin_lock) { |
| 2399 | verbose(env, "bpf_spin_unlock without taking a lock\n"); |
| 2400 | return -EINVAL; |
| 2401 | } |
| 2402 | if (cur->active_spin_lock != reg->id) { |
| 2403 | verbose(env, "bpf_spin_unlock of different lock\n"); |
| 2404 | return -EINVAL; |
| 2405 | } |
| 2406 | cur->active_spin_lock = 0; |
| 2407 | } |
| 2408 | return 0; |
| 2409 | } |
| 2410 | |
Daniel Borkmann | 9013341 | 2018-01-20 01:24:29 +0100 | [diff] [blame] | 2411 | static bool arg_type_is_mem_ptr(enum bpf_arg_type type) |
| 2412 | { |
| 2413 | return type == ARG_PTR_TO_MEM || |
| 2414 | type == ARG_PTR_TO_MEM_OR_NULL || |
| 2415 | type == ARG_PTR_TO_UNINIT_MEM; |
| 2416 | } |
| 2417 | |
| 2418 | static bool arg_type_is_mem_size(enum bpf_arg_type type) |
| 2419 | { |
| 2420 | return type == ARG_CONST_SIZE || |
| 2421 | type == ARG_CONST_SIZE_OR_ZERO; |
| 2422 | } |
| 2423 | |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 2424 | static int check_func_arg(struct bpf_verifier_env *env, u32 regno, |
Daniel Borkmann | 33ff982 | 2016-04-13 00:10:50 +0200 | [diff] [blame] | 2425 | enum bpf_arg_type arg_type, |
| 2426 | struct bpf_call_arg_meta *meta) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2427 | { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 2428 | struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; |
Alexei Starovoitov | 6841de8 | 2016-08-11 18:17:16 -0700 | [diff] [blame] | 2429 | enum bpf_reg_type expected_type, type = reg->type; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2430 | int err = 0; |
| 2431 | |
Daniel Borkmann | 80f1d68 | 2015-03-12 17:21:42 +0100 | [diff] [blame] | 2432 | if (arg_type == ARG_DONTCARE) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2433 | return 0; |
| 2434 | |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 2435 | err = check_reg_arg(env, regno, SRC_OP); |
| 2436 | if (err) |
| 2437 | return err; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2438 | |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 2439 | if (arg_type == ARG_ANYTHING) { |
| 2440 | if (is_pointer_value(env, regno)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2441 | verbose(env, "R%d leaks addr into helper function\n", |
| 2442 | regno); |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 2443 | return -EACCES; |
| 2444 | } |
Daniel Borkmann | 80f1d68 | 2015-03-12 17:21:42 +0100 | [diff] [blame] | 2445 | return 0; |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 2446 | } |
Daniel Borkmann | 80f1d68 | 2015-03-12 17:21:42 +0100 | [diff] [blame] | 2447 | |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 2448 | if (type_is_pkt_pointer(type) && |
Thomas Graf | 3a0af8f | 2016-11-30 17:10:10 +0100 | [diff] [blame] | 2449 | !may_access_direct_pkt_data(env, meta, BPF_READ)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2450 | verbose(env, "helper access to the packet is not allowed\n"); |
Alexei Starovoitov | 6841de8 | 2016-08-11 18:17:16 -0700 | [diff] [blame] | 2451 | return -EACCES; |
| 2452 | } |
| 2453 | |
Daniel Borkmann | 8e2fe1d9 | 2016-02-19 23:05:22 +0100 | [diff] [blame] | 2454 | if (arg_type == ARG_PTR_TO_MAP_KEY || |
Mauricio Vasquez B | 2ea864c | 2018-10-18 15:16:20 +0200 | [diff] [blame] | 2455 | arg_type == ARG_PTR_TO_MAP_VALUE || |
| 2456 | arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) { |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2457 | expected_type = PTR_TO_STACK; |
Paul Chaignon | d71962f | 2018-04-24 15:07:54 +0200 | [diff] [blame] | 2458 | if (!type_is_pkt_pointer(type) && type != PTR_TO_MAP_VALUE && |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 2459 | type != expected_type) |
Alexei Starovoitov | 6841de8 | 2016-08-11 18:17:16 -0700 | [diff] [blame] | 2460 | goto err_type; |
Alexei Starovoitov | 39f19ebb | 2017-01-09 10:19:50 -0800 | [diff] [blame] | 2461 | } else if (arg_type == ARG_CONST_SIZE || |
| 2462 | arg_type == ARG_CONST_SIZE_OR_ZERO) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 2463 | expected_type = SCALAR_VALUE; |
| 2464 | if (type != expected_type) |
Alexei Starovoitov | 6841de8 | 2016-08-11 18:17:16 -0700 | [diff] [blame] | 2465 | goto err_type; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2466 | } else if (arg_type == ARG_CONST_MAP_PTR) { |
| 2467 | expected_type = CONST_PTR_TO_MAP; |
Alexei Starovoitov | 6841de8 | 2016-08-11 18:17:16 -0700 | [diff] [blame] | 2468 | if (type != expected_type) |
| 2469 | goto err_type; |
Alexei Starovoitov | 608cd71 | 2015-03-26 19:53:57 -0700 | [diff] [blame] | 2470 | } else if (arg_type == ARG_PTR_TO_CTX) { |
| 2471 | expected_type = PTR_TO_CTX; |
Alexei Starovoitov | 6841de8 | 2016-08-11 18:17:16 -0700 | [diff] [blame] | 2472 | if (type != expected_type) |
| 2473 | goto err_type; |
Daniel Borkmann | 58990d1 | 2018-06-07 17:40:03 +0200 | [diff] [blame] | 2474 | err = check_ctx_reg(env, reg, regno); |
| 2475 | if (err < 0) |
| 2476 | return err; |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 2477 | } else if (arg_type == ARG_PTR_TO_SOCK_COMMON) { |
| 2478 | expected_type = PTR_TO_SOCK_COMMON; |
| 2479 | /* Any sk pointer can be ARG_PTR_TO_SOCK_COMMON */ |
| 2480 | if (!type_is_sk_pointer(type)) |
| 2481 | goto err_type; |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 2482 | if (reg->ref_obj_id) { |
| 2483 | if (meta->ref_obj_id) { |
| 2484 | verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", |
| 2485 | regno, reg->ref_obj_id, |
| 2486 | meta->ref_obj_id); |
| 2487 | return -EFAULT; |
| 2488 | } |
| 2489 | meta->ref_obj_id = reg->ref_obj_id; |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 2490 | } |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 2491 | } else if (arg_type == ARG_PTR_TO_SPIN_LOCK) { |
| 2492 | if (meta->func_id == BPF_FUNC_spin_lock) { |
| 2493 | if (process_spin_lock(env, regno, true)) |
| 2494 | return -EACCES; |
| 2495 | } else if (meta->func_id == BPF_FUNC_spin_unlock) { |
| 2496 | if (process_spin_lock(env, regno, false)) |
| 2497 | return -EACCES; |
| 2498 | } else { |
| 2499 | verbose(env, "verifier internal error\n"); |
| 2500 | return -EFAULT; |
| 2501 | } |
Daniel Borkmann | 9013341 | 2018-01-20 01:24:29 +0100 | [diff] [blame] | 2502 | } else if (arg_type_is_mem_ptr(arg_type)) { |
Daniel Borkmann | 8e2fe1d9 | 2016-02-19 23:05:22 +0100 | [diff] [blame] | 2503 | expected_type = PTR_TO_STACK; |
| 2504 | /* One exception here. In case function allows for NULL to be |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 2505 | * passed in as argument, it's a SCALAR_VALUE type. Final test |
Daniel Borkmann | 8e2fe1d9 | 2016-02-19 23:05:22 +0100 | [diff] [blame] | 2506 | * happens during stack boundary checking. |
| 2507 | */ |
Alexei Starovoitov | 914cb78 | 2017-11-30 21:31:40 -0800 | [diff] [blame] | 2508 | if (register_is_null(reg) && |
Gianluca Borello | db1ac49 | 2017-11-22 18:32:53 +0000 | [diff] [blame] | 2509 | arg_type == ARG_PTR_TO_MEM_OR_NULL) |
Alexei Starovoitov | 6841de8 | 2016-08-11 18:17:16 -0700 | [diff] [blame] | 2510 | /* final test in check_stack_boundary() */; |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 2511 | else if (!type_is_pkt_pointer(type) && |
| 2512 | type != PTR_TO_MAP_VALUE && |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 2513 | type != expected_type) |
Alexei Starovoitov | 6841de8 | 2016-08-11 18:17:16 -0700 | [diff] [blame] | 2514 | goto err_type; |
Alexei Starovoitov | 39f19ebb | 2017-01-09 10:19:50 -0800 | [diff] [blame] | 2515 | meta->raw_mode = arg_type == ARG_PTR_TO_UNINIT_MEM; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2516 | } else { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2517 | verbose(env, "unsupported arg_type %d\n", arg_type); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2518 | return -EFAULT; |
| 2519 | } |
| 2520 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2521 | if (arg_type == ARG_CONST_MAP_PTR) { |
| 2522 | /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ |
Daniel Borkmann | 33ff982 | 2016-04-13 00:10:50 +0200 | [diff] [blame] | 2523 | meta->map_ptr = reg->map_ptr; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2524 | } else if (arg_type == ARG_PTR_TO_MAP_KEY) { |
| 2525 | /* bpf_map_xxx(..., map_ptr, ..., key) call: |
| 2526 | * check that [key, key + map->key_size) are within |
| 2527 | * stack limits and initialized |
| 2528 | */ |
Daniel Borkmann | 33ff982 | 2016-04-13 00:10:50 +0200 | [diff] [blame] | 2529 | if (!meta->map_ptr) { |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2530 | /* in function declaration map_ptr must come before |
| 2531 | * map_key, so that it's verified and known before |
| 2532 | * we have to check map_key here. Otherwise it means |
| 2533 | * that kernel subsystem misconfigured verifier |
| 2534 | */ |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2535 | verbose(env, "invalid map_ptr to access map->key\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2536 | return -EACCES; |
| 2537 | } |
Paul Chaignon | d71962f | 2018-04-24 15:07:54 +0200 | [diff] [blame] | 2538 | err = check_helper_mem_access(env, regno, |
| 2539 | meta->map_ptr->key_size, false, |
| 2540 | NULL); |
Mauricio Vasquez B | 2ea864c | 2018-10-18 15:16:20 +0200 | [diff] [blame] | 2541 | } else if (arg_type == ARG_PTR_TO_MAP_VALUE || |
| 2542 | arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) { |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2543 | /* bpf_map_xxx(..., map_ptr, ..., value) call: |
| 2544 | * check [value, value + map->value_size) validity |
| 2545 | */ |
Daniel Borkmann | 33ff982 | 2016-04-13 00:10:50 +0200 | [diff] [blame] | 2546 | if (!meta->map_ptr) { |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2547 | /* kernel subsystem misconfigured verifier */ |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2548 | verbose(env, "invalid map_ptr to access map->value\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2549 | return -EACCES; |
| 2550 | } |
Mauricio Vasquez B | 2ea864c | 2018-10-18 15:16:20 +0200 | [diff] [blame] | 2551 | meta->raw_mode = (arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE); |
Paul Chaignon | d71962f | 2018-04-24 15:07:54 +0200 | [diff] [blame] | 2552 | err = check_helper_mem_access(env, regno, |
| 2553 | meta->map_ptr->value_size, false, |
Mauricio Vasquez B | 2ea864c | 2018-10-18 15:16:20 +0200 | [diff] [blame] | 2554 | meta); |
Daniel Borkmann | 9013341 | 2018-01-20 01:24:29 +0100 | [diff] [blame] | 2555 | } else if (arg_type_is_mem_size(arg_type)) { |
Alexei Starovoitov | 39f19ebb | 2017-01-09 10:19:50 -0800 | [diff] [blame] | 2556 | bool zero_size_allowed = (arg_type == ARG_CONST_SIZE_OR_ZERO); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2557 | |
Yonghong Song | 849fa50 | 2018-04-28 22:28:09 -0700 | [diff] [blame] | 2558 | /* remember the mem_size which may be used later |
| 2559 | * to refine return values. |
| 2560 | */ |
| 2561 | meta->msize_smax_value = reg->smax_value; |
| 2562 | meta->msize_umax_value = reg->umax_value; |
| 2563 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 2564 | /* The register is SCALAR_VALUE; the access check |
| 2565 | * happens using its boundaries. |
Gianluca Borello | 06c1c04 | 2017-01-09 10:19:49 -0800 | [diff] [blame] | 2566 | */ |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 2567 | if (!tnum_is_const(reg->var_off)) |
Gianluca Borello | 06c1c04 | 2017-01-09 10:19:49 -0800 | [diff] [blame] | 2568 | /* For unprivileged variable accesses, disable raw |
| 2569 | * mode so that the program is required to |
| 2570 | * initialize all the memory that the helper could |
| 2571 | * just partially fill up. |
| 2572 | */ |
| 2573 | meta = NULL; |
| 2574 | |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 2575 | if (reg->smin_value < 0) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2576 | verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n", |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 2577 | regno); |
| 2578 | return -EACCES; |
| 2579 | } |
Gianluca Borello | 06c1c04 | 2017-01-09 10:19:49 -0800 | [diff] [blame] | 2580 | |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 2581 | if (reg->umin_value == 0) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 2582 | err = check_helper_mem_access(env, regno - 1, 0, |
| 2583 | zero_size_allowed, |
| 2584 | meta); |
Gianluca Borello | 06c1c04 | 2017-01-09 10:19:49 -0800 | [diff] [blame] | 2585 | if (err) |
| 2586 | return err; |
Gianluca Borello | 06c1c04 | 2017-01-09 10:19:49 -0800 | [diff] [blame] | 2587 | } |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 2588 | |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 2589 | if (reg->umax_value >= BPF_MAX_VAR_SIZ) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2590 | verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n", |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 2591 | regno); |
| 2592 | return -EACCES; |
| 2593 | } |
| 2594 | err = check_helper_mem_access(env, regno - 1, |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 2595 | reg->umax_value, |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 2596 | zero_size_allowed, meta); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2597 | } |
| 2598 | |
| 2599 | return err; |
Alexei Starovoitov | 6841de8 | 2016-08-11 18:17:16 -0700 | [diff] [blame] | 2600 | err_type: |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2601 | verbose(env, "R%d type=%s expected=%s\n", regno, |
Alexei Starovoitov | 6841de8 | 2016-08-11 18:17:16 -0700 | [diff] [blame] | 2602 | reg_type_str[type], reg_type_str[expected_type]); |
| 2603 | return -EACCES; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2604 | } |
| 2605 | |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2606 | static int check_map_func_compatibility(struct bpf_verifier_env *env, |
| 2607 | struct bpf_map *map, int func_id) |
Kaixu Xia | 35578d7 | 2015-08-06 07:02:35 +0000 | [diff] [blame] | 2608 | { |
Kaixu Xia | 35578d7 | 2015-08-06 07:02:35 +0000 | [diff] [blame] | 2609 | if (!map) |
| 2610 | return 0; |
| 2611 | |
Alexei Starovoitov | 6aff67c | 2016-04-27 18:56:21 -0700 | [diff] [blame] | 2612 | /* We need a two way check, first is from map perspective ... */ |
| 2613 | switch (map->map_type) { |
| 2614 | case BPF_MAP_TYPE_PROG_ARRAY: |
| 2615 | if (func_id != BPF_FUNC_tail_call) |
| 2616 | goto error; |
| 2617 | break; |
| 2618 | case BPF_MAP_TYPE_PERF_EVENT_ARRAY: |
| 2619 | if (func_id != BPF_FUNC_perf_event_read && |
Yonghong Song | 908432c | 2017-10-05 09:19:20 -0700 | [diff] [blame] | 2620 | func_id != BPF_FUNC_perf_event_output && |
| 2621 | func_id != BPF_FUNC_perf_event_read_value) |
Alexei Starovoitov | 6aff67c | 2016-04-27 18:56:21 -0700 | [diff] [blame] | 2622 | goto error; |
| 2623 | break; |
| 2624 | case BPF_MAP_TYPE_STACK_TRACE: |
| 2625 | if (func_id != BPF_FUNC_get_stackid) |
| 2626 | goto error; |
| 2627 | break; |
Martin KaFai Lau | 4ed8ec5 | 2016-06-30 10:28:43 -0700 | [diff] [blame] | 2628 | case BPF_MAP_TYPE_CGROUP_ARRAY: |
David S. Miller | 60747ef | 2016-08-18 01:17:32 -0400 | [diff] [blame] | 2629 | if (func_id != BPF_FUNC_skb_under_cgroup && |
Sargun Dhillon | 60d20f9 | 2016-08-12 08:56:52 -0700 | [diff] [blame] | 2630 | func_id != BPF_FUNC_current_task_under_cgroup) |
Martin KaFai Lau | 4a482f3 | 2016-06-30 10:28:44 -0700 | [diff] [blame] | 2631 | goto error; |
| 2632 | break; |
Roman Gushchin | cd33943 | 2018-08-02 14:27:24 -0700 | [diff] [blame] | 2633 | case BPF_MAP_TYPE_CGROUP_STORAGE: |
Roman Gushchin | b741f16 | 2018-09-28 14:45:43 +0000 | [diff] [blame] | 2634 | case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: |
Roman Gushchin | cd33943 | 2018-08-02 14:27:24 -0700 | [diff] [blame] | 2635 | if (func_id != BPF_FUNC_get_local_storage) |
| 2636 | goto error; |
| 2637 | break; |
John Fastabend | 546ac1f | 2017-07-17 09:28:56 -0700 | [diff] [blame] | 2638 | /* devmap returns a pointer to a live net_device ifindex that we cannot |
| 2639 | * allow to be modified from bpf side. So do not allow lookup elements |
| 2640 | * for now. |
| 2641 | */ |
| 2642 | case BPF_MAP_TYPE_DEVMAP: |
John Fastabend | 2ddf71e | 2017-07-17 09:30:02 -0700 | [diff] [blame] | 2643 | if (func_id != BPF_FUNC_redirect_map) |
John Fastabend | 546ac1f | 2017-07-17 09:28:56 -0700 | [diff] [blame] | 2644 | goto error; |
| 2645 | break; |
Björn Töpel | fbfc504a | 2018-05-02 13:01:28 +0200 | [diff] [blame] | 2646 | /* Restrict bpf side of cpumap and xskmap, open when use-cases |
| 2647 | * appear. |
| 2648 | */ |
Jesper Dangaard Brouer | 6710e11 | 2017-10-16 12:19:28 +0200 | [diff] [blame] | 2649 | case BPF_MAP_TYPE_CPUMAP: |
Björn Töpel | fbfc504a | 2018-05-02 13:01:28 +0200 | [diff] [blame] | 2650 | case BPF_MAP_TYPE_XSKMAP: |
Jesper Dangaard Brouer | 6710e11 | 2017-10-16 12:19:28 +0200 | [diff] [blame] | 2651 | if (func_id != BPF_FUNC_redirect_map) |
| 2652 | goto error; |
| 2653 | break; |
Martin KaFai Lau | 56f668d | 2017-03-22 10:00:33 -0700 | [diff] [blame] | 2654 | case BPF_MAP_TYPE_ARRAY_OF_MAPS: |
Martin KaFai Lau | bcc6b1b | 2017-03-22 10:00:34 -0700 | [diff] [blame] | 2655 | case BPF_MAP_TYPE_HASH_OF_MAPS: |
Martin KaFai Lau | 56f668d | 2017-03-22 10:00:33 -0700 | [diff] [blame] | 2656 | if (func_id != BPF_FUNC_map_lookup_elem) |
| 2657 | goto error; |
Martin KaFai Lau | 16a4362 | 2017-08-17 18:14:43 -0700 | [diff] [blame] | 2658 | break; |
John Fastabend | 174a79f | 2017-08-15 22:32:47 -0700 | [diff] [blame] | 2659 | case BPF_MAP_TYPE_SOCKMAP: |
| 2660 | if (func_id != BPF_FUNC_sk_redirect_map && |
| 2661 | func_id != BPF_FUNC_sock_map_update && |
John Fastabend | 4f738ad | 2018-03-18 12:57:10 -0700 | [diff] [blame] | 2662 | func_id != BPF_FUNC_map_delete_elem && |
| 2663 | func_id != BPF_FUNC_msg_redirect_map) |
John Fastabend | 174a79f | 2017-08-15 22:32:47 -0700 | [diff] [blame] | 2664 | goto error; |
| 2665 | break; |
John Fastabend | 8111038 | 2018-05-14 10:00:17 -0700 | [diff] [blame] | 2666 | case BPF_MAP_TYPE_SOCKHASH: |
| 2667 | if (func_id != BPF_FUNC_sk_redirect_hash && |
| 2668 | func_id != BPF_FUNC_sock_hash_update && |
| 2669 | func_id != BPF_FUNC_map_delete_elem && |
| 2670 | func_id != BPF_FUNC_msg_redirect_hash) |
| 2671 | goto error; |
| 2672 | break; |
Martin KaFai Lau | 2dbb9b9 | 2018-08-08 01:01:25 -0700 | [diff] [blame] | 2673 | case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: |
| 2674 | if (func_id != BPF_FUNC_sk_select_reuseport) |
| 2675 | goto error; |
| 2676 | break; |
Mauricio Vasquez B | f1a2e44 | 2018-10-18 15:16:25 +0200 | [diff] [blame] | 2677 | case BPF_MAP_TYPE_QUEUE: |
| 2678 | case BPF_MAP_TYPE_STACK: |
| 2679 | if (func_id != BPF_FUNC_map_peek_elem && |
| 2680 | func_id != BPF_FUNC_map_pop_elem && |
| 2681 | func_id != BPF_FUNC_map_push_elem) |
| 2682 | goto error; |
| 2683 | break; |
Alexei Starovoitov | 6aff67c | 2016-04-27 18:56:21 -0700 | [diff] [blame] | 2684 | default: |
| 2685 | break; |
| 2686 | } |
| 2687 | |
| 2688 | /* ... and second from the function itself. */ |
| 2689 | switch (func_id) { |
| 2690 | case BPF_FUNC_tail_call: |
| 2691 | if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY) |
| 2692 | goto error; |
Jiong Wang | f910cef | 2018-05-02 16:17:17 -0400 | [diff] [blame] | 2693 | if (env->subprog_cnt > 1) { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 2694 | verbose(env, "tail_calls are not allowed in programs with bpf-to-bpf calls\n"); |
| 2695 | return -EINVAL; |
| 2696 | } |
Alexei Starovoitov | 6aff67c | 2016-04-27 18:56:21 -0700 | [diff] [blame] | 2697 | break; |
| 2698 | case BPF_FUNC_perf_event_read: |
| 2699 | case BPF_FUNC_perf_event_output: |
Yonghong Song | 908432c | 2017-10-05 09:19:20 -0700 | [diff] [blame] | 2700 | case BPF_FUNC_perf_event_read_value: |
Alexei Starovoitov | 6aff67c | 2016-04-27 18:56:21 -0700 | [diff] [blame] | 2701 | if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) |
| 2702 | goto error; |
| 2703 | break; |
| 2704 | case BPF_FUNC_get_stackid: |
| 2705 | if (map->map_type != BPF_MAP_TYPE_STACK_TRACE) |
| 2706 | goto error; |
| 2707 | break; |
Sargun Dhillon | 60d20f9 | 2016-08-12 08:56:52 -0700 | [diff] [blame] | 2708 | case BPF_FUNC_current_task_under_cgroup: |
Daniel Borkmann | 747ea55 | 2016-08-12 22:17:17 +0200 | [diff] [blame] | 2709 | case BPF_FUNC_skb_under_cgroup: |
Martin KaFai Lau | 4a482f3 | 2016-06-30 10:28:44 -0700 | [diff] [blame] | 2710 | if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY) |
| 2711 | goto error; |
| 2712 | break; |
John Fastabend | 97f91a7 | 2017-07-17 09:29:18 -0700 | [diff] [blame] | 2713 | case BPF_FUNC_redirect_map: |
Jesper Dangaard Brouer | 9c270af | 2017-10-16 12:19:34 +0200 | [diff] [blame] | 2714 | if (map->map_type != BPF_MAP_TYPE_DEVMAP && |
Björn Töpel | fbfc504a | 2018-05-02 13:01:28 +0200 | [diff] [blame] | 2715 | map->map_type != BPF_MAP_TYPE_CPUMAP && |
| 2716 | map->map_type != BPF_MAP_TYPE_XSKMAP) |
John Fastabend | 97f91a7 | 2017-07-17 09:29:18 -0700 | [diff] [blame] | 2717 | goto error; |
| 2718 | break; |
John Fastabend | 174a79f | 2017-08-15 22:32:47 -0700 | [diff] [blame] | 2719 | case BPF_FUNC_sk_redirect_map: |
John Fastabend | 4f738ad | 2018-03-18 12:57:10 -0700 | [diff] [blame] | 2720 | case BPF_FUNC_msg_redirect_map: |
John Fastabend | 8111038 | 2018-05-14 10:00:17 -0700 | [diff] [blame] | 2721 | case BPF_FUNC_sock_map_update: |
John Fastabend | 174a79f | 2017-08-15 22:32:47 -0700 | [diff] [blame] | 2722 | if (map->map_type != BPF_MAP_TYPE_SOCKMAP) |
| 2723 | goto error; |
| 2724 | break; |
John Fastabend | 8111038 | 2018-05-14 10:00:17 -0700 | [diff] [blame] | 2725 | case BPF_FUNC_sk_redirect_hash: |
| 2726 | case BPF_FUNC_msg_redirect_hash: |
| 2727 | case BPF_FUNC_sock_hash_update: |
| 2728 | if (map->map_type != BPF_MAP_TYPE_SOCKHASH) |
John Fastabend | 174a79f | 2017-08-15 22:32:47 -0700 | [diff] [blame] | 2729 | goto error; |
| 2730 | break; |
Roman Gushchin | cd33943 | 2018-08-02 14:27:24 -0700 | [diff] [blame] | 2731 | case BPF_FUNC_get_local_storage: |
Roman Gushchin | b741f16 | 2018-09-28 14:45:43 +0000 | [diff] [blame] | 2732 | if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && |
| 2733 | map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) |
Roman Gushchin | cd33943 | 2018-08-02 14:27:24 -0700 | [diff] [blame] | 2734 | goto error; |
| 2735 | break; |
Martin KaFai Lau | 2dbb9b9 | 2018-08-08 01:01:25 -0700 | [diff] [blame] | 2736 | case BPF_FUNC_sk_select_reuseport: |
| 2737 | if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) |
| 2738 | goto error; |
| 2739 | break; |
Mauricio Vasquez B | f1a2e44 | 2018-10-18 15:16:25 +0200 | [diff] [blame] | 2740 | case BPF_FUNC_map_peek_elem: |
| 2741 | case BPF_FUNC_map_pop_elem: |
| 2742 | case BPF_FUNC_map_push_elem: |
| 2743 | if (map->map_type != BPF_MAP_TYPE_QUEUE && |
| 2744 | map->map_type != BPF_MAP_TYPE_STACK) |
| 2745 | goto error; |
| 2746 | break; |
Alexei Starovoitov | 6aff67c | 2016-04-27 18:56:21 -0700 | [diff] [blame] | 2747 | default: |
| 2748 | break; |
Kaixu Xia | 35578d7 | 2015-08-06 07:02:35 +0000 | [diff] [blame] | 2749 | } |
| 2750 | |
| 2751 | return 0; |
Alexei Starovoitov | 6aff67c | 2016-04-27 18:56:21 -0700 | [diff] [blame] | 2752 | error: |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2753 | verbose(env, "cannot pass map_type %d into func %s#%d\n", |
Thomas Graf | ebb676d | 2016-10-27 11:23:51 +0200 | [diff] [blame] | 2754 | map->map_type, func_id_name(func_id), func_id); |
Alexei Starovoitov | 6aff67c | 2016-04-27 18:56:21 -0700 | [diff] [blame] | 2755 | return -EINVAL; |
Kaixu Xia | 35578d7 | 2015-08-06 07:02:35 +0000 | [diff] [blame] | 2756 | } |
| 2757 | |
Daniel Borkmann | 9013341 | 2018-01-20 01:24:29 +0100 | [diff] [blame] | 2758 | static bool check_raw_mode_ok(const struct bpf_func_proto *fn) |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 2759 | { |
| 2760 | int count = 0; |
| 2761 | |
Alexei Starovoitov | 39f19ebb | 2017-01-09 10:19:50 -0800 | [diff] [blame] | 2762 | if (fn->arg1_type == ARG_PTR_TO_UNINIT_MEM) |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 2763 | count++; |
Alexei Starovoitov | 39f19ebb | 2017-01-09 10:19:50 -0800 | [diff] [blame] | 2764 | if (fn->arg2_type == ARG_PTR_TO_UNINIT_MEM) |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 2765 | count++; |
Alexei Starovoitov | 39f19ebb | 2017-01-09 10:19:50 -0800 | [diff] [blame] | 2766 | if (fn->arg3_type == ARG_PTR_TO_UNINIT_MEM) |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 2767 | count++; |
Alexei Starovoitov | 39f19ebb | 2017-01-09 10:19:50 -0800 | [diff] [blame] | 2768 | if (fn->arg4_type == ARG_PTR_TO_UNINIT_MEM) |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 2769 | count++; |
Alexei Starovoitov | 39f19ebb | 2017-01-09 10:19:50 -0800 | [diff] [blame] | 2770 | if (fn->arg5_type == ARG_PTR_TO_UNINIT_MEM) |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 2771 | count++; |
| 2772 | |
Daniel Borkmann | 9013341 | 2018-01-20 01:24:29 +0100 | [diff] [blame] | 2773 | /* We only support one arg being in raw mode at the moment, |
| 2774 | * which is sufficient for the helper functions we have |
| 2775 | * right now. |
| 2776 | */ |
| 2777 | return count <= 1; |
| 2778 | } |
| 2779 | |
| 2780 | static bool check_args_pair_invalid(enum bpf_arg_type arg_curr, |
| 2781 | enum bpf_arg_type arg_next) |
| 2782 | { |
| 2783 | return (arg_type_is_mem_ptr(arg_curr) && |
| 2784 | !arg_type_is_mem_size(arg_next)) || |
| 2785 | (!arg_type_is_mem_ptr(arg_curr) && |
| 2786 | arg_type_is_mem_size(arg_next)); |
| 2787 | } |
| 2788 | |
| 2789 | static bool check_arg_pair_ok(const struct bpf_func_proto *fn) |
| 2790 | { |
| 2791 | /* bpf_xxx(..., buf, len) call will access 'len' |
| 2792 | * bytes from memory 'buf'. Both arg types need |
| 2793 | * to be paired, so make sure there's no buggy |
| 2794 | * helper function specification. |
| 2795 | */ |
| 2796 | if (arg_type_is_mem_size(fn->arg1_type) || |
| 2797 | arg_type_is_mem_ptr(fn->arg5_type) || |
| 2798 | check_args_pair_invalid(fn->arg1_type, fn->arg2_type) || |
| 2799 | check_args_pair_invalid(fn->arg2_type, fn->arg3_type) || |
| 2800 | check_args_pair_invalid(fn->arg3_type, fn->arg4_type) || |
| 2801 | check_args_pair_invalid(fn->arg4_type, fn->arg5_type)) |
| 2802 | return false; |
| 2803 | |
| 2804 | return true; |
| 2805 | } |
| 2806 | |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 2807 | static bool check_refcount_ok(const struct bpf_func_proto *fn, int func_id) |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 2808 | { |
| 2809 | int count = 0; |
| 2810 | |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 2811 | if (arg_type_may_be_refcounted(fn->arg1_type)) |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 2812 | count++; |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 2813 | if (arg_type_may_be_refcounted(fn->arg2_type)) |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 2814 | count++; |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 2815 | if (arg_type_may_be_refcounted(fn->arg3_type)) |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 2816 | count++; |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 2817 | if (arg_type_may_be_refcounted(fn->arg4_type)) |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 2818 | count++; |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 2819 | if (arg_type_may_be_refcounted(fn->arg5_type)) |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 2820 | count++; |
| 2821 | |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 2822 | /* A reference acquiring function cannot acquire |
| 2823 | * another refcounted ptr. |
| 2824 | */ |
| 2825 | if (is_acquire_function(func_id) && count) |
| 2826 | return false; |
| 2827 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 2828 | /* We only support one arg being unreferenced at the moment, |
| 2829 | * which is sufficient for the helper functions we have right now. |
| 2830 | */ |
| 2831 | return count <= 1; |
| 2832 | } |
| 2833 | |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 2834 | static int check_func_proto(const struct bpf_func_proto *fn, int func_id) |
Daniel Borkmann | 9013341 | 2018-01-20 01:24:29 +0100 | [diff] [blame] | 2835 | { |
| 2836 | return check_raw_mode_ok(fn) && |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 2837 | check_arg_pair_ok(fn) && |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 2838 | check_refcount_ok(fn, func_id) ? 0 : -EINVAL; |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 2839 | } |
| 2840 | |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 2841 | /* Packet data might have moved, any old PTR_TO_PACKET[_META,_END] |
| 2842 | * are now invalid, so turn them into unknown SCALAR_VALUE. |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 2843 | */ |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 2844 | static void __clear_all_pkt_pointers(struct bpf_verifier_env *env, |
| 2845 | struct bpf_func_state *state) |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 2846 | { |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 2847 | struct bpf_reg_state *regs = state->regs, *reg; |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 2848 | int i; |
| 2849 | |
| 2850 | for (i = 0; i < MAX_BPF_REG; i++) |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 2851 | if (reg_is_pkt_pointer_any(®s[i])) |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2852 | mark_reg_unknown(env, regs, i); |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 2853 | |
Joe Stringer | f3709f6 | 2018-10-02 13:35:29 -0700 | [diff] [blame] | 2854 | bpf_for_each_spilled_reg(i, state, reg) { |
| 2855 | if (!reg) |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 2856 | continue; |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 2857 | if (reg_is_pkt_pointer_any(reg)) |
| 2858 | __mark_reg_unknown(reg); |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 2859 | } |
| 2860 | } |
| 2861 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 2862 | static void clear_all_pkt_pointers(struct bpf_verifier_env *env) |
| 2863 | { |
| 2864 | struct bpf_verifier_state *vstate = env->cur_state; |
| 2865 | int i; |
| 2866 | |
| 2867 | for (i = 0; i <= vstate->curframe; i++) |
| 2868 | __clear_all_pkt_pointers(env, vstate->frame[i]); |
| 2869 | } |
| 2870 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 2871 | static void release_reg_references(struct bpf_verifier_env *env, |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 2872 | struct bpf_func_state *state, |
| 2873 | int ref_obj_id) |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 2874 | { |
| 2875 | struct bpf_reg_state *regs = state->regs, *reg; |
| 2876 | int i; |
| 2877 | |
| 2878 | for (i = 0; i < MAX_BPF_REG; i++) |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 2879 | if (regs[i].ref_obj_id == ref_obj_id) |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 2880 | mark_reg_unknown(env, regs, i); |
| 2881 | |
| 2882 | bpf_for_each_spilled_reg(i, state, reg) { |
| 2883 | if (!reg) |
| 2884 | continue; |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 2885 | if (reg->ref_obj_id == ref_obj_id) |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 2886 | __mark_reg_unknown(reg); |
| 2887 | } |
| 2888 | } |
| 2889 | |
| 2890 | /* The pointer with the specified id has released its reference to kernel |
| 2891 | * resources. Identify all copies of the same pointer and clear the reference. |
| 2892 | */ |
| 2893 | static int release_reference(struct bpf_verifier_env *env, |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 2894 | int ref_obj_id) |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 2895 | { |
| 2896 | struct bpf_verifier_state *vstate = env->cur_state; |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 2897 | int err; |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 2898 | int i; |
| 2899 | |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 2900 | err = release_reference_state(cur_func(env), ref_obj_id); |
| 2901 | if (err) |
| 2902 | return err; |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 2903 | |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 2904 | for (i = 0; i <= vstate->curframe; i++) |
| 2905 | release_reg_references(env, vstate->frame[i], ref_obj_id); |
| 2906 | |
| 2907 | return 0; |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 2908 | } |
| 2909 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 2910 | static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, |
| 2911 | int *insn_idx) |
| 2912 | { |
| 2913 | struct bpf_verifier_state *state = env->cur_state; |
| 2914 | struct bpf_func_state *caller, *callee; |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 2915 | int i, err, subprog, target_insn; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 2916 | |
Alexei Starovoitov | aada9ce | 2017-12-25 13:15:42 -0800 | [diff] [blame] | 2917 | if (state->curframe + 1 >= MAX_CALL_FRAMES) { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 2918 | verbose(env, "the call stack of %d frames is too deep\n", |
Alexei Starovoitov | aada9ce | 2017-12-25 13:15:42 -0800 | [diff] [blame] | 2919 | state->curframe + 2); |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 2920 | return -E2BIG; |
| 2921 | } |
| 2922 | |
| 2923 | target_insn = *insn_idx + insn->imm; |
| 2924 | subprog = find_subprog(env, target_insn + 1); |
| 2925 | if (subprog < 0) { |
| 2926 | verbose(env, "verifier bug. No program starts at insn %d\n", |
| 2927 | target_insn + 1); |
| 2928 | return -EFAULT; |
| 2929 | } |
| 2930 | |
| 2931 | caller = state->frame[state->curframe]; |
| 2932 | if (state->frame[state->curframe + 1]) { |
| 2933 | verbose(env, "verifier bug. Frame %d already allocated\n", |
| 2934 | state->curframe + 1); |
| 2935 | return -EFAULT; |
| 2936 | } |
| 2937 | |
| 2938 | callee = kzalloc(sizeof(*callee), GFP_KERNEL); |
| 2939 | if (!callee) |
| 2940 | return -ENOMEM; |
| 2941 | state->frame[state->curframe + 1] = callee; |
| 2942 | |
| 2943 | /* callee cannot access r0, r6 - r9 for reading and has to write |
| 2944 | * into its own stack before reading from it. |
| 2945 | * callee can read/write into caller's stack |
| 2946 | */ |
| 2947 | init_func_state(env, callee, |
| 2948 | /* remember the callsite, it will be used by bpf_exit */ |
| 2949 | *insn_idx /* callsite */, |
| 2950 | state->curframe + 1 /* frameno within this callchain */, |
Jiong Wang | f910cef | 2018-05-02 16:17:17 -0400 | [diff] [blame] | 2951 | subprog /* subprog number within this prog */); |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 2952 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 2953 | /* Transfer references to the callee */ |
| 2954 | err = transfer_reference_state(callee, caller); |
| 2955 | if (err) |
| 2956 | return err; |
| 2957 | |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 2958 | /* copy r1 - r5 args that callee can access. The copy includes parent |
| 2959 | * pointers, which connects us up to the liveness chain |
| 2960 | */ |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 2961 | for (i = BPF_REG_1; i <= BPF_REG_5; i++) |
| 2962 | callee->regs[i] = caller->regs[i]; |
| 2963 | |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 2964 | /* after the call registers r0 - r5 were scratched */ |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 2965 | for (i = 0; i < CALLER_SAVED_REGS; i++) { |
| 2966 | mark_reg_not_init(env, caller->regs, caller_saved[i]); |
| 2967 | check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); |
| 2968 | } |
| 2969 | |
| 2970 | /* only increment it after check_reg_arg() finished */ |
| 2971 | state->curframe++; |
| 2972 | |
| 2973 | /* and go analyze first insn of the callee */ |
| 2974 | *insn_idx = target_insn; |
| 2975 | |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 2976 | if (env->log.level & BPF_LOG_LEVEL) { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 2977 | verbose(env, "caller:\n"); |
| 2978 | print_verifier_state(env, caller); |
| 2979 | verbose(env, "callee:\n"); |
| 2980 | print_verifier_state(env, callee); |
| 2981 | } |
| 2982 | return 0; |
| 2983 | } |
| 2984 | |
| 2985 | static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) |
| 2986 | { |
| 2987 | struct bpf_verifier_state *state = env->cur_state; |
| 2988 | struct bpf_func_state *caller, *callee; |
| 2989 | struct bpf_reg_state *r0; |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 2990 | int err; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 2991 | |
| 2992 | callee = state->frame[state->curframe]; |
| 2993 | r0 = &callee->regs[BPF_REG_0]; |
| 2994 | if (r0->type == PTR_TO_STACK) { |
| 2995 | /* technically it's ok to return caller's stack pointer |
| 2996 | * (or caller's caller's pointer) back to the caller, |
| 2997 | * since these pointers are valid. Only current stack |
| 2998 | * pointer will be invalid as soon as function exits, |
| 2999 | * but let's be conservative |
| 3000 | */ |
| 3001 | verbose(env, "cannot return stack pointer to the caller\n"); |
| 3002 | return -EINVAL; |
| 3003 | } |
| 3004 | |
| 3005 | state->curframe--; |
| 3006 | caller = state->frame[state->curframe]; |
| 3007 | /* return to the caller whatever r0 had in the callee */ |
| 3008 | caller->regs[BPF_REG_0] = *r0; |
| 3009 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 3010 | /* Transfer references to the caller */ |
| 3011 | err = transfer_reference_state(caller, callee); |
| 3012 | if (err) |
| 3013 | return err; |
| 3014 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 3015 | *insn_idx = callee->callsite + 1; |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 3016 | if (env->log.level & BPF_LOG_LEVEL) { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 3017 | verbose(env, "returning from callee:\n"); |
| 3018 | print_verifier_state(env, callee); |
| 3019 | verbose(env, "to caller at %d:\n", *insn_idx); |
| 3020 | print_verifier_state(env, caller); |
| 3021 | } |
| 3022 | /* clear everything in the callee */ |
| 3023 | free_func_state(callee); |
| 3024 | state->frame[state->curframe + 1] = NULL; |
| 3025 | return 0; |
| 3026 | } |
| 3027 | |
Yonghong Song | 849fa50 | 2018-04-28 22:28:09 -0700 | [diff] [blame] | 3028 | static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type, |
| 3029 | int func_id, |
| 3030 | struct bpf_call_arg_meta *meta) |
| 3031 | { |
| 3032 | struct bpf_reg_state *ret_reg = ®s[BPF_REG_0]; |
| 3033 | |
| 3034 | if (ret_type != RET_INTEGER || |
| 3035 | (func_id != BPF_FUNC_get_stack && |
| 3036 | func_id != BPF_FUNC_probe_read_str)) |
| 3037 | return; |
| 3038 | |
| 3039 | ret_reg->smax_value = meta->msize_smax_value; |
| 3040 | ret_reg->umax_value = meta->msize_umax_value; |
| 3041 | __reg_deduce_bounds(ret_reg); |
| 3042 | __reg_bound_offset(ret_reg); |
| 3043 | } |
| 3044 | |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 3045 | static int |
| 3046 | record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, |
| 3047 | int func_id, int insn_idx) |
| 3048 | { |
| 3049 | struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; |
| 3050 | |
| 3051 | if (func_id != BPF_FUNC_tail_call && |
Daniel Borkmann | 09772d9 | 2018-06-02 23:06:35 +0200 | [diff] [blame] | 3052 | func_id != BPF_FUNC_map_lookup_elem && |
| 3053 | func_id != BPF_FUNC_map_update_elem && |
Mauricio Vasquez B | f1a2e44 | 2018-10-18 15:16:25 +0200 | [diff] [blame] | 3054 | func_id != BPF_FUNC_map_delete_elem && |
| 3055 | func_id != BPF_FUNC_map_push_elem && |
| 3056 | func_id != BPF_FUNC_map_pop_elem && |
| 3057 | func_id != BPF_FUNC_map_peek_elem) |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 3058 | return 0; |
Daniel Borkmann | 09772d9 | 2018-06-02 23:06:35 +0200 | [diff] [blame] | 3059 | |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 3060 | if (meta->map_ptr == NULL) { |
| 3061 | verbose(env, "kernel subsystem misconfigured verifier\n"); |
| 3062 | return -EINVAL; |
| 3063 | } |
| 3064 | |
| 3065 | if (!BPF_MAP_PTR(aux->map_state)) |
| 3066 | bpf_map_ptr_store(aux, meta->map_ptr, |
| 3067 | meta->map_ptr->unpriv_array); |
| 3068 | else if (BPF_MAP_PTR(aux->map_state) != meta->map_ptr) |
| 3069 | bpf_map_ptr_store(aux, BPF_MAP_PTR_POISON, |
| 3070 | meta->map_ptr->unpriv_array); |
| 3071 | return 0; |
| 3072 | } |
| 3073 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 3074 | static int check_reference_leak(struct bpf_verifier_env *env) |
| 3075 | { |
| 3076 | struct bpf_func_state *state = cur_func(env); |
| 3077 | int i; |
| 3078 | |
| 3079 | for (i = 0; i < state->acquired_refs; i++) { |
| 3080 | verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", |
| 3081 | state->refs[i].id, state->refs[i].insn_idx); |
| 3082 | } |
| 3083 | return state->acquired_refs ? -EINVAL : 0; |
| 3084 | } |
| 3085 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 3086 | static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn_idx) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3087 | { |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3088 | const struct bpf_func_proto *fn = NULL; |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 3089 | struct bpf_reg_state *regs; |
Daniel Borkmann | 33ff982 | 2016-04-13 00:10:50 +0200 | [diff] [blame] | 3090 | struct bpf_call_arg_meta meta; |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 3091 | bool changes_data; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3092 | int i, err; |
| 3093 | |
| 3094 | /* find function prototype */ |
| 3095 | if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 3096 | verbose(env, "invalid func %s#%d\n", func_id_name(func_id), |
| 3097 | func_id); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3098 | return -EINVAL; |
| 3099 | } |
| 3100 | |
Jakub Kicinski | 00176a3 | 2017-10-16 16:40:54 -0700 | [diff] [blame] | 3101 | if (env->ops->get_func_proto) |
Andrey Ignatov | 5e43f89 | 2018-03-30 15:08:00 -0700 | [diff] [blame] | 3102 | fn = env->ops->get_func_proto(func_id, env->prog); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3103 | if (!fn) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 3104 | verbose(env, "unknown func %s#%d\n", func_id_name(func_id), |
| 3105 | func_id); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3106 | return -EINVAL; |
| 3107 | } |
| 3108 | |
| 3109 | /* eBPF programs must be GPL compatible to use GPL-ed functions */ |
Daniel Borkmann | 24701ec | 2015-03-01 12:31:47 +0100 | [diff] [blame] | 3110 | if (!env->prog->gpl_compatible && fn->gpl_only) { |
Daniel Borkmann | 3fe2867 | 2018-06-02 23:06:33 +0200 | [diff] [blame] | 3111 | verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3112 | return -EINVAL; |
| 3113 | } |
| 3114 | |
Daniel Borkmann | 04514d1 | 2017-12-14 21:07:25 +0100 | [diff] [blame] | 3115 | /* With LD_ABS/IND some JITs save/restore skb from r1. */ |
Martin KaFai Lau | 17bedab | 2016-12-07 15:53:11 -0800 | [diff] [blame] | 3116 | changes_data = bpf_helper_changes_pkt_data(fn->func); |
Daniel Borkmann | 04514d1 | 2017-12-14 21:07:25 +0100 | [diff] [blame] | 3117 | if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) { |
| 3118 | verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n", |
| 3119 | func_id_name(func_id), func_id); |
| 3120 | return -EINVAL; |
| 3121 | } |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 3122 | |
Daniel Borkmann | 33ff982 | 2016-04-13 00:10:50 +0200 | [diff] [blame] | 3123 | memset(&meta, 0, sizeof(meta)); |
Daniel Borkmann | 36bbef5 | 2016-09-20 00:26:13 +0200 | [diff] [blame] | 3124 | meta.pkt_access = fn->pkt_access; |
Daniel Borkmann | 33ff982 | 2016-04-13 00:10:50 +0200 | [diff] [blame] | 3125 | |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 3126 | err = check_func_proto(fn, func_id); |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 3127 | if (err) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 3128 | verbose(env, "kernel subsystem misconfigured func %s#%d\n", |
Thomas Graf | ebb676d | 2016-10-27 11:23:51 +0200 | [diff] [blame] | 3129 | func_id_name(func_id), func_id); |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 3130 | return err; |
| 3131 | } |
| 3132 | |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 3133 | meta.func_id = func_id; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3134 | /* check args */ |
Daniel Borkmann | 33ff982 | 2016-04-13 00:10:50 +0200 | [diff] [blame] | 3135 | err = check_func_arg(env, BPF_REG_1, fn->arg1_type, &meta); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3136 | if (err) |
| 3137 | return err; |
Daniel Borkmann | 33ff982 | 2016-04-13 00:10:50 +0200 | [diff] [blame] | 3138 | err = check_func_arg(env, BPF_REG_2, fn->arg2_type, &meta); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3139 | if (err) |
| 3140 | return err; |
Daniel Borkmann | 33ff982 | 2016-04-13 00:10:50 +0200 | [diff] [blame] | 3141 | err = check_func_arg(env, BPF_REG_3, fn->arg3_type, &meta); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3142 | if (err) |
| 3143 | return err; |
Daniel Borkmann | 33ff982 | 2016-04-13 00:10:50 +0200 | [diff] [blame] | 3144 | err = check_func_arg(env, BPF_REG_4, fn->arg4_type, &meta); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3145 | if (err) |
| 3146 | return err; |
Daniel Borkmann | 33ff982 | 2016-04-13 00:10:50 +0200 | [diff] [blame] | 3147 | err = check_func_arg(env, BPF_REG_5, fn->arg5_type, &meta); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3148 | if (err) |
| 3149 | return err; |
| 3150 | |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 3151 | err = record_func_map(env, &meta, func_id, insn_idx); |
| 3152 | if (err) |
| 3153 | return err; |
| 3154 | |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 3155 | /* Mark slots with STACK_MISC in case of raw mode, stack offset |
| 3156 | * is inferred from register state. |
| 3157 | */ |
| 3158 | for (i = 0; i < meta.access_size; i++) { |
Daniel Borkmann | ca36960 | 2018-02-23 22:29:05 +0100 | [diff] [blame] | 3159 | err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B, |
| 3160 | BPF_WRITE, -1, false); |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 3161 | if (err) |
| 3162 | return err; |
| 3163 | } |
| 3164 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 3165 | if (func_id == BPF_FUNC_tail_call) { |
| 3166 | err = check_reference_leak(env); |
| 3167 | if (err) { |
| 3168 | verbose(env, "tail_call would lead to reference leak\n"); |
| 3169 | return err; |
| 3170 | } |
| 3171 | } else if (is_release_function(func_id)) { |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 3172 | err = release_reference(env, meta.ref_obj_id); |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 3173 | if (err) { |
| 3174 | verbose(env, "func %s#%d reference has not been acquired before\n", |
| 3175 | func_id_name(func_id), func_id); |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 3176 | return err; |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 3177 | } |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 3178 | } |
| 3179 | |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 3180 | regs = cur_regs(env); |
Roman Gushchin | cd33943 | 2018-08-02 14:27:24 -0700 | [diff] [blame] | 3181 | |
| 3182 | /* check that flags argument in get_local_storage(map, flags) is 0, |
| 3183 | * this is required because get_local_storage() can't return an error. |
| 3184 | */ |
| 3185 | if (func_id == BPF_FUNC_get_local_storage && |
| 3186 | !register_is_null(®s[BPF_REG_2])) { |
| 3187 | verbose(env, "get_local_storage() doesn't support non-zero flags\n"); |
| 3188 | return -EINVAL; |
| 3189 | } |
| 3190 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3191 | /* reset caller saved regs */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 3192 | for (i = 0; i < CALLER_SAVED_REGS; i++) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 3193 | mark_reg_not_init(env, regs, caller_saved[i]); |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 3194 | check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); |
| 3195 | } |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3196 | |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 3197 | /* update return register (already marked as written above) */ |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3198 | if (fn->ret_type == RET_INTEGER) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3199 | /* sets type to SCALAR_VALUE */ |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 3200 | mark_reg_unknown(env, regs, BPF_REG_0); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3201 | } else if (fn->ret_type == RET_VOID) { |
| 3202 | regs[BPF_REG_0].type = NOT_INIT; |
Roman Gushchin | 3e6a4b3 | 2018-08-02 14:27:22 -0700 | [diff] [blame] | 3203 | } else if (fn->ret_type == RET_PTR_TO_MAP_VALUE_OR_NULL || |
| 3204 | fn->ret_type == RET_PTR_TO_MAP_VALUE) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3205 | /* There is no offset yet applied, variable or fixed */ |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 3206 | mark_reg_known_zero(env, regs, BPF_REG_0); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3207 | /* remember map_ptr, so that check_map_access() |
| 3208 | * can check 'value_size' boundary of memory access |
| 3209 | * to map element returned from bpf_map_lookup_elem() |
| 3210 | */ |
Daniel Borkmann | 33ff982 | 2016-04-13 00:10:50 +0200 | [diff] [blame] | 3211 | if (meta.map_ptr == NULL) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 3212 | verbose(env, |
| 3213 | "kernel subsystem misconfigured verifier\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3214 | return -EINVAL; |
| 3215 | } |
Daniel Borkmann | 33ff982 | 2016-04-13 00:10:50 +0200 | [diff] [blame] | 3216 | regs[BPF_REG_0].map_ptr = meta.map_ptr; |
Daniel Borkmann | 4d31f30 | 2018-11-01 00:05:53 +0100 | [diff] [blame] | 3217 | if (fn->ret_type == RET_PTR_TO_MAP_VALUE) { |
| 3218 | regs[BPF_REG_0].type = PTR_TO_MAP_VALUE; |
Alexei Starovoitov | e16d2f1 | 2019-01-31 15:40:05 -0800 | [diff] [blame] | 3219 | if (map_value_has_spin_lock(meta.map_ptr)) |
| 3220 | regs[BPF_REG_0].id = ++env->id_gen; |
Daniel Borkmann | 4d31f30 | 2018-11-01 00:05:53 +0100 | [diff] [blame] | 3221 | } else { |
| 3222 | regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL; |
| 3223 | regs[BPF_REG_0].id = ++env->id_gen; |
| 3224 | } |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 3225 | } else if (fn->ret_type == RET_PTR_TO_SOCKET_OR_NULL) { |
| 3226 | mark_reg_known_zero(env, regs, BPF_REG_0); |
| 3227 | regs[BPF_REG_0].type = PTR_TO_SOCKET_OR_NULL; |
Lorenz Bauer | 0f3adc2 | 2019-03-22 09:53:59 +0800 | [diff] [blame] | 3228 | regs[BPF_REG_0].id = ++env->id_gen; |
Lorenz Bauer | 85a51f8 | 2019-03-22 09:54:00 +0800 | [diff] [blame] | 3229 | } else if (fn->ret_type == RET_PTR_TO_SOCK_COMMON_OR_NULL) { |
| 3230 | mark_reg_known_zero(env, regs, BPF_REG_0); |
| 3231 | regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON_OR_NULL; |
| 3232 | regs[BPF_REG_0].id = ++env->id_gen; |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 3233 | } else if (fn->ret_type == RET_PTR_TO_TCP_SOCK_OR_NULL) { |
| 3234 | mark_reg_known_zero(env, regs, BPF_REG_0); |
| 3235 | regs[BPF_REG_0].type = PTR_TO_TCP_SOCK_OR_NULL; |
| 3236 | regs[BPF_REG_0].id = ++env->id_gen; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3237 | } else { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 3238 | verbose(env, "unknown return type %d of func %s#%d\n", |
Thomas Graf | ebb676d | 2016-10-27 11:23:51 +0200 | [diff] [blame] | 3239 | fn->ret_type, func_id_name(func_id), func_id); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3240 | return -EINVAL; |
| 3241 | } |
Alexei Starovoitov | 04fd61ab | 2015-05-19 16:59:03 -0700 | [diff] [blame] | 3242 | |
Lorenz Bauer | 0f3adc2 | 2019-03-22 09:53:59 +0800 | [diff] [blame] | 3243 | if (is_ptr_cast_function(func_id)) { |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 3244 | /* For release_reference() */ |
| 3245 | regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id; |
Lorenz Bauer | 0f3adc2 | 2019-03-22 09:53:59 +0800 | [diff] [blame] | 3246 | } else if (is_acquire_function(func_id)) { |
| 3247 | int id = acquire_reference_state(env, insn_idx); |
| 3248 | |
| 3249 | if (id < 0) |
| 3250 | return id; |
| 3251 | /* For mark_ptr_or_null_reg() */ |
| 3252 | regs[BPF_REG_0].id = id; |
| 3253 | /* For release_reference() */ |
| 3254 | regs[BPF_REG_0].ref_obj_id = id; |
| 3255 | } |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 3256 | |
Yonghong Song | 849fa50 | 2018-04-28 22:28:09 -0700 | [diff] [blame] | 3257 | do_refine_retval_range(regs, fn->ret_type, func_id, &meta); |
| 3258 | |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 3259 | err = check_map_func_compatibility(env, meta.map_ptr, func_id); |
Kaixu Xia | 35578d7 | 2015-08-06 07:02:35 +0000 | [diff] [blame] | 3260 | if (err) |
| 3261 | return err; |
Alexei Starovoitov | 04fd61ab | 2015-05-19 16:59:03 -0700 | [diff] [blame] | 3262 | |
Yonghong Song | c195651e | 2018-04-28 22:28:08 -0700 | [diff] [blame] | 3263 | if (func_id == BPF_FUNC_get_stack && !env->prog->has_callchain_buf) { |
| 3264 | const char *err_str; |
| 3265 | |
| 3266 | #ifdef CONFIG_PERF_EVENTS |
| 3267 | err = get_callchain_buffers(sysctl_perf_event_max_stack); |
| 3268 | err_str = "cannot get callchain buffer for func %s#%d\n"; |
| 3269 | #else |
| 3270 | err = -ENOTSUPP; |
| 3271 | err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n"; |
| 3272 | #endif |
| 3273 | if (err) { |
| 3274 | verbose(env, err_str, func_id_name(func_id), func_id); |
| 3275 | return err; |
| 3276 | } |
| 3277 | |
| 3278 | env->prog->has_callchain_buf = true; |
| 3279 | } |
| 3280 | |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 3281 | if (changes_data) |
| 3282 | clear_all_pkt_pointers(env); |
| 3283 | return 0; |
| 3284 | } |
| 3285 | |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3286 | static bool signed_add_overflows(s64 a, s64 b) |
| 3287 | { |
| 3288 | /* Do the add in u64, where overflow is well-defined */ |
| 3289 | s64 res = (s64)((u64)a + (u64)b); |
| 3290 | |
| 3291 | if (b < 0) |
| 3292 | return res > a; |
| 3293 | return res < a; |
| 3294 | } |
| 3295 | |
| 3296 | static bool signed_sub_overflows(s64 a, s64 b) |
| 3297 | { |
| 3298 | /* Do the sub in u64, where overflow is well-defined */ |
| 3299 | s64 res = (s64)((u64)a - (u64)b); |
| 3300 | |
| 3301 | if (b < 0) |
| 3302 | return res < a; |
| 3303 | return res > a; |
David S. Miller | d117441 | 2017-05-10 11:22:52 -0700 | [diff] [blame] | 3304 | } |
| 3305 | |
Alexei Starovoitov | bb7f0f9 | 2017-12-18 20:12:00 -0800 | [diff] [blame] | 3306 | static bool check_reg_sane_offset(struct bpf_verifier_env *env, |
| 3307 | const struct bpf_reg_state *reg, |
| 3308 | enum bpf_reg_type type) |
| 3309 | { |
| 3310 | bool known = tnum_is_const(reg->var_off); |
| 3311 | s64 val = reg->var_off.value; |
| 3312 | s64 smin = reg->smin_value; |
| 3313 | |
| 3314 | if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) { |
| 3315 | verbose(env, "math between %s pointer and %lld is not allowed\n", |
| 3316 | reg_type_str[type], val); |
| 3317 | return false; |
| 3318 | } |
| 3319 | |
| 3320 | if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) { |
| 3321 | verbose(env, "%s pointer offset %d is not allowed\n", |
| 3322 | reg_type_str[type], reg->off); |
| 3323 | return false; |
| 3324 | } |
| 3325 | |
| 3326 | if (smin == S64_MIN) { |
| 3327 | verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n", |
| 3328 | reg_type_str[type]); |
| 3329 | return false; |
| 3330 | } |
| 3331 | |
| 3332 | if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) { |
| 3333 | verbose(env, "value %lld makes %s pointer be out of bounds\n", |
| 3334 | smin, reg_type_str[type]); |
| 3335 | return false; |
| 3336 | } |
| 3337 | |
| 3338 | return true; |
| 3339 | } |
| 3340 | |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 3341 | static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env) |
| 3342 | { |
| 3343 | return &env->insn_aux_data[env->insn_idx]; |
| 3344 | } |
| 3345 | |
| 3346 | static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, |
| 3347 | u32 *ptr_limit, u8 opcode, bool off_is_neg) |
| 3348 | { |
| 3349 | bool mask_to_left = (opcode == BPF_ADD && off_is_neg) || |
| 3350 | (opcode == BPF_SUB && !off_is_neg); |
| 3351 | u32 off; |
| 3352 | |
| 3353 | switch (ptr_reg->type) { |
| 3354 | case PTR_TO_STACK: |
Andrey Ignatov | 088ec26 | 2019-04-03 23:22:39 -0700 | [diff] [blame] | 3355 | /* Indirect variable offset stack access is prohibited in |
| 3356 | * unprivileged mode so it's not handled here. |
| 3357 | */ |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 3358 | off = ptr_reg->off + ptr_reg->var_off.value; |
| 3359 | if (mask_to_left) |
| 3360 | *ptr_limit = MAX_BPF_STACK + off; |
| 3361 | else |
| 3362 | *ptr_limit = -off; |
| 3363 | return 0; |
| 3364 | case PTR_TO_MAP_VALUE: |
| 3365 | if (mask_to_left) { |
| 3366 | *ptr_limit = ptr_reg->umax_value + ptr_reg->off; |
| 3367 | } else { |
| 3368 | off = ptr_reg->smin_value + ptr_reg->off; |
| 3369 | *ptr_limit = ptr_reg->map_ptr->value_size - off; |
| 3370 | } |
| 3371 | return 0; |
| 3372 | default: |
| 3373 | return -EINVAL; |
| 3374 | } |
| 3375 | } |
| 3376 | |
Daniel Borkmann | d3bd741 | 2019-01-06 00:54:37 +0100 | [diff] [blame] | 3377 | static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env, |
| 3378 | const struct bpf_insn *insn) |
| 3379 | { |
| 3380 | return env->allow_ptr_leaks || BPF_SRC(insn->code) == BPF_K; |
| 3381 | } |
| 3382 | |
| 3383 | static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux, |
| 3384 | u32 alu_state, u32 alu_limit) |
| 3385 | { |
| 3386 | /* If we arrived here from different branches with different |
| 3387 | * state or limits to sanitize, then this won't work. |
| 3388 | */ |
| 3389 | if (aux->alu_state && |
| 3390 | (aux->alu_state != alu_state || |
| 3391 | aux->alu_limit != alu_limit)) |
| 3392 | return -EACCES; |
| 3393 | |
| 3394 | /* Corresponding fixup done in fixup_bpf_calls(). */ |
| 3395 | aux->alu_state = alu_state; |
| 3396 | aux->alu_limit = alu_limit; |
| 3397 | return 0; |
| 3398 | } |
| 3399 | |
| 3400 | static int sanitize_val_alu(struct bpf_verifier_env *env, |
| 3401 | struct bpf_insn *insn) |
| 3402 | { |
| 3403 | struct bpf_insn_aux_data *aux = cur_aux(env); |
| 3404 | |
| 3405 | if (can_skip_alu_sanitation(env, insn)) |
| 3406 | return 0; |
| 3407 | |
| 3408 | return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0); |
| 3409 | } |
| 3410 | |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 3411 | static int sanitize_ptr_alu(struct bpf_verifier_env *env, |
| 3412 | struct bpf_insn *insn, |
| 3413 | const struct bpf_reg_state *ptr_reg, |
| 3414 | struct bpf_reg_state *dst_reg, |
| 3415 | bool off_is_neg) |
| 3416 | { |
| 3417 | struct bpf_verifier_state *vstate = env->cur_state; |
| 3418 | struct bpf_insn_aux_data *aux = cur_aux(env); |
| 3419 | bool ptr_is_dst_reg = ptr_reg == dst_reg; |
| 3420 | u8 opcode = BPF_OP(insn->code); |
| 3421 | u32 alu_state, alu_limit; |
| 3422 | struct bpf_reg_state tmp; |
| 3423 | bool ret; |
| 3424 | |
Daniel Borkmann | d3bd741 | 2019-01-06 00:54:37 +0100 | [diff] [blame] | 3425 | if (can_skip_alu_sanitation(env, insn)) |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 3426 | return 0; |
| 3427 | |
| 3428 | /* We already marked aux for masking from non-speculative |
| 3429 | * paths, thus we got here in the first place. We only care |
| 3430 | * to explore bad access from here. |
| 3431 | */ |
| 3432 | if (vstate->speculative) |
| 3433 | goto do_sim; |
| 3434 | |
| 3435 | alu_state = off_is_neg ? BPF_ALU_NEG_VALUE : 0; |
| 3436 | alu_state |= ptr_is_dst_reg ? |
| 3437 | BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST; |
| 3438 | |
| 3439 | if (retrieve_ptr_limit(ptr_reg, &alu_limit, opcode, off_is_neg)) |
| 3440 | return 0; |
Daniel Borkmann | d3bd741 | 2019-01-06 00:54:37 +0100 | [diff] [blame] | 3441 | if (update_alu_sanitation_state(aux, alu_state, alu_limit)) |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 3442 | return -EACCES; |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 3443 | do_sim: |
| 3444 | /* Simulate and find potential out-of-bounds access under |
| 3445 | * speculative execution from truncation as a result of |
| 3446 | * masking when off was not within expected range. If off |
| 3447 | * sits in dst, then we temporarily need to move ptr there |
| 3448 | * to simulate dst (== 0) +/-= ptr. Needed, for example, |
| 3449 | * for cases where we use K-based arithmetic in one direction |
| 3450 | * and truncated reg-based in the other in order to explore |
| 3451 | * bad access. |
| 3452 | */ |
| 3453 | if (!ptr_is_dst_reg) { |
| 3454 | tmp = *dst_reg; |
| 3455 | *dst_reg = *ptr_reg; |
| 3456 | } |
| 3457 | ret = push_stack(env, env->insn_idx + 1, env->insn_idx, true); |
Xu Yu | 0803278 | 2019-03-21 18:00:35 +0800 | [diff] [blame] | 3458 | if (!ptr_is_dst_reg && ret) |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 3459 | *dst_reg = tmp; |
| 3460 | return !ret ? -EFAULT : 0; |
| 3461 | } |
| 3462 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3463 | /* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off. |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3464 | * Caller should also handle BPF_MOV case separately. |
| 3465 | * If we return -EACCES, caller may want to try again treating pointer as a |
| 3466 | * scalar. So we only emit a diagnostic if !env->allow_ptr_leaks. |
| 3467 | */ |
| 3468 | static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, |
| 3469 | struct bpf_insn *insn, |
| 3470 | const struct bpf_reg_state *ptr_reg, |
| 3471 | const struct bpf_reg_state *off_reg) |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 3472 | { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 3473 | struct bpf_verifier_state *vstate = env->cur_state; |
| 3474 | struct bpf_func_state *state = vstate->frame[vstate->curframe]; |
| 3475 | struct bpf_reg_state *regs = state->regs, *dst_reg; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3476 | bool known = tnum_is_const(off_reg->var_off); |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3477 | s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value, |
| 3478 | smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value; |
| 3479 | u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value, |
| 3480 | umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value; |
Daniel Borkmann | 9d7ecee | 2019-01-03 00:58:32 +0100 | [diff] [blame] | 3481 | u32 dst = insn->dst_reg, src = insn->src_reg; |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 3482 | u8 opcode = BPF_OP(insn->code); |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 3483 | int ret; |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 3484 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3485 | dst_reg = ®s[dst]; |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 3486 | |
Daniel Borkmann | 6f16101 | 2018-01-18 01:15:21 +0100 | [diff] [blame] | 3487 | if ((known && (smin_val != smax_val || umin_val != umax_val)) || |
| 3488 | smin_val > smax_val || umin_val > umax_val) { |
| 3489 | /* Taint dst register if offset had invalid bounds derived from |
| 3490 | * e.g. dead branches. |
| 3491 | */ |
| 3492 | __mark_reg_unknown(dst_reg); |
| 3493 | return 0; |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 3494 | } |
| 3495 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3496 | if (BPF_CLASS(insn->code) != BPF_ALU64) { |
| 3497 | /* 32-bit ALU ops on pointers produce (meaningless) scalars */ |
Alexei Starovoitov | 82abbf8 | 2017-12-18 20:15:20 -0800 | [diff] [blame] | 3498 | verbose(env, |
| 3499 | "R%d 32-bit pointer arithmetic prohibited\n", |
| 3500 | dst); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3501 | return -EACCES; |
| 3502 | } |
David S. Miller | d117441 | 2017-05-10 11:22:52 -0700 | [diff] [blame] | 3503 | |
Joe Stringer | aad2eea | 2018-10-02 13:35:30 -0700 | [diff] [blame] | 3504 | switch (ptr_reg->type) { |
| 3505 | case PTR_TO_MAP_VALUE_OR_NULL: |
| 3506 | verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", |
| 3507 | dst, reg_type_str[ptr_reg->type]); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3508 | return -EACCES; |
Joe Stringer | aad2eea | 2018-10-02 13:35:30 -0700 | [diff] [blame] | 3509 | case CONST_PTR_TO_MAP: |
| 3510 | case PTR_TO_PACKET_END: |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 3511 | case PTR_TO_SOCKET: |
| 3512 | case PTR_TO_SOCKET_OR_NULL: |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 3513 | case PTR_TO_SOCK_COMMON: |
| 3514 | case PTR_TO_SOCK_COMMON_OR_NULL: |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 3515 | case PTR_TO_TCP_SOCK: |
| 3516 | case PTR_TO_TCP_SOCK_OR_NULL: |
Joe Stringer | aad2eea | 2018-10-02 13:35:30 -0700 | [diff] [blame] | 3517 | verbose(env, "R%d pointer arithmetic on %s prohibited\n", |
| 3518 | dst, reg_type_str[ptr_reg->type]); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3519 | return -EACCES; |
Daniel Borkmann | 9d7ecee | 2019-01-03 00:58:32 +0100 | [diff] [blame] | 3520 | case PTR_TO_MAP_VALUE: |
| 3521 | if (!env->allow_ptr_leaks && !known && (smin_val < 0) != (smax_val < 0)) { |
| 3522 | verbose(env, "R%d has unknown scalar with mixed signed bounds, pointer arithmetic with it prohibited for !root\n", |
| 3523 | off_reg == dst_reg ? dst : src); |
| 3524 | return -EACCES; |
| 3525 | } |
| 3526 | /* fall-through */ |
Joe Stringer | aad2eea | 2018-10-02 13:35:30 -0700 | [diff] [blame] | 3527 | default: |
| 3528 | break; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3529 | } |
| 3530 | |
| 3531 | /* In case of 'scalar += pointer', dst_reg inherits pointer type and id. |
| 3532 | * The id may be overwritten later if we create a new variable offset. |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 3533 | */ |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3534 | dst_reg->type = ptr_reg->type; |
| 3535 | dst_reg->id = ptr_reg->id; |
Josef Bacik | f23cc64 | 2016-11-14 15:45:36 -0500 | [diff] [blame] | 3536 | |
Alexei Starovoitov | bb7f0f9 | 2017-12-18 20:12:00 -0800 | [diff] [blame] | 3537 | if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) || |
| 3538 | !check_reg_sane_offset(env, ptr_reg, ptr_reg->type)) |
| 3539 | return -EINVAL; |
| 3540 | |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 3541 | switch (opcode) { |
| 3542 | case BPF_ADD: |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 3543 | ret = sanitize_ptr_alu(env, insn, ptr_reg, dst_reg, smin_val < 0); |
| 3544 | if (ret < 0) { |
| 3545 | verbose(env, "R%d tried to add from different maps or paths\n", dst); |
| 3546 | return ret; |
| 3547 | } |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3548 | /* We can take a fixed offset as long as it doesn't overflow |
| 3549 | * the s32 'off' field |
| 3550 | */ |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3551 | if (known && (ptr_reg->off + smin_val == |
| 3552 | (s64)(s32)(ptr_reg->off + smin_val))) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3553 | /* pointer += K. Accumulate it into fixed offset */ |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3554 | dst_reg->smin_value = smin_ptr; |
| 3555 | dst_reg->smax_value = smax_ptr; |
| 3556 | dst_reg->umin_value = umin_ptr; |
| 3557 | dst_reg->umax_value = umax_ptr; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3558 | dst_reg->var_off = ptr_reg->var_off; |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3559 | dst_reg->off = ptr_reg->off + smin_val; |
Daniel Borkmann | 0962590 | 2018-11-01 00:05:52 +0100 | [diff] [blame] | 3560 | dst_reg->raw = ptr_reg->raw; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3561 | break; |
| 3562 | } |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3563 | /* A new variable offset is created. Note that off_reg->off |
| 3564 | * == 0, since it's a scalar. |
| 3565 | * dst_reg gets the pointer type and since some positive |
| 3566 | * integer value was added to the pointer, give it a new 'id' |
| 3567 | * if it's a PTR_TO_PACKET. |
| 3568 | * this creates a new 'base' pointer, off_reg (variable) gets |
| 3569 | * added into the variable offset, and we copy the fixed offset |
| 3570 | * from ptr_reg. |
| 3571 | */ |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3572 | if (signed_add_overflows(smin_ptr, smin_val) || |
| 3573 | signed_add_overflows(smax_ptr, smax_val)) { |
| 3574 | dst_reg->smin_value = S64_MIN; |
| 3575 | dst_reg->smax_value = S64_MAX; |
| 3576 | } else { |
| 3577 | dst_reg->smin_value = smin_ptr + smin_val; |
| 3578 | dst_reg->smax_value = smax_ptr + smax_val; |
| 3579 | } |
| 3580 | if (umin_ptr + umin_val < umin_ptr || |
| 3581 | umax_ptr + umax_val < umax_ptr) { |
| 3582 | dst_reg->umin_value = 0; |
| 3583 | dst_reg->umax_value = U64_MAX; |
| 3584 | } else { |
| 3585 | dst_reg->umin_value = umin_ptr + umin_val; |
| 3586 | dst_reg->umax_value = umax_ptr + umax_val; |
| 3587 | } |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3588 | dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); |
| 3589 | dst_reg->off = ptr_reg->off; |
Daniel Borkmann | 0962590 | 2018-11-01 00:05:52 +0100 | [diff] [blame] | 3590 | dst_reg->raw = ptr_reg->raw; |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 3591 | if (reg_is_pkt_pointer(ptr_reg)) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3592 | dst_reg->id = ++env->id_gen; |
| 3593 | /* something was added to pkt_ptr, set range to zero */ |
Daniel Borkmann | 0962590 | 2018-11-01 00:05:52 +0100 | [diff] [blame] | 3594 | dst_reg->raw = 0; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3595 | } |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 3596 | break; |
| 3597 | case BPF_SUB: |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 3598 | ret = sanitize_ptr_alu(env, insn, ptr_reg, dst_reg, smin_val < 0); |
| 3599 | if (ret < 0) { |
| 3600 | verbose(env, "R%d tried to sub from different maps or paths\n", dst); |
| 3601 | return ret; |
| 3602 | } |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3603 | if (dst_reg == off_reg) { |
| 3604 | /* scalar -= pointer. Creates an unknown scalar */ |
Alexei Starovoitov | 82abbf8 | 2017-12-18 20:15:20 -0800 | [diff] [blame] | 3605 | verbose(env, "R%d tried to subtract pointer from scalar\n", |
| 3606 | dst); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3607 | return -EACCES; |
| 3608 | } |
| 3609 | /* We don't allow subtraction from FP, because (according to |
| 3610 | * test_verifier.c test "invalid fp arithmetic", JITs might not |
| 3611 | * be able to deal with it. |
Edward Cree | 9305706 | 2017-07-21 14:37:34 +0100 | [diff] [blame] | 3612 | */ |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3613 | if (ptr_reg->type == PTR_TO_STACK) { |
Alexei Starovoitov | 82abbf8 | 2017-12-18 20:15:20 -0800 | [diff] [blame] | 3614 | verbose(env, "R%d subtraction from stack pointer prohibited\n", |
| 3615 | dst); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3616 | return -EACCES; |
| 3617 | } |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3618 | if (known && (ptr_reg->off - smin_val == |
| 3619 | (s64)(s32)(ptr_reg->off - smin_val))) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3620 | /* pointer -= K. Subtract it from fixed offset */ |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3621 | dst_reg->smin_value = smin_ptr; |
| 3622 | dst_reg->smax_value = smax_ptr; |
| 3623 | dst_reg->umin_value = umin_ptr; |
| 3624 | dst_reg->umax_value = umax_ptr; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3625 | dst_reg->var_off = ptr_reg->var_off; |
| 3626 | dst_reg->id = ptr_reg->id; |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3627 | dst_reg->off = ptr_reg->off - smin_val; |
Daniel Borkmann | 0962590 | 2018-11-01 00:05:52 +0100 | [diff] [blame] | 3628 | dst_reg->raw = ptr_reg->raw; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3629 | break; |
| 3630 | } |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3631 | /* A new variable offset is created. If the subtrahend is known |
| 3632 | * nonnegative, then any reg->range we had before is still good. |
| 3633 | */ |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3634 | if (signed_sub_overflows(smin_ptr, smax_val) || |
| 3635 | signed_sub_overflows(smax_ptr, smin_val)) { |
| 3636 | /* Overflow possible, we know nothing */ |
| 3637 | dst_reg->smin_value = S64_MIN; |
| 3638 | dst_reg->smax_value = S64_MAX; |
| 3639 | } else { |
| 3640 | dst_reg->smin_value = smin_ptr - smax_val; |
| 3641 | dst_reg->smax_value = smax_ptr - smin_val; |
| 3642 | } |
| 3643 | if (umin_ptr < umax_val) { |
| 3644 | /* Overflow possible, we know nothing */ |
| 3645 | dst_reg->umin_value = 0; |
| 3646 | dst_reg->umax_value = U64_MAX; |
| 3647 | } else { |
| 3648 | /* Cannot overflow (as long as bounds are consistent) */ |
| 3649 | dst_reg->umin_value = umin_ptr - umax_val; |
| 3650 | dst_reg->umax_value = umax_ptr - umin_val; |
| 3651 | } |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3652 | dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off); |
| 3653 | dst_reg->off = ptr_reg->off; |
Daniel Borkmann | 0962590 | 2018-11-01 00:05:52 +0100 | [diff] [blame] | 3654 | dst_reg->raw = ptr_reg->raw; |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 3655 | if (reg_is_pkt_pointer(ptr_reg)) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3656 | dst_reg->id = ++env->id_gen; |
| 3657 | /* something was added to pkt_ptr, set range to zero */ |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3658 | if (smin_val < 0) |
Daniel Borkmann | 0962590 | 2018-11-01 00:05:52 +0100 | [diff] [blame] | 3659 | dst_reg->raw = 0; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3660 | } |
| 3661 | break; |
| 3662 | case BPF_AND: |
| 3663 | case BPF_OR: |
| 3664 | case BPF_XOR: |
Alexei Starovoitov | 82abbf8 | 2017-12-18 20:15:20 -0800 | [diff] [blame] | 3665 | /* bitwise ops on pointers are troublesome, prohibit. */ |
| 3666 | verbose(env, "R%d bitwise operator %s on pointer prohibited\n", |
| 3667 | dst, bpf_alu_string[opcode >> 4]); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3668 | return -EACCES; |
| 3669 | default: |
| 3670 | /* other operators (e.g. MUL,LSH) produce non-pointer results */ |
Alexei Starovoitov | 82abbf8 | 2017-12-18 20:15:20 -0800 | [diff] [blame] | 3671 | verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", |
| 3672 | dst, bpf_alu_string[opcode >> 4]); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3673 | return -EACCES; |
| 3674 | } |
| 3675 | |
Alexei Starovoitov | bb7f0f9 | 2017-12-18 20:12:00 -0800 | [diff] [blame] | 3676 | if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) |
| 3677 | return -EINVAL; |
| 3678 | |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3679 | __update_reg_bounds(dst_reg); |
| 3680 | __reg_deduce_bounds(dst_reg); |
| 3681 | __reg_bound_offset(dst_reg); |
Daniel Borkmann | 0d6303d | 2019-01-03 00:58:30 +0100 | [diff] [blame] | 3682 | |
| 3683 | /* For unprivileged we require that resulting offset must be in bounds |
| 3684 | * in order to be able to sanitize access later on. |
| 3685 | */ |
Daniel Borkmann | e4298d2 | 2019-01-03 00:58:31 +0100 | [diff] [blame] | 3686 | if (!env->allow_ptr_leaks) { |
| 3687 | if (dst_reg->type == PTR_TO_MAP_VALUE && |
| 3688 | check_map_access(env, dst, dst_reg->off, 1, false)) { |
| 3689 | verbose(env, "R%d pointer arithmetic of map value goes out of range, " |
| 3690 | "prohibited for !root\n", dst); |
| 3691 | return -EACCES; |
| 3692 | } else if (dst_reg->type == PTR_TO_STACK && |
| 3693 | check_stack_access(env, dst_reg, dst_reg->off + |
| 3694 | dst_reg->var_off.value, 1)) { |
| 3695 | verbose(env, "R%d stack pointer arithmetic goes out of range, " |
| 3696 | "prohibited for !root\n", dst); |
| 3697 | return -EACCES; |
| 3698 | } |
Daniel Borkmann | 0d6303d | 2019-01-03 00:58:30 +0100 | [diff] [blame] | 3699 | } |
| 3700 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3701 | return 0; |
| 3702 | } |
| 3703 | |
Jann Horn | 468f6ea | 2017-12-18 20:11:56 -0800 | [diff] [blame] | 3704 | /* WARNING: This function does calculations on 64-bit values, but the actual |
| 3705 | * execution may occur on 32-bit values. Therefore, things like bitshifts |
| 3706 | * need extra checks in the 32-bit case. |
| 3707 | */ |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3708 | static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, |
| 3709 | struct bpf_insn *insn, |
| 3710 | struct bpf_reg_state *dst_reg, |
| 3711 | struct bpf_reg_state src_reg) |
| 3712 | { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 3713 | struct bpf_reg_state *regs = cur_regs(env); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3714 | u8 opcode = BPF_OP(insn->code); |
| 3715 | bool src_known, dst_known; |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3716 | s64 smin_val, smax_val; |
| 3717 | u64 umin_val, umax_val; |
Jann Horn | 468f6ea | 2017-12-18 20:11:56 -0800 | [diff] [blame] | 3718 | u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32; |
Daniel Borkmann | d3bd741 | 2019-01-06 00:54:37 +0100 | [diff] [blame] | 3719 | u32 dst = insn->dst_reg; |
| 3720 | int ret; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3721 | |
Jann Horn | b799207 | 2018-10-05 18:17:59 +0200 | [diff] [blame] | 3722 | if (insn_bitness == 32) { |
| 3723 | /* Relevant for 32-bit RSH: Information can propagate towards |
| 3724 | * LSB, so it isn't sufficient to only truncate the output to |
| 3725 | * 32 bits. |
| 3726 | */ |
| 3727 | coerce_reg_to_size(dst_reg, 4); |
| 3728 | coerce_reg_to_size(&src_reg, 4); |
| 3729 | } |
| 3730 | |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3731 | smin_val = src_reg.smin_value; |
| 3732 | smax_val = src_reg.smax_value; |
| 3733 | umin_val = src_reg.umin_value; |
| 3734 | umax_val = src_reg.umax_value; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3735 | src_known = tnum_is_const(src_reg.var_off); |
| 3736 | dst_known = tnum_is_const(dst_reg->var_off); |
| 3737 | |
Daniel Borkmann | 6f16101 | 2018-01-18 01:15:21 +0100 | [diff] [blame] | 3738 | if ((src_known && (smin_val != smax_val || umin_val != umax_val)) || |
| 3739 | smin_val > smax_val || umin_val > umax_val) { |
| 3740 | /* Taint dst register if offset had invalid bounds derived from |
| 3741 | * e.g. dead branches. |
| 3742 | */ |
| 3743 | __mark_reg_unknown(dst_reg); |
| 3744 | return 0; |
| 3745 | } |
| 3746 | |
Alexei Starovoitov | bb7f0f9 | 2017-12-18 20:12:00 -0800 | [diff] [blame] | 3747 | if (!src_known && |
| 3748 | opcode != BPF_ADD && opcode != BPF_SUB && opcode != BPF_AND) { |
| 3749 | __mark_reg_unknown(dst_reg); |
| 3750 | return 0; |
| 3751 | } |
| 3752 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3753 | switch (opcode) { |
| 3754 | case BPF_ADD: |
Daniel Borkmann | d3bd741 | 2019-01-06 00:54:37 +0100 | [diff] [blame] | 3755 | ret = sanitize_val_alu(env, insn); |
| 3756 | if (ret < 0) { |
| 3757 | verbose(env, "R%d tried to add from different pointers or scalars\n", dst); |
| 3758 | return ret; |
| 3759 | } |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3760 | if (signed_add_overflows(dst_reg->smin_value, smin_val) || |
| 3761 | signed_add_overflows(dst_reg->smax_value, smax_val)) { |
| 3762 | dst_reg->smin_value = S64_MIN; |
| 3763 | dst_reg->smax_value = S64_MAX; |
| 3764 | } else { |
| 3765 | dst_reg->smin_value += smin_val; |
| 3766 | dst_reg->smax_value += smax_val; |
| 3767 | } |
| 3768 | if (dst_reg->umin_value + umin_val < umin_val || |
| 3769 | dst_reg->umax_value + umax_val < umax_val) { |
| 3770 | dst_reg->umin_value = 0; |
| 3771 | dst_reg->umax_value = U64_MAX; |
| 3772 | } else { |
| 3773 | dst_reg->umin_value += umin_val; |
| 3774 | dst_reg->umax_value += umax_val; |
| 3775 | } |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3776 | dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off); |
| 3777 | break; |
| 3778 | case BPF_SUB: |
Daniel Borkmann | d3bd741 | 2019-01-06 00:54:37 +0100 | [diff] [blame] | 3779 | ret = sanitize_val_alu(env, insn); |
| 3780 | if (ret < 0) { |
| 3781 | verbose(env, "R%d tried to sub from different pointers or scalars\n", dst); |
| 3782 | return ret; |
| 3783 | } |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3784 | if (signed_sub_overflows(dst_reg->smin_value, smax_val) || |
| 3785 | signed_sub_overflows(dst_reg->smax_value, smin_val)) { |
| 3786 | /* Overflow possible, we know nothing */ |
| 3787 | dst_reg->smin_value = S64_MIN; |
| 3788 | dst_reg->smax_value = S64_MAX; |
| 3789 | } else { |
| 3790 | dst_reg->smin_value -= smax_val; |
| 3791 | dst_reg->smax_value -= smin_val; |
| 3792 | } |
| 3793 | if (dst_reg->umin_value < umax_val) { |
| 3794 | /* Overflow possible, we know nothing */ |
| 3795 | dst_reg->umin_value = 0; |
| 3796 | dst_reg->umax_value = U64_MAX; |
| 3797 | } else { |
| 3798 | /* Cannot overflow (as long as bounds are consistent) */ |
| 3799 | dst_reg->umin_value -= umax_val; |
| 3800 | dst_reg->umax_value -= umin_val; |
| 3801 | } |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3802 | dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off); |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 3803 | break; |
| 3804 | case BPF_MUL: |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3805 | dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off); |
| 3806 | if (smin_val < 0 || dst_reg->smin_value < 0) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3807 | /* Ain't nobody got time to multiply that sign */ |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3808 | __mark_reg_unbounded(dst_reg); |
| 3809 | __update_reg_bounds(dst_reg); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3810 | break; |
| 3811 | } |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3812 | /* Both values are positive, so we can work with unsigned and |
| 3813 | * copy the result to signed (unless it exceeds S64_MAX). |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3814 | */ |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3815 | if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) { |
| 3816 | /* Potential overflow, we know nothing */ |
| 3817 | __mark_reg_unbounded(dst_reg); |
| 3818 | /* (except what we can learn from the var_off) */ |
| 3819 | __update_reg_bounds(dst_reg); |
| 3820 | break; |
| 3821 | } |
| 3822 | dst_reg->umin_value *= umin_val; |
| 3823 | dst_reg->umax_value *= umax_val; |
| 3824 | if (dst_reg->umax_value > S64_MAX) { |
| 3825 | /* Overflow possible, we know nothing */ |
| 3826 | dst_reg->smin_value = S64_MIN; |
| 3827 | dst_reg->smax_value = S64_MAX; |
| 3828 | } else { |
| 3829 | dst_reg->smin_value = dst_reg->umin_value; |
| 3830 | dst_reg->smax_value = dst_reg->umax_value; |
| 3831 | } |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 3832 | break; |
| 3833 | case BPF_AND: |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3834 | if (src_known && dst_known) { |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3835 | __mark_reg_known(dst_reg, dst_reg->var_off.value & |
| 3836 | src_reg.var_off.value); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3837 | break; |
| 3838 | } |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3839 | /* We get our minimum from the var_off, since that's inherently |
| 3840 | * bitwise. Our maximum is the minimum of the operands' maxima. |
Josef Bacik | f23cc64 | 2016-11-14 15:45:36 -0500 | [diff] [blame] | 3841 | */ |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3842 | dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3843 | dst_reg->umin_value = dst_reg->var_off.value; |
| 3844 | dst_reg->umax_value = min(dst_reg->umax_value, umax_val); |
| 3845 | if (dst_reg->smin_value < 0 || smin_val < 0) { |
| 3846 | /* Lose signed bounds when ANDing negative numbers, |
| 3847 | * ain't nobody got time for that. |
| 3848 | */ |
| 3849 | dst_reg->smin_value = S64_MIN; |
| 3850 | dst_reg->smax_value = S64_MAX; |
| 3851 | } else { |
| 3852 | /* ANDing two positives gives a positive, so safe to |
| 3853 | * cast result into s64. |
| 3854 | */ |
| 3855 | dst_reg->smin_value = dst_reg->umin_value; |
| 3856 | dst_reg->smax_value = dst_reg->umax_value; |
| 3857 | } |
| 3858 | /* We may learn something more from the var_off */ |
| 3859 | __update_reg_bounds(dst_reg); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3860 | break; |
| 3861 | case BPF_OR: |
| 3862 | if (src_known && dst_known) { |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3863 | __mark_reg_known(dst_reg, dst_reg->var_off.value | |
| 3864 | src_reg.var_off.value); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3865 | break; |
| 3866 | } |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3867 | /* We get our maximum from the var_off, and our minimum is the |
| 3868 | * maximum of the operands' minima |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3869 | */ |
| 3870 | dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3871 | dst_reg->umin_value = max(dst_reg->umin_value, umin_val); |
| 3872 | dst_reg->umax_value = dst_reg->var_off.value | |
| 3873 | dst_reg->var_off.mask; |
| 3874 | if (dst_reg->smin_value < 0 || smin_val < 0) { |
| 3875 | /* Lose signed bounds when ORing negative numbers, |
| 3876 | * ain't nobody got time for that. |
| 3877 | */ |
| 3878 | dst_reg->smin_value = S64_MIN; |
| 3879 | dst_reg->smax_value = S64_MAX; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3880 | } else { |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3881 | /* ORing two positives gives a positive, so safe to |
| 3882 | * cast result into s64. |
| 3883 | */ |
| 3884 | dst_reg->smin_value = dst_reg->umin_value; |
| 3885 | dst_reg->smax_value = dst_reg->umax_value; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3886 | } |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3887 | /* We may learn something more from the var_off */ |
| 3888 | __update_reg_bounds(dst_reg); |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 3889 | break; |
| 3890 | case BPF_LSH: |
Jann Horn | 468f6ea | 2017-12-18 20:11:56 -0800 | [diff] [blame] | 3891 | if (umax_val >= insn_bitness) { |
| 3892 | /* Shifts greater than 31 or 63 are undefined. |
| 3893 | * This includes shifts by a negative number. |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3894 | */ |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 3895 | mark_reg_unknown(env, regs, insn->dst_reg); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3896 | break; |
| 3897 | } |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3898 | /* We lose all sign bit information (except what we can pick |
| 3899 | * up from var_off) |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 3900 | */ |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3901 | dst_reg->smin_value = S64_MIN; |
| 3902 | dst_reg->smax_value = S64_MAX; |
| 3903 | /* If we might shift our top bit out, then we know nothing */ |
| 3904 | if (dst_reg->umax_value > 1ULL << (63 - umax_val)) { |
| 3905 | dst_reg->umin_value = 0; |
| 3906 | dst_reg->umax_value = U64_MAX; |
David S. Miller | d117441 | 2017-05-10 11:22:52 -0700 | [diff] [blame] | 3907 | } else { |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3908 | dst_reg->umin_value <<= umin_val; |
| 3909 | dst_reg->umax_value <<= umax_val; |
David S. Miller | d117441 | 2017-05-10 11:22:52 -0700 | [diff] [blame] | 3910 | } |
Yonghong Song | afbe1a5 | 2018-04-28 22:28:10 -0700 | [diff] [blame] | 3911 | dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val); |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3912 | /* We may learn something more from the var_off */ |
| 3913 | __update_reg_bounds(dst_reg); |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 3914 | break; |
| 3915 | case BPF_RSH: |
Jann Horn | 468f6ea | 2017-12-18 20:11:56 -0800 | [diff] [blame] | 3916 | if (umax_val >= insn_bitness) { |
| 3917 | /* Shifts greater than 31 or 63 are undefined. |
| 3918 | * This includes shifts by a negative number. |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3919 | */ |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 3920 | mark_reg_unknown(env, regs, insn->dst_reg); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3921 | break; |
| 3922 | } |
Edward Cree | 4374f25 | 2017-12-18 20:11:53 -0800 | [diff] [blame] | 3923 | /* BPF_RSH is an unsigned shift. If the value in dst_reg might |
| 3924 | * be negative, then either: |
| 3925 | * 1) src_reg might be zero, so the sign bit of the result is |
| 3926 | * unknown, so we lose our signed bounds |
| 3927 | * 2) it's known negative, thus the unsigned bounds capture the |
| 3928 | * signed bounds |
| 3929 | * 3) the signed bounds cross zero, so they tell us nothing |
| 3930 | * about the result |
| 3931 | * If the value in dst_reg is known nonnegative, then again the |
| 3932 | * unsigned bounts capture the signed bounds. |
| 3933 | * Thus, in all cases it suffices to blow away our signed bounds |
| 3934 | * and rely on inferring new ones from the unsigned bounds and |
| 3935 | * var_off of the result. |
| 3936 | */ |
| 3937 | dst_reg->smin_value = S64_MIN; |
| 3938 | dst_reg->smax_value = S64_MAX; |
Yonghong Song | afbe1a5 | 2018-04-28 22:28:10 -0700 | [diff] [blame] | 3939 | dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val); |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3940 | dst_reg->umin_value >>= umax_val; |
| 3941 | dst_reg->umax_value >>= umin_val; |
| 3942 | /* We may learn something more from the var_off */ |
| 3943 | __update_reg_bounds(dst_reg); |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 3944 | break; |
Yonghong Song | 9cbe1f5a | 2018-04-28 22:28:11 -0700 | [diff] [blame] | 3945 | case BPF_ARSH: |
| 3946 | if (umax_val >= insn_bitness) { |
| 3947 | /* Shifts greater than 31 or 63 are undefined. |
| 3948 | * This includes shifts by a negative number. |
| 3949 | */ |
| 3950 | mark_reg_unknown(env, regs, insn->dst_reg); |
| 3951 | break; |
| 3952 | } |
| 3953 | |
| 3954 | /* Upon reaching here, src_known is true and |
| 3955 | * umax_val is equal to umin_val. |
| 3956 | */ |
| 3957 | dst_reg->smin_value >>= umin_val; |
| 3958 | dst_reg->smax_value >>= umin_val; |
| 3959 | dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val); |
| 3960 | |
| 3961 | /* blow away the dst_reg umin_value/umax_value and rely on |
| 3962 | * dst_reg var_off to refine the result. |
| 3963 | */ |
| 3964 | dst_reg->umin_value = 0; |
| 3965 | dst_reg->umax_value = U64_MAX; |
| 3966 | __update_reg_bounds(dst_reg); |
| 3967 | break; |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 3968 | default: |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 3969 | mark_reg_unknown(env, regs, insn->dst_reg); |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 3970 | break; |
| 3971 | } |
| 3972 | |
Jann Horn | 468f6ea | 2017-12-18 20:11:56 -0800 | [diff] [blame] | 3973 | if (BPF_CLASS(insn->code) != BPF_ALU64) { |
| 3974 | /* 32-bit ALU ops are (32,32)->32 */ |
| 3975 | coerce_reg_to_size(dst_reg, 4); |
Jann Horn | 468f6ea | 2017-12-18 20:11:56 -0800 | [diff] [blame] | 3976 | } |
| 3977 | |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3978 | __reg_deduce_bounds(dst_reg); |
| 3979 | __reg_bound_offset(dst_reg); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3980 | return 0; |
| 3981 | } |
| 3982 | |
| 3983 | /* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max |
| 3984 | * and var_off. |
| 3985 | */ |
| 3986 | static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, |
| 3987 | struct bpf_insn *insn) |
| 3988 | { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 3989 | struct bpf_verifier_state *vstate = env->cur_state; |
| 3990 | struct bpf_func_state *state = vstate->frame[vstate->curframe]; |
| 3991 | struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3992 | struct bpf_reg_state *ptr_reg = NULL, off_reg = {0}; |
| 3993 | u8 opcode = BPF_OP(insn->code); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3994 | |
| 3995 | dst_reg = ®s[insn->dst_reg]; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3996 | src_reg = NULL; |
| 3997 | if (dst_reg->type != SCALAR_VALUE) |
| 3998 | ptr_reg = dst_reg; |
| 3999 | if (BPF_SRC(insn->code) == BPF_X) { |
| 4000 | src_reg = ®s[insn->src_reg]; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4001 | if (src_reg->type != SCALAR_VALUE) { |
| 4002 | if (dst_reg->type != SCALAR_VALUE) { |
| 4003 | /* Combining two pointers by any ALU op yields |
Alexei Starovoitov | 82abbf8 | 2017-12-18 20:15:20 -0800 | [diff] [blame] | 4004 | * an arbitrary scalar. Disallow all math except |
| 4005 | * pointer subtraction |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4006 | */ |
Alexei Starovoitov | dd06682 | 2018-09-12 14:06:10 -0700 | [diff] [blame] | 4007 | if (opcode == BPF_SUB && env->allow_ptr_leaks) { |
Alexei Starovoitov | 82abbf8 | 2017-12-18 20:15:20 -0800 | [diff] [blame] | 4008 | mark_reg_unknown(env, regs, insn->dst_reg); |
| 4009 | return 0; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4010 | } |
Alexei Starovoitov | 82abbf8 | 2017-12-18 20:15:20 -0800 | [diff] [blame] | 4011 | verbose(env, "R%d pointer %s pointer prohibited\n", |
| 4012 | insn->dst_reg, |
| 4013 | bpf_alu_string[opcode >> 4]); |
| 4014 | return -EACCES; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4015 | } else { |
| 4016 | /* scalar += pointer |
| 4017 | * This is legal, but we have to reverse our |
| 4018 | * src/dest handling in computing the range |
| 4019 | */ |
Alexei Starovoitov | 82abbf8 | 2017-12-18 20:15:20 -0800 | [diff] [blame] | 4020 | return adjust_ptr_min_max_vals(env, insn, |
| 4021 | src_reg, dst_reg); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4022 | } |
| 4023 | } else if (ptr_reg) { |
| 4024 | /* pointer += scalar */ |
Alexei Starovoitov | 82abbf8 | 2017-12-18 20:15:20 -0800 | [diff] [blame] | 4025 | return adjust_ptr_min_max_vals(env, insn, |
| 4026 | dst_reg, src_reg); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4027 | } |
| 4028 | } else { |
| 4029 | /* Pretend the src is a reg with a known value, since we only |
| 4030 | * need to be able to read from this state. |
| 4031 | */ |
| 4032 | off_reg.type = SCALAR_VALUE; |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 4033 | __mark_reg_known(&off_reg, insn->imm); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4034 | src_reg = &off_reg; |
Alexei Starovoitov | 82abbf8 | 2017-12-18 20:15:20 -0800 | [diff] [blame] | 4035 | if (ptr_reg) /* pointer += K */ |
| 4036 | return adjust_ptr_min_max_vals(env, insn, |
| 4037 | ptr_reg, src_reg); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4038 | } |
| 4039 | |
| 4040 | /* Got here implies adding two SCALAR_VALUEs */ |
| 4041 | if (WARN_ON_ONCE(ptr_reg)) { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 4042 | print_verifier_state(env, state); |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 4043 | verbose(env, "verifier internal error: unexpected ptr_reg\n"); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4044 | return -EINVAL; |
| 4045 | } |
| 4046 | if (WARN_ON(!src_reg)) { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 4047 | print_verifier_state(env, state); |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 4048 | verbose(env, "verifier internal error: no src_reg\n"); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4049 | return -EINVAL; |
| 4050 | } |
| 4051 | return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 4052 | } |
| 4053 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4054 | /* check validity of 32-bit and 64-bit arithmetic operations */ |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 4055 | static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4056 | { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 4057 | struct bpf_reg_state *regs = cur_regs(env); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4058 | u8 opcode = BPF_OP(insn->code); |
| 4059 | int err; |
| 4060 | |
| 4061 | if (opcode == BPF_END || opcode == BPF_NEG) { |
| 4062 | if (opcode == BPF_NEG) { |
| 4063 | if (BPF_SRC(insn->code) != 0 || |
| 4064 | insn->src_reg != BPF_REG_0 || |
| 4065 | insn->off != 0 || insn->imm != 0) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 4066 | verbose(env, "BPF_NEG uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4067 | return -EINVAL; |
| 4068 | } |
| 4069 | } else { |
| 4070 | if (insn->src_reg != BPF_REG_0 || insn->off != 0 || |
Edward Cree | e67b8a6 | 2017-09-15 14:37:38 +0100 | [diff] [blame] | 4071 | (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) || |
| 4072 | BPF_CLASS(insn->code) == BPF_ALU64) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 4073 | verbose(env, "BPF_END uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4074 | return -EINVAL; |
| 4075 | } |
| 4076 | } |
| 4077 | |
| 4078 | /* check src operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 4079 | err = check_reg_arg(env, insn->dst_reg, SRC_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4080 | if (err) |
| 4081 | return err; |
| 4082 | |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 4083 | if (is_pointer_value(env, insn->dst_reg)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 4084 | verbose(env, "R%d pointer arithmetic prohibited\n", |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 4085 | insn->dst_reg); |
| 4086 | return -EACCES; |
| 4087 | } |
| 4088 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4089 | /* check dest operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 4090 | err = check_reg_arg(env, insn->dst_reg, DST_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4091 | if (err) |
| 4092 | return err; |
| 4093 | |
| 4094 | } else if (opcode == BPF_MOV) { |
| 4095 | |
| 4096 | if (BPF_SRC(insn->code) == BPF_X) { |
| 4097 | if (insn->imm != 0 || insn->off != 0) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 4098 | verbose(env, "BPF_MOV uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4099 | return -EINVAL; |
| 4100 | } |
| 4101 | |
| 4102 | /* check src operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 4103 | err = check_reg_arg(env, insn->src_reg, SRC_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4104 | if (err) |
| 4105 | return err; |
| 4106 | } else { |
| 4107 | if (insn->src_reg != BPF_REG_0 || insn->off != 0) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 4108 | verbose(env, "BPF_MOV uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4109 | return -EINVAL; |
| 4110 | } |
| 4111 | } |
| 4112 | |
Arthur Fabre | fbeb160 | 2018-07-31 18:17:22 +0100 | [diff] [blame] | 4113 | /* check dest operand, mark as required later */ |
| 4114 | err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4115 | if (err) |
| 4116 | return err; |
| 4117 | |
| 4118 | if (BPF_SRC(insn->code) == BPF_X) { |
Jiong Wang | e434b8c | 2018-12-07 12:16:18 -0500 | [diff] [blame] | 4119 | struct bpf_reg_state *src_reg = regs + insn->src_reg; |
| 4120 | struct bpf_reg_state *dst_reg = regs + insn->dst_reg; |
| 4121 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4122 | if (BPF_CLASS(insn->code) == BPF_ALU64) { |
| 4123 | /* case: R1 = R2 |
| 4124 | * copy register state to dest reg |
| 4125 | */ |
Jiong Wang | e434b8c | 2018-12-07 12:16:18 -0500 | [diff] [blame] | 4126 | *dst_reg = *src_reg; |
| 4127 | dst_reg->live |= REG_LIVE_WRITTEN; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4128 | } else { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4129 | /* R1 = (u32) R2 */ |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 4130 | if (is_pointer_value(env, insn->src_reg)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 4131 | verbose(env, |
| 4132 | "R%d partial copy of pointer\n", |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 4133 | insn->src_reg); |
| 4134 | return -EACCES; |
Jiong Wang | e434b8c | 2018-12-07 12:16:18 -0500 | [diff] [blame] | 4135 | } else if (src_reg->type == SCALAR_VALUE) { |
| 4136 | *dst_reg = *src_reg; |
| 4137 | dst_reg->live |= REG_LIVE_WRITTEN; |
| 4138 | } else { |
| 4139 | mark_reg_unknown(env, regs, |
| 4140 | insn->dst_reg); |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 4141 | } |
Jiong Wang | e434b8c | 2018-12-07 12:16:18 -0500 | [diff] [blame] | 4142 | coerce_reg_to_size(dst_reg, 4); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4143 | } |
| 4144 | } else { |
| 4145 | /* case: R = imm |
| 4146 | * remember the value we stored into this reg |
| 4147 | */ |
Arthur Fabre | fbeb160 | 2018-07-31 18:17:22 +0100 | [diff] [blame] | 4148 | /* clear any state __mark_reg_known doesn't set */ |
| 4149 | mark_reg_unknown(env, regs, insn->dst_reg); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4150 | regs[insn->dst_reg].type = SCALAR_VALUE; |
Jann Horn | 95a762e | 2017-12-18 20:11:54 -0800 | [diff] [blame] | 4151 | if (BPF_CLASS(insn->code) == BPF_ALU64) { |
| 4152 | __mark_reg_known(regs + insn->dst_reg, |
| 4153 | insn->imm); |
| 4154 | } else { |
| 4155 | __mark_reg_known(regs + insn->dst_reg, |
| 4156 | (u32)insn->imm); |
| 4157 | } |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4158 | } |
| 4159 | |
| 4160 | } else if (opcode > BPF_END) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 4161 | verbose(env, "invalid BPF_ALU opcode %x\n", opcode); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4162 | return -EINVAL; |
| 4163 | |
| 4164 | } else { /* all other ALU ops: and, sub, xor, add, ... */ |
| 4165 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4166 | if (BPF_SRC(insn->code) == BPF_X) { |
| 4167 | if (insn->imm != 0 || insn->off != 0) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 4168 | verbose(env, "BPF_ALU uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4169 | return -EINVAL; |
| 4170 | } |
| 4171 | /* check src1 operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 4172 | err = check_reg_arg(env, insn->src_reg, SRC_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4173 | if (err) |
| 4174 | return err; |
| 4175 | } else { |
| 4176 | if (insn->src_reg != BPF_REG_0 || insn->off != 0) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 4177 | verbose(env, "BPF_ALU uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4178 | return -EINVAL; |
| 4179 | } |
| 4180 | } |
| 4181 | |
| 4182 | /* check src2 operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 4183 | err = check_reg_arg(env, insn->dst_reg, SRC_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4184 | if (err) |
| 4185 | return err; |
| 4186 | |
| 4187 | if ((opcode == BPF_MOD || opcode == BPF_DIV) && |
| 4188 | BPF_SRC(insn->code) == BPF_K && insn->imm == 0) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 4189 | verbose(env, "div by zero\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4190 | return -EINVAL; |
| 4191 | } |
| 4192 | |
Rabin Vincent | 229394e8 | 2016-01-12 20:17:08 +0100 | [diff] [blame] | 4193 | if ((opcode == BPF_LSH || opcode == BPF_RSH || |
| 4194 | opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) { |
| 4195 | int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32; |
| 4196 | |
| 4197 | if (insn->imm < 0 || insn->imm >= size) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 4198 | verbose(env, "invalid shift %d\n", insn->imm); |
Rabin Vincent | 229394e8 | 2016-01-12 20:17:08 +0100 | [diff] [blame] | 4199 | return -EINVAL; |
| 4200 | } |
| 4201 | } |
| 4202 | |
Alexei Starovoitov | 1a0dc1a | 2016-05-05 19:49:09 -0700 | [diff] [blame] | 4203 | /* check dest operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 4204 | err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); |
Alexei Starovoitov | 1a0dc1a | 2016-05-05 19:49:09 -0700 | [diff] [blame] | 4205 | if (err) |
| 4206 | return err; |
| 4207 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4208 | return adjust_reg_min_max_vals(env, insn); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4209 | } |
| 4210 | |
| 4211 | return 0; |
| 4212 | } |
| 4213 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 4214 | static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 4215 | struct bpf_reg_state *dst_reg, |
David S. Miller | f8ddadc | 2017-10-22 13:36:53 +0100 | [diff] [blame] | 4216 | enum bpf_reg_type type, |
Daniel Borkmann | fb2a311 | 2017-10-21 02:34:21 +0200 | [diff] [blame] | 4217 | bool range_right_open) |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 4218 | { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 4219 | struct bpf_func_state *state = vstate->frame[vstate->curframe]; |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 4220 | struct bpf_reg_state *regs = state->regs, *reg; |
Daniel Borkmann | fb2a311 | 2017-10-21 02:34:21 +0200 | [diff] [blame] | 4221 | u16 new_range; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 4222 | int i, j; |
Daniel Borkmann | 2d2be8c | 2016-09-08 01:03:42 +0200 | [diff] [blame] | 4223 | |
Daniel Borkmann | fb2a311 | 2017-10-21 02:34:21 +0200 | [diff] [blame] | 4224 | if (dst_reg->off < 0 || |
| 4225 | (dst_reg->off == 0 && range_right_open)) |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4226 | /* This doesn't give us any range */ |
| 4227 | return; |
| 4228 | |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 4229 | if (dst_reg->umax_value > MAX_PACKET_OFF || |
| 4230 | dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF) |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4231 | /* Risk of overflow. For instance, ptr + (1<<63) may be less |
| 4232 | * than pkt_end, but that's because it's also less than pkt. |
| 4233 | */ |
| 4234 | return; |
| 4235 | |
Daniel Borkmann | fb2a311 | 2017-10-21 02:34:21 +0200 | [diff] [blame] | 4236 | new_range = dst_reg->off; |
| 4237 | if (range_right_open) |
| 4238 | new_range--; |
| 4239 | |
| 4240 | /* Examples for register markings: |
Daniel Borkmann | 2d2be8c | 2016-09-08 01:03:42 +0200 | [diff] [blame] | 4241 | * |
Daniel Borkmann | fb2a311 | 2017-10-21 02:34:21 +0200 | [diff] [blame] | 4242 | * pkt_data in dst register: |
Daniel Borkmann | 2d2be8c | 2016-09-08 01:03:42 +0200 | [diff] [blame] | 4243 | * |
| 4244 | * r2 = r3; |
| 4245 | * r2 += 8; |
| 4246 | * if (r2 > pkt_end) goto <handle exception> |
| 4247 | * <access okay> |
| 4248 | * |
Daniel Borkmann | b4e432f | 2017-08-10 01:40:02 +0200 | [diff] [blame] | 4249 | * r2 = r3; |
| 4250 | * r2 += 8; |
| 4251 | * if (r2 < pkt_end) goto <access okay> |
| 4252 | * <handle exception> |
| 4253 | * |
Daniel Borkmann | 2d2be8c | 2016-09-08 01:03:42 +0200 | [diff] [blame] | 4254 | * Where: |
| 4255 | * r2 == dst_reg, pkt_end == src_reg |
| 4256 | * r2=pkt(id=n,off=8,r=0) |
| 4257 | * r3=pkt(id=n,off=0,r=0) |
| 4258 | * |
Daniel Borkmann | fb2a311 | 2017-10-21 02:34:21 +0200 | [diff] [blame] | 4259 | * pkt_data in src register: |
Daniel Borkmann | 2d2be8c | 2016-09-08 01:03:42 +0200 | [diff] [blame] | 4260 | * |
| 4261 | * r2 = r3; |
| 4262 | * r2 += 8; |
| 4263 | * if (pkt_end >= r2) goto <access okay> |
| 4264 | * <handle exception> |
| 4265 | * |
Daniel Borkmann | b4e432f | 2017-08-10 01:40:02 +0200 | [diff] [blame] | 4266 | * r2 = r3; |
| 4267 | * r2 += 8; |
| 4268 | * if (pkt_end <= r2) goto <handle exception> |
| 4269 | * <access okay> |
| 4270 | * |
Daniel Borkmann | 2d2be8c | 2016-09-08 01:03:42 +0200 | [diff] [blame] | 4271 | * Where: |
| 4272 | * pkt_end == dst_reg, r2 == src_reg |
| 4273 | * r2=pkt(id=n,off=8,r=0) |
| 4274 | * r3=pkt(id=n,off=0,r=0) |
| 4275 | * |
| 4276 | * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8) |
Daniel Borkmann | fb2a311 | 2017-10-21 02:34:21 +0200 | [diff] [blame] | 4277 | * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8) |
| 4278 | * and [r3, r3 + 8-1) respectively is safe to access depending on |
| 4279 | * the check. |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 4280 | */ |
Daniel Borkmann | 2d2be8c | 2016-09-08 01:03:42 +0200 | [diff] [blame] | 4281 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4282 | /* If our ids match, then we must have the same max_value. And we |
| 4283 | * don't care about the other reg's fixed offset, since if it's too big |
| 4284 | * the range won't allow anything. |
| 4285 | * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16. |
| 4286 | */ |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 4287 | for (i = 0; i < MAX_BPF_REG; i++) |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 4288 | if (regs[i].type == type && regs[i].id == dst_reg->id) |
Alexei Starovoitov | b197768 | 2017-03-24 15:57:33 -0700 | [diff] [blame] | 4289 | /* keep the maximum range already checked */ |
Daniel Borkmann | fb2a311 | 2017-10-21 02:34:21 +0200 | [diff] [blame] | 4290 | regs[i].range = max(regs[i].range, new_range); |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 4291 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 4292 | for (j = 0; j <= vstate->curframe; j++) { |
| 4293 | state = vstate->frame[j]; |
Joe Stringer | f3709f6 | 2018-10-02 13:35:29 -0700 | [diff] [blame] | 4294 | bpf_for_each_spilled_reg(i, state, reg) { |
| 4295 | if (!reg) |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 4296 | continue; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 4297 | if (reg->type == type && reg->id == dst_reg->id) |
| 4298 | reg->range = max(reg->range, new_range); |
| 4299 | } |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 4300 | } |
| 4301 | } |
| 4302 | |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 4303 | /* compute branch direction of the expression "if (reg opcode val) goto target;" |
| 4304 | * and return: |
| 4305 | * 1 - branch will be taken and "goto target" will be executed |
| 4306 | * 0 - branch will not be taken and fall-through to next insn |
| 4307 | * -1 - unknown. Example: "if (reg < 5)" is unknown when register value range [0,10] |
| 4308 | */ |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4309 | static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode, |
| 4310 | bool is_jmp32) |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 4311 | { |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4312 | struct bpf_reg_state reg_lo; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4313 | s64 sval; |
| 4314 | |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 4315 | if (__is_pointer_value(false, reg)) |
| 4316 | return -1; |
| 4317 | |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4318 | if (is_jmp32) { |
| 4319 | reg_lo = *reg; |
| 4320 | reg = ®_lo; |
| 4321 | /* For JMP32, only low 32 bits are compared, coerce_reg_to_size |
| 4322 | * could truncate high bits and update umin/umax according to |
| 4323 | * information of low bits. |
| 4324 | */ |
| 4325 | coerce_reg_to_size(reg, 4); |
| 4326 | /* smin/smax need special handling. For example, after coerce, |
| 4327 | * if smin_value is 0x00000000ffffffffLL, the value is -1 when |
| 4328 | * used as operand to JMP32. It is a negative number from s32's |
| 4329 | * point of view, while it is a positive number when seen as |
| 4330 | * s64. The smin/smax are kept as s64, therefore, when used with |
| 4331 | * JMP32, they need to be transformed into s32, then sign |
| 4332 | * extended back to s64. |
| 4333 | * |
| 4334 | * Also, smin/smax were copied from umin/umax. If umin/umax has |
| 4335 | * different sign bit, then min/max relationship doesn't |
| 4336 | * maintain after casting into s32, for this case, set smin/smax |
| 4337 | * to safest range. |
| 4338 | */ |
| 4339 | if ((reg->umax_value ^ reg->umin_value) & |
| 4340 | (1ULL << 31)) { |
| 4341 | reg->smin_value = S32_MIN; |
| 4342 | reg->smax_value = S32_MAX; |
| 4343 | } |
| 4344 | reg->smin_value = (s64)(s32)reg->smin_value; |
| 4345 | reg->smax_value = (s64)(s32)reg->smax_value; |
| 4346 | |
| 4347 | val = (u32)val; |
| 4348 | sval = (s64)(s32)val; |
| 4349 | } else { |
| 4350 | sval = (s64)val; |
| 4351 | } |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4352 | |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 4353 | switch (opcode) { |
| 4354 | case BPF_JEQ: |
| 4355 | if (tnum_is_const(reg->var_off)) |
| 4356 | return !!tnum_equals_const(reg->var_off, val); |
| 4357 | break; |
| 4358 | case BPF_JNE: |
| 4359 | if (tnum_is_const(reg->var_off)) |
| 4360 | return !tnum_equals_const(reg->var_off, val); |
| 4361 | break; |
Jakub Kicinski | 960ea05 | 2018-12-19 22:13:04 -0800 | [diff] [blame] | 4362 | case BPF_JSET: |
| 4363 | if ((~reg->var_off.mask & reg->var_off.value) & val) |
| 4364 | return 1; |
| 4365 | if (!((reg->var_off.mask | reg->var_off.value) & val)) |
| 4366 | return 0; |
| 4367 | break; |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 4368 | case BPF_JGT: |
| 4369 | if (reg->umin_value > val) |
| 4370 | return 1; |
| 4371 | else if (reg->umax_value <= val) |
| 4372 | return 0; |
| 4373 | break; |
| 4374 | case BPF_JSGT: |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4375 | if (reg->smin_value > sval) |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 4376 | return 1; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4377 | else if (reg->smax_value < sval) |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 4378 | return 0; |
| 4379 | break; |
| 4380 | case BPF_JLT: |
| 4381 | if (reg->umax_value < val) |
| 4382 | return 1; |
| 4383 | else if (reg->umin_value >= val) |
| 4384 | return 0; |
| 4385 | break; |
| 4386 | case BPF_JSLT: |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4387 | if (reg->smax_value < sval) |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 4388 | return 1; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4389 | else if (reg->smin_value >= sval) |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 4390 | return 0; |
| 4391 | break; |
| 4392 | case BPF_JGE: |
| 4393 | if (reg->umin_value >= val) |
| 4394 | return 1; |
| 4395 | else if (reg->umax_value < val) |
| 4396 | return 0; |
| 4397 | break; |
| 4398 | case BPF_JSGE: |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4399 | if (reg->smin_value >= sval) |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 4400 | return 1; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4401 | else if (reg->smax_value < sval) |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 4402 | return 0; |
| 4403 | break; |
| 4404 | case BPF_JLE: |
| 4405 | if (reg->umax_value <= val) |
| 4406 | return 1; |
| 4407 | else if (reg->umin_value > val) |
| 4408 | return 0; |
| 4409 | break; |
| 4410 | case BPF_JSLE: |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4411 | if (reg->smax_value <= sval) |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 4412 | return 1; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4413 | else if (reg->smin_value > sval) |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 4414 | return 0; |
| 4415 | break; |
| 4416 | } |
| 4417 | |
| 4418 | return -1; |
| 4419 | } |
| 4420 | |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4421 | /* Generate min value of the high 32-bit from TNUM info. */ |
| 4422 | static u64 gen_hi_min(struct tnum var) |
| 4423 | { |
| 4424 | return var.value & ~0xffffffffULL; |
| 4425 | } |
| 4426 | |
| 4427 | /* Generate max value of the high 32-bit from TNUM info. */ |
| 4428 | static u64 gen_hi_max(struct tnum var) |
| 4429 | { |
| 4430 | return (var.value | var.mask) & ~0xffffffffULL; |
| 4431 | } |
| 4432 | |
| 4433 | /* Return true if VAL is compared with a s64 sign extended from s32, and they |
| 4434 | * are with the same signedness. |
| 4435 | */ |
| 4436 | static bool cmp_val_with_extended_s64(s64 sval, struct bpf_reg_state *reg) |
| 4437 | { |
| 4438 | return ((s32)sval >= 0 && |
| 4439 | reg->smin_value >= 0 && reg->smax_value <= S32_MAX) || |
| 4440 | ((s32)sval < 0 && |
| 4441 | reg->smax_value <= 0 && reg->smin_value >= S32_MIN); |
| 4442 | } |
| 4443 | |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 4444 | /* Adjusts the register min/max values in the case that the dst_reg is the |
| 4445 | * variable register that we are working on, and src_reg is a constant or we're |
| 4446 | * simply doing a BPF_K check. |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4447 | * In JEQ/JNE cases we also adjust the var_off values. |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 4448 | */ |
| 4449 | static void reg_set_min_max(struct bpf_reg_state *true_reg, |
| 4450 | struct bpf_reg_state *false_reg, u64 val, |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4451 | u8 opcode, bool is_jmp32) |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 4452 | { |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4453 | s64 sval; |
| 4454 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4455 | /* If the dst_reg is a pointer, we can't learn anything about its |
| 4456 | * variable offset from the compare (unless src_reg were a pointer into |
| 4457 | * the same object, but we don't bother with that. |
| 4458 | * Since false_reg and true_reg have the same type by construction, we |
| 4459 | * only need to check one of them for pointerness. |
| 4460 | */ |
| 4461 | if (__is_pointer_value(false, false_reg)) |
| 4462 | return; |
Daniel Borkmann | 4cabc5b | 2017-07-21 00:00:21 +0200 | [diff] [blame] | 4463 | |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4464 | val = is_jmp32 ? (u32)val : val; |
| 4465 | sval = is_jmp32 ? (s64)(s32)val : (s64)val; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4466 | |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 4467 | switch (opcode) { |
| 4468 | case BPF_JEQ: |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 4469 | case BPF_JNE: |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4470 | { |
| 4471 | struct bpf_reg_state *reg = |
| 4472 | opcode == BPF_JEQ ? true_reg : false_reg; |
| 4473 | |
| 4474 | /* For BPF_JEQ, if this is false we know nothing Jon Snow, but |
| 4475 | * if it is true we know the value for sure. Likewise for |
| 4476 | * BPF_JNE. |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 4477 | */ |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4478 | if (is_jmp32) { |
| 4479 | u64 old_v = reg->var_off.value; |
| 4480 | u64 hi_mask = ~0xffffffffULL; |
| 4481 | |
| 4482 | reg->var_off.value = (old_v & hi_mask) | val; |
| 4483 | reg->var_off.mask &= hi_mask; |
| 4484 | } else { |
| 4485 | __mark_reg_known(reg, val); |
| 4486 | } |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 4487 | break; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4488 | } |
Jakub Kicinski | 960ea05 | 2018-12-19 22:13:04 -0800 | [diff] [blame] | 4489 | case BPF_JSET: |
| 4490 | false_reg->var_off = tnum_and(false_reg->var_off, |
| 4491 | tnum_const(~val)); |
| 4492 | if (is_power_of_2(val)) |
| 4493 | true_reg->var_off = tnum_or(true_reg->var_off, |
| 4494 | tnum_const(val)); |
| 4495 | break; |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 4496 | case BPF_JGE: |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4497 | case BPF_JGT: |
| 4498 | { |
| 4499 | u64 false_umax = opcode == BPF_JGT ? val : val - 1; |
| 4500 | u64 true_umin = opcode == BPF_JGT ? val + 1 : val; |
| 4501 | |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4502 | if (is_jmp32) { |
| 4503 | false_umax += gen_hi_max(false_reg->var_off); |
| 4504 | true_umin += gen_hi_min(true_reg->var_off); |
| 4505 | } |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4506 | false_reg->umax_value = min(false_reg->umax_value, false_umax); |
| 4507 | true_reg->umin_value = max(true_reg->umin_value, true_umin); |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 4508 | break; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4509 | } |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 4510 | case BPF_JSGE: |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4511 | case BPF_JSGT: |
| 4512 | { |
| 4513 | s64 false_smax = opcode == BPF_JSGT ? sval : sval - 1; |
| 4514 | s64 true_smin = opcode == BPF_JSGT ? sval + 1 : sval; |
| 4515 | |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4516 | /* If the full s64 was not sign-extended from s32 then don't |
| 4517 | * deduct further info. |
| 4518 | */ |
| 4519 | if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg)) |
| 4520 | break; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4521 | false_reg->smax_value = min(false_reg->smax_value, false_smax); |
| 4522 | true_reg->smin_value = max(true_reg->smin_value, true_smin); |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 4523 | break; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4524 | } |
Daniel Borkmann | b4e432f | 2017-08-10 01:40:02 +0200 | [diff] [blame] | 4525 | case BPF_JLE: |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4526 | case BPF_JLT: |
| 4527 | { |
| 4528 | u64 false_umin = opcode == BPF_JLT ? val : val + 1; |
| 4529 | u64 true_umax = opcode == BPF_JLT ? val - 1 : val; |
| 4530 | |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4531 | if (is_jmp32) { |
| 4532 | false_umin += gen_hi_min(false_reg->var_off); |
| 4533 | true_umax += gen_hi_max(true_reg->var_off); |
| 4534 | } |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4535 | false_reg->umin_value = max(false_reg->umin_value, false_umin); |
| 4536 | true_reg->umax_value = min(true_reg->umax_value, true_umax); |
Daniel Borkmann | b4e432f | 2017-08-10 01:40:02 +0200 | [diff] [blame] | 4537 | break; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4538 | } |
Daniel Borkmann | b4e432f | 2017-08-10 01:40:02 +0200 | [diff] [blame] | 4539 | case BPF_JSLE: |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4540 | case BPF_JSLT: |
| 4541 | { |
| 4542 | s64 false_smin = opcode == BPF_JSLT ? sval : sval + 1; |
| 4543 | s64 true_smax = opcode == BPF_JSLT ? sval - 1 : sval; |
| 4544 | |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4545 | if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg)) |
| 4546 | break; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4547 | false_reg->smin_value = max(false_reg->smin_value, false_smin); |
| 4548 | true_reg->smax_value = min(true_reg->smax_value, true_smax); |
Daniel Borkmann | b4e432f | 2017-08-10 01:40:02 +0200 | [diff] [blame] | 4549 | break; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4550 | } |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 4551 | default: |
| 4552 | break; |
| 4553 | } |
| 4554 | |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 4555 | __reg_deduce_bounds(false_reg); |
| 4556 | __reg_deduce_bounds(true_reg); |
| 4557 | /* We might have learned some bits from the bounds. */ |
| 4558 | __reg_bound_offset(false_reg); |
| 4559 | __reg_bound_offset(true_reg); |
| 4560 | /* Intersecting with the old var_off might have improved our bounds |
| 4561 | * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), |
| 4562 | * then new var_off is (0; 0x7f...fc) which improves our umax. |
| 4563 | */ |
| 4564 | __update_reg_bounds(false_reg); |
| 4565 | __update_reg_bounds(true_reg); |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 4566 | } |
| 4567 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4568 | /* Same as above, but for the case that dst_reg holds a constant and src_reg is |
| 4569 | * the variable reg. |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 4570 | */ |
| 4571 | static void reg_set_min_max_inv(struct bpf_reg_state *true_reg, |
| 4572 | struct bpf_reg_state *false_reg, u64 val, |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4573 | u8 opcode, bool is_jmp32) |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 4574 | { |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4575 | s64 sval; |
| 4576 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4577 | if (__is_pointer_value(false, false_reg)) |
| 4578 | return; |
Daniel Borkmann | 4cabc5b | 2017-07-21 00:00:21 +0200 | [diff] [blame] | 4579 | |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4580 | val = is_jmp32 ? (u32)val : val; |
| 4581 | sval = is_jmp32 ? (s64)(s32)val : (s64)val; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4582 | |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 4583 | switch (opcode) { |
| 4584 | case BPF_JEQ: |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 4585 | case BPF_JNE: |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4586 | { |
| 4587 | struct bpf_reg_state *reg = |
| 4588 | opcode == BPF_JEQ ? true_reg : false_reg; |
| 4589 | |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4590 | if (is_jmp32) { |
| 4591 | u64 old_v = reg->var_off.value; |
| 4592 | u64 hi_mask = ~0xffffffffULL; |
| 4593 | |
| 4594 | reg->var_off.value = (old_v & hi_mask) | val; |
| 4595 | reg->var_off.mask &= hi_mask; |
| 4596 | } else { |
| 4597 | __mark_reg_known(reg, val); |
| 4598 | } |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 4599 | break; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4600 | } |
Jakub Kicinski | 960ea05 | 2018-12-19 22:13:04 -0800 | [diff] [blame] | 4601 | case BPF_JSET: |
| 4602 | false_reg->var_off = tnum_and(false_reg->var_off, |
| 4603 | tnum_const(~val)); |
| 4604 | if (is_power_of_2(val)) |
| 4605 | true_reg->var_off = tnum_or(true_reg->var_off, |
| 4606 | tnum_const(val)); |
| 4607 | break; |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 4608 | case BPF_JGE: |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4609 | case BPF_JGT: |
| 4610 | { |
| 4611 | u64 false_umin = opcode == BPF_JGT ? val : val + 1; |
| 4612 | u64 true_umax = opcode == BPF_JGT ? val - 1 : val; |
| 4613 | |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4614 | if (is_jmp32) { |
| 4615 | false_umin += gen_hi_min(false_reg->var_off); |
| 4616 | true_umax += gen_hi_max(true_reg->var_off); |
| 4617 | } |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4618 | false_reg->umin_value = max(false_reg->umin_value, false_umin); |
| 4619 | true_reg->umax_value = min(true_reg->umax_value, true_umax); |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 4620 | break; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4621 | } |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 4622 | case BPF_JSGE: |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4623 | case BPF_JSGT: |
| 4624 | { |
| 4625 | s64 false_smin = opcode == BPF_JSGT ? sval : sval + 1; |
| 4626 | s64 true_smax = opcode == BPF_JSGT ? sval - 1 : sval; |
| 4627 | |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4628 | if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg)) |
| 4629 | break; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4630 | false_reg->smin_value = max(false_reg->smin_value, false_smin); |
| 4631 | true_reg->smax_value = min(true_reg->smax_value, true_smax); |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 4632 | break; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4633 | } |
Daniel Borkmann | b4e432f | 2017-08-10 01:40:02 +0200 | [diff] [blame] | 4634 | case BPF_JLE: |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4635 | case BPF_JLT: |
| 4636 | { |
| 4637 | u64 false_umax = opcode == BPF_JLT ? val : val - 1; |
| 4638 | u64 true_umin = opcode == BPF_JLT ? val + 1 : val; |
| 4639 | |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4640 | if (is_jmp32) { |
| 4641 | false_umax += gen_hi_max(false_reg->var_off); |
| 4642 | true_umin += gen_hi_min(true_reg->var_off); |
| 4643 | } |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4644 | false_reg->umax_value = min(false_reg->umax_value, false_umax); |
| 4645 | true_reg->umin_value = max(true_reg->umin_value, true_umin); |
Daniel Borkmann | b4e432f | 2017-08-10 01:40:02 +0200 | [diff] [blame] | 4646 | break; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4647 | } |
Daniel Borkmann | b4e432f | 2017-08-10 01:40:02 +0200 | [diff] [blame] | 4648 | case BPF_JSLE: |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4649 | case BPF_JSLT: |
| 4650 | { |
| 4651 | s64 false_smax = opcode == BPF_JSLT ? sval : sval - 1; |
| 4652 | s64 true_smin = opcode == BPF_JSLT ? sval + 1 : sval; |
| 4653 | |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4654 | if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg)) |
| 4655 | break; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4656 | false_reg->smax_value = min(false_reg->smax_value, false_smax); |
| 4657 | true_reg->smin_value = max(true_reg->smin_value, true_smin); |
Daniel Borkmann | b4e432f | 2017-08-10 01:40:02 +0200 | [diff] [blame] | 4658 | break; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 4659 | } |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 4660 | default: |
| 4661 | break; |
| 4662 | } |
| 4663 | |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 4664 | __reg_deduce_bounds(false_reg); |
| 4665 | __reg_deduce_bounds(true_reg); |
| 4666 | /* We might have learned some bits from the bounds. */ |
| 4667 | __reg_bound_offset(false_reg); |
| 4668 | __reg_bound_offset(true_reg); |
| 4669 | /* Intersecting with the old var_off might have improved our bounds |
| 4670 | * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), |
| 4671 | * then new var_off is (0; 0x7f...fc) which improves our umax. |
| 4672 | */ |
| 4673 | __update_reg_bounds(false_reg); |
| 4674 | __update_reg_bounds(true_reg); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4675 | } |
| 4676 | |
| 4677 | /* Regs are known to be equal, so intersect their min/max/var_off */ |
| 4678 | static void __reg_combine_min_max(struct bpf_reg_state *src_reg, |
| 4679 | struct bpf_reg_state *dst_reg) |
| 4680 | { |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 4681 | src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value, |
| 4682 | dst_reg->umin_value); |
| 4683 | src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value, |
| 4684 | dst_reg->umax_value); |
| 4685 | src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value, |
| 4686 | dst_reg->smin_value); |
| 4687 | src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value, |
| 4688 | dst_reg->smax_value); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4689 | src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off, |
| 4690 | dst_reg->var_off); |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 4691 | /* We might have learned new bounds from the var_off. */ |
| 4692 | __update_reg_bounds(src_reg); |
| 4693 | __update_reg_bounds(dst_reg); |
| 4694 | /* We might have learned something about the sign bit. */ |
| 4695 | __reg_deduce_bounds(src_reg); |
| 4696 | __reg_deduce_bounds(dst_reg); |
| 4697 | /* We might have learned some bits from the bounds. */ |
| 4698 | __reg_bound_offset(src_reg); |
| 4699 | __reg_bound_offset(dst_reg); |
| 4700 | /* Intersecting with the old var_off might have improved our bounds |
| 4701 | * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), |
| 4702 | * then new var_off is (0; 0x7f...fc) which improves our umax. |
| 4703 | */ |
| 4704 | __update_reg_bounds(src_reg); |
| 4705 | __update_reg_bounds(dst_reg); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4706 | } |
| 4707 | |
| 4708 | static void reg_combine_min_max(struct bpf_reg_state *true_src, |
| 4709 | struct bpf_reg_state *true_dst, |
| 4710 | struct bpf_reg_state *false_src, |
| 4711 | struct bpf_reg_state *false_dst, |
| 4712 | u8 opcode) |
| 4713 | { |
| 4714 | switch (opcode) { |
| 4715 | case BPF_JEQ: |
| 4716 | __reg_combine_min_max(true_src, true_dst); |
| 4717 | break; |
| 4718 | case BPF_JNE: |
| 4719 | __reg_combine_min_max(false_src, false_dst); |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 4720 | break; |
Daniel Borkmann | 4cabc5b | 2017-07-21 00:00:21 +0200 | [diff] [blame] | 4721 | } |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 4722 | } |
| 4723 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 4724 | static void mark_ptr_or_null_reg(struct bpf_func_state *state, |
| 4725 | struct bpf_reg_state *reg, u32 id, |
Joe Stringer | 840b961 | 2018-10-02 13:35:32 -0700 | [diff] [blame] | 4726 | bool is_null) |
Thomas Graf | 57a09bf | 2016-10-18 19:51:19 +0200 | [diff] [blame] | 4727 | { |
Joe Stringer | 840b961 | 2018-10-02 13:35:32 -0700 | [diff] [blame] | 4728 | if (reg_type_may_be_null(reg->type) && reg->id == id) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4729 | /* Old offset (both fixed and variable parts) should |
| 4730 | * have been known-zero, because we don't allow pointer |
| 4731 | * arithmetic on pointers that might be NULL. |
| 4732 | */ |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 4733 | if (WARN_ON_ONCE(reg->smin_value || reg->smax_value || |
| 4734 | !tnum_equals_const(reg->var_off, 0) || |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4735 | reg->off)) { |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 4736 | __mark_reg_known_zero(reg); |
| 4737 | reg->off = 0; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4738 | } |
| 4739 | if (is_null) { |
| 4740 | reg->type = SCALAR_VALUE; |
Joe Stringer | 840b961 | 2018-10-02 13:35:32 -0700 | [diff] [blame] | 4741 | } else if (reg->type == PTR_TO_MAP_VALUE_OR_NULL) { |
| 4742 | if (reg->map_ptr->inner_map_meta) { |
| 4743 | reg->type = CONST_PTR_TO_MAP; |
| 4744 | reg->map_ptr = reg->map_ptr->inner_map_meta; |
| 4745 | } else { |
| 4746 | reg->type = PTR_TO_MAP_VALUE; |
| 4747 | } |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 4748 | } else if (reg->type == PTR_TO_SOCKET_OR_NULL) { |
| 4749 | reg->type = PTR_TO_SOCKET; |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 4750 | } else if (reg->type == PTR_TO_SOCK_COMMON_OR_NULL) { |
| 4751 | reg->type = PTR_TO_SOCK_COMMON; |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 4752 | } else if (reg->type == PTR_TO_TCP_SOCK_OR_NULL) { |
| 4753 | reg->type = PTR_TO_TCP_SOCK; |
Martin KaFai Lau | 56f668d | 2017-03-22 10:00:33 -0700 | [diff] [blame] | 4754 | } |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 4755 | if (is_null) { |
| 4756 | /* We don't need id and ref_obj_id from this point |
| 4757 | * onwards anymore, thus we should better reset it, |
| 4758 | * so that state pruning has chances to take effect. |
| 4759 | */ |
| 4760 | reg->id = 0; |
| 4761 | reg->ref_obj_id = 0; |
| 4762 | } else if (!reg_may_point_to_spin_lock(reg)) { |
| 4763 | /* For not-NULL ptr, reg->ref_obj_id will be reset |
| 4764 | * in release_reg_references(). |
| 4765 | * |
| 4766 | * reg->id is still used by spin_lock ptr. Other |
| 4767 | * than spin_lock ptr type, reg->id can be reset. |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 4768 | */ |
| 4769 | reg->id = 0; |
| 4770 | } |
Thomas Graf | 57a09bf | 2016-10-18 19:51:19 +0200 | [diff] [blame] | 4771 | } |
| 4772 | } |
| 4773 | |
| 4774 | /* The logic is similar to find_good_pkt_pointers(), both could eventually |
| 4775 | * be folded together at some point. |
| 4776 | */ |
Joe Stringer | 840b961 | 2018-10-02 13:35:32 -0700 | [diff] [blame] | 4777 | static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno, |
| 4778 | bool is_null) |
Thomas Graf | 57a09bf | 2016-10-18 19:51:19 +0200 | [diff] [blame] | 4779 | { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 4780 | struct bpf_func_state *state = vstate->frame[vstate->curframe]; |
Joe Stringer | f3709f6 | 2018-10-02 13:35:29 -0700 | [diff] [blame] | 4781 | struct bpf_reg_state *reg, *regs = state->regs; |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 4782 | u32 ref_obj_id = regs[regno].ref_obj_id; |
Daniel Borkmann | a08dd0d | 2016-12-15 01:30:06 +0100 | [diff] [blame] | 4783 | u32 id = regs[regno].id; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 4784 | int i, j; |
Thomas Graf | 57a09bf | 2016-10-18 19:51:19 +0200 | [diff] [blame] | 4785 | |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 4786 | if (ref_obj_id && ref_obj_id == id && is_null) |
| 4787 | /* regs[regno] is in the " == NULL" branch. |
| 4788 | * No one could have freed the reference state before |
| 4789 | * doing the NULL check. |
| 4790 | */ |
| 4791 | WARN_ON_ONCE(release_reference_state(state, id)); |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 4792 | |
Thomas Graf | 57a09bf | 2016-10-18 19:51:19 +0200 | [diff] [blame] | 4793 | for (i = 0; i < MAX_BPF_REG; i++) |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 4794 | mark_ptr_or_null_reg(state, ®s[i], id, is_null); |
Thomas Graf | 57a09bf | 2016-10-18 19:51:19 +0200 | [diff] [blame] | 4795 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 4796 | for (j = 0; j <= vstate->curframe; j++) { |
| 4797 | state = vstate->frame[j]; |
Joe Stringer | f3709f6 | 2018-10-02 13:35:29 -0700 | [diff] [blame] | 4798 | bpf_for_each_spilled_reg(i, state, reg) { |
| 4799 | if (!reg) |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 4800 | continue; |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 4801 | mark_ptr_or_null_reg(state, reg, id, is_null); |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 4802 | } |
Thomas Graf | 57a09bf | 2016-10-18 19:51:19 +0200 | [diff] [blame] | 4803 | } |
| 4804 | } |
| 4805 | |
Daniel Borkmann | 5beca08 | 2017-11-01 23:58:10 +0100 | [diff] [blame] | 4806 | static bool try_match_pkt_pointers(const struct bpf_insn *insn, |
| 4807 | struct bpf_reg_state *dst_reg, |
| 4808 | struct bpf_reg_state *src_reg, |
| 4809 | struct bpf_verifier_state *this_branch, |
| 4810 | struct bpf_verifier_state *other_branch) |
| 4811 | { |
| 4812 | if (BPF_SRC(insn->code) != BPF_X) |
| 4813 | return false; |
| 4814 | |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4815 | /* Pointers are always 64-bit. */ |
| 4816 | if (BPF_CLASS(insn->code) == BPF_JMP32) |
| 4817 | return false; |
| 4818 | |
Daniel Borkmann | 5beca08 | 2017-11-01 23:58:10 +0100 | [diff] [blame] | 4819 | switch (BPF_OP(insn->code)) { |
| 4820 | case BPF_JGT: |
| 4821 | if ((dst_reg->type == PTR_TO_PACKET && |
| 4822 | src_reg->type == PTR_TO_PACKET_END) || |
| 4823 | (dst_reg->type == PTR_TO_PACKET_META && |
| 4824 | reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { |
| 4825 | /* pkt_data' > pkt_end, pkt_meta' > pkt_data */ |
| 4826 | find_good_pkt_pointers(this_branch, dst_reg, |
| 4827 | dst_reg->type, false); |
| 4828 | } else if ((dst_reg->type == PTR_TO_PACKET_END && |
| 4829 | src_reg->type == PTR_TO_PACKET) || |
| 4830 | (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && |
| 4831 | src_reg->type == PTR_TO_PACKET_META)) { |
| 4832 | /* pkt_end > pkt_data', pkt_data > pkt_meta' */ |
| 4833 | find_good_pkt_pointers(other_branch, src_reg, |
| 4834 | src_reg->type, true); |
| 4835 | } else { |
| 4836 | return false; |
| 4837 | } |
| 4838 | break; |
| 4839 | case BPF_JLT: |
| 4840 | if ((dst_reg->type == PTR_TO_PACKET && |
| 4841 | src_reg->type == PTR_TO_PACKET_END) || |
| 4842 | (dst_reg->type == PTR_TO_PACKET_META && |
| 4843 | reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { |
| 4844 | /* pkt_data' < pkt_end, pkt_meta' < pkt_data */ |
| 4845 | find_good_pkt_pointers(other_branch, dst_reg, |
| 4846 | dst_reg->type, true); |
| 4847 | } else if ((dst_reg->type == PTR_TO_PACKET_END && |
| 4848 | src_reg->type == PTR_TO_PACKET) || |
| 4849 | (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && |
| 4850 | src_reg->type == PTR_TO_PACKET_META)) { |
| 4851 | /* pkt_end < pkt_data', pkt_data > pkt_meta' */ |
| 4852 | find_good_pkt_pointers(this_branch, src_reg, |
| 4853 | src_reg->type, false); |
| 4854 | } else { |
| 4855 | return false; |
| 4856 | } |
| 4857 | break; |
| 4858 | case BPF_JGE: |
| 4859 | if ((dst_reg->type == PTR_TO_PACKET && |
| 4860 | src_reg->type == PTR_TO_PACKET_END) || |
| 4861 | (dst_reg->type == PTR_TO_PACKET_META && |
| 4862 | reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { |
| 4863 | /* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */ |
| 4864 | find_good_pkt_pointers(this_branch, dst_reg, |
| 4865 | dst_reg->type, true); |
| 4866 | } else if ((dst_reg->type == PTR_TO_PACKET_END && |
| 4867 | src_reg->type == PTR_TO_PACKET) || |
| 4868 | (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && |
| 4869 | src_reg->type == PTR_TO_PACKET_META)) { |
| 4870 | /* pkt_end >= pkt_data', pkt_data >= pkt_meta' */ |
| 4871 | find_good_pkt_pointers(other_branch, src_reg, |
| 4872 | src_reg->type, false); |
| 4873 | } else { |
| 4874 | return false; |
| 4875 | } |
| 4876 | break; |
| 4877 | case BPF_JLE: |
| 4878 | if ((dst_reg->type == PTR_TO_PACKET && |
| 4879 | src_reg->type == PTR_TO_PACKET_END) || |
| 4880 | (dst_reg->type == PTR_TO_PACKET_META && |
| 4881 | reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { |
| 4882 | /* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */ |
| 4883 | find_good_pkt_pointers(other_branch, dst_reg, |
| 4884 | dst_reg->type, false); |
| 4885 | } else if ((dst_reg->type == PTR_TO_PACKET_END && |
| 4886 | src_reg->type == PTR_TO_PACKET) || |
| 4887 | (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && |
| 4888 | src_reg->type == PTR_TO_PACKET_META)) { |
| 4889 | /* pkt_end <= pkt_data', pkt_data <= pkt_meta' */ |
| 4890 | find_good_pkt_pointers(this_branch, src_reg, |
| 4891 | src_reg->type, true); |
| 4892 | } else { |
| 4893 | return false; |
| 4894 | } |
| 4895 | break; |
| 4896 | default: |
| 4897 | return false; |
| 4898 | } |
| 4899 | |
| 4900 | return true; |
| 4901 | } |
| 4902 | |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 4903 | static int check_cond_jmp_op(struct bpf_verifier_env *env, |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4904 | struct bpf_insn *insn, int *insn_idx) |
| 4905 | { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 4906 | struct bpf_verifier_state *this_branch = env->cur_state; |
| 4907 | struct bpf_verifier_state *other_branch; |
| 4908 | struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs; |
| 4909 | struct bpf_reg_state *dst_reg, *other_branch_regs; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4910 | u8 opcode = BPF_OP(insn->code); |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4911 | bool is_jmp32; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4912 | int err; |
| 4913 | |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4914 | /* Only conditional jumps are expected to reach here. */ |
| 4915 | if (opcode == BPF_JA || opcode > BPF_JSLE) { |
| 4916 | verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4917 | return -EINVAL; |
| 4918 | } |
| 4919 | |
| 4920 | if (BPF_SRC(insn->code) == BPF_X) { |
| 4921 | if (insn->imm != 0) { |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4922 | verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4923 | return -EINVAL; |
| 4924 | } |
| 4925 | |
| 4926 | /* check src1 operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 4927 | err = check_reg_arg(env, insn->src_reg, SRC_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4928 | if (err) |
| 4929 | return err; |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 4930 | |
| 4931 | if (is_pointer_value(env, insn->src_reg)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 4932 | verbose(env, "R%d pointer comparison prohibited\n", |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 4933 | insn->src_reg); |
| 4934 | return -EACCES; |
| 4935 | } |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4936 | } else { |
| 4937 | if (insn->src_reg != BPF_REG_0) { |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4938 | verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4939 | return -EINVAL; |
| 4940 | } |
| 4941 | } |
| 4942 | |
| 4943 | /* check src2 operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 4944 | err = check_reg_arg(env, insn->dst_reg, SRC_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4945 | if (err) |
| 4946 | return err; |
| 4947 | |
Alexei Starovoitov | 1a0dc1a | 2016-05-05 19:49:09 -0700 | [diff] [blame] | 4948 | dst_reg = ®s[insn->dst_reg]; |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4949 | is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; |
Alexei Starovoitov | 1a0dc1a | 2016-05-05 19:49:09 -0700 | [diff] [blame] | 4950 | |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 4951 | if (BPF_SRC(insn->code) == BPF_K) { |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4952 | int pred = is_branch_taken(dst_reg, insn->imm, opcode, |
| 4953 | is_jmp32); |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 4954 | |
| 4955 | if (pred == 1) { |
| 4956 | /* only follow the goto, ignore fall-through */ |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4957 | *insn_idx += insn->off; |
| 4958 | return 0; |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 4959 | } else if (pred == 0) { |
| 4960 | /* only follow fall-through branch, since |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4961 | * that's where the program will go |
| 4962 | */ |
| 4963 | return 0; |
| 4964 | } |
| 4965 | } |
| 4966 | |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 4967 | other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, |
| 4968 | false); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4969 | if (!other_branch) |
| 4970 | return -EFAULT; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 4971 | other_branch_regs = other_branch->frame[other_branch->curframe]->regs; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4972 | |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 4973 | /* detect if we are comparing against a constant value so we can adjust |
| 4974 | * our min/max values for our dst register. |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4975 | * this is only legit if both are scalars (or pointers to the same |
| 4976 | * object, I suppose, but we don't support that right now), because |
| 4977 | * otherwise the different base pointers mean the offsets aren't |
| 4978 | * comparable. |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 4979 | */ |
| 4980 | if (BPF_SRC(insn->code) == BPF_X) { |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4981 | struct bpf_reg_state *src_reg = ®s[insn->src_reg]; |
| 4982 | struct bpf_reg_state lo_reg0 = *dst_reg; |
| 4983 | struct bpf_reg_state lo_reg1 = *src_reg; |
| 4984 | struct bpf_reg_state *src_lo, *dst_lo; |
| 4985 | |
| 4986 | dst_lo = &lo_reg0; |
| 4987 | src_lo = &lo_reg1; |
| 4988 | coerce_reg_to_size(dst_lo, 4); |
| 4989 | coerce_reg_to_size(src_lo, 4); |
| 4990 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4991 | if (dst_reg->type == SCALAR_VALUE && |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4992 | src_reg->type == SCALAR_VALUE) { |
| 4993 | if (tnum_is_const(src_reg->var_off) || |
| 4994 | (is_jmp32 && tnum_is_const(src_lo->var_off))) |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 4995 | reg_set_min_max(&other_branch_regs[insn->dst_reg], |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 4996 | dst_reg, |
| 4997 | is_jmp32 |
| 4998 | ? src_lo->var_off.value |
| 4999 | : src_reg->var_off.value, |
| 5000 | opcode, is_jmp32); |
| 5001 | else if (tnum_is_const(dst_reg->var_off) || |
| 5002 | (is_jmp32 && tnum_is_const(dst_lo->var_off))) |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5003 | reg_set_min_max_inv(&other_branch_regs[insn->src_reg], |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 5004 | src_reg, |
| 5005 | is_jmp32 |
| 5006 | ? dst_lo->var_off.value |
| 5007 | : dst_reg->var_off.value, |
| 5008 | opcode, is_jmp32); |
| 5009 | else if (!is_jmp32 && |
| 5010 | (opcode == BPF_JEQ || opcode == BPF_JNE)) |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 5011 | /* Comparing for equality, we can combine knowledge */ |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5012 | reg_combine_min_max(&other_branch_regs[insn->src_reg], |
| 5013 | &other_branch_regs[insn->dst_reg], |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 5014 | src_reg, dst_reg, opcode); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 5015 | } |
| 5016 | } else if (dst_reg->type == SCALAR_VALUE) { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5017 | reg_set_min_max(&other_branch_regs[insn->dst_reg], |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 5018 | dst_reg, insn->imm, opcode, is_jmp32); |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 5019 | } |
| 5020 | |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 5021 | /* detect if R == 0 where R is returned from bpf_map_lookup_elem(). |
| 5022 | * NOTE: these optimizations below are related with pointer comparison |
| 5023 | * which will never be JMP32. |
| 5024 | */ |
| 5025 | if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K && |
Alexei Starovoitov | 1a0dc1a | 2016-05-05 19:49:09 -0700 | [diff] [blame] | 5026 | insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) && |
Joe Stringer | 840b961 | 2018-10-02 13:35:32 -0700 | [diff] [blame] | 5027 | reg_type_may_be_null(dst_reg->type)) { |
| 5028 | /* Mark all identical registers in each branch as either |
Thomas Graf | 57a09bf | 2016-10-18 19:51:19 +0200 | [diff] [blame] | 5029 | * safe or unknown depending R == 0 or R != 0 conditional. |
| 5030 | */ |
Joe Stringer | 840b961 | 2018-10-02 13:35:32 -0700 | [diff] [blame] | 5031 | mark_ptr_or_null_regs(this_branch, insn->dst_reg, |
| 5032 | opcode == BPF_JNE); |
| 5033 | mark_ptr_or_null_regs(other_branch, insn->dst_reg, |
| 5034 | opcode == BPF_JEQ); |
Daniel Borkmann | 5beca08 | 2017-11-01 23:58:10 +0100 | [diff] [blame] | 5035 | } else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], |
| 5036 | this_branch, other_branch) && |
| 5037 | is_pointer_value(env, insn->dst_reg)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5038 | verbose(env, "R%d pointer comparison prohibited\n", |
| 5039 | insn->dst_reg); |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 5040 | return -EACCES; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 5041 | } |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 5042 | if (env->log.level & BPF_LOG_LEVEL) |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5043 | print_verifier_state(env, this_branch->frame[this_branch->curframe]); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 5044 | return 0; |
| 5045 | } |
| 5046 | |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 5047 | /* return the map pointer stored inside BPF_LD_IMM64 instruction */ |
| 5048 | static struct bpf_map *ld_imm64_to_map_ptr(struct bpf_insn *insn) |
| 5049 | { |
| 5050 | u64 imm64 = ((u64) (u32) insn[0].imm) | ((u64) (u32) insn[1].imm) << 32; |
| 5051 | |
| 5052 | return (struct bpf_map *) (unsigned long) imm64; |
| 5053 | } |
| 5054 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 5055 | /* verify BPF_LD_IMM64 instruction */ |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 5056 | static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 5057 | { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 5058 | struct bpf_reg_state *regs = cur_regs(env); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 5059 | int err; |
| 5060 | |
| 5061 | if (BPF_SIZE(insn->code) != BPF_DW) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5062 | verbose(env, "invalid BPF_LD_IMM insn\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 5063 | return -EINVAL; |
| 5064 | } |
| 5065 | if (insn->off != 0) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5066 | verbose(env, "BPF_LD_IMM64 uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 5067 | return -EINVAL; |
| 5068 | } |
| 5069 | |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 5070 | err = check_reg_arg(env, insn->dst_reg, DST_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 5071 | if (err) |
| 5072 | return err; |
| 5073 | |
Jakub Kicinski | 6b17387 | 2016-09-21 11:43:59 +0100 | [diff] [blame] | 5074 | if (insn->src_reg == 0) { |
Jakub Kicinski | 6b17387 | 2016-09-21 11:43:59 +0100 | [diff] [blame] | 5075 | u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; |
| 5076 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 5077 | regs[insn->dst_reg].type = SCALAR_VALUE; |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 5078 | __mark_reg_known(®s[insn->dst_reg], imm); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 5079 | return 0; |
Jakub Kicinski | 6b17387 | 2016-09-21 11:43:59 +0100 | [diff] [blame] | 5080 | } |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 5081 | |
| 5082 | /* replace_map_fd_with_map_ptr() should have caught bad ld_imm64 */ |
| 5083 | BUG_ON(insn->src_reg != BPF_PSEUDO_MAP_FD); |
| 5084 | |
| 5085 | regs[insn->dst_reg].type = CONST_PTR_TO_MAP; |
| 5086 | regs[insn->dst_reg].map_ptr = ld_imm64_to_map_ptr(insn); |
| 5087 | return 0; |
| 5088 | } |
| 5089 | |
Daniel Borkmann | 96be432 | 2015-03-01 12:31:46 +0100 | [diff] [blame] | 5090 | static bool may_access_skb(enum bpf_prog_type type) |
| 5091 | { |
| 5092 | switch (type) { |
| 5093 | case BPF_PROG_TYPE_SOCKET_FILTER: |
| 5094 | case BPF_PROG_TYPE_SCHED_CLS: |
Daniel Borkmann | 94caee8c | 2015-03-20 15:11:11 +0100 | [diff] [blame] | 5095 | case BPF_PROG_TYPE_SCHED_ACT: |
Daniel Borkmann | 96be432 | 2015-03-01 12:31:46 +0100 | [diff] [blame] | 5096 | return true; |
| 5097 | default: |
| 5098 | return false; |
| 5099 | } |
| 5100 | } |
| 5101 | |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 5102 | /* verify safety of LD_ABS|LD_IND instructions: |
| 5103 | * - they can only appear in the programs where ctx == skb |
| 5104 | * - since they are wrappers of function calls, they scratch R1-R5 registers, |
| 5105 | * preserve R6-R9, and store return value into R0 |
| 5106 | * |
| 5107 | * Implicit input: |
| 5108 | * ctx == skb == R6 == CTX |
| 5109 | * |
| 5110 | * Explicit input: |
| 5111 | * SRC == any register |
| 5112 | * IMM == 32-bit immediate |
| 5113 | * |
| 5114 | * Output: |
| 5115 | * R0 - 8/16/32-bit skb data converted to cpu endianness |
| 5116 | */ |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 5117 | static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 5118 | { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 5119 | struct bpf_reg_state *regs = cur_regs(env); |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 5120 | u8 mode = BPF_MODE(insn->code); |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 5121 | int i, err; |
| 5122 | |
Daniel Borkmann | 24701ec | 2015-03-01 12:31:47 +0100 | [diff] [blame] | 5123 | if (!may_access_skb(env->prog->type)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5124 | verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n"); |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 5125 | return -EINVAL; |
| 5126 | } |
| 5127 | |
Daniel Borkmann | e0cea7c | 2018-05-04 01:08:14 +0200 | [diff] [blame] | 5128 | if (!env->ops->gen_ld_abs) { |
| 5129 | verbose(env, "bpf verifier is misconfigured\n"); |
| 5130 | return -EINVAL; |
| 5131 | } |
| 5132 | |
Jiong Wang | f910cef | 2018-05-02 16:17:17 -0400 | [diff] [blame] | 5133 | if (env->subprog_cnt > 1) { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5134 | /* when program has LD_ABS insn JITs and interpreter assume |
| 5135 | * that r1 == ctx == skb which is not the case for callees |
| 5136 | * that can have arbitrary arguments. It's problematic |
| 5137 | * for main prog as well since JITs would need to analyze |
| 5138 | * all functions in order to make proper register save/restore |
| 5139 | * decisions in the main prog. Hence disallow LD_ABS with calls |
| 5140 | */ |
| 5141 | verbose(env, "BPF_LD_[ABS|IND] instructions cannot be mixed with bpf-to-bpf calls\n"); |
| 5142 | return -EINVAL; |
| 5143 | } |
| 5144 | |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 5145 | if (insn->dst_reg != BPF_REG_0 || insn->off != 0 || |
Alexei Starovoitov | d82bccc | 2016-04-12 10:26:19 -0700 | [diff] [blame] | 5146 | BPF_SIZE(insn->code) == BPF_DW || |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 5147 | (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5148 | verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n"); |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 5149 | return -EINVAL; |
| 5150 | } |
| 5151 | |
| 5152 | /* check whether implicit source operand (register R6) is readable */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 5153 | err = check_reg_arg(env, BPF_REG_6, SRC_OP); |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 5154 | if (err) |
| 5155 | return err; |
| 5156 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 5157 | /* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as |
| 5158 | * gen_ld_abs() may terminate the program at runtime, leading to |
| 5159 | * reference leak. |
| 5160 | */ |
| 5161 | err = check_reference_leak(env); |
| 5162 | if (err) { |
| 5163 | verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n"); |
| 5164 | return err; |
| 5165 | } |
| 5166 | |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 5167 | if (env->cur_state->active_spin_lock) { |
| 5168 | verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n"); |
| 5169 | return -EINVAL; |
| 5170 | } |
| 5171 | |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 5172 | if (regs[BPF_REG_6].type != PTR_TO_CTX) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5173 | verbose(env, |
| 5174 | "at the time of BPF_LD_ABS|IND R6 != pointer to skb\n"); |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 5175 | return -EINVAL; |
| 5176 | } |
| 5177 | |
| 5178 | if (mode == BPF_IND) { |
| 5179 | /* check explicit source operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 5180 | err = check_reg_arg(env, insn->src_reg, SRC_OP); |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 5181 | if (err) |
| 5182 | return err; |
| 5183 | } |
| 5184 | |
| 5185 | /* reset caller saved regs to unreadable */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 5186 | for (i = 0; i < CALLER_SAVED_REGS; i++) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5187 | mark_reg_not_init(env, regs, caller_saved[i]); |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 5188 | check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); |
| 5189 | } |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 5190 | |
| 5191 | /* mark destination R0 register as readable, since it contains |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 5192 | * the value fetched from the packet. |
| 5193 | * Already marked as written above. |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 5194 | */ |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5195 | mark_reg_unknown(env, regs, BPF_REG_0); |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 5196 | return 0; |
| 5197 | } |
| 5198 | |
Alexei Starovoitov | 390ee7e | 2017-10-02 22:50:23 -0700 | [diff] [blame] | 5199 | static int check_return_code(struct bpf_verifier_env *env) |
| 5200 | { |
| 5201 | struct bpf_reg_state *reg; |
| 5202 | struct tnum range = tnum_range(0, 1); |
| 5203 | |
| 5204 | switch (env->prog->type) { |
| 5205 | case BPF_PROG_TYPE_CGROUP_SKB: |
| 5206 | case BPF_PROG_TYPE_CGROUP_SOCK: |
Andrey Ignatov | 4fbac77 | 2018-03-30 15:08:02 -0700 | [diff] [blame] | 5207 | case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: |
Alexei Starovoitov | 390ee7e | 2017-10-02 22:50:23 -0700 | [diff] [blame] | 5208 | case BPF_PROG_TYPE_SOCK_OPS: |
Roman Gushchin | ebc614f | 2017-11-05 08:15:32 -0500 | [diff] [blame] | 5209 | case BPF_PROG_TYPE_CGROUP_DEVICE: |
Alexei Starovoitov | 390ee7e | 2017-10-02 22:50:23 -0700 | [diff] [blame] | 5210 | break; |
| 5211 | default: |
| 5212 | return 0; |
| 5213 | } |
| 5214 | |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 5215 | reg = cur_regs(env) + BPF_REG_0; |
Alexei Starovoitov | 390ee7e | 2017-10-02 22:50:23 -0700 | [diff] [blame] | 5216 | if (reg->type != SCALAR_VALUE) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5217 | verbose(env, "At program exit the register R0 is not a known value (%s)\n", |
Alexei Starovoitov | 390ee7e | 2017-10-02 22:50:23 -0700 | [diff] [blame] | 5218 | reg_type_str[reg->type]); |
| 5219 | return -EINVAL; |
| 5220 | } |
| 5221 | |
| 5222 | if (!tnum_in(range, reg->var_off)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5223 | verbose(env, "At program exit the register R0 "); |
Alexei Starovoitov | 390ee7e | 2017-10-02 22:50:23 -0700 | [diff] [blame] | 5224 | if (!tnum_is_unknown(reg->var_off)) { |
| 5225 | char tn_buf[48]; |
| 5226 | |
| 5227 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5228 | verbose(env, "has value %s", tn_buf); |
Alexei Starovoitov | 390ee7e | 2017-10-02 22:50:23 -0700 | [diff] [blame] | 5229 | } else { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5230 | verbose(env, "has unknown scalar value"); |
Alexei Starovoitov | 390ee7e | 2017-10-02 22:50:23 -0700 | [diff] [blame] | 5231 | } |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5232 | verbose(env, " should have been 0 or 1\n"); |
Alexei Starovoitov | 390ee7e | 2017-10-02 22:50:23 -0700 | [diff] [blame] | 5233 | return -EINVAL; |
| 5234 | } |
| 5235 | return 0; |
| 5236 | } |
| 5237 | |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 5238 | /* non-recursive DFS pseudo code |
| 5239 | * 1 procedure DFS-iterative(G,v): |
| 5240 | * 2 label v as discovered |
| 5241 | * 3 let S be a stack |
| 5242 | * 4 S.push(v) |
| 5243 | * 5 while S is not empty |
| 5244 | * 6 t <- S.pop() |
| 5245 | * 7 if t is what we're looking for: |
| 5246 | * 8 return t |
| 5247 | * 9 for all edges e in G.adjacentEdges(t) do |
| 5248 | * 10 if edge e is already labelled |
| 5249 | * 11 continue with the next edge |
| 5250 | * 12 w <- G.adjacentVertex(t,e) |
| 5251 | * 13 if vertex w is not discovered and not explored |
| 5252 | * 14 label e as tree-edge |
| 5253 | * 15 label w as discovered |
| 5254 | * 16 S.push(w) |
| 5255 | * 17 continue at 5 |
| 5256 | * 18 else if vertex w is discovered |
| 5257 | * 19 label e as back-edge |
| 5258 | * 20 else |
| 5259 | * 21 // vertex w is explored |
| 5260 | * 22 label e as forward- or cross-edge |
| 5261 | * 23 label t as explored |
| 5262 | * 24 S.pop() |
| 5263 | * |
| 5264 | * convention: |
| 5265 | * 0x10 - discovered |
| 5266 | * 0x11 - discovered and fall-through edge labelled |
| 5267 | * 0x12 - discovered and fall-through and branch edges labelled |
| 5268 | * 0x20 - explored |
| 5269 | */ |
| 5270 | |
| 5271 | enum { |
| 5272 | DISCOVERED = 0x10, |
| 5273 | EXPLORED = 0x20, |
| 5274 | FALLTHROUGH = 1, |
| 5275 | BRANCH = 2, |
| 5276 | }; |
| 5277 | |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 5278 | #define STATE_LIST_MARK ((struct bpf_verifier_state_list *) -1L) |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 5279 | |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 5280 | static int *insn_stack; /* stack of insns to process */ |
| 5281 | static int cur_stack; /* current stack index */ |
| 5282 | static int *insn_state; |
| 5283 | |
| 5284 | /* t, w, e - match pseudo-code above: |
| 5285 | * t - index of current instruction |
| 5286 | * w - next instruction |
| 5287 | * e - edge |
| 5288 | */ |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 5289 | static int push_insn(int t, int w, int e, struct bpf_verifier_env *env) |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 5290 | { |
| 5291 | if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH)) |
| 5292 | return 0; |
| 5293 | |
| 5294 | if (e == BRANCH && insn_state[t] >= (DISCOVERED | BRANCH)) |
| 5295 | return 0; |
| 5296 | |
| 5297 | if (w < 0 || w >= env->prog->len) { |
Martin KaFai Lau | d9762e8 | 2018-12-13 10:41:48 -0800 | [diff] [blame] | 5298 | verbose_linfo(env, t, "%d: ", t); |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5299 | verbose(env, "jump out of range from insn %d to %d\n", t, w); |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 5300 | return -EINVAL; |
| 5301 | } |
| 5302 | |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 5303 | if (e == BRANCH) |
| 5304 | /* mark branch target for state pruning */ |
| 5305 | env->explored_states[w] = STATE_LIST_MARK; |
| 5306 | |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 5307 | if (insn_state[w] == 0) { |
| 5308 | /* tree-edge */ |
| 5309 | insn_state[t] = DISCOVERED | e; |
| 5310 | insn_state[w] = DISCOVERED; |
| 5311 | if (cur_stack >= env->prog->len) |
| 5312 | return -E2BIG; |
| 5313 | insn_stack[cur_stack++] = w; |
| 5314 | return 1; |
| 5315 | } else if ((insn_state[w] & 0xF0) == DISCOVERED) { |
Martin KaFai Lau | d9762e8 | 2018-12-13 10:41:48 -0800 | [diff] [blame] | 5316 | verbose_linfo(env, t, "%d: ", t); |
| 5317 | verbose_linfo(env, w, "%d: ", w); |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5318 | verbose(env, "back-edge from insn %d to %d\n", t, w); |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 5319 | return -EINVAL; |
| 5320 | } else if (insn_state[w] == EXPLORED) { |
| 5321 | /* forward- or cross-edge */ |
| 5322 | insn_state[t] = DISCOVERED | e; |
| 5323 | } else { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5324 | verbose(env, "insn state internal bug\n"); |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 5325 | return -EFAULT; |
| 5326 | } |
| 5327 | return 0; |
| 5328 | } |
| 5329 | |
| 5330 | /* non-recursive depth-first-search to detect loops in BPF program |
| 5331 | * loop == back-edge in directed graph |
| 5332 | */ |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 5333 | static int check_cfg(struct bpf_verifier_env *env) |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 5334 | { |
| 5335 | struct bpf_insn *insns = env->prog->insnsi; |
| 5336 | int insn_cnt = env->prog->len; |
| 5337 | int ret = 0; |
| 5338 | int i, t; |
| 5339 | |
Alexei Starovoitov | 71dde68 | 2019-04-01 21:27:43 -0700 | [diff] [blame] | 5340 | insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 5341 | if (!insn_state) |
| 5342 | return -ENOMEM; |
| 5343 | |
Alexei Starovoitov | 71dde68 | 2019-04-01 21:27:43 -0700 | [diff] [blame] | 5344 | insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 5345 | if (!insn_stack) { |
Alexei Starovoitov | 71dde68 | 2019-04-01 21:27:43 -0700 | [diff] [blame] | 5346 | kvfree(insn_state); |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 5347 | return -ENOMEM; |
| 5348 | } |
| 5349 | |
| 5350 | insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */ |
| 5351 | insn_stack[0] = 0; /* 0 is the first instruction */ |
| 5352 | cur_stack = 1; |
| 5353 | |
| 5354 | peek_stack: |
| 5355 | if (cur_stack == 0) |
| 5356 | goto check_state; |
| 5357 | t = insn_stack[cur_stack - 1]; |
| 5358 | |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 5359 | if (BPF_CLASS(insns[t].code) == BPF_JMP || |
| 5360 | BPF_CLASS(insns[t].code) == BPF_JMP32) { |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 5361 | u8 opcode = BPF_OP(insns[t].code); |
| 5362 | |
| 5363 | if (opcode == BPF_EXIT) { |
| 5364 | goto mark_explored; |
| 5365 | } else if (opcode == BPF_CALL) { |
| 5366 | ret = push_insn(t, t + 1, FALLTHROUGH, env); |
| 5367 | if (ret == 1) |
| 5368 | goto peek_stack; |
| 5369 | else if (ret < 0) |
| 5370 | goto err_free; |
Daniel Borkmann | 0701615 | 2016-04-05 22:33:17 +0200 | [diff] [blame] | 5371 | if (t + 1 < insn_cnt) |
| 5372 | env->explored_states[t + 1] = STATE_LIST_MARK; |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 5373 | if (insns[t].src_reg == BPF_PSEUDO_CALL) { |
| 5374 | env->explored_states[t] = STATE_LIST_MARK; |
| 5375 | ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env); |
| 5376 | if (ret == 1) |
| 5377 | goto peek_stack; |
| 5378 | else if (ret < 0) |
| 5379 | goto err_free; |
| 5380 | } |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 5381 | } else if (opcode == BPF_JA) { |
| 5382 | if (BPF_SRC(insns[t].code) != BPF_K) { |
| 5383 | ret = -EINVAL; |
| 5384 | goto err_free; |
| 5385 | } |
| 5386 | /* unconditional jump with single edge */ |
| 5387 | ret = push_insn(t, t + insns[t].off + 1, |
| 5388 | FALLTHROUGH, env); |
| 5389 | if (ret == 1) |
| 5390 | goto peek_stack; |
| 5391 | else if (ret < 0) |
| 5392 | goto err_free; |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 5393 | /* tell verifier to check for equivalent states |
| 5394 | * after every call and jump |
| 5395 | */ |
Alexei Starovoitov | c3de631 | 2015-04-14 15:57:13 -0700 | [diff] [blame] | 5396 | if (t + 1 < insn_cnt) |
| 5397 | env->explored_states[t + 1] = STATE_LIST_MARK; |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 5398 | } else { |
| 5399 | /* conditional jump with two edges */ |
Daniel Borkmann | 3c2ce60 | 2017-05-18 03:00:06 +0200 | [diff] [blame] | 5400 | env->explored_states[t] = STATE_LIST_MARK; |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 5401 | ret = push_insn(t, t + 1, FALLTHROUGH, env); |
| 5402 | if (ret == 1) |
| 5403 | goto peek_stack; |
| 5404 | else if (ret < 0) |
| 5405 | goto err_free; |
| 5406 | |
| 5407 | ret = push_insn(t, t + insns[t].off + 1, BRANCH, env); |
| 5408 | if (ret == 1) |
| 5409 | goto peek_stack; |
| 5410 | else if (ret < 0) |
| 5411 | goto err_free; |
| 5412 | } |
| 5413 | } else { |
| 5414 | /* all other non-branch instructions with single |
| 5415 | * fall-through edge |
| 5416 | */ |
| 5417 | ret = push_insn(t, t + 1, FALLTHROUGH, env); |
| 5418 | if (ret == 1) |
| 5419 | goto peek_stack; |
| 5420 | else if (ret < 0) |
| 5421 | goto err_free; |
| 5422 | } |
| 5423 | |
| 5424 | mark_explored: |
| 5425 | insn_state[t] = EXPLORED; |
| 5426 | if (cur_stack-- <= 0) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5427 | verbose(env, "pop stack internal bug\n"); |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 5428 | ret = -EFAULT; |
| 5429 | goto err_free; |
| 5430 | } |
| 5431 | goto peek_stack; |
| 5432 | |
| 5433 | check_state: |
| 5434 | for (i = 0; i < insn_cnt; i++) { |
| 5435 | if (insn_state[i] != EXPLORED) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5436 | verbose(env, "unreachable insn %d\n", i); |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 5437 | ret = -EINVAL; |
| 5438 | goto err_free; |
| 5439 | } |
| 5440 | } |
| 5441 | ret = 0; /* cfg looks good */ |
| 5442 | |
| 5443 | err_free: |
Alexei Starovoitov | 71dde68 | 2019-04-01 21:27:43 -0700 | [diff] [blame] | 5444 | kvfree(insn_state); |
| 5445 | kvfree(insn_stack); |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 5446 | return ret; |
| 5447 | } |
| 5448 | |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 5449 | /* The minimum supported BTF func info size */ |
| 5450 | #define MIN_BPF_FUNCINFO_SIZE 8 |
| 5451 | #define MAX_FUNCINFO_REC_SIZE 252 |
| 5452 | |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 5453 | static int check_btf_func(struct bpf_verifier_env *env, |
| 5454 | const union bpf_attr *attr, |
| 5455 | union bpf_attr __user *uattr) |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 5456 | { |
Peter Oskolkov | d0b2818 | 2019-01-16 10:43:01 -0800 | [diff] [blame] | 5457 | u32 i, nfuncs, urec_size, min_size; |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 5458 | u32 krec_size = sizeof(struct bpf_func_info); |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 5459 | struct bpf_func_info *krecord; |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 5460 | const struct btf_type *type; |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 5461 | struct bpf_prog *prog; |
| 5462 | const struct btf *btf; |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 5463 | void __user *urecord; |
Peter Oskolkov | d0b2818 | 2019-01-16 10:43:01 -0800 | [diff] [blame] | 5464 | u32 prev_offset = 0; |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 5465 | int ret = 0; |
| 5466 | |
| 5467 | nfuncs = attr->func_info_cnt; |
| 5468 | if (!nfuncs) |
| 5469 | return 0; |
| 5470 | |
| 5471 | if (nfuncs != env->subprog_cnt) { |
| 5472 | verbose(env, "number of funcs in func_info doesn't match number of subprogs\n"); |
| 5473 | return -EINVAL; |
| 5474 | } |
| 5475 | |
| 5476 | urec_size = attr->func_info_rec_size; |
| 5477 | if (urec_size < MIN_BPF_FUNCINFO_SIZE || |
| 5478 | urec_size > MAX_FUNCINFO_REC_SIZE || |
| 5479 | urec_size % sizeof(u32)) { |
| 5480 | verbose(env, "invalid func info rec size %u\n", urec_size); |
| 5481 | return -EINVAL; |
| 5482 | } |
| 5483 | |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 5484 | prog = env->prog; |
| 5485 | btf = prog->aux->btf; |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 5486 | |
| 5487 | urecord = u64_to_user_ptr(attr->func_info); |
| 5488 | min_size = min_t(u32, krec_size, urec_size); |
| 5489 | |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 5490 | krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN); |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 5491 | if (!krecord) |
| 5492 | return -ENOMEM; |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 5493 | |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 5494 | for (i = 0; i < nfuncs; i++) { |
| 5495 | ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size); |
| 5496 | if (ret) { |
| 5497 | if (ret == -E2BIG) { |
| 5498 | verbose(env, "nonzero tailing record in func info"); |
| 5499 | /* set the size kernel expects so loader can zero |
| 5500 | * out the rest of the record. |
| 5501 | */ |
| 5502 | if (put_user(min_size, &uattr->func_info_rec_size)) |
| 5503 | ret = -EFAULT; |
| 5504 | } |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 5505 | goto err_free; |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 5506 | } |
| 5507 | |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 5508 | if (copy_from_user(&krecord[i], urecord, min_size)) { |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 5509 | ret = -EFAULT; |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 5510 | goto err_free; |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 5511 | } |
| 5512 | |
Martin KaFai Lau | d30d42e | 2018-12-05 17:35:44 -0800 | [diff] [blame] | 5513 | /* check insn_off */ |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 5514 | if (i == 0) { |
Martin KaFai Lau | d30d42e | 2018-12-05 17:35:44 -0800 | [diff] [blame] | 5515 | if (krecord[i].insn_off) { |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 5516 | verbose(env, |
Martin KaFai Lau | d30d42e | 2018-12-05 17:35:44 -0800 | [diff] [blame] | 5517 | "nonzero insn_off %u for the first func info record", |
| 5518 | krecord[i].insn_off); |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 5519 | ret = -EINVAL; |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 5520 | goto err_free; |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 5521 | } |
Martin KaFai Lau | d30d42e | 2018-12-05 17:35:44 -0800 | [diff] [blame] | 5522 | } else if (krecord[i].insn_off <= prev_offset) { |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 5523 | verbose(env, |
| 5524 | "same or smaller insn offset (%u) than previous func info record (%u)", |
Martin KaFai Lau | d30d42e | 2018-12-05 17:35:44 -0800 | [diff] [blame] | 5525 | krecord[i].insn_off, prev_offset); |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 5526 | ret = -EINVAL; |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 5527 | goto err_free; |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 5528 | } |
| 5529 | |
Martin KaFai Lau | d30d42e | 2018-12-05 17:35:44 -0800 | [diff] [blame] | 5530 | if (env->subprog_info[i].start != krecord[i].insn_off) { |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 5531 | verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); |
| 5532 | ret = -EINVAL; |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 5533 | goto err_free; |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 5534 | } |
| 5535 | |
| 5536 | /* check type_id */ |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 5537 | type = btf_type_by_id(btf, krecord[i].type_id); |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 5538 | if (!type || BTF_INFO_KIND(type->info) != BTF_KIND_FUNC) { |
| 5539 | verbose(env, "invalid type id %d in func info", |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 5540 | krecord[i].type_id); |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 5541 | ret = -EINVAL; |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 5542 | goto err_free; |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 5543 | } |
| 5544 | |
Martin KaFai Lau | d30d42e | 2018-12-05 17:35:44 -0800 | [diff] [blame] | 5545 | prev_offset = krecord[i].insn_off; |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 5546 | urecord += urec_size; |
| 5547 | } |
| 5548 | |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 5549 | prog->aux->func_info = krecord; |
| 5550 | prog->aux->func_info_cnt = nfuncs; |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 5551 | return 0; |
| 5552 | |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 5553 | err_free: |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 5554 | kvfree(krecord); |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 5555 | return ret; |
| 5556 | } |
| 5557 | |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 5558 | static void adjust_btf_func(struct bpf_verifier_env *env) |
| 5559 | { |
| 5560 | int i; |
| 5561 | |
| 5562 | if (!env->prog->aux->func_info) |
| 5563 | return; |
| 5564 | |
| 5565 | for (i = 0; i < env->subprog_cnt; i++) |
Martin KaFai Lau | d30d42e | 2018-12-05 17:35:44 -0800 | [diff] [blame] | 5566 | env->prog->aux->func_info[i].insn_off = env->subprog_info[i].start; |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 5567 | } |
| 5568 | |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 5569 | #define MIN_BPF_LINEINFO_SIZE (offsetof(struct bpf_line_info, line_col) + \ |
| 5570 | sizeof(((struct bpf_line_info *)(0))->line_col)) |
| 5571 | #define MAX_LINEINFO_REC_SIZE MAX_FUNCINFO_REC_SIZE |
| 5572 | |
| 5573 | static int check_btf_line(struct bpf_verifier_env *env, |
| 5574 | const union bpf_attr *attr, |
| 5575 | union bpf_attr __user *uattr) |
| 5576 | { |
| 5577 | u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0; |
| 5578 | struct bpf_subprog_info *sub; |
| 5579 | struct bpf_line_info *linfo; |
| 5580 | struct bpf_prog *prog; |
| 5581 | const struct btf *btf; |
| 5582 | void __user *ulinfo; |
| 5583 | int err; |
| 5584 | |
| 5585 | nr_linfo = attr->line_info_cnt; |
| 5586 | if (!nr_linfo) |
| 5587 | return 0; |
| 5588 | |
| 5589 | rec_size = attr->line_info_rec_size; |
| 5590 | if (rec_size < MIN_BPF_LINEINFO_SIZE || |
| 5591 | rec_size > MAX_LINEINFO_REC_SIZE || |
| 5592 | rec_size & (sizeof(u32) - 1)) |
| 5593 | return -EINVAL; |
| 5594 | |
| 5595 | /* Need to zero it in case the userspace may |
| 5596 | * pass in a smaller bpf_line_info object. |
| 5597 | */ |
| 5598 | linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info), |
| 5599 | GFP_KERNEL | __GFP_NOWARN); |
| 5600 | if (!linfo) |
| 5601 | return -ENOMEM; |
| 5602 | |
| 5603 | prog = env->prog; |
| 5604 | btf = prog->aux->btf; |
| 5605 | |
| 5606 | s = 0; |
| 5607 | sub = env->subprog_info; |
| 5608 | ulinfo = u64_to_user_ptr(attr->line_info); |
| 5609 | expected_size = sizeof(struct bpf_line_info); |
| 5610 | ncopy = min_t(u32, expected_size, rec_size); |
| 5611 | for (i = 0; i < nr_linfo; i++) { |
| 5612 | err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size); |
| 5613 | if (err) { |
| 5614 | if (err == -E2BIG) { |
| 5615 | verbose(env, "nonzero tailing record in line_info"); |
| 5616 | if (put_user(expected_size, |
| 5617 | &uattr->line_info_rec_size)) |
| 5618 | err = -EFAULT; |
| 5619 | } |
| 5620 | goto err_free; |
| 5621 | } |
| 5622 | |
| 5623 | if (copy_from_user(&linfo[i], ulinfo, ncopy)) { |
| 5624 | err = -EFAULT; |
| 5625 | goto err_free; |
| 5626 | } |
| 5627 | |
| 5628 | /* |
| 5629 | * Check insn_off to ensure |
| 5630 | * 1) strictly increasing AND |
| 5631 | * 2) bounded by prog->len |
| 5632 | * |
| 5633 | * The linfo[0].insn_off == 0 check logically falls into |
| 5634 | * the later "missing bpf_line_info for func..." case |
| 5635 | * because the first linfo[0].insn_off must be the |
| 5636 | * first sub also and the first sub must have |
| 5637 | * subprog_info[0].start == 0. |
| 5638 | */ |
| 5639 | if ((i && linfo[i].insn_off <= prev_offset) || |
| 5640 | linfo[i].insn_off >= prog->len) { |
| 5641 | verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n", |
| 5642 | i, linfo[i].insn_off, prev_offset, |
| 5643 | prog->len); |
| 5644 | err = -EINVAL; |
| 5645 | goto err_free; |
| 5646 | } |
| 5647 | |
Martin KaFai Lau | fdbaa0b | 2018-12-19 13:01:01 -0800 | [diff] [blame] | 5648 | if (!prog->insnsi[linfo[i].insn_off].code) { |
| 5649 | verbose(env, |
| 5650 | "Invalid insn code at line_info[%u].insn_off\n", |
| 5651 | i); |
| 5652 | err = -EINVAL; |
| 5653 | goto err_free; |
| 5654 | } |
| 5655 | |
Martin KaFai Lau | 23127b3 | 2018-12-13 10:41:46 -0800 | [diff] [blame] | 5656 | if (!btf_name_by_offset(btf, linfo[i].line_off) || |
| 5657 | !btf_name_by_offset(btf, linfo[i].file_name_off)) { |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 5658 | verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i); |
| 5659 | err = -EINVAL; |
| 5660 | goto err_free; |
| 5661 | } |
| 5662 | |
| 5663 | if (s != env->subprog_cnt) { |
| 5664 | if (linfo[i].insn_off == sub[s].start) { |
| 5665 | sub[s].linfo_idx = i; |
| 5666 | s++; |
| 5667 | } else if (sub[s].start < linfo[i].insn_off) { |
| 5668 | verbose(env, "missing bpf_line_info for func#%u\n", s); |
| 5669 | err = -EINVAL; |
| 5670 | goto err_free; |
| 5671 | } |
| 5672 | } |
| 5673 | |
| 5674 | prev_offset = linfo[i].insn_off; |
| 5675 | ulinfo += rec_size; |
| 5676 | } |
| 5677 | |
| 5678 | if (s != env->subprog_cnt) { |
| 5679 | verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n", |
| 5680 | env->subprog_cnt - s, s); |
| 5681 | err = -EINVAL; |
| 5682 | goto err_free; |
| 5683 | } |
| 5684 | |
| 5685 | prog->aux->linfo = linfo; |
| 5686 | prog->aux->nr_linfo = nr_linfo; |
| 5687 | |
| 5688 | return 0; |
| 5689 | |
| 5690 | err_free: |
| 5691 | kvfree(linfo); |
| 5692 | return err; |
| 5693 | } |
| 5694 | |
| 5695 | static int check_btf_info(struct bpf_verifier_env *env, |
| 5696 | const union bpf_attr *attr, |
| 5697 | union bpf_attr __user *uattr) |
| 5698 | { |
| 5699 | struct btf *btf; |
| 5700 | int err; |
| 5701 | |
| 5702 | if (!attr->func_info_cnt && !attr->line_info_cnt) |
| 5703 | return 0; |
| 5704 | |
| 5705 | btf = btf_get_by_fd(attr->prog_btf_fd); |
| 5706 | if (IS_ERR(btf)) |
| 5707 | return PTR_ERR(btf); |
| 5708 | env->prog->aux->btf = btf; |
| 5709 | |
| 5710 | err = check_btf_func(env, attr, uattr); |
| 5711 | if (err) |
| 5712 | return err; |
| 5713 | |
| 5714 | err = check_btf_line(env, attr, uattr); |
| 5715 | if (err) |
| 5716 | return err; |
| 5717 | |
| 5718 | return 0; |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 5719 | } |
| 5720 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 5721 | /* check %cur's range satisfies %old's */ |
| 5722 | static bool range_within(struct bpf_reg_state *old, |
| 5723 | struct bpf_reg_state *cur) |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 5724 | { |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 5725 | return old->umin_value <= cur->umin_value && |
| 5726 | old->umax_value >= cur->umax_value && |
| 5727 | old->smin_value <= cur->smin_value && |
| 5728 | old->smax_value >= cur->smax_value; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 5729 | } |
| 5730 | |
| 5731 | /* Maximum number of register states that can exist at once */ |
| 5732 | #define ID_MAP_SIZE (MAX_BPF_REG + MAX_BPF_STACK / BPF_REG_SIZE) |
| 5733 | struct idpair { |
| 5734 | u32 old; |
| 5735 | u32 cur; |
| 5736 | }; |
| 5737 | |
| 5738 | /* If in the old state two registers had the same id, then they need to have |
| 5739 | * the same id in the new state as well. But that id could be different from |
| 5740 | * the old state, so we need to track the mapping from old to new ids. |
| 5741 | * Once we have seen that, say, a reg with old id 5 had new id 9, any subsequent |
| 5742 | * regs with old id 5 must also have new id 9 for the new state to be safe. But |
| 5743 | * regs with a different old id could still have new id 9, we don't care about |
| 5744 | * that. |
| 5745 | * So we look through our idmap to see if this old id has been seen before. If |
| 5746 | * so, we require the new id to match; otherwise, we add the id pair to the map. |
| 5747 | */ |
| 5748 | static bool check_ids(u32 old_id, u32 cur_id, struct idpair *idmap) |
| 5749 | { |
| 5750 | unsigned int i; |
| 5751 | |
| 5752 | for (i = 0; i < ID_MAP_SIZE; i++) { |
| 5753 | if (!idmap[i].old) { |
| 5754 | /* Reached an empty slot; haven't seen this id before */ |
| 5755 | idmap[i].old = old_id; |
| 5756 | idmap[i].cur = cur_id; |
| 5757 | return true; |
| 5758 | } |
| 5759 | if (idmap[i].old == old_id) |
| 5760 | return idmap[i].cur == cur_id; |
| 5761 | } |
| 5762 | /* We ran out of idmap slots, which should be impossible */ |
| 5763 | WARN_ON_ONCE(1); |
| 5764 | return false; |
| 5765 | } |
| 5766 | |
Alexei Starovoitov | 9242b5f | 2018-12-13 11:42:34 -0800 | [diff] [blame] | 5767 | static void clean_func_state(struct bpf_verifier_env *env, |
| 5768 | struct bpf_func_state *st) |
| 5769 | { |
| 5770 | enum bpf_reg_liveness live; |
| 5771 | int i, j; |
| 5772 | |
| 5773 | for (i = 0; i < BPF_REG_FP; i++) { |
| 5774 | live = st->regs[i].live; |
| 5775 | /* liveness must not touch this register anymore */ |
| 5776 | st->regs[i].live |= REG_LIVE_DONE; |
| 5777 | if (!(live & REG_LIVE_READ)) |
| 5778 | /* since the register is unused, clear its state |
| 5779 | * to make further comparison simpler |
| 5780 | */ |
| 5781 | __mark_reg_not_init(&st->regs[i]); |
| 5782 | } |
| 5783 | |
| 5784 | for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) { |
| 5785 | live = st->stack[i].spilled_ptr.live; |
| 5786 | /* liveness must not touch this stack slot anymore */ |
| 5787 | st->stack[i].spilled_ptr.live |= REG_LIVE_DONE; |
| 5788 | if (!(live & REG_LIVE_READ)) { |
| 5789 | __mark_reg_not_init(&st->stack[i].spilled_ptr); |
| 5790 | for (j = 0; j < BPF_REG_SIZE; j++) |
| 5791 | st->stack[i].slot_type[j] = STACK_INVALID; |
| 5792 | } |
| 5793 | } |
| 5794 | } |
| 5795 | |
| 5796 | static void clean_verifier_state(struct bpf_verifier_env *env, |
| 5797 | struct bpf_verifier_state *st) |
| 5798 | { |
| 5799 | int i; |
| 5800 | |
| 5801 | if (st->frame[0]->regs[0].live & REG_LIVE_DONE) |
| 5802 | /* all regs in this state in all frames were already marked */ |
| 5803 | return; |
| 5804 | |
| 5805 | for (i = 0; i <= st->curframe; i++) |
| 5806 | clean_func_state(env, st->frame[i]); |
| 5807 | } |
| 5808 | |
| 5809 | /* the parentage chains form a tree. |
| 5810 | * the verifier states are added to state lists at given insn and |
| 5811 | * pushed into state stack for future exploration. |
| 5812 | * when the verifier reaches bpf_exit insn some of the verifer states |
| 5813 | * stored in the state lists have their final liveness state already, |
| 5814 | * but a lot of states will get revised from liveness point of view when |
| 5815 | * the verifier explores other branches. |
| 5816 | * Example: |
| 5817 | * 1: r0 = 1 |
| 5818 | * 2: if r1 == 100 goto pc+1 |
| 5819 | * 3: r0 = 2 |
| 5820 | * 4: exit |
| 5821 | * when the verifier reaches exit insn the register r0 in the state list of |
| 5822 | * insn 2 will be seen as !REG_LIVE_READ. Then the verifier pops the other_branch |
| 5823 | * of insn 2 and goes exploring further. At the insn 4 it will walk the |
| 5824 | * parentage chain from insn 4 into insn 2 and will mark r0 as REG_LIVE_READ. |
| 5825 | * |
| 5826 | * Since the verifier pushes the branch states as it sees them while exploring |
| 5827 | * the program the condition of walking the branch instruction for the second |
| 5828 | * time means that all states below this branch were already explored and |
| 5829 | * their final liveness markes are already propagated. |
| 5830 | * Hence when the verifier completes the search of state list in is_state_visited() |
| 5831 | * we can call this clean_live_states() function to mark all liveness states |
| 5832 | * as REG_LIVE_DONE to indicate that 'parent' pointers of 'struct bpf_reg_state' |
| 5833 | * will not be used. |
| 5834 | * This function also clears the registers and stack for states that !READ |
| 5835 | * to simplify state merging. |
| 5836 | * |
| 5837 | * Important note here that walking the same branch instruction in the callee |
| 5838 | * doesn't meant that the states are DONE. The verifier has to compare |
| 5839 | * the callsites |
| 5840 | */ |
| 5841 | static void clean_live_states(struct bpf_verifier_env *env, int insn, |
| 5842 | struct bpf_verifier_state *cur) |
| 5843 | { |
| 5844 | struct bpf_verifier_state_list *sl; |
| 5845 | int i; |
| 5846 | |
| 5847 | sl = env->explored_states[insn]; |
| 5848 | if (!sl) |
| 5849 | return; |
| 5850 | |
| 5851 | while (sl != STATE_LIST_MARK) { |
| 5852 | if (sl->state.curframe != cur->curframe) |
| 5853 | goto next; |
| 5854 | for (i = 0; i <= cur->curframe; i++) |
| 5855 | if (sl->state.frame[i]->callsite != cur->frame[i]->callsite) |
| 5856 | goto next; |
| 5857 | clean_verifier_state(env, &sl->state); |
| 5858 | next: |
| 5859 | sl = sl->next; |
| 5860 | } |
| 5861 | } |
| 5862 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 5863 | /* Returns true if (rold safe implies rcur safe) */ |
Edward Cree | 1b688a1 | 2017-08-23 15:10:50 +0100 | [diff] [blame] | 5864 | static bool regsafe(struct bpf_reg_state *rold, struct bpf_reg_state *rcur, |
| 5865 | struct idpair *idmap) |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 5866 | { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5867 | bool equal; |
| 5868 | |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 5869 | if (!(rold->live & REG_LIVE_READ)) |
| 5870 | /* explored state didn't use this */ |
| 5871 | return true; |
| 5872 | |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 5873 | equal = memcmp(rold, rcur, offsetof(struct bpf_reg_state, parent)) == 0; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5874 | |
| 5875 | if (rold->type == PTR_TO_STACK) |
| 5876 | /* two stack pointers are equal only if they're pointing to |
| 5877 | * the same stack frame, since fp-8 in foo != fp-8 in bar |
| 5878 | */ |
| 5879 | return equal && rold->frameno == rcur->frameno; |
| 5880 | |
| 5881 | if (equal) |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 5882 | return true; |
| 5883 | |
| 5884 | if (rold->type == NOT_INIT) |
| 5885 | /* explored state can't have used this */ |
| 5886 | return true; |
| 5887 | if (rcur->type == NOT_INIT) |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 5888 | return false; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 5889 | switch (rold->type) { |
| 5890 | case SCALAR_VALUE: |
| 5891 | if (rcur->type == SCALAR_VALUE) { |
| 5892 | /* new val must satisfy old val knowledge */ |
| 5893 | return range_within(rold, rcur) && |
| 5894 | tnum_in(rold->var_off, rcur->var_off); |
| 5895 | } else { |
Jann Horn | 179d1c5 | 2017-12-18 20:11:59 -0800 | [diff] [blame] | 5896 | /* We're trying to use a pointer in place of a scalar. |
| 5897 | * Even if the scalar was unbounded, this could lead to |
| 5898 | * pointer leaks because scalars are allowed to leak |
| 5899 | * while pointers are not. We could make this safe in |
| 5900 | * special cases if root is calling us, but it's |
| 5901 | * probably not worth the hassle. |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 5902 | */ |
Jann Horn | 179d1c5 | 2017-12-18 20:11:59 -0800 | [diff] [blame] | 5903 | return false; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 5904 | } |
| 5905 | case PTR_TO_MAP_VALUE: |
Edward Cree | 1b688a1 | 2017-08-23 15:10:50 +0100 | [diff] [blame] | 5906 | /* If the new min/max/var_off satisfy the old ones and |
| 5907 | * everything else matches, we are OK. |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 5908 | * 'id' is not compared, since it's only used for maps with |
| 5909 | * bpf_spin_lock inside map element and in such cases if |
| 5910 | * the rest of the prog is valid for one map element then |
| 5911 | * it's valid for all map elements regardless of the key |
| 5912 | * used in bpf_map_lookup() |
Edward Cree | 1b688a1 | 2017-08-23 15:10:50 +0100 | [diff] [blame] | 5913 | */ |
| 5914 | return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 && |
| 5915 | range_within(rold, rcur) && |
| 5916 | tnum_in(rold->var_off, rcur->var_off); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 5917 | case PTR_TO_MAP_VALUE_OR_NULL: |
| 5918 | /* a PTR_TO_MAP_VALUE could be safe to use as a |
| 5919 | * PTR_TO_MAP_VALUE_OR_NULL into the same map. |
| 5920 | * However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL- |
| 5921 | * checked, doing so could have affected others with the same |
| 5922 | * id, and we can't check for that because we lost the id when |
| 5923 | * we converted to a PTR_TO_MAP_VALUE. |
| 5924 | */ |
| 5925 | if (rcur->type != PTR_TO_MAP_VALUE_OR_NULL) |
| 5926 | return false; |
| 5927 | if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id))) |
| 5928 | return false; |
| 5929 | /* Check our ids match any regs they're supposed to */ |
| 5930 | return check_ids(rold->id, rcur->id, idmap); |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 5931 | case PTR_TO_PACKET_META: |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 5932 | case PTR_TO_PACKET: |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 5933 | if (rcur->type != rold->type) |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 5934 | return false; |
| 5935 | /* We must have at least as much range as the old ptr |
| 5936 | * did, so that any accesses which were safe before are |
| 5937 | * still safe. This is true even if old range < old off, |
| 5938 | * since someone could have accessed through (ptr - k), or |
| 5939 | * even done ptr -= k in a register, to get a safe access. |
| 5940 | */ |
| 5941 | if (rold->range > rcur->range) |
| 5942 | return false; |
| 5943 | /* If the offsets don't match, we can't trust our alignment; |
| 5944 | * nor can we be sure that we won't fall out of range. |
| 5945 | */ |
| 5946 | if (rold->off != rcur->off) |
| 5947 | return false; |
| 5948 | /* id relations must be preserved */ |
| 5949 | if (rold->id && !check_ids(rold->id, rcur->id, idmap)) |
| 5950 | return false; |
| 5951 | /* new val must satisfy old val knowledge */ |
| 5952 | return range_within(rold, rcur) && |
| 5953 | tnum_in(rold->var_off, rcur->var_off); |
| 5954 | case PTR_TO_CTX: |
| 5955 | case CONST_PTR_TO_MAP: |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 5956 | case PTR_TO_PACKET_END: |
Petar Penkov | d58e468 | 2018-09-14 07:46:18 -0700 | [diff] [blame] | 5957 | case PTR_TO_FLOW_KEYS: |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 5958 | case PTR_TO_SOCKET: |
| 5959 | case PTR_TO_SOCKET_OR_NULL: |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 5960 | case PTR_TO_SOCK_COMMON: |
| 5961 | case PTR_TO_SOCK_COMMON_OR_NULL: |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 5962 | case PTR_TO_TCP_SOCK: |
| 5963 | case PTR_TO_TCP_SOCK_OR_NULL: |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 5964 | /* Only valid matches are exact, which memcmp() above |
| 5965 | * would have accepted |
| 5966 | */ |
| 5967 | default: |
| 5968 | /* Don't know what's going on, just say it's not safe */ |
| 5969 | return false; |
| 5970 | } |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 5971 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 5972 | /* Shouldn't get here; if we do, say it's not safe */ |
| 5973 | WARN_ON_ONCE(1); |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 5974 | return false; |
| 5975 | } |
| 5976 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5977 | static bool stacksafe(struct bpf_func_state *old, |
| 5978 | struct bpf_func_state *cur, |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 5979 | struct idpair *idmap) |
| 5980 | { |
| 5981 | int i, spi; |
| 5982 | |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 5983 | /* walk slots of the explored stack and ignore any additional |
| 5984 | * slots in the current stack, since explored(safe) state |
| 5985 | * didn't use them |
| 5986 | */ |
| 5987 | for (i = 0; i < old->allocated_stack; i++) { |
| 5988 | spi = i / BPF_REG_SIZE; |
| 5989 | |
Alexei Starovoitov | b233920 | 2018-12-13 11:42:31 -0800 | [diff] [blame] | 5990 | if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ)) { |
| 5991 | i += BPF_REG_SIZE - 1; |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 5992 | /* explored state didn't use this */ |
Gianluca Borello | fd05e57 | 2017-12-23 10:09:55 +0000 | [diff] [blame] | 5993 | continue; |
Alexei Starovoitov | b233920 | 2018-12-13 11:42:31 -0800 | [diff] [blame] | 5994 | } |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 5995 | |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 5996 | if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID) |
| 5997 | continue; |
Alexei Starovoitov | 19e2dbb | 2018-12-13 11:42:33 -0800 | [diff] [blame] | 5998 | |
| 5999 | /* explored stack has more populated slots than current stack |
| 6000 | * and these slots were used |
| 6001 | */ |
| 6002 | if (i >= cur->allocated_stack) |
| 6003 | return false; |
| 6004 | |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 6005 | /* if old state was safe with misc data in the stack |
| 6006 | * it will be safe with zero-initialized stack. |
| 6007 | * The opposite is not true |
| 6008 | */ |
| 6009 | if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC && |
| 6010 | cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO) |
| 6011 | continue; |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 6012 | if (old->stack[spi].slot_type[i % BPF_REG_SIZE] != |
| 6013 | cur->stack[spi].slot_type[i % BPF_REG_SIZE]) |
| 6014 | /* Ex: old explored (safe) state has STACK_SPILL in |
| 6015 | * this stack slot, but current has has STACK_MISC -> |
| 6016 | * this verifier states are not equivalent, |
| 6017 | * return false to continue verification of this path |
| 6018 | */ |
| 6019 | return false; |
| 6020 | if (i % BPF_REG_SIZE) |
| 6021 | continue; |
| 6022 | if (old->stack[spi].slot_type[0] != STACK_SPILL) |
| 6023 | continue; |
| 6024 | if (!regsafe(&old->stack[spi].spilled_ptr, |
| 6025 | &cur->stack[spi].spilled_ptr, |
| 6026 | idmap)) |
| 6027 | /* when explored and current stack slot are both storing |
| 6028 | * spilled registers, check that stored pointers types |
| 6029 | * are the same as well. |
| 6030 | * Ex: explored safe path could have stored |
| 6031 | * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -8} |
| 6032 | * but current path has stored: |
| 6033 | * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -16} |
| 6034 | * such verifier states are not equivalent. |
| 6035 | * return false to continue verification of this path |
| 6036 | */ |
| 6037 | return false; |
| 6038 | } |
| 6039 | return true; |
| 6040 | } |
| 6041 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 6042 | static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur) |
| 6043 | { |
| 6044 | if (old->acquired_refs != cur->acquired_refs) |
| 6045 | return false; |
| 6046 | return !memcmp(old->refs, cur->refs, |
| 6047 | sizeof(*old->refs) * old->acquired_refs); |
| 6048 | } |
| 6049 | |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 6050 | /* compare two verifier states |
| 6051 | * |
| 6052 | * all states stored in state_list are known to be valid, since |
| 6053 | * verifier reached 'bpf_exit' instruction through them |
| 6054 | * |
| 6055 | * this function is called when verifier exploring different branches of |
| 6056 | * execution popped from the state stack. If it sees an old state that has |
| 6057 | * more strict register state and more strict stack state then this execution |
| 6058 | * branch doesn't need to be explored further, since verifier already |
| 6059 | * concluded that more strict state leads to valid finish. |
| 6060 | * |
| 6061 | * Therefore two states are equivalent if register state is more conservative |
| 6062 | * and explored stack state is more conservative than the current one. |
| 6063 | * Example: |
| 6064 | * explored current |
| 6065 | * (slot1=INV slot2=MISC) == (slot1=MISC slot2=MISC) |
| 6066 | * (slot1=MISC slot2=MISC) != (slot1=INV slot2=MISC) |
| 6067 | * |
| 6068 | * In other words if current stack state (one being explored) has more |
| 6069 | * valid slots than old one that already passed validation, it means |
| 6070 | * the verifier can stop exploring and conclude that current state is valid too |
| 6071 | * |
| 6072 | * Similarly with registers. If explored state has register type as invalid |
| 6073 | * whereas register type in current state is meaningful, it means that |
| 6074 | * the current state will reach 'bpf_exit' instruction safely |
| 6075 | */ |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 6076 | static bool func_states_equal(struct bpf_func_state *old, |
| 6077 | struct bpf_func_state *cur) |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 6078 | { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 6079 | struct idpair *idmap; |
| 6080 | bool ret = false; |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 6081 | int i; |
| 6082 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 6083 | idmap = kcalloc(ID_MAP_SIZE, sizeof(struct idpair), GFP_KERNEL); |
| 6084 | /* If we failed to allocate the idmap, just say it's not safe */ |
| 6085 | if (!idmap) |
Alexei Starovoitov | 1a0dc1a | 2016-05-05 19:49:09 -0700 | [diff] [blame] | 6086 | return false; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 6087 | |
| 6088 | for (i = 0; i < MAX_BPF_REG; i++) { |
Edward Cree | 1b688a1 | 2017-08-23 15:10:50 +0100 | [diff] [blame] | 6089 | if (!regsafe(&old->regs[i], &cur->regs[i], idmap)) |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 6090 | goto out_free; |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 6091 | } |
| 6092 | |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 6093 | if (!stacksafe(old, cur, idmap)) |
| 6094 | goto out_free; |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 6095 | |
| 6096 | if (!refsafe(old, cur)) |
| 6097 | goto out_free; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 6098 | ret = true; |
| 6099 | out_free: |
| 6100 | kfree(idmap); |
| 6101 | return ret; |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 6102 | } |
| 6103 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 6104 | static bool states_equal(struct bpf_verifier_env *env, |
| 6105 | struct bpf_verifier_state *old, |
| 6106 | struct bpf_verifier_state *cur) |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 6107 | { |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 6108 | int i; |
| 6109 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 6110 | if (old->curframe != cur->curframe) |
| 6111 | return false; |
| 6112 | |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 6113 | /* Verification state from speculative execution simulation |
| 6114 | * must never prune a non-speculative execution one. |
| 6115 | */ |
| 6116 | if (old->speculative && !cur->speculative) |
| 6117 | return false; |
| 6118 | |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 6119 | if (old->active_spin_lock != cur->active_spin_lock) |
| 6120 | return false; |
| 6121 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 6122 | /* for states to be equal callsites have to be the same |
| 6123 | * and all frame states need to be equivalent |
| 6124 | */ |
| 6125 | for (i = 0; i <= old->curframe; i++) { |
| 6126 | if (old->frame[i]->callsite != cur->frame[i]->callsite) |
| 6127 | return false; |
| 6128 | if (!func_states_equal(old->frame[i], cur->frame[i])) |
| 6129 | return false; |
| 6130 | } |
| 6131 | return true; |
| 6132 | } |
| 6133 | |
| 6134 | /* A write screens off any subsequent reads; but write marks come from the |
| 6135 | * straight-line code between a state and its parent. When we arrive at an |
| 6136 | * equivalent state (jump target or such) we didn't arrive by the straight-line |
| 6137 | * code, so read marks in the state must propagate to the parent regardless |
| 6138 | * of the state's write marks. That's what 'parent == state->parent' comparison |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 6139 | * in mark_reg_read() is for. |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 6140 | */ |
| 6141 | static int propagate_liveness(struct bpf_verifier_env *env, |
| 6142 | const struct bpf_verifier_state *vstate, |
| 6143 | struct bpf_verifier_state *vparent) |
| 6144 | { |
| 6145 | int i, frame, err = 0; |
| 6146 | struct bpf_func_state *state, *parent; |
| 6147 | |
| 6148 | if (vparent->curframe != vstate->curframe) { |
| 6149 | WARN(1, "propagate_live: parent frame %d current frame %d\n", |
| 6150 | vparent->curframe, vstate->curframe); |
| 6151 | return -EFAULT; |
| 6152 | } |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 6153 | /* Propagate read liveness of registers... */ |
| 6154 | BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG); |
Jakub Kicinski | 83d1631 | 2019-03-21 14:34:36 -0700 | [diff] [blame] | 6155 | for (frame = 0; frame <= vstate->curframe; frame++) { |
| 6156 | /* We don't need to worry about FP liveness, it's read-only */ |
| 6157 | for (i = frame < vstate->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) { |
| 6158 | if (vparent->frame[frame]->regs[i].live & REG_LIVE_READ) |
| 6159 | continue; |
| 6160 | if (vstate->frame[frame]->regs[i].live & REG_LIVE_READ) { |
| 6161 | err = mark_reg_read(env, &vstate->frame[frame]->regs[i], |
| 6162 | &vparent->frame[frame]->regs[i]); |
| 6163 | if (err) |
| 6164 | return err; |
| 6165 | } |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 6166 | } |
| 6167 | } |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 6168 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 6169 | /* ... and stack slots */ |
| 6170 | for (frame = 0; frame <= vstate->curframe; frame++) { |
| 6171 | state = vstate->frame[frame]; |
| 6172 | parent = vparent->frame[frame]; |
| 6173 | for (i = 0; i < state->allocated_stack / BPF_REG_SIZE && |
| 6174 | i < parent->allocated_stack / BPF_REG_SIZE; i++) { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 6175 | if (parent->stack[i].spilled_ptr.live & REG_LIVE_READ) |
| 6176 | continue; |
| 6177 | if (state->stack[i].spilled_ptr.live & REG_LIVE_READ) |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 6178 | mark_reg_read(env, &state->stack[i].spilled_ptr, |
| 6179 | &parent->stack[i].spilled_ptr); |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 6180 | } |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 6181 | } |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 6182 | return err; |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 6183 | } |
| 6184 | |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 6185 | static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 6186 | { |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 6187 | struct bpf_verifier_state_list *new_sl; |
Alexei Starovoitov | 9f4686c | 2019-04-01 21:27:41 -0700 | [diff] [blame] | 6188 | struct bpf_verifier_state_list *sl, **pprev; |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 6189 | struct bpf_verifier_state *cur = env->cur_state, *new; |
Alexei Starovoitov | ceefbc9 | 2018-12-03 22:46:06 -0800 | [diff] [blame] | 6190 | int i, j, err, states_cnt = 0; |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 6191 | |
Alexei Starovoitov | 9f4686c | 2019-04-01 21:27:41 -0700 | [diff] [blame] | 6192 | pprev = &env->explored_states[insn_idx]; |
| 6193 | sl = *pprev; |
| 6194 | |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 6195 | if (!sl) |
| 6196 | /* this 'insn_idx' instruction wasn't marked, so we will not |
| 6197 | * be doing state search here |
| 6198 | */ |
| 6199 | return 0; |
| 6200 | |
Alexei Starovoitov | 9242b5f | 2018-12-13 11:42:34 -0800 | [diff] [blame] | 6201 | clean_live_states(env, insn_idx, cur); |
| 6202 | |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 6203 | while (sl != STATE_LIST_MARK) { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 6204 | if (states_equal(env, &sl->state, cur)) { |
Alexei Starovoitov | 9f4686c | 2019-04-01 21:27:41 -0700 | [diff] [blame] | 6205 | sl->hit_cnt++; |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 6206 | /* reached equivalent register/stack state, |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 6207 | * prune the search. |
| 6208 | * Registers read by the continuation are read by us. |
Edward Cree | 8e9cd9c | 2017-08-23 15:11:21 +0100 | [diff] [blame] | 6209 | * If we have any write marks in env->cur_state, they |
| 6210 | * will prevent corresponding reads in the continuation |
| 6211 | * from reaching our parent (an explored_state). Our |
| 6212 | * own state will get the read marks recorded, but |
| 6213 | * they'll be immediately forgotten as we're pruning |
| 6214 | * this state and will pop a new one. |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 6215 | */ |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 6216 | err = propagate_liveness(env, &sl->state, cur); |
| 6217 | if (err) |
| 6218 | return err; |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 6219 | return 1; |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 6220 | } |
Alexei Starovoitov | ceefbc9 | 2018-12-03 22:46:06 -0800 | [diff] [blame] | 6221 | states_cnt++; |
Alexei Starovoitov | 9f4686c | 2019-04-01 21:27:41 -0700 | [diff] [blame] | 6222 | sl->miss_cnt++; |
| 6223 | /* heuristic to determine whether this state is beneficial |
| 6224 | * to keep checking from state equivalence point of view. |
| 6225 | * Higher numbers increase max_states_per_insn and verification time, |
| 6226 | * but do not meaningfully decrease insn_processed. |
| 6227 | */ |
| 6228 | if (sl->miss_cnt > sl->hit_cnt * 3 + 3) { |
| 6229 | /* the state is unlikely to be useful. Remove it to |
| 6230 | * speed up verification |
| 6231 | */ |
| 6232 | *pprev = sl->next; |
| 6233 | if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE) { |
| 6234 | free_verifier_state(&sl->state, false); |
| 6235 | kfree(sl); |
| 6236 | env->peak_states--; |
| 6237 | } else { |
| 6238 | /* cannot free this state, since parentage chain may |
| 6239 | * walk it later. Add it for free_list instead to |
| 6240 | * be freed at the end of verification |
| 6241 | */ |
| 6242 | sl->next = env->free_list; |
| 6243 | env->free_list = sl; |
| 6244 | } |
| 6245 | sl = *pprev; |
| 6246 | continue; |
| 6247 | } |
| 6248 | pprev = &sl->next; |
| 6249 | sl = *pprev; |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 6250 | } |
| 6251 | |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 6252 | if (env->max_states_per_insn < states_cnt) |
| 6253 | env->max_states_per_insn = states_cnt; |
| 6254 | |
Alexei Starovoitov | ceefbc9 | 2018-12-03 22:46:06 -0800 | [diff] [blame] | 6255 | if (!env->allow_ptr_leaks && states_cnt > BPF_COMPLEXITY_LIMIT_STATES) |
| 6256 | return 0; |
| 6257 | |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 6258 | /* there were no equivalent states, remember current one. |
| 6259 | * technically the current state is not proven to be safe yet, |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 6260 | * but it will either reach outer most bpf_exit (which means it's safe) |
| 6261 | * or it will be rejected. Since there are no loops, we won't be |
| 6262 | * seeing this tuple (frame[0].callsite, frame[1].callsite, .. insn_idx) |
| 6263 | * again on the way to bpf_exit |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 6264 | */ |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 6265 | new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL); |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 6266 | if (!new_sl) |
| 6267 | return -ENOMEM; |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 6268 | env->total_states++; |
| 6269 | env->peak_states++; |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 6270 | |
| 6271 | /* add new state to the head of linked list */ |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 6272 | new = &new_sl->state; |
| 6273 | err = copy_verifier_state(new, cur); |
Alexei Starovoitov | 1969db4 | 2017-11-01 00:08:04 -0700 | [diff] [blame] | 6274 | if (err) { |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 6275 | free_verifier_state(new, false); |
Alexei Starovoitov | 1969db4 | 2017-11-01 00:08:04 -0700 | [diff] [blame] | 6276 | kfree(new_sl); |
| 6277 | return err; |
| 6278 | } |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 6279 | new_sl->next = env->explored_states[insn_idx]; |
| 6280 | env->explored_states[insn_idx] = new_sl; |
Jakub Kicinski | 7640ead | 2018-12-12 16:29:07 -0800 | [diff] [blame] | 6281 | /* connect new state to parentage chain. Current frame needs all |
| 6282 | * registers connected. Only r6 - r9 of the callers are alive (pushed |
| 6283 | * to the stack implicitly by JITs) so in callers' frames connect just |
| 6284 | * r6 - r9 as an optimization. Callers will have r1 - r5 connected to |
| 6285 | * the state of the call instruction (with WRITTEN set), and r0 comes |
| 6286 | * from callee with its full parentage chain, anyway. |
| 6287 | */ |
| 6288 | for (j = 0; j <= cur->curframe; j++) |
| 6289 | for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) |
| 6290 | cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i]; |
Edward Cree | 8e9cd9c | 2017-08-23 15:11:21 +0100 | [diff] [blame] | 6291 | /* clear write marks in current state: the writes we did are not writes |
| 6292 | * our child did, so they don't screen off its reads from us. |
| 6293 | * (There are no read marks in current state, because reads always mark |
| 6294 | * their parent and current state never has children yet. Only |
| 6295 | * explored_states can get read marks.) |
| 6296 | */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 6297 | for (i = 0; i < BPF_REG_FP; i++) |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 6298 | cur->frame[cur->curframe]->regs[i].live = REG_LIVE_NONE; |
| 6299 | |
| 6300 | /* all stack frames are accessible from callee, clear them all */ |
| 6301 | for (j = 0; j <= cur->curframe; j++) { |
| 6302 | struct bpf_func_state *frame = cur->frame[j]; |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 6303 | struct bpf_func_state *newframe = new->frame[j]; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 6304 | |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 6305 | for (i = 0; i < frame->allocated_stack / BPF_REG_SIZE; i++) { |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 6306 | frame->stack[i].spilled_ptr.live = REG_LIVE_NONE; |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 6307 | frame->stack[i].spilled_ptr.parent = |
| 6308 | &newframe->stack[i].spilled_ptr; |
| 6309 | } |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 6310 | } |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 6311 | return 0; |
| 6312 | } |
| 6313 | |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 6314 | /* Return true if it's OK to have the same insn return a different type. */ |
| 6315 | static bool reg_type_mismatch_ok(enum bpf_reg_type type) |
| 6316 | { |
| 6317 | switch (type) { |
| 6318 | case PTR_TO_CTX: |
| 6319 | case PTR_TO_SOCKET: |
| 6320 | case PTR_TO_SOCKET_OR_NULL: |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 6321 | case PTR_TO_SOCK_COMMON: |
| 6322 | case PTR_TO_SOCK_COMMON_OR_NULL: |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 6323 | case PTR_TO_TCP_SOCK: |
| 6324 | case PTR_TO_TCP_SOCK_OR_NULL: |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 6325 | return false; |
| 6326 | default: |
| 6327 | return true; |
| 6328 | } |
| 6329 | } |
| 6330 | |
| 6331 | /* If an instruction was previously used with particular pointer types, then we |
| 6332 | * need to be careful to avoid cases such as the below, where it may be ok |
| 6333 | * for one branch accessing the pointer, but not ok for the other branch: |
| 6334 | * |
| 6335 | * R1 = sock_ptr |
| 6336 | * goto X; |
| 6337 | * ... |
| 6338 | * R1 = some_other_valid_ptr; |
| 6339 | * goto X; |
| 6340 | * ... |
| 6341 | * R2 = *(u32 *)(R1 + 0); |
| 6342 | */ |
| 6343 | static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev) |
| 6344 | { |
| 6345 | return src != prev && (!reg_type_mismatch_ok(src) || |
| 6346 | !reg_type_mismatch_ok(prev)); |
| 6347 | } |
| 6348 | |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 6349 | static int do_check(struct bpf_verifier_env *env) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6350 | { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 6351 | struct bpf_verifier_state *state; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6352 | struct bpf_insn *insns = env->prog->insnsi; |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 6353 | struct bpf_reg_state *regs; |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 6354 | int insn_cnt = env->prog->len; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6355 | bool do_print_state = false; |
| 6356 | |
Martin KaFai Lau | d9762e8 | 2018-12-13 10:41:48 -0800 | [diff] [blame] | 6357 | env->prev_linfo = NULL; |
| 6358 | |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 6359 | state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL); |
| 6360 | if (!state) |
| 6361 | return -ENOMEM; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 6362 | state->curframe = 0; |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 6363 | state->speculative = false; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 6364 | state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL); |
| 6365 | if (!state->frame[0]) { |
| 6366 | kfree(state); |
| 6367 | return -ENOMEM; |
| 6368 | } |
| 6369 | env->cur_state = state; |
| 6370 | init_func_state(env, state->frame[0], |
| 6371 | BPF_MAIN_FUNC /* callsite */, |
| 6372 | 0 /* frameno */, |
| 6373 | 0 /* subprogno, zero == main subprog */); |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 6374 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6375 | for (;;) { |
| 6376 | struct bpf_insn *insn; |
| 6377 | u8 class; |
| 6378 | int err; |
| 6379 | |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 6380 | if (env->insn_idx >= insn_cnt) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6381 | verbose(env, "invalid insn idx %d insn_cnt %d\n", |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 6382 | env->insn_idx, insn_cnt); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6383 | return -EFAULT; |
| 6384 | } |
| 6385 | |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 6386 | insn = &insns[env->insn_idx]; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6387 | class = BPF_CLASS(insn->code); |
| 6388 | |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 6389 | if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6390 | verbose(env, |
| 6391 | "BPF program is too large. Processed %d insn\n", |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 6392 | env->insn_processed); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6393 | return -E2BIG; |
| 6394 | } |
| 6395 | |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 6396 | err = is_state_visited(env, env->insn_idx); |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 6397 | if (err < 0) |
| 6398 | return err; |
| 6399 | if (err == 1) { |
| 6400 | /* found equivalent state, can prune the search */ |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 6401 | if (env->log.level & BPF_LOG_LEVEL) { |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 6402 | if (do_print_state) |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 6403 | verbose(env, "\nfrom %d to %d%s: safe\n", |
| 6404 | env->prev_insn_idx, env->insn_idx, |
| 6405 | env->cur_state->speculative ? |
| 6406 | " (speculative execution)" : ""); |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 6407 | else |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 6408 | verbose(env, "%d: safe\n", env->insn_idx); |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 6409 | } |
| 6410 | goto process_bpf_exit; |
| 6411 | } |
| 6412 | |
Alexei Starovoitov | c349480 | 2018-12-03 22:46:04 -0800 | [diff] [blame] | 6413 | if (signal_pending(current)) |
| 6414 | return -EAGAIN; |
| 6415 | |
Daniel Borkmann | 3c2ce60 | 2017-05-18 03:00:06 +0200 | [diff] [blame] | 6416 | if (need_resched()) |
| 6417 | cond_resched(); |
| 6418 | |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 6419 | if (env->log.level & BPF_LOG_LEVEL2 || |
| 6420 | (env->log.level & BPF_LOG_LEVEL && do_print_state)) { |
| 6421 | if (env->log.level & BPF_LOG_LEVEL2) |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 6422 | verbose(env, "%d:", env->insn_idx); |
David S. Miller | c5fc969 | 2017-05-10 11:25:17 -0700 | [diff] [blame] | 6423 | else |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 6424 | verbose(env, "\nfrom %d to %d%s:", |
| 6425 | env->prev_insn_idx, env->insn_idx, |
| 6426 | env->cur_state->speculative ? |
| 6427 | " (speculative execution)" : ""); |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 6428 | print_verifier_state(env, state->frame[state->curframe]); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6429 | do_print_state = false; |
| 6430 | } |
| 6431 | |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 6432 | if (env->log.level & BPF_LOG_LEVEL) { |
Daniel Borkmann | 7105e82 | 2017-12-20 13:42:57 +0100 | [diff] [blame] | 6433 | const struct bpf_insn_cbs cbs = { |
| 6434 | .cb_print = verbose, |
Jiri Olsa | abe0884 | 2018-03-23 11:41:28 +0100 | [diff] [blame] | 6435 | .private_data = env, |
Daniel Borkmann | 7105e82 | 2017-12-20 13:42:57 +0100 | [diff] [blame] | 6436 | }; |
| 6437 | |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 6438 | verbose_linfo(env, env->insn_idx, "; "); |
| 6439 | verbose(env, "%d: ", env->insn_idx); |
Jiri Olsa | abe0884 | 2018-03-23 11:41:28 +0100 | [diff] [blame] | 6440 | print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6441 | } |
| 6442 | |
Jakub Kicinski | cae1927 | 2017-12-27 18:39:05 -0800 | [diff] [blame] | 6443 | if (bpf_prog_is_dev_bound(env->prog->aux)) { |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 6444 | err = bpf_prog_offload_verify_insn(env, env->insn_idx, |
| 6445 | env->prev_insn_idx); |
Jakub Kicinski | cae1927 | 2017-12-27 18:39:05 -0800 | [diff] [blame] | 6446 | if (err) |
| 6447 | return err; |
| 6448 | } |
Jakub Kicinski | 13a27df | 2016-09-21 11:43:58 +0100 | [diff] [blame] | 6449 | |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 6450 | regs = cur_regs(env); |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 6451 | env->insn_aux_data[env->insn_idx].seen = true; |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 6452 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6453 | if (class == BPF_ALU || class == BPF_ALU64) { |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 6454 | err = check_alu_op(env, insn); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6455 | if (err) |
| 6456 | return err; |
| 6457 | |
| 6458 | } else if (class == BPF_LDX) { |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 6459 | enum bpf_reg_type *prev_src_type, src_reg_type; |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 6460 | |
| 6461 | /* check for reserved fields is already done */ |
| 6462 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6463 | /* check src operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 6464 | err = check_reg_arg(env, insn->src_reg, SRC_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6465 | if (err) |
| 6466 | return err; |
| 6467 | |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 6468 | err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6469 | if (err) |
| 6470 | return err; |
| 6471 | |
Alexei Starovoitov | 725f9dc | 2015-04-15 16:19:33 -0700 | [diff] [blame] | 6472 | src_reg_type = regs[insn->src_reg].type; |
| 6473 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6474 | /* check that memory (src_reg + off) is readable, |
| 6475 | * the state of dst_reg will be updated by this func |
| 6476 | */ |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 6477 | err = check_mem_access(env, env->insn_idx, insn->src_reg, |
| 6478 | insn->off, BPF_SIZE(insn->code), |
| 6479 | BPF_READ, insn->dst_reg, false); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6480 | if (err) |
| 6481 | return err; |
| 6482 | |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 6483 | prev_src_type = &env->insn_aux_data[env->insn_idx].ptr_type; |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 6484 | |
| 6485 | if (*prev_src_type == NOT_INIT) { |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 6486 | /* saw a valid insn |
| 6487 | * dst_reg = *(u32 *)(src_reg + off) |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 6488 | * save type to validate intersecting paths |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 6489 | */ |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 6490 | *prev_src_type = src_reg_type; |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 6491 | |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 6492 | } else if (reg_type_mismatch(src_reg_type, *prev_src_type)) { |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 6493 | /* ABuser program is trying to use the same insn |
| 6494 | * dst_reg = *(u32*) (src_reg + off) |
| 6495 | * with different pointer types: |
| 6496 | * src_reg == ctx in one branch and |
| 6497 | * src_reg == stack|map in some other branch. |
| 6498 | * Reject it. |
| 6499 | */ |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6500 | verbose(env, "same insn cannot be used with different pointers\n"); |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 6501 | return -EINVAL; |
| 6502 | } |
| 6503 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6504 | } else if (class == BPF_STX) { |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 6505 | enum bpf_reg_type *prev_dst_type, dst_reg_type; |
Alexei Starovoitov | d691f9e | 2015-06-04 10:11:54 -0700 | [diff] [blame] | 6506 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6507 | if (BPF_MODE(insn->code) == BPF_XADD) { |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 6508 | err = check_xadd(env, env->insn_idx, insn); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6509 | if (err) |
| 6510 | return err; |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 6511 | env->insn_idx++; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6512 | continue; |
| 6513 | } |
| 6514 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6515 | /* check src1 operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 6516 | err = check_reg_arg(env, insn->src_reg, SRC_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6517 | if (err) |
| 6518 | return err; |
| 6519 | /* check src2 operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 6520 | err = check_reg_arg(env, insn->dst_reg, SRC_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6521 | if (err) |
| 6522 | return err; |
| 6523 | |
Alexei Starovoitov | d691f9e | 2015-06-04 10:11:54 -0700 | [diff] [blame] | 6524 | dst_reg_type = regs[insn->dst_reg].type; |
| 6525 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6526 | /* check that memory (dst_reg + off) is writeable */ |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 6527 | err = check_mem_access(env, env->insn_idx, insn->dst_reg, |
| 6528 | insn->off, BPF_SIZE(insn->code), |
| 6529 | BPF_WRITE, insn->src_reg, false); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6530 | if (err) |
| 6531 | return err; |
| 6532 | |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 6533 | prev_dst_type = &env->insn_aux_data[env->insn_idx].ptr_type; |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 6534 | |
| 6535 | if (*prev_dst_type == NOT_INIT) { |
| 6536 | *prev_dst_type = dst_reg_type; |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 6537 | } else if (reg_type_mismatch(dst_reg_type, *prev_dst_type)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6538 | verbose(env, "same insn cannot be used with different pointers\n"); |
Alexei Starovoitov | d691f9e | 2015-06-04 10:11:54 -0700 | [diff] [blame] | 6539 | return -EINVAL; |
| 6540 | } |
| 6541 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6542 | } else if (class == BPF_ST) { |
| 6543 | if (BPF_MODE(insn->code) != BPF_MEM || |
| 6544 | insn->src_reg != BPF_REG_0) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6545 | verbose(env, "BPF_ST uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6546 | return -EINVAL; |
| 6547 | } |
| 6548 | /* check src operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 6549 | err = check_reg_arg(env, insn->dst_reg, SRC_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6550 | if (err) |
| 6551 | return err; |
| 6552 | |
Daniel Borkmann | f37a8cb | 2018-01-16 23:30:10 +0100 | [diff] [blame] | 6553 | if (is_ctx_reg(env, insn->dst_reg)) { |
Joe Stringer | 9d2be44 | 2018-10-02 13:35:31 -0700 | [diff] [blame] | 6554 | verbose(env, "BPF_ST stores into R%d %s is not allowed\n", |
Daniel Borkmann | 2a159c6 | 2018-10-21 02:09:24 +0200 | [diff] [blame] | 6555 | insn->dst_reg, |
| 6556 | reg_type_str[reg_state(env, insn->dst_reg)->type]); |
Daniel Borkmann | f37a8cb | 2018-01-16 23:30:10 +0100 | [diff] [blame] | 6557 | return -EACCES; |
| 6558 | } |
| 6559 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6560 | /* check that memory (dst_reg + off) is writeable */ |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 6561 | err = check_mem_access(env, env->insn_idx, insn->dst_reg, |
| 6562 | insn->off, BPF_SIZE(insn->code), |
| 6563 | BPF_WRITE, -1, false); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6564 | if (err) |
| 6565 | return err; |
| 6566 | |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 6567 | } else if (class == BPF_JMP || class == BPF_JMP32) { |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6568 | u8 opcode = BPF_OP(insn->code); |
| 6569 | |
| 6570 | if (opcode == BPF_CALL) { |
| 6571 | if (BPF_SRC(insn->code) != BPF_K || |
| 6572 | insn->off != 0 || |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 6573 | (insn->src_reg != BPF_REG_0 && |
| 6574 | insn->src_reg != BPF_PSEUDO_CALL) || |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 6575 | insn->dst_reg != BPF_REG_0 || |
| 6576 | class == BPF_JMP32) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6577 | verbose(env, "BPF_CALL uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6578 | return -EINVAL; |
| 6579 | } |
| 6580 | |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 6581 | if (env->cur_state->active_spin_lock && |
| 6582 | (insn->src_reg == BPF_PSEUDO_CALL || |
| 6583 | insn->imm != BPF_FUNC_spin_unlock)) { |
| 6584 | verbose(env, "function calls are not allowed while holding a lock\n"); |
| 6585 | return -EINVAL; |
| 6586 | } |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 6587 | if (insn->src_reg == BPF_PSEUDO_CALL) |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 6588 | err = check_func_call(env, insn, &env->insn_idx); |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 6589 | else |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 6590 | err = check_helper_call(env, insn->imm, env->insn_idx); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6591 | if (err) |
| 6592 | return err; |
| 6593 | |
| 6594 | } else if (opcode == BPF_JA) { |
| 6595 | if (BPF_SRC(insn->code) != BPF_K || |
| 6596 | insn->imm != 0 || |
| 6597 | insn->src_reg != BPF_REG_0 || |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 6598 | insn->dst_reg != BPF_REG_0 || |
| 6599 | class == BPF_JMP32) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6600 | verbose(env, "BPF_JA uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6601 | return -EINVAL; |
| 6602 | } |
| 6603 | |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 6604 | env->insn_idx += insn->off + 1; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6605 | continue; |
| 6606 | |
| 6607 | } else if (opcode == BPF_EXIT) { |
| 6608 | if (BPF_SRC(insn->code) != BPF_K || |
| 6609 | insn->imm != 0 || |
| 6610 | insn->src_reg != BPF_REG_0 || |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 6611 | insn->dst_reg != BPF_REG_0 || |
| 6612 | class == BPF_JMP32) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6613 | verbose(env, "BPF_EXIT uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6614 | return -EINVAL; |
| 6615 | } |
| 6616 | |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 6617 | if (env->cur_state->active_spin_lock) { |
| 6618 | verbose(env, "bpf_spin_unlock is missing\n"); |
| 6619 | return -EINVAL; |
| 6620 | } |
| 6621 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 6622 | if (state->curframe) { |
| 6623 | /* exit from nested function */ |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 6624 | env->prev_insn_idx = env->insn_idx; |
| 6625 | err = prepare_func_exit(env, &env->insn_idx); |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 6626 | if (err) |
| 6627 | return err; |
| 6628 | do_print_state = true; |
| 6629 | continue; |
| 6630 | } |
| 6631 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 6632 | err = check_reference_leak(env); |
| 6633 | if (err) |
| 6634 | return err; |
| 6635 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6636 | /* eBPF calling convetion is such that R0 is used |
| 6637 | * to return the value from eBPF program. |
| 6638 | * Make sure that it's readable at this time |
| 6639 | * of bpf_exit, which means that program wrote |
| 6640 | * something into it earlier |
| 6641 | */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 6642 | err = check_reg_arg(env, BPF_REG_0, SRC_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6643 | if (err) |
| 6644 | return err; |
| 6645 | |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 6646 | if (is_pointer_value(env, BPF_REG_0)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6647 | verbose(env, "R0 leaks addr as return value\n"); |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 6648 | return -EACCES; |
| 6649 | } |
| 6650 | |
Alexei Starovoitov | 390ee7e | 2017-10-02 22:50:23 -0700 | [diff] [blame] | 6651 | err = check_return_code(env); |
| 6652 | if (err) |
| 6653 | return err; |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 6654 | process_bpf_exit: |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 6655 | err = pop_stack(env, &env->prev_insn_idx, |
| 6656 | &env->insn_idx); |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 6657 | if (err < 0) { |
| 6658 | if (err != -ENOENT) |
| 6659 | return err; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6660 | break; |
| 6661 | } else { |
| 6662 | do_print_state = true; |
| 6663 | continue; |
| 6664 | } |
| 6665 | } else { |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 6666 | err = check_cond_jmp_op(env, insn, &env->insn_idx); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6667 | if (err) |
| 6668 | return err; |
| 6669 | } |
| 6670 | } else if (class == BPF_LD) { |
| 6671 | u8 mode = BPF_MODE(insn->code); |
| 6672 | |
| 6673 | if (mode == BPF_ABS || mode == BPF_IND) { |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 6674 | err = check_ld_abs(env, insn); |
| 6675 | if (err) |
| 6676 | return err; |
| 6677 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6678 | } else if (mode == BPF_IMM) { |
| 6679 | err = check_ld_imm(env, insn); |
| 6680 | if (err) |
| 6681 | return err; |
| 6682 | |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 6683 | env->insn_idx++; |
| 6684 | env->insn_aux_data[env->insn_idx].seen = true; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6685 | } else { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6686 | verbose(env, "invalid BPF_LD mode\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6687 | return -EINVAL; |
| 6688 | } |
| 6689 | } else { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6690 | verbose(env, "unknown insn class %d\n", class); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6691 | return -EINVAL; |
| 6692 | } |
| 6693 | |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 6694 | env->insn_idx++; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6695 | } |
| 6696 | |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 6697 | env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6698 | return 0; |
| 6699 | } |
| 6700 | |
Martin KaFai Lau | 56f668d | 2017-03-22 10:00:33 -0700 | [diff] [blame] | 6701 | static int check_map_prealloc(struct bpf_map *map) |
| 6702 | { |
| 6703 | return (map->map_type != BPF_MAP_TYPE_HASH && |
Martin KaFai Lau | bcc6b1b | 2017-03-22 10:00:34 -0700 | [diff] [blame] | 6704 | map->map_type != BPF_MAP_TYPE_PERCPU_HASH && |
| 6705 | map->map_type != BPF_MAP_TYPE_HASH_OF_MAPS) || |
Martin KaFai Lau | 56f668d | 2017-03-22 10:00:33 -0700 | [diff] [blame] | 6706 | !(map->map_flags & BPF_F_NO_PREALLOC); |
| 6707 | } |
| 6708 | |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 6709 | static bool is_tracing_prog_type(enum bpf_prog_type type) |
| 6710 | { |
| 6711 | switch (type) { |
| 6712 | case BPF_PROG_TYPE_KPROBE: |
| 6713 | case BPF_PROG_TYPE_TRACEPOINT: |
| 6714 | case BPF_PROG_TYPE_PERF_EVENT: |
| 6715 | case BPF_PROG_TYPE_RAW_TRACEPOINT: |
| 6716 | return true; |
| 6717 | default: |
| 6718 | return false; |
| 6719 | } |
| 6720 | } |
| 6721 | |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6722 | static int check_map_prog_compatibility(struct bpf_verifier_env *env, |
| 6723 | struct bpf_map *map, |
Alexei Starovoitov | fdc15d3 | 2016-09-01 18:37:23 -0700 | [diff] [blame] | 6724 | struct bpf_prog *prog) |
| 6725 | |
| 6726 | { |
Martin KaFai Lau | 56f668d | 2017-03-22 10:00:33 -0700 | [diff] [blame] | 6727 | /* Make sure that BPF_PROG_TYPE_PERF_EVENT programs only use |
| 6728 | * preallocated hash maps, since doing memory allocation |
| 6729 | * in overflow_handler can crash depending on where nmi got |
| 6730 | * triggered. |
| 6731 | */ |
| 6732 | if (prog->type == BPF_PROG_TYPE_PERF_EVENT) { |
| 6733 | if (!check_map_prealloc(map)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6734 | verbose(env, "perf_event programs can only use preallocated hash map\n"); |
Martin KaFai Lau | 56f668d | 2017-03-22 10:00:33 -0700 | [diff] [blame] | 6735 | return -EINVAL; |
| 6736 | } |
| 6737 | if (map->inner_map_meta && |
| 6738 | !check_map_prealloc(map->inner_map_meta)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6739 | verbose(env, "perf_event programs can only use preallocated inner hash map\n"); |
Martin KaFai Lau | 56f668d | 2017-03-22 10:00:33 -0700 | [diff] [blame] | 6740 | return -EINVAL; |
| 6741 | } |
Alexei Starovoitov | fdc15d3 | 2016-09-01 18:37:23 -0700 | [diff] [blame] | 6742 | } |
Jakub Kicinski | a388457 | 2018-01-11 20:29:09 -0800 | [diff] [blame] | 6743 | |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 6744 | if ((is_tracing_prog_type(prog->type) || |
| 6745 | prog->type == BPF_PROG_TYPE_SOCKET_FILTER) && |
| 6746 | map_value_has_spin_lock(map)) { |
| 6747 | verbose(env, "tracing progs cannot use bpf_spin_lock yet\n"); |
| 6748 | return -EINVAL; |
| 6749 | } |
| 6750 | |
Jakub Kicinski | a388457 | 2018-01-11 20:29:09 -0800 | [diff] [blame] | 6751 | if ((bpf_prog_is_dev_bound(prog->aux) || bpf_map_is_dev_bound(map)) && |
Jakub Kicinski | 0972826 | 2018-07-17 10:53:23 -0700 | [diff] [blame] | 6752 | !bpf_offload_prog_map_match(prog, map)) { |
Jakub Kicinski | a388457 | 2018-01-11 20:29:09 -0800 | [diff] [blame] | 6753 | verbose(env, "offload device mismatch between prog and map\n"); |
| 6754 | return -EINVAL; |
| 6755 | } |
| 6756 | |
Alexei Starovoitov | fdc15d3 | 2016-09-01 18:37:23 -0700 | [diff] [blame] | 6757 | return 0; |
| 6758 | } |
| 6759 | |
Roman Gushchin | b741f16 | 2018-09-28 14:45:43 +0000 | [diff] [blame] | 6760 | static bool bpf_map_is_cgroup_storage(struct bpf_map *map) |
| 6761 | { |
| 6762 | return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE || |
| 6763 | map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE); |
| 6764 | } |
| 6765 | |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 6766 | /* look for pseudo eBPF instructions that access map FDs and |
| 6767 | * replace them with actual map pointers |
| 6768 | */ |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 6769 | static int replace_map_fd_with_map_ptr(struct bpf_verifier_env *env) |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 6770 | { |
| 6771 | struct bpf_insn *insn = env->prog->insnsi; |
| 6772 | int insn_cnt = env->prog->len; |
Alexei Starovoitov | fdc15d3 | 2016-09-01 18:37:23 -0700 | [diff] [blame] | 6773 | int i, j, err; |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 6774 | |
Daniel Borkmann | f1f7714 | 2017-01-13 23:38:15 +0100 | [diff] [blame] | 6775 | err = bpf_prog_calc_tag(env->prog); |
Daniel Borkmann | aafe6ae | 2016-12-18 01:52:57 +0100 | [diff] [blame] | 6776 | if (err) |
| 6777 | return err; |
| 6778 | |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 6779 | for (i = 0; i < insn_cnt; i++, insn++) { |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 6780 | if (BPF_CLASS(insn->code) == BPF_LDX && |
Alexei Starovoitov | d691f9e | 2015-06-04 10:11:54 -0700 | [diff] [blame] | 6781 | (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6782 | verbose(env, "BPF_LDX uses reserved fields\n"); |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 6783 | return -EINVAL; |
| 6784 | } |
| 6785 | |
Alexei Starovoitov | d691f9e | 2015-06-04 10:11:54 -0700 | [diff] [blame] | 6786 | if (BPF_CLASS(insn->code) == BPF_STX && |
| 6787 | ((BPF_MODE(insn->code) != BPF_MEM && |
| 6788 | BPF_MODE(insn->code) != BPF_XADD) || insn->imm != 0)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6789 | verbose(env, "BPF_STX uses reserved fields\n"); |
Alexei Starovoitov | d691f9e | 2015-06-04 10:11:54 -0700 | [diff] [blame] | 6790 | return -EINVAL; |
| 6791 | } |
| 6792 | |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 6793 | if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) { |
| 6794 | struct bpf_map *map; |
| 6795 | struct fd f; |
| 6796 | |
| 6797 | if (i == insn_cnt - 1 || insn[1].code != 0 || |
| 6798 | insn[1].dst_reg != 0 || insn[1].src_reg != 0 || |
| 6799 | insn[1].off != 0) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6800 | verbose(env, "invalid bpf_ld_imm64 insn\n"); |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 6801 | return -EINVAL; |
| 6802 | } |
| 6803 | |
| 6804 | if (insn->src_reg == 0) |
| 6805 | /* valid generic load 64-bit imm */ |
| 6806 | goto next_insn; |
| 6807 | |
Daniel Borkmann | 2018239 | 2019-03-04 21:08:53 +0100 | [diff] [blame] | 6808 | if (insn[0].src_reg != BPF_PSEUDO_MAP_FD || |
| 6809 | insn[1].imm != 0) { |
| 6810 | verbose(env, "unrecognized bpf_ld_imm64 insn\n"); |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 6811 | return -EINVAL; |
| 6812 | } |
| 6813 | |
Daniel Borkmann | 2018239 | 2019-03-04 21:08:53 +0100 | [diff] [blame] | 6814 | f = fdget(insn[0].imm); |
Daniel Borkmann | c210129 | 2015-10-29 14:58:07 +0100 | [diff] [blame] | 6815 | map = __bpf_map_get(f); |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 6816 | if (IS_ERR(map)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6817 | verbose(env, "fd %d is not pointing to valid bpf_map\n", |
Daniel Borkmann | 2018239 | 2019-03-04 21:08:53 +0100 | [diff] [blame] | 6818 | insn[0].imm); |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 6819 | return PTR_ERR(map); |
| 6820 | } |
| 6821 | |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6822 | err = check_map_prog_compatibility(env, map, env->prog); |
Alexei Starovoitov | fdc15d3 | 2016-09-01 18:37:23 -0700 | [diff] [blame] | 6823 | if (err) { |
| 6824 | fdput(f); |
| 6825 | return err; |
| 6826 | } |
| 6827 | |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 6828 | /* store map pointer inside BPF_LD_IMM64 instruction */ |
| 6829 | insn[0].imm = (u32) (unsigned long) map; |
| 6830 | insn[1].imm = ((u64) (unsigned long) map) >> 32; |
| 6831 | |
| 6832 | /* check whether we recorded this map already */ |
| 6833 | for (j = 0; j < env->used_map_cnt; j++) |
| 6834 | if (env->used_maps[j] == map) { |
| 6835 | fdput(f); |
| 6836 | goto next_insn; |
| 6837 | } |
| 6838 | |
| 6839 | if (env->used_map_cnt >= MAX_USED_MAPS) { |
| 6840 | fdput(f); |
| 6841 | return -E2BIG; |
| 6842 | } |
| 6843 | |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 6844 | /* hold the map. If the program is rejected by verifier, |
| 6845 | * the map will be released by release_maps() or it |
| 6846 | * will be used by the valid program until it's unloaded |
Jakub Kicinski | ab7f5bf | 2018-05-03 18:37:17 -0700 | [diff] [blame] | 6847 | * and all maps are released in free_used_maps() |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 6848 | */ |
Alexei Starovoitov | 92117d8 | 2016-04-27 18:56:20 -0700 | [diff] [blame] | 6849 | map = bpf_map_inc(map, false); |
| 6850 | if (IS_ERR(map)) { |
| 6851 | fdput(f); |
| 6852 | return PTR_ERR(map); |
| 6853 | } |
| 6854 | env->used_maps[env->used_map_cnt++] = map; |
| 6855 | |
Roman Gushchin | b741f16 | 2018-09-28 14:45:43 +0000 | [diff] [blame] | 6856 | if (bpf_map_is_cgroup_storage(map) && |
Roman Gushchin | de9cbba | 2018-08-02 14:27:18 -0700 | [diff] [blame] | 6857 | bpf_cgroup_storage_assign(env->prog, map)) { |
Roman Gushchin | b741f16 | 2018-09-28 14:45:43 +0000 | [diff] [blame] | 6858 | verbose(env, "only one cgroup storage of each type is allowed\n"); |
Roman Gushchin | de9cbba | 2018-08-02 14:27:18 -0700 | [diff] [blame] | 6859 | fdput(f); |
| 6860 | return -EBUSY; |
| 6861 | } |
| 6862 | |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 6863 | fdput(f); |
| 6864 | next_insn: |
| 6865 | insn++; |
| 6866 | i++; |
Daniel Borkmann | 5e581da | 2018-01-26 23:33:38 +0100 | [diff] [blame] | 6867 | continue; |
| 6868 | } |
| 6869 | |
| 6870 | /* Basic sanity check before we invest more work here. */ |
| 6871 | if (!bpf_opcode_in_insntable(insn->code)) { |
| 6872 | verbose(env, "unknown opcode %02x\n", insn->code); |
| 6873 | return -EINVAL; |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 6874 | } |
| 6875 | } |
| 6876 | |
| 6877 | /* now all pseudo BPF_LD_IMM64 instructions load valid |
| 6878 | * 'struct bpf_map *' into a register instead of user map_fd. |
| 6879 | * These pointers will be used later by verifier to validate map access. |
| 6880 | */ |
| 6881 | return 0; |
| 6882 | } |
| 6883 | |
| 6884 | /* drop refcnt of maps used by the rejected program */ |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 6885 | static void release_maps(struct bpf_verifier_env *env) |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 6886 | { |
Roman Gushchin | 8bad74f | 2018-09-28 14:45:36 +0000 | [diff] [blame] | 6887 | enum bpf_cgroup_storage_type stype; |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 6888 | int i; |
| 6889 | |
Roman Gushchin | 8bad74f | 2018-09-28 14:45:36 +0000 | [diff] [blame] | 6890 | for_each_cgroup_storage_type(stype) { |
| 6891 | if (!env->prog->aux->cgroup_storage[stype]) |
| 6892 | continue; |
Roman Gushchin | de9cbba | 2018-08-02 14:27:18 -0700 | [diff] [blame] | 6893 | bpf_cgroup_storage_release(env->prog, |
Roman Gushchin | 8bad74f | 2018-09-28 14:45:36 +0000 | [diff] [blame] | 6894 | env->prog->aux->cgroup_storage[stype]); |
| 6895 | } |
Roman Gushchin | de9cbba | 2018-08-02 14:27:18 -0700 | [diff] [blame] | 6896 | |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 6897 | for (i = 0; i < env->used_map_cnt; i++) |
| 6898 | bpf_map_put(env->used_maps[i]); |
| 6899 | } |
| 6900 | |
| 6901 | /* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */ |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 6902 | static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env) |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 6903 | { |
| 6904 | struct bpf_insn *insn = env->prog->insnsi; |
| 6905 | int insn_cnt = env->prog->len; |
| 6906 | int i; |
| 6907 | |
| 6908 | for (i = 0; i < insn_cnt; i++, insn++) |
| 6909 | if (insn->code == (BPF_LD | BPF_IMM | BPF_DW)) |
| 6910 | insn->src_reg = 0; |
| 6911 | } |
| 6912 | |
Alexei Starovoitov | 8041902 | 2017-03-15 18:26:41 -0700 | [diff] [blame] | 6913 | /* single env->prog->insni[off] instruction was replaced with the range |
| 6914 | * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying |
| 6915 | * [0, off) and [off, end) to new locations, so the patched range stays zero |
| 6916 | */ |
| 6917 | static int adjust_insn_aux_data(struct bpf_verifier_env *env, u32 prog_len, |
| 6918 | u32 off, u32 cnt) |
| 6919 | { |
| 6920 | struct bpf_insn_aux_data *new_data, *old_data = env->insn_aux_data; |
Alexei Starovoitov | c131187 | 2017-11-22 16:42:05 -0800 | [diff] [blame] | 6921 | int i; |
Alexei Starovoitov | 8041902 | 2017-03-15 18:26:41 -0700 | [diff] [blame] | 6922 | |
| 6923 | if (cnt == 1) |
| 6924 | return 0; |
Kees Cook | fad953c | 2018-06-12 14:27:37 -0700 | [diff] [blame] | 6925 | new_data = vzalloc(array_size(prog_len, |
| 6926 | sizeof(struct bpf_insn_aux_data))); |
Alexei Starovoitov | 8041902 | 2017-03-15 18:26:41 -0700 | [diff] [blame] | 6927 | if (!new_data) |
| 6928 | return -ENOMEM; |
| 6929 | memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off); |
| 6930 | memcpy(new_data + off + cnt - 1, old_data + off, |
| 6931 | sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); |
Alexei Starovoitov | c131187 | 2017-11-22 16:42:05 -0800 | [diff] [blame] | 6932 | for (i = off; i < off + cnt - 1; i++) |
| 6933 | new_data[i].seen = true; |
Alexei Starovoitov | 8041902 | 2017-03-15 18:26:41 -0700 | [diff] [blame] | 6934 | env->insn_aux_data = new_data; |
| 6935 | vfree(old_data); |
| 6936 | return 0; |
| 6937 | } |
| 6938 | |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 6939 | static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) |
| 6940 | { |
| 6941 | int i; |
| 6942 | |
| 6943 | if (len == 1) |
| 6944 | return; |
Jiong Wang | 4cb3d99 | 2018-05-02 16:17:19 -0400 | [diff] [blame] | 6945 | /* NOTE: fake 'exit' subprog should be updated as well. */ |
| 6946 | for (i = 0; i <= env->subprog_cnt; i++) { |
Edward Cree | afd5942 | 2018-11-16 12:00:07 +0000 | [diff] [blame] | 6947 | if (env->subprog_info[i].start <= off) |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 6948 | continue; |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 6949 | env->subprog_info[i].start += len - 1; |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 6950 | } |
| 6951 | } |
| 6952 | |
Alexei Starovoitov | 8041902 | 2017-03-15 18:26:41 -0700 | [diff] [blame] | 6953 | static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, |
| 6954 | const struct bpf_insn *patch, u32 len) |
| 6955 | { |
| 6956 | struct bpf_prog *new_prog; |
| 6957 | |
| 6958 | new_prog = bpf_patch_insn_single(env->prog, off, patch, len); |
Alexei Starovoitov | 4f73379 | 2019-04-01 21:27:44 -0700 | [diff] [blame] | 6959 | if (IS_ERR(new_prog)) { |
| 6960 | if (PTR_ERR(new_prog) == -ERANGE) |
| 6961 | verbose(env, |
| 6962 | "insn %d cannot be patched due to 16-bit range\n", |
| 6963 | env->insn_aux_data[off].orig_idx); |
Alexei Starovoitov | 8041902 | 2017-03-15 18:26:41 -0700 | [diff] [blame] | 6964 | return NULL; |
Alexei Starovoitov | 4f73379 | 2019-04-01 21:27:44 -0700 | [diff] [blame] | 6965 | } |
Alexei Starovoitov | 8041902 | 2017-03-15 18:26:41 -0700 | [diff] [blame] | 6966 | if (adjust_insn_aux_data(env, new_prog->len, off, len)) |
| 6967 | return NULL; |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 6968 | adjust_subprog_starts(env, off, len); |
Alexei Starovoitov | 8041902 | 2017-03-15 18:26:41 -0700 | [diff] [blame] | 6969 | return new_prog; |
| 6970 | } |
| 6971 | |
Jakub Kicinski | 52875a0 | 2019-01-22 22:45:20 -0800 | [diff] [blame] | 6972 | static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, |
| 6973 | u32 off, u32 cnt) |
| 6974 | { |
| 6975 | int i, j; |
| 6976 | |
| 6977 | /* find first prog starting at or after off (first to remove) */ |
| 6978 | for (i = 0; i < env->subprog_cnt; i++) |
| 6979 | if (env->subprog_info[i].start >= off) |
| 6980 | break; |
| 6981 | /* find first prog starting at or after off + cnt (first to stay) */ |
| 6982 | for (j = i; j < env->subprog_cnt; j++) |
| 6983 | if (env->subprog_info[j].start >= off + cnt) |
| 6984 | break; |
| 6985 | /* if j doesn't start exactly at off + cnt, we are just removing |
| 6986 | * the front of previous prog |
| 6987 | */ |
| 6988 | if (env->subprog_info[j].start != off + cnt) |
| 6989 | j--; |
| 6990 | |
| 6991 | if (j > i) { |
| 6992 | struct bpf_prog_aux *aux = env->prog->aux; |
| 6993 | int move; |
| 6994 | |
| 6995 | /* move fake 'exit' subprog as well */ |
| 6996 | move = env->subprog_cnt + 1 - j; |
| 6997 | |
| 6998 | memmove(env->subprog_info + i, |
| 6999 | env->subprog_info + j, |
| 7000 | sizeof(*env->subprog_info) * move); |
| 7001 | env->subprog_cnt -= j - i; |
| 7002 | |
| 7003 | /* remove func_info */ |
| 7004 | if (aux->func_info) { |
| 7005 | move = aux->func_info_cnt - j; |
| 7006 | |
| 7007 | memmove(aux->func_info + i, |
| 7008 | aux->func_info + j, |
| 7009 | sizeof(*aux->func_info) * move); |
| 7010 | aux->func_info_cnt -= j - i; |
| 7011 | /* func_info->insn_off is set after all code rewrites, |
| 7012 | * in adjust_btf_func() - no need to adjust |
| 7013 | */ |
| 7014 | } |
| 7015 | } else { |
| 7016 | /* convert i from "first prog to remove" to "first to adjust" */ |
| 7017 | if (env->subprog_info[i].start == off) |
| 7018 | i++; |
| 7019 | } |
| 7020 | |
| 7021 | /* update fake 'exit' subprog as well */ |
| 7022 | for (; i <= env->subprog_cnt; i++) |
| 7023 | env->subprog_info[i].start -= cnt; |
| 7024 | |
| 7025 | return 0; |
| 7026 | } |
| 7027 | |
| 7028 | static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, |
| 7029 | u32 cnt) |
| 7030 | { |
| 7031 | struct bpf_prog *prog = env->prog; |
| 7032 | u32 i, l_off, l_cnt, nr_linfo; |
| 7033 | struct bpf_line_info *linfo; |
| 7034 | |
| 7035 | nr_linfo = prog->aux->nr_linfo; |
| 7036 | if (!nr_linfo) |
| 7037 | return 0; |
| 7038 | |
| 7039 | linfo = prog->aux->linfo; |
| 7040 | |
| 7041 | /* find first line info to remove, count lines to be removed */ |
| 7042 | for (i = 0; i < nr_linfo; i++) |
| 7043 | if (linfo[i].insn_off >= off) |
| 7044 | break; |
| 7045 | |
| 7046 | l_off = i; |
| 7047 | l_cnt = 0; |
| 7048 | for (; i < nr_linfo; i++) |
| 7049 | if (linfo[i].insn_off < off + cnt) |
| 7050 | l_cnt++; |
| 7051 | else |
| 7052 | break; |
| 7053 | |
| 7054 | /* First live insn doesn't match first live linfo, it needs to "inherit" |
| 7055 | * last removed linfo. prog is already modified, so prog->len == off |
| 7056 | * means no live instructions after (tail of the program was removed). |
| 7057 | */ |
| 7058 | if (prog->len != off && l_cnt && |
| 7059 | (i == nr_linfo || linfo[i].insn_off != off + cnt)) { |
| 7060 | l_cnt--; |
| 7061 | linfo[--i].insn_off = off + cnt; |
| 7062 | } |
| 7063 | |
| 7064 | /* remove the line info which refer to the removed instructions */ |
| 7065 | if (l_cnt) { |
| 7066 | memmove(linfo + l_off, linfo + i, |
| 7067 | sizeof(*linfo) * (nr_linfo - i)); |
| 7068 | |
| 7069 | prog->aux->nr_linfo -= l_cnt; |
| 7070 | nr_linfo = prog->aux->nr_linfo; |
| 7071 | } |
| 7072 | |
| 7073 | /* pull all linfo[i].insn_off >= off + cnt in by cnt */ |
| 7074 | for (i = l_off; i < nr_linfo; i++) |
| 7075 | linfo[i].insn_off -= cnt; |
| 7076 | |
| 7077 | /* fix up all subprogs (incl. 'exit') which start >= off */ |
| 7078 | for (i = 0; i <= env->subprog_cnt; i++) |
| 7079 | if (env->subprog_info[i].linfo_idx > l_off) { |
| 7080 | /* program may have started in the removed region but |
| 7081 | * may not be fully removed |
| 7082 | */ |
| 7083 | if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) |
| 7084 | env->subprog_info[i].linfo_idx -= l_cnt; |
| 7085 | else |
| 7086 | env->subprog_info[i].linfo_idx = l_off; |
| 7087 | } |
| 7088 | |
| 7089 | return 0; |
| 7090 | } |
| 7091 | |
| 7092 | static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) |
| 7093 | { |
| 7094 | struct bpf_insn_aux_data *aux_data = env->insn_aux_data; |
| 7095 | unsigned int orig_prog_len = env->prog->len; |
| 7096 | int err; |
| 7097 | |
Jakub Kicinski | 08ca90a | 2019-01-22 22:45:24 -0800 | [diff] [blame] | 7098 | if (bpf_prog_is_dev_bound(env->prog->aux)) |
| 7099 | bpf_prog_offload_remove_insns(env, off, cnt); |
| 7100 | |
Jakub Kicinski | 52875a0 | 2019-01-22 22:45:20 -0800 | [diff] [blame] | 7101 | err = bpf_remove_insns(env->prog, off, cnt); |
| 7102 | if (err) |
| 7103 | return err; |
| 7104 | |
| 7105 | err = adjust_subprog_starts_after_remove(env, off, cnt); |
| 7106 | if (err) |
| 7107 | return err; |
| 7108 | |
| 7109 | err = bpf_adj_linfo_after_remove(env, off, cnt); |
| 7110 | if (err) |
| 7111 | return err; |
| 7112 | |
| 7113 | memmove(aux_data + off, aux_data + off + cnt, |
| 7114 | sizeof(*aux_data) * (orig_prog_len - off - cnt)); |
| 7115 | |
| 7116 | return 0; |
| 7117 | } |
| 7118 | |
Daniel Borkmann | 2a5418a | 2018-01-26 23:33:37 +0100 | [diff] [blame] | 7119 | /* The verifier does more data flow analysis than llvm and will not |
| 7120 | * explore branches that are dead at run time. Malicious programs can |
| 7121 | * have dead code too. Therefore replace all dead at-run-time code |
| 7122 | * with 'ja -1'. |
| 7123 | * |
| 7124 | * Just nops are not optimal, e.g. if they would sit at the end of the |
| 7125 | * program and through another bug we would manage to jump there, then |
| 7126 | * we'd execute beyond program memory otherwise. Returning exception |
| 7127 | * code also wouldn't work since we can have subprogs where the dead |
| 7128 | * code could be located. |
Alexei Starovoitov | c131187 | 2017-11-22 16:42:05 -0800 | [diff] [blame] | 7129 | */ |
| 7130 | static void sanitize_dead_code(struct bpf_verifier_env *env) |
| 7131 | { |
| 7132 | struct bpf_insn_aux_data *aux_data = env->insn_aux_data; |
Daniel Borkmann | 2a5418a | 2018-01-26 23:33:37 +0100 | [diff] [blame] | 7133 | struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); |
Alexei Starovoitov | c131187 | 2017-11-22 16:42:05 -0800 | [diff] [blame] | 7134 | struct bpf_insn *insn = env->prog->insnsi; |
| 7135 | const int insn_cnt = env->prog->len; |
| 7136 | int i; |
| 7137 | |
| 7138 | for (i = 0; i < insn_cnt; i++) { |
| 7139 | if (aux_data[i].seen) |
| 7140 | continue; |
Daniel Borkmann | 2a5418a | 2018-01-26 23:33:37 +0100 | [diff] [blame] | 7141 | memcpy(insn + i, &trap, sizeof(trap)); |
Alexei Starovoitov | c131187 | 2017-11-22 16:42:05 -0800 | [diff] [blame] | 7142 | } |
| 7143 | } |
| 7144 | |
Jakub Kicinski | e2ae4ca | 2019-01-22 22:45:19 -0800 | [diff] [blame] | 7145 | static bool insn_is_cond_jump(u8 code) |
| 7146 | { |
| 7147 | u8 op; |
| 7148 | |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 7149 | if (BPF_CLASS(code) == BPF_JMP32) |
| 7150 | return true; |
| 7151 | |
Jakub Kicinski | e2ae4ca | 2019-01-22 22:45:19 -0800 | [diff] [blame] | 7152 | if (BPF_CLASS(code) != BPF_JMP) |
| 7153 | return false; |
| 7154 | |
| 7155 | op = BPF_OP(code); |
| 7156 | return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL; |
| 7157 | } |
| 7158 | |
| 7159 | static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) |
| 7160 | { |
| 7161 | struct bpf_insn_aux_data *aux_data = env->insn_aux_data; |
| 7162 | struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); |
| 7163 | struct bpf_insn *insn = env->prog->insnsi; |
| 7164 | const int insn_cnt = env->prog->len; |
| 7165 | int i; |
| 7166 | |
| 7167 | for (i = 0; i < insn_cnt; i++, insn++) { |
| 7168 | if (!insn_is_cond_jump(insn->code)) |
| 7169 | continue; |
| 7170 | |
| 7171 | if (!aux_data[i + 1].seen) |
| 7172 | ja.off = insn->off; |
| 7173 | else if (!aux_data[i + 1 + insn->off].seen) |
| 7174 | ja.off = 0; |
| 7175 | else |
| 7176 | continue; |
| 7177 | |
Jakub Kicinski | 08ca90a | 2019-01-22 22:45:24 -0800 | [diff] [blame] | 7178 | if (bpf_prog_is_dev_bound(env->prog->aux)) |
| 7179 | bpf_prog_offload_replace_insn(env, i, &ja); |
| 7180 | |
Jakub Kicinski | e2ae4ca | 2019-01-22 22:45:19 -0800 | [diff] [blame] | 7181 | memcpy(insn, &ja, sizeof(ja)); |
| 7182 | } |
| 7183 | } |
| 7184 | |
Jakub Kicinski | 52875a0 | 2019-01-22 22:45:20 -0800 | [diff] [blame] | 7185 | static int opt_remove_dead_code(struct bpf_verifier_env *env) |
| 7186 | { |
| 7187 | struct bpf_insn_aux_data *aux_data = env->insn_aux_data; |
| 7188 | int insn_cnt = env->prog->len; |
| 7189 | int i, err; |
| 7190 | |
| 7191 | for (i = 0; i < insn_cnt; i++) { |
| 7192 | int j; |
| 7193 | |
| 7194 | j = 0; |
| 7195 | while (i + j < insn_cnt && !aux_data[i + j].seen) |
| 7196 | j++; |
| 7197 | if (!j) |
| 7198 | continue; |
| 7199 | |
| 7200 | err = verifier_remove_insns(env, i, j); |
| 7201 | if (err) |
| 7202 | return err; |
| 7203 | insn_cnt = env->prog->len; |
| 7204 | } |
| 7205 | |
| 7206 | return 0; |
| 7207 | } |
| 7208 | |
Jakub Kicinski | a1b14ab | 2019-01-22 22:45:21 -0800 | [diff] [blame] | 7209 | static int opt_remove_nops(struct bpf_verifier_env *env) |
| 7210 | { |
| 7211 | const struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); |
| 7212 | struct bpf_insn *insn = env->prog->insnsi; |
| 7213 | int insn_cnt = env->prog->len; |
| 7214 | int i, err; |
| 7215 | |
| 7216 | for (i = 0; i < insn_cnt; i++) { |
| 7217 | if (memcmp(&insn[i], &ja, sizeof(ja))) |
| 7218 | continue; |
| 7219 | |
| 7220 | err = verifier_remove_insns(env, i, 1); |
| 7221 | if (err) |
| 7222 | return err; |
| 7223 | insn_cnt--; |
| 7224 | i--; |
| 7225 | } |
| 7226 | |
| 7227 | return 0; |
| 7228 | } |
| 7229 | |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 7230 | /* convert load instructions that access fields of a context type into a |
| 7231 | * sequence of instructions that access fields of the underlying structure: |
| 7232 | * struct __sk_buff -> struct sk_buff |
| 7233 | * struct bpf_sock_ops -> struct sock |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 7234 | */ |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 7235 | static int convert_ctx_accesses(struct bpf_verifier_env *env) |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 7236 | { |
Jakub Kicinski | 00176a3 | 2017-10-16 16:40:54 -0700 | [diff] [blame] | 7237 | const struct bpf_verifier_ops *ops = env->ops; |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 7238 | int i, cnt, size, ctx_field_size, delta = 0; |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 7239 | const int insn_cnt = env->prog->len; |
Daniel Borkmann | 36bbef5 | 2016-09-20 00:26:13 +0200 | [diff] [blame] | 7240 | struct bpf_insn insn_buf[16], *insn; |
Andrey Ignatov | 46f53a6 | 2018-11-10 22:15:13 -0800 | [diff] [blame] | 7241 | u32 target_size, size_default, off; |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 7242 | struct bpf_prog *new_prog; |
Alexei Starovoitov | d691f9e | 2015-06-04 10:11:54 -0700 | [diff] [blame] | 7243 | enum bpf_access_type type; |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 7244 | bool is_narrower_load; |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 7245 | |
Daniel Borkmann | b09928b | 2018-10-24 22:05:49 +0200 | [diff] [blame] | 7246 | if (ops->gen_prologue || env->seen_direct_write) { |
| 7247 | if (!ops->gen_prologue) { |
| 7248 | verbose(env, "bpf verifier is misconfigured\n"); |
| 7249 | return -EINVAL; |
| 7250 | } |
Daniel Borkmann | 36bbef5 | 2016-09-20 00:26:13 +0200 | [diff] [blame] | 7251 | cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, |
| 7252 | env->prog); |
| 7253 | if (cnt >= ARRAY_SIZE(insn_buf)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 7254 | verbose(env, "bpf verifier is misconfigured\n"); |
Daniel Borkmann | 36bbef5 | 2016-09-20 00:26:13 +0200 | [diff] [blame] | 7255 | return -EINVAL; |
| 7256 | } else if (cnt) { |
Alexei Starovoitov | 8041902 | 2017-03-15 18:26:41 -0700 | [diff] [blame] | 7257 | new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); |
Daniel Borkmann | 36bbef5 | 2016-09-20 00:26:13 +0200 | [diff] [blame] | 7258 | if (!new_prog) |
| 7259 | return -ENOMEM; |
Alexei Starovoitov | 8041902 | 2017-03-15 18:26:41 -0700 | [diff] [blame] | 7260 | |
Daniel Borkmann | 36bbef5 | 2016-09-20 00:26:13 +0200 | [diff] [blame] | 7261 | env->prog = new_prog; |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 7262 | delta += cnt - 1; |
Daniel Borkmann | 36bbef5 | 2016-09-20 00:26:13 +0200 | [diff] [blame] | 7263 | } |
| 7264 | } |
| 7265 | |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 7266 | if (bpf_prog_is_dev_bound(env->prog->aux)) |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 7267 | return 0; |
| 7268 | |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 7269 | insn = env->prog->insnsi + delta; |
Daniel Borkmann | 36bbef5 | 2016-09-20 00:26:13 +0200 | [diff] [blame] | 7270 | |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 7271 | for (i = 0; i < insn_cnt; i++, insn++) { |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 7272 | bpf_convert_ctx_access_t convert_ctx_access; |
| 7273 | |
Daniel Borkmann | 62c7989 | 2017-01-12 11:51:33 +0100 | [diff] [blame] | 7274 | if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) || |
| 7275 | insn->code == (BPF_LDX | BPF_MEM | BPF_H) || |
| 7276 | insn->code == (BPF_LDX | BPF_MEM | BPF_W) || |
Alexei Starovoitov | ea2e7ce | 2016-09-01 18:37:21 -0700 | [diff] [blame] | 7277 | insn->code == (BPF_LDX | BPF_MEM | BPF_DW)) |
Alexei Starovoitov | d691f9e | 2015-06-04 10:11:54 -0700 | [diff] [blame] | 7278 | type = BPF_READ; |
Daniel Borkmann | 62c7989 | 2017-01-12 11:51:33 +0100 | [diff] [blame] | 7279 | else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) || |
| 7280 | insn->code == (BPF_STX | BPF_MEM | BPF_H) || |
| 7281 | insn->code == (BPF_STX | BPF_MEM | BPF_W) || |
Alexei Starovoitov | ea2e7ce | 2016-09-01 18:37:21 -0700 | [diff] [blame] | 7282 | insn->code == (BPF_STX | BPF_MEM | BPF_DW)) |
Alexei Starovoitov | d691f9e | 2015-06-04 10:11:54 -0700 | [diff] [blame] | 7283 | type = BPF_WRITE; |
| 7284 | else |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 7285 | continue; |
| 7286 | |
Alexei Starovoitov | af86ca4 | 2018-05-15 09:27:05 -0700 | [diff] [blame] | 7287 | if (type == BPF_WRITE && |
| 7288 | env->insn_aux_data[i + delta].sanitize_stack_off) { |
| 7289 | struct bpf_insn patch[] = { |
| 7290 | /* Sanitize suspicious stack slot with zero. |
| 7291 | * There are no memory dependencies for this store, |
| 7292 | * since it's only using frame pointer and immediate |
| 7293 | * constant of zero |
| 7294 | */ |
| 7295 | BPF_ST_MEM(BPF_DW, BPF_REG_FP, |
| 7296 | env->insn_aux_data[i + delta].sanitize_stack_off, |
| 7297 | 0), |
| 7298 | /* the original STX instruction will immediately |
| 7299 | * overwrite the same stack slot with appropriate value |
| 7300 | */ |
| 7301 | *insn, |
| 7302 | }; |
| 7303 | |
| 7304 | cnt = ARRAY_SIZE(patch); |
| 7305 | new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt); |
| 7306 | if (!new_prog) |
| 7307 | return -ENOMEM; |
| 7308 | |
| 7309 | delta += cnt - 1; |
| 7310 | env->prog = new_prog; |
| 7311 | insn = new_prog->insnsi + i + delta; |
| 7312 | continue; |
| 7313 | } |
| 7314 | |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 7315 | switch (env->insn_aux_data[i + delta].ptr_type) { |
| 7316 | case PTR_TO_CTX: |
| 7317 | if (!ops->convert_ctx_access) |
| 7318 | continue; |
| 7319 | convert_ctx_access = ops->convert_ctx_access; |
| 7320 | break; |
| 7321 | case PTR_TO_SOCKET: |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 7322 | case PTR_TO_SOCK_COMMON: |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 7323 | convert_ctx_access = bpf_sock_convert_ctx_access; |
| 7324 | break; |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 7325 | case PTR_TO_TCP_SOCK: |
| 7326 | convert_ctx_access = bpf_tcp_sock_convert_ctx_access; |
| 7327 | break; |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 7328 | default: |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 7329 | continue; |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 7330 | } |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 7331 | |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 7332 | ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 7333 | size = BPF_LDST_BYTES(insn); |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 7334 | |
| 7335 | /* If the read access is a narrower load of the field, |
| 7336 | * convert to a 4/8-byte load, to minimum program type specific |
| 7337 | * convert_ctx_access changes. If conversion is successful, |
| 7338 | * we will apply proper mask to the result. |
| 7339 | */ |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 7340 | is_narrower_load = size < ctx_field_size; |
Andrey Ignatov | 46f53a6 | 2018-11-10 22:15:13 -0800 | [diff] [blame] | 7341 | size_default = bpf_ctx_off_adjust_machine(ctx_field_size); |
| 7342 | off = insn->off; |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 7343 | if (is_narrower_load) { |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 7344 | u8 size_code; |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 7345 | |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 7346 | if (type == BPF_WRITE) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 7347 | verbose(env, "bpf verifier narrow ctx access misconfigured\n"); |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 7348 | return -EINVAL; |
| 7349 | } |
| 7350 | |
| 7351 | size_code = BPF_H; |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 7352 | if (ctx_field_size == 4) |
| 7353 | size_code = BPF_W; |
| 7354 | else if (ctx_field_size == 8) |
| 7355 | size_code = BPF_DW; |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 7356 | |
Daniel Borkmann | bc23105 | 2018-06-02 23:06:39 +0200 | [diff] [blame] | 7357 | insn->off = off & ~(size_default - 1); |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 7358 | insn->code = BPF_LDX | BPF_MEM | size_code; |
| 7359 | } |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 7360 | |
| 7361 | target_size = 0; |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 7362 | cnt = convert_ctx_access(type, insn, insn_buf, env->prog, |
| 7363 | &target_size); |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 7364 | if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf) || |
| 7365 | (ctx_field_size && !target_size)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 7366 | verbose(env, "bpf verifier is misconfigured\n"); |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 7367 | return -EINVAL; |
| 7368 | } |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 7369 | |
| 7370 | if (is_narrower_load && size < target_size) { |
Andrey Ignatov | 46f53a6 | 2018-11-10 22:15:13 -0800 | [diff] [blame] | 7371 | u8 shift = (off & (size_default - 1)) * 8; |
| 7372 | |
| 7373 | if (ctx_field_size <= 4) { |
| 7374 | if (shift) |
| 7375 | insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH, |
| 7376 | insn->dst_reg, |
| 7377 | shift); |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 7378 | insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 7379 | (1 << size * 8) - 1); |
Andrey Ignatov | 46f53a6 | 2018-11-10 22:15:13 -0800 | [diff] [blame] | 7380 | } else { |
| 7381 | if (shift) |
| 7382 | insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH, |
| 7383 | insn->dst_reg, |
| 7384 | shift); |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 7385 | insn_buf[cnt++] = BPF_ALU64_IMM(BPF_AND, insn->dst_reg, |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 7386 | (1 << size * 8) - 1); |
Andrey Ignatov | 46f53a6 | 2018-11-10 22:15:13 -0800 | [diff] [blame] | 7387 | } |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 7388 | } |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 7389 | |
Alexei Starovoitov | 8041902 | 2017-03-15 18:26:41 -0700 | [diff] [blame] | 7390 | new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 7391 | if (!new_prog) |
| 7392 | return -ENOMEM; |
| 7393 | |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 7394 | delta += cnt - 1; |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 7395 | |
| 7396 | /* keep walking new program and skip insns we just inserted */ |
| 7397 | env->prog = new_prog; |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 7398 | insn = new_prog->insnsi + i + delta; |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 7399 | } |
| 7400 | |
| 7401 | return 0; |
| 7402 | } |
| 7403 | |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7404 | static int jit_subprogs(struct bpf_verifier_env *env) |
| 7405 | { |
| 7406 | struct bpf_prog *prog = env->prog, **func, *tmp; |
| 7407 | int i, j, subprog_start, subprog_end = 0, len, subprog; |
Daniel Borkmann | 7105e82 | 2017-12-20 13:42:57 +0100 | [diff] [blame] | 7408 | struct bpf_insn *insn; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7409 | void *old_bpf_func; |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 7410 | int err; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7411 | |
Jiong Wang | f910cef | 2018-05-02 16:17:17 -0400 | [diff] [blame] | 7412 | if (env->subprog_cnt <= 1) |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7413 | return 0; |
| 7414 | |
Daniel Borkmann | 7105e82 | 2017-12-20 13:42:57 +0100 | [diff] [blame] | 7415 | for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7416 | if (insn->code != (BPF_JMP | BPF_CALL) || |
| 7417 | insn->src_reg != BPF_PSEUDO_CALL) |
| 7418 | continue; |
Daniel Borkmann | c7a8978 | 2018-07-12 21:44:28 +0200 | [diff] [blame] | 7419 | /* Upon error here we cannot fall back to interpreter but |
| 7420 | * need a hard reject of the program. Thus -EFAULT is |
| 7421 | * propagated in any case. |
| 7422 | */ |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7423 | subprog = find_subprog(env, i + insn->imm + 1); |
| 7424 | if (subprog < 0) { |
| 7425 | WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", |
| 7426 | i + insn->imm + 1); |
| 7427 | return -EFAULT; |
| 7428 | } |
| 7429 | /* temporarily remember subprog id inside insn instead of |
| 7430 | * aux_data, since next loop will split up all insns into funcs |
| 7431 | */ |
Jiong Wang | f910cef | 2018-05-02 16:17:17 -0400 | [diff] [blame] | 7432 | insn->off = subprog; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7433 | /* remember original imm in case JIT fails and fallback |
| 7434 | * to interpreter will be needed |
| 7435 | */ |
| 7436 | env->insn_aux_data[i].call_imm = insn->imm; |
| 7437 | /* point imm to __bpf_call_base+1 from JITs point of view */ |
| 7438 | insn->imm = 1; |
| 7439 | } |
| 7440 | |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 7441 | err = bpf_prog_alloc_jited_linfo(prog); |
| 7442 | if (err) |
| 7443 | goto out_undo_insn; |
| 7444 | |
| 7445 | err = -ENOMEM; |
Kees Cook | 6396bb2 | 2018-06-12 14:03:40 -0700 | [diff] [blame] | 7446 | func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL); |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7447 | if (!func) |
Daniel Borkmann | c7a8978 | 2018-07-12 21:44:28 +0200 | [diff] [blame] | 7448 | goto out_undo_insn; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7449 | |
Jiong Wang | f910cef | 2018-05-02 16:17:17 -0400 | [diff] [blame] | 7450 | for (i = 0; i < env->subprog_cnt; i++) { |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7451 | subprog_start = subprog_end; |
Jiong Wang | 4cb3d99 | 2018-05-02 16:17:19 -0400 | [diff] [blame] | 7452 | subprog_end = env->subprog_info[i + 1].start; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7453 | |
| 7454 | len = subprog_end - subprog_start; |
Alexei Starovoitov | 492ecee | 2019-02-25 14:28:39 -0800 | [diff] [blame] | 7455 | /* BPF_PROG_RUN doesn't call subprogs directly, |
| 7456 | * hence main prog stats include the runtime of subprogs. |
| 7457 | * subprogs don't have IDs and not reachable via prog_get_next_id |
| 7458 | * func[i]->aux->stats will never be accessed and stays NULL |
| 7459 | */ |
| 7460 | func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER); |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7461 | if (!func[i]) |
| 7462 | goto out_free; |
| 7463 | memcpy(func[i]->insnsi, &prog->insnsi[subprog_start], |
| 7464 | len * sizeof(struct bpf_insn)); |
Daniel Borkmann | 4f74d80 | 2017-12-20 13:42:56 +0100 | [diff] [blame] | 7465 | func[i]->type = prog->type; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7466 | func[i]->len = len; |
Daniel Borkmann | 4f74d80 | 2017-12-20 13:42:56 +0100 | [diff] [blame] | 7467 | if (bpf_prog_calc_tag(func[i])) |
| 7468 | goto out_free; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7469 | func[i]->is_func = 1; |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 7470 | func[i]->aux->func_idx = i; |
| 7471 | /* the btf and func_info will be freed only at prog->aux */ |
| 7472 | func[i]->aux->btf = prog->aux->btf; |
| 7473 | func[i]->aux->func_info = prog->aux->func_info; |
| 7474 | |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7475 | /* Use bpf_prog_F_tag to indicate functions in stack traces. |
| 7476 | * Long term would need debug info to populate names |
| 7477 | */ |
| 7478 | func[i]->aux->name[0] = 'F'; |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 7479 | func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7480 | func[i]->jit_requested = 1; |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 7481 | func[i]->aux->linfo = prog->aux->linfo; |
| 7482 | func[i]->aux->nr_linfo = prog->aux->nr_linfo; |
| 7483 | func[i]->aux->jited_linfo = prog->aux->jited_linfo; |
| 7484 | func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7485 | func[i] = bpf_int_jit_compile(func[i]); |
| 7486 | if (!func[i]->jited) { |
| 7487 | err = -ENOTSUPP; |
| 7488 | goto out_free; |
| 7489 | } |
| 7490 | cond_resched(); |
| 7491 | } |
| 7492 | /* at this point all bpf functions were successfully JITed |
| 7493 | * now populate all bpf_calls with correct addresses and |
| 7494 | * run last pass of JIT |
| 7495 | */ |
Jiong Wang | f910cef | 2018-05-02 16:17:17 -0400 | [diff] [blame] | 7496 | for (i = 0; i < env->subprog_cnt; i++) { |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7497 | insn = func[i]->insnsi; |
| 7498 | for (j = 0; j < func[i]->len; j++, insn++) { |
| 7499 | if (insn->code != (BPF_JMP | BPF_CALL) || |
| 7500 | insn->src_reg != BPF_PSEUDO_CALL) |
| 7501 | continue; |
| 7502 | subprog = insn->off; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7503 | insn->imm = (u64 (*)(u64, u64, u64, u64, u64)) |
| 7504 | func[subprog]->bpf_func - |
| 7505 | __bpf_call_base; |
| 7506 | } |
Sandipan Das | 2162fed | 2018-05-24 12:26:45 +0530 | [diff] [blame] | 7507 | |
| 7508 | /* we use the aux data to keep a list of the start addresses |
| 7509 | * of the JITed images for each function in the program |
| 7510 | * |
| 7511 | * for some architectures, such as powerpc64, the imm field |
| 7512 | * might not be large enough to hold the offset of the start |
| 7513 | * address of the callee's JITed image from __bpf_call_base |
| 7514 | * |
| 7515 | * in such cases, we can lookup the start address of a callee |
| 7516 | * by using its subprog id, available from the off field of |
| 7517 | * the call instruction, as an index for this list |
| 7518 | */ |
| 7519 | func[i]->aux->func = func; |
| 7520 | func[i]->aux->func_cnt = env->subprog_cnt; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7521 | } |
Jiong Wang | f910cef | 2018-05-02 16:17:17 -0400 | [diff] [blame] | 7522 | for (i = 0; i < env->subprog_cnt; i++) { |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7523 | old_bpf_func = func[i]->bpf_func; |
| 7524 | tmp = bpf_int_jit_compile(func[i]); |
| 7525 | if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) { |
| 7526 | verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); |
Daniel Borkmann | c7a8978 | 2018-07-12 21:44:28 +0200 | [diff] [blame] | 7527 | err = -ENOTSUPP; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7528 | goto out_free; |
| 7529 | } |
| 7530 | cond_resched(); |
| 7531 | } |
| 7532 | |
| 7533 | /* finally lock prog and jit images for all functions and |
| 7534 | * populate kallsysm |
| 7535 | */ |
Jiong Wang | f910cef | 2018-05-02 16:17:17 -0400 | [diff] [blame] | 7536 | for (i = 0; i < env->subprog_cnt; i++) { |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7537 | bpf_prog_lock_ro(func[i]); |
| 7538 | bpf_prog_kallsyms_add(func[i]); |
| 7539 | } |
Daniel Borkmann | 7105e82 | 2017-12-20 13:42:57 +0100 | [diff] [blame] | 7540 | |
| 7541 | /* Last step: make now unused interpreter insns from main |
| 7542 | * prog consistent for later dump requests, so they can |
| 7543 | * later look the same as if they were interpreted only. |
| 7544 | */ |
| 7545 | for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { |
Daniel Borkmann | 7105e82 | 2017-12-20 13:42:57 +0100 | [diff] [blame] | 7546 | if (insn->code != (BPF_JMP | BPF_CALL) || |
| 7547 | insn->src_reg != BPF_PSEUDO_CALL) |
| 7548 | continue; |
| 7549 | insn->off = env->insn_aux_data[i].call_imm; |
| 7550 | subprog = find_subprog(env, i + insn->off + 1); |
Sandipan Das | dbecd73 | 2018-05-24 12:26:48 +0530 | [diff] [blame] | 7551 | insn->imm = subprog; |
Daniel Borkmann | 7105e82 | 2017-12-20 13:42:57 +0100 | [diff] [blame] | 7552 | } |
| 7553 | |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7554 | prog->jited = 1; |
| 7555 | prog->bpf_func = func[0]->bpf_func; |
| 7556 | prog->aux->func = func; |
Jiong Wang | f910cef | 2018-05-02 16:17:17 -0400 | [diff] [blame] | 7557 | prog->aux->func_cnt = env->subprog_cnt; |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 7558 | bpf_prog_free_unused_jited_linfo(prog); |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7559 | return 0; |
| 7560 | out_free: |
Jiong Wang | f910cef | 2018-05-02 16:17:17 -0400 | [diff] [blame] | 7561 | for (i = 0; i < env->subprog_cnt; i++) |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7562 | if (func[i]) |
| 7563 | bpf_jit_free(func[i]); |
| 7564 | kfree(func); |
Daniel Borkmann | c7a8978 | 2018-07-12 21:44:28 +0200 | [diff] [blame] | 7565 | out_undo_insn: |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7566 | /* cleanup main prog to be interpreted */ |
| 7567 | prog->jit_requested = 0; |
| 7568 | for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { |
| 7569 | if (insn->code != (BPF_JMP | BPF_CALL) || |
| 7570 | insn->src_reg != BPF_PSEUDO_CALL) |
| 7571 | continue; |
| 7572 | insn->off = 0; |
| 7573 | insn->imm = env->insn_aux_data[i].call_imm; |
| 7574 | } |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 7575 | bpf_prog_free_jited_linfo(prog); |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7576 | return err; |
| 7577 | } |
| 7578 | |
Alexei Starovoitov | 1ea47e0 | 2017-12-14 17:55:13 -0800 | [diff] [blame] | 7579 | static int fixup_call_args(struct bpf_verifier_env *env) |
| 7580 | { |
David S. Miller | 19d28fb | 2018-01-11 21:27:54 -0500 | [diff] [blame] | 7581 | #ifndef CONFIG_BPF_JIT_ALWAYS_ON |
Alexei Starovoitov | 1ea47e0 | 2017-12-14 17:55:13 -0800 | [diff] [blame] | 7582 | struct bpf_prog *prog = env->prog; |
| 7583 | struct bpf_insn *insn = prog->insnsi; |
| 7584 | int i, depth; |
David S. Miller | 19d28fb | 2018-01-11 21:27:54 -0500 | [diff] [blame] | 7585 | #endif |
Quentin Monnet | e4052d0 | 2018-10-07 12:56:58 +0100 | [diff] [blame] | 7586 | int err = 0; |
Alexei Starovoitov | 1ea47e0 | 2017-12-14 17:55:13 -0800 | [diff] [blame] | 7587 | |
Quentin Monnet | e4052d0 | 2018-10-07 12:56:58 +0100 | [diff] [blame] | 7588 | if (env->prog->jit_requested && |
| 7589 | !bpf_prog_is_dev_bound(env->prog->aux)) { |
David S. Miller | 19d28fb | 2018-01-11 21:27:54 -0500 | [diff] [blame] | 7590 | err = jit_subprogs(env); |
| 7591 | if (err == 0) |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 7592 | return 0; |
Daniel Borkmann | c7a8978 | 2018-07-12 21:44:28 +0200 | [diff] [blame] | 7593 | if (err == -EFAULT) |
| 7594 | return err; |
David S. Miller | 19d28fb | 2018-01-11 21:27:54 -0500 | [diff] [blame] | 7595 | } |
| 7596 | #ifndef CONFIG_BPF_JIT_ALWAYS_ON |
Alexei Starovoitov | 1ea47e0 | 2017-12-14 17:55:13 -0800 | [diff] [blame] | 7597 | for (i = 0; i < prog->len; i++, insn++) { |
| 7598 | if (insn->code != (BPF_JMP | BPF_CALL) || |
| 7599 | insn->src_reg != BPF_PSEUDO_CALL) |
| 7600 | continue; |
| 7601 | depth = get_callee_stack_depth(env, insn, i); |
| 7602 | if (depth < 0) |
| 7603 | return depth; |
| 7604 | bpf_patch_call_args(insn, depth); |
| 7605 | } |
David S. Miller | 19d28fb | 2018-01-11 21:27:54 -0500 | [diff] [blame] | 7606 | err = 0; |
| 7607 | #endif |
| 7608 | return err; |
Alexei Starovoitov | 1ea47e0 | 2017-12-14 17:55:13 -0800 | [diff] [blame] | 7609 | } |
| 7610 | |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 7611 | /* fixup insn->imm field of bpf_call instructions |
Alexei Starovoitov | 81ed18a | 2017-03-15 18:26:42 -0700 | [diff] [blame] | 7612 | * and inline eligible helpers as explicit sequence of BPF instructions |
Alexei Starovoitov | e245c5c6 | 2017-03-15 18:26:39 -0700 | [diff] [blame] | 7613 | * |
| 7614 | * this function is called after eBPF program passed verification |
| 7615 | */ |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 7616 | static int fixup_bpf_calls(struct bpf_verifier_env *env) |
Alexei Starovoitov | e245c5c6 | 2017-03-15 18:26:39 -0700 | [diff] [blame] | 7617 | { |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 7618 | struct bpf_prog *prog = env->prog; |
| 7619 | struct bpf_insn *insn = prog->insnsi; |
Alexei Starovoitov | e245c5c6 | 2017-03-15 18:26:39 -0700 | [diff] [blame] | 7620 | const struct bpf_func_proto *fn; |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 7621 | const int insn_cnt = prog->len; |
Daniel Borkmann | 09772d9 | 2018-06-02 23:06:35 +0200 | [diff] [blame] | 7622 | const struct bpf_map_ops *ops; |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 7623 | struct bpf_insn_aux_data *aux; |
Alexei Starovoitov | 81ed18a | 2017-03-15 18:26:42 -0700 | [diff] [blame] | 7624 | struct bpf_insn insn_buf[16]; |
| 7625 | struct bpf_prog *new_prog; |
| 7626 | struct bpf_map *map_ptr; |
| 7627 | int i, cnt, delta = 0; |
Alexei Starovoitov | e245c5c6 | 2017-03-15 18:26:39 -0700 | [diff] [blame] | 7628 | |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 7629 | for (i = 0; i < insn_cnt; i++, insn++) { |
Daniel Borkmann | f6b1b3b | 2018-01-26 23:33:39 +0100 | [diff] [blame] | 7630 | if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) || |
| 7631 | insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || |
| 7632 | insn->code == (BPF_ALU | BPF_MOD | BPF_X) || |
Alexei Starovoitov | 68fda45 | 2018-01-12 18:59:52 -0800 | [diff] [blame] | 7633 | insn->code == (BPF_ALU | BPF_DIV | BPF_X)) { |
Daniel Borkmann | f6b1b3b | 2018-01-26 23:33:39 +0100 | [diff] [blame] | 7634 | bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; |
| 7635 | struct bpf_insn mask_and_div[] = { |
| 7636 | BPF_MOV32_REG(insn->src_reg, insn->src_reg), |
| 7637 | /* Rx div 0 -> 0 */ |
| 7638 | BPF_JMP_IMM(BPF_JNE, insn->src_reg, 0, 2), |
| 7639 | BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg), |
| 7640 | BPF_JMP_IMM(BPF_JA, 0, 0, 1), |
| 7641 | *insn, |
| 7642 | }; |
| 7643 | struct bpf_insn mask_and_mod[] = { |
| 7644 | BPF_MOV32_REG(insn->src_reg, insn->src_reg), |
| 7645 | /* Rx mod 0 -> Rx */ |
| 7646 | BPF_JMP_IMM(BPF_JEQ, insn->src_reg, 0, 1), |
| 7647 | *insn, |
| 7648 | }; |
| 7649 | struct bpf_insn *patchlet; |
| 7650 | |
| 7651 | if (insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || |
| 7652 | insn->code == (BPF_ALU | BPF_DIV | BPF_X)) { |
| 7653 | patchlet = mask_and_div + (is64 ? 1 : 0); |
| 7654 | cnt = ARRAY_SIZE(mask_and_div) - (is64 ? 1 : 0); |
| 7655 | } else { |
| 7656 | patchlet = mask_and_mod + (is64 ? 1 : 0); |
| 7657 | cnt = ARRAY_SIZE(mask_and_mod) - (is64 ? 1 : 0); |
| 7658 | } |
| 7659 | |
| 7660 | new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt); |
Alexei Starovoitov | 68fda45 | 2018-01-12 18:59:52 -0800 | [diff] [blame] | 7661 | if (!new_prog) |
| 7662 | return -ENOMEM; |
| 7663 | |
| 7664 | delta += cnt - 1; |
| 7665 | env->prog = prog = new_prog; |
| 7666 | insn = new_prog->insnsi + i + delta; |
| 7667 | continue; |
| 7668 | } |
| 7669 | |
Daniel Borkmann | e0cea7c | 2018-05-04 01:08:14 +0200 | [diff] [blame] | 7670 | if (BPF_CLASS(insn->code) == BPF_LD && |
| 7671 | (BPF_MODE(insn->code) == BPF_ABS || |
| 7672 | BPF_MODE(insn->code) == BPF_IND)) { |
| 7673 | cnt = env->ops->gen_ld_abs(insn, insn_buf); |
| 7674 | if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) { |
| 7675 | verbose(env, "bpf verifier is misconfigured\n"); |
| 7676 | return -EINVAL; |
| 7677 | } |
| 7678 | |
| 7679 | new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); |
| 7680 | if (!new_prog) |
| 7681 | return -ENOMEM; |
| 7682 | |
| 7683 | delta += cnt - 1; |
| 7684 | env->prog = prog = new_prog; |
| 7685 | insn = new_prog->insnsi + i + delta; |
| 7686 | continue; |
| 7687 | } |
| 7688 | |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 7689 | if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) || |
| 7690 | insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) { |
| 7691 | const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X; |
| 7692 | const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X; |
| 7693 | struct bpf_insn insn_buf[16]; |
| 7694 | struct bpf_insn *patch = &insn_buf[0]; |
| 7695 | bool issrc, isneg; |
| 7696 | u32 off_reg; |
| 7697 | |
| 7698 | aux = &env->insn_aux_data[i + delta]; |
Daniel Borkmann | 3612af7 | 2019-03-01 22:05:29 +0100 | [diff] [blame] | 7699 | if (!aux->alu_state || |
| 7700 | aux->alu_state == BPF_ALU_NON_POINTER) |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 7701 | continue; |
| 7702 | |
| 7703 | isneg = aux->alu_state & BPF_ALU_NEG_VALUE; |
| 7704 | issrc = (aux->alu_state & BPF_ALU_SANITIZE) == |
| 7705 | BPF_ALU_SANITIZE_SRC; |
| 7706 | |
| 7707 | off_reg = issrc ? insn->src_reg : insn->dst_reg; |
| 7708 | if (isneg) |
| 7709 | *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); |
| 7710 | *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit - 1); |
| 7711 | *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg); |
| 7712 | *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg); |
| 7713 | *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0); |
| 7714 | *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63); |
| 7715 | if (issrc) { |
| 7716 | *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, |
| 7717 | off_reg); |
| 7718 | insn->src_reg = BPF_REG_AX; |
| 7719 | } else { |
| 7720 | *patch++ = BPF_ALU64_REG(BPF_AND, off_reg, |
| 7721 | BPF_REG_AX); |
| 7722 | } |
| 7723 | if (isneg) |
| 7724 | insn->code = insn->code == code_add ? |
| 7725 | code_sub : code_add; |
| 7726 | *patch++ = *insn; |
| 7727 | if (issrc && isneg) |
| 7728 | *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); |
| 7729 | cnt = patch - insn_buf; |
| 7730 | |
| 7731 | new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); |
| 7732 | if (!new_prog) |
| 7733 | return -ENOMEM; |
| 7734 | |
| 7735 | delta += cnt - 1; |
| 7736 | env->prog = prog = new_prog; |
| 7737 | insn = new_prog->insnsi + i + delta; |
| 7738 | continue; |
| 7739 | } |
| 7740 | |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 7741 | if (insn->code != (BPF_JMP | BPF_CALL)) |
| 7742 | continue; |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 7743 | if (insn->src_reg == BPF_PSEUDO_CALL) |
| 7744 | continue; |
Alexei Starovoitov | e245c5c6 | 2017-03-15 18:26:39 -0700 | [diff] [blame] | 7745 | |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 7746 | if (insn->imm == BPF_FUNC_get_route_realm) |
| 7747 | prog->dst_needed = 1; |
| 7748 | if (insn->imm == BPF_FUNC_get_prandom_u32) |
| 7749 | bpf_user_rnd_init_once(); |
Josef Bacik | 9802d86 | 2017-12-11 11:36:48 -0500 | [diff] [blame] | 7750 | if (insn->imm == BPF_FUNC_override_return) |
| 7751 | prog->kprobe_override = 1; |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 7752 | if (insn->imm == BPF_FUNC_tail_call) { |
David S. Miller | 7b9f6da | 2017-04-20 10:35:33 -0400 | [diff] [blame] | 7753 | /* If we tail call into other programs, we |
| 7754 | * cannot make any assumptions since they can |
| 7755 | * be replaced dynamically during runtime in |
| 7756 | * the program array. |
| 7757 | */ |
| 7758 | prog->cb_access = 1; |
Alexei Starovoitov | 80a58d0 | 2017-05-30 13:31:30 -0700 | [diff] [blame] | 7759 | env->prog->aux->stack_depth = MAX_BPF_STACK; |
Jiong Wang | e647815 | 2018-11-08 04:08:42 -0500 | [diff] [blame] | 7760 | env->prog->aux->max_pkt_offset = MAX_PACKET_OFF; |
David S. Miller | 7b9f6da | 2017-04-20 10:35:33 -0400 | [diff] [blame] | 7761 | |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 7762 | /* mark bpf_tail_call as different opcode to avoid |
| 7763 | * conditional branch in the interpeter for every normal |
| 7764 | * call and to prevent accidental JITing by JIT compiler |
| 7765 | * that doesn't support bpf_tail_call yet |
Alexei Starovoitov | e245c5c6 | 2017-03-15 18:26:39 -0700 | [diff] [blame] | 7766 | */ |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 7767 | insn->imm = 0; |
Alexei Starovoitov | 71189fa | 2017-05-30 13:31:27 -0700 | [diff] [blame] | 7768 | insn->code = BPF_JMP | BPF_TAIL_CALL; |
Alexei Starovoitov | b215739 | 2018-01-07 17:33:02 -0800 | [diff] [blame] | 7769 | |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 7770 | aux = &env->insn_aux_data[i + delta]; |
| 7771 | if (!bpf_map_ptr_unpriv(aux)) |
| 7772 | continue; |
| 7773 | |
Alexei Starovoitov | b215739 | 2018-01-07 17:33:02 -0800 | [diff] [blame] | 7774 | /* instead of changing every JIT dealing with tail_call |
| 7775 | * emit two extra insns: |
| 7776 | * if (index >= max_entries) goto out; |
| 7777 | * index &= array->index_mask; |
| 7778 | * to avoid out-of-bounds cpu speculation |
| 7779 | */ |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 7780 | if (bpf_map_ptr_poisoned(aux)) { |
Colin Ian King | 4095034 | 2018-01-10 09:20:54 +0000 | [diff] [blame] | 7781 | verbose(env, "tail_call abusing map_ptr\n"); |
Alexei Starovoitov | b215739 | 2018-01-07 17:33:02 -0800 | [diff] [blame] | 7782 | return -EINVAL; |
| 7783 | } |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 7784 | |
| 7785 | map_ptr = BPF_MAP_PTR(aux->map_state); |
Alexei Starovoitov | b215739 | 2018-01-07 17:33:02 -0800 | [diff] [blame] | 7786 | insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3, |
| 7787 | map_ptr->max_entries, 2); |
| 7788 | insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3, |
| 7789 | container_of(map_ptr, |
| 7790 | struct bpf_array, |
| 7791 | map)->index_mask); |
| 7792 | insn_buf[2] = *insn; |
| 7793 | cnt = 3; |
| 7794 | new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); |
| 7795 | if (!new_prog) |
| 7796 | return -ENOMEM; |
| 7797 | |
| 7798 | delta += cnt - 1; |
| 7799 | env->prog = prog = new_prog; |
| 7800 | insn = new_prog->insnsi + i + delta; |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 7801 | continue; |
Alexei Starovoitov | e245c5c6 | 2017-03-15 18:26:39 -0700 | [diff] [blame] | 7802 | } |
Alexei Starovoitov | e245c5c6 | 2017-03-15 18:26:39 -0700 | [diff] [blame] | 7803 | |
Daniel Borkmann | 89c6307 | 2017-08-19 03:12:45 +0200 | [diff] [blame] | 7804 | /* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup |
Daniel Borkmann | 09772d9 | 2018-06-02 23:06:35 +0200 | [diff] [blame] | 7805 | * and other inlining handlers are currently limited to 64 bit |
| 7806 | * only. |
Daniel Borkmann | 89c6307 | 2017-08-19 03:12:45 +0200 | [diff] [blame] | 7807 | */ |
Alexei Starovoitov | 60b58afc | 2017-12-14 17:55:14 -0800 | [diff] [blame] | 7808 | if (prog->jit_requested && BITS_PER_LONG == 64 && |
Daniel Borkmann | 09772d9 | 2018-06-02 23:06:35 +0200 | [diff] [blame] | 7809 | (insn->imm == BPF_FUNC_map_lookup_elem || |
| 7810 | insn->imm == BPF_FUNC_map_update_elem || |
Daniel Borkmann | 84430d4 | 2018-10-21 02:09:27 +0200 | [diff] [blame] | 7811 | insn->imm == BPF_FUNC_map_delete_elem || |
| 7812 | insn->imm == BPF_FUNC_map_push_elem || |
| 7813 | insn->imm == BPF_FUNC_map_pop_elem || |
| 7814 | insn->imm == BPF_FUNC_map_peek_elem)) { |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 7815 | aux = &env->insn_aux_data[i + delta]; |
| 7816 | if (bpf_map_ptr_poisoned(aux)) |
| 7817 | goto patch_call_imm; |
| 7818 | |
| 7819 | map_ptr = BPF_MAP_PTR(aux->map_state); |
Daniel Borkmann | 09772d9 | 2018-06-02 23:06:35 +0200 | [diff] [blame] | 7820 | ops = map_ptr->ops; |
| 7821 | if (insn->imm == BPF_FUNC_map_lookup_elem && |
| 7822 | ops->map_gen_lookup) { |
| 7823 | cnt = ops->map_gen_lookup(map_ptr, insn_buf); |
| 7824 | if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) { |
| 7825 | verbose(env, "bpf verifier is misconfigured\n"); |
| 7826 | return -EINVAL; |
| 7827 | } |
Alexei Starovoitov | 81ed18a | 2017-03-15 18:26:42 -0700 | [diff] [blame] | 7828 | |
Daniel Borkmann | 09772d9 | 2018-06-02 23:06:35 +0200 | [diff] [blame] | 7829 | new_prog = bpf_patch_insn_data(env, i + delta, |
| 7830 | insn_buf, cnt); |
| 7831 | if (!new_prog) |
| 7832 | return -ENOMEM; |
| 7833 | |
| 7834 | delta += cnt - 1; |
| 7835 | env->prog = prog = new_prog; |
| 7836 | insn = new_prog->insnsi + i + delta; |
| 7837 | continue; |
Alexei Starovoitov | 81ed18a | 2017-03-15 18:26:42 -0700 | [diff] [blame] | 7838 | } |
| 7839 | |
Daniel Borkmann | 09772d9 | 2018-06-02 23:06:35 +0200 | [diff] [blame] | 7840 | BUILD_BUG_ON(!__same_type(ops->map_lookup_elem, |
| 7841 | (void *(*)(struct bpf_map *map, void *key))NULL)); |
| 7842 | BUILD_BUG_ON(!__same_type(ops->map_delete_elem, |
| 7843 | (int (*)(struct bpf_map *map, void *key))NULL)); |
| 7844 | BUILD_BUG_ON(!__same_type(ops->map_update_elem, |
| 7845 | (int (*)(struct bpf_map *map, void *key, void *value, |
| 7846 | u64 flags))NULL)); |
Daniel Borkmann | 84430d4 | 2018-10-21 02:09:27 +0200 | [diff] [blame] | 7847 | BUILD_BUG_ON(!__same_type(ops->map_push_elem, |
| 7848 | (int (*)(struct bpf_map *map, void *value, |
| 7849 | u64 flags))NULL)); |
| 7850 | BUILD_BUG_ON(!__same_type(ops->map_pop_elem, |
| 7851 | (int (*)(struct bpf_map *map, void *value))NULL)); |
| 7852 | BUILD_BUG_ON(!__same_type(ops->map_peek_elem, |
| 7853 | (int (*)(struct bpf_map *map, void *value))NULL)); |
| 7854 | |
Daniel Borkmann | 09772d9 | 2018-06-02 23:06:35 +0200 | [diff] [blame] | 7855 | switch (insn->imm) { |
| 7856 | case BPF_FUNC_map_lookup_elem: |
| 7857 | insn->imm = BPF_CAST_CALL(ops->map_lookup_elem) - |
| 7858 | __bpf_call_base; |
| 7859 | continue; |
| 7860 | case BPF_FUNC_map_update_elem: |
| 7861 | insn->imm = BPF_CAST_CALL(ops->map_update_elem) - |
| 7862 | __bpf_call_base; |
| 7863 | continue; |
| 7864 | case BPF_FUNC_map_delete_elem: |
| 7865 | insn->imm = BPF_CAST_CALL(ops->map_delete_elem) - |
| 7866 | __bpf_call_base; |
| 7867 | continue; |
Daniel Borkmann | 84430d4 | 2018-10-21 02:09:27 +0200 | [diff] [blame] | 7868 | case BPF_FUNC_map_push_elem: |
| 7869 | insn->imm = BPF_CAST_CALL(ops->map_push_elem) - |
| 7870 | __bpf_call_base; |
| 7871 | continue; |
| 7872 | case BPF_FUNC_map_pop_elem: |
| 7873 | insn->imm = BPF_CAST_CALL(ops->map_pop_elem) - |
| 7874 | __bpf_call_base; |
| 7875 | continue; |
| 7876 | case BPF_FUNC_map_peek_elem: |
| 7877 | insn->imm = BPF_CAST_CALL(ops->map_peek_elem) - |
| 7878 | __bpf_call_base; |
| 7879 | continue; |
Daniel Borkmann | 09772d9 | 2018-06-02 23:06:35 +0200 | [diff] [blame] | 7880 | } |
Alexei Starovoitov | 81ed18a | 2017-03-15 18:26:42 -0700 | [diff] [blame] | 7881 | |
Daniel Borkmann | 09772d9 | 2018-06-02 23:06:35 +0200 | [diff] [blame] | 7882 | goto patch_call_imm; |
Alexei Starovoitov | 81ed18a | 2017-03-15 18:26:42 -0700 | [diff] [blame] | 7883 | } |
| 7884 | |
| 7885 | patch_call_imm: |
Andrey Ignatov | 5e43f89 | 2018-03-30 15:08:00 -0700 | [diff] [blame] | 7886 | fn = env->ops->get_func_proto(insn->imm, env->prog); |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 7887 | /* all functions that have prototype and verifier allowed |
| 7888 | * programs to call them, must be real in-kernel functions |
| 7889 | */ |
| 7890 | if (!fn->func) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 7891 | verbose(env, |
| 7892 | "kernel subsystem misconfigured func %s#%d\n", |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 7893 | func_id_name(insn->imm), insn->imm); |
| 7894 | return -EFAULT; |
| 7895 | } |
| 7896 | insn->imm = fn->func - __bpf_call_base; |
| 7897 | } |
| 7898 | |
| 7899 | return 0; |
| 7900 | } |
Alexei Starovoitov | e245c5c6 | 2017-03-15 18:26:39 -0700 | [diff] [blame] | 7901 | |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 7902 | static void free_states(struct bpf_verifier_env *env) |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 7903 | { |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 7904 | struct bpf_verifier_state_list *sl, *sln; |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 7905 | int i; |
| 7906 | |
Alexei Starovoitov | 9f4686c | 2019-04-01 21:27:41 -0700 | [diff] [blame] | 7907 | sl = env->free_list; |
| 7908 | while (sl) { |
| 7909 | sln = sl->next; |
| 7910 | free_verifier_state(&sl->state, false); |
| 7911 | kfree(sl); |
| 7912 | sl = sln; |
| 7913 | } |
| 7914 | |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 7915 | if (!env->explored_states) |
| 7916 | return; |
| 7917 | |
| 7918 | for (i = 0; i < env->prog->len; i++) { |
| 7919 | sl = env->explored_states[i]; |
| 7920 | |
| 7921 | if (sl) |
| 7922 | while (sl != STATE_LIST_MARK) { |
| 7923 | sln = sl->next; |
Alexei Starovoitov | 1969db4 | 2017-11-01 00:08:04 -0700 | [diff] [blame] | 7924 | free_verifier_state(&sl->state, false); |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 7925 | kfree(sl); |
| 7926 | sl = sln; |
| 7927 | } |
| 7928 | } |
| 7929 | |
Alexei Starovoitov | 71dde68 | 2019-04-01 21:27:43 -0700 | [diff] [blame] | 7930 | kvfree(env->explored_states); |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 7931 | } |
| 7932 | |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 7933 | static void print_verification_stats(struct bpf_verifier_env *env) |
| 7934 | { |
| 7935 | int i; |
| 7936 | |
| 7937 | if (env->log.level & BPF_LOG_STATS) { |
| 7938 | verbose(env, "verification time %lld usec\n", |
| 7939 | div_u64(env->verification_time, 1000)); |
| 7940 | verbose(env, "stack depth "); |
| 7941 | for (i = 0; i < env->subprog_cnt; i++) { |
| 7942 | u32 depth = env->subprog_info[i].stack_depth; |
| 7943 | |
| 7944 | verbose(env, "%d", depth); |
| 7945 | if (i + 1 < env->subprog_cnt) |
| 7946 | verbose(env, "+"); |
| 7947 | } |
| 7948 | verbose(env, "\n"); |
| 7949 | } |
| 7950 | verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " |
| 7951 | "total_states %d peak_states %d mark_read %d\n", |
| 7952 | env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS, |
| 7953 | env->max_states_per_insn, env->total_states, |
| 7954 | env->peak_states, env->longest_mark_read_walk); |
| 7955 | } |
| 7956 | |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 7957 | int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, |
| 7958 | union bpf_attr __user *uattr) |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 7959 | { |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 7960 | u64 start_time = ktime_get_ns(); |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 7961 | struct bpf_verifier_env *env; |
Martin KaFai Lau | b9193c1 | 2018-03-24 11:44:22 -0700 | [diff] [blame] | 7962 | struct bpf_verifier_log *log; |
Jakub Kicinski | 9e4c24e | 2019-01-22 22:45:23 -0800 | [diff] [blame] | 7963 | int i, len, ret = -EINVAL; |
Jakub Kicinski | e2ae4ca | 2019-01-22 22:45:19 -0800 | [diff] [blame] | 7964 | bool is_priv; |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 7965 | |
Arnd Bergmann | eba0c92 | 2017-11-02 12:05:52 +0100 | [diff] [blame] | 7966 | /* no program is valid */ |
| 7967 | if (ARRAY_SIZE(bpf_verifier_ops) == 0) |
| 7968 | return -EINVAL; |
| 7969 | |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 7970 | /* 'struct bpf_verifier_env' can be global, but since it's not small, |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 7971 | * allocate/free it every time bpf_check() is called |
| 7972 | */ |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 7973 | env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL); |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 7974 | if (!env) |
| 7975 | return -ENOMEM; |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 7976 | log = &env->log; |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 7977 | |
Jakub Kicinski | 9e4c24e | 2019-01-22 22:45:23 -0800 | [diff] [blame] | 7978 | len = (*prog)->len; |
Kees Cook | fad953c | 2018-06-12 14:27:37 -0700 | [diff] [blame] | 7979 | env->insn_aux_data = |
Jakub Kicinski | 9e4c24e | 2019-01-22 22:45:23 -0800 | [diff] [blame] | 7980 | vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len)); |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 7981 | ret = -ENOMEM; |
| 7982 | if (!env->insn_aux_data) |
| 7983 | goto err_free_env; |
Jakub Kicinski | 9e4c24e | 2019-01-22 22:45:23 -0800 | [diff] [blame] | 7984 | for (i = 0; i < len; i++) |
| 7985 | env->insn_aux_data[i].orig_idx = i; |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 7986 | env->prog = *prog; |
Jakub Kicinski | 00176a3 | 2017-10-16 16:40:54 -0700 | [diff] [blame] | 7987 | env->ops = bpf_verifier_ops[env->prog->type]; |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 7988 | |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 7989 | /* grab the mutex to protect few globals used by verifier */ |
| 7990 | mutex_lock(&bpf_verifier_lock); |
| 7991 | |
| 7992 | if (attr->log_level || attr->log_buf || attr->log_size) { |
| 7993 | /* user requested verbose verifier output |
| 7994 | * and supplied buffer to store the verification trace |
| 7995 | */ |
Jakub Kicinski | e7bf824 | 2017-10-09 10:30:10 -0700 | [diff] [blame] | 7996 | log->level = attr->log_level; |
| 7997 | log->ubuf = (char __user *) (unsigned long) attr->log_buf; |
| 7998 | log->len_total = attr->log_size; |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 7999 | |
| 8000 | ret = -EINVAL; |
Jakub Kicinski | e7bf824 | 2017-10-09 10:30:10 -0700 | [diff] [blame] | 8001 | /* log attributes have to be sane */ |
Alexei Starovoitov | 7a9f5c6 | 2019-04-01 21:27:46 -0700 | [diff] [blame] | 8002 | if (log->len_total < 128 || log->len_total > UINT_MAX >> 2 || |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 8003 | !log->level || !log->ubuf || log->level & ~BPF_LOG_MASK) |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 8004 | goto err_unlock; |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 8005 | } |
Daniel Borkmann | 1ad2f58 | 2017-05-25 01:05:05 +0200 | [diff] [blame] | 8006 | |
| 8007 | env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT); |
| 8008 | if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) |
David S. Miller | e07b98d | 2017-05-10 11:38:07 -0700 | [diff] [blame] | 8009 | env->strict_alignment = true; |
David Miller | e9ee9ef | 2018-11-30 21:08:14 -0800 | [diff] [blame] | 8010 | if (attr->prog_flags & BPF_F_ANY_ALIGNMENT) |
| 8011 | env->strict_alignment = false; |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 8012 | |
Jakub Kicinski | e2ae4ca | 2019-01-22 22:45:19 -0800 | [diff] [blame] | 8013 | is_priv = capable(CAP_SYS_ADMIN); |
| 8014 | env->allow_ptr_leaks = is_priv; |
| 8015 | |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 8016 | ret = replace_map_fd_with_map_ptr(env); |
| 8017 | if (ret < 0) |
| 8018 | goto skip_full_check; |
| 8019 | |
Jakub Kicinski | f4e3ec0 | 2018-05-03 18:37:11 -0700 | [diff] [blame] | 8020 | if (bpf_prog_is_dev_bound(env->prog->aux)) { |
Quentin Monnet | a40a263 | 2018-11-09 13:03:31 +0000 | [diff] [blame] | 8021 | ret = bpf_prog_offload_verifier_prep(env->prog); |
Jakub Kicinski | f4e3ec0 | 2018-05-03 18:37:11 -0700 | [diff] [blame] | 8022 | if (ret) |
| 8023 | goto skip_full_check; |
| 8024 | } |
| 8025 | |
Alexei Starovoitov | 71dde68 | 2019-04-01 21:27:43 -0700 | [diff] [blame] | 8026 | env->explored_states = kvcalloc(env->prog->len, |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 8027 | sizeof(struct bpf_verifier_state_list *), |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 8028 | GFP_USER); |
| 8029 | ret = -ENOMEM; |
| 8030 | if (!env->explored_states) |
| 8031 | goto skip_full_check; |
| 8032 | |
Martin KaFai Lau | d9762e8 | 2018-12-13 10:41:48 -0800 | [diff] [blame] | 8033 | ret = check_subprogs(env); |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 8034 | if (ret < 0) |
| 8035 | goto skip_full_check; |
| 8036 | |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 8037 | ret = check_btf_info(env, attr, uattr); |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 8038 | if (ret < 0) |
| 8039 | goto skip_full_check; |
| 8040 | |
Martin KaFai Lau | d9762e8 | 2018-12-13 10:41:48 -0800 | [diff] [blame] | 8041 | ret = check_cfg(env); |
| 8042 | if (ret < 0) |
| 8043 | goto skip_full_check; |
| 8044 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8045 | ret = do_check(env); |
Craig Gallek | 8c01c4f | 2017-11-02 11:18:01 -0400 | [diff] [blame] | 8046 | if (env->cur_state) { |
| 8047 | free_verifier_state(env->cur_state, true); |
| 8048 | env->cur_state = NULL; |
| 8049 | } |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 8050 | |
Quentin Monnet | c941ce9 | 2018-10-07 12:56:47 +0100 | [diff] [blame] | 8051 | if (ret == 0 && bpf_prog_is_dev_bound(env->prog->aux)) |
| 8052 | ret = bpf_prog_offload_finalize(env); |
| 8053 | |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 8054 | skip_full_check: |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 8055 | while (!pop_stack(env, NULL, NULL)); |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 8056 | free_states(env); |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 8057 | |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 8058 | if (ret == 0) |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 8059 | ret = check_max_stack_depth(env); |
| 8060 | |
Jakub Kicinski | 9b38c40 | 2018-12-19 22:13:06 -0800 | [diff] [blame] | 8061 | /* instruction rewrites happen after this point */ |
Jakub Kicinski | e2ae4ca | 2019-01-22 22:45:19 -0800 | [diff] [blame] | 8062 | if (is_priv) { |
| 8063 | if (ret == 0) |
| 8064 | opt_hard_wire_dead_code_branches(env); |
Jakub Kicinski | 52875a0 | 2019-01-22 22:45:20 -0800 | [diff] [blame] | 8065 | if (ret == 0) |
| 8066 | ret = opt_remove_dead_code(env); |
Jakub Kicinski | a1b14ab | 2019-01-22 22:45:21 -0800 | [diff] [blame] | 8067 | if (ret == 0) |
| 8068 | ret = opt_remove_nops(env); |
Jakub Kicinski | 52875a0 | 2019-01-22 22:45:20 -0800 | [diff] [blame] | 8069 | } else { |
| 8070 | if (ret == 0) |
| 8071 | sanitize_dead_code(env); |
Jakub Kicinski | e2ae4ca | 2019-01-22 22:45:19 -0800 | [diff] [blame] | 8072 | } |
| 8073 | |
Jakub Kicinski | 9b38c40 | 2018-12-19 22:13:06 -0800 | [diff] [blame] | 8074 | if (ret == 0) |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 8075 | /* program is valid, convert *(u32*)(ctx + off) accesses */ |
| 8076 | ret = convert_ctx_accesses(env); |
| 8077 | |
Alexei Starovoitov | e245c5c6 | 2017-03-15 18:26:39 -0700 | [diff] [blame] | 8078 | if (ret == 0) |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 8079 | ret = fixup_bpf_calls(env); |
Alexei Starovoitov | e245c5c6 | 2017-03-15 18:26:39 -0700 | [diff] [blame] | 8080 | |
Alexei Starovoitov | 1ea47e0 | 2017-12-14 17:55:13 -0800 | [diff] [blame] | 8081 | if (ret == 0) |
| 8082 | ret = fixup_call_args(env); |
| 8083 | |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 8084 | env->verification_time = ktime_get_ns() - start_time; |
| 8085 | print_verification_stats(env); |
| 8086 | |
Jakub Kicinski | a2a7d57 | 2017-10-09 10:30:15 -0700 | [diff] [blame] | 8087 | if (log->level && bpf_verifier_log_full(log)) |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 8088 | ret = -ENOSPC; |
Jakub Kicinski | a2a7d57 | 2017-10-09 10:30:15 -0700 | [diff] [blame] | 8089 | if (log->level && !log->ubuf) { |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 8090 | ret = -EFAULT; |
Jakub Kicinski | a2a7d57 | 2017-10-09 10:30:15 -0700 | [diff] [blame] | 8091 | goto err_release_maps; |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 8092 | } |
| 8093 | |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 8094 | if (ret == 0 && env->used_map_cnt) { |
| 8095 | /* if program passed verifier, update used_maps in bpf_prog_info */ |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 8096 | env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt, |
| 8097 | sizeof(env->used_maps[0]), |
| 8098 | GFP_KERNEL); |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 8099 | |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 8100 | if (!env->prog->aux->used_maps) { |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 8101 | ret = -ENOMEM; |
Jakub Kicinski | a2a7d57 | 2017-10-09 10:30:15 -0700 | [diff] [blame] | 8102 | goto err_release_maps; |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 8103 | } |
| 8104 | |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 8105 | memcpy(env->prog->aux->used_maps, env->used_maps, |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 8106 | sizeof(env->used_maps[0]) * env->used_map_cnt); |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 8107 | env->prog->aux->used_map_cnt = env->used_map_cnt; |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 8108 | |
| 8109 | /* program is valid. Convert pseudo bpf_ld_imm64 into generic |
| 8110 | * bpf_ld_imm64 instructions |
| 8111 | */ |
| 8112 | convert_pseudo_ld_imm64(env); |
| 8113 | } |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 8114 | |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 8115 | if (ret == 0) |
| 8116 | adjust_btf_func(env); |
| 8117 | |
Jakub Kicinski | a2a7d57 | 2017-10-09 10:30:15 -0700 | [diff] [blame] | 8118 | err_release_maps: |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 8119 | if (!env->prog->aux->used_maps) |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 8120 | /* if we didn't copy map pointers into bpf_prog_info, release |
Jakub Kicinski | ab7f5bf | 2018-05-03 18:37:17 -0700 | [diff] [blame] | 8121 | * them now. Otherwise free_used_maps() will release them. |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 8122 | */ |
| 8123 | release_maps(env); |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 8124 | *prog = env->prog; |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 8125 | err_unlock: |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 8126 | mutex_unlock(&bpf_verifier_lock); |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 8127 | vfree(env->insn_aux_data); |
| 8128 | err_free_env: |
| 8129 | kfree(env); |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 8130 | return ret; |
| 8131 | } |