Thomas Gleixner | 5b497af | 2019-05-29 07:18:09 -0700 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-only |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 2 | /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 3 | * Copyright (c) 2016 Facebook |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 4 | * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 5 | */ |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 6 | #include <uapi/linux/btf.h> |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 7 | #include <linux/kernel.h> |
| 8 | #include <linux/types.h> |
| 9 | #include <linux/slab.h> |
| 10 | #include <linux/bpf.h> |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 11 | #include <linux/btf.h> |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 12 | #include <linux/bpf_verifier.h> |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 13 | #include <linux/filter.h> |
| 14 | #include <net/netlink.h> |
| 15 | #include <linux/file.h> |
| 16 | #include <linux/vmalloc.h> |
Thomas Graf | ebb676d | 2016-10-27 11:23:51 +0200 | [diff] [blame] | 17 | #include <linux/stringify.h> |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 18 | #include <linux/bsearch.h> |
| 19 | #include <linux/sort.h> |
Yonghong Song | c195651e | 2018-04-28 22:28:08 -0700 | [diff] [blame] | 20 | #include <linux/perf_event.h> |
Martin KaFai Lau | d9762e8 | 2018-12-13 10:41:48 -0800 | [diff] [blame] | 21 | #include <linux/ctype.h> |
KP Singh | 6ba43b7 | 2020-03-04 20:18:50 +0100 | [diff] [blame] | 22 | #include <linux/error-injection.h> |
KP Singh | 9e4e01d | 2020-03-29 01:43:52 +0100 | [diff] [blame] | 23 | #include <linux/bpf_lsm.h> |
Alexei Starovoitov | 1e6c62a | 2020-08-27 15:01:11 -0700 | [diff] [blame] | 24 | #include <linux/btf_ids.h> |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 25 | |
Jakub Kicinski | f4ac7e0 | 2017-10-09 10:30:12 -0700 | [diff] [blame] | 26 | #include "disasm.h" |
| 27 | |
Jakub Kicinski | 00176a3 | 2017-10-16 16:40:54 -0700 | [diff] [blame] | 28 | static const struct bpf_verifier_ops * const bpf_verifier_ops[] = { |
Alexei Starovoitov | 91cc1a9 | 2019-11-14 10:57:15 -0800 | [diff] [blame] | 29 | #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ |
Jakub Kicinski | 00176a3 | 2017-10-16 16:40:54 -0700 | [diff] [blame] | 30 | [_id] = & _name ## _verifier_ops, |
| 31 | #define BPF_MAP_TYPE(_id, _ops) |
Andrii Nakryiko | f2e10bf | 2020-04-28 17:16:08 -0700 | [diff] [blame] | 32 | #define BPF_LINK_TYPE(_id, _name) |
Jakub Kicinski | 00176a3 | 2017-10-16 16:40:54 -0700 | [diff] [blame] | 33 | #include <linux/bpf_types.h> |
| 34 | #undef BPF_PROG_TYPE |
| 35 | #undef BPF_MAP_TYPE |
Andrii Nakryiko | f2e10bf | 2020-04-28 17:16:08 -0700 | [diff] [blame] | 36 | #undef BPF_LINK_TYPE |
Jakub Kicinski | 00176a3 | 2017-10-16 16:40:54 -0700 | [diff] [blame] | 37 | }; |
| 38 | |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 39 | /* bpf_check() is a static code analyzer that walks eBPF program |
| 40 | * instruction by instruction and updates register/stack state. |
| 41 | * All paths of conditional branches are analyzed until 'bpf_exit' insn. |
| 42 | * |
| 43 | * The first pass is depth-first-search to check that the program is a DAG. |
| 44 | * It rejects the following programs: |
| 45 | * - larger than BPF_MAXINSNS insns |
| 46 | * - if loop is present (detected via back-edge) |
| 47 | * - unreachable insns exist (shouldn't be a forest. program = one function) |
| 48 | * - out of bounds or malformed jumps |
| 49 | * The second pass is all possible path descent from the 1st insn. |
Zhen Lei | 8fb33b6 | 2021-05-25 10:56:59 +0800 | [diff] [blame] | 50 | * Since it's analyzing all paths through the program, the length of the |
Gary Lin | eba38a9 | 2017-03-01 16:25:51 +0800 | [diff] [blame] | 51 | * analysis is limited to 64k insn, which may be hit even if total number of |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 52 | * insn is less then 4K, but there are too many branches that change stack/regs. |
| 53 | * Number of 'branches to be analyzed' is limited to 1k |
| 54 | * |
| 55 | * On entry to each instruction, each register has a type, and the instruction |
| 56 | * changes the types of the registers depending on instruction semantics. |
| 57 | * If instruction is BPF_MOV64_REG(BPF_REG_1, BPF_REG_5), then type of R5 is |
| 58 | * copied to R1. |
| 59 | * |
| 60 | * All registers are 64-bit. |
| 61 | * R0 - return register |
| 62 | * R1-R5 argument passing registers |
| 63 | * R6-R9 callee saved registers |
| 64 | * R10 - frame pointer read-only |
| 65 | * |
| 66 | * At the start of BPF program the register R1 contains a pointer to bpf_context |
| 67 | * and has type PTR_TO_CTX. |
| 68 | * |
| 69 | * Verifier tracks arithmetic operations on pointers in case: |
| 70 | * BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), |
| 71 | * BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -20), |
| 72 | * 1st insn copies R10 (which has FRAME_PTR) type into R1 |
| 73 | * and 2nd arithmetic instruction is pattern matched to recognize |
| 74 | * that it wants to construct a pointer to some element within stack. |
| 75 | * So after 2nd insn, the register R1 has type PTR_TO_STACK |
| 76 | * (and -20 constant is saved for further stack bounds checking). |
| 77 | * Meaning that this reg is a pointer to stack plus known immediate constant. |
| 78 | * |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 79 | * Most of the time the registers have SCALAR_VALUE type, which |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 80 | * means the register has some value, but it's not a valid pointer. |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 81 | * (like pointer plus pointer becomes SCALAR_VALUE type) |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 82 | * |
| 83 | * When verifier sees load or store instructions the type of base register |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 84 | * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are |
| 85 | * four pointer types recognized by check_mem_access() function. |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 86 | * |
| 87 | * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value' |
| 88 | * and the range of [ptr, ptr + map's value_size) is accessible. |
| 89 | * |
| 90 | * registers used to pass values to function calls are checked against |
| 91 | * function argument constraints. |
| 92 | * |
| 93 | * ARG_PTR_TO_MAP_KEY is one of such argument constraints. |
| 94 | * It means that the register type passed to this function must be |
| 95 | * PTR_TO_STACK and it will be used inside the function as |
| 96 | * 'pointer to map element key' |
| 97 | * |
| 98 | * For example the argument constraints for bpf_map_lookup_elem(): |
| 99 | * .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, |
| 100 | * .arg1_type = ARG_CONST_MAP_PTR, |
| 101 | * .arg2_type = ARG_PTR_TO_MAP_KEY, |
| 102 | * |
| 103 | * ret_type says that this function returns 'pointer to map elem value or null' |
| 104 | * function expects 1st argument to be a const pointer to 'struct bpf_map' and |
| 105 | * 2nd argument should be a pointer to stack, which will be used inside |
| 106 | * the helper function as a pointer to map element key. |
| 107 | * |
| 108 | * On the kernel side the helper function looks like: |
| 109 | * u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) |
| 110 | * { |
| 111 | * struct bpf_map *map = (struct bpf_map *) (unsigned long) r1; |
| 112 | * void *key = (void *) (unsigned long) r2; |
| 113 | * void *value; |
| 114 | * |
| 115 | * here kernel can access 'key' and 'map' pointers safely, knowing that |
| 116 | * [key, key + map->key_size) bytes are valid and were initialized on |
| 117 | * the stack of eBPF program. |
| 118 | * } |
| 119 | * |
| 120 | * Corresponding eBPF program may look like: |
| 121 | * BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), // after this insn R2 type is FRAME_PTR |
| 122 | * BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), // after this insn R2 type is PTR_TO_STACK |
| 123 | * BPF_LD_MAP_FD(BPF_REG_1, map_fd), // after this insn R1 type is CONST_PTR_TO_MAP |
| 124 | * BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem), |
| 125 | * here verifier looks at prototype of map_lookup_elem() and sees: |
| 126 | * .arg1_type == ARG_CONST_MAP_PTR and R1->type == CONST_PTR_TO_MAP, which is ok, |
| 127 | * Now verifier knows that this map has key of R1->map_ptr->key_size bytes |
| 128 | * |
| 129 | * Then .arg2_type == ARG_PTR_TO_MAP_KEY and R2->type == PTR_TO_STACK, ok so far, |
| 130 | * Now verifier checks that [R2, R2 + map's key_size) are within stack limits |
| 131 | * and were initialized prior to this call. |
| 132 | * If it's ok, then verifier allows this BPF_CALL insn and looks at |
| 133 | * .ret_type which is RET_PTR_TO_MAP_VALUE_OR_NULL, so it sets |
| 134 | * R0->type = PTR_TO_MAP_VALUE_OR_NULL which means bpf_map_lookup_elem() function |
Zhen Lei | 8fb33b6 | 2021-05-25 10:56:59 +0800 | [diff] [blame] | 135 | * returns either pointer to map value or NULL. |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 136 | * |
| 137 | * When type PTR_TO_MAP_VALUE_OR_NULL passes through 'if (reg != 0) goto +off' |
| 138 | * insn, the register holding that pointer in the true branch changes state to |
| 139 | * PTR_TO_MAP_VALUE and the same register changes state to CONST_IMM in the false |
| 140 | * branch. See check_cond_jmp_op(). |
| 141 | * |
| 142 | * After the call R0 is set to return type of the function and registers R1-R5 |
| 143 | * are set to NOT_INIT to indicate that they are no longer readable. |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 144 | * |
| 145 | * The following reference types represent a potential reference to a kernel |
| 146 | * resource which, after first being allocated, must be checked and freed by |
| 147 | * the BPF program: |
| 148 | * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET |
| 149 | * |
| 150 | * When the verifier sees a helper call return a reference type, it allocates a |
| 151 | * pointer id for the reference and stores it in the current function state. |
| 152 | * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into |
| 153 | * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type |
| 154 | * passes through a NULL-check conditional. For the branch wherein the state is |
| 155 | * changed to CONST_IMM, the verifier releases the reference. |
Joe Stringer | 6acc9b4 | 2018-10-02 13:35:36 -0700 | [diff] [blame] | 156 | * |
| 157 | * For each helper function that allocates a reference, such as |
| 158 | * bpf_sk_lookup_tcp(), there is a corresponding release function, such as |
| 159 | * bpf_sk_release(). When a reference type passes into the release function, |
| 160 | * the verifier also releases the reference. If any unchecked or unreleased |
| 161 | * reference remains at the end of the program, the verifier rejects it. |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 162 | */ |
| 163 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 164 | /* verifier_state + insn_idx are pushed to stack when branch is encountered */ |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 165 | struct bpf_verifier_stack_elem { |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 166 | /* verifer state is 'st' |
| 167 | * before processing instruction 'insn_idx' |
| 168 | * and after processing instruction 'prev_insn_idx' |
| 169 | */ |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 170 | struct bpf_verifier_state st; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 171 | int insn_idx; |
| 172 | int prev_insn_idx; |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 173 | struct bpf_verifier_stack_elem *next; |
Andrii Nakryiko | 6f8a57c | 2020-04-23 12:58:50 -0700 | [diff] [blame] | 174 | /* length of verifier log at the time this state was pushed on stack */ |
| 175 | u32 log_pos; |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 176 | }; |
| 177 | |
Alexei Starovoitov | b285fcb | 2019-05-21 20:14:19 -0700 | [diff] [blame] | 178 | #define BPF_COMPLEXITY_LIMIT_JMP_SEQ 8192 |
Alexei Starovoitov | ceefbc9 | 2018-12-03 22:46:06 -0800 | [diff] [blame] | 179 | #define BPF_COMPLEXITY_LIMIT_STATES 64 |
Daniel Borkmann | 0701615 | 2016-04-05 22:33:17 +0200 | [diff] [blame] | 180 | |
Daniel Borkmann | d2e4c1e | 2019-11-22 21:07:59 +0100 | [diff] [blame] | 181 | #define BPF_MAP_KEY_POISON (1ULL << 63) |
| 182 | #define BPF_MAP_KEY_SEEN (1ULL << 62) |
| 183 | |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 184 | #define BPF_MAP_PTR_UNPRIV 1UL |
| 185 | #define BPF_MAP_PTR_POISON ((void *)((0xeB9FUL << 1) + \ |
| 186 | POISON_POINTER_DELTA)) |
| 187 | #define BPF_MAP_PTR(X) ((struct bpf_map *)((X) & ~BPF_MAP_PTR_UNPRIV)) |
| 188 | |
| 189 | static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux) |
| 190 | { |
Daniel Borkmann | d2e4c1e | 2019-11-22 21:07:59 +0100 | [diff] [blame] | 191 | return BPF_MAP_PTR(aux->map_ptr_state) == BPF_MAP_PTR_POISON; |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 192 | } |
| 193 | |
| 194 | static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux) |
| 195 | { |
Daniel Borkmann | d2e4c1e | 2019-11-22 21:07:59 +0100 | [diff] [blame] | 196 | return aux->map_ptr_state & BPF_MAP_PTR_UNPRIV; |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 197 | } |
| 198 | |
| 199 | static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux, |
| 200 | const struct bpf_map *map, bool unpriv) |
| 201 | { |
| 202 | BUILD_BUG_ON((unsigned long)BPF_MAP_PTR_POISON & BPF_MAP_PTR_UNPRIV); |
| 203 | unpriv |= bpf_map_ptr_unpriv(aux); |
Daniel Borkmann | d2e4c1e | 2019-11-22 21:07:59 +0100 | [diff] [blame] | 204 | aux->map_ptr_state = (unsigned long)map | |
| 205 | (unpriv ? BPF_MAP_PTR_UNPRIV : 0UL); |
| 206 | } |
| 207 | |
| 208 | static bool bpf_map_key_poisoned(const struct bpf_insn_aux_data *aux) |
| 209 | { |
| 210 | return aux->map_key_state & BPF_MAP_KEY_POISON; |
| 211 | } |
| 212 | |
| 213 | static bool bpf_map_key_unseen(const struct bpf_insn_aux_data *aux) |
| 214 | { |
| 215 | return !(aux->map_key_state & BPF_MAP_KEY_SEEN); |
| 216 | } |
| 217 | |
| 218 | static u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux) |
| 219 | { |
| 220 | return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON); |
| 221 | } |
| 222 | |
| 223 | static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state) |
| 224 | { |
| 225 | bool poisoned = bpf_map_key_poisoned(aux); |
| 226 | |
| 227 | aux->map_key_state = state | BPF_MAP_KEY_SEEN | |
| 228 | (poisoned ? BPF_MAP_KEY_POISON : 0ULL); |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 229 | } |
Martin KaFai Lau | fad73a1 | 2017-03-22 10:00:32 -0700 | [diff] [blame] | 230 | |
Yonghong Song | 23a2d70 | 2021-02-04 15:48:27 -0800 | [diff] [blame] | 231 | static bool bpf_pseudo_call(const struct bpf_insn *insn) |
| 232 | { |
| 233 | return insn->code == (BPF_JMP | BPF_CALL) && |
| 234 | insn->src_reg == BPF_PSEUDO_CALL; |
| 235 | } |
| 236 | |
Martin KaFai Lau | e6ac245 | 2021-03-24 18:51:42 -0700 | [diff] [blame] | 237 | static bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn) |
| 238 | { |
| 239 | return insn->code == (BPF_JMP | BPF_CALL) && |
| 240 | insn->src_reg == BPF_PSEUDO_KFUNC_CALL; |
| 241 | } |
| 242 | |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 243 | static bool bpf_pseudo_func(const struct bpf_insn *insn) |
| 244 | { |
| 245 | return insn->code == (BPF_LD | BPF_IMM | BPF_DW) && |
| 246 | insn->src_reg == BPF_PSEUDO_FUNC; |
| 247 | } |
| 248 | |
Daniel Borkmann | 33ff982 | 2016-04-13 00:10:50 +0200 | [diff] [blame] | 249 | struct bpf_call_arg_meta { |
| 250 | struct bpf_map *map_ptr; |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 251 | bool raw_mode; |
Daniel Borkmann | 36bbef5 | 2016-09-20 00:26:13 +0200 | [diff] [blame] | 252 | bool pkt_access; |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 253 | int regno; |
| 254 | int access_size; |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 255 | int mem_size; |
John Fastabend | 1006050 | 2020-03-30 14:36:19 -0700 | [diff] [blame] | 256 | u64 msize_max_value; |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 257 | int ref_obj_id; |
Alexei Starovoitov | 3e8ce29 | 2021-07-14 17:54:11 -0700 | [diff] [blame] | 258 | int map_uid; |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 259 | int func_id; |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 260 | struct btf *btf; |
Hao Luo | eaa6bcb | 2020-09-29 16:50:47 -0700 | [diff] [blame] | 261 | u32 btf_id; |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 262 | struct btf *ret_btf; |
Hao Luo | eaa6bcb | 2020-09-29 16:50:47 -0700 | [diff] [blame] | 263 | u32 ret_btf_id; |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 264 | u32 subprogno; |
Daniel Borkmann | 33ff982 | 2016-04-13 00:10:50 +0200 | [diff] [blame] | 265 | }; |
| 266 | |
Alexei Starovoitov | 8580ac9 | 2019-10-15 20:24:57 -0700 | [diff] [blame] | 267 | struct btf *btf_vmlinux; |
| 268 | |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 269 | static DEFINE_MUTEX(bpf_verifier_lock); |
| 270 | |
Martin KaFai Lau | d9762e8 | 2018-12-13 10:41:48 -0800 | [diff] [blame] | 271 | static const struct bpf_line_info * |
| 272 | find_linfo(const struct bpf_verifier_env *env, u32 insn_off) |
| 273 | { |
| 274 | const struct bpf_line_info *linfo; |
| 275 | const struct bpf_prog *prog; |
| 276 | u32 i, nr_linfo; |
| 277 | |
| 278 | prog = env->prog; |
| 279 | nr_linfo = prog->aux->nr_linfo; |
| 280 | |
| 281 | if (!nr_linfo || insn_off >= prog->len) |
| 282 | return NULL; |
| 283 | |
| 284 | linfo = prog->aux->linfo; |
| 285 | for (i = 1; i < nr_linfo; i++) |
| 286 | if (insn_off < linfo[i].insn_off) |
| 287 | break; |
| 288 | |
| 289 | return &linfo[i - 1]; |
| 290 | } |
| 291 | |
Martin KaFai Lau | 77d2e05 | 2018-03-24 11:44:23 -0700 | [diff] [blame] | 292 | void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt, |
| 293 | va_list args) |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 294 | { |
Jakub Kicinski | a2a7d57 | 2017-10-09 10:30:15 -0700 | [diff] [blame] | 295 | unsigned int n; |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 296 | |
Jakub Kicinski | a2a7d57 | 2017-10-09 10:30:15 -0700 | [diff] [blame] | 297 | n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args); |
Jakub Kicinski | a2a7d57 | 2017-10-09 10:30:15 -0700 | [diff] [blame] | 298 | |
| 299 | WARN_ONCE(n >= BPF_VERIFIER_TMP_LOG_SIZE - 1, |
| 300 | "verifier log line truncated - local buffer too short\n"); |
| 301 | |
| 302 | n = min(log->len_total - log->len_used - 1, n); |
| 303 | log->kbuf[n] = '\0'; |
| 304 | |
Alexei Starovoitov | 8580ac9 | 2019-10-15 20:24:57 -0700 | [diff] [blame] | 305 | if (log->level == BPF_LOG_KERNEL) { |
| 306 | pr_err("BPF:%s\n", log->kbuf); |
| 307 | return; |
| 308 | } |
Jakub Kicinski | a2a7d57 | 2017-10-09 10:30:15 -0700 | [diff] [blame] | 309 | if (!copy_to_user(log->ubuf + log->len_used, log->kbuf, n + 1)) |
| 310 | log->len_used += n; |
| 311 | else |
| 312 | log->ubuf = NULL; |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 313 | } |
Jiri Olsa | abe0884 | 2018-03-23 11:41:28 +0100 | [diff] [blame] | 314 | |
Andrii Nakryiko | 6f8a57c | 2020-04-23 12:58:50 -0700 | [diff] [blame] | 315 | static void bpf_vlog_reset(struct bpf_verifier_log *log, u32 new_pos) |
| 316 | { |
| 317 | char zero = 0; |
| 318 | |
| 319 | if (!bpf_verifier_log_needed(log)) |
| 320 | return; |
| 321 | |
| 322 | log->len_used = new_pos; |
| 323 | if (put_user(zero, log->ubuf + new_pos)) |
| 324 | log->ubuf = NULL; |
| 325 | } |
| 326 | |
Jiri Olsa | abe0884 | 2018-03-23 11:41:28 +0100 | [diff] [blame] | 327 | /* log_level controls verbosity level of eBPF verifier. |
| 328 | * bpf_verifier_log_write() is used to dump the verification trace to the log, |
| 329 | * so the user can figure out what's wrong with the program |
Quentin Monnet | 430e68d | 2018-01-10 12:26:06 +0000 | [diff] [blame] | 330 | */ |
Jiri Olsa | abe0884 | 2018-03-23 11:41:28 +0100 | [diff] [blame] | 331 | __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, |
| 332 | const char *fmt, ...) |
| 333 | { |
| 334 | va_list args; |
| 335 | |
Martin KaFai Lau | 77d2e05 | 2018-03-24 11:44:23 -0700 | [diff] [blame] | 336 | if (!bpf_verifier_log_needed(&env->log)) |
| 337 | return; |
| 338 | |
Jiri Olsa | abe0884 | 2018-03-23 11:41:28 +0100 | [diff] [blame] | 339 | va_start(args, fmt); |
Martin KaFai Lau | 77d2e05 | 2018-03-24 11:44:23 -0700 | [diff] [blame] | 340 | bpf_verifier_vlog(&env->log, fmt, args); |
Jiri Olsa | abe0884 | 2018-03-23 11:41:28 +0100 | [diff] [blame] | 341 | va_end(args); |
| 342 | } |
| 343 | EXPORT_SYMBOL_GPL(bpf_verifier_log_write); |
| 344 | |
| 345 | __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...) |
| 346 | { |
Martin KaFai Lau | 77d2e05 | 2018-03-24 11:44:23 -0700 | [diff] [blame] | 347 | struct bpf_verifier_env *env = private_data; |
Jiri Olsa | abe0884 | 2018-03-23 11:41:28 +0100 | [diff] [blame] | 348 | va_list args; |
| 349 | |
Martin KaFai Lau | 77d2e05 | 2018-03-24 11:44:23 -0700 | [diff] [blame] | 350 | if (!bpf_verifier_log_needed(&env->log)) |
| 351 | return; |
| 352 | |
Jiri Olsa | abe0884 | 2018-03-23 11:41:28 +0100 | [diff] [blame] | 353 | va_start(args, fmt); |
Martin KaFai Lau | 77d2e05 | 2018-03-24 11:44:23 -0700 | [diff] [blame] | 354 | bpf_verifier_vlog(&env->log, fmt, args); |
Jiri Olsa | abe0884 | 2018-03-23 11:41:28 +0100 | [diff] [blame] | 355 | va_end(args); |
| 356 | } |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 357 | |
Alexei Starovoitov | 9e15db6 | 2019-10-15 20:25:00 -0700 | [diff] [blame] | 358 | __printf(2, 3) void bpf_log(struct bpf_verifier_log *log, |
| 359 | const char *fmt, ...) |
| 360 | { |
| 361 | va_list args; |
| 362 | |
| 363 | if (!bpf_verifier_log_needed(log)) |
| 364 | return; |
| 365 | |
| 366 | va_start(args, fmt); |
| 367 | bpf_verifier_vlog(log, fmt, args); |
| 368 | va_end(args); |
| 369 | } |
| 370 | |
Martin KaFai Lau | d9762e8 | 2018-12-13 10:41:48 -0800 | [diff] [blame] | 371 | static const char *ltrim(const char *s) |
| 372 | { |
| 373 | while (isspace(*s)) |
| 374 | s++; |
| 375 | |
| 376 | return s; |
| 377 | } |
| 378 | |
| 379 | __printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env, |
| 380 | u32 insn_off, |
| 381 | const char *prefix_fmt, ...) |
| 382 | { |
| 383 | const struct bpf_line_info *linfo; |
| 384 | |
| 385 | if (!bpf_verifier_log_needed(&env->log)) |
| 386 | return; |
| 387 | |
| 388 | linfo = find_linfo(env, insn_off); |
| 389 | if (!linfo || linfo == env->prev_linfo) |
| 390 | return; |
| 391 | |
| 392 | if (prefix_fmt) { |
| 393 | va_list args; |
| 394 | |
| 395 | va_start(args, prefix_fmt); |
| 396 | bpf_verifier_vlog(&env->log, prefix_fmt, args); |
| 397 | va_end(args); |
| 398 | } |
| 399 | |
| 400 | verbose(env, "%s\n", |
| 401 | ltrim(btf_name_by_offset(env->prog->aux->btf, |
| 402 | linfo->line_off))); |
| 403 | |
| 404 | env->prev_linfo = linfo; |
| 405 | } |
| 406 | |
Yonghong Song | bc2591d | 2021-02-26 12:49:22 -0800 | [diff] [blame] | 407 | static void verbose_invalid_scalar(struct bpf_verifier_env *env, |
| 408 | struct bpf_reg_state *reg, |
| 409 | struct tnum *range, const char *ctx, |
| 410 | const char *reg_name) |
| 411 | { |
| 412 | char tn_buf[48]; |
| 413 | |
| 414 | verbose(env, "At %s the register %s ", ctx, reg_name); |
| 415 | if (!tnum_is_unknown(reg->var_off)) { |
| 416 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
| 417 | verbose(env, "has value %s", tn_buf); |
| 418 | } else { |
| 419 | verbose(env, "has unknown scalar value"); |
| 420 | } |
| 421 | tnum_strn(tn_buf, sizeof(tn_buf), *range); |
| 422 | verbose(env, " should have been in %s\n", tn_buf); |
| 423 | } |
| 424 | |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 425 | static bool type_is_pkt_pointer(enum bpf_reg_type type) |
| 426 | { |
| 427 | return type == PTR_TO_PACKET || |
| 428 | type == PTR_TO_PACKET_META; |
| 429 | } |
| 430 | |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 431 | static bool type_is_sk_pointer(enum bpf_reg_type type) |
| 432 | { |
| 433 | return type == PTR_TO_SOCKET || |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 434 | type == PTR_TO_SOCK_COMMON || |
Jonathan Lemon | fada7fd | 2019-06-06 13:59:40 -0700 | [diff] [blame] | 435 | type == PTR_TO_TCP_SOCK || |
| 436 | type == PTR_TO_XDP_SOCK; |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 437 | } |
| 438 | |
John Fastabend | cac616d | 2020-05-21 13:07:26 -0700 | [diff] [blame] | 439 | static bool reg_type_not_null(enum bpf_reg_type type) |
| 440 | { |
| 441 | return type == PTR_TO_SOCKET || |
| 442 | type == PTR_TO_TCP_SOCK || |
| 443 | type == PTR_TO_MAP_VALUE || |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 444 | type == PTR_TO_MAP_KEY || |
Yonghong Song | 01c66c4 | 2020-06-30 10:12:40 -0700 | [diff] [blame] | 445 | type == PTR_TO_SOCK_COMMON; |
John Fastabend | cac616d | 2020-05-21 13:07:26 -0700 | [diff] [blame] | 446 | } |
| 447 | |
Joe Stringer | 840b961 | 2018-10-02 13:35:32 -0700 | [diff] [blame] | 448 | static bool reg_type_may_be_null(enum bpf_reg_type type) |
| 449 | { |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 450 | return type == PTR_TO_MAP_VALUE_OR_NULL || |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 451 | type == PTR_TO_SOCKET_OR_NULL || |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 452 | type == PTR_TO_SOCK_COMMON_OR_NULL || |
Yonghong Song | b121b34 | 2020-05-09 10:59:12 -0700 | [diff] [blame] | 453 | type == PTR_TO_TCP_SOCK_OR_NULL || |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 454 | type == PTR_TO_BTF_ID_OR_NULL || |
Yonghong Song | afbf21d | 2020-07-23 11:41:11 -0700 | [diff] [blame] | 455 | type == PTR_TO_MEM_OR_NULL || |
| 456 | type == PTR_TO_RDONLY_BUF_OR_NULL || |
| 457 | type == PTR_TO_RDWR_BUF_OR_NULL; |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 458 | } |
| 459 | |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 460 | static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) |
| 461 | { |
| 462 | return reg->type == PTR_TO_MAP_VALUE && |
| 463 | map_value_has_spin_lock(reg->map_ptr); |
| 464 | } |
| 465 | |
Martin KaFai Lau | cba368c | 2019-03-18 10:37:13 -0700 | [diff] [blame] | 466 | static bool reg_type_may_be_refcounted_or_null(enum bpf_reg_type type) |
| 467 | { |
| 468 | return type == PTR_TO_SOCKET || |
| 469 | type == PTR_TO_SOCKET_OR_NULL || |
| 470 | type == PTR_TO_TCP_SOCK || |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 471 | type == PTR_TO_TCP_SOCK_OR_NULL || |
| 472 | type == PTR_TO_MEM || |
| 473 | type == PTR_TO_MEM_OR_NULL; |
Martin KaFai Lau | cba368c | 2019-03-18 10:37:13 -0700 | [diff] [blame] | 474 | } |
| 475 | |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 476 | static bool arg_type_may_be_refcounted(enum bpf_arg_type type) |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 477 | { |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 478 | return type == ARG_PTR_TO_SOCK_COMMON; |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 479 | } |
| 480 | |
Lorenz Bauer | fd1b0d6 | 2020-09-21 13:12:26 +0100 | [diff] [blame] | 481 | static bool arg_type_may_be_null(enum bpf_arg_type type) |
| 482 | { |
| 483 | return type == ARG_PTR_TO_MAP_VALUE_OR_NULL || |
| 484 | type == ARG_PTR_TO_MEM_OR_NULL || |
| 485 | type == ARG_PTR_TO_CTX_OR_NULL || |
| 486 | type == ARG_PTR_TO_SOCKET_OR_NULL || |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 487 | type == ARG_PTR_TO_ALLOC_MEM_OR_NULL || |
| 488 | type == ARG_PTR_TO_STACK_OR_NULL; |
Lorenz Bauer | fd1b0d6 | 2020-09-21 13:12:26 +0100 | [diff] [blame] | 489 | } |
| 490 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 491 | /* Determine whether the function releases some resources allocated by another |
| 492 | * function call. The first reference type argument will be assumed to be |
| 493 | * released by release_reference(). |
| 494 | */ |
| 495 | static bool is_release_function(enum bpf_func_id func_id) |
| 496 | { |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 497 | return func_id == BPF_FUNC_sk_release || |
| 498 | func_id == BPF_FUNC_ringbuf_submit || |
| 499 | func_id == BPF_FUNC_ringbuf_discard; |
Joe Stringer | 840b961 | 2018-10-02 13:35:32 -0700 | [diff] [blame] | 500 | } |
| 501 | |
Jakub Sitnicki | 64d8529 | 2020-04-29 20:11:52 +0200 | [diff] [blame] | 502 | static bool may_be_acquire_function(enum bpf_func_id func_id) |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 503 | { |
| 504 | return func_id == BPF_FUNC_sk_lookup_tcp || |
Lorenz Bauer | edbf8c0 | 2019-03-22 09:54:01 +0800 | [diff] [blame] | 505 | func_id == BPF_FUNC_sk_lookup_udp || |
Jakub Sitnicki | 64d8529 | 2020-04-29 20:11:52 +0200 | [diff] [blame] | 506 | func_id == BPF_FUNC_skc_lookup_tcp || |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 507 | func_id == BPF_FUNC_map_lookup_elem || |
| 508 | func_id == BPF_FUNC_ringbuf_reserve; |
Jakub Sitnicki | 64d8529 | 2020-04-29 20:11:52 +0200 | [diff] [blame] | 509 | } |
| 510 | |
| 511 | static bool is_acquire_function(enum bpf_func_id func_id, |
| 512 | const struct bpf_map *map) |
| 513 | { |
| 514 | enum bpf_map_type map_type = map ? map->map_type : BPF_MAP_TYPE_UNSPEC; |
| 515 | |
| 516 | if (func_id == BPF_FUNC_sk_lookup_tcp || |
| 517 | func_id == BPF_FUNC_sk_lookup_udp || |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 518 | func_id == BPF_FUNC_skc_lookup_tcp || |
| 519 | func_id == BPF_FUNC_ringbuf_reserve) |
Jakub Sitnicki | 64d8529 | 2020-04-29 20:11:52 +0200 | [diff] [blame] | 520 | return true; |
| 521 | |
| 522 | if (func_id == BPF_FUNC_map_lookup_elem && |
| 523 | (map_type == BPF_MAP_TYPE_SOCKMAP || |
| 524 | map_type == BPF_MAP_TYPE_SOCKHASH)) |
| 525 | return true; |
| 526 | |
| 527 | return false; |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 528 | } |
| 529 | |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 530 | static bool is_ptr_cast_function(enum bpf_func_id func_id) |
| 531 | { |
| 532 | return func_id == BPF_FUNC_tcp_sock || |
Martin KaFai Lau | 1df8f55 | 2020-09-24 17:03:50 -0700 | [diff] [blame] | 533 | func_id == BPF_FUNC_sk_fullsock || |
| 534 | func_id == BPF_FUNC_skc_to_tcp_sock || |
| 535 | func_id == BPF_FUNC_skc_to_tcp6_sock || |
| 536 | func_id == BPF_FUNC_skc_to_udp6_sock || |
| 537 | func_id == BPF_FUNC_skc_to_tcp_timewait_sock || |
| 538 | func_id == BPF_FUNC_skc_to_tcp_request_sock; |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 539 | } |
| 540 | |
Brendan Jackman | 3949186 | 2021-03-04 18:56:46 -0800 | [diff] [blame] | 541 | static bool is_cmpxchg_insn(const struct bpf_insn *insn) |
| 542 | { |
| 543 | return BPF_CLASS(insn->code) == BPF_STX && |
| 544 | BPF_MODE(insn->code) == BPF_ATOMIC && |
| 545 | insn->imm == BPF_CMPXCHG; |
| 546 | } |
| 547 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 548 | /* string representation of 'enum bpf_reg_type' */ |
| 549 | static const char * const reg_type_str[] = { |
| 550 | [NOT_INIT] = "?", |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 551 | [SCALAR_VALUE] = "inv", |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 552 | [PTR_TO_CTX] = "ctx", |
| 553 | [CONST_PTR_TO_MAP] = "map_ptr", |
| 554 | [PTR_TO_MAP_VALUE] = "map_value", |
| 555 | [PTR_TO_MAP_VALUE_OR_NULL] = "map_value_or_null", |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 556 | [PTR_TO_STACK] = "fp", |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 557 | [PTR_TO_PACKET] = "pkt", |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 558 | [PTR_TO_PACKET_META] = "pkt_meta", |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 559 | [PTR_TO_PACKET_END] = "pkt_end", |
Petar Penkov | d58e468 | 2018-09-14 07:46:18 -0700 | [diff] [blame] | 560 | [PTR_TO_FLOW_KEYS] = "flow_keys", |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 561 | [PTR_TO_SOCKET] = "sock", |
| 562 | [PTR_TO_SOCKET_OR_NULL] = "sock_or_null", |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 563 | [PTR_TO_SOCK_COMMON] = "sock_common", |
| 564 | [PTR_TO_SOCK_COMMON_OR_NULL] = "sock_common_or_null", |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 565 | [PTR_TO_TCP_SOCK] = "tcp_sock", |
| 566 | [PTR_TO_TCP_SOCK_OR_NULL] = "tcp_sock_or_null", |
Matt Mullins | 9df1c28 | 2019-04-26 11:49:47 -0700 | [diff] [blame] | 567 | [PTR_TO_TP_BUFFER] = "tp_buffer", |
Jonathan Lemon | fada7fd | 2019-06-06 13:59:40 -0700 | [diff] [blame] | 568 | [PTR_TO_XDP_SOCK] = "xdp_sock", |
Alexei Starovoitov | 9e15db6 | 2019-10-15 20:25:00 -0700 | [diff] [blame] | 569 | [PTR_TO_BTF_ID] = "ptr_", |
Yonghong Song | b121b34 | 2020-05-09 10:59:12 -0700 | [diff] [blame] | 570 | [PTR_TO_BTF_ID_OR_NULL] = "ptr_or_null_", |
Hao Luo | eaa6bcb | 2020-09-29 16:50:47 -0700 | [diff] [blame] | 571 | [PTR_TO_PERCPU_BTF_ID] = "percpu_ptr_", |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 572 | [PTR_TO_MEM] = "mem", |
| 573 | [PTR_TO_MEM_OR_NULL] = "mem_or_null", |
Yonghong Song | afbf21d | 2020-07-23 11:41:11 -0700 | [diff] [blame] | 574 | [PTR_TO_RDONLY_BUF] = "rdonly_buf", |
| 575 | [PTR_TO_RDONLY_BUF_OR_NULL] = "rdonly_buf_or_null", |
| 576 | [PTR_TO_RDWR_BUF] = "rdwr_buf", |
| 577 | [PTR_TO_RDWR_BUF_OR_NULL] = "rdwr_buf_or_null", |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 578 | [PTR_TO_FUNC] = "func", |
| 579 | [PTR_TO_MAP_KEY] = "map_key", |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 580 | }; |
| 581 | |
Edward Cree | 8efea21 | 2018-08-22 20:02:44 +0100 | [diff] [blame] | 582 | static char slot_type_char[] = { |
| 583 | [STACK_INVALID] = '?', |
| 584 | [STACK_SPILL] = 'r', |
| 585 | [STACK_MISC] = 'm', |
| 586 | [STACK_ZERO] = '0', |
| 587 | }; |
| 588 | |
Alexei Starovoitov | 4e92024 | 2017-11-30 21:31:36 -0800 | [diff] [blame] | 589 | static void print_liveness(struct bpf_verifier_env *env, |
| 590 | enum bpf_reg_liveness live) |
| 591 | { |
Alexei Starovoitov | 9242b5f | 2018-12-13 11:42:34 -0800 | [diff] [blame] | 592 | if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE)) |
Alexei Starovoitov | 4e92024 | 2017-11-30 21:31:36 -0800 | [diff] [blame] | 593 | verbose(env, "_"); |
| 594 | if (live & REG_LIVE_READ) |
| 595 | verbose(env, "r"); |
| 596 | if (live & REG_LIVE_WRITTEN) |
| 597 | verbose(env, "w"); |
Alexei Starovoitov | 9242b5f | 2018-12-13 11:42:34 -0800 | [diff] [blame] | 598 | if (live & REG_LIVE_DONE) |
| 599 | verbose(env, "D"); |
Alexei Starovoitov | 4e92024 | 2017-11-30 21:31:36 -0800 | [diff] [blame] | 600 | } |
| 601 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 602 | static struct bpf_func_state *func(struct bpf_verifier_env *env, |
| 603 | const struct bpf_reg_state *reg) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 604 | { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 605 | struct bpf_verifier_state *cur = env->cur_state; |
| 606 | |
| 607 | return cur->frame[reg->frameno]; |
| 608 | } |
| 609 | |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 610 | static const char *kernel_type_name(const struct btf* btf, u32 id) |
Alexei Starovoitov | 9e15db6 | 2019-10-15 20:25:00 -0700 | [diff] [blame] | 611 | { |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 612 | return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off); |
Alexei Starovoitov | 9e15db6 | 2019-10-15 20:25:00 -0700 | [diff] [blame] | 613 | } |
| 614 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 615 | static void print_verifier_state(struct bpf_verifier_env *env, |
| 616 | const struct bpf_func_state *state) |
| 617 | { |
| 618 | const struct bpf_reg_state *reg; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 619 | enum bpf_reg_type t; |
| 620 | int i; |
| 621 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 622 | if (state->frameno) |
| 623 | verbose(env, " frame%d:", state->frameno); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 624 | for (i = 0; i < MAX_BPF_REG; i++) { |
Alexei Starovoitov | 1a0dc1a | 2016-05-05 19:49:09 -0700 | [diff] [blame] | 625 | reg = &state->regs[i]; |
| 626 | t = reg->type; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 627 | if (t == NOT_INIT) |
| 628 | continue; |
Alexei Starovoitov | 4e92024 | 2017-11-30 21:31:36 -0800 | [diff] [blame] | 629 | verbose(env, " R%d", i); |
| 630 | print_liveness(env, reg->live); |
| 631 | verbose(env, "=%s", reg_type_str[t]); |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 632 | if (t == SCALAR_VALUE && reg->precise) |
| 633 | verbose(env, "P"); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 634 | if ((t == SCALAR_VALUE || t == PTR_TO_STACK) && |
| 635 | tnum_is_const(reg->var_off)) { |
| 636 | /* reg->off should be 0 for SCALAR_VALUE */ |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 637 | verbose(env, "%lld", reg->var_off.value + reg->off); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 638 | } else { |
Hao Luo | eaa6bcb | 2020-09-29 16:50:47 -0700 | [diff] [blame] | 639 | if (t == PTR_TO_BTF_ID || |
| 640 | t == PTR_TO_BTF_ID_OR_NULL || |
| 641 | t == PTR_TO_PERCPU_BTF_ID) |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 642 | verbose(env, "%s", kernel_type_name(reg->btf, reg->btf_id)); |
Martin KaFai Lau | cba368c | 2019-03-18 10:37:13 -0700 | [diff] [blame] | 643 | verbose(env, "(id=%d", reg->id); |
| 644 | if (reg_type_may_be_refcounted_or_null(t)) |
| 645 | verbose(env, ",ref_obj_id=%d", reg->ref_obj_id); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 646 | if (t != SCALAR_VALUE) |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 647 | verbose(env, ",off=%d", reg->off); |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 648 | if (type_is_pkt_pointer(t)) |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 649 | verbose(env, ",r=%d", reg->range); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 650 | else if (t == CONST_PTR_TO_MAP || |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 651 | t == PTR_TO_MAP_KEY || |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 652 | t == PTR_TO_MAP_VALUE || |
| 653 | t == PTR_TO_MAP_VALUE_OR_NULL) |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 654 | verbose(env, ",ks=%d,vs=%d", |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 655 | reg->map_ptr->key_size, |
| 656 | reg->map_ptr->value_size); |
Edward Cree | 7d1238f | 2017-08-07 15:26:56 +0100 | [diff] [blame] | 657 | if (tnum_is_const(reg->var_off)) { |
| 658 | /* Typically an immediate SCALAR_VALUE, but |
| 659 | * could be a pointer whose offset is too big |
| 660 | * for reg->off |
| 661 | */ |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 662 | verbose(env, ",imm=%llx", reg->var_off.value); |
Edward Cree | 7d1238f | 2017-08-07 15:26:56 +0100 | [diff] [blame] | 663 | } else { |
| 664 | if (reg->smin_value != reg->umin_value && |
| 665 | reg->smin_value != S64_MIN) |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 666 | verbose(env, ",smin_value=%lld", |
Edward Cree | 7d1238f | 2017-08-07 15:26:56 +0100 | [diff] [blame] | 667 | (long long)reg->smin_value); |
| 668 | if (reg->smax_value != reg->umax_value && |
| 669 | reg->smax_value != S64_MAX) |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 670 | verbose(env, ",smax_value=%lld", |
Edward Cree | 7d1238f | 2017-08-07 15:26:56 +0100 | [diff] [blame] | 671 | (long long)reg->smax_value); |
| 672 | if (reg->umin_value != 0) |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 673 | verbose(env, ",umin_value=%llu", |
Edward Cree | 7d1238f | 2017-08-07 15:26:56 +0100 | [diff] [blame] | 674 | (unsigned long long)reg->umin_value); |
| 675 | if (reg->umax_value != U64_MAX) |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 676 | verbose(env, ",umax_value=%llu", |
Edward Cree | 7d1238f | 2017-08-07 15:26:56 +0100 | [diff] [blame] | 677 | (unsigned long long)reg->umax_value); |
| 678 | if (!tnum_is_unknown(reg->var_off)) { |
| 679 | char tn_buf[48]; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 680 | |
Edward Cree | 7d1238f | 2017-08-07 15:26:56 +0100 | [diff] [blame] | 681 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 682 | verbose(env, ",var_off=%s", tn_buf); |
Edward Cree | 7d1238f | 2017-08-07 15:26:56 +0100 | [diff] [blame] | 683 | } |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 684 | if (reg->s32_min_value != reg->smin_value && |
| 685 | reg->s32_min_value != S32_MIN) |
| 686 | verbose(env, ",s32_min_value=%d", |
| 687 | (int)(reg->s32_min_value)); |
| 688 | if (reg->s32_max_value != reg->smax_value && |
| 689 | reg->s32_max_value != S32_MAX) |
| 690 | verbose(env, ",s32_max_value=%d", |
| 691 | (int)(reg->s32_max_value)); |
| 692 | if (reg->u32_min_value != reg->umin_value && |
| 693 | reg->u32_min_value != U32_MIN) |
| 694 | verbose(env, ",u32_min_value=%d", |
| 695 | (int)(reg->u32_min_value)); |
| 696 | if (reg->u32_max_value != reg->umax_value && |
| 697 | reg->u32_max_value != U32_MAX) |
| 698 | verbose(env, ",u32_max_value=%d", |
| 699 | (int)(reg->u32_max_value)); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 700 | } |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 701 | verbose(env, ")"); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 702 | } |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 703 | } |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 704 | for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { |
Edward Cree | 8efea21 | 2018-08-22 20:02:44 +0100 | [diff] [blame] | 705 | char types_buf[BPF_REG_SIZE + 1]; |
| 706 | bool valid = false; |
| 707 | int j; |
| 708 | |
| 709 | for (j = 0; j < BPF_REG_SIZE; j++) { |
| 710 | if (state->stack[i].slot_type[j] != STACK_INVALID) |
| 711 | valid = true; |
| 712 | types_buf[j] = slot_type_char[ |
| 713 | state->stack[i].slot_type[j]]; |
| 714 | } |
| 715 | types_buf[BPF_REG_SIZE] = 0; |
| 716 | if (!valid) |
| 717 | continue; |
| 718 | verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); |
| 719 | print_liveness(env, state->stack[i].spilled_ptr.live); |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 720 | if (state->stack[i].slot_type[0] == STACK_SPILL) { |
| 721 | reg = &state->stack[i].spilled_ptr; |
| 722 | t = reg->type; |
| 723 | verbose(env, "=%s", reg_type_str[t]); |
| 724 | if (t == SCALAR_VALUE && reg->precise) |
| 725 | verbose(env, "P"); |
| 726 | if (t == SCALAR_VALUE && tnum_is_const(reg->var_off)) |
| 727 | verbose(env, "%lld", reg->var_off.value + reg->off); |
| 728 | } else { |
Edward Cree | 8efea21 | 2018-08-22 20:02:44 +0100 | [diff] [blame] | 729 | verbose(env, "=%s", types_buf); |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 730 | } |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 731 | } |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 732 | if (state->acquired_refs && state->refs[0].id) { |
| 733 | verbose(env, " refs=%d", state->refs[0].id); |
| 734 | for (i = 1; i < state->acquired_refs; i++) |
| 735 | if (state->refs[i].id) |
| 736 | verbose(env, ",%d", state->refs[i].id); |
| 737 | } |
Alexei Starovoitov | bfc6bb7 | 2021-07-14 17:54:14 -0700 | [diff] [blame] | 738 | if (state->in_callback_fn) |
| 739 | verbose(env, " cb"); |
| 740 | if (state->in_async_callback_fn) |
| 741 | verbose(env, " async_cb"); |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 742 | verbose(env, "\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 743 | } |
| 744 | |
Lorenz Bauer | c69431a | 2021-04-29 14:46:54 +0100 | [diff] [blame] | 745 | /* copy array src of length n * size bytes to dst. dst is reallocated if it's too |
| 746 | * small to hold src. This is different from krealloc since we don't want to preserve |
| 747 | * the contents of dst. |
| 748 | * |
| 749 | * Leaves dst untouched if src is NULL or length is zero. Returns NULL if memory could |
| 750 | * not be allocated. |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 751 | */ |
Lorenz Bauer | c69431a | 2021-04-29 14:46:54 +0100 | [diff] [blame] | 752 | static void *copy_array(void *dst, const void *src, size_t n, size_t size, gfp_t flags) |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 753 | { |
Lorenz Bauer | c69431a | 2021-04-29 14:46:54 +0100 | [diff] [blame] | 754 | size_t bytes; |
| 755 | |
| 756 | if (ZERO_OR_NULL_PTR(src)) |
| 757 | goto out; |
| 758 | |
| 759 | if (unlikely(check_mul_overflow(n, size, &bytes))) |
| 760 | return NULL; |
| 761 | |
| 762 | if (ksize(dst) < bytes) { |
| 763 | kfree(dst); |
| 764 | dst = kmalloc_track_caller(bytes, flags); |
| 765 | if (!dst) |
| 766 | return NULL; |
| 767 | } |
| 768 | |
| 769 | memcpy(dst, src, bytes); |
| 770 | out: |
| 771 | return dst ? dst : ZERO_SIZE_PTR; |
| 772 | } |
| 773 | |
| 774 | /* resize an array from old_n items to new_n items. the array is reallocated if it's too |
| 775 | * small to hold new_n items. new items are zeroed out if the array grows. |
| 776 | * |
| 777 | * Contrary to krealloc_array, does not free arr if new_n is zero. |
| 778 | */ |
| 779 | static void *realloc_array(void *arr, size_t old_n, size_t new_n, size_t size) |
| 780 | { |
| 781 | if (!new_n || old_n == new_n) |
| 782 | goto out; |
| 783 | |
| 784 | arr = krealloc_array(arr, new_n, size, GFP_KERNEL); |
| 785 | if (!arr) |
| 786 | return NULL; |
| 787 | |
| 788 | if (new_n > old_n) |
| 789 | memset(arr + old_n * size, 0, (new_n - old_n) * size); |
| 790 | |
| 791 | out: |
| 792 | return arr ? arr : ZERO_SIZE_PTR; |
| 793 | } |
| 794 | |
| 795 | static int copy_reference_state(struct bpf_func_state *dst, const struct bpf_func_state *src) |
| 796 | { |
| 797 | dst->refs = copy_array(dst->refs, src->refs, src->acquired_refs, |
| 798 | sizeof(struct bpf_reference_state), GFP_KERNEL); |
| 799 | if (!dst->refs) |
| 800 | return -ENOMEM; |
| 801 | |
| 802 | dst->acquired_refs = src->acquired_refs; |
| 803 | return 0; |
| 804 | } |
| 805 | |
| 806 | static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_state *src) |
| 807 | { |
| 808 | size_t n = src->allocated_stack / BPF_REG_SIZE; |
| 809 | |
| 810 | dst->stack = copy_array(dst->stack, src->stack, n, sizeof(struct bpf_stack_state), |
| 811 | GFP_KERNEL); |
| 812 | if (!dst->stack) |
| 813 | return -ENOMEM; |
| 814 | |
| 815 | dst->allocated_stack = src->allocated_stack; |
| 816 | return 0; |
| 817 | } |
| 818 | |
| 819 | static int resize_reference_state(struct bpf_func_state *state, size_t n) |
| 820 | { |
| 821 | state->refs = realloc_array(state->refs, state->acquired_refs, n, |
| 822 | sizeof(struct bpf_reference_state)); |
| 823 | if (!state->refs) |
| 824 | return -ENOMEM; |
| 825 | |
| 826 | state->acquired_refs = n; |
| 827 | return 0; |
| 828 | } |
| 829 | |
| 830 | static int grow_stack_state(struct bpf_func_state *state, int size) |
| 831 | { |
| 832 | size_t old_n = state->allocated_stack / BPF_REG_SIZE, n = size / BPF_REG_SIZE; |
| 833 | |
| 834 | if (old_n >= n) |
| 835 | return 0; |
| 836 | |
| 837 | state->stack = realloc_array(state->stack, old_n, n, sizeof(struct bpf_stack_state)); |
| 838 | if (!state->stack) |
| 839 | return -ENOMEM; |
| 840 | |
| 841 | state->allocated_stack = size; |
| 842 | return 0; |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 843 | } |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 844 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 845 | /* Acquire a pointer id from the env and update the state->refs to include |
| 846 | * this new pointer reference. |
| 847 | * On success, returns a valid pointer id to associate with the register |
| 848 | * On failure, returns a negative errno. |
| 849 | */ |
| 850 | static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx) |
| 851 | { |
| 852 | struct bpf_func_state *state = cur_func(env); |
| 853 | int new_ofs = state->acquired_refs; |
| 854 | int id, err; |
| 855 | |
Lorenz Bauer | c69431a | 2021-04-29 14:46:54 +0100 | [diff] [blame] | 856 | err = resize_reference_state(state, state->acquired_refs + 1); |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 857 | if (err) |
| 858 | return err; |
| 859 | id = ++env->id_gen; |
| 860 | state->refs[new_ofs].id = id; |
| 861 | state->refs[new_ofs].insn_idx = insn_idx; |
| 862 | |
| 863 | return id; |
| 864 | } |
| 865 | |
| 866 | /* release function corresponding to acquire_reference_state(). Idempotent. */ |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 867 | static int release_reference_state(struct bpf_func_state *state, int ptr_id) |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 868 | { |
| 869 | int i, last_idx; |
| 870 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 871 | last_idx = state->acquired_refs - 1; |
| 872 | for (i = 0; i < state->acquired_refs; i++) { |
| 873 | if (state->refs[i].id == ptr_id) { |
| 874 | if (last_idx && i != last_idx) |
| 875 | memcpy(&state->refs[i], &state->refs[last_idx], |
| 876 | sizeof(*state->refs)); |
| 877 | memset(&state->refs[last_idx], 0, sizeof(*state->refs)); |
| 878 | state->acquired_refs--; |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 879 | return 0; |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 880 | } |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 881 | } |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 882 | return -EINVAL; |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 883 | } |
| 884 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 885 | static void free_func_state(struct bpf_func_state *state) |
| 886 | { |
Alexei Starovoitov | 5896351 | 2018-01-08 07:51:17 -0800 | [diff] [blame] | 887 | if (!state) |
| 888 | return; |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 889 | kfree(state->refs); |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 890 | kfree(state->stack); |
| 891 | kfree(state); |
| 892 | } |
| 893 | |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 894 | static void clear_jmp_history(struct bpf_verifier_state *state) |
| 895 | { |
| 896 | kfree(state->jmp_history); |
| 897 | state->jmp_history = NULL; |
| 898 | state->jmp_history_cnt = 0; |
| 899 | } |
| 900 | |
Alexei Starovoitov | 1969db4 | 2017-11-01 00:08:04 -0700 | [diff] [blame] | 901 | static void free_verifier_state(struct bpf_verifier_state *state, |
| 902 | bool free_self) |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 903 | { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 904 | int i; |
| 905 | |
| 906 | for (i = 0; i <= state->curframe; i++) { |
| 907 | free_func_state(state->frame[i]); |
| 908 | state->frame[i] = NULL; |
| 909 | } |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 910 | clear_jmp_history(state); |
Alexei Starovoitov | 1969db4 | 2017-11-01 00:08:04 -0700 | [diff] [blame] | 911 | if (free_self) |
| 912 | kfree(state); |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 913 | } |
| 914 | |
| 915 | /* copy verifier state from src to dst growing dst stack space |
| 916 | * when necessary to accommodate larger src stack |
| 917 | */ |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 918 | static int copy_func_state(struct bpf_func_state *dst, |
| 919 | const struct bpf_func_state *src) |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 920 | { |
| 921 | int err; |
| 922 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 923 | memcpy(dst, src, offsetof(struct bpf_func_state, acquired_refs)); |
| 924 | err = copy_reference_state(dst, src); |
| 925 | if (err) |
| 926 | return err; |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 927 | return copy_stack_state(dst, src); |
| 928 | } |
| 929 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 930 | static int copy_verifier_state(struct bpf_verifier_state *dst_state, |
| 931 | const struct bpf_verifier_state *src) |
| 932 | { |
| 933 | struct bpf_func_state *dst; |
| 934 | int i, err; |
| 935 | |
Lorenz Bauer | 06ab6a5 | 2021-04-29 14:46:55 +0100 | [diff] [blame] | 936 | dst_state->jmp_history = copy_array(dst_state->jmp_history, src->jmp_history, |
| 937 | src->jmp_history_cnt, sizeof(struct bpf_idx_pair), |
| 938 | GFP_USER); |
| 939 | if (!dst_state->jmp_history) |
| 940 | return -ENOMEM; |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 941 | dst_state->jmp_history_cnt = src->jmp_history_cnt; |
| 942 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 943 | /* if dst has more stack frames then src frame, free them */ |
| 944 | for (i = src->curframe + 1; i <= dst_state->curframe; i++) { |
| 945 | free_func_state(dst_state->frame[i]); |
| 946 | dst_state->frame[i] = NULL; |
| 947 | } |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 948 | dst_state->speculative = src->speculative; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 949 | dst_state->curframe = src->curframe; |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 950 | dst_state->active_spin_lock = src->active_spin_lock; |
Alexei Starovoitov | 2589726 | 2019-06-15 12:12:20 -0700 | [diff] [blame] | 951 | dst_state->branches = src->branches; |
| 952 | dst_state->parent = src->parent; |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 953 | dst_state->first_insn_idx = src->first_insn_idx; |
| 954 | dst_state->last_insn_idx = src->last_insn_idx; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 955 | for (i = 0; i <= src->curframe; i++) { |
| 956 | dst = dst_state->frame[i]; |
| 957 | if (!dst) { |
| 958 | dst = kzalloc(sizeof(*dst), GFP_KERNEL); |
| 959 | if (!dst) |
| 960 | return -ENOMEM; |
| 961 | dst_state->frame[i] = dst; |
| 962 | } |
| 963 | err = copy_func_state(dst, src->frame[i]); |
| 964 | if (err) |
| 965 | return err; |
| 966 | } |
| 967 | return 0; |
| 968 | } |
| 969 | |
Alexei Starovoitov | 2589726 | 2019-06-15 12:12:20 -0700 | [diff] [blame] | 970 | static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st) |
| 971 | { |
| 972 | while (st) { |
| 973 | u32 br = --st->branches; |
| 974 | |
| 975 | /* WARN_ON(br > 1) technically makes sense here, |
| 976 | * but see comment in push_stack(), hence: |
| 977 | */ |
| 978 | WARN_ONCE((int)br < 0, |
| 979 | "BUG update_branch_counts:branches_to_explore=%d\n", |
| 980 | br); |
| 981 | if (br) |
| 982 | break; |
| 983 | st = st->parent; |
| 984 | } |
| 985 | } |
| 986 | |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 987 | static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, |
Andrii Nakryiko | 6f8a57c | 2020-04-23 12:58:50 -0700 | [diff] [blame] | 988 | int *insn_idx, bool pop_log) |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 989 | { |
| 990 | struct bpf_verifier_state *cur = env->cur_state; |
| 991 | struct bpf_verifier_stack_elem *elem, *head = env->head; |
| 992 | int err; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 993 | |
| 994 | if (env->head == NULL) |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 995 | return -ENOENT; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 996 | |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 997 | if (cur) { |
| 998 | err = copy_verifier_state(cur, &head->st); |
| 999 | if (err) |
| 1000 | return err; |
| 1001 | } |
Andrii Nakryiko | 6f8a57c | 2020-04-23 12:58:50 -0700 | [diff] [blame] | 1002 | if (pop_log) |
| 1003 | bpf_vlog_reset(&env->log, head->log_pos); |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 1004 | if (insn_idx) |
| 1005 | *insn_idx = head->insn_idx; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1006 | if (prev_insn_idx) |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 1007 | *prev_insn_idx = head->prev_insn_idx; |
| 1008 | elem = head->next; |
Alexei Starovoitov | 1969db4 | 2017-11-01 00:08:04 -0700 | [diff] [blame] | 1009 | free_verifier_state(&head->st, false); |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 1010 | kfree(head); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1011 | env->head = elem; |
| 1012 | env->stack_size--; |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 1013 | return 0; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1014 | } |
| 1015 | |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 1016 | static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 1017 | int insn_idx, int prev_insn_idx, |
| 1018 | bool speculative) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1019 | { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 1020 | struct bpf_verifier_state *cur = env->cur_state; |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 1021 | struct bpf_verifier_stack_elem *elem; |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 1022 | int err; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1023 | |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 1024 | elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1025 | if (!elem) |
| 1026 | goto err; |
| 1027 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1028 | elem->insn_idx = insn_idx; |
| 1029 | elem->prev_insn_idx = prev_insn_idx; |
| 1030 | elem->next = env->head; |
Andrii Nakryiko | 6f8a57c | 2020-04-23 12:58:50 -0700 | [diff] [blame] | 1031 | elem->log_pos = env->log.len_used; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1032 | env->head = elem; |
| 1033 | env->stack_size++; |
Alexei Starovoitov | 1969db4 | 2017-11-01 00:08:04 -0700 | [diff] [blame] | 1034 | err = copy_verifier_state(&elem->st, cur); |
| 1035 | if (err) |
| 1036 | goto err; |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 1037 | elem->st.speculative |= speculative; |
Alexei Starovoitov | b285fcb | 2019-05-21 20:14:19 -0700 | [diff] [blame] | 1038 | if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { |
| 1039 | verbose(env, "The sequence of %d jumps is too complex.\n", |
| 1040 | env->stack_size); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1041 | goto err; |
| 1042 | } |
Alexei Starovoitov | 2589726 | 2019-06-15 12:12:20 -0700 | [diff] [blame] | 1043 | if (elem->st.parent) { |
| 1044 | ++elem->st.parent->branches; |
| 1045 | /* WARN_ON(branches > 2) technically makes sense here, |
| 1046 | * but |
| 1047 | * 1. speculative states will bump 'branches' for non-branch |
| 1048 | * instructions |
| 1049 | * 2. is_state_visited() heuristics may decide not to create |
| 1050 | * a new state for a sequence of branches and all such current |
| 1051 | * and cloned states will be pointing to a single parent state |
| 1052 | * which might have large 'branches' count. |
| 1053 | */ |
| 1054 | } |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1055 | return &elem->st; |
| 1056 | err: |
Alexei Starovoitov | 5896351 | 2018-01-08 07:51:17 -0800 | [diff] [blame] | 1057 | free_verifier_state(env->cur_state, true); |
| 1058 | env->cur_state = NULL; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1059 | /* pop all elements and return */ |
Andrii Nakryiko | 6f8a57c | 2020-04-23 12:58:50 -0700 | [diff] [blame] | 1060 | while (!pop_stack(env, NULL, NULL, false)); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1061 | return NULL; |
| 1062 | } |
| 1063 | |
| 1064 | #define CALLER_SAVED_REGS 6 |
| 1065 | static const int caller_saved[CALLER_SAVED_REGS] = { |
| 1066 | BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5 |
| 1067 | }; |
| 1068 | |
Daniel Borkmann | f54c789 | 2019-12-22 23:37:40 +0100 | [diff] [blame] | 1069 | static void __mark_reg_not_init(const struct bpf_verifier_env *env, |
| 1070 | struct bpf_reg_state *reg); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1071 | |
Alexei Starovoitov | e688c3d | 2020-10-14 10:56:08 -0700 | [diff] [blame] | 1072 | /* This helper doesn't clear reg->id */ |
| 1073 | static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm) |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 1074 | { |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 1075 | reg->var_off = tnum_const(imm); |
| 1076 | reg->smin_value = (s64)imm; |
| 1077 | reg->smax_value = (s64)imm; |
| 1078 | reg->umin_value = imm; |
| 1079 | reg->umax_value = imm; |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 1080 | |
| 1081 | reg->s32_min_value = (s32)imm; |
| 1082 | reg->s32_max_value = (s32)imm; |
| 1083 | reg->u32_min_value = (u32)imm; |
| 1084 | reg->u32_max_value = (u32)imm; |
| 1085 | } |
| 1086 | |
Alexei Starovoitov | e688c3d | 2020-10-14 10:56:08 -0700 | [diff] [blame] | 1087 | /* Mark the unknown part of a register (variable offset or scalar value) as |
| 1088 | * known to have the value @imm. |
| 1089 | */ |
| 1090 | static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm) |
| 1091 | { |
| 1092 | /* Clear id, off, and union(map_ptr, range) */ |
| 1093 | memset(((u8 *)reg) + sizeof(reg->type), 0, |
| 1094 | offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type)); |
| 1095 | ___mark_reg_known(reg, imm); |
| 1096 | } |
| 1097 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 1098 | static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm) |
| 1099 | { |
| 1100 | reg->var_off = tnum_const_subreg(reg->var_off, imm); |
| 1101 | reg->s32_min_value = (s32)imm; |
| 1102 | reg->s32_max_value = (s32)imm; |
| 1103 | reg->u32_min_value = (u32)imm; |
| 1104 | reg->u32_max_value = (u32)imm; |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 1105 | } |
| 1106 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1107 | /* Mark the 'variable offset' part of a register as zero. This should be |
| 1108 | * used only on registers holding a pointer type. |
| 1109 | */ |
| 1110 | static void __mark_reg_known_zero(struct bpf_reg_state *reg) |
| 1111 | { |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 1112 | __mark_reg_known(reg, 0); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1113 | } |
| 1114 | |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 1115 | static void __mark_reg_const_zero(struct bpf_reg_state *reg) |
| 1116 | { |
| 1117 | __mark_reg_known(reg, 0); |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 1118 | reg->type = SCALAR_VALUE; |
| 1119 | } |
| 1120 | |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1121 | static void mark_reg_known_zero(struct bpf_verifier_env *env, |
| 1122 | struct bpf_reg_state *regs, u32 regno) |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1123 | { |
| 1124 | if (WARN_ON(regno >= MAX_BPF_REG)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1125 | verbose(env, "mark_reg_known_zero(regs, %u)\n", regno); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1126 | /* Something bad happened, let's kill all regs */ |
| 1127 | for (regno = 0; regno < MAX_BPF_REG; regno++) |
Daniel Borkmann | f54c789 | 2019-12-22 23:37:40 +0100 | [diff] [blame] | 1128 | __mark_reg_not_init(env, regs + regno); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1129 | return; |
| 1130 | } |
| 1131 | __mark_reg_known_zero(regs + regno); |
| 1132 | } |
| 1133 | |
Dmitrii Banshchikov | 4ddb741 | 2021-02-13 00:56:40 +0400 | [diff] [blame] | 1134 | static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) |
| 1135 | { |
| 1136 | switch (reg->type) { |
| 1137 | case PTR_TO_MAP_VALUE_OR_NULL: { |
| 1138 | const struct bpf_map *map = reg->map_ptr; |
| 1139 | |
| 1140 | if (map->inner_map_meta) { |
| 1141 | reg->type = CONST_PTR_TO_MAP; |
| 1142 | reg->map_ptr = map->inner_map_meta; |
Alexei Starovoitov | 3e8ce29 | 2021-07-14 17:54:11 -0700 | [diff] [blame] | 1143 | /* transfer reg's id which is unique for every map_lookup_elem |
| 1144 | * as UID of the inner map. |
| 1145 | */ |
| 1146 | reg->map_uid = reg->id; |
Dmitrii Banshchikov | 4ddb741 | 2021-02-13 00:56:40 +0400 | [diff] [blame] | 1147 | } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { |
| 1148 | reg->type = PTR_TO_XDP_SOCK; |
| 1149 | } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || |
| 1150 | map->map_type == BPF_MAP_TYPE_SOCKHASH) { |
| 1151 | reg->type = PTR_TO_SOCKET; |
| 1152 | } else { |
| 1153 | reg->type = PTR_TO_MAP_VALUE; |
| 1154 | } |
| 1155 | break; |
| 1156 | } |
| 1157 | case PTR_TO_SOCKET_OR_NULL: |
| 1158 | reg->type = PTR_TO_SOCKET; |
| 1159 | break; |
| 1160 | case PTR_TO_SOCK_COMMON_OR_NULL: |
| 1161 | reg->type = PTR_TO_SOCK_COMMON; |
| 1162 | break; |
| 1163 | case PTR_TO_TCP_SOCK_OR_NULL: |
| 1164 | reg->type = PTR_TO_TCP_SOCK; |
| 1165 | break; |
| 1166 | case PTR_TO_BTF_ID_OR_NULL: |
| 1167 | reg->type = PTR_TO_BTF_ID; |
| 1168 | break; |
| 1169 | case PTR_TO_MEM_OR_NULL: |
| 1170 | reg->type = PTR_TO_MEM; |
| 1171 | break; |
| 1172 | case PTR_TO_RDONLY_BUF_OR_NULL: |
| 1173 | reg->type = PTR_TO_RDONLY_BUF; |
| 1174 | break; |
| 1175 | case PTR_TO_RDWR_BUF_OR_NULL: |
| 1176 | reg->type = PTR_TO_RDWR_BUF; |
| 1177 | break; |
| 1178 | default: |
Dan Carpenter | 33ccec5 | 2021-02-17 10:45:25 +0300 | [diff] [blame] | 1179 | WARN_ONCE(1, "unknown nullable register type"); |
Dmitrii Banshchikov | 4ddb741 | 2021-02-13 00:56:40 +0400 | [diff] [blame] | 1180 | } |
| 1181 | } |
| 1182 | |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 1183 | static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg) |
| 1184 | { |
| 1185 | return type_is_pkt_pointer(reg->type); |
| 1186 | } |
| 1187 | |
| 1188 | static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg) |
| 1189 | { |
| 1190 | return reg_is_pkt_pointer(reg) || |
| 1191 | reg->type == PTR_TO_PACKET_END; |
| 1192 | } |
| 1193 | |
| 1194 | /* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */ |
| 1195 | static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg, |
| 1196 | enum bpf_reg_type which) |
| 1197 | { |
| 1198 | /* The register can already have a range from prior markings. |
| 1199 | * This is fine as long as it hasn't been advanced from its |
| 1200 | * origin. |
| 1201 | */ |
| 1202 | return reg->type == which && |
| 1203 | reg->id == 0 && |
| 1204 | reg->off == 0 && |
| 1205 | tnum_equals_const(reg->var_off, 0); |
| 1206 | } |
| 1207 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 1208 | /* Reset the min/max bounds of a register */ |
| 1209 | static void __mark_reg_unbounded(struct bpf_reg_state *reg) |
| 1210 | { |
| 1211 | reg->smin_value = S64_MIN; |
| 1212 | reg->smax_value = S64_MAX; |
| 1213 | reg->umin_value = 0; |
| 1214 | reg->umax_value = U64_MAX; |
| 1215 | |
| 1216 | reg->s32_min_value = S32_MIN; |
| 1217 | reg->s32_max_value = S32_MAX; |
| 1218 | reg->u32_min_value = 0; |
| 1219 | reg->u32_max_value = U32_MAX; |
| 1220 | } |
| 1221 | |
| 1222 | static void __mark_reg64_unbounded(struct bpf_reg_state *reg) |
| 1223 | { |
| 1224 | reg->smin_value = S64_MIN; |
| 1225 | reg->smax_value = S64_MAX; |
| 1226 | reg->umin_value = 0; |
| 1227 | reg->umax_value = U64_MAX; |
| 1228 | } |
| 1229 | |
| 1230 | static void __mark_reg32_unbounded(struct bpf_reg_state *reg) |
| 1231 | { |
| 1232 | reg->s32_min_value = S32_MIN; |
| 1233 | reg->s32_max_value = S32_MAX; |
| 1234 | reg->u32_min_value = 0; |
| 1235 | reg->u32_max_value = U32_MAX; |
| 1236 | } |
| 1237 | |
| 1238 | static void __update_reg32_bounds(struct bpf_reg_state *reg) |
| 1239 | { |
| 1240 | struct tnum var32_off = tnum_subreg(reg->var_off); |
| 1241 | |
| 1242 | /* min signed is max(sign bit) | min(other bits) */ |
| 1243 | reg->s32_min_value = max_t(s32, reg->s32_min_value, |
| 1244 | var32_off.value | (var32_off.mask & S32_MIN)); |
| 1245 | /* max signed is min(sign bit) | max(other bits) */ |
| 1246 | reg->s32_max_value = min_t(s32, reg->s32_max_value, |
| 1247 | var32_off.value | (var32_off.mask & S32_MAX)); |
| 1248 | reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)var32_off.value); |
| 1249 | reg->u32_max_value = min(reg->u32_max_value, |
| 1250 | (u32)(var32_off.value | var32_off.mask)); |
| 1251 | } |
| 1252 | |
| 1253 | static void __update_reg64_bounds(struct bpf_reg_state *reg) |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 1254 | { |
| 1255 | /* min signed is max(sign bit) | min(other bits) */ |
| 1256 | reg->smin_value = max_t(s64, reg->smin_value, |
| 1257 | reg->var_off.value | (reg->var_off.mask & S64_MIN)); |
| 1258 | /* max signed is min(sign bit) | max(other bits) */ |
| 1259 | reg->smax_value = min_t(s64, reg->smax_value, |
| 1260 | reg->var_off.value | (reg->var_off.mask & S64_MAX)); |
| 1261 | reg->umin_value = max(reg->umin_value, reg->var_off.value); |
| 1262 | reg->umax_value = min(reg->umax_value, |
| 1263 | reg->var_off.value | reg->var_off.mask); |
| 1264 | } |
| 1265 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 1266 | static void __update_reg_bounds(struct bpf_reg_state *reg) |
| 1267 | { |
| 1268 | __update_reg32_bounds(reg); |
| 1269 | __update_reg64_bounds(reg); |
| 1270 | } |
| 1271 | |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 1272 | /* Uses signed min/max values to inform unsigned, and vice-versa */ |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 1273 | static void __reg32_deduce_bounds(struct bpf_reg_state *reg) |
| 1274 | { |
| 1275 | /* Learn sign from signed bounds. |
| 1276 | * If we cannot cross the sign boundary, then signed and unsigned bounds |
| 1277 | * are the same, so combine. This works even in the negative case, e.g. |
| 1278 | * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. |
| 1279 | */ |
| 1280 | if (reg->s32_min_value >= 0 || reg->s32_max_value < 0) { |
| 1281 | reg->s32_min_value = reg->u32_min_value = |
| 1282 | max_t(u32, reg->s32_min_value, reg->u32_min_value); |
| 1283 | reg->s32_max_value = reg->u32_max_value = |
| 1284 | min_t(u32, reg->s32_max_value, reg->u32_max_value); |
| 1285 | return; |
| 1286 | } |
| 1287 | /* Learn sign from unsigned bounds. Signed bounds cross the sign |
| 1288 | * boundary, so we must be careful. |
| 1289 | */ |
| 1290 | if ((s32)reg->u32_max_value >= 0) { |
| 1291 | /* Positive. We can't learn anything from the smin, but smax |
| 1292 | * is positive, hence safe. |
| 1293 | */ |
| 1294 | reg->s32_min_value = reg->u32_min_value; |
| 1295 | reg->s32_max_value = reg->u32_max_value = |
| 1296 | min_t(u32, reg->s32_max_value, reg->u32_max_value); |
| 1297 | } else if ((s32)reg->u32_min_value < 0) { |
| 1298 | /* Negative. We can't learn anything from the smax, but smin |
| 1299 | * is negative, hence safe. |
| 1300 | */ |
| 1301 | reg->s32_min_value = reg->u32_min_value = |
| 1302 | max_t(u32, reg->s32_min_value, reg->u32_min_value); |
| 1303 | reg->s32_max_value = reg->u32_max_value; |
| 1304 | } |
| 1305 | } |
| 1306 | |
| 1307 | static void __reg64_deduce_bounds(struct bpf_reg_state *reg) |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 1308 | { |
| 1309 | /* Learn sign from signed bounds. |
| 1310 | * If we cannot cross the sign boundary, then signed and unsigned bounds |
| 1311 | * are the same, so combine. This works even in the negative case, e.g. |
| 1312 | * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. |
| 1313 | */ |
| 1314 | if (reg->smin_value >= 0 || reg->smax_value < 0) { |
| 1315 | reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, |
| 1316 | reg->umin_value); |
| 1317 | reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, |
| 1318 | reg->umax_value); |
| 1319 | return; |
| 1320 | } |
| 1321 | /* Learn sign from unsigned bounds. Signed bounds cross the sign |
| 1322 | * boundary, so we must be careful. |
| 1323 | */ |
| 1324 | if ((s64)reg->umax_value >= 0) { |
| 1325 | /* Positive. We can't learn anything from the smin, but smax |
| 1326 | * is positive, hence safe. |
| 1327 | */ |
| 1328 | reg->smin_value = reg->umin_value; |
| 1329 | reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, |
| 1330 | reg->umax_value); |
| 1331 | } else if ((s64)reg->umin_value < 0) { |
| 1332 | /* Negative. We can't learn anything from the smax, but smin |
| 1333 | * is negative, hence safe. |
| 1334 | */ |
| 1335 | reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, |
| 1336 | reg->umin_value); |
| 1337 | reg->smax_value = reg->umax_value; |
| 1338 | } |
| 1339 | } |
| 1340 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 1341 | static void __reg_deduce_bounds(struct bpf_reg_state *reg) |
| 1342 | { |
| 1343 | __reg32_deduce_bounds(reg); |
| 1344 | __reg64_deduce_bounds(reg); |
| 1345 | } |
| 1346 | |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 1347 | /* Attempts to improve var_off based on unsigned min/max information */ |
| 1348 | static void __reg_bound_offset(struct bpf_reg_state *reg) |
| 1349 | { |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 1350 | struct tnum var64_off = tnum_intersect(reg->var_off, |
| 1351 | tnum_range(reg->umin_value, |
| 1352 | reg->umax_value)); |
| 1353 | struct tnum var32_off = tnum_intersect(tnum_subreg(reg->var_off), |
| 1354 | tnum_range(reg->u32_min_value, |
| 1355 | reg->u32_max_value)); |
| 1356 | |
| 1357 | reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off); |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 1358 | } |
| 1359 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 1360 | static void __reg_assign_32_into_64(struct bpf_reg_state *reg) |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 1361 | { |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 1362 | reg->umin_value = reg->u32_min_value; |
| 1363 | reg->umax_value = reg->u32_max_value; |
| 1364 | /* Attempt to pull 32-bit signed bounds into 64-bit bounds |
| 1365 | * but must be positive otherwise set to worse case bounds |
| 1366 | * and refine later from tnum. |
| 1367 | */ |
John Fastabend | 3a71dc3 | 2020-05-29 10:28:40 -0700 | [diff] [blame] | 1368 | if (reg->s32_min_value >= 0 && reg->s32_max_value >= 0) |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 1369 | reg->smax_value = reg->s32_max_value; |
| 1370 | else |
| 1371 | reg->smax_value = U32_MAX; |
John Fastabend | 3a71dc3 | 2020-05-29 10:28:40 -0700 | [diff] [blame] | 1372 | if (reg->s32_min_value >= 0) |
| 1373 | reg->smin_value = reg->s32_min_value; |
| 1374 | else |
| 1375 | reg->smin_value = 0; |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 1376 | } |
| 1377 | |
| 1378 | static void __reg_combine_32_into_64(struct bpf_reg_state *reg) |
| 1379 | { |
| 1380 | /* special case when 64-bit register has upper 32-bit register |
| 1381 | * zeroed. Typically happens after zext or <<32, >>32 sequence |
| 1382 | * allowing us to use 32-bit bounds directly, |
| 1383 | */ |
| 1384 | if (tnum_equals_const(tnum_clear_subreg(reg->var_off), 0)) { |
| 1385 | __reg_assign_32_into_64(reg); |
| 1386 | } else { |
| 1387 | /* Otherwise the best we can do is push lower 32bit known and |
| 1388 | * unknown bits into register (var_off set from jmp logic) |
| 1389 | * then learn as much as possible from the 64-bit tnum |
| 1390 | * known and unknown bits. The previous smin/smax bounds are |
| 1391 | * invalid here because of jmp32 compare so mark them unknown |
| 1392 | * so they do not impact tnum bounds calculation. |
| 1393 | */ |
| 1394 | __mark_reg64_unbounded(reg); |
| 1395 | __update_reg_bounds(reg); |
| 1396 | } |
| 1397 | |
| 1398 | /* Intersecting with the old var_off might have improved our bounds |
| 1399 | * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), |
| 1400 | * then new var_off is (0; 0x7f...fc) which improves our umax. |
| 1401 | */ |
| 1402 | __reg_deduce_bounds(reg); |
| 1403 | __reg_bound_offset(reg); |
| 1404 | __update_reg_bounds(reg); |
| 1405 | } |
| 1406 | |
| 1407 | static bool __reg64_bound_s32(s64 a) |
| 1408 | { |
Alexei Starovoitov | b0270958 | 2020-12-08 19:01:51 +0100 | [diff] [blame] | 1409 | return a > S32_MIN && a < S32_MAX; |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 1410 | } |
| 1411 | |
| 1412 | static bool __reg64_bound_u32(u64 a) |
| 1413 | { |
Daniel Borkmann | 10bf4e8 | 2021-04-23 13:59:55 +0000 | [diff] [blame] | 1414 | return a > U32_MIN && a < U32_MAX; |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 1415 | } |
| 1416 | |
| 1417 | static void __reg_combine_64_into_32(struct bpf_reg_state *reg) |
| 1418 | { |
| 1419 | __mark_reg32_unbounded(reg); |
| 1420 | |
Alexei Starovoitov | b0270958 | 2020-12-08 19:01:51 +0100 | [diff] [blame] | 1421 | if (__reg64_bound_s32(reg->smin_value) && __reg64_bound_s32(reg->smax_value)) { |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 1422 | reg->s32_min_value = (s32)reg->smin_value; |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 1423 | reg->s32_max_value = (s32)reg->smax_value; |
Alexei Starovoitov | b0270958 | 2020-12-08 19:01:51 +0100 | [diff] [blame] | 1424 | } |
Daniel Borkmann | 10bf4e8 | 2021-04-23 13:59:55 +0000 | [diff] [blame] | 1425 | if (__reg64_bound_u32(reg->umin_value) && __reg64_bound_u32(reg->umax_value)) { |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 1426 | reg->u32_min_value = (u32)reg->umin_value; |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 1427 | reg->u32_max_value = (u32)reg->umax_value; |
Daniel Borkmann | 10bf4e8 | 2021-04-23 13:59:55 +0000 | [diff] [blame] | 1428 | } |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 1429 | |
| 1430 | /* Intersecting with the old var_off might have improved our bounds |
| 1431 | * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), |
| 1432 | * then new var_off is (0; 0x7f...fc) which improves our umax. |
| 1433 | */ |
| 1434 | __reg_deduce_bounds(reg); |
| 1435 | __reg_bound_offset(reg); |
| 1436 | __update_reg_bounds(reg); |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 1437 | } |
| 1438 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1439 | /* Mark a register as having a completely unknown (scalar) value. */ |
Daniel Borkmann | f54c789 | 2019-12-22 23:37:40 +0100 | [diff] [blame] | 1440 | static void __mark_reg_unknown(const struct bpf_verifier_env *env, |
| 1441 | struct bpf_reg_state *reg) |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1442 | { |
Alexei Starovoitov | a9c676b | 2018-09-04 19:13:44 -0700 | [diff] [blame] | 1443 | /* |
| 1444 | * Clear type, id, off, and union(map_ptr, range) and |
| 1445 | * padding between 'type' and union |
| 1446 | */ |
| 1447 | memset(reg, 0, offsetof(struct bpf_reg_state, var_off)); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1448 | reg->type = SCALAR_VALUE; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1449 | reg->var_off = tnum_unknown; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1450 | reg->frameno = 0; |
Alexei Starovoitov | 2c78ee8 | 2020-05-13 16:03:54 -0700 | [diff] [blame] | 1451 | reg->precise = env->subprog_cnt > 1 || !env->bpf_capable; |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 1452 | __mark_reg_unbounded(reg); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1453 | } |
| 1454 | |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1455 | static void mark_reg_unknown(struct bpf_verifier_env *env, |
| 1456 | struct bpf_reg_state *regs, u32 regno) |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1457 | { |
| 1458 | if (WARN_ON(regno >= MAX_BPF_REG)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1459 | verbose(env, "mark_reg_unknown(regs, %u)\n", regno); |
Alexei Starovoitov | 19ceb41 | 2017-11-30 21:31:37 -0800 | [diff] [blame] | 1460 | /* Something bad happened, let's kill all regs except FP */ |
| 1461 | for (regno = 0; regno < BPF_REG_FP; regno++) |
Daniel Borkmann | f54c789 | 2019-12-22 23:37:40 +0100 | [diff] [blame] | 1462 | __mark_reg_not_init(env, regs + regno); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1463 | return; |
| 1464 | } |
Daniel Borkmann | f54c789 | 2019-12-22 23:37:40 +0100 | [diff] [blame] | 1465 | __mark_reg_unknown(env, regs + regno); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1466 | } |
| 1467 | |
Daniel Borkmann | f54c789 | 2019-12-22 23:37:40 +0100 | [diff] [blame] | 1468 | static void __mark_reg_not_init(const struct bpf_verifier_env *env, |
| 1469 | struct bpf_reg_state *reg) |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1470 | { |
Daniel Borkmann | f54c789 | 2019-12-22 23:37:40 +0100 | [diff] [blame] | 1471 | __mark_reg_unknown(env, reg); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1472 | reg->type = NOT_INIT; |
| 1473 | } |
| 1474 | |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1475 | static void mark_reg_not_init(struct bpf_verifier_env *env, |
| 1476 | struct bpf_reg_state *regs, u32 regno) |
Daniel Borkmann | a9789ef | 2017-05-25 01:05:06 +0200 | [diff] [blame] | 1477 | { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1478 | if (WARN_ON(regno >= MAX_BPF_REG)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1479 | verbose(env, "mark_reg_not_init(regs, %u)\n", regno); |
Alexei Starovoitov | 19ceb41 | 2017-11-30 21:31:37 -0800 | [diff] [blame] | 1480 | /* Something bad happened, let's kill all regs except FP */ |
| 1481 | for (regno = 0; regno < BPF_REG_FP; regno++) |
Daniel Borkmann | f54c789 | 2019-12-22 23:37:40 +0100 | [diff] [blame] | 1482 | __mark_reg_not_init(env, regs + regno); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1483 | return; |
| 1484 | } |
Daniel Borkmann | f54c789 | 2019-12-22 23:37:40 +0100 | [diff] [blame] | 1485 | __mark_reg_not_init(env, regs + regno); |
Daniel Borkmann | a9789ef | 2017-05-25 01:05:06 +0200 | [diff] [blame] | 1486 | } |
| 1487 | |
Andrey Ignatov | 41c48f3 | 2020-06-19 14:11:43 -0700 | [diff] [blame] | 1488 | static void mark_btf_ld_reg(struct bpf_verifier_env *env, |
| 1489 | struct bpf_reg_state *regs, u32 regno, |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 1490 | enum bpf_reg_type reg_type, |
| 1491 | struct btf *btf, u32 btf_id) |
Andrey Ignatov | 41c48f3 | 2020-06-19 14:11:43 -0700 | [diff] [blame] | 1492 | { |
| 1493 | if (reg_type == SCALAR_VALUE) { |
| 1494 | mark_reg_unknown(env, regs, regno); |
| 1495 | return; |
| 1496 | } |
| 1497 | mark_reg_known_zero(env, regs, regno); |
| 1498 | regs[regno].type = PTR_TO_BTF_ID; |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 1499 | regs[regno].btf = btf; |
Andrey Ignatov | 41c48f3 | 2020-06-19 14:11:43 -0700 | [diff] [blame] | 1500 | regs[regno].btf_id = btf_id; |
| 1501 | } |
| 1502 | |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 1503 | #define DEF_NOT_SUBREG (0) |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1504 | static void init_reg_state(struct bpf_verifier_env *env, |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1505 | struct bpf_func_state *state) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1506 | { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1507 | struct bpf_reg_state *regs = state->regs; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1508 | int i; |
| 1509 | |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 1510 | for (i = 0; i < MAX_BPF_REG; i++) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1511 | mark_reg_not_init(env, regs, i); |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 1512 | regs[i].live = REG_LIVE_NONE; |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 1513 | regs[i].parent = NULL; |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 1514 | regs[i].subreg_def = DEF_NOT_SUBREG; |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 1515 | } |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1516 | |
| 1517 | /* frame pointer */ |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 1518 | regs[BPF_REG_FP].type = PTR_TO_STACK; |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 1519 | mark_reg_known_zero(env, regs, BPF_REG_FP); |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1520 | regs[BPF_REG_FP].frameno = state->frameno; |
Daniel Borkmann | 6760bf2 | 2016-12-18 01:52:59 +0100 | [diff] [blame] | 1521 | } |
| 1522 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1523 | #define BPF_MAIN_FUNC (-1) |
| 1524 | static void init_func_state(struct bpf_verifier_env *env, |
| 1525 | struct bpf_func_state *state, |
| 1526 | int callsite, int frameno, int subprogno) |
| 1527 | { |
| 1528 | state->callsite = callsite; |
| 1529 | state->frameno = frameno; |
| 1530 | state->subprogno = subprogno; |
| 1531 | init_reg_state(env, state); |
| 1532 | } |
| 1533 | |
Alexei Starovoitov | bfc6bb7 | 2021-07-14 17:54:14 -0700 | [diff] [blame] | 1534 | /* Similar to push_stack(), but for async callbacks */ |
| 1535 | static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, |
| 1536 | int insn_idx, int prev_insn_idx, |
| 1537 | int subprog) |
| 1538 | { |
| 1539 | struct bpf_verifier_stack_elem *elem; |
| 1540 | struct bpf_func_state *frame; |
| 1541 | |
| 1542 | elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL); |
| 1543 | if (!elem) |
| 1544 | goto err; |
| 1545 | |
| 1546 | elem->insn_idx = insn_idx; |
| 1547 | elem->prev_insn_idx = prev_insn_idx; |
| 1548 | elem->next = env->head; |
| 1549 | elem->log_pos = env->log.len_used; |
| 1550 | env->head = elem; |
| 1551 | env->stack_size++; |
| 1552 | if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { |
| 1553 | verbose(env, |
| 1554 | "The sequence of %d jumps is too complex for async cb.\n", |
| 1555 | env->stack_size); |
| 1556 | goto err; |
| 1557 | } |
| 1558 | /* Unlike push_stack() do not copy_verifier_state(). |
| 1559 | * The caller state doesn't matter. |
| 1560 | * This is async callback. It starts in a fresh stack. |
| 1561 | * Initialize it similar to do_check_common(). |
| 1562 | */ |
| 1563 | elem->st.branches = 1; |
| 1564 | frame = kzalloc(sizeof(*frame), GFP_KERNEL); |
| 1565 | if (!frame) |
| 1566 | goto err; |
| 1567 | init_func_state(env, frame, |
| 1568 | BPF_MAIN_FUNC /* callsite */, |
| 1569 | 0 /* frameno within this callchain */, |
| 1570 | subprog /* subprog number within this prog */); |
| 1571 | elem->st.frame[0] = frame; |
| 1572 | return &elem->st; |
| 1573 | err: |
| 1574 | free_verifier_state(env->cur_state, true); |
| 1575 | env->cur_state = NULL; |
| 1576 | /* pop all elements and return */ |
| 1577 | while (!pop_stack(env, NULL, NULL, false)); |
| 1578 | return NULL; |
| 1579 | } |
| 1580 | |
| 1581 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 1582 | enum reg_arg_type { |
| 1583 | SRC_OP, /* register is used as source operand */ |
| 1584 | DST_OP, /* register is used as destination operand */ |
| 1585 | DST_OP_NO_MARK /* same as above, check only, don't mark */ |
| 1586 | }; |
| 1587 | |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1588 | static int cmp_subprogs(const void *a, const void *b) |
| 1589 | { |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 1590 | return ((struct bpf_subprog_info *)a)->start - |
| 1591 | ((struct bpf_subprog_info *)b)->start; |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1592 | } |
| 1593 | |
| 1594 | static int find_subprog(struct bpf_verifier_env *env, int off) |
| 1595 | { |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 1596 | struct bpf_subprog_info *p; |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1597 | |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 1598 | p = bsearch(&off, env->subprog_info, env->subprog_cnt, |
| 1599 | sizeof(env->subprog_info[0]), cmp_subprogs); |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1600 | if (!p) |
| 1601 | return -ENOENT; |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 1602 | return p - env->subprog_info; |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1603 | |
| 1604 | } |
| 1605 | |
| 1606 | static int add_subprog(struct bpf_verifier_env *env, int off) |
| 1607 | { |
| 1608 | int insn_cnt = env->prog->len; |
| 1609 | int ret; |
| 1610 | |
| 1611 | if (off >= insn_cnt || off < 0) { |
| 1612 | verbose(env, "call to invalid destination\n"); |
| 1613 | return -EINVAL; |
| 1614 | } |
| 1615 | ret = find_subprog(env, off); |
| 1616 | if (ret >= 0) |
Yonghong Song | 282a0f4 | 2021-02-26 12:49:24 -0800 | [diff] [blame] | 1617 | return ret; |
Jiong Wang | 4cb3d99 | 2018-05-02 16:17:19 -0400 | [diff] [blame] | 1618 | if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1619 | verbose(env, "too many subprograms\n"); |
| 1620 | return -E2BIG; |
| 1621 | } |
Martin KaFai Lau | e6ac245 | 2021-03-24 18:51:42 -0700 | [diff] [blame] | 1622 | /* determine subprog starts. The end is one before the next starts */ |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 1623 | env->subprog_info[env->subprog_cnt++].start = off; |
| 1624 | sort(env->subprog_info, env->subprog_cnt, |
| 1625 | sizeof(env->subprog_info[0]), cmp_subprogs, NULL); |
Yonghong Song | 282a0f4 | 2021-02-26 12:49:24 -0800 | [diff] [blame] | 1626 | return env->subprog_cnt - 1; |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1627 | } |
| 1628 | |
Martin KaFai Lau | e6ac245 | 2021-03-24 18:51:42 -0700 | [diff] [blame] | 1629 | struct bpf_kfunc_desc { |
| 1630 | struct btf_func_model func_model; |
| 1631 | u32 func_id; |
| 1632 | s32 imm; |
| 1633 | }; |
| 1634 | |
| 1635 | #define MAX_KFUNC_DESCS 256 |
| 1636 | struct bpf_kfunc_desc_tab { |
| 1637 | struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS]; |
| 1638 | u32 nr_descs; |
| 1639 | }; |
| 1640 | |
| 1641 | static int kfunc_desc_cmp_by_id(const void *a, const void *b) |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1642 | { |
Martin KaFai Lau | e6ac245 | 2021-03-24 18:51:42 -0700 | [diff] [blame] | 1643 | const struct bpf_kfunc_desc *d0 = a; |
| 1644 | const struct bpf_kfunc_desc *d1 = b; |
| 1645 | |
| 1646 | /* func_id is not greater than BTF_MAX_TYPE */ |
| 1647 | return d0->func_id - d1->func_id; |
| 1648 | } |
| 1649 | |
| 1650 | static const struct bpf_kfunc_desc * |
| 1651 | find_kfunc_desc(const struct bpf_prog *prog, u32 func_id) |
| 1652 | { |
| 1653 | struct bpf_kfunc_desc desc = { |
| 1654 | .func_id = func_id, |
| 1655 | }; |
| 1656 | struct bpf_kfunc_desc_tab *tab; |
| 1657 | |
| 1658 | tab = prog->aux->kfunc_tab; |
| 1659 | return bsearch(&desc, tab->descs, tab->nr_descs, |
| 1660 | sizeof(tab->descs[0]), kfunc_desc_cmp_by_id); |
| 1661 | } |
| 1662 | |
| 1663 | static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id) |
| 1664 | { |
| 1665 | const struct btf_type *func, *func_proto; |
| 1666 | struct bpf_kfunc_desc_tab *tab; |
| 1667 | struct bpf_prog_aux *prog_aux; |
| 1668 | struct bpf_kfunc_desc *desc; |
| 1669 | const char *func_name; |
| 1670 | unsigned long addr; |
| 1671 | int err; |
| 1672 | |
| 1673 | prog_aux = env->prog->aux; |
| 1674 | tab = prog_aux->kfunc_tab; |
| 1675 | if (!tab) { |
| 1676 | if (!btf_vmlinux) { |
| 1677 | verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n"); |
| 1678 | return -ENOTSUPP; |
| 1679 | } |
| 1680 | |
| 1681 | if (!env->prog->jit_requested) { |
| 1682 | verbose(env, "JIT is required for calling kernel function\n"); |
| 1683 | return -ENOTSUPP; |
| 1684 | } |
| 1685 | |
| 1686 | if (!bpf_jit_supports_kfunc_call()) { |
| 1687 | verbose(env, "JIT does not support calling kernel function\n"); |
| 1688 | return -ENOTSUPP; |
| 1689 | } |
| 1690 | |
| 1691 | if (!env->prog->gpl_compatible) { |
| 1692 | verbose(env, "cannot call kernel function from non-GPL compatible program\n"); |
| 1693 | return -EINVAL; |
| 1694 | } |
| 1695 | |
| 1696 | tab = kzalloc(sizeof(*tab), GFP_KERNEL); |
| 1697 | if (!tab) |
| 1698 | return -ENOMEM; |
| 1699 | prog_aux->kfunc_tab = tab; |
| 1700 | } |
| 1701 | |
| 1702 | if (find_kfunc_desc(env->prog, func_id)) |
| 1703 | return 0; |
| 1704 | |
| 1705 | if (tab->nr_descs == MAX_KFUNC_DESCS) { |
| 1706 | verbose(env, "too many different kernel function calls\n"); |
| 1707 | return -E2BIG; |
| 1708 | } |
| 1709 | |
| 1710 | func = btf_type_by_id(btf_vmlinux, func_id); |
| 1711 | if (!func || !btf_type_is_func(func)) { |
| 1712 | verbose(env, "kernel btf_id %u is not a function\n", |
| 1713 | func_id); |
| 1714 | return -EINVAL; |
| 1715 | } |
| 1716 | func_proto = btf_type_by_id(btf_vmlinux, func->type); |
| 1717 | if (!func_proto || !btf_type_is_func_proto(func_proto)) { |
| 1718 | verbose(env, "kernel function btf_id %u does not have a valid func_proto\n", |
| 1719 | func_id); |
| 1720 | return -EINVAL; |
| 1721 | } |
| 1722 | |
| 1723 | func_name = btf_name_by_offset(btf_vmlinux, func->name_off); |
| 1724 | addr = kallsyms_lookup_name(func_name); |
| 1725 | if (!addr) { |
| 1726 | verbose(env, "cannot find address for kernel function %s\n", |
| 1727 | func_name); |
| 1728 | return -EINVAL; |
| 1729 | } |
| 1730 | |
| 1731 | desc = &tab->descs[tab->nr_descs++]; |
| 1732 | desc->func_id = func_id; |
| 1733 | desc->imm = BPF_CAST_CALL(addr) - __bpf_call_base; |
| 1734 | err = btf_distill_func_proto(&env->log, btf_vmlinux, |
| 1735 | func_proto, func_name, |
| 1736 | &desc->func_model); |
| 1737 | if (!err) |
| 1738 | sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), |
| 1739 | kfunc_desc_cmp_by_id, NULL); |
| 1740 | return err; |
| 1741 | } |
| 1742 | |
| 1743 | static int kfunc_desc_cmp_by_imm(const void *a, const void *b) |
| 1744 | { |
| 1745 | const struct bpf_kfunc_desc *d0 = a; |
| 1746 | const struct bpf_kfunc_desc *d1 = b; |
| 1747 | |
| 1748 | if (d0->imm > d1->imm) |
| 1749 | return 1; |
| 1750 | else if (d0->imm < d1->imm) |
| 1751 | return -1; |
| 1752 | return 0; |
| 1753 | } |
| 1754 | |
| 1755 | static void sort_kfunc_descs_by_imm(struct bpf_prog *prog) |
| 1756 | { |
| 1757 | struct bpf_kfunc_desc_tab *tab; |
| 1758 | |
| 1759 | tab = prog->aux->kfunc_tab; |
| 1760 | if (!tab) |
| 1761 | return; |
| 1762 | |
| 1763 | sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), |
| 1764 | kfunc_desc_cmp_by_imm, NULL); |
| 1765 | } |
| 1766 | |
| 1767 | bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog) |
| 1768 | { |
| 1769 | return !!prog->aux->kfunc_tab; |
| 1770 | } |
| 1771 | |
| 1772 | const struct btf_func_model * |
| 1773 | bpf_jit_find_kfunc_model(const struct bpf_prog *prog, |
| 1774 | const struct bpf_insn *insn) |
| 1775 | { |
| 1776 | const struct bpf_kfunc_desc desc = { |
| 1777 | .imm = insn->imm, |
| 1778 | }; |
| 1779 | const struct bpf_kfunc_desc *res; |
| 1780 | struct bpf_kfunc_desc_tab *tab; |
| 1781 | |
| 1782 | tab = prog->aux->kfunc_tab; |
| 1783 | res = bsearch(&desc, tab->descs, tab->nr_descs, |
| 1784 | sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm); |
| 1785 | |
| 1786 | return res ? &res->func_model : NULL; |
| 1787 | } |
| 1788 | |
| 1789 | static int add_subprog_and_kfunc(struct bpf_verifier_env *env) |
| 1790 | { |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 1791 | struct bpf_subprog_info *subprog = env->subprog_info; |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1792 | struct bpf_insn *insn = env->prog->insnsi; |
Martin KaFai Lau | e6ac245 | 2021-03-24 18:51:42 -0700 | [diff] [blame] | 1793 | int i, ret, insn_cnt = env->prog->len; |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1794 | |
Jiong Wang | f910cef | 2018-05-02 16:17:17 -0400 | [diff] [blame] | 1795 | /* Add entry function. */ |
| 1796 | ret = add_subprog(env, 0); |
Martin KaFai Lau | e6ac245 | 2021-03-24 18:51:42 -0700 | [diff] [blame] | 1797 | if (ret) |
Jiong Wang | f910cef | 2018-05-02 16:17:17 -0400 | [diff] [blame] | 1798 | return ret; |
| 1799 | |
Martin KaFai Lau | e6ac245 | 2021-03-24 18:51:42 -0700 | [diff] [blame] | 1800 | for (i = 0; i < insn_cnt; i++, insn++) { |
| 1801 | if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn) && |
| 1802 | !bpf_pseudo_kfunc_call(insn)) |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 1803 | continue; |
Martin KaFai Lau | e6ac245 | 2021-03-24 18:51:42 -0700 | [diff] [blame] | 1804 | |
Alexei Starovoitov | 2c78ee8 | 2020-05-13 16:03:54 -0700 | [diff] [blame] | 1805 | if (!env->bpf_capable) { |
Martin KaFai Lau | e6ac245 | 2021-03-24 18:51:42 -0700 | [diff] [blame] | 1806 | verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n"); |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1807 | return -EPERM; |
| 1808 | } |
Martin KaFai Lau | e6ac245 | 2021-03-24 18:51:42 -0700 | [diff] [blame] | 1809 | |
| 1810 | if (bpf_pseudo_func(insn)) { |
| 1811 | ret = add_subprog(env, i + insn->imm + 1); |
| 1812 | if (ret >= 0) |
| 1813 | /* remember subprog */ |
| 1814 | insn[1].imm = ret; |
| 1815 | } else if (bpf_pseudo_call(insn)) { |
| 1816 | ret = add_subprog(env, i + insn->imm + 1); |
| 1817 | } else { |
| 1818 | ret = add_kfunc_call(env, insn->imm); |
| 1819 | } |
| 1820 | |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1821 | if (ret < 0) |
| 1822 | return ret; |
| 1823 | } |
| 1824 | |
Jiong Wang | 4cb3d99 | 2018-05-02 16:17:19 -0400 | [diff] [blame] | 1825 | /* Add a fake 'exit' subprog which could simplify subprog iteration |
| 1826 | * logic. 'subprog_cnt' should not be increased. |
| 1827 | */ |
| 1828 | subprog[env->subprog_cnt].start = insn_cnt; |
| 1829 | |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 1830 | if (env->log.level & BPF_LOG_LEVEL2) |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1831 | for (i = 0; i < env->subprog_cnt; i++) |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 1832 | verbose(env, "func#%d @%d\n", i, subprog[i].start); |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1833 | |
Martin KaFai Lau | e6ac245 | 2021-03-24 18:51:42 -0700 | [diff] [blame] | 1834 | return 0; |
| 1835 | } |
| 1836 | |
| 1837 | static int check_subprogs(struct bpf_verifier_env *env) |
| 1838 | { |
| 1839 | int i, subprog_start, subprog_end, off, cur_subprog = 0; |
| 1840 | struct bpf_subprog_info *subprog = env->subprog_info; |
| 1841 | struct bpf_insn *insn = env->prog->insnsi; |
| 1842 | int insn_cnt = env->prog->len; |
| 1843 | |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1844 | /* now check that all jumps are within the same subprog */ |
Jiong Wang | 4cb3d99 | 2018-05-02 16:17:19 -0400 | [diff] [blame] | 1845 | subprog_start = subprog[cur_subprog].start; |
| 1846 | subprog_end = subprog[cur_subprog + 1].start; |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1847 | for (i = 0; i < insn_cnt; i++) { |
| 1848 | u8 code = insn[i].code; |
| 1849 | |
Maciej Fijalkowski | 7f6e431 | 2020-09-16 23:10:07 +0200 | [diff] [blame] | 1850 | if (code == (BPF_JMP | BPF_CALL) && |
| 1851 | insn[i].imm == BPF_FUNC_tail_call && |
| 1852 | insn[i].src_reg != BPF_PSEUDO_CALL) |
| 1853 | subprog[cur_subprog].has_tail_call = true; |
Alexei Starovoitov | 09b28d7 | 2020-09-17 19:09:18 -0700 | [diff] [blame] | 1854 | if (BPF_CLASS(code) == BPF_LD && |
| 1855 | (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) |
| 1856 | subprog[cur_subprog].has_ld_abs = true; |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 1857 | if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1858 | goto next; |
| 1859 | if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL) |
| 1860 | goto next; |
| 1861 | off = i + insn[i].off + 1; |
| 1862 | if (off < subprog_start || off >= subprog_end) { |
| 1863 | verbose(env, "jump out of range from insn %d to %d\n", i, off); |
| 1864 | return -EINVAL; |
| 1865 | } |
| 1866 | next: |
| 1867 | if (i == subprog_end - 1) { |
| 1868 | /* to avoid fall-through from one subprog into another |
| 1869 | * the last insn of the subprog should be either exit |
| 1870 | * or unconditional jump back |
| 1871 | */ |
| 1872 | if (code != (BPF_JMP | BPF_EXIT) && |
| 1873 | code != (BPF_JMP | BPF_JA)) { |
| 1874 | verbose(env, "last insn is not an exit or jmp\n"); |
| 1875 | return -EINVAL; |
| 1876 | } |
| 1877 | subprog_start = subprog_end; |
Jiong Wang | 4cb3d99 | 2018-05-02 16:17:19 -0400 | [diff] [blame] | 1878 | cur_subprog++; |
| 1879 | if (cur_subprog < env->subprog_cnt) |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 1880 | subprog_end = subprog[cur_subprog + 1].start; |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 1881 | } |
| 1882 | } |
| 1883 | return 0; |
| 1884 | } |
| 1885 | |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 1886 | /* Parentage chain of this register (or stack slot) should take care of all |
| 1887 | * issues like callee-saved registers, stack slot allocation time, etc. |
| 1888 | */ |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1889 | static int mark_reg_read(struct bpf_verifier_env *env, |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 1890 | const struct bpf_reg_state *state, |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 1891 | struct bpf_reg_state *parent, u8 flag) |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1892 | { |
| 1893 | bool writes = parent == state->parent; /* Observe write marks */ |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 1894 | int cnt = 0; |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 1895 | |
| 1896 | while (parent) { |
| 1897 | /* if read wasn't screened by an earlier write ... */ |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 1898 | if (writes && state->live & REG_LIVE_WRITTEN) |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 1899 | break; |
Alexei Starovoitov | 9242b5f | 2018-12-13 11:42:34 -0800 | [diff] [blame] | 1900 | if (parent->live & REG_LIVE_DONE) { |
| 1901 | verbose(env, "verifier BUG type %s var_off %lld off %d\n", |
| 1902 | reg_type_str[parent->type], |
| 1903 | parent->var_off.value, parent->off); |
| 1904 | return -EFAULT; |
| 1905 | } |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 1906 | /* The first condition is more likely to be true than the |
| 1907 | * second, checked it first. |
| 1908 | */ |
| 1909 | if ((parent->live & REG_LIVE_READ) == flag || |
| 1910 | parent->live & REG_LIVE_READ64) |
Alexei Starovoitov | 25af32d | 2019-04-01 21:27:42 -0700 | [diff] [blame] | 1911 | /* The parentage chain never changes and |
| 1912 | * this parent was already marked as LIVE_READ. |
| 1913 | * There is no need to keep walking the chain again and |
| 1914 | * keep re-marking all parents as LIVE_READ. |
| 1915 | * This case happens when the same register is read |
| 1916 | * multiple times without writes into it in-between. |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 1917 | * Also, if parent has the stronger REG_LIVE_READ64 set, |
| 1918 | * then no need to set the weak REG_LIVE_READ32. |
Alexei Starovoitov | 25af32d | 2019-04-01 21:27:42 -0700 | [diff] [blame] | 1919 | */ |
| 1920 | break; |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 1921 | /* ... then we depend on parent's value */ |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 1922 | parent->live |= flag; |
| 1923 | /* REG_LIVE_READ64 overrides REG_LIVE_READ32. */ |
| 1924 | if (flag == REG_LIVE_READ64) |
| 1925 | parent->live &= ~REG_LIVE_READ32; |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 1926 | state = parent; |
| 1927 | parent = state->parent; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1928 | writes = true; |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 1929 | cnt++; |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 1930 | } |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 1931 | |
| 1932 | if (env->longest_mark_read_walk < cnt) |
| 1933 | env->longest_mark_read_walk = cnt; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 1934 | return 0; |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 1935 | } |
| 1936 | |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 1937 | /* This function is supposed to be used by the following 32-bit optimization |
| 1938 | * code only. It returns TRUE if the source or destination register operates |
| 1939 | * on 64-bit, otherwise return FALSE. |
| 1940 | */ |
| 1941 | static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn, |
| 1942 | u32 regno, struct bpf_reg_state *reg, enum reg_arg_type t) |
| 1943 | { |
| 1944 | u8 code, class, op; |
| 1945 | |
| 1946 | code = insn->code; |
| 1947 | class = BPF_CLASS(code); |
| 1948 | op = BPF_OP(code); |
| 1949 | if (class == BPF_JMP) { |
| 1950 | /* BPF_EXIT for "main" will reach here. Return TRUE |
| 1951 | * conservatively. |
| 1952 | */ |
| 1953 | if (op == BPF_EXIT) |
| 1954 | return true; |
| 1955 | if (op == BPF_CALL) { |
| 1956 | /* BPF to BPF call will reach here because of marking |
| 1957 | * caller saved clobber with DST_OP_NO_MARK for which we |
| 1958 | * don't care the register def because they are anyway |
| 1959 | * marked as NOT_INIT already. |
| 1960 | */ |
| 1961 | if (insn->src_reg == BPF_PSEUDO_CALL) |
| 1962 | return false; |
| 1963 | /* Helper call will reach here because of arg type |
| 1964 | * check, conservatively return TRUE. |
| 1965 | */ |
| 1966 | if (t == SRC_OP) |
| 1967 | return true; |
| 1968 | |
| 1969 | return false; |
| 1970 | } |
| 1971 | } |
| 1972 | |
| 1973 | if (class == BPF_ALU64 || class == BPF_JMP || |
| 1974 | /* BPF_END always use BPF_ALU class. */ |
| 1975 | (class == BPF_ALU && op == BPF_END && insn->imm == 64)) |
| 1976 | return true; |
| 1977 | |
| 1978 | if (class == BPF_ALU || class == BPF_JMP32) |
| 1979 | return false; |
| 1980 | |
| 1981 | if (class == BPF_LDX) { |
| 1982 | if (t != SRC_OP) |
| 1983 | return BPF_SIZE(code) == BPF_DW; |
| 1984 | /* LDX source must be ptr. */ |
| 1985 | return true; |
| 1986 | } |
| 1987 | |
| 1988 | if (class == BPF_STX) { |
Ilya Leoshkevich | 83a2881 | 2021-03-01 16:40:19 +0100 | [diff] [blame] | 1989 | /* BPF_STX (including atomic variants) has multiple source |
| 1990 | * operands, one of which is a ptr. Check whether the caller is |
| 1991 | * asking about it. |
| 1992 | */ |
| 1993 | if (t == SRC_OP && reg->type != SCALAR_VALUE) |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 1994 | return true; |
| 1995 | return BPF_SIZE(code) == BPF_DW; |
| 1996 | } |
| 1997 | |
| 1998 | if (class == BPF_LD) { |
| 1999 | u8 mode = BPF_MODE(code); |
| 2000 | |
| 2001 | /* LD_IMM64 */ |
| 2002 | if (mode == BPF_IMM) |
| 2003 | return true; |
| 2004 | |
| 2005 | /* Both LD_IND and LD_ABS return 32-bit data. */ |
| 2006 | if (t != SRC_OP) |
| 2007 | return false; |
| 2008 | |
| 2009 | /* Implicit ctx ptr. */ |
| 2010 | if (regno == BPF_REG_6) |
| 2011 | return true; |
| 2012 | |
| 2013 | /* Explicit source could be any width. */ |
| 2014 | return true; |
| 2015 | } |
| 2016 | |
| 2017 | if (class == BPF_ST) |
| 2018 | /* The only source register for BPF_ST is a ptr. */ |
| 2019 | return true; |
| 2020 | |
| 2021 | /* Conservatively return true at default. */ |
| 2022 | return true; |
| 2023 | } |
| 2024 | |
Ilya Leoshkevich | 83a2881 | 2021-03-01 16:40:19 +0100 | [diff] [blame] | 2025 | /* Return the regno defined by the insn, or -1. */ |
| 2026 | static int insn_def_regno(const struct bpf_insn *insn) |
Jiong Wang | b325fbc | 2019-05-24 23:25:13 +0100 | [diff] [blame] | 2027 | { |
Ilya Leoshkevich | 83a2881 | 2021-03-01 16:40:19 +0100 | [diff] [blame] | 2028 | switch (BPF_CLASS(insn->code)) { |
| 2029 | case BPF_JMP: |
| 2030 | case BPF_JMP32: |
| 2031 | case BPF_ST: |
| 2032 | return -1; |
| 2033 | case BPF_STX: |
| 2034 | if (BPF_MODE(insn->code) == BPF_ATOMIC && |
| 2035 | (insn->imm & BPF_FETCH)) { |
| 2036 | if (insn->imm == BPF_CMPXCHG) |
| 2037 | return BPF_REG_0; |
| 2038 | else |
| 2039 | return insn->src_reg; |
| 2040 | } else { |
| 2041 | return -1; |
| 2042 | } |
| 2043 | default: |
| 2044 | return insn->dst_reg; |
| 2045 | } |
Jiong Wang | b325fbc | 2019-05-24 23:25:13 +0100 | [diff] [blame] | 2046 | } |
| 2047 | |
| 2048 | /* Return TRUE if INSN has defined any 32-bit value explicitly. */ |
| 2049 | static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn) |
| 2050 | { |
Ilya Leoshkevich | 83a2881 | 2021-03-01 16:40:19 +0100 | [diff] [blame] | 2051 | int dst_reg = insn_def_regno(insn); |
| 2052 | |
| 2053 | if (dst_reg == -1) |
Jiong Wang | b325fbc | 2019-05-24 23:25:13 +0100 | [diff] [blame] | 2054 | return false; |
| 2055 | |
Ilya Leoshkevich | 83a2881 | 2021-03-01 16:40:19 +0100 | [diff] [blame] | 2056 | return !is_reg64(env, insn, dst_reg, NULL, DST_OP); |
Jiong Wang | b325fbc | 2019-05-24 23:25:13 +0100 | [diff] [blame] | 2057 | } |
| 2058 | |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 2059 | static void mark_insn_zext(struct bpf_verifier_env *env, |
| 2060 | struct bpf_reg_state *reg) |
| 2061 | { |
| 2062 | s32 def_idx = reg->subreg_def; |
| 2063 | |
| 2064 | if (def_idx == DEF_NOT_SUBREG) |
| 2065 | return; |
| 2066 | |
| 2067 | env->insn_aux_data[def_idx - 1].zext_dst = true; |
| 2068 | /* The dst will be zero extended, so won't be sub-register anymore. */ |
| 2069 | reg->subreg_def = DEF_NOT_SUBREG; |
| 2070 | } |
| 2071 | |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 2072 | static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2073 | enum reg_arg_type t) |
| 2074 | { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 2075 | struct bpf_verifier_state *vstate = env->cur_state; |
| 2076 | struct bpf_func_state *state = vstate->frame[vstate->curframe]; |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 2077 | struct bpf_insn *insn = env->prog->insnsi + env->insn_idx; |
Jiong Wang | c342dc1 | 2019-04-12 22:59:37 +0100 | [diff] [blame] | 2078 | struct bpf_reg_state *reg, *regs = state->regs; |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 2079 | bool rw64; |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 2080 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2081 | if (regno >= MAX_BPF_REG) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2082 | verbose(env, "R%d is invalid\n", regno); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2083 | return -EINVAL; |
| 2084 | } |
| 2085 | |
Jiong Wang | c342dc1 | 2019-04-12 22:59:37 +0100 | [diff] [blame] | 2086 | reg = ®s[regno]; |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 2087 | rw64 = is_reg64(env, insn, regno, reg, t); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2088 | if (t == SRC_OP) { |
| 2089 | /* check whether register used as source operand can be read */ |
Jiong Wang | c342dc1 | 2019-04-12 22:59:37 +0100 | [diff] [blame] | 2090 | if (reg->type == NOT_INIT) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2091 | verbose(env, "R%d !read_ok\n", regno); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2092 | return -EACCES; |
| 2093 | } |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 2094 | /* We don't need to worry about FP liveness because it's read-only */ |
Jiong Wang | c342dc1 | 2019-04-12 22:59:37 +0100 | [diff] [blame] | 2095 | if (regno == BPF_REG_FP) |
| 2096 | return 0; |
| 2097 | |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 2098 | if (rw64) |
| 2099 | mark_insn_zext(env, reg); |
| 2100 | |
| 2101 | return mark_reg_read(env, reg, reg->parent, |
| 2102 | rw64 ? REG_LIVE_READ64 : REG_LIVE_READ32); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2103 | } else { |
| 2104 | /* check whether register used as dest operand can be written to */ |
| 2105 | if (regno == BPF_REG_FP) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2106 | verbose(env, "frame pointer is read only\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2107 | return -EACCES; |
| 2108 | } |
Jiong Wang | c342dc1 | 2019-04-12 22:59:37 +0100 | [diff] [blame] | 2109 | reg->live |= REG_LIVE_WRITTEN; |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 2110 | reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2111 | if (t == DST_OP) |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2112 | mark_reg_unknown(env, regs, regno); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2113 | } |
| 2114 | return 0; |
| 2115 | } |
| 2116 | |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2117 | /* for any branch, call, exit record the history of jmps in the given state */ |
| 2118 | static int push_jmp_history(struct bpf_verifier_env *env, |
| 2119 | struct bpf_verifier_state *cur) |
| 2120 | { |
| 2121 | u32 cnt = cur->jmp_history_cnt; |
| 2122 | struct bpf_idx_pair *p; |
| 2123 | |
| 2124 | cnt++; |
| 2125 | p = krealloc(cur->jmp_history, cnt * sizeof(*p), GFP_USER); |
| 2126 | if (!p) |
| 2127 | return -ENOMEM; |
| 2128 | p[cnt - 1].idx = env->insn_idx; |
| 2129 | p[cnt - 1].prev_idx = env->prev_insn_idx; |
| 2130 | cur->jmp_history = p; |
| 2131 | cur->jmp_history_cnt = cnt; |
| 2132 | return 0; |
| 2133 | } |
| 2134 | |
| 2135 | /* Backtrack one insn at a time. If idx is not at the top of recorded |
| 2136 | * history then previous instruction came from straight line execution. |
| 2137 | */ |
| 2138 | static int get_prev_insn_idx(struct bpf_verifier_state *st, int i, |
| 2139 | u32 *history) |
| 2140 | { |
| 2141 | u32 cnt = *history; |
| 2142 | |
| 2143 | if (cnt && st->jmp_history[cnt - 1].idx == i) { |
| 2144 | i = st->jmp_history[cnt - 1].prev_idx; |
| 2145 | (*history)--; |
| 2146 | } else { |
| 2147 | i--; |
| 2148 | } |
| 2149 | return i; |
| 2150 | } |
| 2151 | |
Martin KaFai Lau | e6ac245 | 2021-03-24 18:51:42 -0700 | [diff] [blame] | 2152 | static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn) |
| 2153 | { |
| 2154 | const struct btf_type *func; |
| 2155 | |
| 2156 | if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL) |
| 2157 | return NULL; |
| 2158 | |
| 2159 | func = btf_type_by_id(btf_vmlinux, insn->imm); |
| 2160 | return btf_name_by_offset(btf_vmlinux, func->name_off); |
| 2161 | } |
| 2162 | |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2163 | /* For given verifier state backtrack_insn() is called from the last insn to |
| 2164 | * the first insn. Its purpose is to compute a bitmask of registers and |
| 2165 | * stack slots that needs precision in the parent verifier state. |
| 2166 | */ |
| 2167 | static int backtrack_insn(struct bpf_verifier_env *env, int idx, |
| 2168 | u32 *reg_mask, u64 *stack_mask) |
| 2169 | { |
| 2170 | const struct bpf_insn_cbs cbs = { |
Martin KaFai Lau | e6ac245 | 2021-03-24 18:51:42 -0700 | [diff] [blame] | 2171 | .cb_call = disasm_kfunc_name, |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2172 | .cb_print = verbose, |
| 2173 | .private_data = env, |
| 2174 | }; |
| 2175 | struct bpf_insn *insn = env->prog->insnsi + idx; |
| 2176 | u8 class = BPF_CLASS(insn->code); |
| 2177 | u8 opcode = BPF_OP(insn->code); |
| 2178 | u8 mode = BPF_MODE(insn->code); |
| 2179 | u32 dreg = 1u << insn->dst_reg; |
| 2180 | u32 sreg = 1u << insn->src_reg; |
| 2181 | u32 spi; |
| 2182 | |
| 2183 | if (insn->code == 0) |
| 2184 | return 0; |
| 2185 | if (env->log.level & BPF_LOG_LEVEL) { |
| 2186 | verbose(env, "regs=%x stack=%llx before ", *reg_mask, *stack_mask); |
| 2187 | verbose(env, "%d: ", idx); |
| 2188 | print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); |
| 2189 | } |
| 2190 | |
| 2191 | if (class == BPF_ALU || class == BPF_ALU64) { |
| 2192 | if (!(*reg_mask & dreg)) |
| 2193 | return 0; |
| 2194 | if (opcode == BPF_MOV) { |
| 2195 | if (BPF_SRC(insn->code) == BPF_X) { |
| 2196 | /* dreg = sreg |
| 2197 | * dreg needs precision after this insn |
| 2198 | * sreg needs precision before this insn |
| 2199 | */ |
| 2200 | *reg_mask &= ~dreg; |
| 2201 | *reg_mask |= sreg; |
| 2202 | } else { |
| 2203 | /* dreg = K |
| 2204 | * dreg needs precision after this insn. |
| 2205 | * Corresponding register is already marked |
| 2206 | * as precise=true in this verifier state. |
| 2207 | * No further markings in parent are necessary |
| 2208 | */ |
| 2209 | *reg_mask &= ~dreg; |
| 2210 | } |
| 2211 | } else { |
| 2212 | if (BPF_SRC(insn->code) == BPF_X) { |
| 2213 | /* dreg += sreg |
| 2214 | * both dreg and sreg need precision |
| 2215 | * before this insn |
| 2216 | */ |
| 2217 | *reg_mask |= sreg; |
| 2218 | } /* else dreg += K |
| 2219 | * dreg still needs precision before this insn |
| 2220 | */ |
| 2221 | } |
| 2222 | } else if (class == BPF_LDX) { |
| 2223 | if (!(*reg_mask & dreg)) |
| 2224 | return 0; |
| 2225 | *reg_mask &= ~dreg; |
| 2226 | |
| 2227 | /* scalars can only be spilled into stack w/o losing precision. |
| 2228 | * Load from any other memory can be zero extended. |
| 2229 | * The desire to keep that precision is already indicated |
| 2230 | * by 'precise' mark in corresponding register of this state. |
| 2231 | * No further tracking necessary. |
| 2232 | */ |
| 2233 | if (insn->src_reg != BPF_REG_FP) |
| 2234 | return 0; |
| 2235 | if (BPF_SIZE(insn->code) != BPF_DW) |
| 2236 | return 0; |
| 2237 | |
| 2238 | /* dreg = *(u64 *)[fp - off] was a fill from the stack. |
| 2239 | * that [fp - off] slot contains scalar that needs to be |
| 2240 | * tracked with precision |
| 2241 | */ |
| 2242 | spi = (-insn->off - 1) / BPF_REG_SIZE; |
| 2243 | if (spi >= 64) { |
| 2244 | verbose(env, "BUG spi %d\n", spi); |
| 2245 | WARN_ONCE(1, "verifier backtracking bug"); |
| 2246 | return -EFAULT; |
| 2247 | } |
| 2248 | *stack_mask |= 1ull << spi; |
Andrii Nakryiko | b3b50f0 | 2019-07-08 20:32:44 -0700 | [diff] [blame] | 2249 | } else if (class == BPF_STX || class == BPF_ST) { |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2250 | if (*reg_mask & dreg) |
Andrii Nakryiko | b3b50f0 | 2019-07-08 20:32:44 -0700 | [diff] [blame] | 2251 | /* stx & st shouldn't be using _scalar_ dst_reg |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2252 | * to access memory. It means backtracking |
| 2253 | * encountered a case of pointer subtraction. |
| 2254 | */ |
| 2255 | return -ENOTSUPP; |
| 2256 | /* scalars can only be spilled into stack */ |
| 2257 | if (insn->dst_reg != BPF_REG_FP) |
| 2258 | return 0; |
| 2259 | if (BPF_SIZE(insn->code) != BPF_DW) |
| 2260 | return 0; |
| 2261 | spi = (-insn->off - 1) / BPF_REG_SIZE; |
| 2262 | if (spi >= 64) { |
| 2263 | verbose(env, "BUG spi %d\n", spi); |
| 2264 | WARN_ONCE(1, "verifier backtracking bug"); |
| 2265 | return -EFAULT; |
| 2266 | } |
| 2267 | if (!(*stack_mask & (1ull << spi))) |
| 2268 | return 0; |
| 2269 | *stack_mask &= ~(1ull << spi); |
Andrii Nakryiko | b3b50f0 | 2019-07-08 20:32:44 -0700 | [diff] [blame] | 2270 | if (class == BPF_STX) |
| 2271 | *reg_mask |= sreg; |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2272 | } else if (class == BPF_JMP || class == BPF_JMP32) { |
| 2273 | if (opcode == BPF_CALL) { |
| 2274 | if (insn->src_reg == BPF_PSEUDO_CALL) |
| 2275 | return -ENOTSUPP; |
| 2276 | /* regular helper call sets R0 */ |
| 2277 | *reg_mask &= ~1; |
| 2278 | if (*reg_mask & 0x3f) { |
| 2279 | /* if backtracing was looking for registers R1-R5 |
| 2280 | * they should have been found already. |
| 2281 | */ |
| 2282 | verbose(env, "BUG regs %x\n", *reg_mask); |
| 2283 | WARN_ONCE(1, "verifier backtracking bug"); |
| 2284 | return -EFAULT; |
| 2285 | } |
| 2286 | } else if (opcode == BPF_EXIT) { |
| 2287 | return -ENOTSUPP; |
| 2288 | } |
| 2289 | } else if (class == BPF_LD) { |
| 2290 | if (!(*reg_mask & dreg)) |
| 2291 | return 0; |
| 2292 | *reg_mask &= ~dreg; |
| 2293 | /* It's ld_imm64 or ld_abs or ld_ind. |
| 2294 | * For ld_imm64 no further tracking of precision |
| 2295 | * into parent is necessary |
| 2296 | */ |
| 2297 | if (mode == BPF_IND || mode == BPF_ABS) |
| 2298 | /* to be analyzed */ |
| 2299 | return -ENOTSUPP; |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2300 | } |
| 2301 | return 0; |
| 2302 | } |
| 2303 | |
| 2304 | /* the scalar precision tracking algorithm: |
| 2305 | * . at the start all registers have precise=false. |
| 2306 | * . scalar ranges are tracked as normal through alu and jmp insns. |
| 2307 | * . once precise value of the scalar register is used in: |
| 2308 | * . ptr + scalar alu |
| 2309 | * . if (scalar cond K|scalar) |
| 2310 | * . helper_call(.., scalar, ...) where ARG_CONST is expected |
| 2311 | * backtrack through the verifier states and mark all registers and |
| 2312 | * stack slots with spilled constants that these scalar regisers |
| 2313 | * should be precise. |
| 2314 | * . during state pruning two registers (or spilled stack slots) |
| 2315 | * are equivalent if both are not precise. |
| 2316 | * |
| 2317 | * Note the verifier cannot simply walk register parentage chain, |
| 2318 | * since many different registers and stack slots could have been |
| 2319 | * used to compute single precise scalar. |
| 2320 | * |
| 2321 | * The approach of starting with precise=true for all registers and then |
| 2322 | * backtrack to mark a register as not precise when the verifier detects |
| 2323 | * that program doesn't care about specific value (e.g., when helper |
| 2324 | * takes register as ARG_ANYTHING parameter) is not safe. |
| 2325 | * |
| 2326 | * It's ok to walk single parentage chain of the verifier states. |
| 2327 | * It's possible that this backtracking will go all the way till 1st insn. |
| 2328 | * All other branches will be explored for needing precision later. |
| 2329 | * |
| 2330 | * The backtracking needs to deal with cases like: |
| 2331 | * R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0) |
| 2332 | * r9 -= r8 |
| 2333 | * r5 = r9 |
| 2334 | * if r5 > 0x79f goto pc+7 |
| 2335 | * R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff)) |
| 2336 | * r5 += 1 |
| 2337 | * ... |
| 2338 | * call bpf_perf_event_output#25 |
| 2339 | * where .arg5_type = ARG_CONST_SIZE_OR_ZERO |
| 2340 | * |
| 2341 | * and this case: |
| 2342 | * r6 = 1 |
| 2343 | * call foo // uses callee's r6 inside to compute r0 |
| 2344 | * r0 += r6 |
| 2345 | * if r0 == 0 goto |
| 2346 | * |
| 2347 | * to track above reg_mask/stack_mask needs to be independent for each frame. |
| 2348 | * |
| 2349 | * Also if parent's curframe > frame where backtracking started, |
| 2350 | * the verifier need to mark registers in both frames, otherwise callees |
| 2351 | * may incorrectly prune callers. This is similar to |
| 2352 | * commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences") |
| 2353 | * |
| 2354 | * For now backtracking falls back into conservative marking. |
| 2355 | */ |
| 2356 | static void mark_all_scalars_precise(struct bpf_verifier_env *env, |
| 2357 | struct bpf_verifier_state *st) |
| 2358 | { |
| 2359 | struct bpf_func_state *func; |
| 2360 | struct bpf_reg_state *reg; |
| 2361 | int i, j; |
| 2362 | |
| 2363 | /* big hammer: mark all scalars precise in this path. |
| 2364 | * pop_stack may still get !precise scalars. |
| 2365 | */ |
| 2366 | for (; st; st = st->parent) |
| 2367 | for (i = 0; i <= st->curframe; i++) { |
| 2368 | func = st->frame[i]; |
| 2369 | for (j = 0; j < BPF_REG_FP; j++) { |
| 2370 | reg = &func->regs[j]; |
| 2371 | if (reg->type != SCALAR_VALUE) |
| 2372 | continue; |
| 2373 | reg->precise = true; |
| 2374 | } |
| 2375 | for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { |
| 2376 | if (func->stack[j].slot_type[0] != STACK_SPILL) |
| 2377 | continue; |
| 2378 | reg = &func->stack[j].spilled_ptr; |
| 2379 | if (reg->type != SCALAR_VALUE) |
| 2380 | continue; |
| 2381 | reg->precise = true; |
| 2382 | } |
| 2383 | } |
| 2384 | } |
| 2385 | |
Alexei Starovoitov | a3ce685 | 2019-06-28 09:24:09 -0700 | [diff] [blame] | 2386 | static int __mark_chain_precision(struct bpf_verifier_env *env, int regno, |
| 2387 | int spi) |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2388 | { |
| 2389 | struct bpf_verifier_state *st = env->cur_state; |
| 2390 | int first_idx = st->first_insn_idx; |
| 2391 | int last_idx = env->insn_idx; |
| 2392 | struct bpf_func_state *func; |
| 2393 | struct bpf_reg_state *reg; |
Alexei Starovoitov | a3ce685 | 2019-06-28 09:24:09 -0700 | [diff] [blame] | 2394 | u32 reg_mask = regno >= 0 ? 1u << regno : 0; |
| 2395 | u64 stack_mask = spi >= 0 ? 1ull << spi : 0; |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2396 | bool skip_first = true; |
Alexei Starovoitov | a3ce685 | 2019-06-28 09:24:09 -0700 | [diff] [blame] | 2397 | bool new_marks = false; |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2398 | int i, err; |
| 2399 | |
Alexei Starovoitov | 2c78ee8 | 2020-05-13 16:03:54 -0700 | [diff] [blame] | 2400 | if (!env->bpf_capable) |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2401 | return 0; |
| 2402 | |
| 2403 | func = st->frame[st->curframe]; |
Alexei Starovoitov | a3ce685 | 2019-06-28 09:24:09 -0700 | [diff] [blame] | 2404 | if (regno >= 0) { |
| 2405 | reg = &func->regs[regno]; |
| 2406 | if (reg->type != SCALAR_VALUE) { |
| 2407 | WARN_ONCE(1, "backtracing misuse"); |
| 2408 | return -EFAULT; |
| 2409 | } |
| 2410 | if (!reg->precise) |
| 2411 | new_marks = true; |
| 2412 | else |
| 2413 | reg_mask = 0; |
| 2414 | reg->precise = true; |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2415 | } |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2416 | |
Alexei Starovoitov | a3ce685 | 2019-06-28 09:24:09 -0700 | [diff] [blame] | 2417 | while (spi >= 0) { |
| 2418 | if (func->stack[spi].slot_type[0] != STACK_SPILL) { |
| 2419 | stack_mask = 0; |
| 2420 | break; |
| 2421 | } |
| 2422 | reg = &func->stack[spi].spilled_ptr; |
| 2423 | if (reg->type != SCALAR_VALUE) { |
| 2424 | stack_mask = 0; |
| 2425 | break; |
| 2426 | } |
| 2427 | if (!reg->precise) |
| 2428 | new_marks = true; |
| 2429 | else |
| 2430 | stack_mask = 0; |
| 2431 | reg->precise = true; |
| 2432 | break; |
| 2433 | } |
| 2434 | |
| 2435 | if (!new_marks) |
| 2436 | return 0; |
| 2437 | if (!reg_mask && !stack_mask) |
| 2438 | return 0; |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2439 | for (;;) { |
| 2440 | DECLARE_BITMAP(mask, 64); |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2441 | u32 history = st->jmp_history_cnt; |
| 2442 | |
| 2443 | if (env->log.level & BPF_LOG_LEVEL) |
| 2444 | verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx); |
| 2445 | for (i = last_idx;;) { |
| 2446 | if (skip_first) { |
| 2447 | err = 0; |
| 2448 | skip_first = false; |
| 2449 | } else { |
| 2450 | err = backtrack_insn(env, i, ®_mask, &stack_mask); |
| 2451 | } |
| 2452 | if (err == -ENOTSUPP) { |
| 2453 | mark_all_scalars_precise(env, st); |
| 2454 | return 0; |
| 2455 | } else if (err) { |
| 2456 | return err; |
| 2457 | } |
| 2458 | if (!reg_mask && !stack_mask) |
| 2459 | /* Found assignment(s) into tracked register in this state. |
| 2460 | * Since this state is already marked, just return. |
| 2461 | * Nothing to be tracked further in the parent state. |
| 2462 | */ |
| 2463 | return 0; |
| 2464 | if (i == first_idx) |
| 2465 | break; |
| 2466 | i = get_prev_insn_idx(st, i, &history); |
| 2467 | if (i >= env->prog->len) { |
| 2468 | /* This can happen if backtracking reached insn 0 |
| 2469 | * and there are still reg_mask or stack_mask |
| 2470 | * to backtrack. |
| 2471 | * It means the backtracking missed the spot where |
| 2472 | * particular register was initialized with a constant. |
| 2473 | */ |
| 2474 | verbose(env, "BUG backtracking idx %d\n", i); |
| 2475 | WARN_ONCE(1, "verifier backtracking bug"); |
| 2476 | return -EFAULT; |
| 2477 | } |
| 2478 | } |
| 2479 | st = st->parent; |
| 2480 | if (!st) |
| 2481 | break; |
| 2482 | |
Alexei Starovoitov | a3ce685 | 2019-06-28 09:24:09 -0700 | [diff] [blame] | 2483 | new_marks = false; |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2484 | func = st->frame[st->curframe]; |
| 2485 | bitmap_from_u64(mask, reg_mask); |
| 2486 | for_each_set_bit(i, mask, 32) { |
| 2487 | reg = &func->regs[i]; |
Alexei Starovoitov | a3ce685 | 2019-06-28 09:24:09 -0700 | [diff] [blame] | 2488 | if (reg->type != SCALAR_VALUE) { |
| 2489 | reg_mask &= ~(1u << i); |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2490 | continue; |
Alexei Starovoitov | a3ce685 | 2019-06-28 09:24:09 -0700 | [diff] [blame] | 2491 | } |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2492 | if (!reg->precise) |
| 2493 | new_marks = true; |
| 2494 | reg->precise = true; |
| 2495 | } |
| 2496 | |
| 2497 | bitmap_from_u64(mask, stack_mask); |
| 2498 | for_each_set_bit(i, mask, 64) { |
| 2499 | if (i >= func->allocated_stack / BPF_REG_SIZE) { |
Alexei Starovoitov | 2339cd6 | 2019-09-03 15:16:17 -0700 | [diff] [blame] | 2500 | /* the sequence of instructions: |
| 2501 | * 2: (bf) r3 = r10 |
| 2502 | * 3: (7b) *(u64 *)(r3 -8) = r0 |
| 2503 | * 4: (79) r4 = *(u64 *)(r10 -8) |
| 2504 | * doesn't contain jmps. It's backtracked |
| 2505 | * as a single block. |
| 2506 | * During backtracking insn 3 is not recognized as |
| 2507 | * stack access, so at the end of backtracking |
| 2508 | * stack slot fp-8 is still marked in stack_mask. |
| 2509 | * However the parent state may not have accessed |
| 2510 | * fp-8 and it's "unallocated" stack space. |
| 2511 | * In such case fallback to conservative. |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2512 | */ |
Alexei Starovoitov | 2339cd6 | 2019-09-03 15:16:17 -0700 | [diff] [blame] | 2513 | mark_all_scalars_precise(env, st); |
| 2514 | return 0; |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2515 | } |
| 2516 | |
Alexei Starovoitov | a3ce685 | 2019-06-28 09:24:09 -0700 | [diff] [blame] | 2517 | if (func->stack[i].slot_type[0] != STACK_SPILL) { |
| 2518 | stack_mask &= ~(1ull << i); |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2519 | continue; |
Alexei Starovoitov | a3ce685 | 2019-06-28 09:24:09 -0700 | [diff] [blame] | 2520 | } |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2521 | reg = &func->stack[i].spilled_ptr; |
Alexei Starovoitov | a3ce685 | 2019-06-28 09:24:09 -0700 | [diff] [blame] | 2522 | if (reg->type != SCALAR_VALUE) { |
| 2523 | stack_mask &= ~(1ull << i); |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2524 | continue; |
Alexei Starovoitov | a3ce685 | 2019-06-28 09:24:09 -0700 | [diff] [blame] | 2525 | } |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2526 | if (!reg->precise) |
| 2527 | new_marks = true; |
| 2528 | reg->precise = true; |
| 2529 | } |
| 2530 | if (env->log.level & BPF_LOG_LEVEL) { |
| 2531 | print_verifier_state(env, func); |
| 2532 | verbose(env, "parent %s regs=%x stack=%llx marks\n", |
| 2533 | new_marks ? "didn't have" : "already had", |
| 2534 | reg_mask, stack_mask); |
| 2535 | } |
| 2536 | |
Alexei Starovoitov | a3ce685 | 2019-06-28 09:24:09 -0700 | [diff] [blame] | 2537 | if (!reg_mask && !stack_mask) |
| 2538 | break; |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2539 | if (!new_marks) |
| 2540 | break; |
| 2541 | |
| 2542 | last_idx = st->last_insn_idx; |
| 2543 | first_idx = st->first_insn_idx; |
| 2544 | } |
| 2545 | return 0; |
| 2546 | } |
| 2547 | |
Alexei Starovoitov | a3ce685 | 2019-06-28 09:24:09 -0700 | [diff] [blame] | 2548 | static int mark_chain_precision(struct bpf_verifier_env *env, int regno) |
| 2549 | { |
| 2550 | return __mark_chain_precision(env, regno, -1); |
| 2551 | } |
| 2552 | |
| 2553 | static int mark_chain_precision_stack(struct bpf_verifier_env *env, int spi) |
| 2554 | { |
| 2555 | return __mark_chain_precision(env, -1, spi); |
| 2556 | } |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2557 | |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 2558 | static bool is_spillable_regtype(enum bpf_reg_type type) |
| 2559 | { |
| 2560 | switch (type) { |
| 2561 | case PTR_TO_MAP_VALUE: |
| 2562 | case PTR_TO_MAP_VALUE_OR_NULL: |
| 2563 | case PTR_TO_STACK: |
| 2564 | case PTR_TO_CTX: |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 2565 | case PTR_TO_PACKET: |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 2566 | case PTR_TO_PACKET_META: |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 2567 | case PTR_TO_PACKET_END: |
Petar Penkov | d58e468 | 2018-09-14 07:46:18 -0700 | [diff] [blame] | 2568 | case PTR_TO_FLOW_KEYS: |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 2569 | case CONST_PTR_TO_MAP: |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 2570 | case PTR_TO_SOCKET: |
| 2571 | case PTR_TO_SOCKET_OR_NULL: |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 2572 | case PTR_TO_SOCK_COMMON: |
| 2573 | case PTR_TO_SOCK_COMMON_OR_NULL: |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 2574 | case PTR_TO_TCP_SOCK: |
| 2575 | case PTR_TO_TCP_SOCK_OR_NULL: |
Jonathan Lemon | fada7fd | 2019-06-06 13:59:40 -0700 | [diff] [blame] | 2576 | case PTR_TO_XDP_SOCK: |
Martin KaFai Lau | 65726b5 | 2020-01-08 16:34:54 -0800 | [diff] [blame] | 2577 | case PTR_TO_BTF_ID: |
Yonghong Song | b121b34 | 2020-05-09 10:59:12 -0700 | [diff] [blame] | 2578 | case PTR_TO_BTF_ID_OR_NULL: |
Yonghong Song | afbf21d | 2020-07-23 11:41:11 -0700 | [diff] [blame] | 2579 | case PTR_TO_RDONLY_BUF: |
| 2580 | case PTR_TO_RDONLY_BUF_OR_NULL: |
| 2581 | case PTR_TO_RDWR_BUF: |
| 2582 | case PTR_TO_RDWR_BUF_OR_NULL: |
Hao Luo | eaa6bcb | 2020-09-29 16:50:47 -0700 | [diff] [blame] | 2583 | case PTR_TO_PERCPU_BTF_ID: |
Gilad Reti | 744ea4e | 2021-01-13 07:38:07 +0200 | [diff] [blame] | 2584 | case PTR_TO_MEM: |
| 2585 | case PTR_TO_MEM_OR_NULL: |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 2586 | case PTR_TO_FUNC: |
| 2587 | case PTR_TO_MAP_KEY: |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 2588 | return true; |
| 2589 | default: |
| 2590 | return false; |
| 2591 | } |
| 2592 | } |
| 2593 | |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 2594 | /* Does this register contain a constant zero? */ |
| 2595 | static bool register_is_null(struct bpf_reg_state *reg) |
| 2596 | { |
| 2597 | return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0); |
| 2598 | } |
| 2599 | |
Alexei Starovoitov | f7cf25b | 2019-06-15 12:12:17 -0700 | [diff] [blame] | 2600 | static bool register_is_const(struct bpf_reg_state *reg) |
| 2601 | { |
| 2602 | return reg->type == SCALAR_VALUE && tnum_is_const(reg->var_off); |
| 2603 | } |
| 2604 | |
Yonghong Song | 5689d49 | 2020-10-08 18:12:38 -0700 | [diff] [blame] | 2605 | static bool __is_scalar_unbounded(struct bpf_reg_state *reg) |
| 2606 | { |
| 2607 | return tnum_is_unknown(reg->var_off) && |
| 2608 | reg->smin_value == S64_MIN && reg->smax_value == S64_MAX && |
| 2609 | reg->umin_value == 0 && reg->umax_value == U64_MAX && |
| 2610 | reg->s32_min_value == S32_MIN && reg->s32_max_value == S32_MAX && |
| 2611 | reg->u32_min_value == 0 && reg->u32_max_value == U32_MAX; |
| 2612 | } |
| 2613 | |
| 2614 | static bool register_is_bounded(struct bpf_reg_state *reg) |
| 2615 | { |
| 2616 | return reg->type == SCALAR_VALUE && !__is_scalar_unbounded(reg); |
| 2617 | } |
| 2618 | |
Jann Horn | 6e7e63c | 2020-04-17 02:00:06 +0200 | [diff] [blame] | 2619 | static bool __is_pointer_value(bool allow_ptr_leaks, |
| 2620 | const struct bpf_reg_state *reg) |
| 2621 | { |
| 2622 | if (allow_ptr_leaks) |
| 2623 | return false; |
| 2624 | |
| 2625 | return reg->type != SCALAR_VALUE; |
| 2626 | } |
| 2627 | |
Alexei Starovoitov | f7cf25b | 2019-06-15 12:12:17 -0700 | [diff] [blame] | 2628 | static void save_register_state(struct bpf_func_state *state, |
| 2629 | int spi, struct bpf_reg_state *reg) |
| 2630 | { |
| 2631 | int i; |
| 2632 | |
| 2633 | state->stack[spi].spilled_ptr = *reg; |
| 2634 | state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; |
| 2635 | |
| 2636 | for (i = 0; i < BPF_REG_SIZE; i++) |
| 2637 | state->stack[spi].slot_type[i] = STACK_SPILL; |
| 2638 | } |
| 2639 | |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 2640 | /* check_stack_{read,write}_fixed_off functions track spill/fill of registers, |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2641 | * stack boundary and alignment are checked in check_mem_access() |
| 2642 | */ |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 2643 | static int check_stack_write_fixed_off(struct bpf_verifier_env *env, |
| 2644 | /* stack frame we're writing to */ |
| 2645 | struct bpf_func_state *state, |
| 2646 | int off, int size, int value_regno, |
| 2647 | int insn_idx) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2648 | { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 2649 | struct bpf_func_state *cur; /* state of the current function */ |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 2650 | int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err; |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2651 | u32 dst_reg = env->prog->insnsi[insn_idx].dst_reg; |
Alexei Starovoitov | f7cf25b | 2019-06-15 12:12:17 -0700 | [diff] [blame] | 2652 | struct bpf_reg_state *reg = NULL; |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 2653 | |
Lorenz Bauer | c69431a | 2021-04-29 14:46:54 +0100 | [diff] [blame] | 2654 | err = grow_stack_state(state, round_up(slot + 1, BPF_REG_SIZE)); |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 2655 | if (err) |
| 2656 | return err; |
Alexei Starovoitov | 9c399760 | 2014-10-28 15:11:41 -0700 | [diff] [blame] | 2657 | /* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0, |
| 2658 | * so it's aligned access and [off, off + size) are within stack limits |
| 2659 | */ |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 2660 | if (!env->allow_ptr_leaks && |
| 2661 | state->stack[spi].slot_type[0] == STACK_SPILL && |
| 2662 | size != BPF_REG_SIZE) { |
| 2663 | verbose(env, "attempt to corrupt spilled pointer on stack\n"); |
| 2664 | return -EACCES; |
| 2665 | } |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2666 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 2667 | cur = env->cur_state->frame[env->cur_state->curframe]; |
Alexei Starovoitov | f7cf25b | 2019-06-15 12:12:17 -0700 | [diff] [blame] | 2668 | if (value_regno >= 0) |
| 2669 | reg = &cur->regs[value_regno]; |
Daniel Borkmann | 2039f26 | 2021-07-13 08:18:31 +0000 | [diff] [blame] | 2670 | if (!env->bypass_spec_v4) { |
| 2671 | bool sanitize = reg && is_spillable_regtype(reg->type); |
| 2672 | |
| 2673 | for (i = 0; i < size; i++) { |
| 2674 | if (state->stack[spi].slot_type[i] == STACK_INVALID) { |
| 2675 | sanitize = true; |
| 2676 | break; |
| 2677 | } |
| 2678 | } |
| 2679 | |
| 2680 | if (sanitize) |
| 2681 | env->insn_aux_data[insn_idx].sanitize_stack_spill = true; |
| 2682 | } |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2683 | |
Yonghong Song | 5689d49 | 2020-10-08 18:12:38 -0700 | [diff] [blame] | 2684 | if (reg && size == BPF_REG_SIZE && register_is_bounded(reg) && |
Alexei Starovoitov | 2c78ee8 | 2020-05-13 16:03:54 -0700 | [diff] [blame] | 2685 | !register_is_null(reg) && env->bpf_capable) { |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2686 | if (dst_reg != BPF_REG_FP) { |
| 2687 | /* The backtracking logic can only recognize explicit |
| 2688 | * stack slot address like [fp - 8]. Other spill of |
Zhen Lei | 8fb33b6 | 2021-05-25 10:56:59 +0800 | [diff] [blame] | 2689 | * scalar via different register has to be conservative. |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2690 | * Backtrack from here and mark all registers as precise |
| 2691 | * that contributed into 'reg' being a constant. |
| 2692 | */ |
| 2693 | err = mark_chain_precision(env, value_regno); |
| 2694 | if (err) |
| 2695 | return err; |
| 2696 | } |
Alexei Starovoitov | f7cf25b | 2019-06-15 12:12:17 -0700 | [diff] [blame] | 2697 | save_register_state(state, spi, reg); |
| 2698 | } else if (reg && is_spillable_regtype(reg->type)) { |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2699 | /* register containing pointer is being spilled into stack */ |
Alexei Starovoitov | 9c399760 | 2014-10-28 15:11:41 -0700 | [diff] [blame] | 2700 | if (size != BPF_REG_SIZE) { |
Alexei Starovoitov | f7cf25b | 2019-06-15 12:12:17 -0700 | [diff] [blame] | 2701 | verbose_linfo(env, insn_idx, "; "); |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2702 | verbose(env, "invalid size of register spill\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2703 | return -EACCES; |
| 2704 | } |
Alexei Starovoitov | f7cf25b | 2019-06-15 12:12:17 -0700 | [diff] [blame] | 2705 | if (state != cur && reg->type == PTR_TO_STACK) { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 2706 | verbose(env, "cannot spill pointers to stack into stack frame of the caller\n"); |
| 2707 | return -EINVAL; |
| 2708 | } |
Alexei Starovoitov | f7cf25b | 2019-06-15 12:12:17 -0700 | [diff] [blame] | 2709 | save_register_state(state, spi, reg); |
Alexei Starovoitov | 9c399760 | 2014-10-28 15:11:41 -0700 | [diff] [blame] | 2710 | } else { |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 2711 | u8 type = STACK_MISC; |
| 2712 | |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 2713 | /* regular write of data into stack destroys any spilled ptr */ |
| 2714 | state->stack[spi].spilled_ptr.type = NOT_INIT; |
Jiong Wang | 0bae2d4 | 2018-12-15 03:34:40 -0500 | [diff] [blame] | 2715 | /* Mark slots as STACK_MISC if they belonged to spilled ptr. */ |
| 2716 | if (state->stack[spi].slot_type[0] == STACK_SPILL) |
| 2717 | for (i = 0; i < BPF_REG_SIZE; i++) |
| 2718 | state->stack[spi].slot_type[i] = STACK_MISC; |
Alexei Starovoitov | 9c399760 | 2014-10-28 15:11:41 -0700 | [diff] [blame] | 2719 | |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 2720 | /* only mark the slot as written if all 8 bytes were written |
| 2721 | * otherwise read propagation may incorrectly stop too soon |
| 2722 | * when stack slots are partially written. |
| 2723 | * This heuristic means that read propagation will be |
| 2724 | * conservative, since it will add reg_live_read marks |
| 2725 | * to stack slots all the way to first state when programs |
| 2726 | * writes+reads less than 8 bytes |
| 2727 | */ |
| 2728 | if (size == BPF_REG_SIZE) |
| 2729 | state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; |
| 2730 | |
| 2731 | /* when we zero initialize stack slots mark them as such */ |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2732 | if (reg && register_is_null(reg)) { |
| 2733 | /* backtracking doesn't work for STACK_ZERO yet. */ |
| 2734 | err = mark_chain_precision(env, value_regno); |
| 2735 | if (err) |
| 2736 | return err; |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 2737 | type = STACK_ZERO; |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 2738 | } |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 2739 | |
Jiong Wang | 0bae2d4 | 2018-12-15 03:34:40 -0500 | [diff] [blame] | 2740 | /* Mark slots affected by this stack write. */ |
Alexei Starovoitov | 9c399760 | 2014-10-28 15:11:41 -0700 | [diff] [blame] | 2741 | for (i = 0; i < size; i++) |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 2742 | state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 2743 | type; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2744 | } |
| 2745 | return 0; |
| 2746 | } |
| 2747 | |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 2748 | /* Write the stack: 'stack[ptr_regno + off] = value_regno'. 'ptr_regno' is |
| 2749 | * known to contain a variable offset. |
| 2750 | * This function checks whether the write is permitted and conservatively |
| 2751 | * tracks the effects of the write, considering that each stack slot in the |
| 2752 | * dynamic range is potentially written to. |
| 2753 | * |
| 2754 | * 'off' includes 'regno->off'. |
| 2755 | * 'value_regno' can be -1, meaning that an unknown value is being written to |
| 2756 | * the stack. |
| 2757 | * |
| 2758 | * Spilled pointers in range are not marked as written because we don't know |
| 2759 | * what's going to be actually written. This means that read propagation for |
| 2760 | * future reads cannot be terminated by this write. |
| 2761 | * |
| 2762 | * For privileged programs, uninitialized stack slots are considered |
| 2763 | * initialized by this write (even though we don't know exactly what offsets |
| 2764 | * are going to be written to). The idea is that we don't want the verifier to |
| 2765 | * reject future reads that access slots written to through variable offsets. |
| 2766 | */ |
| 2767 | static int check_stack_write_var_off(struct bpf_verifier_env *env, |
| 2768 | /* func where register points to */ |
| 2769 | struct bpf_func_state *state, |
| 2770 | int ptr_regno, int off, int size, |
| 2771 | int value_regno, int insn_idx) |
| 2772 | { |
| 2773 | struct bpf_func_state *cur; /* state of the current function */ |
| 2774 | int min_off, max_off; |
| 2775 | int i, err; |
| 2776 | struct bpf_reg_state *ptr_reg = NULL, *value_reg = NULL; |
| 2777 | bool writing_zero = false; |
| 2778 | /* set if the fact that we're writing a zero is used to let any |
| 2779 | * stack slots remain STACK_ZERO |
| 2780 | */ |
| 2781 | bool zero_used = false; |
| 2782 | |
| 2783 | cur = env->cur_state->frame[env->cur_state->curframe]; |
| 2784 | ptr_reg = &cur->regs[ptr_regno]; |
| 2785 | min_off = ptr_reg->smin_value + off; |
| 2786 | max_off = ptr_reg->smax_value + off + size; |
| 2787 | if (value_regno >= 0) |
| 2788 | value_reg = &cur->regs[value_regno]; |
| 2789 | if (value_reg && register_is_null(value_reg)) |
| 2790 | writing_zero = true; |
| 2791 | |
Lorenz Bauer | c69431a | 2021-04-29 14:46:54 +0100 | [diff] [blame] | 2792 | err = grow_stack_state(state, round_up(-min_off, BPF_REG_SIZE)); |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 2793 | if (err) |
| 2794 | return err; |
| 2795 | |
| 2796 | |
| 2797 | /* Variable offset writes destroy any spilled pointers in range. */ |
| 2798 | for (i = min_off; i < max_off; i++) { |
| 2799 | u8 new_type, *stype; |
| 2800 | int slot, spi; |
| 2801 | |
| 2802 | slot = -i - 1; |
| 2803 | spi = slot / BPF_REG_SIZE; |
| 2804 | stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; |
| 2805 | |
| 2806 | if (!env->allow_ptr_leaks |
| 2807 | && *stype != NOT_INIT |
| 2808 | && *stype != SCALAR_VALUE) { |
| 2809 | /* Reject the write if there's are spilled pointers in |
| 2810 | * range. If we didn't reject here, the ptr status |
| 2811 | * would be erased below (even though not all slots are |
| 2812 | * actually overwritten), possibly opening the door to |
| 2813 | * leaks. |
| 2814 | */ |
| 2815 | verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d", |
| 2816 | insn_idx, i); |
| 2817 | return -EINVAL; |
| 2818 | } |
| 2819 | |
| 2820 | /* Erase all spilled pointers. */ |
| 2821 | state->stack[spi].spilled_ptr.type = NOT_INIT; |
| 2822 | |
| 2823 | /* Update the slot type. */ |
| 2824 | new_type = STACK_MISC; |
| 2825 | if (writing_zero && *stype == STACK_ZERO) { |
| 2826 | new_type = STACK_ZERO; |
| 2827 | zero_used = true; |
| 2828 | } |
| 2829 | /* If the slot is STACK_INVALID, we check whether it's OK to |
| 2830 | * pretend that it will be initialized by this write. The slot |
| 2831 | * might not actually be written to, and so if we mark it as |
| 2832 | * initialized future reads might leak uninitialized memory. |
| 2833 | * For privileged programs, we will accept such reads to slots |
| 2834 | * that may or may not be written because, if we're reject |
| 2835 | * them, the error would be too confusing. |
| 2836 | */ |
| 2837 | if (*stype == STACK_INVALID && !env->allow_uninit_stack) { |
| 2838 | verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d", |
| 2839 | insn_idx, i); |
| 2840 | return -EINVAL; |
| 2841 | } |
| 2842 | *stype = new_type; |
| 2843 | } |
| 2844 | if (zero_used) { |
| 2845 | /* backtracking doesn't work for STACK_ZERO yet. */ |
| 2846 | err = mark_chain_precision(env, value_regno); |
| 2847 | if (err) |
| 2848 | return err; |
| 2849 | } |
| 2850 | return 0; |
| 2851 | } |
| 2852 | |
| 2853 | /* When register 'dst_regno' is assigned some values from stack[min_off, |
| 2854 | * max_off), we set the register's type according to the types of the |
| 2855 | * respective stack slots. If all the stack values are known to be zeros, then |
| 2856 | * so is the destination reg. Otherwise, the register is considered to be |
| 2857 | * SCALAR. This function does not deal with register filling; the caller must |
| 2858 | * ensure that all spilled registers in the stack range have been marked as |
| 2859 | * read. |
| 2860 | */ |
| 2861 | static void mark_reg_stack_read(struct bpf_verifier_env *env, |
| 2862 | /* func where src register points to */ |
| 2863 | struct bpf_func_state *ptr_state, |
| 2864 | int min_off, int max_off, int dst_regno) |
| 2865 | { |
| 2866 | struct bpf_verifier_state *vstate = env->cur_state; |
| 2867 | struct bpf_func_state *state = vstate->frame[vstate->curframe]; |
| 2868 | int i, slot, spi; |
| 2869 | u8 *stype; |
| 2870 | int zeros = 0; |
| 2871 | |
| 2872 | for (i = min_off; i < max_off; i++) { |
| 2873 | slot = -i - 1; |
| 2874 | spi = slot / BPF_REG_SIZE; |
| 2875 | stype = ptr_state->stack[spi].slot_type; |
| 2876 | if (stype[slot % BPF_REG_SIZE] != STACK_ZERO) |
| 2877 | break; |
| 2878 | zeros++; |
| 2879 | } |
| 2880 | if (zeros == max_off - min_off) { |
| 2881 | /* any access_size read into register is zero extended, |
| 2882 | * so the whole register == const_zero |
| 2883 | */ |
| 2884 | __mark_reg_const_zero(&state->regs[dst_regno]); |
| 2885 | /* backtracking doesn't support STACK_ZERO yet, |
| 2886 | * so mark it precise here, so that later |
| 2887 | * backtracking can stop here. |
| 2888 | * Backtracking may not need this if this register |
| 2889 | * doesn't participate in pointer adjustment. |
| 2890 | * Forward propagation of precise flag is not |
| 2891 | * necessary either. This mark is only to stop |
| 2892 | * backtracking. Any register that contributed |
| 2893 | * to const 0 was marked precise before spill. |
| 2894 | */ |
| 2895 | state->regs[dst_regno].precise = true; |
| 2896 | } else { |
| 2897 | /* have read misc data from the stack */ |
| 2898 | mark_reg_unknown(env, state->regs, dst_regno); |
| 2899 | } |
| 2900 | state->regs[dst_regno].live |= REG_LIVE_WRITTEN; |
| 2901 | } |
| 2902 | |
| 2903 | /* Read the stack at 'off' and put the results into the register indicated by |
| 2904 | * 'dst_regno'. It handles reg filling if the addressed stack slot is a |
| 2905 | * spilled reg. |
| 2906 | * |
| 2907 | * 'dst_regno' can be -1, meaning that the read value is not going to a |
| 2908 | * register. |
| 2909 | * |
| 2910 | * The access is assumed to be within the current stack bounds. |
| 2911 | */ |
| 2912 | static int check_stack_read_fixed_off(struct bpf_verifier_env *env, |
| 2913 | /* func where src register points to */ |
| 2914 | struct bpf_func_state *reg_state, |
| 2915 | int off, int size, int dst_regno) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2916 | { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 2917 | struct bpf_verifier_state *vstate = env->cur_state; |
| 2918 | struct bpf_func_state *state = vstate->frame[vstate->curframe]; |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 2919 | int i, slot = -off - 1, spi = slot / BPF_REG_SIZE; |
Alexei Starovoitov | f7cf25b | 2019-06-15 12:12:17 -0700 | [diff] [blame] | 2920 | struct bpf_reg_state *reg; |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 2921 | u8 *stype; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2922 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 2923 | stype = reg_state->stack[spi].slot_type; |
Alexei Starovoitov | f7cf25b | 2019-06-15 12:12:17 -0700 | [diff] [blame] | 2924 | reg = ®_state->stack[spi].spilled_ptr; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2925 | |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 2926 | if (stype[0] == STACK_SPILL) { |
Alexei Starovoitov | 9c399760 | 2014-10-28 15:11:41 -0700 | [diff] [blame] | 2927 | if (size != BPF_REG_SIZE) { |
Alexei Starovoitov | f7cf25b | 2019-06-15 12:12:17 -0700 | [diff] [blame] | 2928 | if (reg->type != SCALAR_VALUE) { |
| 2929 | verbose_linfo(env, env->insn_idx, "; "); |
| 2930 | verbose(env, "invalid size of register fill\n"); |
| 2931 | return -EACCES; |
| 2932 | } |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 2933 | if (dst_regno >= 0) { |
| 2934 | mark_reg_unknown(env, state->regs, dst_regno); |
| 2935 | state->regs[dst_regno].live |= REG_LIVE_WRITTEN; |
Alexei Starovoitov | f7cf25b | 2019-06-15 12:12:17 -0700 | [diff] [blame] | 2936 | } |
| 2937 | mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); |
| 2938 | return 0; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2939 | } |
Alexei Starovoitov | 9c399760 | 2014-10-28 15:11:41 -0700 | [diff] [blame] | 2940 | for (i = 1; i < BPF_REG_SIZE; i++) { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 2941 | if (stype[(slot - i) % BPF_REG_SIZE] != STACK_SPILL) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 2942 | verbose(env, "corrupted spill memory\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2943 | return -EACCES; |
| 2944 | } |
| 2945 | } |
| 2946 | |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 2947 | if (dst_regno >= 0) { |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2948 | /* restore register state from stack */ |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 2949 | state->regs[dst_regno] = *reg; |
Alexei Starovoitov | 2f18f62 | 2017-11-30 21:31:38 -0800 | [diff] [blame] | 2950 | /* mark reg as written since spilled pointer state likely |
| 2951 | * has its liveness marks cleared by is_state_visited() |
| 2952 | * which resets stack/reg liveness for state transitions |
| 2953 | */ |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 2954 | state->regs[dst_regno].live |= REG_LIVE_WRITTEN; |
Jann Horn | 6e7e63c | 2020-04-17 02:00:06 +0200 | [diff] [blame] | 2955 | } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) { |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 2956 | /* If dst_regno==-1, the caller is asking us whether |
Jann Horn | 6e7e63c | 2020-04-17 02:00:06 +0200 | [diff] [blame] | 2957 | * it is acceptable to use this value as a SCALAR_VALUE |
| 2958 | * (e.g. for XADD). |
| 2959 | * We must not allow unprivileged callers to do that |
| 2960 | * with spilled pointers. |
| 2961 | */ |
| 2962 | verbose(env, "leaking pointer from stack off %d\n", |
| 2963 | off); |
| 2964 | return -EACCES; |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 2965 | } |
Alexei Starovoitov | f7cf25b | 2019-06-15 12:12:17 -0700 | [diff] [blame] | 2966 | mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2967 | } else { |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 2968 | u8 type; |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 2969 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2970 | for (i = 0; i < size; i++) { |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 2971 | type = stype[(slot - i) % BPF_REG_SIZE]; |
| 2972 | if (type == STACK_MISC) |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 2973 | continue; |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 2974 | if (type == STACK_ZERO) |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 2975 | continue; |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 2976 | verbose(env, "invalid read from stack off %d+%d size %d\n", |
| 2977 | off, i, size); |
| 2978 | return -EACCES; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2979 | } |
Alexei Starovoitov | f7cf25b | 2019-06-15 12:12:17 -0700 | [diff] [blame] | 2980 | mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 2981 | if (dst_regno >= 0) |
| 2982 | mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2983 | } |
Alexei Starovoitov | f7cf25b | 2019-06-15 12:12:17 -0700 | [diff] [blame] | 2984 | return 0; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 2985 | } |
| 2986 | |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 2987 | enum stack_access_src { |
| 2988 | ACCESS_DIRECT = 1, /* the access is performed by an instruction */ |
| 2989 | ACCESS_HELPER = 2, /* the access is performed by a helper */ |
| 2990 | }; |
| 2991 | |
| 2992 | static int check_stack_range_initialized(struct bpf_verifier_env *env, |
| 2993 | int regno, int off, int access_size, |
| 2994 | bool zero_size_allowed, |
| 2995 | enum stack_access_src type, |
| 2996 | struct bpf_call_arg_meta *meta); |
| 2997 | |
| 2998 | static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) |
Daniel Borkmann | e4298d2 | 2019-01-03 00:58:31 +0100 | [diff] [blame] | 2999 | { |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 3000 | return cur_regs(env) + regno; |
| 3001 | } |
| 3002 | |
| 3003 | /* Read the stack at 'ptr_regno + off' and put the result into the register |
| 3004 | * 'dst_regno'. |
| 3005 | * 'off' includes the pointer register's fixed offset(i.e. 'ptr_regno.off'), |
| 3006 | * but not its variable offset. |
| 3007 | * 'size' is assumed to be <= reg size and the access is assumed to be aligned. |
| 3008 | * |
| 3009 | * As opposed to check_stack_read_fixed_off, this function doesn't deal with |
| 3010 | * filling registers (i.e. reads of spilled register cannot be detected when |
| 3011 | * the offset is not fixed). We conservatively mark 'dst_regno' as containing |
| 3012 | * SCALAR_VALUE. That's why we assert that the 'ptr_regno' has a variable |
| 3013 | * offset; for a fixed offset check_stack_read_fixed_off should be used |
| 3014 | * instead. |
| 3015 | */ |
| 3016 | static int check_stack_read_var_off(struct bpf_verifier_env *env, |
| 3017 | int ptr_regno, int off, int size, int dst_regno) |
| 3018 | { |
| 3019 | /* The state of the source register. */ |
| 3020 | struct bpf_reg_state *reg = reg_state(env, ptr_regno); |
| 3021 | struct bpf_func_state *ptr_state = func(env, reg); |
| 3022 | int err; |
| 3023 | int min_off, max_off; |
| 3024 | |
| 3025 | /* Note that we pass a NULL meta, so raw access will not be permitted. |
Daniel Borkmann | e4298d2 | 2019-01-03 00:58:31 +0100 | [diff] [blame] | 3026 | */ |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 3027 | err = check_stack_range_initialized(env, ptr_regno, off, size, |
| 3028 | false, ACCESS_DIRECT, NULL); |
| 3029 | if (err) |
| 3030 | return err; |
| 3031 | |
| 3032 | min_off = reg->smin_value + off; |
| 3033 | max_off = reg->smax_value + off; |
| 3034 | mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno); |
| 3035 | return 0; |
| 3036 | } |
| 3037 | |
| 3038 | /* check_stack_read dispatches to check_stack_read_fixed_off or |
| 3039 | * check_stack_read_var_off. |
| 3040 | * |
| 3041 | * The caller must ensure that the offset falls within the allocated stack |
| 3042 | * bounds. |
| 3043 | * |
| 3044 | * 'dst_regno' is a register which will receive the value from the stack. It |
| 3045 | * can be -1, meaning that the read value is not going to a register. |
| 3046 | */ |
| 3047 | static int check_stack_read(struct bpf_verifier_env *env, |
| 3048 | int ptr_regno, int off, int size, |
| 3049 | int dst_regno) |
| 3050 | { |
| 3051 | struct bpf_reg_state *reg = reg_state(env, ptr_regno); |
| 3052 | struct bpf_func_state *state = func(env, reg); |
| 3053 | int err; |
| 3054 | /* Some accesses are only permitted with a static offset. */ |
| 3055 | bool var_off = !tnum_is_const(reg->var_off); |
| 3056 | |
| 3057 | /* The offset is required to be static when reads don't go to a |
| 3058 | * register, in order to not leak pointers (see |
| 3059 | * check_stack_read_fixed_off). |
| 3060 | */ |
| 3061 | if (dst_regno < 0 && var_off) { |
Daniel Borkmann | e4298d2 | 2019-01-03 00:58:31 +0100 | [diff] [blame] | 3062 | char tn_buf[48]; |
| 3063 | |
| 3064 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 3065 | verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n", |
Daniel Borkmann | e4298d2 | 2019-01-03 00:58:31 +0100 | [diff] [blame] | 3066 | tn_buf, off, size); |
| 3067 | return -EACCES; |
| 3068 | } |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 3069 | /* Variable offset is prohibited for unprivileged mode for simplicity |
| 3070 | * since it requires corresponding support in Spectre masking for stack |
| 3071 | * ALU. See also retrieve_ptr_limit(). |
| 3072 | */ |
| 3073 | if (!env->bypass_spec_v1 && var_off) { |
| 3074 | char tn_buf[48]; |
Daniel Borkmann | e4298d2 | 2019-01-03 00:58:31 +0100 | [diff] [blame] | 3075 | |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 3076 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
| 3077 | verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n", |
| 3078 | ptr_regno, tn_buf); |
Daniel Borkmann | e4298d2 | 2019-01-03 00:58:31 +0100 | [diff] [blame] | 3079 | return -EACCES; |
| 3080 | } |
| 3081 | |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 3082 | if (!var_off) { |
| 3083 | off += reg->var_off.value; |
| 3084 | err = check_stack_read_fixed_off(env, state, off, size, |
| 3085 | dst_regno); |
| 3086 | } else { |
| 3087 | /* Variable offset stack reads need more conservative handling |
| 3088 | * than fixed offset ones. Note that dst_regno >= 0 on this |
| 3089 | * branch. |
| 3090 | */ |
| 3091 | err = check_stack_read_var_off(env, ptr_regno, off, size, |
| 3092 | dst_regno); |
| 3093 | } |
| 3094 | return err; |
| 3095 | } |
| 3096 | |
| 3097 | |
| 3098 | /* check_stack_write dispatches to check_stack_write_fixed_off or |
| 3099 | * check_stack_write_var_off. |
| 3100 | * |
| 3101 | * 'ptr_regno' is the register used as a pointer into the stack. |
| 3102 | * 'off' includes 'ptr_regno->off', but not its variable offset (if any). |
| 3103 | * 'value_regno' is the register whose value we're writing to the stack. It can |
| 3104 | * be -1, meaning that we're not writing from a register. |
| 3105 | * |
| 3106 | * The caller must ensure that the offset falls within the maximum stack size. |
| 3107 | */ |
| 3108 | static int check_stack_write(struct bpf_verifier_env *env, |
| 3109 | int ptr_regno, int off, int size, |
| 3110 | int value_regno, int insn_idx) |
| 3111 | { |
| 3112 | struct bpf_reg_state *reg = reg_state(env, ptr_regno); |
| 3113 | struct bpf_func_state *state = func(env, reg); |
| 3114 | int err; |
| 3115 | |
| 3116 | if (tnum_is_const(reg->var_off)) { |
| 3117 | off += reg->var_off.value; |
| 3118 | err = check_stack_write_fixed_off(env, state, off, size, |
| 3119 | value_regno, insn_idx); |
| 3120 | } else { |
| 3121 | /* Variable offset stack reads need more conservative handling |
| 3122 | * than fixed offset ones. |
| 3123 | */ |
| 3124 | err = check_stack_write_var_off(env, state, |
| 3125 | ptr_regno, off, size, |
| 3126 | value_regno, insn_idx); |
| 3127 | } |
| 3128 | return err; |
Daniel Borkmann | e4298d2 | 2019-01-03 00:58:31 +0100 | [diff] [blame] | 3129 | } |
| 3130 | |
Daniel Borkmann | 591fe98 | 2019-04-09 23:20:05 +0200 | [diff] [blame] | 3131 | static int check_map_access_type(struct bpf_verifier_env *env, u32 regno, |
| 3132 | int off, int size, enum bpf_access_type type) |
| 3133 | { |
| 3134 | struct bpf_reg_state *regs = cur_regs(env); |
| 3135 | struct bpf_map *map = regs[regno].map_ptr; |
| 3136 | u32 cap = bpf_map_flags_to_cap(map); |
| 3137 | |
| 3138 | if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) { |
| 3139 | verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n", |
| 3140 | map->value_size, off, size); |
| 3141 | return -EACCES; |
| 3142 | } |
| 3143 | |
| 3144 | if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) { |
| 3145 | verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n", |
| 3146 | map->value_size, off, size); |
| 3147 | return -EACCES; |
| 3148 | } |
| 3149 | |
| 3150 | return 0; |
| 3151 | } |
| 3152 | |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 3153 | /* check read/write into memory region (e.g., map value, ringbuf sample, etc) */ |
| 3154 | static int __check_mem_access(struct bpf_verifier_env *env, int regno, |
| 3155 | int off, int size, u32 mem_size, |
| 3156 | bool zero_size_allowed) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3157 | { |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 3158 | bool size_ok = size > 0 || (size == 0 && zero_size_allowed); |
| 3159 | struct bpf_reg_state *reg; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3160 | |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 3161 | if (off >= 0 && size_ok && (u64)off + size <= mem_size) |
| 3162 | return 0; |
| 3163 | |
| 3164 | reg = &cur_regs(env)[regno]; |
| 3165 | switch (reg->type) { |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 3166 | case PTR_TO_MAP_KEY: |
| 3167 | verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n", |
| 3168 | mem_size, off, size); |
| 3169 | break; |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 3170 | case PTR_TO_MAP_VALUE: |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 3171 | verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n", |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 3172 | mem_size, off, size); |
| 3173 | break; |
| 3174 | case PTR_TO_PACKET: |
| 3175 | case PTR_TO_PACKET_META: |
| 3176 | case PTR_TO_PACKET_END: |
| 3177 | verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n", |
| 3178 | off, size, regno, reg->id, off, mem_size); |
| 3179 | break; |
| 3180 | case PTR_TO_MEM: |
| 3181 | default: |
| 3182 | verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n", |
| 3183 | mem_size, off, size); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3184 | } |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 3185 | |
| 3186 | return -EACCES; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3187 | } |
| 3188 | |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 3189 | /* check read/write into a memory region with possible variable offset */ |
| 3190 | static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno, |
| 3191 | int off, int size, u32 mem_size, |
| 3192 | bool zero_size_allowed) |
Gianluca Borello | dbcfe5f | 2017-01-09 10:19:46 -0800 | [diff] [blame] | 3193 | { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 3194 | struct bpf_verifier_state *vstate = env->cur_state; |
| 3195 | struct bpf_func_state *state = vstate->frame[vstate->curframe]; |
Gianluca Borello | dbcfe5f | 2017-01-09 10:19:46 -0800 | [diff] [blame] | 3196 | struct bpf_reg_state *reg = &state->regs[regno]; |
| 3197 | int err; |
| 3198 | |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 3199 | /* We may have adjusted the register pointing to memory region, so we |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3200 | * need to try adding each of min_value and max_value to off |
| 3201 | * to make sure our theoretical access will be safe. |
Gianluca Borello | dbcfe5f | 2017-01-09 10:19:46 -0800 | [diff] [blame] | 3202 | */ |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 3203 | if (env->log.level & BPF_LOG_LEVEL) |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 3204 | print_verifier_state(env, state); |
Daniel Borkmann | b7137c4 | 2019-01-03 00:58:33 +0100 | [diff] [blame] | 3205 | |
Gianluca Borello | dbcfe5f | 2017-01-09 10:19:46 -0800 | [diff] [blame] | 3206 | /* The minimum value is only important with signed |
| 3207 | * comparisons where we can't assume the floor of a |
| 3208 | * value is 0. If we are using signed variables for our |
| 3209 | * index'es we need to make sure that whatever we use |
| 3210 | * will have a set floor within our range. |
| 3211 | */ |
Daniel Borkmann | b7137c4 | 2019-01-03 00:58:33 +0100 | [diff] [blame] | 3212 | if (reg->smin_value < 0 && |
| 3213 | (reg->smin_value == S64_MIN || |
| 3214 | (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) || |
| 3215 | reg->smin_value + off < 0)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 3216 | verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", |
Gianluca Borello | dbcfe5f | 2017-01-09 10:19:46 -0800 | [diff] [blame] | 3217 | regno); |
| 3218 | return -EACCES; |
| 3219 | } |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 3220 | err = __check_mem_access(env, regno, reg->smin_value + off, size, |
| 3221 | mem_size, zero_size_allowed); |
Gianluca Borello | dbcfe5f | 2017-01-09 10:19:46 -0800 | [diff] [blame] | 3222 | if (err) { |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 3223 | verbose(env, "R%d min value is outside of the allowed memory range\n", |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 3224 | regno); |
Gianluca Borello | dbcfe5f | 2017-01-09 10:19:46 -0800 | [diff] [blame] | 3225 | return err; |
| 3226 | } |
| 3227 | |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3228 | /* If we haven't set a max value then we need to bail since we can't be |
| 3229 | * sure we won't do bad things. |
| 3230 | * If reg->umax_value + off could overflow, treat that as unbounded too. |
Gianluca Borello | dbcfe5f | 2017-01-09 10:19:46 -0800 | [diff] [blame] | 3231 | */ |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3232 | if (reg->umax_value >= BPF_MAX_VAR_OFF) { |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 3233 | verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n", |
Gianluca Borello | dbcfe5f | 2017-01-09 10:19:46 -0800 | [diff] [blame] | 3234 | regno); |
| 3235 | return -EACCES; |
| 3236 | } |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 3237 | err = __check_mem_access(env, regno, reg->umax_value + off, size, |
| 3238 | mem_size, zero_size_allowed); |
| 3239 | if (err) { |
| 3240 | verbose(env, "R%d max value is outside of the allowed memory range\n", |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 3241 | regno); |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 3242 | return err; |
| 3243 | } |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 3244 | |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 3245 | return 0; |
| 3246 | } |
| 3247 | |
| 3248 | /* check read/write into a map element with possible variable offset */ |
| 3249 | static int check_map_access(struct bpf_verifier_env *env, u32 regno, |
| 3250 | int off, int size, bool zero_size_allowed) |
| 3251 | { |
| 3252 | struct bpf_verifier_state *vstate = env->cur_state; |
| 3253 | struct bpf_func_state *state = vstate->frame[vstate->curframe]; |
| 3254 | struct bpf_reg_state *reg = &state->regs[regno]; |
| 3255 | struct bpf_map *map = reg->map_ptr; |
| 3256 | int err; |
| 3257 | |
| 3258 | err = check_mem_region_access(env, regno, off, size, map->value_size, |
| 3259 | zero_size_allowed); |
| 3260 | if (err) |
| 3261 | return err; |
| 3262 | |
| 3263 | if (map_value_has_spin_lock(map)) { |
| 3264 | u32 lock = map->spin_lock_off; |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 3265 | |
| 3266 | /* if any part of struct bpf_spin_lock can be touched by |
| 3267 | * load/store reject this program. |
| 3268 | * To check that [x1, x2) overlaps with [y1, y2) |
| 3269 | * it is sufficient to check x1 < y2 && y1 < x2. |
| 3270 | */ |
| 3271 | if (reg->smin_value + off < lock + sizeof(struct bpf_spin_lock) && |
| 3272 | lock < reg->umax_value + off + size) { |
| 3273 | verbose(env, "bpf_spin_lock cannot be accessed directly by load/store\n"); |
| 3274 | return -EACCES; |
| 3275 | } |
| 3276 | } |
Alexei Starovoitov | 6813466 | 2021-07-14 17:54:10 -0700 | [diff] [blame] | 3277 | if (map_value_has_timer(map)) { |
| 3278 | u32 t = map->timer_off; |
| 3279 | |
| 3280 | if (reg->smin_value + off < t + sizeof(struct bpf_timer) && |
| 3281 | t < reg->umax_value + off + size) { |
| 3282 | verbose(env, "bpf_timer cannot be accessed directly by load/store\n"); |
| 3283 | return -EACCES; |
| 3284 | } |
| 3285 | } |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3286 | return err; |
Gianluca Borello | dbcfe5f | 2017-01-09 10:19:46 -0800 | [diff] [blame] | 3287 | } |
| 3288 | |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 3289 | #define MAX_PACKET_OFF 0xffff |
| 3290 | |
Udip Pant | 7e40781 | 2020-08-25 16:20:00 -0700 | [diff] [blame] | 3291 | static enum bpf_prog_type resolve_prog_type(struct bpf_prog *prog) |
| 3292 | { |
Toke Høiland-Jørgensen | 3aac1ea | 2020-09-29 14:45:50 +0200 | [diff] [blame] | 3293 | return prog->aux->dst_prog ? prog->aux->dst_prog->type : prog->type; |
Udip Pant | 7e40781 | 2020-08-25 16:20:00 -0700 | [diff] [blame] | 3294 | } |
| 3295 | |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 3296 | static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, |
Thomas Graf | 3a0af8f | 2016-11-30 17:10:10 +0100 | [diff] [blame] | 3297 | const struct bpf_call_arg_meta *meta, |
| 3298 | enum bpf_access_type t) |
Brenden Blanco | 4acf6c0 | 2016-07-19 12:16:56 -0700 | [diff] [blame] | 3299 | { |
Udip Pant | 7e40781 | 2020-08-25 16:20:00 -0700 | [diff] [blame] | 3300 | enum bpf_prog_type prog_type = resolve_prog_type(env->prog); |
| 3301 | |
| 3302 | switch (prog_type) { |
Daniel Borkmann | 5d66fa7 | 2018-10-24 22:05:45 +0200 | [diff] [blame] | 3303 | /* Program types only with direct read access go here! */ |
Thomas Graf | 3a0af8f | 2016-11-30 17:10:10 +0100 | [diff] [blame] | 3304 | case BPF_PROG_TYPE_LWT_IN: |
| 3305 | case BPF_PROG_TYPE_LWT_OUT: |
Mathieu Xhonneux | 004d4b2 | 2018-05-20 14:58:16 +0100 | [diff] [blame] | 3306 | case BPF_PROG_TYPE_LWT_SEG6LOCAL: |
Martin KaFai Lau | 2dbb9b9 | 2018-08-08 01:01:25 -0700 | [diff] [blame] | 3307 | case BPF_PROG_TYPE_SK_REUSEPORT: |
Daniel Borkmann | 5d66fa7 | 2018-10-24 22:05:45 +0200 | [diff] [blame] | 3308 | case BPF_PROG_TYPE_FLOW_DISSECTOR: |
Daniel Borkmann | d5563d3 | 2018-10-24 22:05:46 +0200 | [diff] [blame] | 3309 | case BPF_PROG_TYPE_CGROUP_SKB: |
Thomas Graf | 3a0af8f | 2016-11-30 17:10:10 +0100 | [diff] [blame] | 3310 | if (t == BPF_WRITE) |
| 3311 | return false; |
Gustavo A. R. Silva | 8731745 | 2020-10-02 18:42:17 -0500 | [diff] [blame] | 3312 | fallthrough; |
Daniel Borkmann | 5d66fa7 | 2018-10-24 22:05:45 +0200 | [diff] [blame] | 3313 | |
| 3314 | /* Program types with direct read + write access go here! */ |
Daniel Borkmann | 36bbef5 | 2016-09-20 00:26:13 +0200 | [diff] [blame] | 3315 | case BPF_PROG_TYPE_SCHED_CLS: |
| 3316 | case BPF_PROG_TYPE_SCHED_ACT: |
Brenden Blanco | 4acf6c0 | 2016-07-19 12:16:56 -0700 | [diff] [blame] | 3317 | case BPF_PROG_TYPE_XDP: |
Thomas Graf | 3a0af8f | 2016-11-30 17:10:10 +0100 | [diff] [blame] | 3318 | case BPF_PROG_TYPE_LWT_XMIT: |
John Fastabend | 8a31db5 | 2017-08-15 22:33:09 -0700 | [diff] [blame] | 3319 | case BPF_PROG_TYPE_SK_SKB: |
John Fastabend | 4f738ad | 2018-03-18 12:57:10 -0700 | [diff] [blame] | 3320 | case BPF_PROG_TYPE_SK_MSG: |
Daniel Borkmann | 36bbef5 | 2016-09-20 00:26:13 +0200 | [diff] [blame] | 3321 | if (meta) |
| 3322 | return meta->pkt_access; |
| 3323 | |
| 3324 | env->seen_direct_write = true; |
Brenden Blanco | 4acf6c0 | 2016-07-19 12:16:56 -0700 | [diff] [blame] | 3325 | return true; |
Stanislav Fomichev | 0d01da6 | 2019-06-27 13:38:47 -0700 | [diff] [blame] | 3326 | |
| 3327 | case BPF_PROG_TYPE_CGROUP_SOCKOPT: |
| 3328 | if (t == BPF_WRITE) |
| 3329 | env->seen_direct_write = true; |
| 3330 | |
| 3331 | return true; |
| 3332 | |
Brenden Blanco | 4acf6c0 | 2016-07-19 12:16:56 -0700 | [diff] [blame] | 3333 | default: |
| 3334 | return false; |
| 3335 | } |
| 3336 | } |
| 3337 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3338 | static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, |
Yonghong Song | 9fd29c0 | 2017-11-12 14:49:09 -0800 | [diff] [blame] | 3339 | int size, bool zero_size_allowed) |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3340 | { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 3341 | struct bpf_reg_state *regs = cur_regs(env); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3342 | struct bpf_reg_state *reg = ®s[regno]; |
| 3343 | int err; |
| 3344 | |
| 3345 | /* We may have added a variable offset to the packet pointer; but any |
| 3346 | * reg->range we have comes after that. We are only checking the fixed |
| 3347 | * offset. |
| 3348 | */ |
| 3349 | |
| 3350 | /* We don't allow negative numbers, because we aren't tracking enough |
| 3351 | * detail to prove they're safe. |
| 3352 | */ |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 3353 | if (reg->smin_value < 0) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 3354 | verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3355 | regno); |
| 3356 | return -EACCES; |
| 3357 | } |
Alexei Starovoitov | 6d94e74 | 2020-11-10 19:12:11 -0800 | [diff] [blame] | 3358 | |
| 3359 | err = reg->range < 0 ? -EINVAL : |
| 3360 | __check_mem_access(env, regno, off, size, reg->range, |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 3361 | zero_size_allowed); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3362 | if (err) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 3363 | verbose(env, "R%d offset is outside of the packet\n", regno); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3364 | return err; |
| 3365 | } |
Jiong Wang | e647815 | 2018-11-08 04:08:42 -0500 | [diff] [blame] | 3366 | |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 3367 | /* __check_mem_access has made sure "off + size - 1" is within u16. |
Jiong Wang | e647815 | 2018-11-08 04:08:42 -0500 | [diff] [blame] | 3368 | * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff, |
| 3369 | * otherwise find_good_pkt_pointers would have refused to set range info |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 3370 | * that __check_mem_access would have rejected this pkt access. |
Jiong Wang | e647815 | 2018-11-08 04:08:42 -0500 | [diff] [blame] | 3371 | * Therefore, "off + reg->umax_value + size - 1" won't overflow u32. |
| 3372 | */ |
| 3373 | env->prog->aux->max_pkt_offset = |
| 3374 | max_t(u32, env->prog->aux->max_pkt_offset, |
| 3375 | off + reg->umax_value + size - 1); |
| 3376 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3377 | return err; |
| 3378 | } |
| 3379 | |
| 3380 | /* check access to 'struct bpf_context' fields. Supports fixed offsets only */ |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 3381 | static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size, |
Alexei Starovoitov | 9e15db6 | 2019-10-15 20:25:00 -0700 | [diff] [blame] | 3382 | enum bpf_access_type t, enum bpf_reg_type *reg_type, |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 3383 | struct btf **btf, u32 *btf_id) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3384 | { |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 3385 | struct bpf_insn_access_aux info = { |
| 3386 | .reg_type = *reg_type, |
Alexei Starovoitov | 9e15db6 | 2019-10-15 20:25:00 -0700 | [diff] [blame] | 3387 | .log = &env->log, |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 3388 | }; |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 3389 | |
Jakub Kicinski | 4f9218a | 2017-10-16 16:40:55 -0700 | [diff] [blame] | 3390 | if (env->ops->is_valid_access && |
Andrey Ignatov | 5e43f89 | 2018-03-30 15:08:00 -0700 | [diff] [blame] | 3391 | env->ops->is_valid_access(off, size, t, env->prog, &info)) { |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 3392 | /* A non zero info.ctx_field_size indicates that this field is a |
| 3393 | * candidate for later verifier transformation to load the whole |
| 3394 | * field and then apply a mask when accessed with a narrower |
| 3395 | * access than actual ctx access size. A zero info.ctx_field_size |
| 3396 | * will only allow for whole field access and rejects any other |
| 3397 | * type of narrower access. |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 3398 | */ |
Yonghong Song | 2399463 | 2017-06-22 15:07:39 -0700 | [diff] [blame] | 3399 | *reg_type = info.reg_type; |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 3400 | |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 3401 | if (*reg_type == PTR_TO_BTF_ID || *reg_type == PTR_TO_BTF_ID_OR_NULL) { |
| 3402 | *btf = info.btf; |
Alexei Starovoitov | 9e15db6 | 2019-10-15 20:25:00 -0700 | [diff] [blame] | 3403 | *btf_id = info.btf_id; |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 3404 | } else { |
Alexei Starovoitov | 9e15db6 | 2019-10-15 20:25:00 -0700 | [diff] [blame] | 3405 | env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 3406 | } |
Alexei Starovoitov | 32bbe00 | 2016-04-06 18:43:28 -0700 | [diff] [blame] | 3407 | /* remember the offset of last byte accessed in ctx */ |
| 3408 | if (env->prog->aux->max_ctx_offset < off + size) |
| 3409 | env->prog->aux->max_ctx_offset = off + size; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3410 | return 0; |
Alexei Starovoitov | 32bbe00 | 2016-04-06 18:43:28 -0700 | [diff] [blame] | 3411 | } |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3412 | |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 3413 | verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 3414 | return -EACCES; |
| 3415 | } |
| 3416 | |
Petar Penkov | d58e468 | 2018-09-14 07:46:18 -0700 | [diff] [blame] | 3417 | static int check_flow_keys_access(struct bpf_verifier_env *env, int off, |
| 3418 | int size) |
| 3419 | { |
| 3420 | if (size < 0 || off < 0 || |
| 3421 | (u64)off + size > sizeof(struct bpf_flow_keys)) { |
| 3422 | verbose(env, "invalid access to flow keys off=%d size=%d\n", |
| 3423 | off, size); |
| 3424 | return -EACCES; |
| 3425 | } |
| 3426 | return 0; |
| 3427 | } |
| 3428 | |
Martin KaFai Lau | 5f45664 | 2019-02-08 22:25:54 -0800 | [diff] [blame] | 3429 | static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, |
| 3430 | u32 regno, int off, int size, |
| 3431 | enum bpf_access_type t) |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 3432 | { |
| 3433 | struct bpf_reg_state *regs = cur_regs(env); |
| 3434 | struct bpf_reg_state *reg = ®s[regno]; |
Martin KaFai Lau | 5f45664 | 2019-02-08 22:25:54 -0800 | [diff] [blame] | 3435 | struct bpf_insn_access_aux info = {}; |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 3436 | bool valid; |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 3437 | |
| 3438 | if (reg->smin_value < 0) { |
| 3439 | verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", |
| 3440 | regno); |
| 3441 | return -EACCES; |
| 3442 | } |
| 3443 | |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 3444 | switch (reg->type) { |
| 3445 | case PTR_TO_SOCK_COMMON: |
| 3446 | valid = bpf_sock_common_is_valid_access(off, size, t, &info); |
| 3447 | break; |
| 3448 | case PTR_TO_SOCKET: |
| 3449 | valid = bpf_sock_is_valid_access(off, size, t, &info); |
| 3450 | break; |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 3451 | case PTR_TO_TCP_SOCK: |
| 3452 | valid = bpf_tcp_sock_is_valid_access(off, size, t, &info); |
| 3453 | break; |
Jonathan Lemon | fada7fd | 2019-06-06 13:59:40 -0700 | [diff] [blame] | 3454 | case PTR_TO_XDP_SOCK: |
| 3455 | valid = bpf_xdp_sock_is_valid_access(off, size, t, &info); |
| 3456 | break; |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 3457 | default: |
| 3458 | valid = false; |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 3459 | } |
| 3460 | |
Martin KaFai Lau | 5f45664 | 2019-02-08 22:25:54 -0800 | [diff] [blame] | 3461 | |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 3462 | if (valid) { |
| 3463 | env->insn_aux_data[insn_idx].ctx_field_size = |
| 3464 | info.ctx_field_size; |
| 3465 | return 0; |
| 3466 | } |
| 3467 | |
| 3468 | verbose(env, "R%d invalid %s access off=%d size=%d\n", |
| 3469 | regno, reg_type_str[reg->type], off, size); |
| 3470 | |
| 3471 | return -EACCES; |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 3472 | } |
| 3473 | |
Daniel Borkmann | 4cabc5b | 2017-07-21 00:00:21 +0200 | [diff] [blame] | 3474 | static bool is_pointer_value(struct bpf_verifier_env *env, int regno) |
| 3475 | { |
Daniel Borkmann | 2a159c6 | 2018-10-21 02:09:24 +0200 | [diff] [blame] | 3476 | return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno)); |
Daniel Borkmann | 4cabc5b | 2017-07-21 00:00:21 +0200 | [diff] [blame] | 3477 | } |
| 3478 | |
Daniel Borkmann | f37a8cb | 2018-01-16 23:30:10 +0100 | [diff] [blame] | 3479 | static bool is_ctx_reg(struct bpf_verifier_env *env, int regno) |
| 3480 | { |
Daniel Borkmann | 2a159c6 | 2018-10-21 02:09:24 +0200 | [diff] [blame] | 3481 | const struct bpf_reg_state *reg = reg_state(env, regno); |
Daniel Borkmann | f37a8cb | 2018-01-16 23:30:10 +0100 | [diff] [blame] | 3482 | |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 3483 | return reg->type == PTR_TO_CTX; |
| 3484 | } |
| 3485 | |
| 3486 | static bool is_sk_reg(struct bpf_verifier_env *env, int regno) |
| 3487 | { |
| 3488 | const struct bpf_reg_state *reg = reg_state(env, regno); |
| 3489 | |
| 3490 | return type_is_sk_pointer(reg->type); |
Daniel Borkmann | f37a8cb | 2018-01-16 23:30:10 +0100 | [diff] [blame] | 3491 | } |
| 3492 | |
Daniel Borkmann | ca36960 | 2018-02-23 22:29:05 +0100 | [diff] [blame] | 3493 | static bool is_pkt_reg(struct bpf_verifier_env *env, int regno) |
| 3494 | { |
Daniel Borkmann | 2a159c6 | 2018-10-21 02:09:24 +0200 | [diff] [blame] | 3495 | const struct bpf_reg_state *reg = reg_state(env, regno); |
Daniel Borkmann | ca36960 | 2018-02-23 22:29:05 +0100 | [diff] [blame] | 3496 | |
| 3497 | return type_is_pkt_pointer(reg->type); |
| 3498 | } |
| 3499 | |
Daniel Borkmann | 4b5defd | 2018-10-21 02:09:25 +0200 | [diff] [blame] | 3500 | static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno) |
| 3501 | { |
| 3502 | const struct bpf_reg_state *reg = reg_state(env, regno); |
| 3503 | |
| 3504 | /* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */ |
| 3505 | return reg->type == PTR_TO_FLOW_KEYS; |
| 3506 | } |
| 3507 | |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 3508 | static int check_pkt_ptr_alignment(struct bpf_verifier_env *env, |
| 3509 | const struct bpf_reg_state *reg, |
David S. Miller | d117441 | 2017-05-10 11:22:52 -0700 | [diff] [blame] | 3510 | int off, int size, bool strict) |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 3511 | { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3512 | struct tnum reg_off; |
David S. Miller | e07b98d | 2017-05-10 11:38:07 -0700 | [diff] [blame] | 3513 | int ip_align; |
David S. Miller | d117441 | 2017-05-10 11:22:52 -0700 | [diff] [blame] | 3514 | |
| 3515 | /* Byte size accesses are always allowed. */ |
| 3516 | if (!strict || size == 1) |
| 3517 | return 0; |
| 3518 | |
David S. Miller | e4eda88 | 2017-05-22 12:27:07 -0400 | [diff] [blame] | 3519 | /* For platforms that do not have a Kconfig enabling |
| 3520 | * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of |
| 3521 | * NET_IP_ALIGN is universally set to '2'. And on platforms |
| 3522 | * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get |
| 3523 | * to this code only in strict mode where we want to emulate |
| 3524 | * the NET_IP_ALIGN==2 checking. Therefore use an |
| 3525 | * unconditional IP align value of '2'. |
David S. Miller | e07b98d | 2017-05-10 11:38:07 -0700 | [diff] [blame] | 3526 | */ |
David S. Miller | e4eda88 | 2017-05-22 12:27:07 -0400 | [diff] [blame] | 3527 | ip_align = 2; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3528 | |
| 3529 | reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off)); |
| 3530 | if (!tnum_is_aligned(reg_off, size)) { |
| 3531 | char tn_buf[48]; |
| 3532 | |
| 3533 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 3534 | verbose(env, |
| 3535 | "misaligned packet access off %d+%s+%d+%d size %d\n", |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3536 | ip_align, tn_buf, reg->off, off, size); |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 3537 | return -EACCES; |
| 3538 | } |
Daniel Borkmann | 79adffc | 2017-03-31 02:24:03 +0200 | [diff] [blame] | 3539 | |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 3540 | return 0; |
| 3541 | } |
| 3542 | |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 3543 | static int check_generic_ptr_alignment(struct bpf_verifier_env *env, |
| 3544 | const struct bpf_reg_state *reg, |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3545 | const char *pointer_desc, |
| 3546 | int off, int size, bool strict) |
Daniel Borkmann | 79adffc | 2017-03-31 02:24:03 +0200 | [diff] [blame] | 3547 | { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3548 | struct tnum reg_off; |
| 3549 | |
| 3550 | /* Byte size accesses are always allowed. */ |
| 3551 | if (!strict || size == 1) |
| 3552 | return 0; |
| 3553 | |
| 3554 | reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off)); |
| 3555 | if (!tnum_is_aligned(reg_off, size)) { |
| 3556 | char tn_buf[48]; |
| 3557 | |
| 3558 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 3559 | verbose(env, "misaligned %saccess off %s+%d+%d size %d\n", |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3560 | pointer_desc, tn_buf, reg->off, off, size); |
Daniel Borkmann | 79adffc | 2017-03-31 02:24:03 +0200 | [diff] [blame] | 3561 | return -EACCES; |
| 3562 | } |
| 3563 | |
| 3564 | return 0; |
| 3565 | } |
| 3566 | |
David S. Miller | e07b98d | 2017-05-10 11:38:07 -0700 | [diff] [blame] | 3567 | static int check_ptr_alignment(struct bpf_verifier_env *env, |
Daniel Borkmann | ca36960 | 2018-02-23 22:29:05 +0100 | [diff] [blame] | 3568 | const struct bpf_reg_state *reg, int off, |
| 3569 | int size, bool strict_alignment_once) |
Daniel Borkmann | 79adffc | 2017-03-31 02:24:03 +0200 | [diff] [blame] | 3570 | { |
Daniel Borkmann | ca36960 | 2018-02-23 22:29:05 +0100 | [diff] [blame] | 3571 | bool strict = env->strict_alignment || strict_alignment_once; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3572 | const char *pointer_desc = ""; |
David S. Miller | d117441 | 2017-05-10 11:22:52 -0700 | [diff] [blame] | 3573 | |
Daniel Borkmann | 79adffc | 2017-03-31 02:24:03 +0200 | [diff] [blame] | 3574 | switch (reg->type) { |
| 3575 | case PTR_TO_PACKET: |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 3576 | case PTR_TO_PACKET_META: |
| 3577 | /* Special case, because of NET_IP_ALIGN. Given metadata sits |
| 3578 | * right in front, treat it the very same way. |
| 3579 | */ |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 3580 | return check_pkt_ptr_alignment(env, reg, off, size, strict); |
Petar Penkov | d58e468 | 2018-09-14 07:46:18 -0700 | [diff] [blame] | 3581 | case PTR_TO_FLOW_KEYS: |
| 3582 | pointer_desc = "flow keys "; |
| 3583 | break; |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 3584 | case PTR_TO_MAP_KEY: |
| 3585 | pointer_desc = "key "; |
| 3586 | break; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3587 | case PTR_TO_MAP_VALUE: |
| 3588 | pointer_desc = "value "; |
| 3589 | break; |
| 3590 | case PTR_TO_CTX: |
| 3591 | pointer_desc = "context "; |
| 3592 | break; |
| 3593 | case PTR_TO_STACK: |
| 3594 | pointer_desc = "stack "; |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 3595 | /* The stack spill tracking logic in check_stack_write_fixed_off() |
| 3596 | * and check_stack_read_fixed_off() relies on stack accesses being |
Jann Horn | a5ec6ae | 2017-12-18 20:11:58 -0800 | [diff] [blame] | 3597 | * aligned. |
| 3598 | */ |
| 3599 | strict = true; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3600 | break; |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 3601 | case PTR_TO_SOCKET: |
| 3602 | pointer_desc = "sock "; |
| 3603 | break; |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 3604 | case PTR_TO_SOCK_COMMON: |
| 3605 | pointer_desc = "sock_common "; |
| 3606 | break; |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 3607 | case PTR_TO_TCP_SOCK: |
| 3608 | pointer_desc = "tcp_sock "; |
| 3609 | break; |
Jonathan Lemon | fada7fd | 2019-06-06 13:59:40 -0700 | [diff] [blame] | 3610 | case PTR_TO_XDP_SOCK: |
| 3611 | pointer_desc = "xdp_sock "; |
| 3612 | break; |
Daniel Borkmann | 79adffc | 2017-03-31 02:24:03 +0200 | [diff] [blame] | 3613 | default: |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 3614 | break; |
Daniel Borkmann | 79adffc | 2017-03-31 02:24:03 +0200 | [diff] [blame] | 3615 | } |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 3616 | return check_generic_ptr_alignment(env, reg, pointer_desc, off, size, |
| 3617 | strict); |
Daniel Borkmann | 79adffc | 2017-03-31 02:24:03 +0200 | [diff] [blame] | 3618 | } |
| 3619 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 3620 | static int update_stack_depth(struct bpf_verifier_env *env, |
| 3621 | const struct bpf_func_state *func, |
| 3622 | int off) |
| 3623 | { |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 3624 | u16 stack = env->subprog_info[func->subprogno].stack_depth; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 3625 | |
| 3626 | if (stack >= -off) |
| 3627 | return 0; |
| 3628 | |
| 3629 | /* update known max for given subprogram */ |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 3630 | env->subprog_info[func->subprogno].stack_depth = -off; |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 3631 | return 0; |
| 3632 | } |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 3633 | |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 3634 | /* starting from main bpf function walk all instructions of the function |
| 3635 | * and recursively walk all callees that given function can call. |
| 3636 | * Ignore jump and exit insns. |
| 3637 | * Since recursion is prevented by check_cfg() this algorithm |
| 3638 | * only needs a local stack of MAX_CALL_FRAMES to remember callsites |
| 3639 | */ |
| 3640 | static int check_max_stack_depth(struct bpf_verifier_env *env) |
| 3641 | { |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 3642 | int depth = 0, frame = 0, idx = 0, i = 0, subprog_end; |
| 3643 | struct bpf_subprog_info *subprog = env->subprog_info; |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 3644 | struct bpf_insn *insn = env->prog->insnsi; |
Maciej Fijalkowski | ebf7d1f | 2020-09-16 23:10:08 +0200 | [diff] [blame] | 3645 | bool tail_call_reachable = false; |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 3646 | int ret_insn[MAX_CALL_FRAMES]; |
| 3647 | int ret_prog[MAX_CALL_FRAMES]; |
Maciej Fijalkowski | ebf7d1f | 2020-09-16 23:10:08 +0200 | [diff] [blame] | 3648 | int j; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 3649 | |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 3650 | process_func: |
Maciej Fijalkowski | 7f6e431 | 2020-09-16 23:10:07 +0200 | [diff] [blame] | 3651 | /* protect against potential stack overflow that might happen when |
| 3652 | * bpf2bpf calls get combined with tailcalls. Limit the caller's stack |
| 3653 | * depth for such case down to 256 so that the worst case scenario |
| 3654 | * would result in 8k stack size (32 which is tailcall limit * 256 = |
| 3655 | * 8k). |
| 3656 | * |
| 3657 | * To get the idea what might happen, see an example: |
| 3658 | * func1 -> sub rsp, 128 |
| 3659 | * subfunc1 -> sub rsp, 256 |
| 3660 | * tailcall1 -> add rsp, 256 |
| 3661 | * func2 -> sub rsp, 192 (total stack size = 128 + 192 = 320) |
| 3662 | * subfunc2 -> sub rsp, 64 |
| 3663 | * subfunc22 -> sub rsp, 128 |
| 3664 | * tailcall2 -> add rsp, 128 |
| 3665 | * func3 -> sub rsp, 32 (total stack size 128 + 192 + 64 + 32 = 416) |
| 3666 | * |
| 3667 | * tailcall will unwind the current stack frame but it will not get rid |
| 3668 | * of caller's stack as shown on the example above. |
| 3669 | */ |
| 3670 | if (idx && subprog[idx].has_tail_call && depth >= 256) { |
| 3671 | verbose(env, |
| 3672 | "tail_calls are not allowed when call stack of previous frames is %d bytes. Too large\n", |
| 3673 | depth); |
| 3674 | return -EACCES; |
| 3675 | } |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 3676 | /* round up to 32-bytes, since this is granularity |
| 3677 | * of interpreter stack size |
| 3678 | */ |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 3679 | depth += round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 3680 | if (depth > MAX_BPF_STACK) { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 3681 | verbose(env, "combined stack size of %d calls is %d. Too large\n", |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 3682 | frame + 1, depth); |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 3683 | return -EACCES; |
| 3684 | } |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 3685 | continue_func: |
Jiong Wang | 4cb3d99 | 2018-05-02 16:17:19 -0400 | [diff] [blame] | 3686 | subprog_end = subprog[idx + 1].start; |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 3687 | for (; i < subprog_end; i++) { |
Alexei Starovoitov | 7ddc80a | 2021-07-14 17:54:15 -0700 | [diff] [blame] | 3688 | int next_insn; |
| 3689 | |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 3690 | if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i)) |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 3691 | continue; |
| 3692 | /* remember insn and function to return to */ |
| 3693 | ret_insn[frame] = i + 1; |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 3694 | ret_prog[frame] = idx; |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 3695 | |
| 3696 | /* find the callee */ |
Alexei Starovoitov | 7ddc80a | 2021-07-14 17:54:15 -0700 | [diff] [blame] | 3697 | next_insn = i + insn[i].imm + 1; |
| 3698 | idx = find_subprog(env, next_insn); |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 3699 | if (idx < 0) { |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 3700 | WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", |
Alexei Starovoitov | 7ddc80a | 2021-07-14 17:54:15 -0700 | [diff] [blame] | 3701 | next_insn); |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 3702 | return -EFAULT; |
| 3703 | } |
Alexei Starovoitov | 7ddc80a | 2021-07-14 17:54:15 -0700 | [diff] [blame] | 3704 | if (subprog[idx].is_async_cb) { |
| 3705 | if (subprog[idx].has_tail_call) { |
| 3706 | verbose(env, "verifier bug. subprog has tail_call and async cb\n"); |
| 3707 | return -EFAULT; |
| 3708 | } |
| 3709 | /* async callbacks don't increase bpf prog stack size */ |
| 3710 | continue; |
| 3711 | } |
| 3712 | i = next_insn; |
Maciej Fijalkowski | ebf7d1f | 2020-09-16 23:10:08 +0200 | [diff] [blame] | 3713 | |
| 3714 | if (subprog[idx].has_tail_call) |
| 3715 | tail_call_reachable = true; |
| 3716 | |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 3717 | frame++; |
| 3718 | if (frame >= MAX_CALL_FRAMES) { |
Paul Chaignon | 927cb78 | 2019-03-20 13:58:27 +0100 | [diff] [blame] | 3719 | verbose(env, "the call stack of %d frames is too deep !\n", |
| 3720 | frame); |
| 3721 | return -E2BIG; |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 3722 | } |
| 3723 | goto process_func; |
| 3724 | } |
Maciej Fijalkowski | ebf7d1f | 2020-09-16 23:10:08 +0200 | [diff] [blame] | 3725 | /* if tail call got detected across bpf2bpf calls then mark each of the |
| 3726 | * currently present subprog frames as tail call reachable subprogs; |
| 3727 | * this info will be utilized by JIT so that we will be preserving the |
| 3728 | * tail call counter throughout bpf2bpf calls combined with tailcalls |
| 3729 | */ |
| 3730 | if (tail_call_reachable) |
| 3731 | for (j = 0; j < frame; j++) |
| 3732 | subprog[ret_prog[j]].tail_call_reachable = true; |
Daniel Borkmann | 5dd0a6b | 2021-07-12 22:57:35 +0200 | [diff] [blame] | 3733 | if (subprog[0].tail_call_reachable) |
| 3734 | env->prog->aux->tail_call_reachable = true; |
Maciej Fijalkowski | ebf7d1f | 2020-09-16 23:10:08 +0200 | [diff] [blame] | 3735 | |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 3736 | /* end of for() loop means the last insn of the 'subprog' |
| 3737 | * was reached. Doesn't matter whether it was JA or EXIT |
| 3738 | */ |
| 3739 | if (frame == 0) |
| 3740 | return 0; |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 3741 | depth -= round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 3742 | frame--; |
| 3743 | i = ret_insn[frame]; |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 3744 | idx = ret_prog[frame]; |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 3745 | goto continue_func; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 3746 | } |
| 3747 | |
David S. Miller | 19d28fb | 2018-01-11 21:27:54 -0500 | [diff] [blame] | 3748 | #ifndef CONFIG_BPF_JIT_ALWAYS_ON |
Alexei Starovoitov | 1ea47e0 | 2017-12-14 17:55:13 -0800 | [diff] [blame] | 3749 | static int get_callee_stack_depth(struct bpf_verifier_env *env, |
| 3750 | const struct bpf_insn *insn, int idx) |
| 3751 | { |
| 3752 | int start = idx + insn->imm + 1, subprog; |
| 3753 | |
| 3754 | subprog = find_subprog(env, start); |
| 3755 | if (subprog < 0) { |
| 3756 | WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", |
| 3757 | start); |
| 3758 | return -EFAULT; |
| 3759 | } |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 3760 | return env->subprog_info[subprog].stack_depth; |
Alexei Starovoitov | 1ea47e0 | 2017-12-14 17:55:13 -0800 | [diff] [blame] | 3761 | } |
David S. Miller | 19d28fb | 2018-01-11 21:27:54 -0500 | [diff] [blame] | 3762 | #endif |
Alexei Starovoitov | 1ea47e0 | 2017-12-14 17:55:13 -0800 | [diff] [blame] | 3763 | |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 3764 | int check_ctx_reg(struct bpf_verifier_env *env, |
| 3765 | const struct bpf_reg_state *reg, int regno) |
Daniel Borkmann | 58990d1 | 2018-06-07 17:40:03 +0200 | [diff] [blame] | 3766 | { |
| 3767 | /* Access to ctx or passing it to a helper is only allowed in |
| 3768 | * its original, unmodified form. |
| 3769 | */ |
| 3770 | |
| 3771 | if (reg->off) { |
| 3772 | verbose(env, "dereference of modified ctx ptr R%d off=%d disallowed\n", |
| 3773 | regno, reg->off); |
| 3774 | return -EACCES; |
| 3775 | } |
| 3776 | |
| 3777 | if (!tnum_is_const(reg->var_off) || reg->var_off.value) { |
| 3778 | char tn_buf[48]; |
| 3779 | |
| 3780 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
| 3781 | verbose(env, "variable ctx access var_off=%s disallowed\n", tn_buf); |
| 3782 | return -EACCES; |
| 3783 | } |
| 3784 | |
| 3785 | return 0; |
| 3786 | } |
| 3787 | |
Yonghong Song | afbf21d | 2020-07-23 11:41:11 -0700 | [diff] [blame] | 3788 | static int __check_buffer_access(struct bpf_verifier_env *env, |
| 3789 | const char *buf_info, |
| 3790 | const struct bpf_reg_state *reg, |
| 3791 | int regno, int off, int size) |
Matt Mullins | 9df1c28 | 2019-04-26 11:49:47 -0700 | [diff] [blame] | 3792 | { |
| 3793 | if (off < 0) { |
| 3794 | verbose(env, |
Yonghong Song | 4fc00b7 | 2020-07-28 15:18:01 -0700 | [diff] [blame] | 3795 | "R%d invalid %s buffer access: off=%d, size=%d\n", |
Yonghong Song | afbf21d | 2020-07-23 11:41:11 -0700 | [diff] [blame] | 3796 | regno, buf_info, off, size); |
Matt Mullins | 9df1c28 | 2019-04-26 11:49:47 -0700 | [diff] [blame] | 3797 | return -EACCES; |
| 3798 | } |
| 3799 | if (!tnum_is_const(reg->var_off) || reg->var_off.value) { |
| 3800 | char tn_buf[48]; |
| 3801 | |
| 3802 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
| 3803 | verbose(env, |
Yonghong Song | 4fc00b7 | 2020-07-28 15:18:01 -0700 | [diff] [blame] | 3804 | "R%d invalid variable buffer offset: off=%d, var_off=%s\n", |
Matt Mullins | 9df1c28 | 2019-04-26 11:49:47 -0700 | [diff] [blame] | 3805 | regno, off, tn_buf); |
| 3806 | return -EACCES; |
| 3807 | } |
Yonghong Song | afbf21d | 2020-07-23 11:41:11 -0700 | [diff] [blame] | 3808 | |
| 3809 | return 0; |
| 3810 | } |
| 3811 | |
| 3812 | static int check_tp_buffer_access(struct bpf_verifier_env *env, |
| 3813 | const struct bpf_reg_state *reg, |
| 3814 | int regno, int off, int size) |
| 3815 | { |
| 3816 | int err; |
| 3817 | |
| 3818 | err = __check_buffer_access(env, "tracepoint", reg, regno, off, size); |
| 3819 | if (err) |
| 3820 | return err; |
| 3821 | |
Matt Mullins | 9df1c28 | 2019-04-26 11:49:47 -0700 | [diff] [blame] | 3822 | if (off + size > env->prog->aux->max_tp_access) |
| 3823 | env->prog->aux->max_tp_access = off + size; |
| 3824 | |
| 3825 | return 0; |
| 3826 | } |
| 3827 | |
Yonghong Song | afbf21d | 2020-07-23 11:41:11 -0700 | [diff] [blame] | 3828 | static int check_buffer_access(struct bpf_verifier_env *env, |
| 3829 | const struct bpf_reg_state *reg, |
| 3830 | int regno, int off, int size, |
| 3831 | bool zero_size_allowed, |
| 3832 | const char *buf_info, |
| 3833 | u32 *max_access) |
| 3834 | { |
| 3835 | int err; |
| 3836 | |
| 3837 | err = __check_buffer_access(env, buf_info, reg, regno, off, size); |
| 3838 | if (err) |
| 3839 | return err; |
| 3840 | |
| 3841 | if (off + size > *max_access) |
| 3842 | *max_access = off + size; |
| 3843 | |
| 3844 | return 0; |
| 3845 | } |
| 3846 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 3847 | /* BPF architecture zero extends alu32 ops into 64-bit registesr */ |
| 3848 | static void zext_32_to_64(struct bpf_reg_state *reg) |
| 3849 | { |
| 3850 | reg->var_off = tnum_subreg(reg->var_off); |
| 3851 | __reg_assign_32_into_64(reg); |
| 3852 | } |
Matt Mullins | 9df1c28 | 2019-04-26 11:49:47 -0700 | [diff] [blame] | 3853 | |
Jann Horn | 0c17d1d | 2017-12-18 20:11:55 -0800 | [diff] [blame] | 3854 | /* truncate register to smaller size (in bytes) |
| 3855 | * must be called with size < BPF_REG_SIZE |
| 3856 | */ |
| 3857 | static void coerce_reg_to_size(struct bpf_reg_state *reg, int size) |
| 3858 | { |
| 3859 | u64 mask; |
| 3860 | |
| 3861 | /* clear high bits in bit representation */ |
| 3862 | reg->var_off = tnum_cast(reg->var_off, size); |
| 3863 | |
| 3864 | /* fix arithmetic bounds */ |
| 3865 | mask = ((u64)1 << (size * 8)) - 1; |
| 3866 | if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) { |
| 3867 | reg->umin_value &= mask; |
| 3868 | reg->umax_value &= mask; |
| 3869 | } else { |
| 3870 | reg->umin_value = 0; |
| 3871 | reg->umax_value = mask; |
| 3872 | } |
| 3873 | reg->smin_value = reg->umin_value; |
| 3874 | reg->smax_value = reg->umax_value; |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 3875 | |
| 3876 | /* If size is smaller than 32bit register the 32bit register |
| 3877 | * values are also truncated so we push 64-bit bounds into |
| 3878 | * 32-bit bounds. Above were truncated < 32-bits already. |
| 3879 | */ |
| 3880 | if (size >= 4) |
| 3881 | return; |
| 3882 | __reg_combine_64_into_32(reg); |
Jann Horn | 0c17d1d | 2017-12-18 20:11:55 -0800 | [diff] [blame] | 3883 | } |
| 3884 | |
Andrii Nakryiko | a23740e | 2019-10-09 13:14:57 -0700 | [diff] [blame] | 3885 | static bool bpf_map_is_rdonly(const struct bpf_map *map) |
| 3886 | { |
| 3887 | return (map->map_flags & BPF_F_RDONLY_PROG) && map->frozen; |
| 3888 | } |
| 3889 | |
| 3890 | static int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val) |
| 3891 | { |
| 3892 | void *ptr; |
| 3893 | u64 addr; |
| 3894 | int err; |
| 3895 | |
| 3896 | err = map->ops->map_direct_value_addr(map, &addr, off); |
| 3897 | if (err) |
| 3898 | return err; |
Andrii Nakryiko | 2dedd7d | 2019-10-11 10:20:53 -0700 | [diff] [blame] | 3899 | ptr = (void *)(long)addr + off; |
Andrii Nakryiko | a23740e | 2019-10-09 13:14:57 -0700 | [diff] [blame] | 3900 | |
| 3901 | switch (size) { |
| 3902 | case sizeof(u8): |
| 3903 | *val = (u64)*(u8 *)ptr; |
| 3904 | break; |
| 3905 | case sizeof(u16): |
| 3906 | *val = (u64)*(u16 *)ptr; |
| 3907 | break; |
| 3908 | case sizeof(u32): |
| 3909 | *val = (u64)*(u32 *)ptr; |
| 3910 | break; |
| 3911 | case sizeof(u64): |
| 3912 | *val = *(u64 *)ptr; |
| 3913 | break; |
| 3914 | default: |
| 3915 | return -EINVAL; |
| 3916 | } |
| 3917 | return 0; |
| 3918 | } |
| 3919 | |
Alexei Starovoitov | 9e15db6 | 2019-10-15 20:25:00 -0700 | [diff] [blame] | 3920 | static int check_ptr_to_btf_access(struct bpf_verifier_env *env, |
| 3921 | struct bpf_reg_state *regs, |
| 3922 | int regno, int off, int size, |
| 3923 | enum bpf_access_type atype, |
| 3924 | int value_regno) |
| 3925 | { |
| 3926 | struct bpf_reg_state *reg = regs + regno; |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 3927 | const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id); |
| 3928 | const char *tname = btf_name_by_offset(reg->btf, t->name_off); |
Alexei Starovoitov | 9e15db6 | 2019-10-15 20:25:00 -0700 | [diff] [blame] | 3929 | u32 btf_id; |
| 3930 | int ret; |
| 3931 | |
Alexei Starovoitov | 9e15db6 | 2019-10-15 20:25:00 -0700 | [diff] [blame] | 3932 | if (off < 0) { |
| 3933 | verbose(env, |
| 3934 | "R%d is ptr_%s invalid negative access: off=%d\n", |
| 3935 | regno, tname, off); |
| 3936 | return -EACCES; |
| 3937 | } |
| 3938 | if (!tnum_is_const(reg->var_off) || reg->var_off.value) { |
| 3939 | char tn_buf[48]; |
| 3940 | |
| 3941 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
| 3942 | verbose(env, |
| 3943 | "R%d is ptr_%s invalid variable offset: off=%d, var_off=%s\n", |
| 3944 | regno, tname, off, tn_buf); |
| 3945 | return -EACCES; |
| 3946 | } |
| 3947 | |
Martin KaFai Lau | 27ae7997 | 2020-01-08 16:35:03 -0800 | [diff] [blame] | 3948 | if (env->ops->btf_struct_access) { |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 3949 | ret = env->ops->btf_struct_access(&env->log, reg->btf, t, |
| 3950 | off, size, atype, &btf_id); |
Martin KaFai Lau | 27ae7997 | 2020-01-08 16:35:03 -0800 | [diff] [blame] | 3951 | } else { |
| 3952 | if (atype != BPF_READ) { |
| 3953 | verbose(env, "only read is supported\n"); |
| 3954 | return -EACCES; |
| 3955 | } |
| 3956 | |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 3957 | ret = btf_struct_access(&env->log, reg->btf, t, off, size, |
| 3958 | atype, &btf_id); |
Martin KaFai Lau | 27ae7997 | 2020-01-08 16:35:03 -0800 | [diff] [blame] | 3959 | } |
| 3960 | |
Alexei Starovoitov | 9e15db6 | 2019-10-15 20:25:00 -0700 | [diff] [blame] | 3961 | if (ret < 0) |
| 3962 | return ret; |
| 3963 | |
Andrey Ignatov | 41c48f3 | 2020-06-19 14:11:43 -0700 | [diff] [blame] | 3964 | if (atype == BPF_READ && value_regno >= 0) |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 3965 | mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id); |
Martin KaFai Lau | 27ae7997 | 2020-01-08 16:35:03 -0800 | [diff] [blame] | 3966 | |
Alexei Starovoitov | 9e15db6 | 2019-10-15 20:25:00 -0700 | [diff] [blame] | 3967 | return 0; |
| 3968 | } |
| 3969 | |
Andrey Ignatov | 41c48f3 | 2020-06-19 14:11:43 -0700 | [diff] [blame] | 3970 | static int check_ptr_to_map_access(struct bpf_verifier_env *env, |
| 3971 | struct bpf_reg_state *regs, |
| 3972 | int regno, int off, int size, |
| 3973 | enum bpf_access_type atype, |
| 3974 | int value_regno) |
| 3975 | { |
| 3976 | struct bpf_reg_state *reg = regs + regno; |
| 3977 | struct bpf_map *map = reg->map_ptr; |
| 3978 | const struct btf_type *t; |
| 3979 | const char *tname; |
| 3980 | u32 btf_id; |
| 3981 | int ret; |
| 3982 | |
| 3983 | if (!btf_vmlinux) { |
| 3984 | verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n"); |
| 3985 | return -ENOTSUPP; |
| 3986 | } |
| 3987 | |
| 3988 | if (!map->ops->map_btf_id || !*map->ops->map_btf_id) { |
| 3989 | verbose(env, "map_ptr access not supported for map type %d\n", |
| 3990 | map->map_type); |
| 3991 | return -ENOTSUPP; |
| 3992 | } |
| 3993 | |
| 3994 | t = btf_type_by_id(btf_vmlinux, *map->ops->map_btf_id); |
| 3995 | tname = btf_name_by_offset(btf_vmlinux, t->name_off); |
| 3996 | |
| 3997 | if (!env->allow_ptr_to_map_access) { |
| 3998 | verbose(env, |
| 3999 | "%s access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", |
| 4000 | tname); |
| 4001 | return -EPERM; |
| 4002 | } |
| 4003 | |
| 4004 | if (off < 0) { |
| 4005 | verbose(env, "R%d is %s invalid negative access: off=%d\n", |
| 4006 | regno, tname, off); |
| 4007 | return -EACCES; |
| 4008 | } |
| 4009 | |
| 4010 | if (atype != BPF_READ) { |
| 4011 | verbose(env, "only read from %s is supported\n", tname); |
| 4012 | return -EACCES; |
| 4013 | } |
| 4014 | |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 4015 | ret = btf_struct_access(&env->log, btf_vmlinux, t, off, size, atype, &btf_id); |
Andrey Ignatov | 41c48f3 | 2020-06-19 14:11:43 -0700 | [diff] [blame] | 4016 | if (ret < 0) |
| 4017 | return ret; |
| 4018 | |
| 4019 | if (value_regno >= 0) |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 4020 | mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id); |
Andrey Ignatov | 41c48f3 | 2020-06-19 14:11:43 -0700 | [diff] [blame] | 4021 | |
| 4022 | return 0; |
| 4023 | } |
| 4024 | |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 4025 | /* Check that the stack access at the given offset is within bounds. The |
| 4026 | * maximum valid offset is -1. |
| 4027 | * |
| 4028 | * The minimum valid offset is -MAX_BPF_STACK for writes, and |
| 4029 | * -state->allocated_stack for reads. |
| 4030 | */ |
| 4031 | static int check_stack_slot_within_bounds(int off, |
| 4032 | struct bpf_func_state *state, |
| 4033 | enum bpf_access_type t) |
| 4034 | { |
| 4035 | int min_valid_off; |
| 4036 | |
| 4037 | if (t == BPF_WRITE) |
| 4038 | min_valid_off = -MAX_BPF_STACK; |
| 4039 | else |
| 4040 | min_valid_off = -state->allocated_stack; |
| 4041 | |
| 4042 | if (off < min_valid_off || off > -1) |
| 4043 | return -EACCES; |
| 4044 | return 0; |
| 4045 | } |
| 4046 | |
| 4047 | /* Check that the stack access at 'regno + off' falls within the maximum stack |
| 4048 | * bounds. |
| 4049 | * |
| 4050 | * 'off' includes `regno->offset`, but not its dynamic part (if any). |
| 4051 | */ |
| 4052 | static int check_stack_access_within_bounds( |
| 4053 | struct bpf_verifier_env *env, |
| 4054 | int regno, int off, int access_size, |
| 4055 | enum stack_access_src src, enum bpf_access_type type) |
| 4056 | { |
| 4057 | struct bpf_reg_state *regs = cur_regs(env); |
| 4058 | struct bpf_reg_state *reg = regs + regno; |
| 4059 | struct bpf_func_state *state = func(env, reg); |
| 4060 | int min_off, max_off; |
| 4061 | int err; |
| 4062 | char *err_extra; |
| 4063 | |
| 4064 | if (src == ACCESS_HELPER) |
| 4065 | /* We don't know if helpers are reading or writing (or both). */ |
| 4066 | err_extra = " indirect access to"; |
| 4067 | else if (type == BPF_READ) |
| 4068 | err_extra = " read from"; |
| 4069 | else |
| 4070 | err_extra = " write to"; |
| 4071 | |
| 4072 | if (tnum_is_const(reg->var_off)) { |
| 4073 | min_off = reg->var_off.value + off; |
| 4074 | if (access_size > 0) |
| 4075 | max_off = min_off + access_size - 1; |
| 4076 | else |
| 4077 | max_off = min_off; |
| 4078 | } else { |
| 4079 | if (reg->smax_value >= BPF_MAX_VAR_OFF || |
| 4080 | reg->smin_value <= -BPF_MAX_VAR_OFF) { |
| 4081 | verbose(env, "invalid unbounded variable-offset%s stack R%d\n", |
| 4082 | err_extra, regno); |
| 4083 | return -EACCES; |
| 4084 | } |
| 4085 | min_off = reg->smin_value + off; |
| 4086 | if (access_size > 0) |
| 4087 | max_off = reg->smax_value + off + access_size - 1; |
| 4088 | else |
| 4089 | max_off = min_off; |
| 4090 | } |
| 4091 | |
| 4092 | err = check_stack_slot_within_bounds(min_off, state, type); |
| 4093 | if (!err) |
| 4094 | err = check_stack_slot_within_bounds(max_off, state, type); |
| 4095 | |
| 4096 | if (err) { |
| 4097 | if (tnum_is_const(reg->var_off)) { |
| 4098 | verbose(env, "invalid%s stack R%d off=%d size=%d\n", |
| 4099 | err_extra, regno, off, access_size); |
| 4100 | } else { |
| 4101 | char tn_buf[48]; |
| 4102 | |
| 4103 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
| 4104 | verbose(env, "invalid variable-offset%s stack R%d var_off=%s size=%d\n", |
| 4105 | err_extra, regno, tn_buf, access_size); |
| 4106 | } |
| 4107 | } |
| 4108 | return err; |
| 4109 | } |
Andrey Ignatov | 41c48f3 | 2020-06-19 14:11:43 -0700 | [diff] [blame] | 4110 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4111 | /* check whether memory at (regno + off) is accessible for t = (read | write) |
| 4112 | * if t==write, value_regno is a register which value is stored into memory |
| 4113 | * if t==read, value_regno is a register which will receive the value from memory |
| 4114 | * if t==write && value_regno==-1, some unknown value is stored into memory |
| 4115 | * if t==read && value_regno==-1, don't care what we read from memory |
| 4116 | */ |
Daniel Borkmann | ca36960 | 2018-02-23 22:29:05 +0100 | [diff] [blame] | 4117 | static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, |
| 4118 | int off, int bpf_size, enum bpf_access_type t, |
| 4119 | int value_regno, bool strict_alignment_once) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4120 | { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 4121 | struct bpf_reg_state *regs = cur_regs(env); |
| 4122 | struct bpf_reg_state *reg = regs + regno; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 4123 | struct bpf_func_state *state; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4124 | int size, err = 0; |
| 4125 | |
| 4126 | size = bpf_size_to_bytes(bpf_size); |
| 4127 | if (size < 0) |
| 4128 | return size; |
| 4129 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4130 | /* alignment checks will add in reg->off themselves */ |
Daniel Borkmann | ca36960 | 2018-02-23 22:29:05 +0100 | [diff] [blame] | 4131 | err = check_ptr_alignment(env, reg, off, size, strict_alignment_once); |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 4132 | if (err) |
| 4133 | return err; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4134 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4135 | /* for access checks, reg->off is just part of off */ |
| 4136 | off += reg->off; |
| 4137 | |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 4138 | if (reg->type == PTR_TO_MAP_KEY) { |
| 4139 | if (t == BPF_WRITE) { |
| 4140 | verbose(env, "write to change key R%d not allowed\n", regno); |
| 4141 | return -EACCES; |
| 4142 | } |
| 4143 | |
| 4144 | err = check_mem_region_access(env, regno, off, size, |
| 4145 | reg->map_ptr->key_size, false); |
| 4146 | if (err) |
| 4147 | return err; |
| 4148 | if (value_regno >= 0) |
| 4149 | mark_reg_unknown(env, regs, value_regno); |
| 4150 | } else if (reg->type == PTR_TO_MAP_VALUE) { |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 4151 | if (t == BPF_WRITE && value_regno >= 0 && |
| 4152 | is_pointer_value(env, value_regno)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 4153 | verbose(env, "R%d leaks addr into map\n", value_regno); |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 4154 | return -EACCES; |
| 4155 | } |
Daniel Borkmann | 591fe98 | 2019-04-09 23:20:05 +0200 | [diff] [blame] | 4156 | err = check_map_access_type(env, regno, off, size, t); |
| 4157 | if (err) |
| 4158 | return err; |
Yonghong Song | 9fd29c0 | 2017-11-12 14:49:09 -0800 | [diff] [blame] | 4159 | err = check_map_access(env, regno, off, size, false); |
Andrii Nakryiko | a23740e | 2019-10-09 13:14:57 -0700 | [diff] [blame] | 4160 | if (!err && t == BPF_READ && value_regno >= 0) { |
| 4161 | struct bpf_map *map = reg->map_ptr; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4162 | |
Andrii Nakryiko | a23740e | 2019-10-09 13:14:57 -0700 | [diff] [blame] | 4163 | /* if map is read-only, track its contents as scalars */ |
| 4164 | if (tnum_is_const(reg->var_off) && |
| 4165 | bpf_map_is_rdonly(map) && |
| 4166 | map->ops->map_direct_value_addr) { |
| 4167 | int map_off = off + reg->var_off.value; |
| 4168 | u64 val = 0; |
| 4169 | |
| 4170 | err = bpf_map_direct_read(map, map_off, size, |
| 4171 | &val); |
| 4172 | if (err) |
| 4173 | return err; |
| 4174 | |
| 4175 | regs[value_regno].type = SCALAR_VALUE; |
| 4176 | __mark_reg_known(®s[value_regno], val); |
| 4177 | } else { |
| 4178 | mark_reg_unknown(env, regs, value_regno); |
| 4179 | } |
| 4180 | } |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 4181 | } else if (reg->type == PTR_TO_MEM) { |
| 4182 | if (t == BPF_WRITE && value_regno >= 0 && |
| 4183 | is_pointer_value(env, value_regno)) { |
| 4184 | verbose(env, "R%d leaks addr into mem\n", value_regno); |
| 4185 | return -EACCES; |
| 4186 | } |
| 4187 | err = check_mem_region_access(env, regno, off, size, |
| 4188 | reg->mem_size, false); |
| 4189 | if (!err && t == BPF_READ && value_regno >= 0) |
| 4190 | mark_reg_unknown(env, regs, value_regno); |
Alexei Starovoitov | 1a0dc1a | 2016-05-05 19:49:09 -0700 | [diff] [blame] | 4191 | } else if (reg->type == PTR_TO_CTX) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4192 | enum bpf_reg_type reg_type = SCALAR_VALUE; |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 4193 | struct btf *btf = NULL; |
Alexei Starovoitov | 9e15db6 | 2019-10-15 20:25:00 -0700 | [diff] [blame] | 4194 | u32 btf_id = 0; |
Alexei Starovoitov | 19de99f | 2016-06-15 18:25:38 -0700 | [diff] [blame] | 4195 | |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 4196 | if (t == BPF_WRITE && value_regno >= 0 && |
| 4197 | is_pointer_value(env, value_regno)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 4198 | verbose(env, "R%d leaks addr into ctx\n", value_regno); |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 4199 | return -EACCES; |
| 4200 | } |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4201 | |
Daniel Borkmann | 58990d1 | 2018-06-07 17:40:03 +0200 | [diff] [blame] | 4202 | err = check_ctx_reg(env, reg, regno); |
| 4203 | if (err < 0) |
| 4204 | return err; |
| 4205 | |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 4206 | err = check_ctx_access(env, insn_idx, off, size, t, ®_type, &btf, &btf_id); |
Alexei Starovoitov | 9e15db6 | 2019-10-15 20:25:00 -0700 | [diff] [blame] | 4207 | if (err) |
| 4208 | verbose_linfo(env, insn_idx, "; "); |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 4209 | if (!err && t == BPF_READ && value_regno >= 0) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4210 | /* ctx access returns either a scalar, or a |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 4211 | * PTR_TO_PACKET[_META,_END]. In the latter |
| 4212 | * case, we know the offset is zero. |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4213 | */ |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 4214 | if (reg_type == SCALAR_VALUE) { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 4215 | mark_reg_unknown(env, regs, value_regno); |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 4216 | } else { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 4217 | mark_reg_known_zero(env, regs, |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 4218 | value_regno); |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 4219 | if (reg_type_may_be_null(reg_type)) |
| 4220 | regs[value_regno].id = ++env->id_gen; |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 4221 | /* A load of ctx field could have different |
| 4222 | * actual load size with the one encoded in the |
| 4223 | * insn. When the dst is PTR, it is for sure not |
| 4224 | * a sub-register. |
| 4225 | */ |
| 4226 | regs[value_regno].subreg_def = DEF_NOT_SUBREG; |
Yonghong Song | b121b34 | 2020-05-09 10:59:12 -0700 | [diff] [blame] | 4227 | if (reg_type == PTR_TO_BTF_ID || |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 4228 | reg_type == PTR_TO_BTF_ID_OR_NULL) { |
| 4229 | regs[value_regno].btf = btf; |
Alexei Starovoitov | 9e15db6 | 2019-10-15 20:25:00 -0700 | [diff] [blame] | 4230 | regs[value_regno].btf_id = btf_id; |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 4231 | } |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 4232 | } |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 4233 | regs[value_regno].type = reg_type; |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 4234 | } |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4235 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4236 | } else if (reg->type == PTR_TO_STACK) { |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 4237 | /* Basic bounds checks. */ |
| 4238 | err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t); |
Daniel Borkmann | e4298d2 | 2019-01-03 00:58:31 +0100 | [diff] [blame] | 4239 | if (err) |
| 4240 | return err; |
Alexei Starovoitov | 8726679 | 2017-05-30 13:31:29 -0700 | [diff] [blame] | 4241 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 4242 | state = func(env, reg); |
| 4243 | err = update_stack_depth(env, state, off); |
| 4244 | if (err) |
| 4245 | return err; |
Alexei Starovoitov | 8726679 | 2017-05-30 13:31:29 -0700 | [diff] [blame] | 4246 | |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 4247 | if (t == BPF_READ) |
| 4248 | err = check_stack_read(env, regno, off, size, |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 4249 | value_regno); |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 4250 | else |
| 4251 | err = check_stack_write(env, regno, off, size, |
| 4252 | value_regno, insn_idx); |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 4253 | } else if (reg_is_pkt_pointer(reg)) { |
Thomas Graf | 3a0af8f | 2016-11-30 17:10:10 +0100 | [diff] [blame] | 4254 | if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 4255 | verbose(env, "cannot write into packet\n"); |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 4256 | return -EACCES; |
| 4257 | } |
Brenden Blanco | 4acf6c0 | 2016-07-19 12:16:56 -0700 | [diff] [blame] | 4258 | if (t == BPF_WRITE && value_regno >= 0 && |
| 4259 | is_pointer_value(env, value_regno)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 4260 | verbose(env, "R%d leaks addr into packet\n", |
| 4261 | value_regno); |
Brenden Blanco | 4acf6c0 | 2016-07-19 12:16:56 -0700 | [diff] [blame] | 4262 | return -EACCES; |
| 4263 | } |
Yonghong Song | 9fd29c0 | 2017-11-12 14:49:09 -0800 | [diff] [blame] | 4264 | err = check_packet_access(env, regno, off, size, false); |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 4265 | if (!err && t == BPF_READ && value_regno >= 0) |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 4266 | mark_reg_unknown(env, regs, value_regno); |
Petar Penkov | d58e468 | 2018-09-14 07:46:18 -0700 | [diff] [blame] | 4267 | } else if (reg->type == PTR_TO_FLOW_KEYS) { |
| 4268 | if (t == BPF_WRITE && value_regno >= 0 && |
| 4269 | is_pointer_value(env, value_regno)) { |
| 4270 | verbose(env, "R%d leaks addr into flow keys\n", |
| 4271 | value_regno); |
| 4272 | return -EACCES; |
| 4273 | } |
| 4274 | |
| 4275 | err = check_flow_keys_access(env, off, size); |
| 4276 | if (!err && t == BPF_READ && value_regno >= 0) |
| 4277 | mark_reg_unknown(env, regs, value_regno); |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 4278 | } else if (type_is_sk_pointer(reg->type)) { |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 4279 | if (t == BPF_WRITE) { |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 4280 | verbose(env, "R%d cannot write into %s\n", |
| 4281 | regno, reg_type_str[reg->type]); |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 4282 | return -EACCES; |
| 4283 | } |
Martin KaFai Lau | 5f45664 | 2019-02-08 22:25:54 -0800 | [diff] [blame] | 4284 | err = check_sock_access(env, insn_idx, regno, off, size, t); |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 4285 | if (!err && value_regno >= 0) |
| 4286 | mark_reg_unknown(env, regs, value_regno); |
Matt Mullins | 9df1c28 | 2019-04-26 11:49:47 -0700 | [diff] [blame] | 4287 | } else if (reg->type == PTR_TO_TP_BUFFER) { |
| 4288 | err = check_tp_buffer_access(env, reg, regno, off, size); |
| 4289 | if (!err && t == BPF_READ && value_regno >= 0) |
| 4290 | mark_reg_unknown(env, regs, value_regno); |
Alexei Starovoitov | 9e15db6 | 2019-10-15 20:25:00 -0700 | [diff] [blame] | 4291 | } else if (reg->type == PTR_TO_BTF_ID) { |
| 4292 | err = check_ptr_to_btf_access(env, regs, regno, off, size, t, |
| 4293 | value_regno); |
Andrey Ignatov | 41c48f3 | 2020-06-19 14:11:43 -0700 | [diff] [blame] | 4294 | } else if (reg->type == CONST_PTR_TO_MAP) { |
| 4295 | err = check_ptr_to_map_access(env, regs, regno, off, size, t, |
| 4296 | value_regno); |
Yonghong Song | afbf21d | 2020-07-23 11:41:11 -0700 | [diff] [blame] | 4297 | } else if (reg->type == PTR_TO_RDONLY_BUF) { |
| 4298 | if (t == BPF_WRITE) { |
| 4299 | verbose(env, "R%d cannot write into %s\n", |
| 4300 | regno, reg_type_str[reg->type]); |
| 4301 | return -EACCES; |
| 4302 | } |
Colin Ian King | f6dfbe31 | 2020-07-27 18:54:11 +0100 | [diff] [blame] | 4303 | err = check_buffer_access(env, reg, regno, off, size, false, |
| 4304 | "rdonly", |
Yonghong Song | afbf21d | 2020-07-23 11:41:11 -0700 | [diff] [blame] | 4305 | &env->prog->aux->max_rdonly_access); |
| 4306 | if (!err && value_regno >= 0) |
| 4307 | mark_reg_unknown(env, regs, value_regno); |
| 4308 | } else if (reg->type == PTR_TO_RDWR_BUF) { |
Colin Ian King | f6dfbe31 | 2020-07-27 18:54:11 +0100 | [diff] [blame] | 4309 | err = check_buffer_access(env, reg, regno, off, size, false, |
| 4310 | "rdwr", |
Yonghong Song | afbf21d | 2020-07-23 11:41:11 -0700 | [diff] [blame] | 4311 | &env->prog->aux->max_rdwr_access); |
| 4312 | if (!err && t == BPF_READ && value_regno >= 0) |
| 4313 | mark_reg_unknown(env, regs, value_regno); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4314 | } else { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 4315 | verbose(env, "R%d invalid mem access '%s'\n", regno, |
| 4316 | reg_type_str[reg->type]); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4317 | return -EACCES; |
| 4318 | } |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 4319 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4320 | if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ && |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 4321 | regs[value_regno].type == SCALAR_VALUE) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4322 | /* b/h/w load zero-extends, mark upper bits as known 0 */ |
Jann Horn | 0c17d1d | 2017-12-18 20:11:55 -0800 | [diff] [blame] | 4323 | coerce_reg_to_size(®s[value_regno], size); |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 4324 | } |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4325 | return err; |
| 4326 | } |
| 4327 | |
Brendan Jackman | 91c960b | 2021-01-14 18:17:44 +0000 | [diff] [blame] | 4328 | static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4329 | { |
Brendan Jackman | 5ffa255 | 2021-01-14 18:17:47 +0000 | [diff] [blame] | 4330 | int load_reg; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4331 | int err; |
| 4332 | |
Brendan Jackman | 5ca419f | 2021-01-14 18:17:46 +0000 | [diff] [blame] | 4333 | switch (insn->imm) { |
| 4334 | case BPF_ADD: |
| 4335 | case BPF_ADD | BPF_FETCH: |
Brendan Jackman | 981f94c | 2021-01-14 18:17:49 +0000 | [diff] [blame] | 4336 | case BPF_AND: |
| 4337 | case BPF_AND | BPF_FETCH: |
| 4338 | case BPF_OR: |
| 4339 | case BPF_OR | BPF_FETCH: |
| 4340 | case BPF_XOR: |
| 4341 | case BPF_XOR | BPF_FETCH: |
Brendan Jackman | 5ffa255 | 2021-01-14 18:17:47 +0000 | [diff] [blame] | 4342 | case BPF_XCHG: |
| 4343 | case BPF_CMPXCHG: |
Brendan Jackman | 5ca419f | 2021-01-14 18:17:46 +0000 | [diff] [blame] | 4344 | break; |
| 4345 | default: |
Brendan Jackman | 91c960b | 2021-01-14 18:17:44 +0000 | [diff] [blame] | 4346 | verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm); |
| 4347 | return -EINVAL; |
| 4348 | } |
| 4349 | |
| 4350 | if (BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) { |
| 4351 | verbose(env, "invalid atomic operand size\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4352 | return -EINVAL; |
| 4353 | } |
| 4354 | |
| 4355 | /* check src1 operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 4356 | err = check_reg_arg(env, insn->src_reg, SRC_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4357 | if (err) |
| 4358 | return err; |
| 4359 | |
| 4360 | /* check src2 operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 4361 | err = check_reg_arg(env, insn->dst_reg, SRC_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4362 | if (err) |
| 4363 | return err; |
| 4364 | |
Brendan Jackman | 5ffa255 | 2021-01-14 18:17:47 +0000 | [diff] [blame] | 4365 | if (insn->imm == BPF_CMPXCHG) { |
| 4366 | /* Check comparison of R0 with memory location */ |
| 4367 | err = check_reg_arg(env, BPF_REG_0, SRC_OP); |
| 4368 | if (err) |
| 4369 | return err; |
| 4370 | } |
| 4371 | |
Daniel Borkmann | 6bdf6ab | 2017-06-29 03:04:59 +0200 | [diff] [blame] | 4372 | if (is_pointer_value(env, insn->src_reg)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 4373 | verbose(env, "R%d leaks addr into mem\n", insn->src_reg); |
Daniel Borkmann | 6bdf6ab | 2017-06-29 03:04:59 +0200 | [diff] [blame] | 4374 | return -EACCES; |
| 4375 | } |
| 4376 | |
Daniel Borkmann | ca36960 | 2018-02-23 22:29:05 +0100 | [diff] [blame] | 4377 | if (is_ctx_reg(env, insn->dst_reg) || |
Daniel Borkmann | 4b5defd | 2018-10-21 02:09:25 +0200 | [diff] [blame] | 4378 | is_pkt_reg(env, insn->dst_reg) || |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 4379 | is_flow_key_reg(env, insn->dst_reg) || |
| 4380 | is_sk_reg(env, insn->dst_reg)) { |
Brendan Jackman | 91c960b | 2021-01-14 18:17:44 +0000 | [diff] [blame] | 4381 | verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n", |
Daniel Borkmann | 2a159c6 | 2018-10-21 02:09:24 +0200 | [diff] [blame] | 4382 | insn->dst_reg, |
| 4383 | reg_type_str[reg_state(env, insn->dst_reg)->type]); |
Daniel Borkmann | f37a8cb | 2018-01-16 23:30:10 +0100 | [diff] [blame] | 4384 | return -EACCES; |
| 4385 | } |
| 4386 | |
Brendan Jackman | 37086bf | 2021-02-02 13:50:02 +0000 | [diff] [blame] | 4387 | if (insn->imm & BPF_FETCH) { |
| 4388 | if (insn->imm == BPF_CMPXCHG) |
| 4389 | load_reg = BPF_REG_0; |
| 4390 | else |
| 4391 | load_reg = insn->src_reg; |
| 4392 | |
| 4393 | /* check and record load of old value */ |
| 4394 | err = check_reg_arg(env, load_reg, DST_OP); |
| 4395 | if (err) |
| 4396 | return err; |
| 4397 | } else { |
| 4398 | /* This instruction accesses a memory location but doesn't |
| 4399 | * actually load it into a register. |
| 4400 | */ |
| 4401 | load_reg = -1; |
| 4402 | } |
| 4403 | |
Brendan Jackman | 91c960b | 2021-01-14 18:17:44 +0000 | [diff] [blame] | 4404 | /* check whether we can read the memory */ |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 4405 | err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, |
Brendan Jackman | 37086bf | 2021-02-02 13:50:02 +0000 | [diff] [blame] | 4406 | BPF_SIZE(insn->code), BPF_READ, load_reg, true); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4407 | if (err) |
| 4408 | return err; |
| 4409 | |
Brendan Jackman | 91c960b | 2021-01-14 18:17:44 +0000 | [diff] [blame] | 4410 | /* check whether we can write into the same memory */ |
Brendan Jackman | 5ca419f | 2021-01-14 18:17:46 +0000 | [diff] [blame] | 4411 | err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, |
| 4412 | BPF_SIZE(insn->code), BPF_WRITE, -1, true); |
| 4413 | if (err) |
| 4414 | return err; |
| 4415 | |
Brendan Jackman | 5ca419f | 2021-01-14 18:17:46 +0000 | [diff] [blame] | 4416 | return 0; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4417 | } |
| 4418 | |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 4419 | /* When register 'regno' is used to read the stack (either directly or through |
| 4420 | * a helper function) make sure that it's within stack boundary and, depending |
| 4421 | * on the access type, that all elements of the stack are initialized. |
| 4422 | * |
| 4423 | * 'off' includes 'regno->off', but not its dynamic part (if any). |
| 4424 | * |
| 4425 | * All registers that have been spilled on the stack in the slots within the |
| 4426 | * read offsets are marked as read. |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4427 | */ |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 4428 | static int check_stack_range_initialized( |
| 4429 | struct bpf_verifier_env *env, int regno, int off, |
| 4430 | int access_size, bool zero_size_allowed, |
| 4431 | enum stack_access_src type, struct bpf_call_arg_meta *meta) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4432 | { |
Daniel Borkmann | 2a159c6 | 2018-10-21 02:09:24 +0200 | [diff] [blame] | 4433 | struct bpf_reg_state *reg = reg_state(env, regno); |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 4434 | struct bpf_func_state *state = func(env, reg); |
Alexei Starovoitov | f7cf25b | 2019-06-15 12:12:17 -0700 | [diff] [blame] | 4435 | int err, min_off, max_off, i, j, slot, spi; |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 4436 | char *err_extra = type == ACCESS_HELPER ? " indirect" : ""; |
| 4437 | enum bpf_access_type bounds_check_type; |
| 4438 | /* Some accesses can write anything into the stack, others are |
| 4439 | * read-only. |
| 4440 | */ |
| 4441 | bool clobber = false; |
| 4442 | |
| 4443 | if (access_size == 0 && !zero_size_allowed) { |
| 4444 | verbose(env, "invalid zero-sized read\n"); |
| 4445 | return -EACCES; |
| 4446 | } |
| 4447 | |
| 4448 | if (type == ACCESS_HELPER) { |
| 4449 | /* The bounds checks for writes are more permissive than for |
| 4450 | * reads. However, if raw_mode is not set, we'll do extra |
| 4451 | * checks below. |
| 4452 | */ |
| 4453 | bounds_check_type = BPF_WRITE; |
| 4454 | clobber = true; |
| 4455 | } else { |
| 4456 | bounds_check_type = BPF_READ; |
| 4457 | } |
| 4458 | err = check_stack_access_within_bounds(env, regno, off, access_size, |
| 4459 | type, bounds_check_type); |
| 4460 | if (err) |
| 4461 | return err; |
| 4462 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4463 | |
Andrey Ignatov | 2011fcc | 2019-03-28 18:01:57 -0700 | [diff] [blame] | 4464 | if (tnum_is_const(reg->var_off)) { |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 4465 | min_off = max_off = reg->var_off.value + off; |
Andrey Ignatov | 2011fcc | 2019-03-28 18:01:57 -0700 | [diff] [blame] | 4466 | } else { |
Andrey Ignatov | 088ec26 | 2019-04-03 23:22:39 -0700 | [diff] [blame] | 4467 | /* Variable offset is prohibited for unprivileged mode for |
| 4468 | * simplicity since it requires corresponding support in |
| 4469 | * Spectre masking for stack ALU. |
| 4470 | * See also retrieve_ptr_limit(). |
| 4471 | */ |
Alexei Starovoitov | 2c78ee8 | 2020-05-13 16:03:54 -0700 | [diff] [blame] | 4472 | if (!env->bypass_spec_v1) { |
Andrey Ignatov | 088ec26 | 2019-04-03 23:22:39 -0700 | [diff] [blame] | 4473 | char tn_buf[48]; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4474 | |
Andrey Ignatov | 088ec26 | 2019-04-03 23:22:39 -0700 | [diff] [blame] | 4475 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 4476 | verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n", |
| 4477 | regno, err_extra, tn_buf); |
Andrey Ignatov | 088ec26 | 2019-04-03 23:22:39 -0700 | [diff] [blame] | 4478 | return -EACCES; |
| 4479 | } |
Andrey Ignatov | f2bcd05 | 2019-04-03 23:22:37 -0700 | [diff] [blame] | 4480 | /* Only initialized buffer on stack is allowed to be accessed |
| 4481 | * with variable offset. With uninitialized buffer it's hard to |
| 4482 | * guarantee that whole memory is marked as initialized on |
| 4483 | * helper return since specific bounds are unknown what may |
| 4484 | * cause uninitialized stack leaking. |
| 4485 | */ |
| 4486 | if (meta && meta->raw_mode) |
| 4487 | meta = NULL; |
| 4488 | |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 4489 | min_off = reg->smin_value + off; |
| 4490 | max_off = reg->smax_value + off; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4491 | } |
| 4492 | |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 4493 | if (meta && meta->raw_mode) { |
| 4494 | meta->access_size = access_size; |
| 4495 | meta->regno = regno; |
| 4496 | return 0; |
| 4497 | } |
| 4498 | |
Andrey Ignatov | 2011fcc | 2019-03-28 18:01:57 -0700 | [diff] [blame] | 4499 | for (i = min_off; i < max_off + access_size; i++) { |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 4500 | u8 *stype; |
| 4501 | |
Andrey Ignatov | 2011fcc | 2019-03-28 18:01:57 -0700 | [diff] [blame] | 4502 | slot = -i - 1; |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 4503 | spi = slot / BPF_REG_SIZE; |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 4504 | if (state->allocated_stack <= slot) |
| 4505 | goto err; |
| 4506 | stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; |
| 4507 | if (*stype == STACK_MISC) |
| 4508 | goto mark; |
| 4509 | if (*stype == STACK_ZERO) { |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 4510 | if (clobber) { |
| 4511 | /* helper can write anything into the stack */ |
| 4512 | *stype = STACK_MISC; |
| 4513 | } |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 4514 | goto mark; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4515 | } |
Yonghong Song | 1d68f22 | 2020-05-09 10:59:15 -0700 | [diff] [blame] | 4516 | |
| 4517 | if (state->stack[spi].slot_type[0] == STACK_SPILL && |
| 4518 | state->stack[spi].spilled_ptr.type == PTR_TO_BTF_ID) |
| 4519 | goto mark; |
| 4520 | |
Alexei Starovoitov | f7cf25b | 2019-06-15 12:12:17 -0700 | [diff] [blame] | 4521 | if (state->stack[spi].slot_type[0] == STACK_SPILL && |
Yonghong Song | cd17d38 | 2020-12-09 17:33:49 -0800 | [diff] [blame] | 4522 | (state->stack[spi].spilled_ptr.type == SCALAR_VALUE || |
| 4523 | env->allow_ptr_leaks)) { |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 4524 | if (clobber) { |
| 4525 | __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); |
| 4526 | for (j = 0; j < BPF_REG_SIZE; j++) |
| 4527 | state->stack[spi].slot_type[j] = STACK_MISC; |
| 4528 | } |
Alexei Starovoitov | f7cf25b | 2019-06-15 12:12:17 -0700 | [diff] [blame] | 4529 | goto mark; |
| 4530 | } |
| 4531 | |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 4532 | err: |
Andrey Ignatov | 2011fcc | 2019-03-28 18:01:57 -0700 | [diff] [blame] | 4533 | if (tnum_is_const(reg->var_off)) { |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 4534 | verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n", |
| 4535 | err_extra, regno, min_off, i - min_off, access_size); |
Andrey Ignatov | 2011fcc | 2019-03-28 18:01:57 -0700 | [diff] [blame] | 4536 | } else { |
| 4537 | char tn_buf[48]; |
| 4538 | |
| 4539 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 4540 | verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n", |
| 4541 | err_extra, regno, tn_buf, i - min_off, access_size); |
Andrey Ignatov | 2011fcc | 2019-03-28 18:01:57 -0700 | [diff] [blame] | 4542 | } |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 4543 | return -EACCES; |
| 4544 | mark: |
| 4545 | /* reading any byte out of 8-byte 'spill_slot' will cause |
| 4546 | * the whole slot to be marked as 'read' |
| 4547 | */ |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 4548 | mark_reg_read(env, &state->stack[spi].spilled_ptr, |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 4549 | state->stack[spi].spilled_ptr.parent, |
| 4550 | REG_LIVE_READ64); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4551 | } |
Andrey Ignatov | 2011fcc | 2019-03-28 18:01:57 -0700 | [diff] [blame] | 4552 | return update_stack_depth(env, state, min_off); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4553 | } |
| 4554 | |
Gianluca Borello | 06c1c04 | 2017-01-09 10:19:49 -0800 | [diff] [blame] | 4555 | static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, |
| 4556 | int access_size, bool zero_size_allowed, |
| 4557 | struct bpf_call_arg_meta *meta) |
| 4558 | { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 4559 | struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; |
Gianluca Borello | 06c1c04 | 2017-01-09 10:19:49 -0800 | [diff] [blame] | 4560 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 4561 | switch (reg->type) { |
Gianluca Borello | 06c1c04 | 2017-01-09 10:19:49 -0800 | [diff] [blame] | 4562 | case PTR_TO_PACKET: |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 4563 | case PTR_TO_PACKET_META: |
Yonghong Song | 9fd29c0 | 2017-11-12 14:49:09 -0800 | [diff] [blame] | 4564 | return check_packet_access(env, regno, reg->off, access_size, |
| 4565 | zero_size_allowed); |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 4566 | case PTR_TO_MAP_KEY: |
| 4567 | return check_mem_region_access(env, regno, reg->off, access_size, |
| 4568 | reg->map_ptr->key_size, false); |
Gianluca Borello | 06c1c04 | 2017-01-09 10:19:49 -0800 | [diff] [blame] | 4569 | case PTR_TO_MAP_VALUE: |
Daniel Borkmann | 591fe98 | 2019-04-09 23:20:05 +0200 | [diff] [blame] | 4570 | if (check_map_access_type(env, regno, reg->off, access_size, |
| 4571 | meta && meta->raw_mode ? BPF_WRITE : |
| 4572 | BPF_READ)) |
| 4573 | return -EACCES; |
Yonghong Song | 9fd29c0 | 2017-11-12 14:49:09 -0800 | [diff] [blame] | 4574 | return check_map_access(env, regno, reg->off, access_size, |
| 4575 | zero_size_allowed); |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 4576 | case PTR_TO_MEM: |
| 4577 | return check_mem_region_access(env, regno, reg->off, |
| 4578 | access_size, reg->mem_size, |
| 4579 | zero_size_allowed); |
Yonghong Song | afbf21d | 2020-07-23 11:41:11 -0700 | [diff] [blame] | 4580 | case PTR_TO_RDONLY_BUF: |
| 4581 | if (meta && meta->raw_mode) |
| 4582 | return -EACCES; |
| 4583 | return check_buffer_access(env, reg, regno, reg->off, |
| 4584 | access_size, zero_size_allowed, |
| 4585 | "rdonly", |
| 4586 | &env->prog->aux->max_rdonly_access); |
| 4587 | case PTR_TO_RDWR_BUF: |
| 4588 | return check_buffer_access(env, reg, regno, reg->off, |
| 4589 | access_size, zero_size_allowed, |
| 4590 | "rdwr", |
| 4591 | &env->prog->aux->max_rdwr_access); |
Lorenz Bauer | 0d004c02 | 2020-09-21 13:12:18 +0100 | [diff] [blame] | 4592 | case PTR_TO_STACK: |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 4593 | return check_stack_range_initialized( |
| 4594 | env, |
| 4595 | regno, reg->off, access_size, |
| 4596 | zero_size_allowed, ACCESS_HELPER, meta); |
Lorenz Bauer | 0d004c02 | 2020-09-21 13:12:18 +0100 | [diff] [blame] | 4597 | default: /* scalar_value or invalid ptr */ |
| 4598 | /* Allow zero-byte read from NULL, regardless of pointer type */ |
| 4599 | if (zero_size_allowed && access_size == 0 && |
| 4600 | register_is_null(reg)) |
| 4601 | return 0; |
| 4602 | |
| 4603 | verbose(env, "R%d type=%s expected=%s\n", regno, |
| 4604 | reg_type_str[reg->type], |
| 4605 | reg_type_str[PTR_TO_STACK]); |
| 4606 | return -EACCES; |
Gianluca Borello | 06c1c04 | 2017-01-09 10:19:49 -0800 | [diff] [blame] | 4607 | } |
| 4608 | } |
| 4609 | |
Dmitrii Banshchikov | e5069b9c | 2021-02-13 00:56:41 +0400 | [diff] [blame] | 4610 | int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, |
| 4611 | u32 regno, u32 mem_size) |
| 4612 | { |
| 4613 | if (register_is_null(reg)) |
| 4614 | return 0; |
| 4615 | |
| 4616 | if (reg_type_may_be_null(reg->type)) { |
| 4617 | /* Assuming that the register contains a value check if the memory |
| 4618 | * access is safe. Temporarily save and restore the register's state as |
| 4619 | * the conversion shouldn't be visible to a caller. |
| 4620 | */ |
| 4621 | const struct bpf_reg_state saved_reg = *reg; |
| 4622 | int rv; |
| 4623 | |
| 4624 | mark_ptr_not_null_reg(reg); |
| 4625 | rv = check_helper_mem_access(env, regno, mem_size, true, NULL); |
| 4626 | *reg = saved_reg; |
| 4627 | return rv; |
| 4628 | } |
| 4629 | |
| 4630 | return check_helper_mem_access(env, regno, mem_size, true, NULL); |
| 4631 | } |
| 4632 | |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 4633 | /* Implementation details: |
| 4634 | * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL |
| 4635 | * Two bpf_map_lookups (even with the same key) will have different reg->id. |
| 4636 | * For traditional PTR_TO_MAP_VALUE the verifier clears reg->id after |
| 4637 | * value_or_null->value transition, since the verifier only cares about |
| 4638 | * the range of access to valid map value pointer and doesn't care about actual |
| 4639 | * address of the map element. |
| 4640 | * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps |
| 4641 | * reg->id > 0 after value_or_null->value transition. By doing so |
| 4642 | * two bpf_map_lookups will be considered two different pointers that |
| 4643 | * point to different bpf_spin_locks. |
| 4644 | * The verifier allows taking only one bpf_spin_lock at a time to avoid |
| 4645 | * dead-locks. |
| 4646 | * Since only one bpf_spin_lock is allowed the checks are simpler than |
| 4647 | * reg_is_refcounted() logic. The verifier needs to remember only |
| 4648 | * one spin_lock instead of array of acquired_refs. |
| 4649 | * cur_state->active_spin_lock remembers which map value element got locked |
| 4650 | * and clears it after bpf_spin_unlock. |
| 4651 | */ |
| 4652 | static int process_spin_lock(struct bpf_verifier_env *env, int regno, |
| 4653 | bool is_lock) |
| 4654 | { |
| 4655 | struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; |
| 4656 | struct bpf_verifier_state *cur = env->cur_state; |
| 4657 | bool is_const = tnum_is_const(reg->var_off); |
| 4658 | struct bpf_map *map = reg->map_ptr; |
| 4659 | u64 val = reg->var_off.value; |
| 4660 | |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 4661 | if (!is_const) { |
| 4662 | verbose(env, |
| 4663 | "R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n", |
| 4664 | regno); |
| 4665 | return -EINVAL; |
| 4666 | } |
| 4667 | if (!map->btf) { |
| 4668 | verbose(env, |
| 4669 | "map '%s' has to have BTF in order to use bpf_spin_lock\n", |
| 4670 | map->name); |
| 4671 | return -EINVAL; |
| 4672 | } |
| 4673 | if (!map_value_has_spin_lock(map)) { |
| 4674 | if (map->spin_lock_off == -E2BIG) |
| 4675 | verbose(env, |
| 4676 | "map '%s' has more than one 'struct bpf_spin_lock'\n", |
| 4677 | map->name); |
| 4678 | else if (map->spin_lock_off == -ENOENT) |
| 4679 | verbose(env, |
| 4680 | "map '%s' doesn't have 'struct bpf_spin_lock'\n", |
| 4681 | map->name); |
| 4682 | else |
| 4683 | verbose(env, |
| 4684 | "map '%s' is not a struct type or bpf_spin_lock is mangled\n", |
| 4685 | map->name); |
| 4686 | return -EINVAL; |
| 4687 | } |
| 4688 | if (map->spin_lock_off != val + reg->off) { |
| 4689 | verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock'\n", |
| 4690 | val + reg->off); |
| 4691 | return -EINVAL; |
| 4692 | } |
| 4693 | if (is_lock) { |
| 4694 | if (cur->active_spin_lock) { |
| 4695 | verbose(env, |
| 4696 | "Locking two bpf_spin_locks are not allowed\n"); |
| 4697 | return -EINVAL; |
| 4698 | } |
| 4699 | cur->active_spin_lock = reg->id; |
| 4700 | } else { |
| 4701 | if (!cur->active_spin_lock) { |
| 4702 | verbose(env, "bpf_spin_unlock without taking a lock\n"); |
| 4703 | return -EINVAL; |
| 4704 | } |
| 4705 | if (cur->active_spin_lock != reg->id) { |
| 4706 | verbose(env, "bpf_spin_unlock of different lock\n"); |
| 4707 | return -EINVAL; |
| 4708 | } |
| 4709 | cur->active_spin_lock = 0; |
| 4710 | } |
| 4711 | return 0; |
| 4712 | } |
| 4713 | |
Alexei Starovoitov | b00628b | 2021-07-14 17:54:09 -0700 | [diff] [blame] | 4714 | static int process_timer_func(struct bpf_verifier_env *env, int regno, |
| 4715 | struct bpf_call_arg_meta *meta) |
| 4716 | { |
| 4717 | struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; |
| 4718 | bool is_const = tnum_is_const(reg->var_off); |
| 4719 | struct bpf_map *map = reg->map_ptr; |
| 4720 | u64 val = reg->var_off.value; |
| 4721 | |
| 4722 | if (!is_const) { |
| 4723 | verbose(env, |
| 4724 | "R%d doesn't have constant offset. bpf_timer has to be at the constant offset\n", |
| 4725 | regno); |
| 4726 | return -EINVAL; |
| 4727 | } |
| 4728 | if (!map->btf) { |
| 4729 | verbose(env, "map '%s' has to have BTF in order to use bpf_timer\n", |
| 4730 | map->name); |
| 4731 | return -EINVAL; |
| 4732 | } |
Alexei Starovoitov | 6813466 | 2021-07-14 17:54:10 -0700 | [diff] [blame] | 4733 | if (!map_value_has_timer(map)) { |
| 4734 | if (map->timer_off == -E2BIG) |
| 4735 | verbose(env, |
| 4736 | "map '%s' has more than one 'struct bpf_timer'\n", |
| 4737 | map->name); |
| 4738 | else if (map->timer_off == -ENOENT) |
| 4739 | verbose(env, |
| 4740 | "map '%s' doesn't have 'struct bpf_timer'\n", |
| 4741 | map->name); |
| 4742 | else |
| 4743 | verbose(env, |
| 4744 | "map '%s' is not a struct type or bpf_timer is mangled\n", |
| 4745 | map->name); |
| 4746 | return -EINVAL; |
| 4747 | } |
| 4748 | if (map->timer_off != val + reg->off) { |
| 4749 | verbose(env, "off %lld doesn't point to 'struct bpf_timer' that is at %d\n", |
| 4750 | val + reg->off, map->timer_off); |
Alexei Starovoitov | b00628b | 2021-07-14 17:54:09 -0700 | [diff] [blame] | 4751 | return -EINVAL; |
| 4752 | } |
| 4753 | if (meta->map_ptr) { |
| 4754 | verbose(env, "verifier bug. Two map pointers in a timer helper\n"); |
| 4755 | return -EFAULT; |
| 4756 | } |
Alexei Starovoitov | 3e8ce29 | 2021-07-14 17:54:11 -0700 | [diff] [blame] | 4757 | meta->map_uid = reg->map_uid; |
Alexei Starovoitov | b00628b | 2021-07-14 17:54:09 -0700 | [diff] [blame] | 4758 | meta->map_ptr = map; |
| 4759 | return 0; |
| 4760 | } |
| 4761 | |
Daniel Borkmann | 9013341 | 2018-01-20 01:24:29 +0100 | [diff] [blame] | 4762 | static bool arg_type_is_mem_ptr(enum bpf_arg_type type) |
| 4763 | { |
| 4764 | return type == ARG_PTR_TO_MEM || |
| 4765 | type == ARG_PTR_TO_MEM_OR_NULL || |
| 4766 | type == ARG_PTR_TO_UNINIT_MEM; |
| 4767 | } |
| 4768 | |
| 4769 | static bool arg_type_is_mem_size(enum bpf_arg_type type) |
| 4770 | { |
| 4771 | return type == ARG_CONST_SIZE || |
| 4772 | type == ARG_CONST_SIZE_OR_ZERO; |
| 4773 | } |
| 4774 | |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 4775 | static bool arg_type_is_alloc_size(enum bpf_arg_type type) |
| 4776 | { |
| 4777 | return type == ARG_CONST_ALLOC_SIZE_OR_ZERO; |
| 4778 | } |
| 4779 | |
Andrey Ignatov | 57c3bb7 | 2019-03-18 16:57:10 -0700 | [diff] [blame] | 4780 | static bool arg_type_is_int_ptr(enum bpf_arg_type type) |
| 4781 | { |
| 4782 | return type == ARG_PTR_TO_INT || |
| 4783 | type == ARG_PTR_TO_LONG; |
| 4784 | } |
| 4785 | |
| 4786 | static int int_ptr_type_to_size(enum bpf_arg_type type) |
| 4787 | { |
| 4788 | if (type == ARG_PTR_TO_INT) |
| 4789 | return sizeof(u32); |
| 4790 | else if (type == ARG_PTR_TO_LONG) |
| 4791 | return sizeof(u64); |
| 4792 | |
| 4793 | return -EINVAL; |
| 4794 | } |
| 4795 | |
Lorenz Bauer | 912f442 | 2020-08-21 11:29:46 +0100 | [diff] [blame] | 4796 | static int resolve_map_arg_type(struct bpf_verifier_env *env, |
| 4797 | const struct bpf_call_arg_meta *meta, |
| 4798 | enum bpf_arg_type *arg_type) |
| 4799 | { |
| 4800 | if (!meta->map_ptr) { |
| 4801 | /* kernel subsystem misconfigured verifier */ |
| 4802 | verbose(env, "invalid map_ptr to access map->type\n"); |
| 4803 | return -EACCES; |
| 4804 | } |
| 4805 | |
| 4806 | switch (meta->map_ptr->map_type) { |
| 4807 | case BPF_MAP_TYPE_SOCKMAP: |
| 4808 | case BPF_MAP_TYPE_SOCKHASH: |
| 4809 | if (*arg_type == ARG_PTR_TO_MAP_VALUE) { |
Lorenz Bauer | 6550f2d | 2020-09-28 10:08:02 +0100 | [diff] [blame] | 4810 | *arg_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON; |
Lorenz Bauer | 912f442 | 2020-08-21 11:29:46 +0100 | [diff] [blame] | 4811 | } else { |
| 4812 | verbose(env, "invalid arg_type for sockmap/sockhash\n"); |
| 4813 | return -EINVAL; |
| 4814 | } |
| 4815 | break; |
| 4816 | |
| 4817 | default: |
| 4818 | break; |
| 4819 | } |
| 4820 | return 0; |
| 4821 | } |
| 4822 | |
Lorenz Bauer | f79e7ea | 2020-09-21 13:12:27 +0100 | [diff] [blame] | 4823 | struct bpf_reg_types { |
| 4824 | const enum bpf_reg_type types[10]; |
Martin KaFai Lau | 1df8f55 | 2020-09-24 17:03:50 -0700 | [diff] [blame] | 4825 | u32 *btf_id; |
Lorenz Bauer | f79e7ea | 2020-09-21 13:12:27 +0100 | [diff] [blame] | 4826 | }; |
| 4827 | |
| 4828 | static const struct bpf_reg_types map_key_value_types = { |
| 4829 | .types = { |
| 4830 | PTR_TO_STACK, |
| 4831 | PTR_TO_PACKET, |
| 4832 | PTR_TO_PACKET_META, |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 4833 | PTR_TO_MAP_KEY, |
Lorenz Bauer | f79e7ea | 2020-09-21 13:12:27 +0100 | [diff] [blame] | 4834 | PTR_TO_MAP_VALUE, |
| 4835 | }, |
| 4836 | }; |
| 4837 | |
| 4838 | static const struct bpf_reg_types sock_types = { |
| 4839 | .types = { |
| 4840 | PTR_TO_SOCK_COMMON, |
| 4841 | PTR_TO_SOCKET, |
| 4842 | PTR_TO_TCP_SOCK, |
| 4843 | PTR_TO_XDP_SOCK, |
| 4844 | }, |
| 4845 | }; |
| 4846 | |
Randy Dunlap | 49a2a4d | 2020-10-06 19:16:13 -0700 | [diff] [blame] | 4847 | #ifdef CONFIG_NET |
Martin KaFai Lau | 1df8f55 | 2020-09-24 17:03:50 -0700 | [diff] [blame] | 4848 | static const struct bpf_reg_types btf_id_sock_common_types = { |
| 4849 | .types = { |
| 4850 | PTR_TO_SOCK_COMMON, |
| 4851 | PTR_TO_SOCKET, |
| 4852 | PTR_TO_TCP_SOCK, |
| 4853 | PTR_TO_XDP_SOCK, |
| 4854 | PTR_TO_BTF_ID, |
| 4855 | }, |
| 4856 | .btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], |
| 4857 | }; |
Randy Dunlap | 49a2a4d | 2020-10-06 19:16:13 -0700 | [diff] [blame] | 4858 | #endif |
Martin KaFai Lau | 1df8f55 | 2020-09-24 17:03:50 -0700 | [diff] [blame] | 4859 | |
Lorenz Bauer | f79e7ea | 2020-09-21 13:12:27 +0100 | [diff] [blame] | 4860 | static const struct bpf_reg_types mem_types = { |
| 4861 | .types = { |
| 4862 | PTR_TO_STACK, |
| 4863 | PTR_TO_PACKET, |
| 4864 | PTR_TO_PACKET_META, |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 4865 | PTR_TO_MAP_KEY, |
Lorenz Bauer | f79e7ea | 2020-09-21 13:12:27 +0100 | [diff] [blame] | 4866 | PTR_TO_MAP_VALUE, |
| 4867 | PTR_TO_MEM, |
| 4868 | PTR_TO_RDONLY_BUF, |
| 4869 | PTR_TO_RDWR_BUF, |
| 4870 | }, |
| 4871 | }; |
| 4872 | |
| 4873 | static const struct bpf_reg_types int_ptr_types = { |
| 4874 | .types = { |
| 4875 | PTR_TO_STACK, |
| 4876 | PTR_TO_PACKET, |
| 4877 | PTR_TO_PACKET_META, |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 4878 | PTR_TO_MAP_KEY, |
Lorenz Bauer | f79e7ea | 2020-09-21 13:12:27 +0100 | [diff] [blame] | 4879 | PTR_TO_MAP_VALUE, |
| 4880 | }, |
| 4881 | }; |
| 4882 | |
| 4883 | static const struct bpf_reg_types fullsock_types = { .types = { PTR_TO_SOCKET } }; |
| 4884 | static const struct bpf_reg_types scalar_types = { .types = { SCALAR_VALUE } }; |
| 4885 | static const struct bpf_reg_types context_types = { .types = { PTR_TO_CTX } }; |
| 4886 | static const struct bpf_reg_types alloc_mem_types = { .types = { PTR_TO_MEM } }; |
| 4887 | static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } }; |
| 4888 | static const struct bpf_reg_types btf_ptr_types = { .types = { PTR_TO_BTF_ID } }; |
| 4889 | static const struct bpf_reg_types spin_lock_types = { .types = { PTR_TO_MAP_VALUE } }; |
Hao Luo | eaa6bcb | 2020-09-29 16:50:47 -0700 | [diff] [blame] | 4890 | static const struct bpf_reg_types percpu_btf_ptr_types = { .types = { PTR_TO_PERCPU_BTF_ID } }; |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 4891 | static const struct bpf_reg_types func_ptr_types = { .types = { PTR_TO_FUNC } }; |
| 4892 | static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK } }; |
Florent Revest | fff13c4 | 2021-04-19 17:52:39 +0200 | [diff] [blame] | 4893 | static const struct bpf_reg_types const_str_ptr_types = { .types = { PTR_TO_MAP_VALUE } }; |
Alexei Starovoitov | b00628b | 2021-07-14 17:54:09 -0700 | [diff] [blame] | 4894 | static const struct bpf_reg_types timer_types = { .types = { PTR_TO_MAP_VALUE } }; |
Lorenz Bauer | f79e7ea | 2020-09-21 13:12:27 +0100 | [diff] [blame] | 4895 | |
Lorenz Bauer | 0789e13b | 2020-09-23 17:01:55 +0100 | [diff] [blame] | 4896 | static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { |
Lorenz Bauer | f79e7ea | 2020-09-21 13:12:27 +0100 | [diff] [blame] | 4897 | [ARG_PTR_TO_MAP_KEY] = &map_key_value_types, |
| 4898 | [ARG_PTR_TO_MAP_VALUE] = &map_key_value_types, |
| 4899 | [ARG_PTR_TO_UNINIT_MAP_VALUE] = &map_key_value_types, |
| 4900 | [ARG_PTR_TO_MAP_VALUE_OR_NULL] = &map_key_value_types, |
| 4901 | [ARG_CONST_SIZE] = &scalar_types, |
| 4902 | [ARG_CONST_SIZE_OR_ZERO] = &scalar_types, |
| 4903 | [ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types, |
| 4904 | [ARG_CONST_MAP_PTR] = &const_map_ptr_types, |
| 4905 | [ARG_PTR_TO_CTX] = &context_types, |
| 4906 | [ARG_PTR_TO_CTX_OR_NULL] = &context_types, |
| 4907 | [ARG_PTR_TO_SOCK_COMMON] = &sock_types, |
Randy Dunlap | 49a2a4d | 2020-10-06 19:16:13 -0700 | [diff] [blame] | 4908 | #ifdef CONFIG_NET |
Martin KaFai Lau | 1df8f55 | 2020-09-24 17:03:50 -0700 | [diff] [blame] | 4909 | [ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types, |
Randy Dunlap | 49a2a4d | 2020-10-06 19:16:13 -0700 | [diff] [blame] | 4910 | #endif |
Lorenz Bauer | f79e7ea | 2020-09-21 13:12:27 +0100 | [diff] [blame] | 4911 | [ARG_PTR_TO_SOCKET] = &fullsock_types, |
| 4912 | [ARG_PTR_TO_SOCKET_OR_NULL] = &fullsock_types, |
| 4913 | [ARG_PTR_TO_BTF_ID] = &btf_ptr_types, |
| 4914 | [ARG_PTR_TO_SPIN_LOCK] = &spin_lock_types, |
| 4915 | [ARG_PTR_TO_MEM] = &mem_types, |
| 4916 | [ARG_PTR_TO_MEM_OR_NULL] = &mem_types, |
| 4917 | [ARG_PTR_TO_UNINIT_MEM] = &mem_types, |
| 4918 | [ARG_PTR_TO_ALLOC_MEM] = &alloc_mem_types, |
| 4919 | [ARG_PTR_TO_ALLOC_MEM_OR_NULL] = &alloc_mem_types, |
| 4920 | [ARG_PTR_TO_INT] = &int_ptr_types, |
| 4921 | [ARG_PTR_TO_LONG] = &int_ptr_types, |
Hao Luo | eaa6bcb | 2020-09-29 16:50:47 -0700 | [diff] [blame] | 4922 | [ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types, |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 4923 | [ARG_PTR_TO_FUNC] = &func_ptr_types, |
| 4924 | [ARG_PTR_TO_STACK_OR_NULL] = &stack_ptr_types, |
Florent Revest | fff13c4 | 2021-04-19 17:52:39 +0200 | [diff] [blame] | 4925 | [ARG_PTR_TO_CONST_STR] = &const_str_ptr_types, |
Alexei Starovoitov | b00628b | 2021-07-14 17:54:09 -0700 | [diff] [blame] | 4926 | [ARG_PTR_TO_TIMER] = &timer_types, |
Lorenz Bauer | f79e7ea | 2020-09-21 13:12:27 +0100 | [diff] [blame] | 4927 | }; |
| 4928 | |
| 4929 | static int check_reg_type(struct bpf_verifier_env *env, u32 regno, |
Martin KaFai Lau | a968d5e | 2020-09-24 17:03:44 -0700 | [diff] [blame] | 4930 | enum bpf_arg_type arg_type, |
| 4931 | const u32 *arg_btf_id) |
Lorenz Bauer | f79e7ea | 2020-09-21 13:12:27 +0100 | [diff] [blame] | 4932 | { |
| 4933 | struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; |
| 4934 | enum bpf_reg_type expected, type = reg->type; |
Martin KaFai Lau | a968d5e | 2020-09-24 17:03:44 -0700 | [diff] [blame] | 4935 | const struct bpf_reg_types *compatible; |
Lorenz Bauer | f79e7ea | 2020-09-21 13:12:27 +0100 | [diff] [blame] | 4936 | int i, j; |
| 4937 | |
Martin KaFai Lau | a968d5e | 2020-09-24 17:03:44 -0700 | [diff] [blame] | 4938 | compatible = compatible_reg_types[arg_type]; |
| 4939 | if (!compatible) { |
| 4940 | verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type); |
| 4941 | return -EFAULT; |
| 4942 | } |
| 4943 | |
Lorenz Bauer | f79e7ea | 2020-09-21 13:12:27 +0100 | [diff] [blame] | 4944 | for (i = 0; i < ARRAY_SIZE(compatible->types); i++) { |
| 4945 | expected = compatible->types[i]; |
| 4946 | if (expected == NOT_INIT) |
| 4947 | break; |
| 4948 | |
| 4949 | if (type == expected) |
Martin KaFai Lau | a968d5e | 2020-09-24 17:03:44 -0700 | [diff] [blame] | 4950 | goto found; |
Lorenz Bauer | f79e7ea | 2020-09-21 13:12:27 +0100 | [diff] [blame] | 4951 | } |
| 4952 | |
| 4953 | verbose(env, "R%d type=%s expected=", regno, reg_type_str[type]); |
| 4954 | for (j = 0; j + 1 < i; j++) |
| 4955 | verbose(env, "%s, ", reg_type_str[compatible->types[j]]); |
| 4956 | verbose(env, "%s\n", reg_type_str[compatible->types[j]]); |
| 4957 | return -EACCES; |
Martin KaFai Lau | a968d5e | 2020-09-24 17:03:44 -0700 | [diff] [blame] | 4958 | |
| 4959 | found: |
| 4960 | if (type == PTR_TO_BTF_ID) { |
Martin KaFai Lau | 1df8f55 | 2020-09-24 17:03:50 -0700 | [diff] [blame] | 4961 | if (!arg_btf_id) { |
| 4962 | if (!compatible->btf_id) { |
| 4963 | verbose(env, "verifier internal error: missing arg compatible BTF ID\n"); |
| 4964 | return -EFAULT; |
| 4965 | } |
| 4966 | arg_btf_id = compatible->btf_id; |
| 4967 | } |
| 4968 | |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 4969 | if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, |
| 4970 | btf_vmlinux, *arg_btf_id)) { |
Martin KaFai Lau | a968d5e | 2020-09-24 17:03:44 -0700 | [diff] [blame] | 4971 | verbose(env, "R%d is of type %s but %s is expected\n", |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 4972 | regno, kernel_type_name(reg->btf, reg->btf_id), |
| 4973 | kernel_type_name(btf_vmlinux, *arg_btf_id)); |
Martin KaFai Lau | a968d5e | 2020-09-24 17:03:44 -0700 | [diff] [blame] | 4974 | return -EACCES; |
| 4975 | } |
| 4976 | |
| 4977 | if (!tnum_is_const(reg->var_off) || reg->var_off.value) { |
| 4978 | verbose(env, "R%d is a pointer to in-kernel struct with non-zero offset\n", |
| 4979 | regno); |
| 4980 | return -EACCES; |
| 4981 | } |
| 4982 | } |
| 4983 | |
| 4984 | return 0; |
Lorenz Bauer | f79e7ea | 2020-09-21 13:12:27 +0100 | [diff] [blame] | 4985 | } |
| 4986 | |
Yonghong Song | af7ec13 | 2020-06-23 16:08:09 -0700 | [diff] [blame] | 4987 | static int check_func_arg(struct bpf_verifier_env *env, u32 arg, |
| 4988 | struct bpf_call_arg_meta *meta, |
| 4989 | const struct bpf_func_proto *fn) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4990 | { |
Yonghong Song | af7ec13 | 2020-06-23 16:08:09 -0700 | [diff] [blame] | 4991 | u32 regno = BPF_REG_1 + arg; |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 4992 | struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; |
Yonghong Song | af7ec13 | 2020-06-23 16:08:09 -0700 | [diff] [blame] | 4993 | enum bpf_arg_type arg_type = fn->arg_type[arg]; |
Lorenz Bauer | f79e7ea | 2020-09-21 13:12:27 +0100 | [diff] [blame] | 4994 | enum bpf_reg_type type = reg->type; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4995 | int err = 0; |
| 4996 | |
Daniel Borkmann | 80f1d68 | 2015-03-12 17:21:42 +0100 | [diff] [blame] | 4997 | if (arg_type == ARG_DONTCARE) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 4998 | return 0; |
| 4999 | |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 5000 | err = check_reg_arg(env, regno, SRC_OP); |
| 5001 | if (err) |
| 5002 | return err; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 5003 | |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 5004 | if (arg_type == ARG_ANYTHING) { |
| 5005 | if (is_pointer_value(env, regno)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5006 | verbose(env, "R%d leaks addr into helper function\n", |
| 5007 | regno); |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 5008 | return -EACCES; |
| 5009 | } |
Daniel Borkmann | 80f1d68 | 2015-03-12 17:21:42 +0100 | [diff] [blame] | 5010 | return 0; |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 5011 | } |
Daniel Borkmann | 80f1d68 | 2015-03-12 17:21:42 +0100 | [diff] [blame] | 5012 | |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 5013 | if (type_is_pkt_pointer(type) && |
Thomas Graf | 3a0af8f | 2016-11-30 17:10:10 +0100 | [diff] [blame] | 5014 | !may_access_direct_pkt_data(env, meta, BPF_READ)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5015 | verbose(env, "helper access to the packet is not allowed\n"); |
Alexei Starovoitov | 6841de8 | 2016-08-11 18:17:16 -0700 | [diff] [blame] | 5016 | return -EACCES; |
| 5017 | } |
| 5018 | |
Lorenz Bauer | 912f442 | 2020-08-21 11:29:46 +0100 | [diff] [blame] | 5019 | if (arg_type == ARG_PTR_TO_MAP_VALUE || |
| 5020 | arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE || |
| 5021 | arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL) { |
| 5022 | err = resolve_map_arg_type(env, meta, &arg_type); |
| 5023 | if (err) |
| 5024 | return err; |
| 5025 | } |
| 5026 | |
Lorenz Bauer | fd1b0d6 | 2020-09-21 13:12:26 +0100 | [diff] [blame] | 5027 | if (register_is_null(reg) && arg_type_may_be_null(arg_type)) |
| 5028 | /* A NULL register has a SCALAR_VALUE type, so skip |
| 5029 | * type checking. |
| 5030 | */ |
| 5031 | goto skip_type_check; |
Jiri Olsa | faaf4a7 | 2020-08-25 21:21:18 +0200 | [diff] [blame] | 5032 | |
Martin KaFai Lau | a968d5e | 2020-09-24 17:03:44 -0700 | [diff] [blame] | 5033 | err = check_reg_type(env, regno, arg_type, fn->arg_btf_id[arg]); |
Lorenz Bauer | f79e7ea | 2020-09-21 13:12:27 +0100 | [diff] [blame] | 5034 | if (err) |
| 5035 | return err; |
| 5036 | |
Martin KaFai Lau | a968d5e | 2020-09-24 17:03:44 -0700 | [diff] [blame] | 5037 | if (type == PTR_TO_CTX) { |
Lorenz Bauer | feec704 | 2020-09-21 13:12:23 +0100 | [diff] [blame] | 5038 | err = check_ctx_reg(env, reg, regno); |
| 5039 | if (err < 0) |
| 5040 | return err; |
Lorenz Bauer | d7b9454 | 2020-09-21 13:12:21 +0100 | [diff] [blame] | 5041 | } |
| 5042 | |
Lorenz Bauer | fd1b0d6 | 2020-09-21 13:12:26 +0100 | [diff] [blame] | 5043 | skip_type_check: |
Lorenz Bauer | 02f7c95 | 2020-09-21 13:12:22 +0100 | [diff] [blame] | 5044 | if (reg->ref_obj_id) { |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 5045 | if (meta->ref_obj_id) { |
| 5046 | verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", |
| 5047 | regno, reg->ref_obj_id, |
| 5048 | meta->ref_obj_id); |
| 5049 | return -EFAULT; |
| 5050 | } |
| 5051 | meta->ref_obj_id = reg->ref_obj_id; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 5052 | } |
| 5053 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 5054 | if (arg_type == ARG_CONST_MAP_PTR) { |
| 5055 | /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ |
Alexei Starovoitov | 3e8ce29 | 2021-07-14 17:54:11 -0700 | [diff] [blame] | 5056 | if (meta->map_ptr) { |
| 5057 | /* Use map_uid (which is unique id of inner map) to reject: |
| 5058 | * inner_map1 = bpf_map_lookup_elem(outer_map, key1) |
| 5059 | * inner_map2 = bpf_map_lookup_elem(outer_map, key2) |
| 5060 | * if (inner_map1 && inner_map2) { |
| 5061 | * timer = bpf_map_lookup_elem(inner_map1); |
| 5062 | * if (timer) |
| 5063 | * // mismatch would have been allowed |
| 5064 | * bpf_timer_init(timer, inner_map2); |
| 5065 | * } |
| 5066 | * |
| 5067 | * Comparing map_ptr is enough to distinguish normal and outer maps. |
| 5068 | */ |
| 5069 | if (meta->map_ptr != reg->map_ptr || |
| 5070 | meta->map_uid != reg->map_uid) { |
| 5071 | verbose(env, |
| 5072 | "timer pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n", |
| 5073 | meta->map_uid, reg->map_uid); |
| 5074 | return -EINVAL; |
| 5075 | } |
Alexei Starovoitov | b00628b | 2021-07-14 17:54:09 -0700 | [diff] [blame] | 5076 | } |
Daniel Borkmann | 33ff982 | 2016-04-13 00:10:50 +0200 | [diff] [blame] | 5077 | meta->map_ptr = reg->map_ptr; |
Alexei Starovoitov | 3e8ce29 | 2021-07-14 17:54:11 -0700 | [diff] [blame] | 5078 | meta->map_uid = reg->map_uid; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 5079 | } else if (arg_type == ARG_PTR_TO_MAP_KEY) { |
| 5080 | /* bpf_map_xxx(..., map_ptr, ..., key) call: |
| 5081 | * check that [key, key + map->key_size) are within |
| 5082 | * stack limits and initialized |
| 5083 | */ |
Daniel Borkmann | 33ff982 | 2016-04-13 00:10:50 +0200 | [diff] [blame] | 5084 | if (!meta->map_ptr) { |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 5085 | /* in function declaration map_ptr must come before |
| 5086 | * map_key, so that it's verified and known before |
| 5087 | * we have to check map_key here. Otherwise it means |
| 5088 | * that kernel subsystem misconfigured verifier |
| 5089 | */ |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5090 | verbose(env, "invalid map_ptr to access map->key\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 5091 | return -EACCES; |
| 5092 | } |
Paul Chaignon | d71962f | 2018-04-24 15:07:54 +0200 | [diff] [blame] | 5093 | err = check_helper_mem_access(env, regno, |
| 5094 | meta->map_ptr->key_size, false, |
| 5095 | NULL); |
Mauricio Vasquez B | 2ea864c | 2018-10-18 15:16:20 +0200 | [diff] [blame] | 5096 | } else if (arg_type == ARG_PTR_TO_MAP_VALUE || |
Martin KaFai Lau | 6ac99e8 | 2019-04-26 16:39:39 -0700 | [diff] [blame] | 5097 | (arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL && |
| 5098 | !register_is_null(reg)) || |
Mauricio Vasquez B | 2ea864c | 2018-10-18 15:16:20 +0200 | [diff] [blame] | 5099 | arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) { |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 5100 | /* bpf_map_xxx(..., map_ptr, ..., value) call: |
| 5101 | * check [value, value + map->value_size) validity |
| 5102 | */ |
Daniel Borkmann | 33ff982 | 2016-04-13 00:10:50 +0200 | [diff] [blame] | 5103 | if (!meta->map_ptr) { |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 5104 | /* kernel subsystem misconfigured verifier */ |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5105 | verbose(env, "invalid map_ptr to access map->value\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 5106 | return -EACCES; |
| 5107 | } |
Mauricio Vasquez B | 2ea864c | 2018-10-18 15:16:20 +0200 | [diff] [blame] | 5108 | meta->raw_mode = (arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE); |
Paul Chaignon | d71962f | 2018-04-24 15:07:54 +0200 | [diff] [blame] | 5109 | err = check_helper_mem_access(env, regno, |
| 5110 | meta->map_ptr->value_size, false, |
Mauricio Vasquez B | 2ea864c | 2018-10-18 15:16:20 +0200 | [diff] [blame] | 5111 | meta); |
Hao Luo | eaa6bcb | 2020-09-29 16:50:47 -0700 | [diff] [blame] | 5112 | } else if (arg_type == ARG_PTR_TO_PERCPU_BTF_ID) { |
| 5113 | if (!reg->btf_id) { |
| 5114 | verbose(env, "Helper has invalid btf_id in R%d\n", regno); |
| 5115 | return -EACCES; |
| 5116 | } |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 5117 | meta->ret_btf = reg->btf; |
Hao Luo | eaa6bcb | 2020-09-29 16:50:47 -0700 | [diff] [blame] | 5118 | meta->ret_btf_id = reg->btf_id; |
Lorenz Bauer | c18f0b6 | 2020-09-21 13:12:25 +0100 | [diff] [blame] | 5119 | } else if (arg_type == ARG_PTR_TO_SPIN_LOCK) { |
| 5120 | if (meta->func_id == BPF_FUNC_spin_lock) { |
| 5121 | if (process_spin_lock(env, regno, true)) |
| 5122 | return -EACCES; |
| 5123 | } else if (meta->func_id == BPF_FUNC_spin_unlock) { |
| 5124 | if (process_spin_lock(env, regno, false)) |
| 5125 | return -EACCES; |
| 5126 | } else { |
| 5127 | verbose(env, "verifier internal error\n"); |
| 5128 | return -EFAULT; |
| 5129 | } |
Alexei Starovoitov | b00628b | 2021-07-14 17:54:09 -0700 | [diff] [blame] | 5130 | } else if (arg_type == ARG_PTR_TO_TIMER) { |
| 5131 | if (process_timer_func(env, regno, meta)) |
| 5132 | return -EACCES; |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 5133 | } else if (arg_type == ARG_PTR_TO_FUNC) { |
| 5134 | meta->subprogno = reg->subprogno; |
Lorenz Bauer | a2bbe7c | 2020-09-21 13:12:24 +0100 | [diff] [blame] | 5135 | } else if (arg_type_is_mem_ptr(arg_type)) { |
| 5136 | /* The access to this pointer is only checked when we hit the |
| 5137 | * next is_mem_size argument below. |
| 5138 | */ |
| 5139 | meta->raw_mode = (arg_type == ARG_PTR_TO_UNINIT_MEM); |
Daniel Borkmann | 9013341 | 2018-01-20 01:24:29 +0100 | [diff] [blame] | 5140 | } else if (arg_type_is_mem_size(arg_type)) { |
Alexei Starovoitov | 39f19ebb | 2017-01-09 10:19:50 -0800 | [diff] [blame] | 5141 | bool zero_size_allowed = (arg_type == ARG_CONST_SIZE_OR_ZERO); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 5142 | |
John Fastabend | 1006050 | 2020-03-30 14:36:19 -0700 | [diff] [blame] | 5143 | /* This is used to refine r0 return value bounds for helpers |
| 5144 | * that enforce this value as an upper bound on return values. |
| 5145 | * See do_refine_retval_range() for helpers that can refine |
| 5146 | * the return value. C type of helper is u32 so we pull register |
| 5147 | * bound from umax_value however, if negative verifier errors |
| 5148 | * out. Only upper bounds can be learned because retval is an |
| 5149 | * int type and negative retvals are allowed. |
Yonghong Song | 849fa50 | 2018-04-28 22:28:09 -0700 | [diff] [blame] | 5150 | */ |
John Fastabend | 1006050 | 2020-03-30 14:36:19 -0700 | [diff] [blame] | 5151 | meta->msize_max_value = reg->umax_value; |
Yonghong Song | 849fa50 | 2018-04-28 22:28:09 -0700 | [diff] [blame] | 5152 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 5153 | /* The register is SCALAR_VALUE; the access check |
| 5154 | * happens using its boundaries. |
Gianluca Borello | 06c1c04 | 2017-01-09 10:19:49 -0800 | [diff] [blame] | 5155 | */ |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 5156 | if (!tnum_is_const(reg->var_off)) |
Gianluca Borello | 06c1c04 | 2017-01-09 10:19:49 -0800 | [diff] [blame] | 5157 | /* For unprivileged variable accesses, disable raw |
| 5158 | * mode so that the program is required to |
| 5159 | * initialize all the memory that the helper could |
| 5160 | * just partially fill up. |
| 5161 | */ |
| 5162 | meta = NULL; |
| 5163 | |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 5164 | if (reg->smin_value < 0) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5165 | verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n", |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 5166 | regno); |
| 5167 | return -EACCES; |
| 5168 | } |
Gianluca Borello | 06c1c04 | 2017-01-09 10:19:49 -0800 | [diff] [blame] | 5169 | |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 5170 | if (reg->umin_value == 0) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 5171 | err = check_helper_mem_access(env, regno - 1, 0, |
| 5172 | zero_size_allowed, |
| 5173 | meta); |
Gianluca Borello | 06c1c04 | 2017-01-09 10:19:49 -0800 | [diff] [blame] | 5174 | if (err) |
| 5175 | return err; |
Gianluca Borello | 06c1c04 | 2017-01-09 10:19:49 -0800 | [diff] [blame] | 5176 | } |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 5177 | |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 5178 | if (reg->umax_value >= BPF_MAX_VAR_SIZ) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5179 | verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n", |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 5180 | regno); |
| 5181 | return -EACCES; |
| 5182 | } |
| 5183 | err = check_helper_mem_access(env, regno - 1, |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 5184 | reg->umax_value, |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 5185 | zero_size_allowed, meta); |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 5186 | if (!err) |
| 5187 | err = mark_chain_precision(env, regno); |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 5188 | } else if (arg_type_is_alloc_size(arg_type)) { |
| 5189 | if (!tnum_is_const(reg->var_off)) { |
Brendan Jackman | 28a8add | 2021-01-12 12:39:13 +0000 | [diff] [blame] | 5190 | verbose(env, "R%d is not a known constant'\n", |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 5191 | regno); |
| 5192 | return -EACCES; |
| 5193 | } |
| 5194 | meta->mem_size = reg->var_off.value; |
Andrey Ignatov | 57c3bb7 | 2019-03-18 16:57:10 -0700 | [diff] [blame] | 5195 | } else if (arg_type_is_int_ptr(arg_type)) { |
| 5196 | int size = int_ptr_type_to_size(arg_type); |
| 5197 | |
| 5198 | err = check_helper_mem_access(env, regno, size, false, meta); |
| 5199 | if (err) |
| 5200 | return err; |
| 5201 | err = check_ptr_alignment(env, reg, 0, size, true); |
Florent Revest | fff13c4 | 2021-04-19 17:52:39 +0200 | [diff] [blame] | 5202 | } else if (arg_type == ARG_PTR_TO_CONST_STR) { |
| 5203 | struct bpf_map *map = reg->map_ptr; |
| 5204 | int map_off; |
| 5205 | u64 map_addr; |
| 5206 | char *str_ptr; |
| 5207 | |
Florent Revest | a8fad73 | 2021-04-23 01:55:43 +0200 | [diff] [blame] | 5208 | if (!bpf_map_is_rdonly(map)) { |
Florent Revest | fff13c4 | 2021-04-19 17:52:39 +0200 | [diff] [blame] | 5209 | verbose(env, "R%d does not point to a readonly map'\n", regno); |
| 5210 | return -EACCES; |
| 5211 | } |
| 5212 | |
| 5213 | if (!tnum_is_const(reg->var_off)) { |
| 5214 | verbose(env, "R%d is not a constant address'\n", regno); |
| 5215 | return -EACCES; |
| 5216 | } |
| 5217 | |
| 5218 | if (!map->ops->map_direct_value_addr) { |
| 5219 | verbose(env, "no direct value access support for this map type\n"); |
| 5220 | return -EACCES; |
| 5221 | } |
| 5222 | |
| 5223 | err = check_map_access(env, regno, reg->off, |
| 5224 | map->value_size - reg->off, false); |
| 5225 | if (err) |
| 5226 | return err; |
| 5227 | |
| 5228 | map_off = reg->off + reg->var_off.value; |
| 5229 | err = map->ops->map_direct_value_addr(map, &map_addr, map_off); |
| 5230 | if (err) { |
| 5231 | verbose(env, "direct value access on string failed\n"); |
| 5232 | return err; |
| 5233 | } |
| 5234 | |
| 5235 | str_ptr = (char *)(long)(map_addr); |
| 5236 | if (!strnchr(str_ptr + map_off, map->value_size - map_off, 0)) { |
| 5237 | verbose(env, "string is not zero-terminated\n"); |
| 5238 | return -EINVAL; |
| 5239 | } |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 5240 | } |
| 5241 | |
| 5242 | return err; |
| 5243 | } |
| 5244 | |
Lorenz Bauer | 0126240 | 2020-08-21 11:29:47 +0100 | [diff] [blame] | 5245 | static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) |
| 5246 | { |
| 5247 | enum bpf_attach_type eatype = env->prog->expected_attach_type; |
Udip Pant | 7e40781 | 2020-08-25 16:20:00 -0700 | [diff] [blame] | 5248 | enum bpf_prog_type type = resolve_prog_type(env->prog); |
Lorenz Bauer | 0126240 | 2020-08-21 11:29:47 +0100 | [diff] [blame] | 5249 | |
| 5250 | if (func_id != BPF_FUNC_map_update_elem) |
| 5251 | return false; |
| 5252 | |
| 5253 | /* It's not possible to get access to a locked struct sock in these |
| 5254 | * contexts, so updating is safe. |
| 5255 | */ |
| 5256 | switch (type) { |
| 5257 | case BPF_PROG_TYPE_TRACING: |
| 5258 | if (eatype == BPF_TRACE_ITER) |
| 5259 | return true; |
| 5260 | break; |
| 5261 | case BPF_PROG_TYPE_SOCKET_FILTER: |
| 5262 | case BPF_PROG_TYPE_SCHED_CLS: |
| 5263 | case BPF_PROG_TYPE_SCHED_ACT: |
| 5264 | case BPF_PROG_TYPE_XDP: |
| 5265 | case BPF_PROG_TYPE_SK_REUSEPORT: |
| 5266 | case BPF_PROG_TYPE_FLOW_DISSECTOR: |
| 5267 | case BPF_PROG_TYPE_SK_LOOKUP: |
| 5268 | return true; |
| 5269 | default: |
| 5270 | break; |
| 5271 | } |
| 5272 | |
| 5273 | verbose(env, "cannot update sockmap in this context\n"); |
| 5274 | return false; |
| 5275 | } |
| 5276 | |
Maciej Fijalkowski | e411901 | 2020-09-16 23:10:09 +0200 | [diff] [blame] | 5277 | static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env) |
| 5278 | { |
| 5279 | return env->prog->jit_requested && IS_ENABLED(CONFIG_X86_64); |
| 5280 | } |
| 5281 | |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5282 | static int check_map_func_compatibility(struct bpf_verifier_env *env, |
| 5283 | struct bpf_map *map, int func_id) |
Kaixu Xia | 35578d7 | 2015-08-06 07:02:35 +0000 | [diff] [blame] | 5284 | { |
Kaixu Xia | 35578d7 | 2015-08-06 07:02:35 +0000 | [diff] [blame] | 5285 | if (!map) |
| 5286 | return 0; |
| 5287 | |
Alexei Starovoitov | 6aff67c | 2016-04-27 18:56:21 -0700 | [diff] [blame] | 5288 | /* We need a two way check, first is from map perspective ... */ |
| 5289 | switch (map->map_type) { |
| 5290 | case BPF_MAP_TYPE_PROG_ARRAY: |
| 5291 | if (func_id != BPF_FUNC_tail_call) |
| 5292 | goto error; |
| 5293 | break; |
| 5294 | case BPF_MAP_TYPE_PERF_EVENT_ARRAY: |
| 5295 | if (func_id != BPF_FUNC_perf_event_read && |
Yonghong Song | 908432c | 2017-10-05 09:19:20 -0700 | [diff] [blame] | 5296 | func_id != BPF_FUNC_perf_event_output && |
Alexei Starovoitov | a7658e1 | 2019-10-15 20:25:04 -0700 | [diff] [blame] | 5297 | func_id != BPF_FUNC_skb_output && |
Eelco Chaudron | d831ee8 | 2020-03-06 08:59:23 +0000 | [diff] [blame] | 5298 | func_id != BPF_FUNC_perf_event_read_value && |
| 5299 | func_id != BPF_FUNC_xdp_output) |
Alexei Starovoitov | 6aff67c | 2016-04-27 18:56:21 -0700 | [diff] [blame] | 5300 | goto error; |
| 5301 | break; |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 5302 | case BPF_MAP_TYPE_RINGBUF: |
| 5303 | if (func_id != BPF_FUNC_ringbuf_output && |
| 5304 | func_id != BPF_FUNC_ringbuf_reserve && |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 5305 | func_id != BPF_FUNC_ringbuf_query) |
| 5306 | goto error; |
| 5307 | break; |
Alexei Starovoitov | 6aff67c | 2016-04-27 18:56:21 -0700 | [diff] [blame] | 5308 | case BPF_MAP_TYPE_STACK_TRACE: |
| 5309 | if (func_id != BPF_FUNC_get_stackid) |
| 5310 | goto error; |
| 5311 | break; |
Martin KaFai Lau | 4ed8ec5 | 2016-06-30 10:28:43 -0700 | [diff] [blame] | 5312 | case BPF_MAP_TYPE_CGROUP_ARRAY: |
David S. Miller | 60747ef | 2016-08-18 01:17:32 -0400 | [diff] [blame] | 5313 | if (func_id != BPF_FUNC_skb_under_cgroup && |
Sargun Dhillon | 60d20f9 | 2016-08-12 08:56:52 -0700 | [diff] [blame] | 5314 | func_id != BPF_FUNC_current_task_under_cgroup) |
Martin KaFai Lau | 4a482f3 | 2016-06-30 10:28:44 -0700 | [diff] [blame] | 5315 | goto error; |
| 5316 | break; |
Roman Gushchin | cd33943 | 2018-08-02 14:27:24 -0700 | [diff] [blame] | 5317 | case BPF_MAP_TYPE_CGROUP_STORAGE: |
Roman Gushchin | b741f16 | 2018-09-28 14:45:43 +0000 | [diff] [blame] | 5318 | case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: |
Roman Gushchin | cd33943 | 2018-08-02 14:27:24 -0700 | [diff] [blame] | 5319 | if (func_id != BPF_FUNC_get_local_storage) |
| 5320 | goto error; |
| 5321 | break; |
John Fastabend | 546ac1f | 2017-07-17 09:28:56 -0700 | [diff] [blame] | 5322 | case BPF_MAP_TYPE_DEVMAP: |
Toke Høiland-Jørgensen | 6f9d451 | 2019-07-26 18:06:55 +0200 | [diff] [blame] | 5323 | case BPF_MAP_TYPE_DEVMAP_HASH: |
Toke Høiland-Jørgensen | 0cdbb4b | 2019-06-28 11:12:35 +0200 | [diff] [blame] | 5324 | if (func_id != BPF_FUNC_redirect_map && |
| 5325 | func_id != BPF_FUNC_map_lookup_elem) |
John Fastabend | 546ac1f | 2017-07-17 09:28:56 -0700 | [diff] [blame] | 5326 | goto error; |
| 5327 | break; |
Björn Töpel | fbfc504a | 2018-05-02 13:01:28 +0200 | [diff] [blame] | 5328 | /* Restrict bpf side of cpumap and xskmap, open when use-cases |
| 5329 | * appear. |
| 5330 | */ |
Jesper Dangaard Brouer | 6710e11 | 2017-10-16 12:19:28 +0200 | [diff] [blame] | 5331 | case BPF_MAP_TYPE_CPUMAP: |
| 5332 | if (func_id != BPF_FUNC_redirect_map) |
| 5333 | goto error; |
| 5334 | break; |
Jonathan Lemon | fada7fd | 2019-06-06 13:59:40 -0700 | [diff] [blame] | 5335 | case BPF_MAP_TYPE_XSKMAP: |
| 5336 | if (func_id != BPF_FUNC_redirect_map && |
| 5337 | func_id != BPF_FUNC_map_lookup_elem) |
| 5338 | goto error; |
| 5339 | break; |
Martin KaFai Lau | 56f668d | 2017-03-22 10:00:33 -0700 | [diff] [blame] | 5340 | case BPF_MAP_TYPE_ARRAY_OF_MAPS: |
Martin KaFai Lau | bcc6b1b | 2017-03-22 10:00:34 -0700 | [diff] [blame] | 5341 | case BPF_MAP_TYPE_HASH_OF_MAPS: |
Martin KaFai Lau | 56f668d | 2017-03-22 10:00:33 -0700 | [diff] [blame] | 5342 | if (func_id != BPF_FUNC_map_lookup_elem) |
| 5343 | goto error; |
Martin KaFai Lau | 16a4362 | 2017-08-17 18:14:43 -0700 | [diff] [blame] | 5344 | break; |
John Fastabend | 174a79f | 2017-08-15 22:32:47 -0700 | [diff] [blame] | 5345 | case BPF_MAP_TYPE_SOCKMAP: |
| 5346 | if (func_id != BPF_FUNC_sk_redirect_map && |
| 5347 | func_id != BPF_FUNC_sock_map_update && |
John Fastabend | 4f738ad | 2018-03-18 12:57:10 -0700 | [diff] [blame] | 5348 | func_id != BPF_FUNC_map_delete_elem && |
Jakub Sitnicki | 9fed900 | 2020-02-18 17:10:20 +0000 | [diff] [blame] | 5349 | func_id != BPF_FUNC_msg_redirect_map && |
Jakub Sitnicki | 64d8529 | 2020-04-29 20:11:52 +0200 | [diff] [blame] | 5350 | func_id != BPF_FUNC_sk_select_reuseport && |
Lorenz Bauer | 0126240 | 2020-08-21 11:29:47 +0100 | [diff] [blame] | 5351 | func_id != BPF_FUNC_map_lookup_elem && |
| 5352 | !may_update_sockmap(env, func_id)) |
John Fastabend | 174a79f | 2017-08-15 22:32:47 -0700 | [diff] [blame] | 5353 | goto error; |
| 5354 | break; |
John Fastabend | 8111038 | 2018-05-14 10:00:17 -0700 | [diff] [blame] | 5355 | case BPF_MAP_TYPE_SOCKHASH: |
| 5356 | if (func_id != BPF_FUNC_sk_redirect_hash && |
| 5357 | func_id != BPF_FUNC_sock_hash_update && |
| 5358 | func_id != BPF_FUNC_map_delete_elem && |
Jakub Sitnicki | 9fed900 | 2020-02-18 17:10:20 +0000 | [diff] [blame] | 5359 | func_id != BPF_FUNC_msg_redirect_hash && |
Jakub Sitnicki | 64d8529 | 2020-04-29 20:11:52 +0200 | [diff] [blame] | 5360 | func_id != BPF_FUNC_sk_select_reuseport && |
Lorenz Bauer | 0126240 | 2020-08-21 11:29:47 +0100 | [diff] [blame] | 5361 | func_id != BPF_FUNC_map_lookup_elem && |
| 5362 | !may_update_sockmap(env, func_id)) |
John Fastabend | 8111038 | 2018-05-14 10:00:17 -0700 | [diff] [blame] | 5363 | goto error; |
| 5364 | break; |
Martin KaFai Lau | 2dbb9b9 | 2018-08-08 01:01:25 -0700 | [diff] [blame] | 5365 | case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: |
| 5366 | if (func_id != BPF_FUNC_sk_select_reuseport) |
| 5367 | goto error; |
| 5368 | break; |
Mauricio Vasquez B | f1a2e44 | 2018-10-18 15:16:25 +0200 | [diff] [blame] | 5369 | case BPF_MAP_TYPE_QUEUE: |
| 5370 | case BPF_MAP_TYPE_STACK: |
| 5371 | if (func_id != BPF_FUNC_map_peek_elem && |
| 5372 | func_id != BPF_FUNC_map_pop_elem && |
| 5373 | func_id != BPF_FUNC_map_push_elem) |
| 5374 | goto error; |
| 5375 | break; |
Martin KaFai Lau | 6ac99e8 | 2019-04-26 16:39:39 -0700 | [diff] [blame] | 5376 | case BPF_MAP_TYPE_SK_STORAGE: |
| 5377 | if (func_id != BPF_FUNC_sk_storage_get && |
| 5378 | func_id != BPF_FUNC_sk_storage_delete) |
| 5379 | goto error; |
| 5380 | break; |
KP Singh | 8ea6368 | 2020-08-25 20:29:17 +0200 | [diff] [blame] | 5381 | case BPF_MAP_TYPE_INODE_STORAGE: |
| 5382 | if (func_id != BPF_FUNC_inode_storage_get && |
| 5383 | func_id != BPF_FUNC_inode_storage_delete) |
| 5384 | goto error; |
| 5385 | break; |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 5386 | case BPF_MAP_TYPE_TASK_STORAGE: |
| 5387 | if (func_id != BPF_FUNC_task_storage_get && |
| 5388 | func_id != BPF_FUNC_task_storage_delete) |
| 5389 | goto error; |
| 5390 | break; |
Alexei Starovoitov | 6aff67c | 2016-04-27 18:56:21 -0700 | [diff] [blame] | 5391 | default: |
| 5392 | break; |
| 5393 | } |
| 5394 | |
| 5395 | /* ... and second from the function itself. */ |
| 5396 | switch (func_id) { |
| 5397 | case BPF_FUNC_tail_call: |
| 5398 | if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY) |
| 5399 | goto error; |
Maciej Fijalkowski | e411901 | 2020-09-16 23:10:09 +0200 | [diff] [blame] | 5400 | if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) { |
| 5401 | verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5402 | return -EINVAL; |
| 5403 | } |
Alexei Starovoitov | 6aff67c | 2016-04-27 18:56:21 -0700 | [diff] [blame] | 5404 | break; |
| 5405 | case BPF_FUNC_perf_event_read: |
| 5406 | case BPF_FUNC_perf_event_output: |
Yonghong Song | 908432c | 2017-10-05 09:19:20 -0700 | [diff] [blame] | 5407 | case BPF_FUNC_perf_event_read_value: |
Alexei Starovoitov | a7658e1 | 2019-10-15 20:25:04 -0700 | [diff] [blame] | 5408 | case BPF_FUNC_skb_output: |
Eelco Chaudron | d831ee8 | 2020-03-06 08:59:23 +0000 | [diff] [blame] | 5409 | case BPF_FUNC_xdp_output: |
Alexei Starovoitov | 6aff67c | 2016-04-27 18:56:21 -0700 | [diff] [blame] | 5410 | if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) |
| 5411 | goto error; |
| 5412 | break; |
Daniel Borkmann | 5b029a3 | 2021-08-23 21:02:09 +0200 | [diff] [blame] | 5413 | case BPF_FUNC_ringbuf_output: |
| 5414 | case BPF_FUNC_ringbuf_reserve: |
| 5415 | case BPF_FUNC_ringbuf_query: |
| 5416 | if (map->map_type != BPF_MAP_TYPE_RINGBUF) |
| 5417 | goto error; |
| 5418 | break; |
Alexei Starovoitov | 6aff67c | 2016-04-27 18:56:21 -0700 | [diff] [blame] | 5419 | case BPF_FUNC_get_stackid: |
| 5420 | if (map->map_type != BPF_MAP_TYPE_STACK_TRACE) |
| 5421 | goto error; |
| 5422 | break; |
Sargun Dhillon | 60d20f9 | 2016-08-12 08:56:52 -0700 | [diff] [blame] | 5423 | case BPF_FUNC_current_task_under_cgroup: |
Daniel Borkmann | 747ea55 | 2016-08-12 22:17:17 +0200 | [diff] [blame] | 5424 | case BPF_FUNC_skb_under_cgroup: |
Martin KaFai Lau | 4a482f3 | 2016-06-30 10:28:44 -0700 | [diff] [blame] | 5425 | if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY) |
| 5426 | goto error; |
| 5427 | break; |
John Fastabend | 97f91a7 | 2017-07-17 09:29:18 -0700 | [diff] [blame] | 5428 | case BPF_FUNC_redirect_map: |
Jesper Dangaard Brouer | 9c270af | 2017-10-16 12:19:34 +0200 | [diff] [blame] | 5429 | if (map->map_type != BPF_MAP_TYPE_DEVMAP && |
Toke Høiland-Jørgensen | 6f9d451 | 2019-07-26 18:06:55 +0200 | [diff] [blame] | 5430 | map->map_type != BPF_MAP_TYPE_DEVMAP_HASH && |
Björn Töpel | fbfc504a | 2018-05-02 13:01:28 +0200 | [diff] [blame] | 5431 | map->map_type != BPF_MAP_TYPE_CPUMAP && |
| 5432 | map->map_type != BPF_MAP_TYPE_XSKMAP) |
John Fastabend | 97f91a7 | 2017-07-17 09:29:18 -0700 | [diff] [blame] | 5433 | goto error; |
| 5434 | break; |
John Fastabend | 174a79f | 2017-08-15 22:32:47 -0700 | [diff] [blame] | 5435 | case BPF_FUNC_sk_redirect_map: |
John Fastabend | 4f738ad | 2018-03-18 12:57:10 -0700 | [diff] [blame] | 5436 | case BPF_FUNC_msg_redirect_map: |
John Fastabend | 8111038 | 2018-05-14 10:00:17 -0700 | [diff] [blame] | 5437 | case BPF_FUNC_sock_map_update: |
John Fastabend | 174a79f | 2017-08-15 22:32:47 -0700 | [diff] [blame] | 5438 | if (map->map_type != BPF_MAP_TYPE_SOCKMAP) |
| 5439 | goto error; |
| 5440 | break; |
John Fastabend | 8111038 | 2018-05-14 10:00:17 -0700 | [diff] [blame] | 5441 | case BPF_FUNC_sk_redirect_hash: |
| 5442 | case BPF_FUNC_msg_redirect_hash: |
| 5443 | case BPF_FUNC_sock_hash_update: |
| 5444 | if (map->map_type != BPF_MAP_TYPE_SOCKHASH) |
John Fastabend | 174a79f | 2017-08-15 22:32:47 -0700 | [diff] [blame] | 5445 | goto error; |
| 5446 | break; |
Roman Gushchin | cd33943 | 2018-08-02 14:27:24 -0700 | [diff] [blame] | 5447 | case BPF_FUNC_get_local_storage: |
Roman Gushchin | b741f16 | 2018-09-28 14:45:43 +0000 | [diff] [blame] | 5448 | if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && |
| 5449 | map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) |
Roman Gushchin | cd33943 | 2018-08-02 14:27:24 -0700 | [diff] [blame] | 5450 | goto error; |
| 5451 | break; |
Martin KaFai Lau | 2dbb9b9 | 2018-08-08 01:01:25 -0700 | [diff] [blame] | 5452 | case BPF_FUNC_sk_select_reuseport: |
Jakub Sitnicki | 9fed900 | 2020-02-18 17:10:20 +0000 | [diff] [blame] | 5453 | if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY && |
| 5454 | map->map_type != BPF_MAP_TYPE_SOCKMAP && |
| 5455 | map->map_type != BPF_MAP_TYPE_SOCKHASH) |
Martin KaFai Lau | 2dbb9b9 | 2018-08-08 01:01:25 -0700 | [diff] [blame] | 5456 | goto error; |
| 5457 | break; |
Mauricio Vasquez B | f1a2e44 | 2018-10-18 15:16:25 +0200 | [diff] [blame] | 5458 | case BPF_FUNC_map_peek_elem: |
| 5459 | case BPF_FUNC_map_pop_elem: |
| 5460 | case BPF_FUNC_map_push_elem: |
| 5461 | if (map->map_type != BPF_MAP_TYPE_QUEUE && |
| 5462 | map->map_type != BPF_MAP_TYPE_STACK) |
| 5463 | goto error; |
| 5464 | break; |
Martin KaFai Lau | 6ac99e8 | 2019-04-26 16:39:39 -0700 | [diff] [blame] | 5465 | case BPF_FUNC_sk_storage_get: |
| 5466 | case BPF_FUNC_sk_storage_delete: |
| 5467 | if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) |
| 5468 | goto error; |
| 5469 | break; |
KP Singh | 8ea6368 | 2020-08-25 20:29:17 +0200 | [diff] [blame] | 5470 | case BPF_FUNC_inode_storage_get: |
| 5471 | case BPF_FUNC_inode_storage_delete: |
| 5472 | if (map->map_type != BPF_MAP_TYPE_INODE_STORAGE) |
| 5473 | goto error; |
| 5474 | break; |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 5475 | case BPF_FUNC_task_storage_get: |
| 5476 | case BPF_FUNC_task_storage_delete: |
| 5477 | if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE) |
| 5478 | goto error; |
| 5479 | break; |
Alexei Starovoitov | 6aff67c | 2016-04-27 18:56:21 -0700 | [diff] [blame] | 5480 | default: |
| 5481 | break; |
Kaixu Xia | 35578d7 | 2015-08-06 07:02:35 +0000 | [diff] [blame] | 5482 | } |
| 5483 | |
| 5484 | return 0; |
Alexei Starovoitov | 6aff67c | 2016-04-27 18:56:21 -0700 | [diff] [blame] | 5485 | error: |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5486 | verbose(env, "cannot pass map_type %d into func %s#%d\n", |
Thomas Graf | ebb676d | 2016-10-27 11:23:51 +0200 | [diff] [blame] | 5487 | map->map_type, func_id_name(func_id), func_id); |
Alexei Starovoitov | 6aff67c | 2016-04-27 18:56:21 -0700 | [diff] [blame] | 5488 | return -EINVAL; |
Kaixu Xia | 35578d7 | 2015-08-06 07:02:35 +0000 | [diff] [blame] | 5489 | } |
| 5490 | |
Daniel Borkmann | 9013341 | 2018-01-20 01:24:29 +0100 | [diff] [blame] | 5491 | static bool check_raw_mode_ok(const struct bpf_func_proto *fn) |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 5492 | { |
| 5493 | int count = 0; |
| 5494 | |
Alexei Starovoitov | 39f19ebb | 2017-01-09 10:19:50 -0800 | [diff] [blame] | 5495 | if (fn->arg1_type == ARG_PTR_TO_UNINIT_MEM) |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 5496 | count++; |
Alexei Starovoitov | 39f19ebb | 2017-01-09 10:19:50 -0800 | [diff] [blame] | 5497 | if (fn->arg2_type == ARG_PTR_TO_UNINIT_MEM) |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 5498 | count++; |
Alexei Starovoitov | 39f19ebb | 2017-01-09 10:19:50 -0800 | [diff] [blame] | 5499 | if (fn->arg3_type == ARG_PTR_TO_UNINIT_MEM) |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 5500 | count++; |
Alexei Starovoitov | 39f19ebb | 2017-01-09 10:19:50 -0800 | [diff] [blame] | 5501 | if (fn->arg4_type == ARG_PTR_TO_UNINIT_MEM) |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 5502 | count++; |
Alexei Starovoitov | 39f19ebb | 2017-01-09 10:19:50 -0800 | [diff] [blame] | 5503 | if (fn->arg5_type == ARG_PTR_TO_UNINIT_MEM) |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 5504 | count++; |
| 5505 | |
Daniel Borkmann | 9013341 | 2018-01-20 01:24:29 +0100 | [diff] [blame] | 5506 | /* We only support one arg being in raw mode at the moment, |
| 5507 | * which is sufficient for the helper functions we have |
| 5508 | * right now. |
| 5509 | */ |
| 5510 | return count <= 1; |
| 5511 | } |
| 5512 | |
| 5513 | static bool check_args_pair_invalid(enum bpf_arg_type arg_curr, |
| 5514 | enum bpf_arg_type arg_next) |
| 5515 | { |
| 5516 | return (arg_type_is_mem_ptr(arg_curr) && |
| 5517 | !arg_type_is_mem_size(arg_next)) || |
| 5518 | (!arg_type_is_mem_ptr(arg_curr) && |
| 5519 | arg_type_is_mem_size(arg_next)); |
| 5520 | } |
| 5521 | |
| 5522 | static bool check_arg_pair_ok(const struct bpf_func_proto *fn) |
| 5523 | { |
| 5524 | /* bpf_xxx(..., buf, len) call will access 'len' |
| 5525 | * bytes from memory 'buf'. Both arg types need |
| 5526 | * to be paired, so make sure there's no buggy |
| 5527 | * helper function specification. |
| 5528 | */ |
| 5529 | if (arg_type_is_mem_size(fn->arg1_type) || |
| 5530 | arg_type_is_mem_ptr(fn->arg5_type) || |
| 5531 | check_args_pair_invalid(fn->arg1_type, fn->arg2_type) || |
| 5532 | check_args_pair_invalid(fn->arg2_type, fn->arg3_type) || |
| 5533 | check_args_pair_invalid(fn->arg3_type, fn->arg4_type) || |
| 5534 | check_args_pair_invalid(fn->arg4_type, fn->arg5_type)) |
| 5535 | return false; |
| 5536 | |
| 5537 | return true; |
| 5538 | } |
| 5539 | |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 5540 | static bool check_refcount_ok(const struct bpf_func_proto *fn, int func_id) |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 5541 | { |
| 5542 | int count = 0; |
| 5543 | |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 5544 | if (arg_type_may_be_refcounted(fn->arg1_type)) |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 5545 | count++; |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 5546 | if (arg_type_may_be_refcounted(fn->arg2_type)) |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 5547 | count++; |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 5548 | if (arg_type_may_be_refcounted(fn->arg3_type)) |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 5549 | count++; |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 5550 | if (arg_type_may_be_refcounted(fn->arg4_type)) |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 5551 | count++; |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 5552 | if (arg_type_may_be_refcounted(fn->arg5_type)) |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 5553 | count++; |
| 5554 | |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 5555 | /* A reference acquiring function cannot acquire |
| 5556 | * another refcounted ptr. |
| 5557 | */ |
Jakub Sitnicki | 64d8529 | 2020-04-29 20:11:52 +0200 | [diff] [blame] | 5558 | if (may_be_acquire_function(func_id) && count) |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 5559 | return false; |
| 5560 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 5561 | /* We only support one arg being unreferenced at the moment, |
| 5562 | * which is sufficient for the helper functions we have right now. |
| 5563 | */ |
| 5564 | return count <= 1; |
| 5565 | } |
| 5566 | |
Lorenz Bauer | 9436ef6 | 2020-09-21 13:12:20 +0100 | [diff] [blame] | 5567 | static bool check_btf_id_ok(const struct bpf_func_proto *fn) |
| 5568 | { |
| 5569 | int i; |
| 5570 | |
Martin KaFai Lau | 1df8f55 | 2020-09-24 17:03:50 -0700 | [diff] [blame] | 5571 | for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { |
Lorenz Bauer | 9436ef6 | 2020-09-21 13:12:20 +0100 | [diff] [blame] | 5572 | if (fn->arg_type[i] == ARG_PTR_TO_BTF_ID && !fn->arg_btf_id[i]) |
| 5573 | return false; |
| 5574 | |
Martin KaFai Lau | 1df8f55 | 2020-09-24 17:03:50 -0700 | [diff] [blame] | 5575 | if (fn->arg_type[i] != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i]) |
| 5576 | return false; |
| 5577 | } |
| 5578 | |
Lorenz Bauer | 9436ef6 | 2020-09-21 13:12:20 +0100 | [diff] [blame] | 5579 | return true; |
| 5580 | } |
| 5581 | |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 5582 | static int check_func_proto(const struct bpf_func_proto *fn, int func_id) |
Daniel Borkmann | 9013341 | 2018-01-20 01:24:29 +0100 | [diff] [blame] | 5583 | { |
| 5584 | return check_raw_mode_ok(fn) && |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 5585 | check_arg_pair_ok(fn) && |
Lorenz Bauer | 9436ef6 | 2020-09-21 13:12:20 +0100 | [diff] [blame] | 5586 | check_btf_id_ok(fn) && |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 5587 | check_refcount_ok(fn, func_id) ? 0 : -EINVAL; |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 5588 | } |
| 5589 | |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 5590 | /* Packet data might have moved, any old PTR_TO_PACKET[_META,_END] |
| 5591 | * are now invalid, so turn them into unknown SCALAR_VALUE. |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 5592 | */ |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5593 | static void __clear_all_pkt_pointers(struct bpf_verifier_env *env, |
| 5594 | struct bpf_func_state *state) |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 5595 | { |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 5596 | struct bpf_reg_state *regs = state->regs, *reg; |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 5597 | int i; |
| 5598 | |
| 5599 | for (i = 0; i < MAX_BPF_REG; i++) |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 5600 | if (reg_is_pkt_pointer_any(®s[i])) |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 5601 | mark_reg_unknown(env, regs, i); |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 5602 | |
Joe Stringer | f3709f6 | 2018-10-02 13:35:29 -0700 | [diff] [blame] | 5603 | bpf_for_each_spilled_reg(i, state, reg) { |
| 5604 | if (!reg) |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 5605 | continue; |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 5606 | if (reg_is_pkt_pointer_any(reg)) |
Daniel Borkmann | f54c789 | 2019-12-22 23:37:40 +0100 | [diff] [blame] | 5607 | __mark_reg_unknown(env, reg); |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 5608 | } |
| 5609 | } |
| 5610 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5611 | static void clear_all_pkt_pointers(struct bpf_verifier_env *env) |
| 5612 | { |
| 5613 | struct bpf_verifier_state *vstate = env->cur_state; |
| 5614 | int i; |
| 5615 | |
| 5616 | for (i = 0; i <= vstate->curframe; i++) |
| 5617 | __clear_all_pkt_pointers(env, vstate->frame[i]); |
| 5618 | } |
| 5619 | |
Alexei Starovoitov | 6d94e74 | 2020-11-10 19:12:11 -0800 | [diff] [blame] | 5620 | enum { |
| 5621 | AT_PKT_END = -1, |
| 5622 | BEYOND_PKT_END = -2, |
| 5623 | }; |
| 5624 | |
| 5625 | static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range_open) |
| 5626 | { |
| 5627 | struct bpf_func_state *state = vstate->frame[vstate->curframe]; |
| 5628 | struct bpf_reg_state *reg = &state->regs[regn]; |
| 5629 | |
| 5630 | if (reg->type != PTR_TO_PACKET) |
| 5631 | /* PTR_TO_PACKET_META is not supported yet */ |
| 5632 | return; |
| 5633 | |
| 5634 | /* The 'reg' is pkt > pkt_end or pkt >= pkt_end. |
| 5635 | * How far beyond pkt_end it goes is unknown. |
| 5636 | * if (!range_open) it's the case of pkt >= pkt_end |
| 5637 | * if (range_open) it's the case of pkt > pkt_end |
| 5638 | * hence this pointer is at least 1 byte bigger than pkt_end |
| 5639 | */ |
| 5640 | if (range_open) |
| 5641 | reg->range = BEYOND_PKT_END; |
| 5642 | else |
| 5643 | reg->range = AT_PKT_END; |
| 5644 | } |
| 5645 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 5646 | static void release_reg_references(struct bpf_verifier_env *env, |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 5647 | struct bpf_func_state *state, |
| 5648 | int ref_obj_id) |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 5649 | { |
| 5650 | struct bpf_reg_state *regs = state->regs, *reg; |
| 5651 | int i; |
| 5652 | |
| 5653 | for (i = 0; i < MAX_BPF_REG; i++) |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 5654 | if (regs[i].ref_obj_id == ref_obj_id) |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 5655 | mark_reg_unknown(env, regs, i); |
| 5656 | |
| 5657 | bpf_for_each_spilled_reg(i, state, reg) { |
| 5658 | if (!reg) |
| 5659 | continue; |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 5660 | if (reg->ref_obj_id == ref_obj_id) |
Daniel Borkmann | f54c789 | 2019-12-22 23:37:40 +0100 | [diff] [blame] | 5661 | __mark_reg_unknown(env, reg); |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 5662 | } |
| 5663 | } |
| 5664 | |
| 5665 | /* The pointer with the specified id has released its reference to kernel |
| 5666 | * resources. Identify all copies of the same pointer and clear the reference. |
| 5667 | */ |
| 5668 | static int release_reference(struct bpf_verifier_env *env, |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 5669 | int ref_obj_id) |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 5670 | { |
| 5671 | struct bpf_verifier_state *vstate = env->cur_state; |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 5672 | int err; |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 5673 | int i; |
| 5674 | |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 5675 | err = release_reference_state(cur_func(env), ref_obj_id); |
| 5676 | if (err) |
| 5677 | return err; |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 5678 | |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 5679 | for (i = 0; i <= vstate->curframe; i++) |
| 5680 | release_reg_references(env, vstate->frame[i], ref_obj_id); |
| 5681 | |
| 5682 | return 0; |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 5683 | } |
| 5684 | |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 5685 | static void clear_caller_saved_regs(struct bpf_verifier_env *env, |
| 5686 | struct bpf_reg_state *regs) |
| 5687 | { |
| 5688 | int i; |
| 5689 | |
| 5690 | /* after the call registers r0 - r5 were scratched */ |
| 5691 | for (i = 0; i < CALLER_SAVED_REGS; i++) { |
| 5692 | mark_reg_not_init(env, regs, caller_saved[i]); |
| 5693 | check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); |
| 5694 | } |
| 5695 | } |
| 5696 | |
Yonghong Song | 1435137 | 2021-02-26 12:49:23 -0800 | [diff] [blame] | 5697 | typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env, |
| 5698 | struct bpf_func_state *caller, |
| 5699 | struct bpf_func_state *callee, |
| 5700 | int insn_idx); |
| 5701 | |
| 5702 | static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, |
| 5703 | int *insn_idx, int subprog, |
| 5704 | set_callee_state_fn set_callee_state_cb) |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5705 | { |
| 5706 | struct bpf_verifier_state *state = env->cur_state; |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 5707 | struct bpf_func_info_aux *func_info_aux; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5708 | struct bpf_func_state *caller, *callee; |
Yonghong Song | 1435137 | 2021-02-26 12:49:23 -0800 | [diff] [blame] | 5709 | int err; |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 5710 | bool is_global = false; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5711 | |
Alexei Starovoitov | aada9ce | 2017-12-25 13:15:42 -0800 | [diff] [blame] | 5712 | if (state->curframe + 1 >= MAX_CALL_FRAMES) { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5713 | verbose(env, "the call stack of %d frames is too deep\n", |
Alexei Starovoitov | aada9ce | 2017-12-25 13:15:42 -0800 | [diff] [blame] | 5714 | state->curframe + 2); |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5715 | return -E2BIG; |
| 5716 | } |
| 5717 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5718 | caller = state->frame[state->curframe]; |
| 5719 | if (state->frame[state->curframe + 1]) { |
| 5720 | verbose(env, "verifier bug. Frame %d already allocated\n", |
| 5721 | state->curframe + 1); |
| 5722 | return -EFAULT; |
| 5723 | } |
| 5724 | |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 5725 | func_info_aux = env->prog->aux->func_info_aux; |
| 5726 | if (func_info_aux) |
| 5727 | is_global = func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL; |
Martin KaFai Lau | 34747c4 | 2021-03-24 18:51:36 -0700 | [diff] [blame] | 5728 | err = btf_check_subprog_arg_match(env, subprog, caller->regs); |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 5729 | if (err == -EFAULT) |
| 5730 | return err; |
| 5731 | if (is_global) { |
| 5732 | if (err) { |
| 5733 | verbose(env, "Caller passes invalid args into func#%d\n", |
| 5734 | subprog); |
| 5735 | return err; |
| 5736 | } else { |
| 5737 | if (env->log.level & BPF_LOG_LEVEL) |
| 5738 | verbose(env, |
| 5739 | "Func#%d is global and valid. Skipping.\n", |
| 5740 | subprog); |
| 5741 | clear_caller_saved_regs(env, caller->regs); |
| 5742 | |
Ilya Leoshkevich | 45159b2 | 2021-02-12 05:04:08 +0100 | [diff] [blame] | 5743 | /* All global functions return a 64-bit SCALAR_VALUE */ |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 5744 | mark_reg_unknown(env, caller->regs, BPF_REG_0); |
Ilya Leoshkevich | 45159b2 | 2021-02-12 05:04:08 +0100 | [diff] [blame] | 5745 | caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 5746 | |
| 5747 | /* continue with next insn after call */ |
| 5748 | return 0; |
| 5749 | } |
| 5750 | } |
| 5751 | |
Alexei Starovoitov | bfc6bb7 | 2021-07-14 17:54:14 -0700 | [diff] [blame] | 5752 | if (insn->code == (BPF_JMP | BPF_CALL) && |
| 5753 | insn->imm == BPF_FUNC_timer_set_callback) { |
| 5754 | struct bpf_verifier_state *async_cb; |
| 5755 | |
| 5756 | /* there is no real recursion here. timer callbacks are async */ |
Alexei Starovoitov | 7ddc80a | 2021-07-14 17:54:15 -0700 | [diff] [blame] | 5757 | env->subprog_info[subprog].is_async_cb = true; |
Alexei Starovoitov | bfc6bb7 | 2021-07-14 17:54:14 -0700 | [diff] [blame] | 5758 | async_cb = push_async_cb(env, env->subprog_info[subprog].start, |
| 5759 | *insn_idx, subprog); |
| 5760 | if (!async_cb) |
| 5761 | return -EFAULT; |
| 5762 | callee = async_cb->frame[0]; |
| 5763 | callee->async_entry_cnt = caller->async_entry_cnt + 1; |
| 5764 | |
| 5765 | /* Convert bpf_timer_set_callback() args into timer callback args */ |
| 5766 | err = set_callee_state_cb(env, caller, callee, *insn_idx); |
| 5767 | if (err) |
| 5768 | return err; |
| 5769 | |
| 5770 | clear_caller_saved_regs(env, caller->regs); |
| 5771 | mark_reg_unknown(env, caller->regs, BPF_REG_0); |
| 5772 | caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; |
| 5773 | /* continue with next insn after call */ |
| 5774 | return 0; |
| 5775 | } |
| 5776 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5777 | callee = kzalloc(sizeof(*callee), GFP_KERNEL); |
| 5778 | if (!callee) |
| 5779 | return -ENOMEM; |
| 5780 | state->frame[state->curframe + 1] = callee; |
| 5781 | |
| 5782 | /* callee cannot access r0, r6 - r9 for reading and has to write |
| 5783 | * into its own stack before reading from it. |
| 5784 | * callee can read/write into caller's stack |
| 5785 | */ |
| 5786 | init_func_state(env, callee, |
| 5787 | /* remember the callsite, it will be used by bpf_exit */ |
| 5788 | *insn_idx /* callsite */, |
| 5789 | state->curframe + 1 /* frameno within this callchain */, |
Jiong Wang | f910cef | 2018-05-02 16:17:17 -0400 | [diff] [blame] | 5790 | subprog /* subprog number within this prog */); |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5791 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 5792 | /* Transfer references to the callee */ |
Lorenz Bauer | c69431a | 2021-04-29 14:46:54 +0100 | [diff] [blame] | 5793 | err = copy_reference_state(callee, caller); |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 5794 | if (err) |
| 5795 | return err; |
| 5796 | |
Yonghong Song | 1435137 | 2021-02-26 12:49:23 -0800 | [diff] [blame] | 5797 | err = set_callee_state_cb(env, caller, callee, *insn_idx); |
| 5798 | if (err) |
| 5799 | return err; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5800 | |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 5801 | clear_caller_saved_regs(env, caller->regs); |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5802 | |
| 5803 | /* only increment it after check_reg_arg() finished */ |
| 5804 | state->curframe++; |
| 5805 | |
| 5806 | /* and go analyze first insn of the callee */ |
Yonghong Song | 1435137 | 2021-02-26 12:49:23 -0800 | [diff] [blame] | 5807 | *insn_idx = env->subprog_info[subprog].start - 1; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5808 | |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 5809 | if (env->log.level & BPF_LOG_LEVEL) { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5810 | verbose(env, "caller:\n"); |
| 5811 | print_verifier_state(env, caller); |
| 5812 | verbose(env, "callee:\n"); |
| 5813 | print_verifier_state(env, callee); |
| 5814 | } |
| 5815 | return 0; |
| 5816 | } |
| 5817 | |
Yonghong Song | 314ee05 | 2021-02-26 12:49:27 -0800 | [diff] [blame] | 5818 | int map_set_for_each_callback_args(struct bpf_verifier_env *env, |
| 5819 | struct bpf_func_state *caller, |
| 5820 | struct bpf_func_state *callee) |
| 5821 | { |
| 5822 | /* bpf_for_each_map_elem(struct bpf_map *map, void *callback_fn, |
| 5823 | * void *callback_ctx, u64 flags); |
| 5824 | * callback_fn(struct bpf_map *map, void *key, void *value, |
| 5825 | * void *callback_ctx); |
| 5826 | */ |
| 5827 | callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1]; |
| 5828 | |
| 5829 | callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; |
| 5830 | __mark_reg_known_zero(&callee->regs[BPF_REG_2]); |
| 5831 | callee->regs[BPF_REG_2].map_ptr = caller->regs[BPF_REG_1].map_ptr; |
| 5832 | |
| 5833 | callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; |
| 5834 | __mark_reg_known_zero(&callee->regs[BPF_REG_3]); |
| 5835 | callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr; |
| 5836 | |
| 5837 | /* pointer to stack or null */ |
| 5838 | callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3]; |
| 5839 | |
| 5840 | /* unused */ |
| 5841 | __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); |
| 5842 | return 0; |
| 5843 | } |
| 5844 | |
Yonghong Song | 1435137 | 2021-02-26 12:49:23 -0800 | [diff] [blame] | 5845 | static int set_callee_state(struct bpf_verifier_env *env, |
| 5846 | struct bpf_func_state *caller, |
| 5847 | struct bpf_func_state *callee, int insn_idx) |
| 5848 | { |
| 5849 | int i; |
| 5850 | |
| 5851 | /* copy r1 - r5 args that callee can access. The copy includes parent |
| 5852 | * pointers, which connects us up to the liveness chain |
| 5853 | */ |
| 5854 | for (i = BPF_REG_1; i <= BPF_REG_5; i++) |
| 5855 | callee->regs[i] = caller->regs[i]; |
| 5856 | return 0; |
| 5857 | } |
| 5858 | |
| 5859 | static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, |
| 5860 | int *insn_idx) |
| 5861 | { |
| 5862 | int subprog, target_insn; |
| 5863 | |
| 5864 | target_insn = *insn_idx + insn->imm + 1; |
| 5865 | subprog = find_subprog(env, target_insn); |
| 5866 | if (subprog < 0) { |
| 5867 | verbose(env, "verifier bug. No program starts at insn %d\n", |
| 5868 | target_insn); |
| 5869 | return -EFAULT; |
| 5870 | } |
| 5871 | |
| 5872 | return __check_func_call(env, insn, insn_idx, subprog, set_callee_state); |
| 5873 | } |
| 5874 | |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 5875 | static int set_map_elem_callback_state(struct bpf_verifier_env *env, |
| 5876 | struct bpf_func_state *caller, |
| 5877 | struct bpf_func_state *callee, |
| 5878 | int insn_idx) |
| 5879 | { |
| 5880 | struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx]; |
| 5881 | struct bpf_map *map; |
| 5882 | int err; |
| 5883 | |
| 5884 | if (bpf_map_ptr_poisoned(insn_aux)) { |
| 5885 | verbose(env, "tail_call abusing map_ptr\n"); |
| 5886 | return -EINVAL; |
| 5887 | } |
| 5888 | |
| 5889 | map = BPF_MAP_PTR(insn_aux->map_ptr_state); |
| 5890 | if (!map->ops->map_set_for_each_callback_args || |
| 5891 | !map->ops->map_for_each_callback) { |
| 5892 | verbose(env, "callback function not allowed for map\n"); |
| 5893 | return -ENOTSUPP; |
| 5894 | } |
| 5895 | |
| 5896 | err = map->ops->map_set_for_each_callback_args(env, caller, callee); |
| 5897 | if (err) |
| 5898 | return err; |
| 5899 | |
| 5900 | callee->in_callback_fn = true; |
| 5901 | return 0; |
| 5902 | } |
| 5903 | |
Alexei Starovoitov | b00628b | 2021-07-14 17:54:09 -0700 | [diff] [blame] | 5904 | static int set_timer_callback_state(struct bpf_verifier_env *env, |
| 5905 | struct bpf_func_state *caller, |
| 5906 | struct bpf_func_state *callee, |
| 5907 | int insn_idx) |
| 5908 | { |
| 5909 | struct bpf_map *map_ptr = caller->regs[BPF_REG_1].map_ptr; |
| 5910 | |
| 5911 | /* bpf_timer_set_callback(struct bpf_timer *timer, void *callback_fn); |
| 5912 | * callback_fn(struct bpf_map *map, void *key, void *value); |
| 5913 | */ |
| 5914 | callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP; |
| 5915 | __mark_reg_known_zero(&callee->regs[BPF_REG_1]); |
| 5916 | callee->regs[BPF_REG_1].map_ptr = map_ptr; |
| 5917 | |
| 5918 | callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; |
| 5919 | __mark_reg_known_zero(&callee->regs[BPF_REG_2]); |
| 5920 | callee->regs[BPF_REG_2].map_ptr = map_ptr; |
| 5921 | |
| 5922 | callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; |
| 5923 | __mark_reg_known_zero(&callee->regs[BPF_REG_3]); |
| 5924 | callee->regs[BPF_REG_3].map_ptr = map_ptr; |
| 5925 | |
| 5926 | /* unused */ |
| 5927 | __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); |
| 5928 | __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); |
Alexei Starovoitov | bfc6bb7 | 2021-07-14 17:54:14 -0700 | [diff] [blame] | 5929 | callee->in_async_callback_fn = true; |
Alexei Starovoitov | b00628b | 2021-07-14 17:54:09 -0700 | [diff] [blame] | 5930 | return 0; |
| 5931 | } |
| 5932 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5933 | static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) |
| 5934 | { |
| 5935 | struct bpf_verifier_state *state = env->cur_state; |
| 5936 | struct bpf_func_state *caller, *callee; |
| 5937 | struct bpf_reg_state *r0; |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 5938 | int err; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5939 | |
| 5940 | callee = state->frame[state->curframe]; |
| 5941 | r0 = &callee->regs[BPF_REG_0]; |
| 5942 | if (r0->type == PTR_TO_STACK) { |
| 5943 | /* technically it's ok to return caller's stack pointer |
| 5944 | * (or caller's caller's pointer) back to the caller, |
| 5945 | * since these pointers are valid. Only current stack |
| 5946 | * pointer will be invalid as soon as function exits, |
| 5947 | * but let's be conservative |
| 5948 | */ |
| 5949 | verbose(env, "cannot return stack pointer to the caller\n"); |
| 5950 | return -EINVAL; |
| 5951 | } |
| 5952 | |
| 5953 | state->curframe--; |
| 5954 | caller = state->frame[state->curframe]; |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 5955 | if (callee->in_callback_fn) { |
| 5956 | /* enforce R0 return value range [0, 1]. */ |
| 5957 | struct tnum range = tnum_range(0, 1); |
| 5958 | |
| 5959 | if (r0->type != SCALAR_VALUE) { |
| 5960 | verbose(env, "R0 not a scalar value\n"); |
| 5961 | return -EACCES; |
| 5962 | } |
| 5963 | if (!tnum_in(range, r0->var_off)) { |
| 5964 | verbose_invalid_scalar(env, r0, &range, "callback return", "R0"); |
| 5965 | return -EINVAL; |
| 5966 | } |
| 5967 | } else { |
| 5968 | /* return to the caller whatever r0 had in the callee */ |
| 5969 | caller->regs[BPF_REG_0] = *r0; |
| 5970 | } |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5971 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 5972 | /* Transfer references to the caller */ |
Lorenz Bauer | c69431a | 2021-04-29 14:46:54 +0100 | [diff] [blame] | 5973 | err = copy_reference_state(caller, callee); |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 5974 | if (err) |
| 5975 | return err; |
| 5976 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5977 | *insn_idx = callee->callsite + 1; |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 5978 | if (env->log.level & BPF_LOG_LEVEL) { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 5979 | verbose(env, "returning from callee:\n"); |
| 5980 | print_verifier_state(env, callee); |
| 5981 | verbose(env, "to caller at %d:\n", *insn_idx); |
| 5982 | print_verifier_state(env, caller); |
| 5983 | } |
| 5984 | /* clear everything in the callee */ |
| 5985 | free_func_state(callee); |
| 5986 | state->frame[state->curframe + 1] = NULL; |
| 5987 | return 0; |
| 5988 | } |
| 5989 | |
Yonghong Song | 849fa50 | 2018-04-28 22:28:09 -0700 | [diff] [blame] | 5990 | static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type, |
| 5991 | int func_id, |
| 5992 | struct bpf_call_arg_meta *meta) |
| 5993 | { |
| 5994 | struct bpf_reg_state *ret_reg = ®s[BPF_REG_0]; |
| 5995 | |
| 5996 | if (ret_type != RET_INTEGER || |
| 5997 | (func_id != BPF_FUNC_get_stack && |
Dave Marchevsky | fd0b88f | 2021-04-16 13:47:02 -0700 | [diff] [blame] | 5998 | func_id != BPF_FUNC_get_task_stack && |
Daniel Borkmann | 47cc0ed | 2020-05-15 12:11:17 +0200 | [diff] [blame] | 5999 | func_id != BPF_FUNC_probe_read_str && |
| 6000 | func_id != BPF_FUNC_probe_read_kernel_str && |
| 6001 | func_id != BPF_FUNC_probe_read_user_str)) |
Yonghong Song | 849fa50 | 2018-04-28 22:28:09 -0700 | [diff] [blame] | 6002 | return; |
| 6003 | |
John Fastabend | 1006050 | 2020-03-30 14:36:19 -0700 | [diff] [blame] | 6004 | ret_reg->smax_value = meta->msize_max_value; |
John Fastabend | fa123ac | 2020-03-30 14:36:59 -0700 | [diff] [blame] | 6005 | ret_reg->s32_max_value = meta->msize_max_value; |
Alexei Starovoitov | b0270958 | 2020-12-08 19:01:51 +0100 | [diff] [blame] | 6006 | ret_reg->smin_value = -MAX_ERRNO; |
| 6007 | ret_reg->s32_min_value = -MAX_ERRNO; |
Yonghong Song | 849fa50 | 2018-04-28 22:28:09 -0700 | [diff] [blame] | 6008 | __reg_deduce_bounds(ret_reg); |
| 6009 | __reg_bound_offset(ret_reg); |
John Fastabend | 1006050 | 2020-03-30 14:36:19 -0700 | [diff] [blame] | 6010 | __update_reg_bounds(ret_reg); |
Yonghong Song | 849fa50 | 2018-04-28 22:28:09 -0700 | [diff] [blame] | 6011 | } |
| 6012 | |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 6013 | static int |
| 6014 | record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, |
| 6015 | int func_id, int insn_idx) |
| 6016 | { |
| 6017 | struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; |
Daniel Borkmann | 591fe98 | 2019-04-09 23:20:05 +0200 | [diff] [blame] | 6018 | struct bpf_map *map = meta->map_ptr; |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 6019 | |
| 6020 | if (func_id != BPF_FUNC_tail_call && |
Daniel Borkmann | 09772d9 | 2018-06-02 23:06:35 +0200 | [diff] [blame] | 6021 | func_id != BPF_FUNC_map_lookup_elem && |
| 6022 | func_id != BPF_FUNC_map_update_elem && |
Mauricio Vasquez B | f1a2e44 | 2018-10-18 15:16:25 +0200 | [diff] [blame] | 6023 | func_id != BPF_FUNC_map_delete_elem && |
| 6024 | func_id != BPF_FUNC_map_push_elem && |
| 6025 | func_id != BPF_FUNC_map_pop_elem && |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 6026 | func_id != BPF_FUNC_map_peek_elem && |
Björn Töpel | e6a4750 | 2021-03-08 12:29:06 +0100 | [diff] [blame] | 6027 | func_id != BPF_FUNC_for_each_map_elem && |
| 6028 | func_id != BPF_FUNC_redirect_map) |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 6029 | return 0; |
Daniel Borkmann | 09772d9 | 2018-06-02 23:06:35 +0200 | [diff] [blame] | 6030 | |
Daniel Borkmann | 591fe98 | 2019-04-09 23:20:05 +0200 | [diff] [blame] | 6031 | if (map == NULL) { |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 6032 | verbose(env, "kernel subsystem misconfigured verifier\n"); |
| 6033 | return -EINVAL; |
| 6034 | } |
| 6035 | |
Daniel Borkmann | 591fe98 | 2019-04-09 23:20:05 +0200 | [diff] [blame] | 6036 | /* In case of read-only, some additional restrictions |
| 6037 | * need to be applied in order to prevent altering the |
| 6038 | * state of the map from program side. |
| 6039 | */ |
| 6040 | if ((map->map_flags & BPF_F_RDONLY_PROG) && |
| 6041 | (func_id == BPF_FUNC_map_delete_elem || |
| 6042 | func_id == BPF_FUNC_map_update_elem || |
| 6043 | func_id == BPF_FUNC_map_push_elem || |
| 6044 | func_id == BPF_FUNC_map_pop_elem)) { |
| 6045 | verbose(env, "write into map forbidden\n"); |
| 6046 | return -EACCES; |
| 6047 | } |
| 6048 | |
Daniel Borkmann | d2e4c1e | 2019-11-22 21:07:59 +0100 | [diff] [blame] | 6049 | if (!BPF_MAP_PTR(aux->map_ptr_state)) |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 6050 | bpf_map_ptr_store(aux, meta->map_ptr, |
Alexei Starovoitov | 2c78ee8 | 2020-05-13 16:03:54 -0700 | [diff] [blame] | 6051 | !meta->map_ptr->bypass_spec_v1); |
Daniel Borkmann | d2e4c1e | 2019-11-22 21:07:59 +0100 | [diff] [blame] | 6052 | else if (BPF_MAP_PTR(aux->map_ptr_state) != meta->map_ptr) |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 6053 | bpf_map_ptr_store(aux, BPF_MAP_PTR_POISON, |
Alexei Starovoitov | 2c78ee8 | 2020-05-13 16:03:54 -0700 | [diff] [blame] | 6054 | !meta->map_ptr->bypass_spec_v1); |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 6055 | return 0; |
| 6056 | } |
| 6057 | |
Daniel Borkmann | d2e4c1e | 2019-11-22 21:07:59 +0100 | [diff] [blame] | 6058 | static int |
| 6059 | record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, |
| 6060 | int func_id, int insn_idx) |
| 6061 | { |
| 6062 | struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; |
| 6063 | struct bpf_reg_state *regs = cur_regs(env), *reg; |
| 6064 | struct bpf_map *map = meta->map_ptr; |
| 6065 | struct tnum range; |
| 6066 | u64 val; |
Daniel Borkmann | cc52d91 | 2019-12-19 22:19:50 +0100 | [diff] [blame] | 6067 | int err; |
Daniel Borkmann | d2e4c1e | 2019-11-22 21:07:59 +0100 | [diff] [blame] | 6068 | |
| 6069 | if (func_id != BPF_FUNC_tail_call) |
| 6070 | return 0; |
| 6071 | if (!map || map->map_type != BPF_MAP_TYPE_PROG_ARRAY) { |
| 6072 | verbose(env, "kernel subsystem misconfigured verifier\n"); |
| 6073 | return -EINVAL; |
| 6074 | } |
| 6075 | |
| 6076 | range = tnum_range(0, map->max_entries - 1); |
| 6077 | reg = ®s[BPF_REG_3]; |
| 6078 | |
| 6079 | if (!register_is_const(reg) || !tnum_in(range, reg->var_off)) { |
| 6080 | bpf_map_key_store(aux, BPF_MAP_KEY_POISON); |
| 6081 | return 0; |
| 6082 | } |
| 6083 | |
Daniel Borkmann | cc52d91 | 2019-12-19 22:19:50 +0100 | [diff] [blame] | 6084 | err = mark_chain_precision(env, BPF_REG_3); |
| 6085 | if (err) |
| 6086 | return err; |
| 6087 | |
Daniel Borkmann | d2e4c1e | 2019-11-22 21:07:59 +0100 | [diff] [blame] | 6088 | val = reg->var_off.value; |
| 6089 | if (bpf_map_key_unseen(aux)) |
| 6090 | bpf_map_key_store(aux, val); |
| 6091 | else if (!bpf_map_key_poisoned(aux) && |
| 6092 | bpf_map_key_immediate(aux) != val) |
| 6093 | bpf_map_key_store(aux, BPF_MAP_KEY_POISON); |
| 6094 | return 0; |
| 6095 | } |
| 6096 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 6097 | static int check_reference_leak(struct bpf_verifier_env *env) |
| 6098 | { |
| 6099 | struct bpf_func_state *state = cur_func(env); |
| 6100 | int i; |
| 6101 | |
| 6102 | for (i = 0; i < state->acquired_refs; i++) { |
| 6103 | verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", |
| 6104 | state->refs[i].id, state->refs[i].insn_idx); |
| 6105 | } |
| 6106 | return state->acquired_refs ? -EINVAL : 0; |
| 6107 | } |
| 6108 | |
Florent Revest | 7b15523 | 2021-04-19 17:52:40 +0200 | [diff] [blame] | 6109 | static int check_bpf_snprintf_call(struct bpf_verifier_env *env, |
| 6110 | struct bpf_reg_state *regs) |
| 6111 | { |
| 6112 | struct bpf_reg_state *fmt_reg = ®s[BPF_REG_3]; |
| 6113 | struct bpf_reg_state *data_len_reg = ®s[BPF_REG_5]; |
| 6114 | struct bpf_map *fmt_map = fmt_reg->map_ptr; |
| 6115 | int err, fmt_map_off, num_args; |
| 6116 | u64 fmt_addr; |
| 6117 | char *fmt; |
| 6118 | |
| 6119 | /* data must be an array of u64 */ |
| 6120 | if (data_len_reg->var_off.value % 8) |
| 6121 | return -EINVAL; |
| 6122 | num_args = data_len_reg->var_off.value / 8; |
| 6123 | |
| 6124 | /* fmt being ARG_PTR_TO_CONST_STR guarantees that var_off is const |
| 6125 | * and map_direct_value_addr is set. |
| 6126 | */ |
| 6127 | fmt_map_off = fmt_reg->off + fmt_reg->var_off.value; |
| 6128 | err = fmt_map->ops->map_direct_value_addr(fmt_map, &fmt_addr, |
| 6129 | fmt_map_off); |
Florent Revest | 8e8ee10 | 2021-04-23 01:55:42 +0200 | [diff] [blame] | 6130 | if (err) { |
| 6131 | verbose(env, "verifier bug\n"); |
| 6132 | return -EFAULT; |
| 6133 | } |
Florent Revest | 7b15523 | 2021-04-19 17:52:40 +0200 | [diff] [blame] | 6134 | fmt = (char *)(long)fmt_addr + fmt_map_off; |
| 6135 | |
| 6136 | /* We are also guaranteed that fmt+fmt_map_off is NULL terminated, we |
| 6137 | * can focus on validating the format specifiers. |
| 6138 | */ |
Florent Revest | 48cac3f | 2021-04-27 19:43:13 +0200 | [diff] [blame] | 6139 | err = bpf_bprintf_prepare(fmt, UINT_MAX, NULL, NULL, num_args); |
Florent Revest | 7b15523 | 2021-04-19 17:52:40 +0200 | [diff] [blame] | 6140 | if (err < 0) |
| 6141 | verbose(env, "Invalid format string\n"); |
| 6142 | |
| 6143 | return err; |
| 6144 | } |
| 6145 | |
Jiri Olsa | 9b99edc | 2021-07-14 11:43:55 +0200 | [diff] [blame] | 6146 | static int check_get_func_ip(struct bpf_verifier_env *env) |
| 6147 | { |
| 6148 | enum bpf_attach_type eatype = env->prog->expected_attach_type; |
| 6149 | enum bpf_prog_type type = resolve_prog_type(env->prog); |
| 6150 | int func_id = BPF_FUNC_get_func_ip; |
| 6151 | |
| 6152 | if (type == BPF_PROG_TYPE_TRACING) { |
| 6153 | if (eatype != BPF_TRACE_FENTRY && eatype != BPF_TRACE_FEXIT && |
| 6154 | eatype != BPF_MODIFY_RETURN) { |
| 6155 | verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n", |
| 6156 | func_id_name(func_id), func_id); |
| 6157 | return -ENOTSUPP; |
| 6158 | } |
| 6159 | return 0; |
Jiri Olsa | 9ffd9f3 | 2021-07-14 11:43:56 +0200 | [diff] [blame] | 6160 | } else if (type == BPF_PROG_TYPE_KPROBE) { |
| 6161 | return 0; |
Jiri Olsa | 9b99edc | 2021-07-14 11:43:55 +0200 | [diff] [blame] | 6162 | } |
| 6163 | |
| 6164 | verbose(env, "func %s#%d not supported for program type %d\n", |
| 6165 | func_id_name(func_id), func_id, type); |
| 6166 | return -ENOTSUPP; |
| 6167 | } |
| 6168 | |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 6169 | static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn, |
| 6170 | int *insn_idx_p) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6171 | { |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6172 | const struct bpf_func_proto *fn = NULL; |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 6173 | struct bpf_reg_state *regs; |
Daniel Borkmann | 33ff982 | 2016-04-13 00:10:50 +0200 | [diff] [blame] | 6174 | struct bpf_call_arg_meta meta; |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 6175 | int insn_idx = *insn_idx_p; |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 6176 | bool changes_data; |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 6177 | int i, err, func_id; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6178 | |
| 6179 | /* find function prototype */ |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 6180 | func_id = insn->imm; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6181 | if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6182 | verbose(env, "invalid func %s#%d\n", func_id_name(func_id), |
| 6183 | func_id); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6184 | return -EINVAL; |
| 6185 | } |
| 6186 | |
Jakub Kicinski | 00176a3 | 2017-10-16 16:40:54 -0700 | [diff] [blame] | 6187 | if (env->ops->get_func_proto) |
Andrey Ignatov | 5e43f89 | 2018-03-30 15:08:00 -0700 | [diff] [blame] | 6188 | fn = env->ops->get_func_proto(func_id, env->prog); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6189 | if (!fn) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6190 | verbose(env, "unknown func %s#%d\n", func_id_name(func_id), |
| 6191 | func_id); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6192 | return -EINVAL; |
| 6193 | } |
| 6194 | |
| 6195 | /* eBPF programs must be GPL compatible to use GPL-ed functions */ |
Daniel Borkmann | 24701ec | 2015-03-01 12:31:47 +0100 | [diff] [blame] | 6196 | if (!env->prog->gpl_compatible && fn->gpl_only) { |
Daniel Borkmann | 3fe2867 | 2018-06-02 23:06:33 +0200 | [diff] [blame] | 6197 | verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6198 | return -EINVAL; |
| 6199 | } |
| 6200 | |
Jiri Olsa | eae2e83 | 2020-08-25 21:21:19 +0200 | [diff] [blame] | 6201 | if (fn->allowed && !fn->allowed(env->prog)) { |
| 6202 | verbose(env, "helper call is not allowed in probe\n"); |
| 6203 | return -EINVAL; |
| 6204 | } |
| 6205 | |
Daniel Borkmann | 04514d1 | 2017-12-14 21:07:25 +0100 | [diff] [blame] | 6206 | /* With LD_ABS/IND some JITs save/restore skb from r1. */ |
Martin KaFai Lau | 17bedab | 2016-12-07 15:53:11 -0800 | [diff] [blame] | 6207 | changes_data = bpf_helper_changes_pkt_data(fn->func); |
Daniel Borkmann | 04514d1 | 2017-12-14 21:07:25 +0100 | [diff] [blame] | 6208 | if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) { |
| 6209 | verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n", |
| 6210 | func_id_name(func_id), func_id); |
| 6211 | return -EINVAL; |
| 6212 | } |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 6213 | |
Daniel Borkmann | 33ff982 | 2016-04-13 00:10:50 +0200 | [diff] [blame] | 6214 | memset(&meta, 0, sizeof(meta)); |
Daniel Borkmann | 36bbef5 | 2016-09-20 00:26:13 +0200 | [diff] [blame] | 6215 | meta.pkt_access = fn->pkt_access; |
Daniel Borkmann | 33ff982 | 2016-04-13 00:10:50 +0200 | [diff] [blame] | 6216 | |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 6217 | err = check_func_proto(fn, func_id); |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 6218 | if (err) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6219 | verbose(env, "kernel subsystem misconfigured func %s#%d\n", |
Thomas Graf | ebb676d | 2016-10-27 11:23:51 +0200 | [diff] [blame] | 6220 | func_id_name(func_id), func_id); |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 6221 | return err; |
| 6222 | } |
| 6223 | |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 6224 | meta.func_id = func_id; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6225 | /* check args */ |
Dmitrii Banshchikov | 523a4cf | 2021-02-26 00:26:29 +0400 | [diff] [blame] | 6226 | for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { |
Yonghong Song | af7ec13 | 2020-06-23 16:08:09 -0700 | [diff] [blame] | 6227 | err = check_func_arg(env, i, &meta, fn); |
Alexei Starovoitov | a7658e1 | 2019-10-15 20:25:04 -0700 | [diff] [blame] | 6228 | if (err) |
| 6229 | return err; |
| 6230 | } |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6231 | |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 6232 | err = record_func_map(env, &meta, func_id, insn_idx); |
| 6233 | if (err) |
| 6234 | return err; |
| 6235 | |
Daniel Borkmann | d2e4c1e | 2019-11-22 21:07:59 +0100 | [diff] [blame] | 6236 | err = record_func_key(env, &meta, func_id, insn_idx); |
| 6237 | if (err) |
| 6238 | return err; |
| 6239 | |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 6240 | /* Mark slots with STACK_MISC in case of raw mode, stack offset |
| 6241 | * is inferred from register state. |
| 6242 | */ |
| 6243 | for (i = 0; i < meta.access_size; i++) { |
Daniel Borkmann | ca36960 | 2018-02-23 22:29:05 +0100 | [diff] [blame] | 6244 | err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B, |
| 6245 | BPF_WRITE, -1, false); |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 6246 | if (err) |
| 6247 | return err; |
| 6248 | } |
| 6249 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 6250 | if (func_id == BPF_FUNC_tail_call) { |
| 6251 | err = check_reference_leak(env); |
| 6252 | if (err) { |
| 6253 | verbose(env, "tail_call would lead to reference leak\n"); |
| 6254 | return err; |
| 6255 | } |
| 6256 | } else if (is_release_function(func_id)) { |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 6257 | err = release_reference(env, meta.ref_obj_id); |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 6258 | if (err) { |
| 6259 | verbose(env, "func %s#%d reference has not been acquired before\n", |
| 6260 | func_id_name(func_id), func_id); |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 6261 | return err; |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 6262 | } |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 6263 | } |
| 6264 | |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 6265 | regs = cur_regs(env); |
Roman Gushchin | cd33943 | 2018-08-02 14:27:24 -0700 | [diff] [blame] | 6266 | |
| 6267 | /* check that flags argument in get_local_storage(map, flags) is 0, |
| 6268 | * this is required because get_local_storage() can't return an error. |
| 6269 | */ |
| 6270 | if (func_id == BPF_FUNC_get_local_storage && |
| 6271 | !register_is_null(®s[BPF_REG_2])) { |
| 6272 | verbose(env, "get_local_storage() doesn't support non-zero flags\n"); |
| 6273 | return -EINVAL; |
| 6274 | } |
| 6275 | |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 6276 | if (func_id == BPF_FUNC_for_each_map_elem) { |
| 6277 | err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, |
| 6278 | set_map_elem_callback_state); |
| 6279 | if (err < 0) |
| 6280 | return -EINVAL; |
| 6281 | } |
| 6282 | |
Alexei Starovoitov | b00628b | 2021-07-14 17:54:09 -0700 | [diff] [blame] | 6283 | if (func_id == BPF_FUNC_timer_set_callback) { |
| 6284 | err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, |
| 6285 | set_timer_callback_state); |
| 6286 | if (err < 0) |
| 6287 | return -EINVAL; |
| 6288 | } |
| 6289 | |
Florent Revest | 7b15523 | 2021-04-19 17:52:40 +0200 | [diff] [blame] | 6290 | if (func_id == BPF_FUNC_snprintf) { |
| 6291 | err = check_bpf_snprintf_call(env, regs); |
| 6292 | if (err < 0) |
| 6293 | return err; |
| 6294 | } |
| 6295 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6296 | /* reset caller saved regs */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 6297 | for (i = 0; i < CALLER_SAVED_REGS; i++) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6298 | mark_reg_not_init(env, regs, caller_saved[i]); |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 6299 | check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); |
| 6300 | } |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6301 | |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 6302 | /* helper call returns 64-bit value. */ |
| 6303 | regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; |
| 6304 | |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 6305 | /* update return register (already marked as written above) */ |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6306 | if (fn->ret_type == RET_INTEGER) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 6307 | /* sets type to SCALAR_VALUE */ |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6308 | mark_reg_unknown(env, regs, BPF_REG_0); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6309 | } else if (fn->ret_type == RET_VOID) { |
| 6310 | regs[BPF_REG_0].type = NOT_INIT; |
Roman Gushchin | 3e6a4b3 | 2018-08-02 14:27:22 -0700 | [diff] [blame] | 6311 | } else if (fn->ret_type == RET_PTR_TO_MAP_VALUE_OR_NULL || |
| 6312 | fn->ret_type == RET_PTR_TO_MAP_VALUE) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 6313 | /* There is no offset yet applied, variable or fixed */ |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6314 | mark_reg_known_zero(env, regs, BPF_REG_0); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6315 | /* remember map_ptr, so that check_map_access() |
| 6316 | * can check 'value_size' boundary of memory access |
| 6317 | * to map element returned from bpf_map_lookup_elem() |
| 6318 | */ |
Daniel Borkmann | 33ff982 | 2016-04-13 00:10:50 +0200 | [diff] [blame] | 6319 | if (meta.map_ptr == NULL) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6320 | verbose(env, |
| 6321 | "kernel subsystem misconfigured verifier\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6322 | return -EINVAL; |
| 6323 | } |
Daniel Borkmann | 33ff982 | 2016-04-13 00:10:50 +0200 | [diff] [blame] | 6324 | regs[BPF_REG_0].map_ptr = meta.map_ptr; |
Alexei Starovoitov | 3e8ce29 | 2021-07-14 17:54:11 -0700 | [diff] [blame] | 6325 | regs[BPF_REG_0].map_uid = meta.map_uid; |
Daniel Borkmann | 4d31f30 | 2018-11-01 00:05:53 +0100 | [diff] [blame] | 6326 | if (fn->ret_type == RET_PTR_TO_MAP_VALUE) { |
| 6327 | regs[BPF_REG_0].type = PTR_TO_MAP_VALUE; |
Alexei Starovoitov | e16d2f1 | 2019-01-31 15:40:05 -0800 | [diff] [blame] | 6328 | if (map_value_has_spin_lock(meta.map_ptr)) |
| 6329 | regs[BPF_REG_0].id = ++env->id_gen; |
Daniel Borkmann | 4d31f30 | 2018-11-01 00:05:53 +0100 | [diff] [blame] | 6330 | } else { |
| 6331 | regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL; |
Daniel Borkmann | 4d31f30 | 2018-11-01 00:05:53 +0100 | [diff] [blame] | 6332 | } |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 6333 | } else if (fn->ret_type == RET_PTR_TO_SOCKET_OR_NULL) { |
| 6334 | mark_reg_known_zero(env, regs, BPF_REG_0); |
| 6335 | regs[BPF_REG_0].type = PTR_TO_SOCKET_OR_NULL; |
Lorenz Bauer | 85a51f8 | 2019-03-22 09:54:00 +0800 | [diff] [blame] | 6336 | } else if (fn->ret_type == RET_PTR_TO_SOCK_COMMON_OR_NULL) { |
| 6337 | mark_reg_known_zero(env, regs, BPF_REG_0); |
| 6338 | regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON_OR_NULL; |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 6339 | } else if (fn->ret_type == RET_PTR_TO_TCP_SOCK_OR_NULL) { |
| 6340 | mark_reg_known_zero(env, regs, BPF_REG_0); |
| 6341 | regs[BPF_REG_0].type = PTR_TO_TCP_SOCK_OR_NULL; |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 6342 | } else if (fn->ret_type == RET_PTR_TO_ALLOC_MEM_OR_NULL) { |
| 6343 | mark_reg_known_zero(env, regs, BPF_REG_0); |
| 6344 | regs[BPF_REG_0].type = PTR_TO_MEM_OR_NULL; |
Andrii Nakryiko | 457f443 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 6345 | regs[BPF_REG_0].mem_size = meta.mem_size; |
Hao Luo | 63d9b80 | 2020-09-29 16:50:48 -0700 | [diff] [blame] | 6346 | } else if (fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL || |
| 6347 | fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID) { |
Hao Luo | eaa6bcb | 2020-09-29 16:50:47 -0700 | [diff] [blame] | 6348 | const struct btf_type *t; |
| 6349 | |
| 6350 | mark_reg_known_zero(env, regs, BPF_REG_0); |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 6351 | t = btf_type_skip_modifiers(meta.ret_btf, meta.ret_btf_id, NULL); |
Hao Luo | eaa6bcb | 2020-09-29 16:50:47 -0700 | [diff] [blame] | 6352 | if (!btf_type_is_struct(t)) { |
| 6353 | u32 tsize; |
| 6354 | const struct btf_type *ret; |
| 6355 | const char *tname; |
| 6356 | |
| 6357 | /* resolve the type size of ksym. */ |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 6358 | ret = btf_resolve_size(meta.ret_btf, t, &tsize); |
Hao Luo | eaa6bcb | 2020-09-29 16:50:47 -0700 | [diff] [blame] | 6359 | if (IS_ERR(ret)) { |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 6360 | tname = btf_name_by_offset(meta.ret_btf, t->name_off); |
Hao Luo | eaa6bcb | 2020-09-29 16:50:47 -0700 | [diff] [blame] | 6361 | verbose(env, "unable to resolve the size of type '%s': %ld\n", |
| 6362 | tname, PTR_ERR(ret)); |
| 6363 | return -EINVAL; |
| 6364 | } |
Hao Luo | 63d9b80 | 2020-09-29 16:50:48 -0700 | [diff] [blame] | 6365 | regs[BPF_REG_0].type = |
| 6366 | fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ? |
| 6367 | PTR_TO_MEM : PTR_TO_MEM_OR_NULL; |
Hao Luo | eaa6bcb | 2020-09-29 16:50:47 -0700 | [diff] [blame] | 6368 | regs[BPF_REG_0].mem_size = tsize; |
| 6369 | } else { |
Hao Luo | 63d9b80 | 2020-09-29 16:50:48 -0700 | [diff] [blame] | 6370 | regs[BPF_REG_0].type = |
| 6371 | fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ? |
| 6372 | PTR_TO_BTF_ID : PTR_TO_BTF_ID_OR_NULL; |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 6373 | regs[BPF_REG_0].btf = meta.ret_btf; |
Hao Luo | eaa6bcb | 2020-09-29 16:50:47 -0700 | [diff] [blame] | 6374 | regs[BPF_REG_0].btf_id = meta.ret_btf_id; |
| 6375 | } |
KP Singh | 3ca1032 | 2020-11-06 10:37:43 +0000 | [diff] [blame] | 6376 | } else if (fn->ret_type == RET_PTR_TO_BTF_ID_OR_NULL || |
| 6377 | fn->ret_type == RET_PTR_TO_BTF_ID) { |
Yonghong Song | af7ec13 | 2020-06-23 16:08:09 -0700 | [diff] [blame] | 6378 | int ret_btf_id; |
| 6379 | |
| 6380 | mark_reg_known_zero(env, regs, BPF_REG_0); |
KP Singh | 3ca1032 | 2020-11-06 10:37:43 +0000 | [diff] [blame] | 6381 | regs[BPF_REG_0].type = fn->ret_type == RET_PTR_TO_BTF_ID ? |
| 6382 | PTR_TO_BTF_ID : |
| 6383 | PTR_TO_BTF_ID_OR_NULL; |
Yonghong Song | af7ec13 | 2020-06-23 16:08:09 -0700 | [diff] [blame] | 6384 | ret_btf_id = *fn->ret_btf_id; |
| 6385 | if (ret_btf_id == 0) { |
| 6386 | verbose(env, "invalid return type %d of func %s#%d\n", |
| 6387 | fn->ret_type, func_id_name(func_id), func_id); |
| 6388 | return -EINVAL; |
| 6389 | } |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 6390 | /* current BPF helper definitions are only coming from |
| 6391 | * built-in code with type IDs from vmlinux BTF |
| 6392 | */ |
| 6393 | regs[BPF_REG_0].btf = btf_vmlinux; |
Yonghong Song | af7ec13 | 2020-06-23 16:08:09 -0700 | [diff] [blame] | 6394 | regs[BPF_REG_0].btf_id = ret_btf_id; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6395 | } else { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6396 | verbose(env, "unknown return type %d of func %s#%d\n", |
Thomas Graf | ebb676d | 2016-10-27 11:23:51 +0200 | [diff] [blame] | 6397 | fn->ret_type, func_id_name(func_id), func_id); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 6398 | return -EINVAL; |
| 6399 | } |
Alexei Starovoitov | 04fd61ab | 2015-05-19 16:59:03 -0700 | [diff] [blame] | 6400 | |
Martin KaFai Lau | 93c230e | 2020-10-19 12:42:12 -0700 | [diff] [blame] | 6401 | if (reg_type_may_be_null(regs[BPF_REG_0].type)) |
| 6402 | regs[BPF_REG_0].id = ++env->id_gen; |
| 6403 | |
Lorenz Bauer | 0f3adc2 | 2019-03-22 09:53:59 +0800 | [diff] [blame] | 6404 | if (is_ptr_cast_function(func_id)) { |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 6405 | /* For release_reference() */ |
| 6406 | regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id; |
Jakub Sitnicki | 64d8529 | 2020-04-29 20:11:52 +0200 | [diff] [blame] | 6407 | } else if (is_acquire_function(func_id, meta.map_ptr)) { |
Lorenz Bauer | 0f3adc2 | 2019-03-22 09:53:59 +0800 | [diff] [blame] | 6408 | int id = acquire_reference_state(env, insn_idx); |
| 6409 | |
| 6410 | if (id < 0) |
| 6411 | return id; |
| 6412 | /* For mark_ptr_or_null_reg() */ |
| 6413 | regs[BPF_REG_0].id = id; |
| 6414 | /* For release_reference() */ |
| 6415 | regs[BPF_REG_0].ref_obj_id = id; |
| 6416 | } |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 6417 | |
Yonghong Song | 849fa50 | 2018-04-28 22:28:09 -0700 | [diff] [blame] | 6418 | do_refine_retval_range(regs, fn->ret_type, func_id, &meta); |
| 6419 | |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 6420 | err = check_map_func_compatibility(env, meta.map_ptr, func_id); |
Kaixu Xia | 35578d7 | 2015-08-06 07:02:35 +0000 | [diff] [blame] | 6421 | if (err) |
| 6422 | return err; |
Alexei Starovoitov | 04fd61ab | 2015-05-19 16:59:03 -0700 | [diff] [blame] | 6423 | |
Song Liu | fa28dcb | 2020-06-29 23:28:44 -0700 | [diff] [blame] | 6424 | if ((func_id == BPF_FUNC_get_stack || |
| 6425 | func_id == BPF_FUNC_get_task_stack) && |
| 6426 | !env->prog->has_callchain_buf) { |
Yonghong Song | c195651e | 2018-04-28 22:28:08 -0700 | [diff] [blame] | 6427 | const char *err_str; |
| 6428 | |
| 6429 | #ifdef CONFIG_PERF_EVENTS |
| 6430 | err = get_callchain_buffers(sysctl_perf_event_max_stack); |
| 6431 | err_str = "cannot get callchain buffer for func %s#%d\n"; |
| 6432 | #else |
| 6433 | err = -ENOTSUPP; |
| 6434 | err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n"; |
| 6435 | #endif |
| 6436 | if (err) { |
| 6437 | verbose(env, err_str, func_id_name(func_id), func_id); |
| 6438 | return err; |
| 6439 | } |
| 6440 | |
| 6441 | env->prog->has_callchain_buf = true; |
| 6442 | } |
| 6443 | |
Song Liu | 5d99cb2c | 2020-07-23 11:06:45 -0700 | [diff] [blame] | 6444 | if (func_id == BPF_FUNC_get_stackid || func_id == BPF_FUNC_get_stack) |
| 6445 | env->prog->call_get_stack = true; |
| 6446 | |
Jiri Olsa | 9b99edc | 2021-07-14 11:43:55 +0200 | [diff] [blame] | 6447 | if (func_id == BPF_FUNC_get_func_ip) { |
| 6448 | if (check_get_func_ip(env)) |
| 6449 | return -ENOTSUPP; |
| 6450 | env->prog->call_get_func_ip = true; |
| 6451 | } |
| 6452 | |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 6453 | if (changes_data) |
| 6454 | clear_all_pkt_pointers(env); |
| 6455 | return 0; |
| 6456 | } |
| 6457 | |
Martin KaFai Lau | e6ac245 | 2021-03-24 18:51:42 -0700 | [diff] [blame] | 6458 | /* mark_btf_func_reg_size() is used when the reg size is determined by |
| 6459 | * the BTF func_proto's return value size and argument. |
| 6460 | */ |
| 6461 | static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno, |
| 6462 | size_t reg_size) |
| 6463 | { |
| 6464 | struct bpf_reg_state *reg = &cur_regs(env)[regno]; |
| 6465 | |
| 6466 | if (regno == BPF_REG_0) { |
| 6467 | /* Function return value */ |
| 6468 | reg->live |= REG_LIVE_WRITTEN; |
| 6469 | reg->subreg_def = reg_size == sizeof(u64) ? |
| 6470 | DEF_NOT_SUBREG : env->insn_idx + 1; |
| 6471 | } else { |
| 6472 | /* Function argument */ |
| 6473 | if (reg_size == sizeof(u64)) { |
| 6474 | mark_insn_zext(env, reg); |
| 6475 | mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); |
| 6476 | } else { |
| 6477 | mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32); |
| 6478 | } |
| 6479 | } |
| 6480 | } |
| 6481 | |
| 6482 | static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn) |
| 6483 | { |
| 6484 | const struct btf_type *t, *func, *func_proto, *ptr_type; |
| 6485 | struct bpf_reg_state *regs = cur_regs(env); |
| 6486 | const char *func_name, *ptr_type_name; |
| 6487 | u32 i, nargs, func_id, ptr_type_id; |
| 6488 | const struct btf_param *args; |
| 6489 | int err; |
| 6490 | |
| 6491 | func_id = insn->imm; |
| 6492 | func = btf_type_by_id(btf_vmlinux, func_id); |
| 6493 | func_name = btf_name_by_offset(btf_vmlinux, func->name_off); |
| 6494 | func_proto = btf_type_by_id(btf_vmlinux, func->type); |
| 6495 | |
| 6496 | if (!env->ops->check_kfunc_call || |
| 6497 | !env->ops->check_kfunc_call(func_id)) { |
| 6498 | verbose(env, "calling kernel function %s is not allowed\n", |
| 6499 | func_name); |
| 6500 | return -EACCES; |
| 6501 | } |
| 6502 | |
| 6503 | /* Check the arguments */ |
| 6504 | err = btf_check_kfunc_arg_match(env, btf_vmlinux, func_id, regs); |
| 6505 | if (err) |
| 6506 | return err; |
| 6507 | |
| 6508 | for (i = 0; i < CALLER_SAVED_REGS; i++) |
| 6509 | mark_reg_not_init(env, regs, caller_saved[i]); |
| 6510 | |
| 6511 | /* Check return type */ |
| 6512 | t = btf_type_skip_modifiers(btf_vmlinux, func_proto->type, NULL); |
| 6513 | if (btf_type_is_scalar(t)) { |
| 6514 | mark_reg_unknown(env, regs, BPF_REG_0); |
| 6515 | mark_btf_func_reg_size(env, BPF_REG_0, t->size); |
| 6516 | } else if (btf_type_is_ptr(t)) { |
| 6517 | ptr_type = btf_type_skip_modifiers(btf_vmlinux, t->type, |
| 6518 | &ptr_type_id); |
| 6519 | if (!btf_type_is_struct(ptr_type)) { |
| 6520 | ptr_type_name = btf_name_by_offset(btf_vmlinux, |
| 6521 | ptr_type->name_off); |
| 6522 | verbose(env, "kernel function %s returns pointer type %s %s is not supported\n", |
| 6523 | func_name, btf_type_str(ptr_type), |
| 6524 | ptr_type_name); |
| 6525 | return -EINVAL; |
| 6526 | } |
| 6527 | mark_reg_known_zero(env, regs, BPF_REG_0); |
| 6528 | regs[BPF_REG_0].btf = btf_vmlinux; |
| 6529 | regs[BPF_REG_0].type = PTR_TO_BTF_ID; |
| 6530 | regs[BPF_REG_0].btf_id = ptr_type_id; |
| 6531 | mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *)); |
| 6532 | } /* else { add_kfunc_call() ensures it is btf_type_is_void(t) } */ |
| 6533 | |
| 6534 | nargs = btf_type_vlen(func_proto); |
| 6535 | args = (const struct btf_param *)(func_proto + 1); |
| 6536 | for (i = 0; i < nargs; i++) { |
| 6537 | u32 regno = i + 1; |
| 6538 | |
| 6539 | t = btf_type_skip_modifiers(btf_vmlinux, args[i].type, NULL); |
| 6540 | if (btf_type_is_ptr(t)) |
| 6541 | mark_btf_func_reg_size(env, regno, sizeof(void *)); |
| 6542 | else |
| 6543 | /* scalar. ensured by btf_check_kfunc_arg_match() */ |
| 6544 | mark_btf_func_reg_size(env, regno, t->size); |
| 6545 | } |
| 6546 | |
| 6547 | return 0; |
| 6548 | } |
| 6549 | |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 6550 | static bool signed_add_overflows(s64 a, s64 b) |
| 6551 | { |
| 6552 | /* Do the add in u64, where overflow is well-defined */ |
| 6553 | s64 res = (s64)((u64)a + (u64)b); |
| 6554 | |
| 6555 | if (b < 0) |
| 6556 | return res > a; |
| 6557 | return res < a; |
| 6558 | } |
| 6559 | |
Daniel Borkmann | bc895e8 | 2021-01-20 00:24:24 +0100 | [diff] [blame] | 6560 | static bool signed_add32_overflows(s32 a, s32 b) |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 6561 | { |
| 6562 | /* Do the add in u32, where overflow is well-defined */ |
| 6563 | s32 res = (s32)((u32)a + (u32)b); |
| 6564 | |
| 6565 | if (b < 0) |
| 6566 | return res > a; |
| 6567 | return res < a; |
| 6568 | } |
| 6569 | |
Daniel Borkmann | bc895e8 | 2021-01-20 00:24:24 +0100 | [diff] [blame] | 6570 | static bool signed_sub_overflows(s64 a, s64 b) |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 6571 | { |
| 6572 | /* Do the sub in u64, where overflow is well-defined */ |
| 6573 | s64 res = (s64)((u64)a - (u64)b); |
| 6574 | |
| 6575 | if (b < 0) |
| 6576 | return res < a; |
| 6577 | return res > a; |
David S. Miller | d117441 | 2017-05-10 11:22:52 -0700 | [diff] [blame] | 6578 | } |
| 6579 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 6580 | static bool signed_sub32_overflows(s32 a, s32 b) |
| 6581 | { |
Daniel Borkmann | bc895e8 | 2021-01-20 00:24:24 +0100 | [diff] [blame] | 6582 | /* Do the sub in u32, where overflow is well-defined */ |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 6583 | s32 res = (s32)((u32)a - (u32)b); |
| 6584 | |
| 6585 | if (b < 0) |
| 6586 | return res < a; |
| 6587 | return res > a; |
| 6588 | } |
| 6589 | |
Alexei Starovoitov | bb7f0f9 | 2017-12-18 20:12:00 -0800 | [diff] [blame] | 6590 | static bool check_reg_sane_offset(struct bpf_verifier_env *env, |
| 6591 | const struct bpf_reg_state *reg, |
| 6592 | enum bpf_reg_type type) |
| 6593 | { |
| 6594 | bool known = tnum_is_const(reg->var_off); |
| 6595 | s64 val = reg->var_off.value; |
| 6596 | s64 smin = reg->smin_value; |
| 6597 | |
| 6598 | if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) { |
| 6599 | verbose(env, "math between %s pointer and %lld is not allowed\n", |
| 6600 | reg_type_str[type], val); |
| 6601 | return false; |
| 6602 | } |
| 6603 | |
| 6604 | if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) { |
| 6605 | verbose(env, "%s pointer offset %d is not allowed\n", |
| 6606 | reg_type_str[type], reg->off); |
| 6607 | return false; |
| 6608 | } |
| 6609 | |
| 6610 | if (smin == S64_MIN) { |
| 6611 | verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n", |
| 6612 | reg_type_str[type]); |
| 6613 | return false; |
| 6614 | } |
| 6615 | |
| 6616 | if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) { |
| 6617 | verbose(env, "value %lld makes %s pointer be out of bounds\n", |
| 6618 | smin, reg_type_str[type]); |
| 6619 | return false; |
| 6620 | } |
| 6621 | |
| 6622 | return true; |
| 6623 | } |
| 6624 | |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 6625 | static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env) |
| 6626 | { |
| 6627 | return &env->insn_aux_data[env->insn_idx]; |
| 6628 | } |
| 6629 | |
Daniel Borkmann | a6aaece | 2021-03-23 09:30:01 +0100 | [diff] [blame] | 6630 | enum { |
| 6631 | REASON_BOUNDS = -1, |
| 6632 | REASON_TYPE = -2, |
| 6633 | REASON_PATHS = -3, |
| 6634 | REASON_LIMIT = -4, |
| 6635 | REASON_STACK = -5, |
| 6636 | }; |
| 6637 | |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 6638 | static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, |
Daniel Borkmann | bb01a1b | 2021-05-21 10:19:22 +0000 | [diff] [blame] | 6639 | u32 *alu_limit, bool mask_to_left) |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 6640 | { |
Daniel Borkmann | 7fedb63 | 2021-03-24 10:38:26 +0100 | [diff] [blame] | 6641 | u32 max = 0, ptr_limit = 0; |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 6642 | |
| 6643 | switch (ptr_reg->type) { |
| 6644 | case PTR_TO_STACK: |
Piotr Krysiuk | 1b1597e | 2021-03-16 09:47:02 +0100 | [diff] [blame] | 6645 | /* Offset 0 is out-of-bounds, but acceptable start for the |
Daniel Borkmann | 7fedb63 | 2021-03-24 10:38:26 +0100 | [diff] [blame] | 6646 | * left direction, see BPF_REG_FP. Also, unknown scalar |
| 6647 | * offset where we would need to deal with min/max bounds is |
| 6648 | * currently prohibited for unprivileged. |
Piotr Krysiuk | 1b1597e | 2021-03-16 09:47:02 +0100 | [diff] [blame] | 6649 | */ |
| 6650 | max = MAX_BPF_STACK + mask_to_left; |
Daniel Borkmann | 7fedb63 | 2021-03-24 10:38:26 +0100 | [diff] [blame] | 6651 | ptr_limit = -(ptr_reg->var_off.value + ptr_reg->off); |
Daniel Borkmann | b658bbb | 2021-03-23 09:04:10 +0100 | [diff] [blame] | 6652 | break; |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 6653 | case PTR_TO_MAP_VALUE: |
Piotr Krysiuk | 1b1597e | 2021-03-16 09:47:02 +0100 | [diff] [blame] | 6654 | max = ptr_reg->map_ptr->value_size; |
Daniel Borkmann | 7fedb63 | 2021-03-24 10:38:26 +0100 | [diff] [blame] | 6655 | ptr_limit = (mask_to_left ? |
| 6656 | ptr_reg->smin_value : |
| 6657 | ptr_reg->umax_value) + ptr_reg->off; |
Daniel Borkmann | b658bbb | 2021-03-23 09:04:10 +0100 | [diff] [blame] | 6658 | break; |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 6659 | default: |
Daniel Borkmann | a6aaece | 2021-03-23 09:30:01 +0100 | [diff] [blame] | 6660 | return REASON_TYPE; |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 6661 | } |
Daniel Borkmann | b658bbb | 2021-03-23 09:04:10 +0100 | [diff] [blame] | 6662 | |
| 6663 | if (ptr_limit >= max) |
Daniel Borkmann | a6aaece | 2021-03-23 09:30:01 +0100 | [diff] [blame] | 6664 | return REASON_LIMIT; |
Daniel Borkmann | b658bbb | 2021-03-23 09:04:10 +0100 | [diff] [blame] | 6665 | *alu_limit = ptr_limit; |
| 6666 | return 0; |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 6667 | } |
| 6668 | |
Daniel Borkmann | d3bd741 | 2019-01-06 00:54:37 +0100 | [diff] [blame] | 6669 | static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env, |
| 6670 | const struct bpf_insn *insn) |
| 6671 | { |
Alexei Starovoitov | 2c78ee8 | 2020-05-13 16:03:54 -0700 | [diff] [blame] | 6672 | return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K; |
Daniel Borkmann | d3bd741 | 2019-01-06 00:54:37 +0100 | [diff] [blame] | 6673 | } |
| 6674 | |
| 6675 | static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux, |
| 6676 | u32 alu_state, u32 alu_limit) |
| 6677 | { |
| 6678 | /* If we arrived here from different branches with different |
| 6679 | * state or limits to sanitize, then this won't work. |
| 6680 | */ |
| 6681 | if (aux->alu_state && |
| 6682 | (aux->alu_state != alu_state || |
| 6683 | aux->alu_limit != alu_limit)) |
Daniel Borkmann | a6aaece | 2021-03-23 09:30:01 +0100 | [diff] [blame] | 6684 | return REASON_PATHS; |
Daniel Borkmann | d3bd741 | 2019-01-06 00:54:37 +0100 | [diff] [blame] | 6685 | |
Brendan Jackman | e6ac593 | 2021-02-17 10:45:09 +0000 | [diff] [blame] | 6686 | /* Corresponding fixup done in do_misc_fixups(). */ |
Daniel Borkmann | d3bd741 | 2019-01-06 00:54:37 +0100 | [diff] [blame] | 6687 | aux->alu_state = alu_state; |
| 6688 | aux->alu_limit = alu_limit; |
| 6689 | return 0; |
| 6690 | } |
| 6691 | |
| 6692 | static int sanitize_val_alu(struct bpf_verifier_env *env, |
| 6693 | struct bpf_insn *insn) |
| 6694 | { |
| 6695 | struct bpf_insn_aux_data *aux = cur_aux(env); |
| 6696 | |
| 6697 | if (can_skip_alu_sanitation(env, insn)) |
| 6698 | return 0; |
| 6699 | |
| 6700 | return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0); |
| 6701 | } |
| 6702 | |
Daniel Borkmann | f528819 | 2021-03-24 11:25:39 +0100 | [diff] [blame] | 6703 | static bool sanitize_needed(u8 opcode) |
| 6704 | { |
| 6705 | return opcode == BPF_ADD || opcode == BPF_SUB; |
| 6706 | } |
| 6707 | |
Daniel Borkmann | 3d0220f | 2021-05-21 10:17:36 +0000 | [diff] [blame] | 6708 | struct bpf_sanitize_info { |
| 6709 | struct bpf_insn_aux_data aux; |
Daniel Borkmann | bb01a1b | 2021-05-21 10:19:22 +0000 | [diff] [blame] | 6710 | bool mask_to_left; |
Daniel Borkmann | 3d0220f | 2021-05-21 10:17:36 +0000 | [diff] [blame] | 6711 | }; |
| 6712 | |
Daniel Borkmann | 9183671a | 2021-05-28 15:47:32 +0000 | [diff] [blame] | 6713 | static struct bpf_verifier_state * |
| 6714 | sanitize_speculative_path(struct bpf_verifier_env *env, |
| 6715 | const struct bpf_insn *insn, |
| 6716 | u32 next_idx, u32 curr_idx) |
| 6717 | { |
| 6718 | struct bpf_verifier_state *branch; |
| 6719 | struct bpf_reg_state *regs; |
| 6720 | |
| 6721 | branch = push_stack(env, next_idx, curr_idx, true); |
| 6722 | if (branch && insn) { |
| 6723 | regs = branch->frame[branch->curframe]->regs; |
| 6724 | if (BPF_SRC(insn->code) == BPF_K) { |
| 6725 | mark_reg_unknown(env, regs, insn->dst_reg); |
| 6726 | } else if (BPF_SRC(insn->code) == BPF_X) { |
| 6727 | mark_reg_unknown(env, regs, insn->dst_reg); |
| 6728 | mark_reg_unknown(env, regs, insn->src_reg); |
| 6729 | } |
| 6730 | } |
| 6731 | return branch; |
| 6732 | } |
| 6733 | |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 6734 | static int sanitize_ptr_alu(struct bpf_verifier_env *env, |
| 6735 | struct bpf_insn *insn, |
| 6736 | const struct bpf_reg_state *ptr_reg, |
Daniel Borkmann | 6f55b2f | 2021-03-22 15:45:52 +0100 | [diff] [blame] | 6737 | const struct bpf_reg_state *off_reg, |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 6738 | struct bpf_reg_state *dst_reg, |
Daniel Borkmann | 3d0220f | 2021-05-21 10:17:36 +0000 | [diff] [blame] | 6739 | struct bpf_sanitize_info *info, |
Daniel Borkmann | 7fedb63 | 2021-03-24 10:38:26 +0100 | [diff] [blame] | 6740 | const bool commit_window) |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 6741 | { |
Daniel Borkmann | 3d0220f | 2021-05-21 10:17:36 +0000 | [diff] [blame] | 6742 | struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux; |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 6743 | struct bpf_verifier_state *vstate = env->cur_state; |
Daniel Borkmann | 801c605 | 2021-04-29 15:19:37 +0000 | [diff] [blame] | 6744 | bool off_is_imm = tnum_is_const(off_reg->var_off); |
Daniel Borkmann | 6f55b2f | 2021-03-22 15:45:52 +0100 | [diff] [blame] | 6745 | bool off_is_neg = off_reg->smin_value < 0; |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 6746 | bool ptr_is_dst_reg = ptr_reg == dst_reg; |
| 6747 | u8 opcode = BPF_OP(insn->code); |
| 6748 | u32 alu_state, alu_limit; |
| 6749 | struct bpf_reg_state tmp; |
| 6750 | bool ret; |
Piotr Krysiuk | f232326 | 2021-03-16 09:47:02 +0100 | [diff] [blame] | 6751 | int err; |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 6752 | |
Daniel Borkmann | d3bd741 | 2019-01-06 00:54:37 +0100 | [diff] [blame] | 6753 | if (can_skip_alu_sanitation(env, insn)) |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 6754 | return 0; |
| 6755 | |
| 6756 | /* We already marked aux for masking from non-speculative |
| 6757 | * paths, thus we got here in the first place. We only care |
| 6758 | * to explore bad access from here. |
| 6759 | */ |
| 6760 | if (vstate->speculative) |
| 6761 | goto do_sim; |
| 6762 | |
Daniel Borkmann | bb01a1b | 2021-05-21 10:19:22 +0000 | [diff] [blame] | 6763 | if (!commit_window) { |
| 6764 | if (!tnum_is_const(off_reg->var_off) && |
| 6765 | (off_reg->smin_value < 0) != (off_reg->smax_value < 0)) |
| 6766 | return REASON_BOUNDS; |
| 6767 | |
| 6768 | info->mask_to_left = (opcode == BPF_ADD && off_is_neg) || |
| 6769 | (opcode == BPF_SUB && !off_is_neg); |
| 6770 | } |
| 6771 | |
| 6772 | err = retrieve_ptr_limit(ptr_reg, &alu_limit, info->mask_to_left); |
Piotr Krysiuk | f232326 | 2021-03-16 09:47:02 +0100 | [diff] [blame] | 6773 | if (err < 0) |
| 6774 | return err; |
| 6775 | |
Daniel Borkmann | 7fedb63 | 2021-03-24 10:38:26 +0100 | [diff] [blame] | 6776 | if (commit_window) { |
| 6777 | /* In commit phase we narrow the masking window based on |
| 6778 | * the observed pointer move after the simulated operation. |
| 6779 | */ |
Daniel Borkmann | 3d0220f | 2021-05-21 10:17:36 +0000 | [diff] [blame] | 6780 | alu_state = info->aux.alu_state; |
| 6781 | alu_limit = abs(info->aux.alu_limit - alu_limit); |
Daniel Borkmann | 7fedb63 | 2021-03-24 10:38:26 +0100 | [diff] [blame] | 6782 | } else { |
| 6783 | alu_state = off_is_neg ? BPF_ALU_NEG_VALUE : 0; |
Daniel Borkmann | 801c605 | 2021-04-29 15:19:37 +0000 | [diff] [blame] | 6784 | alu_state |= off_is_imm ? BPF_ALU_IMMEDIATE : 0; |
Daniel Borkmann | 7fedb63 | 2021-03-24 10:38:26 +0100 | [diff] [blame] | 6785 | alu_state |= ptr_is_dst_reg ? |
| 6786 | BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST; |
Daniel Borkmann | e042aa5 | 2021-07-16 09:18:21 +0000 | [diff] [blame] | 6787 | |
| 6788 | /* Limit pruning on unknown scalars to enable deep search for |
| 6789 | * potential masking differences from other program paths. |
| 6790 | */ |
| 6791 | if (!off_is_imm) |
| 6792 | env->explore_alu_limits = true; |
Daniel Borkmann | 7fedb63 | 2021-03-24 10:38:26 +0100 | [diff] [blame] | 6793 | } |
| 6794 | |
Piotr Krysiuk | f232326 | 2021-03-16 09:47:02 +0100 | [diff] [blame] | 6795 | err = update_alu_sanitation_state(aux, alu_state, alu_limit); |
| 6796 | if (err < 0) |
| 6797 | return err; |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 6798 | do_sim: |
Daniel Borkmann | 7fedb63 | 2021-03-24 10:38:26 +0100 | [diff] [blame] | 6799 | /* If we're in commit phase, we're done here given we already |
| 6800 | * pushed the truncated dst_reg into the speculative verification |
| 6801 | * stack. |
Daniel Borkmann | a703619 | 2021-05-04 08:58:25 +0000 | [diff] [blame] | 6802 | * |
| 6803 | * Also, when register is a known constant, we rewrite register-based |
| 6804 | * operation to immediate-based, and thus do not need masking (and as |
| 6805 | * a consequence, do not need to simulate the zero-truncation either). |
Daniel Borkmann | 7fedb63 | 2021-03-24 10:38:26 +0100 | [diff] [blame] | 6806 | */ |
Daniel Borkmann | a703619 | 2021-05-04 08:58:25 +0000 | [diff] [blame] | 6807 | if (commit_window || off_is_imm) |
Daniel Borkmann | 7fedb63 | 2021-03-24 10:38:26 +0100 | [diff] [blame] | 6808 | return 0; |
| 6809 | |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 6810 | /* Simulate and find potential out-of-bounds access under |
| 6811 | * speculative execution from truncation as a result of |
| 6812 | * masking when off was not within expected range. If off |
| 6813 | * sits in dst, then we temporarily need to move ptr there |
| 6814 | * to simulate dst (== 0) +/-= ptr. Needed, for example, |
| 6815 | * for cases where we use K-based arithmetic in one direction |
| 6816 | * and truncated reg-based in the other in order to explore |
| 6817 | * bad access. |
| 6818 | */ |
| 6819 | if (!ptr_is_dst_reg) { |
| 6820 | tmp = *dst_reg; |
| 6821 | *dst_reg = *ptr_reg; |
| 6822 | } |
Daniel Borkmann | 9183671a | 2021-05-28 15:47:32 +0000 | [diff] [blame] | 6823 | ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1, |
| 6824 | env->insn_idx); |
Xu Yu | 0803278 | 2019-03-21 18:00:35 +0800 | [diff] [blame] | 6825 | if (!ptr_is_dst_reg && ret) |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 6826 | *dst_reg = tmp; |
Daniel Borkmann | a6aaece | 2021-03-23 09:30:01 +0100 | [diff] [blame] | 6827 | return !ret ? REASON_STACK : 0; |
| 6828 | } |
| 6829 | |
Daniel Borkmann | fe9a5ca | 2021-05-28 13:47:27 +0000 | [diff] [blame] | 6830 | static void sanitize_mark_insn_seen(struct bpf_verifier_env *env) |
| 6831 | { |
| 6832 | struct bpf_verifier_state *vstate = env->cur_state; |
| 6833 | |
| 6834 | /* If we simulate paths under speculation, we don't update the |
| 6835 | * insn as 'seen' such that when we verify unreachable paths in |
| 6836 | * the non-speculative domain, sanitize_dead_code() can still |
| 6837 | * rewrite/sanitize them. |
| 6838 | */ |
| 6839 | if (!vstate->speculative) |
| 6840 | env->insn_aux_data[env->insn_idx].seen = env->pass_cnt; |
| 6841 | } |
| 6842 | |
Daniel Borkmann | a6aaece | 2021-03-23 09:30:01 +0100 | [diff] [blame] | 6843 | static int sanitize_err(struct bpf_verifier_env *env, |
| 6844 | const struct bpf_insn *insn, int reason, |
| 6845 | const struct bpf_reg_state *off_reg, |
| 6846 | const struct bpf_reg_state *dst_reg) |
| 6847 | { |
| 6848 | static const char *err = "pointer arithmetic with it prohibited for !root"; |
| 6849 | const char *op = BPF_OP(insn->code) == BPF_ADD ? "add" : "sub"; |
| 6850 | u32 dst = insn->dst_reg, src = insn->src_reg; |
| 6851 | |
| 6852 | switch (reason) { |
| 6853 | case REASON_BOUNDS: |
| 6854 | verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n", |
| 6855 | off_reg == dst_reg ? dst : src, err); |
| 6856 | break; |
| 6857 | case REASON_TYPE: |
| 6858 | verbose(env, "R%d has pointer with unsupported alu operation, %s\n", |
| 6859 | off_reg == dst_reg ? src : dst, err); |
| 6860 | break; |
| 6861 | case REASON_PATHS: |
| 6862 | verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n", |
| 6863 | dst, op, err); |
| 6864 | break; |
| 6865 | case REASON_LIMIT: |
| 6866 | verbose(env, "R%d tried to %s beyond pointer bounds, %s\n", |
| 6867 | dst, op, err); |
| 6868 | break; |
| 6869 | case REASON_STACK: |
| 6870 | verbose(env, "R%d could not be pushed for speculative verification, %s\n", |
| 6871 | dst, err); |
| 6872 | break; |
| 6873 | default: |
| 6874 | verbose(env, "verifier internal error: unknown reason (%d)\n", |
| 6875 | reason); |
| 6876 | break; |
| 6877 | } |
| 6878 | |
| 6879 | return -EACCES; |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 6880 | } |
| 6881 | |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 6882 | /* check that stack access falls within stack limits and that 'reg' doesn't |
| 6883 | * have a variable offset. |
| 6884 | * |
| 6885 | * Variable offset is prohibited for unprivileged mode for simplicity since it |
| 6886 | * requires corresponding support in Spectre masking for stack ALU. See also |
| 6887 | * retrieve_ptr_limit(). |
| 6888 | * |
| 6889 | * |
| 6890 | * 'off' includes 'reg->off'. |
| 6891 | */ |
| 6892 | static int check_stack_access_for_ptr_arithmetic( |
| 6893 | struct bpf_verifier_env *env, |
| 6894 | int regno, |
| 6895 | const struct bpf_reg_state *reg, |
| 6896 | int off) |
| 6897 | { |
| 6898 | if (!tnum_is_const(reg->var_off)) { |
| 6899 | char tn_buf[48]; |
| 6900 | |
| 6901 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
| 6902 | verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n", |
| 6903 | regno, tn_buf, off); |
| 6904 | return -EACCES; |
| 6905 | } |
| 6906 | |
| 6907 | if (off >= 0 || off < -MAX_BPF_STACK) { |
| 6908 | verbose(env, "R%d stack pointer arithmetic goes out of range, " |
| 6909 | "prohibited for !root; off=%d\n", regno, off); |
| 6910 | return -EACCES; |
| 6911 | } |
| 6912 | |
| 6913 | return 0; |
| 6914 | } |
| 6915 | |
Daniel Borkmann | 073815b | 2021-03-23 15:05:48 +0100 | [diff] [blame] | 6916 | static int sanitize_check_bounds(struct bpf_verifier_env *env, |
| 6917 | const struct bpf_insn *insn, |
| 6918 | const struct bpf_reg_state *dst_reg) |
| 6919 | { |
| 6920 | u32 dst = insn->dst_reg; |
| 6921 | |
| 6922 | /* For unprivileged we require that resulting offset must be in bounds |
| 6923 | * in order to be able to sanitize access later on. |
| 6924 | */ |
| 6925 | if (env->bypass_spec_v1) |
| 6926 | return 0; |
| 6927 | |
| 6928 | switch (dst_reg->type) { |
| 6929 | case PTR_TO_STACK: |
| 6930 | if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg, |
| 6931 | dst_reg->off + dst_reg->var_off.value)) |
| 6932 | return -EACCES; |
| 6933 | break; |
| 6934 | case PTR_TO_MAP_VALUE: |
| 6935 | if (check_map_access(env, dst, dst_reg->off, 1, false)) { |
| 6936 | verbose(env, "R%d pointer arithmetic of map value goes out of range, " |
| 6937 | "prohibited for !root\n", dst); |
| 6938 | return -EACCES; |
| 6939 | } |
| 6940 | break; |
| 6941 | default: |
| 6942 | break; |
| 6943 | } |
| 6944 | |
| 6945 | return 0; |
| 6946 | } |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 6947 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 6948 | /* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off. |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 6949 | * Caller should also handle BPF_MOV case separately. |
| 6950 | * If we return -EACCES, caller may want to try again treating pointer as a |
| 6951 | * scalar. So we only emit a diagnostic if !env->allow_ptr_leaks. |
| 6952 | */ |
| 6953 | static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, |
| 6954 | struct bpf_insn *insn, |
| 6955 | const struct bpf_reg_state *ptr_reg, |
| 6956 | const struct bpf_reg_state *off_reg) |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 6957 | { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 6958 | struct bpf_verifier_state *vstate = env->cur_state; |
| 6959 | struct bpf_func_state *state = vstate->frame[vstate->curframe]; |
| 6960 | struct bpf_reg_state *regs = state->regs, *dst_reg; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 6961 | bool known = tnum_is_const(off_reg->var_off); |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 6962 | s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value, |
| 6963 | smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value; |
| 6964 | u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value, |
| 6965 | umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value; |
Daniel Borkmann | 3d0220f | 2021-05-21 10:17:36 +0000 | [diff] [blame] | 6966 | struct bpf_sanitize_info info = {}; |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 6967 | u8 opcode = BPF_OP(insn->code); |
Daniel Borkmann | 24c109b | 2021-03-23 08:51:02 +0100 | [diff] [blame] | 6968 | u32 dst = insn->dst_reg; |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 6969 | int ret; |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 6970 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 6971 | dst_reg = ®s[dst]; |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 6972 | |
Daniel Borkmann | 6f16101 | 2018-01-18 01:15:21 +0100 | [diff] [blame] | 6973 | if ((known && (smin_val != smax_val || umin_val != umax_val)) || |
| 6974 | smin_val > smax_val || umin_val > umax_val) { |
| 6975 | /* Taint dst register if offset had invalid bounds derived from |
| 6976 | * e.g. dead branches. |
| 6977 | */ |
Daniel Borkmann | f54c789 | 2019-12-22 23:37:40 +0100 | [diff] [blame] | 6978 | __mark_reg_unknown(env, dst_reg); |
Daniel Borkmann | 6f16101 | 2018-01-18 01:15:21 +0100 | [diff] [blame] | 6979 | return 0; |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 6980 | } |
| 6981 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 6982 | if (BPF_CLASS(insn->code) != BPF_ALU64) { |
| 6983 | /* 32-bit ALU ops on pointers produce (meaningless) scalars */ |
Yonghong Song | 6c69354 | 2020-06-18 16:46:31 -0700 | [diff] [blame] | 6984 | if (opcode == BPF_SUB && env->allow_ptr_leaks) { |
| 6985 | __mark_reg_unknown(env, dst_reg); |
| 6986 | return 0; |
| 6987 | } |
| 6988 | |
Alexei Starovoitov | 82abbf8 | 2017-12-18 20:15:20 -0800 | [diff] [blame] | 6989 | verbose(env, |
| 6990 | "R%d 32-bit pointer arithmetic prohibited\n", |
| 6991 | dst); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 6992 | return -EACCES; |
| 6993 | } |
David S. Miller | d117441 | 2017-05-10 11:22:52 -0700 | [diff] [blame] | 6994 | |
Joe Stringer | aad2eea | 2018-10-02 13:35:30 -0700 | [diff] [blame] | 6995 | switch (ptr_reg->type) { |
| 6996 | case PTR_TO_MAP_VALUE_OR_NULL: |
| 6997 | verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", |
| 6998 | dst, reg_type_str[ptr_reg->type]); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 6999 | return -EACCES; |
Joe Stringer | aad2eea | 2018-10-02 13:35:30 -0700 | [diff] [blame] | 7000 | case CONST_PTR_TO_MAP: |
Yonghong Song | 7c69673 | 2020-09-08 10:57:02 -0700 | [diff] [blame] | 7001 | /* smin_val represents the known value */ |
| 7002 | if (known && smin_val == 0 && opcode == BPF_ADD) |
| 7003 | break; |
Gustavo A. R. Silva | 8731745 | 2020-10-02 18:42:17 -0500 | [diff] [blame] | 7004 | fallthrough; |
Joe Stringer | aad2eea | 2018-10-02 13:35:30 -0700 | [diff] [blame] | 7005 | case PTR_TO_PACKET_END: |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 7006 | case PTR_TO_SOCKET: |
| 7007 | case PTR_TO_SOCKET_OR_NULL: |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 7008 | case PTR_TO_SOCK_COMMON: |
| 7009 | case PTR_TO_SOCK_COMMON_OR_NULL: |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 7010 | case PTR_TO_TCP_SOCK: |
| 7011 | case PTR_TO_TCP_SOCK_OR_NULL: |
Jonathan Lemon | fada7fd | 2019-06-06 13:59:40 -0700 | [diff] [blame] | 7012 | case PTR_TO_XDP_SOCK: |
Joe Stringer | aad2eea | 2018-10-02 13:35:30 -0700 | [diff] [blame] | 7013 | verbose(env, "R%d pointer arithmetic on %s prohibited\n", |
| 7014 | dst, reg_type_str[ptr_reg->type]); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7015 | return -EACCES; |
Joe Stringer | aad2eea | 2018-10-02 13:35:30 -0700 | [diff] [blame] | 7016 | default: |
| 7017 | break; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7018 | } |
| 7019 | |
| 7020 | /* In case of 'scalar += pointer', dst_reg inherits pointer type and id. |
| 7021 | * The id may be overwritten later if we create a new variable offset. |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 7022 | */ |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7023 | dst_reg->type = ptr_reg->type; |
| 7024 | dst_reg->id = ptr_reg->id; |
Josef Bacik | f23cc64 | 2016-11-14 15:45:36 -0500 | [diff] [blame] | 7025 | |
Alexei Starovoitov | bb7f0f9 | 2017-12-18 20:12:00 -0800 | [diff] [blame] | 7026 | if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) || |
| 7027 | !check_reg_sane_offset(env, ptr_reg, ptr_reg->type)) |
| 7028 | return -EINVAL; |
| 7029 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7030 | /* pointer types do not carry 32-bit bounds at the moment. */ |
| 7031 | __mark_reg32_unbounded(dst_reg); |
| 7032 | |
Daniel Borkmann | 7fedb63 | 2021-03-24 10:38:26 +0100 | [diff] [blame] | 7033 | if (sanitize_needed(opcode)) { |
| 7034 | ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, |
Daniel Borkmann | 3d0220f | 2021-05-21 10:17:36 +0000 | [diff] [blame] | 7035 | &info, false); |
Daniel Borkmann | a6aaece | 2021-03-23 09:30:01 +0100 | [diff] [blame] | 7036 | if (ret < 0) |
| 7037 | return sanitize_err(env, insn, ret, off_reg, dst_reg); |
Daniel Borkmann | 7fedb63 | 2021-03-24 10:38:26 +0100 | [diff] [blame] | 7038 | } |
Daniel Borkmann | a6aaece | 2021-03-23 09:30:01 +0100 | [diff] [blame] | 7039 | |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 7040 | switch (opcode) { |
| 7041 | case BPF_ADD: |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7042 | /* We can take a fixed offset as long as it doesn't overflow |
| 7043 | * the s32 'off' field |
| 7044 | */ |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 7045 | if (known && (ptr_reg->off + smin_val == |
| 7046 | (s64)(s32)(ptr_reg->off + smin_val))) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7047 | /* pointer += K. Accumulate it into fixed offset */ |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 7048 | dst_reg->smin_value = smin_ptr; |
| 7049 | dst_reg->smax_value = smax_ptr; |
| 7050 | dst_reg->umin_value = umin_ptr; |
| 7051 | dst_reg->umax_value = umax_ptr; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7052 | dst_reg->var_off = ptr_reg->var_off; |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 7053 | dst_reg->off = ptr_reg->off + smin_val; |
Daniel Borkmann | 0962590 | 2018-11-01 00:05:52 +0100 | [diff] [blame] | 7054 | dst_reg->raw = ptr_reg->raw; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7055 | break; |
| 7056 | } |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7057 | /* A new variable offset is created. Note that off_reg->off |
| 7058 | * == 0, since it's a scalar. |
| 7059 | * dst_reg gets the pointer type and since some positive |
| 7060 | * integer value was added to the pointer, give it a new 'id' |
| 7061 | * if it's a PTR_TO_PACKET. |
| 7062 | * this creates a new 'base' pointer, off_reg (variable) gets |
| 7063 | * added into the variable offset, and we copy the fixed offset |
| 7064 | * from ptr_reg. |
| 7065 | */ |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 7066 | if (signed_add_overflows(smin_ptr, smin_val) || |
| 7067 | signed_add_overflows(smax_ptr, smax_val)) { |
| 7068 | dst_reg->smin_value = S64_MIN; |
| 7069 | dst_reg->smax_value = S64_MAX; |
| 7070 | } else { |
| 7071 | dst_reg->smin_value = smin_ptr + smin_val; |
| 7072 | dst_reg->smax_value = smax_ptr + smax_val; |
| 7073 | } |
| 7074 | if (umin_ptr + umin_val < umin_ptr || |
| 7075 | umax_ptr + umax_val < umax_ptr) { |
| 7076 | dst_reg->umin_value = 0; |
| 7077 | dst_reg->umax_value = U64_MAX; |
| 7078 | } else { |
| 7079 | dst_reg->umin_value = umin_ptr + umin_val; |
| 7080 | dst_reg->umax_value = umax_ptr + umax_val; |
| 7081 | } |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7082 | dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); |
| 7083 | dst_reg->off = ptr_reg->off; |
Daniel Borkmann | 0962590 | 2018-11-01 00:05:52 +0100 | [diff] [blame] | 7084 | dst_reg->raw = ptr_reg->raw; |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 7085 | if (reg_is_pkt_pointer(ptr_reg)) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7086 | dst_reg->id = ++env->id_gen; |
| 7087 | /* something was added to pkt_ptr, set range to zero */ |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 7088 | memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7089 | } |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 7090 | break; |
| 7091 | case BPF_SUB: |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7092 | if (dst_reg == off_reg) { |
| 7093 | /* scalar -= pointer. Creates an unknown scalar */ |
Alexei Starovoitov | 82abbf8 | 2017-12-18 20:15:20 -0800 | [diff] [blame] | 7094 | verbose(env, "R%d tried to subtract pointer from scalar\n", |
| 7095 | dst); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7096 | return -EACCES; |
| 7097 | } |
| 7098 | /* We don't allow subtraction from FP, because (according to |
| 7099 | * test_verifier.c test "invalid fp arithmetic", JITs might not |
| 7100 | * be able to deal with it. |
Edward Cree | 9305706 | 2017-07-21 14:37:34 +0100 | [diff] [blame] | 7101 | */ |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7102 | if (ptr_reg->type == PTR_TO_STACK) { |
Alexei Starovoitov | 82abbf8 | 2017-12-18 20:15:20 -0800 | [diff] [blame] | 7103 | verbose(env, "R%d subtraction from stack pointer prohibited\n", |
| 7104 | dst); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7105 | return -EACCES; |
| 7106 | } |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 7107 | if (known && (ptr_reg->off - smin_val == |
| 7108 | (s64)(s32)(ptr_reg->off - smin_val))) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7109 | /* pointer -= K. Subtract it from fixed offset */ |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 7110 | dst_reg->smin_value = smin_ptr; |
| 7111 | dst_reg->smax_value = smax_ptr; |
| 7112 | dst_reg->umin_value = umin_ptr; |
| 7113 | dst_reg->umax_value = umax_ptr; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7114 | dst_reg->var_off = ptr_reg->var_off; |
| 7115 | dst_reg->id = ptr_reg->id; |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 7116 | dst_reg->off = ptr_reg->off - smin_val; |
Daniel Borkmann | 0962590 | 2018-11-01 00:05:52 +0100 | [diff] [blame] | 7117 | dst_reg->raw = ptr_reg->raw; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7118 | break; |
| 7119 | } |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7120 | /* A new variable offset is created. If the subtrahend is known |
| 7121 | * nonnegative, then any reg->range we had before is still good. |
| 7122 | */ |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 7123 | if (signed_sub_overflows(smin_ptr, smax_val) || |
| 7124 | signed_sub_overflows(smax_ptr, smin_val)) { |
| 7125 | /* Overflow possible, we know nothing */ |
| 7126 | dst_reg->smin_value = S64_MIN; |
| 7127 | dst_reg->smax_value = S64_MAX; |
| 7128 | } else { |
| 7129 | dst_reg->smin_value = smin_ptr - smax_val; |
| 7130 | dst_reg->smax_value = smax_ptr - smin_val; |
| 7131 | } |
| 7132 | if (umin_ptr < umax_val) { |
| 7133 | /* Overflow possible, we know nothing */ |
| 7134 | dst_reg->umin_value = 0; |
| 7135 | dst_reg->umax_value = U64_MAX; |
| 7136 | } else { |
| 7137 | /* Cannot overflow (as long as bounds are consistent) */ |
| 7138 | dst_reg->umin_value = umin_ptr - umax_val; |
| 7139 | dst_reg->umax_value = umax_ptr - umin_val; |
| 7140 | } |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7141 | dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off); |
| 7142 | dst_reg->off = ptr_reg->off; |
Daniel Borkmann | 0962590 | 2018-11-01 00:05:52 +0100 | [diff] [blame] | 7143 | dst_reg->raw = ptr_reg->raw; |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 7144 | if (reg_is_pkt_pointer(ptr_reg)) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7145 | dst_reg->id = ++env->id_gen; |
| 7146 | /* something was added to pkt_ptr, set range to zero */ |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 7147 | if (smin_val < 0) |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 7148 | memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7149 | } |
| 7150 | break; |
| 7151 | case BPF_AND: |
| 7152 | case BPF_OR: |
| 7153 | case BPF_XOR: |
Alexei Starovoitov | 82abbf8 | 2017-12-18 20:15:20 -0800 | [diff] [blame] | 7154 | /* bitwise ops on pointers are troublesome, prohibit. */ |
| 7155 | verbose(env, "R%d bitwise operator %s on pointer prohibited\n", |
| 7156 | dst, bpf_alu_string[opcode >> 4]); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7157 | return -EACCES; |
| 7158 | default: |
| 7159 | /* other operators (e.g. MUL,LSH) produce non-pointer results */ |
Alexei Starovoitov | 82abbf8 | 2017-12-18 20:15:20 -0800 | [diff] [blame] | 7160 | verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", |
| 7161 | dst, bpf_alu_string[opcode >> 4]); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7162 | return -EACCES; |
| 7163 | } |
| 7164 | |
Alexei Starovoitov | bb7f0f9 | 2017-12-18 20:12:00 -0800 | [diff] [blame] | 7165 | if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) |
| 7166 | return -EINVAL; |
| 7167 | |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 7168 | __update_reg_bounds(dst_reg); |
| 7169 | __reg_deduce_bounds(dst_reg); |
| 7170 | __reg_bound_offset(dst_reg); |
Daniel Borkmann | 0d6303d | 2019-01-03 00:58:30 +0100 | [diff] [blame] | 7171 | |
Daniel Borkmann | 073815b | 2021-03-23 15:05:48 +0100 | [diff] [blame] | 7172 | if (sanitize_check_bounds(env, insn, dst_reg) < 0) |
| 7173 | return -EACCES; |
Daniel Borkmann | 7fedb63 | 2021-03-24 10:38:26 +0100 | [diff] [blame] | 7174 | if (sanitize_needed(opcode)) { |
| 7175 | ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg, |
Daniel Borkmann | 3d0220f | 2021-05-21 10:17:36 +0000 | [diff] [blame] | 7176 | &info, true); |
Daniel Borkmann | 7fedb63 | 2021-03-24 10:38:26 +0100 | [diff] [blame] | 7177 | if (ret < 0) |
| 7178 | return sanitize_err(env, insn, ret, off_reg, dst_reg); |
Daniel Borkmann | 0d6303d | 2019-01-03 00:58:30 +0100 | [diff] [blame] | 7179 | } |
| 7180 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7181 | return 0; |
| 7182 | } |
| 7183 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7184 | static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, |
| 7185 | struct bpf_reg_state *src_reg) |
| 7186 | { |
| 7187 | s32 smin_val = src_reg->s32_min_value; |
| 7188 | s32 smax_val = src_reg->s32_max_value; |
| 7189 | u32 umin_val = src_reg->u32_min_value; |
| 7190 | u32 umax_val = src_reg->u32_max_value; |
| 7191 | |
| 7192 | if (signed_add32_overflows(dst_reg->s32_min_value, smin_val) || |
| 7193 | signed_add32_overflows(dst_reg->s32_max_value, smax_val)) { |
| 7194 | dst_reg->s32_min_value = S32_MIN; |
| 7195 | dst_reg->s32_max_value = S32_MAX; |
| 7196 | } else { |
| 7197 | dst_reg->s32_min_value += smin_val; |
| 7198 | dst_reg->s32_max_value += smax_val; |
| 7199 | } |
| 7200 | if (dst_reg->u32_min_value + umin_val < umin_val || |
| 7201 | dst_reg->u32_max_value + umax_val < umax_val) { |
| 7202 | dst_reg->u32_min_value = 0; |
| 7203 | dst_reg->u32_max_value = U32_MAX; |
| 7204 | } else { |
| 7205 | dst_reg->u32_min_value += umin_val; |
| 7206 | dst_reg->u32_max_value += umax_val; |
| 7207 | } |
| 7208 | } |
| 7209 | |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7210 | static void scalar_min_max_add(struct bpf_reg_state *dst_reg, |
| 7211 | struct bpf_reg_state *src_reg) |
| 7212 | { |
| 7213 | s64 smin_val = src_reg->smin_value; |
| 7214 | s64 smax_val = src_reg->smax_value; |
| 7215 | u64 umin_val = src_reg->umin_value; |
| 7216 | u64 umax_val = src_reg->umax_value; |
| 7217 | |
| 7218 | if (signed_add_overflows(dst_reg->smin_value, smin_val) || |
| 7219 | signed_add_overflows(dst_reg->smax_value, smax_val)) { |
| 7220 | dst_reg->smin_value = S64_MIN; |
| 7221 | dst_reg->smax_value = S64_MAX; |
| 7222 | } else { |
| 7223 | dst_reg->smin_value += smin_val; |
| 7224 | dst_reg->smax_value += smax_val; |
| 7225 | } |
| 7226 | if (dst_reg->umin_value + umin_val < umin_val || |
| 7227 | dst_reg->umax_value + umax_val < umax_val) { |
| 7228 | dst_reg->umin_value = 0; |
| 7229 | dst_reg->umax_value = U64_MAX; |
| 7230 | } else { |
| 7231 | dst_reg->umin_value += umin_val; |
| 7232 | dst_reg->umax_value += umax_val; |
| 7233 | } |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7234 | } |
| 7235 | |
| 7236 | static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg, |
| 7237 | struct bpf_reg_state *src_reg) |
| 7238 | { |
| 7239 | s32 smin_val = src_reg->s32_min_value; |
| 7240 | s32 smax_val = src_reg->s32_max_value; |
| 7241 | u32 umin_val = src_reg->u32_min_value; |
| 7242 | u32 umax_val = src_reg->u32_max_value; |
| 7243 | |
| 7244 | if (signed_sub32_overflows(dst_reg->s32_min_value, smax_val) || |
| 7245 | signed_sub32_overflows(dst_reg->s32_max_value, smin_val)) { |
| 7246 | /* Overflow possible, we know nothing */ |
| 7247 | dst_reg->s32_min_value = S32_MIN; |
| 7248 | dst_reg->s32_max_value = S32_MAX; |
| 7249 | } else { |
| 7250 | dst_reg->s32_min_value -= smax_val; |
| 7251 | dst_reg->s32_max_value -= smin_val; |
| 7252 | } |
| 7253 | if (dst_reg->u32_min_value < umax_val) { |
| 7254 | /* Overflow possible, we know nothing */ |
| 7255 | dst_reg->u32_min_value = 0; |
| 7256 | dst_reg->u32_max_value = U32_MAX; |
| 7257 | } else { |
| 7258 | /* Cannot overflow (as long as bounds are consistent) */ |
| 7259 | dst_reg->u32_min_value -= umax_val; |
| 7260 | dst_reg->u32_max_value -= umin_val; |
| 7261 | } |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7262 | } |
| 7263 | |
| 7264 | static void scalar_min_max_sub(struct bpf_reg_state *dst_reg, |
| 7265 | struct bpf_reg_state *src_reg) |
| 7266 | { |
| 7267 | s64 smin_val = src_reg->smin_value; |
| 7268 | s64 smax_val = src_reg->smax_value; |
| 7269 | u64 umin_val = src_reg->umin_value; |
| 7270 | u64 umax_val = src_reg->umax_value; |
| 7271 | |
| 7272 | if (signed_sub_overflows(dst_reg->smin_value, smax_val) || |
| 7273 | signed_sub_overflows(dst_reg->smax_value, smin_val)) { |
| 7274 | /* Overflow possible, we know nothing */ |
| 7275 | dst_reg->smin_value = S64_MIN; |
| 7276 | dst_reg->smax_value = S64_MAX; |
| 7277 | } else { |
| 7278 | dst_reg->smin_value -= smax_val; |
| 7279 | dst_reg->smax_value -= smin_val; |
| 7280 | } |
| 7281 | if (dst_reg->umin_value < umax_val) { |
| 7282 | /* Overflow possible, we know nothing */ |
| 7283 | dst_reg->umin_value = 0; |
| 7284 | dst_reg->umax_value = U64_MAX; |
| 7285 | } else { |
| 7286 | /* Cannot overflow (as long as bounds are consistent) */ |
| 7287 | dst_reg->umin_value -= umax_val; |
| 7288 | dst_reg->umax_value -= umin_val; |
| 7289 | } |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7290 | } |
| 7291 | |
| 7292 | static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg, |
| 7293 | struct bpf_reg_state *src_reg) |
| 7294 | { |
| 7295 | s32 smin_val = src_reg->s32_min_value; |
| 7296 | u32 umin_val = src_reg->u32_min_value; |
| 7297 | u32 umax_val = src_reg->u32_max_value; |
| 7298 | |
| 7299 | if (smin_val < 0 || dst_reg->s32_min_value < 0) { |
| 7300 | /* Ain't nobody got time to multiply that sign */ |
| 7301 | __mark_reg32_unbounded(dst_reg); |
| 7302 | return; |
| 7303 | } |
| 7304 | /* Both values are positive, so we can work with unsigned and |
| 7305 | * copy the result to signed (unless it exceeds S32_MAX). |
| 7306 | */ |
| 7307 | if (umax_val > U16_MAX || dst_reg->u32_max_value > U16_MAX) { |
| 7308 | /* Potential overflow, we know nothing */ |
| 7309 | __mark_reg32_unbounded(dst_reg); |
| 7310 | return; |
| 7311 | } |
| 7312 | dst_reg->u32_min_value *= umin_val; |
| 7313 | dst_reg->u32_max_value *= umax_val; |
| 7314 | if (dst_reg->u32_max_value > S32_MAX) { |
| 7315 | /* Overflow possible, we know nothing */ |
| 7316 | dst_reg->s32_min_value = S32_MIN; |
| 7317 | dst_reg->s32_max_value = S32_MAX; |
| 7318 | } else { |
| 7319 | dst_reg->s32_min_value = dst_reg->u32_min_value; |
| 7320 | dst_reg->s32_max_value = dst_reg->u32_max_value; |
| 7321 | } |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7322 | } |
| 7323 | |
| 7324 | static void scalar_min_max_mul(struct bpf_reg_state *dst_reg, |
| 7325 | struct bpf_reg_state *src_reg) |
| 7326 | { |
| 7327 | s64 smin_val = src_reg->smin_value; |
| 7328 | u64 umin_val = src_reg->umin_value; |
| 7329 | u64 umax_val = src_reg->umax_value; |
| 7330 | |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7331 | if (smin_val < 0 || dst_reg->smin_value < 0) { |
| 7332 | /* Ain't nobody got time to multiply that sign */ |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7333 | __mark_reg64_unbounded(dst_reg); |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7334 | return; |
| 7335 | } |
| 7336 | /* Both values are positive, so we can work with unsigned and |
| 7337 | * copy the result to signed (unless it exceeds S64_MAX). |
| 7338 | */ |
| 7339 | if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) { |
| 7340 | /* Potential overflow, we know nothing */ |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7341 | __mark_reg64_unbounded(dst_reg); |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7342 | return; |
| 7343 | } |
| 7344 | dst_reg->umin_value *= umin_val; |
| 7345 | dst_reg->umax_value *= umax_val; |
| 7346 | if (dst_reg->umax_value > S64_MAX) { |
| 7347 | /* Overflow possible, we know nothing */ |
| 7348 | dst_reg->smin_value = S64_MIN; |
| 7349 | dst_reg->smax_value = S64_MAX; |
| 7350 | } else { |
| 7351 | dst_reg->smin_value = dst_reg->umin_value; |
| 7352 | dst_reg->smax_value = dst_reg->umax_value; |
| 7353 | } |
| 7354 | } |
| 7355 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7356 | static void scalar32_min_max_and(struct bpf_reg_state *dst_reg, |
| 7357 | struct bpf_reg_state *src_reg) |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7358 | { |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7359 | bool src_known = tnum_subreg_is_const(src_reg->var_off); |
| 7360 | bool dst_known = tnum_subreg_is_const(dst_reg->var_off); |
| 7361 | struct tnum var32_off = tnum_subreg(dst_reg->var_off); |
| 7362 | s32 smin_val = src_reg->s32_min_value; |
| 7363 | u32 umax_val = src_reg->u32_max_value; |
| 7364 | |
Daniel Borkmann | 049c4e1 | 2021-05-10 13:10:44 +0000 | [diff] [blame] | 7365 | if (src_known && dst_known) { |
| 7366 | __mark_reg32_known(dst_reg, var32_off.value); |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7367 | return; |
Daniel Borkmann | 049c4e1 | 2021-05-10 13:10:44 +0000 | [diff] [blame] | 7368 | } |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7369 | |
| 7370 | /* We get our minimum from the var_off, since that's inherently |
| 7371 | * bitwise. Our maximum is the minimum of the operands' maxima. |
| 7372 | */ |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7373 | dst_reg->u32_min_value = var32_off.value; |
| 7374 | dst_reg->u32_max_value = min(dst_reg->u32_max_value, umax_val); |
| 7375 | if (dst_reg->s32_min_value < 0 || smin_val < 0) { |
| 7376 | /* Lose signed bounds when ANDing negative numbers, |
| 7377 | * ain't nobody got time for that. |
| 7378 | */ |
| 7379 | dst_reg->s32_min_value = S32_MIN; |
| 7380 | dst_reg->s32_max_value = S32_MAX; |
| 7381 | } else { |
| 7382 | /* ANDing two positives gives a positive, so safe to |
| 7383 | * cast result into s64. |
| 7384 | */ |
| 7385 | dst_reg->s32_min_value = dst_reg->u32_min_value; |
| 7386 | dst_reg->s32_max_value = dst_reg->u32_max_value; |
| 7387 | } |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7388 | } |
| 7389 | |
| 7390 | static void scalar_min_max_and(struct bpf_reg_state *dst_reg, |
| 7391 | struct bpf_reg_state *src_reg) |
| 7392 | { |
| 7393 | bool src_known = tnum_is_const(src_reg->var_off); |
| 7394 | bool dst_known = tnum_is_const(dst_reg->var_off); |
| 7395 | s64 smin_val = src_reg->smin_value; |
| 7396 | u64 umax_val = src_reg->umax_value; |
| 7397 | |
| 7398 | if (src_known && dst_known) { |
John Fastabend | 4fbb38a | 2020-09-24 11:45:06 -0700 | [diff] [blame] | 7399 | __mark_reg_known(dst_reg, dst_reg->var_off.value); |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7400 | return; |
| 7401 | } |
| 7402 | |
| 7403 | /* We get our minimum from the var_off, since that's inherently |
| 7404 | * bitwise. Our maximum is the minimum of the operands' maxima. |
| 7405 | */ |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7406 | dst_reg->umin_value = dst_reg->var_off.value; |
| 7407 | dst_reg->umax_value = min(dst_reg->umax_value, umax_val); |
| 7408 | if (dst_reg->smin_value < 0 || smin_val < 0) { |
| 7409 | /* Lose signed bounds when ANDing negative numbers, |
| 7410 | * ain't nobody got time for that. |
| 7411 | */ |
| 7412 | dst_reg->smin_value = S64_MIN; |
| 7413 | dst_reg->smax_value = S64_MAX; |
| 7414 | } else { |
| 7415 | /* ANDing two positives gives a positive, so safe to |
| 7416 | * cast result into s64. |
| 7417 | */ |
| 7418 | dst_reg->smin_value = dst_reg->umin_value; |
| 7419 | dst_reg->smax_value = dst_reg->umax_value; |
| 7420 | } |
| 7421 | /* We may learn something more from the var_off */ |
| 7422 | __update_reg_bounds(dst_reg); |
| 7423 | } |
| 7424 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7425 | static void scalar32_min_max_or(struct bpf_reg_state *dst_reg, |
| 7426 | struct bpf_reg_state *src_reg) |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7427 | { |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7428 | bool src_known = tnum_subreg_is_const(src_reg->var_off); |
| 7429 | bool dst_known = tnum_subreg_is_const(dst_reg->var_off); |
| 7430 | struct tnum var32_off = tnum_subreg(dst_reg->var_off); |
Daniel Borkmann | 5b9fbeb | 2020-10-07 15:48:58 +0200 | [diff] [blame] | 7431 | s32 smin_val = src_reg->s32_min_value; |
| 7432 | u32 umin_val = src_reg->u32_min_value; |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7433 | |
Daniel Borkmann | 049c4e1 | 2021-05-10 13:10:44 +0000 | [diff] [blame] | 7434 | if (src_known && dst_known) { |
| 7435 | __mark_reg32_known(dst_reg, var32_off.value); |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7436 | return; |
Daniel Borkmann | 049c4e1 | 2021-05-10 13:10:44 +0000 | [diff] [blame] | 7437 | } |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7438 | |
| 7439 | /* We get our maximum from the var_off, and our minimum is the |
| 7440 | * maximum of the operands' minima |
| 7441 | */ |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7442 | dst_reg->u32_min_value = max(dst_reg->u32_min_value, umin_val); |
| 7443 | dst_reg->u32_max_value = var32_off.value | var32_off.mask; |
| 7444 | if (dst_reg->s32_min_value < 0 || smin_val < 0) { |
| 7445 | /* Lose signed bounds when ORing negative numbers, |
| 7446 | * ain't nobody got time for that. |
| 7447 | */ |
| 7448 | dst_reg->s32_min_value = S32_MIN; |
| 7449 | dst_reg->s32_max_value = S32_MAX; |
| 7450 | } else { |
| 7451 | /* ORing two positives gives a positive, so safe to |
| 7452 | * cast result into s64. |
| 7453 | */ |
Daniel Borkmann | 5b9fbeb | 2020-10-07 15:48:58 +0200 | [diff] [blame] | 7454 | dst_reg->s32_min_value = dst_reg->u32_min_value; |
| 7455 | dst_reg->s32_max_value = dst_reg->u32_max_value; |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7456 | } |
| 7457 | } |
| 7458 | |
| 7459 | static void scalar_min_max_or(struct bpf_reg_state *dst_reg, |
| 7460 | struct bpf_reg_state *src_reg) |
| 7461 | { |
| 7462 | bool src_known = tnum_is_const(src_reg->var_off); |
| 7463 | bool dst_known = tnum_is_const(dst_reg->var_off); |
| 7464 | s64 smin_val = src_reg->smin_value; |
| 7465 | u64 umin_val = src_reg->umin_value; |
| 7466 | |
| 7467 | if (src_known && dst_known) { |
John Fastabend | 4fbb38a | 2020-09-24 11:45:06 -0700 | [diff] [blame] | 7468 | __mark_reg_known(dst_reg, dst_reg->var_off.value); |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7469 | return; |
| 7470 | } |
| 7471 | |
| 7472 | /* We get our maximum from the var_off, and our minimum is the |
| 7473 | * maximum of the operands' minima |
| 7474 | */ |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7475 | dst_reg->umin_value = max(dst_reg->umin_value, umin_val); |
| 7476 | dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; |
| 7477 | if (dst_reg->smin_value < 0 || smin_val < 0) { |
| 7478 | /* Lose signed bounds when ORing negative numbers, |
| 7479 | * ain't nobody got time for that. |
| 7480 | */ |
| 7481 | dst_reg->smin_value = S64_MIN; |
| 7482 | dst_reg->smax_value = S64_MAX; |
| 7483 | } else { |
| 7484 | /* ORing two positives gives a positive, so safe to |
| 7485 | * cast result into s64. |
| 7486 | */ |
| 7487 | dst_reg->smin_value = dst_reg->umin_value; |
| 7488 | dst_reg->smax_value = dst_reg->umax_value; |
| 7489 | } |
| 7490 | /* We may learn something more from the var_off */ |
| 7491 | __update_reg_bounds(dst_reg); |
| 7492 | } |
| 7493 | |
Yonghong Song | 2921c90 | 2020-08-24 23:46:08 -0700 | [diff] [blame] | 7494 | static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg, |
| 7495 | struct bpf_reg_state *src_reg) |
| 7496 | { |
| 7497 | bool src_known = tnum_subreg_is_const(src_reg->var_off); |
| 7498 | bool dst_known = tnum_subreg_is_const(dst_reg->var_off); |
| 7499 | struct tnum var32_off = tnum_subreg(dst_reg->var_off); |
| 7500 | s32 smin_val = src_reg->s32_min_value; |
| 7501 | |
Daniel Borkmann | 049c4e1 | 2021-05-10 13:10:44 +0000 | [diff] [blame] | 7502 | if (src_known && dst_known) { |
| 7503 | __mark_reg32_known(dst_reg, var32_off.value); |
Yonghong Song | 2921c90 | 2020-08-24 23:46:08 -0700 | [diff] [blame] | 7504 | return; |
Daniel Borkmann | 049c4e1 | 2021-05-10 13:10:44 +0000 | [diff] [blame] | 7505 | } |
Yonghong Song | 2921c90 | 2020-08-24 23:46:08 -0700 | [diff] [blame] | 7506 | |
| 7507 | /* We get both minimum and maximum from the var32_off. */ |
| 7508 | dst_reg->u32_min_value = var32_off.value; |
| 7509 | dst_reg->u32_max_value = var32_off.value | var32_off.mask; |
| 7510 | |
| 7511 | if (dst_reg->s32_min_value >= 0 && smin_val >= 0) { |
| 7512 | /* XORing two positive sign numbers gives a positive, |
| 7513 | * so safe to cast u32 result into s32. |
| 7514 | */ |
| 7515 | dst_reg->s32_min_value = dst_reg->u32_min_value; |
| 7516 | dst_reg->s32_max_value = dst_reg->u32_max_value; |
| 7517 | } else { |
| 7518 | dst_reg->s32_min_value = S32_MIN; |
| 7519 | dst_reg->s32_max_value = S32_MAX; |
| 7520 | } |
| 7521 | } |
| 7522 | |
| 7523 | static void scalar_min_max_xor(struct bpf_reg_state *dst_reg, |
| 7524 | struct bpf_reg_state *src_reg) |
| 7525 | { |
| 7526 | bool src_known = tnum_is_const(src_reg->var_off); |
| 7527 | bool dst_known = tnum_is_const(dst_reg->var_off); |
| 7528 | s64 smin_val = src_reg->smin_value; |
| 7529 | |
| 7530 | if (src_known && dst_known) { |
| 7531 | /* dst_reg->var_off.value has been updated earlier */ |
| 7532 | __mark_reg_known(dst_reg, dst_reg->var_off.value); |
| 7533 | return; |
| 7534 | } |
| 7535 | |
| 7536 | /* We get both minimum and maximum from the var_off. */ |
| 7537 | dst_reg->umin_value = dst_reg->var_off.value; |
| 7538 | dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; |
| 7539 | |
| 7540 | if (dst_reg->smin_value >= 0 && smin_val >= 0) { |
| 7541 | /* XORing two positive sign numbers gives a positive, |
| 7542 | * so safe to cast u64 result into s64. |
| 7543 | */ |
| 7544 | dst_reg->smin_value = dst_reg->umin_value; |
| 7545 | dst_reg->smax_value = dst_reg->umax_value; |
| 7546 | } else { |
| 7547 | dst_reg->smin_value = S64_MIN; |
| 7548 | dst_reg->smax_value = S64_MAX; |
| 7549 | } |
| 7550 | |
| 7551 | __update_reg_bounds(dst_reg); |
| 7552 | } |
| 7553 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7554 | static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, |
| 7555 | u64 umin_val, u64 umax_val) |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7556 | { |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7557 | /* We lose all sign bit information (except what we can pick |
| 7558 | * up from var_off) |
| 7559 | */ |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7560 | dst_reg->s32_min_value = S32_MIN; |
| 7561 | dst_reg->s32_max_value = S32_MAX; |
| 7562 | /* If we might shift our top bit out, then we know nothing */ |
| 7563 | if (umax_val > 31 || dst_reg->u32_max_value > 1ULL << (31 - umax_val)) { |
| 7564 | dst_reg->u32_min_value = 0; |
| 7565 | dst_reg->u32_max_value = U32_MAX; |
| 7566 | } else { |
| 7567 | dst_reg->u32_min_value <<= umin_val; |
| 7568 | dst_reg->u32_max_value <<= umax_val; |
| 7569 | } |
| 7570 | } |
| 7571 | |
| 7572 | static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, |
| 7573 | struct bpf_reg_state *src_reg) |
| 7574 | { |
| 7575 | u32 umax_val = src_reg->u32_max_value; |
| 7576 | u32 umin_val = src_reg->u32_min_value; |
| 7577 | /* u32 alu operation will zext upper bits */ |
| 7578 | struct tnum subreg = tnum_subreg(dst_reg->var_off); |
| 7579 | |
| 7580 | __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); |
| 7581 | dst_reg->var_off = tnum_subreg(tnum_lshift(subreg, umin_val)); |
| 7582 | /* Not required but being careful mark reg64 bounds as unknown so |
| 7583 | * that we are forced to pick them up from tnum and zext later and |
| 7584 | * if some path skips this step we are still safe. |
| 7585 | */ |
| 7586 | __mark_reg64_unbounded(dst_reg); |
| 7587 | __update_reg32_bounds(dst_reg); |
| 7588 | } |
| 7589 | |
| 7590 | static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg, |
| 7591 | u64 umin_val, u64 umax_val) |
| 7592 | { |
| 7593 | /* Special case <<32 because it is a common compiler pattern to sign |
| 7594 | * extend subreg by doing <<32 s>>32. In this case if 32bit bounds are |
| 7595 | * positive we know this shift will also be positive so we can track |
| 7596 | * bounds correctly. Otherwise we lose all sign bit information except |
| 7597 | * what we can pick up from var_off. Perhaps we can generalize this |
| 7598 | * later to shifts of any length. |
| 7599 | */ |
| 7600 | if (umin_val == 32 && umax_val == 32 && dst_reg->s32_max_value >= 0) |
| 7601 | dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32; |
| 7602 | else |
| 7603 | dst_reg->smax_value = S64_MAX; |
| 7604 | |
| 7605 | if (umin_val == 32 && umax_val == 32 && dst_reg->s32_min_value >= 0) |
| 7606 | dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32; |
| 7607 | else |
| 7608 | dst_reg->smin_value = S64_MIN; |
| 7609 | |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7610 | /* If we might shift our top bit out, then we know nothing */ |
| 7611 | if (dst_reg->umax_value > 1ULL << (63 - umax_val)) { |
| 7612 | dst_reg->umin_value = 0; |
| 7613 | dst_reg->umax_value = U64_MAX; |
| 7614 | } else { |
| 7615 | dst_reg->umin_value <<= umin_val; |
| 7616 | dst_reg->umax_value <<= umax_val; |
| 7617 | } |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7618 | } |
| 7619 | |
| 7620 | static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg, |
| 7621 | struct bpf_reg_state *src_reg) |
| 7622 | { |
| 7623 | u64 umax_val = src_reg->umax_value; |
| 7624 | u64 umin_val = src_reg->umin_value; |
| 7625 | |
| 7626 | /* scalar64 calc uses 32bit unshifted bounds so must be called first */ |
| 7627 | __scalar64_min_max_lsh(dst_reg, umin_val, umax_val); |
| 7628 | __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); |
| 7629 | |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7630 | dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val); |
| 7631 | /* We may learn something more from the var_off */ |
| 7632 | __update_reg_bounds(dst_reg); |
| 7633 | } |
| 7634 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7635 | static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg, |
| 7636 | struct bpf_reg_state *src_reg) |
| 7637 | { |
| 7638 | struct tnum subreg = tnum_subreg(dst_reg->var_off); |
| 7639 | u32 umax_val = src_reg->u32_max_value; |
| 7640 | u32 umin_val = src_reg->u32_min_value; |
| 7641 | |
| 7642 | /* BPF_RSH is an unsigned shift. If the value in dst_reg might |
| 7643 | * be negative, then either: |
| 7644 | * 1) src_reg might be zero, so the sign bit of the result is |
| 7645 | * unknown, so we lose our signed bounds |
| 7646 | * 2) it's known negative, thus the unsigned bounds capture the |
| 7647 | * signed bounds |
| 7648 | * 3) the signed bounds cross zero, so they tell us nothing |
| 7649 | * about the result |
| 7650 | * If the value in dst_reg is known nonnegative, then again the |
Tobias Klauser | 18b24d7 | 2021-01-21 18:43:24 +0100 | [diff] [blame] | 7651 | * unsigned bounds capture the signed bounds. |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7652 | * Thus, in all cases it suffices to blow away our signed bounds |
| 7653 | * and rely on inferring new ones from the unsigned bounds and |
| 7654 | * var_off of the result. |
| 7655 | */ |
| 7656 | dst_reg->s32_min_value = S32_MIN; |
| 7657 | dst_reg->s32_max_value = S32_MAX; |
| 7658 | |
| 7659 | dst_reg->var_off = tnum_rshift(subreg, umin_val); |
| 7660 | dst_reg->u32_min_value >>= umax_val; |
| 7661 | dst_reg->u32_max_value >>= umin_val; |
| 7662 | |
| 7663 | __mark_reg64_unbounded(dst_reg); |
| 7664 | __update_reg32_bounds(dst_reg); |
| 7665 | } |
| 7666 | |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7667 | static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg, |
| 7668 | struct bpf_reg_state *src_reg) |
| 7669 | { |
| 7670 | u64 umax_val = src_reg->umax_value; |
| 7671 | u64 umin_val = src_reg->umin_value; |
| 7672 | |
| 7673 | /* BPF_RSH is an unsigned shift. If the value in dst_reg might |
| 7674 | * be negative, then either: |
| 7675 | * 1) src_reg might be zero, so the sign bit of the result is |
| 7676 | * unknown, so we lose our signed bounds |
| 7677 | * 2) it's known negative, thus the unsigned bounds capture the |
| 7678 | * signed bounds |
| 7679 | * 3) the signed bounds cross zero, so they tell us nothing |
| 7680 | * about the result |
| 7681 | * If the value in dst_reg is known nonnegative, then again the |
Tobias Klauser | 18b24d7 | 2021-01-21 18:43:24 +0100 | [diff] [blame] | 7682 | * unsigned bounds capture the signed bounds. |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7683 | * Thus, in all cases it suffices to blow away our signed bounds |
| 7684 | * and rely on inferring new ones from the unsigned bounds and |
| 7685 | * var_off of the result. |
| 7686 | */ |
| 7687 | dst_reg->smin_value = S64_MIN; |
| 7688 | dst_reg->smax_value = S64_MAX; |
| 7689 | dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val); |
| 7690 | dst_reg->umin_value >>= umax_val; |
| 7691 | dst_reg->umax_value >>= umin_val; |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7692 | |
| 7693 | /* Its not easy to operate on alu32 bounds here because it depends |
| 7694 | * on bits being shifted in. Take easy way out and mark unbounded |
| 7695 | * so we can recalculate later from tnum. |
| 7696 | */ |
| 7697 | __mark_reg32_unbounded(dst_reg); |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7698 | __update_reg_bounds(dst_reg); |
| 7699 | } |
| 7700 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7701 | static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg, |
| 7702 | struct bpf_reg_state *src_reg) |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7703 | { |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7704 | u64 umin_val = src_reg->u32_min_value; |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7705 | |
| 7706 | /* Upon reaching here, src_known is true and |
| 7707 | * umax_val is equal to umin_val. |
| 7708 | */ |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7709 | dst_reg->s32_min_value = (u32)(((s32)dst_reg->s32_min_value) >> umin_val); |
| 7710 | dst_reg->s32_max_value = (u32)(((s32)dst_reg->s32_max_value) >> umin_val); |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7711 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7712 | dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32); |
| 7713 | |
| 7714 | /* blow away the dst_reg umin_value/umax_value and rely on |
| 7715 | * dst_reg var_off to refine the result. |
| 7716 | */ |
| 7717 | dst_reg->u32_min_value = 0; |
| 7718 | dst_reg->u32_max_value = U32_MAX; |
| 7719 | |
| 7720 | __mark_reg64_unbounded(dst_reg); |
| 7721 | __update_reg32_bounds(dst_reg); |
| 7722 | } |
| 7723 | |
| 7724 | static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg, |
| 7725 | struct bpf_reg_state *src_reg) |
| 7726 | { |
| 7727 | u64 umin_val = src_reg->umin_value; |
| 7728 | |
| 7729 | /* Upon reaching here, src_known is true and umax_val is equal |
| 7730 | * to umin_val. |
| 7731 | */ |
| 7732 | dst_reg->smin_value >>= umin_val; |
| 7733 | dst_reg->smax_value >>= umin_val; |
| 7734 | |
| 7735 | dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64); |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7736 | |
| 7737 | /* blow away the dst_reg umin_value/umax_value and rely on |
| 7738 | * dst_reg var_off to refine the result. |
| 7739 | */ |
| 7740 | dst_reg->umin_value = 0; |
| 7741 | dst_reg->umax_value = U64_MAX; |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7742 | |
| 7743 | /* Its not easy to operate on alu32 bounds here because it depends |
| 7744 | * on bits being shifted in from upper 32-bits. Take easy way out |
| 7745 | * and mark unbounded so we can recalculate later from tnum. |
| 7746 | */ |
| 7747 | __mark_reg32_unbounded(dst_reg); |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7748 | __update_reg_bounds(dst_reg); |
| 7749 | } |
| 7750 | |
Jann Horn | 468f6ea | 2017-12-18 20:11:56 -0800 | [diff] [blame] | 7751 | /* WARNING: This function does calculations on 64-bit values, but the actual |
| 7752 | * execution may occur on 32-bit values. Therefore, things like bitshifts |
| 7753 | * need extra checks in the 32-bit case. |
| 7754 | */ |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7755 | static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, |
| 7756 | struct bpf_insn *insn, |
| 7757 | struct bpf_reg_state *dst_reg, |
| 7758 | struct bpf_reg_state src_reg) |
| 7759 | { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 7760 | struct bpf_reg_state *regs = cur_regs(env); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7761 | u8 opcode = BPF_OP(insn->code); |
Mao Wenan | b0b3fb6 | 2020-04-18 09:37:35 +0800 | [diff] [blame] | 7762 | bool src_known; |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 7763 | s64 smin_val, smax_val; |
| 7764 | u64 umin_val, umax_val; |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7765 | s32 s32_min_val, s32_max_val; |
| 7766 | u32 u32_min_val, u32_max_val; |
Jann Horn | 468f6ea | 2017-12-18 20:11:56 -0800 | [diff] [blame] | 7767 | u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32; |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7768 | bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64); |
Daniel Borkmann | a6aaece | 2021-03-23 09:30:01 +0100 | [diff] [blame] | 7769 | int ret; |
Jann Horn | b799207 | 2018-10-05 18:17:59 +0200 | [diff] [blame] | 7770 | |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 7771 | smin_val = src_reg.smin_value; |
| 7772 | smax_val = src_reg.smax_value; |
| 7773 | umin_val = src_reg.umin_value; |
| 7774 | umax_val = src_reg.umax_value; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7775 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7776 | s32_min_val = src_reg.s32_min_value; |
| 7777 | s32_max_val = src_reg.s32_max_value; |
| 7778 | u32_min_val = src_reg.u32_min_value; |
| 7779 | u32_max_val = src_reg.u32_max_value; |
| 7780 | |
| 7781 | if (alu32) { |
| 7782 | src_known = tnum_subreg_is_const(src_reg.var_off); |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7783 | if ((src_known && |
| 7784 | (s32_min_val != s32_max_val || u32_min_val != u32_max_val)) || |
| 7785 | s32_min_val > s32_max_val || u32_min_val > u32_max_val) { |
| 7786 | /* Taint dst register if offset had invalid bounds |
| 7787 | * derived from e.g. dead branches. |
| 7788 | */ |
| 7789 | __mark_reg_unknown(env, dst_reg); |
| 7790 | return 0; |
| 7791 | } |
| 7792 | } else { |
| 7793 | src_known = tnum_is_const(src_reg.var_off); |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7794 | if ((src_known && |
| 7795 | (smin_val != smax_val || umin_val != umax_val)) || |
| 7796 | smin_val > smax_val || umin_val > umax_val) { |
| 7797 | /* Taint dst register if offset had invalid bounds |
| 7798 | * derived from e.g. dead branches. |
| 7799 | */ |
| 7800 | __mark_reg_unknown(env, dst_reg); |
| 7801 | return 0; |
| 7802 | } |
Daniel Borkmann | 6f16101 | 2018-01-18 01:15:21 +0100 | [diff] [blame] | 7803 | } |
| 7804 | |
Alexei Starovoitov | bb7f0f9 | 2017-12-18 20:12:00 -0800 | [diff] [blame] | 7805 | if (!src_known && |
| 7806 | opcode != BPF_ADD && opcode != BPF_SUB && opcode != BPF_AND) { |
Daniel Borkmann | f54c789 | 2019-12-22 23:37:40 +0100 | [diff] [blame] | 7807 | __mark_reg_unknown(env, dst_reg); |
Alexei Starovoitov | bb7f0f9 | 2017-12-18 20:12:00 -0800 | [diff] [blame] | 7808 | return 0; |
| 7809 | } |
| 7810 | |
Daniel Borkmann | f528819 | 2021-03-24 11:25:39 +0100 | [diff] [blame] | 7811 | if (sanitize_needed(opcode)) { |
| 7812 | ret = sanitize_val_alu(env, insn); |
| 7813 | if (ret < 0) |
| 7814 | return sanitize_err(env, insn, ret, NULL, NULL); |
| 7815 | } |
| 7816 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7817 | /* Calculate sign/unsigned bounds and tnum for alu32 and alu64 bit ops. |
| 7818 | * There are two classes of instructions: The first class we track both |
| 7819 | * alu32 and alu64 sign/unsigned bounds independently this provides the |
| 7820 | * greatest amount of precision when alu operations are mixed with jmp32 |
| 7821 | * operations. These operations are BPF_ADD, BPF_SUB, BPF_MUL, BPF_ADD, |
| 7822 | * and BPF_OR. This is possible because these ops have fairly easy to |
| 7823 | * understand and calculate behavior in both 32-bit and 64-bit alu ops. |
| 7824 | * See alu32 verifier tests for examples. The second class of |
| 7825 | * operations, BPF_LSH, BPF_RSH, and BPF_ARSH, however are not so easy |
| 7826 | * with regards to tracking sign/unsigned bounds because the bits may |
| 7827 | * cross subreg boundaries in the alu64 case. When this happens we mark |
| 7828 | * the reg unbounded in the subreg bound space and use the resulting |
| 7829 | * tnum to calculate an approximation of the sign/unsigned bounds. |
| 7830 | */ |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7831 | switch (opcode) { |
| 7832 | case BPF_ADD: |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7833 | scalar32_min_max_add(dst_reg, &src_reg); |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7834 | scalar_min_max_add(dst_reg, &src_reg); |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7835 | dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7836 | break; |
| 7837 | case BPF_SUB: |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7838 | scalar32_min_max_sub(dst_reg, &src_reg); |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7839 | scalar_min_max_sub(dst_reg, &src_reg); |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7840 | dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off); |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 7841 | break; |
| 7842 | case BPF_MUL: |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7843 | dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off); |
| 7844 | scalar32_min_max_mul(dst_reg, &src_reg); |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7845 | scalar_min_max_mul(dst_reg, &src_reg); |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 7846 | break; |
| 7847 | case BPF_AND: |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7848 | dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); |
| 7849 | scalar32_min_max_and(dst_reg, &src_reg); |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7850 | scalar_min_max_and(dst_reg, &src_reg); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7851 | break; |
| 7852 | case BPF_OR: |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7853 | dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); |
| 7854 | scalar32_min_max_or(dst_reg, &src_reg); |
John Fastabend | 07cd263 | 2020-03-24 10:38:15 -0700 | [diff] [blame] | 7855 | scalar_min_max_or(dst_reg, &src_reg); |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 7856 | break; |
Yonghong Song | 2921c90 | 2020-08-24 23:46:08 -0700 | [diff] [blame] | 7857 | case BPF_XOR: |
| 7858 | dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off); |
| 7859 | scalar32_min_max_xor(dst_reg, &src_reg); |
| 7860 | scalar_min_max_xor(dst_reg, &src_reg); |
| 7861 | break; |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 7862 | case BPF_LSH: |
Jann Horn | 468f6ea | 2017-12-18 20:11:56 -0800 | [diff] [blame] | 7863 | if (umax_val >= insn_bitness) { |
| 7864 | /* Shifts greater than 31 or 63 are undefined. |
| 7865 | * This includes shifts by a negative number. |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 7866 | */ |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 7867 | mark_reg_unknown(env, regs, insn->dst_reg); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7868 | break; |
| 7869 | } |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7870 | if (alu32) |
| 7871 | scalar32_min_max_lsh(dst_reg, &src_reg); |
| 7872 | else |
| 7873 | scalar_min_max_lsh(dst_reg, &src_reg); |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 7874 | break; |
| 7875 | case BPF_RSH: |
Jann Horn | 468f6ea | 2017-12-18 20:11:56 -0800 | [diff] [blame] | 7876 | if (umax_val >= insn_bitness) { |
| 7877 | /* Shifts greater than 31 or 63 are undefined. |
| 7878 | * This includes shifts by a negative number. |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 7879 | */ |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 7880 | mark_reg_unknown(env, regs, insn->dst_reg); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7881 | break; |
| 7882 | } |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7883 | if (alu32) |
| 7884 | scalar32_min_max_rsh(dst_reg, &src_reg); |
| 7885 | else |
| 7886 | scalar_min_max_rsh(dst_reg, &src_reg); |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 7887 | break; |
Yonghong Song | 9cbe1f5a | 2018-04-28 22:28:11 -0700 | [diff] [blame] | 7888 | case BPF_ARSH: |
| 7889 | if (umax_val >= insn_bitness) { |
| 7890 | /* Shifts greater than 31 or 63 are undefined. |
| 7891 | * This includes shifts by a negative number. |
| 7892 | */ |
| 7893 | mark_reg_unknown(env, regs, insn->dst_reg); |
| 7894 | break; |
| 7895 | } |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7896 | if (alu32) |
| 7897 | scalar32_min_max_arsh(dst_reg, &src_reg); |
| 7898 | else |
| 7899 | scalar_min_max_arsh(dst_reg, &src_reg); |
Yonghong Song | 9cbe1f5a | 2018-04-28 22:28:11 -0700 | [diff] [blame] | 7900 | break; |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 7901 | default: |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 7902 | mark_reg_unknown(env, regs, insn->dst_reg); |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 7903 | break; |
| 7904 | } |
| 7905 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 7906 | /* ALU32 ops are zero extended into 64bit register */ |
| 7907 | if (alu32) |
| 7908 | zext_32_to_64(dst_reg); |
Jann Horn | 468f6ea | 2017-12-18 20:11:56 -0800 | [diff] [blame] | 7909 | |
John Fastabend | 294f2fc | 2020-03-24 10:38:37 -0700 | [diff] [blame] | 7910 | __update_reg_bounds(dst_reg); |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 7911 | __reg_deduce_bounds(dst_reg); |
| 7912 | __reg_bound_offset(dst_reg); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7913 | return 0; |
| 7914 | } |
| 7915 | |
| 7916 | /* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max |
| 7917 | * and var_off. |
| 7918 | */ |
| 7919 | static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, |
| 7920 | struct bpf_insn *insn) |
| 7921 | { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 7922 | struct bpf_verifier_state *vstate = env->cur_state; |
| 7923 | struct bpf_func_state *state = vstate->frame[vstate->curframe]; |
| 7924 | struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7925 | struct bpf_reg_state *ptr_reg = NULL, off_reg = {0}; |
| 7926 | u8 opcode = BPF_OP(insn->code); |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 7927 | int err; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7928 | |
| 7929 | dst_reg = ®s[insn->dst_reg]; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7930 | src_reg = NULL; |
| 7931 | if (dst_reg->type != SCALAR_VALUE) |
| 7932 | ptr_reg = dst_reg; |
Alexei Starovoitov | 7574883 | 2020-10-08 18:12:37 -0700 | [diff] [blame] | 7933 | else |
| 7934 | /* Make sure ID is cleared otherwise dst_reg min/max could be |
| 7935 | * incorrectly propagated into other registers by find_equal_scalars() |
| 7936 | */ |
| 7937 | dst_reg->id = 0; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7938 | if (BPF_SRC(insn->code) == BPF_X) { |
| 7939 | src_reg = ®s[insn->src_reg]; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7940 | if (src_reg->type != SCALAR_VALUE) { |
| 7941 | if (dst_reg->type != SCALAR_VALUE) { |
| 7942 | /* Combining two pointers by any ALU op yields |
Alexei Starovoitov | 82abbf8 | 2017-12-18 20:15:20 -0800 | [diff] [blame] | 7943 | * an arbitrary scalar. Disallow all math except |
| 7944 | * pointer subtraction |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7945 | */ |
Alexei Starovoitov | dd06682 | 2018-09-12 14:06:10 -0700 | [diff] [blame] | 7946 | if (opcode == BPF_SUB && env->allow_ptr_leaks) { |
Alexei Starovoitov | 82abbf8 | 2017-12-18 20:15:20 -0800 | [diff] [blame] | 7947 | mark_reg_unknown(env, regs, insn->dst_reg); |
| 7948 | return 0; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7949 | } |
Alexei Starovoitov | 82abbf8 | 2017-12-18 20:15:20 -0800 | [diff] [blame] | 7950 | verbose(env, "R%d pointer %s pointer prohibited\n", |
| 7951 | insn->dst_reg, |
| 7952 | bpf_alu_string[opcode >> 4]); |
| 7953 | return -EACCES; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7954 | } else { |
| 7955 | /* scalar += pointer |
| 7956 | * This is legal, but we have to reverse our |
| 7957 | * src/dest handling in computing the range |
| 7958 | */ |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 7959 | err = mark_chain_precision(env, insn->dst_reg); |
| 7960 | if (err) |
| 7961 | return err; |
Alexei Starovoitov | 82abbf8 | 2017-12-18 20:15:20 -0800 | [diff] [blame] | 7962 | return adjust_ptr_min_max_vals(env, insn, |
| 7963 | src_reg, dst_reg); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7964 | } |
| 7965 | } else if (ptr_reg) { |
| 7966 | /* pointer += scalar */ |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 7967 | err = mark_chain_precision(env, insn->src_reg); |
| 7968 | if (err) |
| 7969 | return err; |
Alexei Starovoitov | 82abbf8 | 2017-12-18 20:15:20 -0800 | [diff] [blame] | 7970 | return adjust_ptr_min_max_vals(env, insn, |
| 7971 | dst_reg, src_reg); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7972 | } |
| 7973 | } else { |
| 7974 | /* Pretend the src is a reg with a known value, since we only |
| 7975 | * need to be able to read from this state. |
| 7976 | */ |
| 7977 | off_reg.type = SCALAR_VALUE; |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 7978 | __mark_reg_known(&off_reg, insn->imm); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7979 | src_reg = &off_reg; |
Alexei Starovoitov | 82abbf8 | 2017-12-18 20:15:20 -0800 | [diff] [blame] | 7980 | if (ptr_reg) /* pointer += K */ |
| 7981 | return adjust_ptr_min_max_vals(env, insn, |
| 7982 | ptr_reg, src_reg); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7983 | } |
| 7984 | |
| 7985 | /* Got here implies adding two SCALAR_VALUEs */ |
| 7986 | if (WARN_ON_ONCE(ptr_reg)) { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 7987 | print_verifier_state(env, state); |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 7988 | verbose(env, "verifier internal error: unexpected ptr_reg\n"); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7989 | return -EINVAL; |
| 7990 | } |
| 7991 | if (WARN_ON(!src_reg)) { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 7992 | print_verifier_state(env, state); |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 7993 | verbose(env, "verifier internal error: no src_reg\n"); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 7994 | return -EINVAL; |
| 7995 | } |
| 7996 | return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 7997 | } |
| 7998 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 7999 | /* check validity of 32-bit and 64-bit arithmetic operations */ |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 8000 | static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8001 | { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 8002 | struct bpf_reg_state *regs = cur_regs(env); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8003 | u8 opcode = BPF_OP(insn->code); |
| 8004 | int err; |
| 8005 | |
| 8006 | if (opcode == BPF_END || opcode == BPF_NEG) { |
| 8007 | if (opcode == BPF_NEG) { |
| 8008 | if (BPF_SRC(insn->code) != 0 || |
| 8009 | insn->src_reg != BPF_REG_0 || |
| 8010 | insn->off != 0 || insn->imm != 0) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 8011 | verbose(env, "BPF_NEG uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8012 | return -EINVAL; |
| 8013 | } |
| 8014 | } else { |
| 8015 | if (insn->src_reg != BPF_REG_0 || insn->off != 0 || |
Edward Cree | e67b8a6 | 2017-09-15 14:37:38 +0100 | [diff] [blame] | 8016 | (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) || |
| 8017 | BPF_CLASS(insn->code) == BPF_ALU64) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 8018 | verbose(env, "BPF_END uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8019 | return -EINVAL; |
| 8020 | } |
| 8021 | } |
| 8022 | |
| 8023 | /* check src operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 8024 | err = check_reg_arg(env, insn->dst_reg, SRC_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8025 | if (err) |
| 8026 | return err; |
| 8027 | |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 8028 | if (is_pointer_value(env, insn->dst_reg)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 8029 | verbose(env, "R%d pointer arithmetic prohibited\n", |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 8030 | insn->dst_reg); |
| 8031 | return -EACCES; |
| 8032 | } |
| 8033 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8034 | /* check dest operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 8035 | err = check_reg_arg(env, insn->dst_reg, DST_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8036 | if (err) |
| 8037 | return err; |
| 8038 | |
| 8039 | } else if (opcode == BPF_MOV) { |
| 8040 | |
| 8041 | if (BPF_SRC(insn->code) == BPF_X) { |
| 8042 | if (insn->imm != 0 || insn->off != 0) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 8043 | verbose(env, "BPF_MOV uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8044 | return -EINVAL; |
| 8045 | } |
| 8046 | |
| 8047 | /* check src operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 8048 | err = check_reg_arg(env, insn->src_reg, SRC_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8049 | if (err) |
| 8050 | return err; |
| 8051 | } else { |
| 8052 | if (insn->src_reg != BPF_REG_0 || insn->off != 0) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 8053 | verbose(env, "BPF_MOV uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8054 | return -EINVAL; |
| 8055 | } |
| 8056 | } |
| 8057 | |
Arthur Fabre | fbeb160 | 2018-07-31 18:17:22 +0100 | [diff] [blame] | 8058 | /* check dest operand, mark as required later */ |
| 8059 | err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8060 | if (err) |
| 8061 | return err; |
| 8062 | |
| 8063 | if (BPF_SRC(insn->code) == BPF_X) { |
Jiong Wang | e434b8c | 2018-12-07 12:16:18 -0500 | [diff] [blame] | 8064 | struct bpf_reg_state *src_reg = regs + insn->src_reg; |
| 8065 | struct bpf_reg_state *dst_reg = regs + insn->dst_reg; |
| 8066 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8067 | if (BPF_CLASS(insn->code) == BPF_ALU64) { |
| 8068 | /* case: R1 = R2 |
| 8069 | * copy register state to dest reg |
| 8070 | */ |
Alexei Starovoitov | 7574883 | 2020-10-08 18:12:37 -0700 | [diff] [blame] | 8071 | if (src_reg->type == SCALAR_VALUE && !src_reg->id) |
| 8072 | /* Assign src and dst registers the same ID |
| 8073 | * that will be used by find_equal_scalars() |
| 8074 | * to propagate min/max range. |
| 8075 | */ |
| 8076 | src_reg->id = ++env->id_gen; |
Jiong Wang | e434b8c | 2018-12-07 12:16:18 -0500 | [diff] [blame] | 8077 | *dst_reg = *src_reg; |
| 8078 | dst_reg->live |= REG_LIVE_WRITTEN; |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 8079 | dst_reg->subreg_def = DEF_NOT_SUBREG; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8080 | } else { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 8081 | /* R1 = (u32) R2 */ |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 8082 | if (is_pointer_value(env, insn->src_reg)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 8083 | verbose(env, |
| 8084 | "R%d partial copy of pointer\n", |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 8085 | insn->src_reg); |
| 8086 | return -EACCES; |
Jiong Wang | e434b8c | 2018-12-07 12:16:18 -0500 | [diff] [blame] | 8087 | } else if (src_reg->type == SCALAR_VALUE) { |
| 8088 | *dst_reg = *src_reg; |
Alexei Starovoitov | 7574883 | 2020-10-08 18:12:37 -0700 | [diff] [blame] | 8089 | /* Make sure ID is cleared otherwise |
| 8090 | * dst_reg min/max could be incorrectly |
| 8091 | * propagated into src_reg by find_equal_scalars() |
| 8092 | */ |
| 8093 | dst_reg->id = 0; |
Jiong Wang | e434b8c | 2018-12-07 12:16:18 -0500 | [diff] [blame] | 8094 | dst_reg->live |= REG_LIVE_WRITTEN; |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 8095 | dst_reg->subreg_def = env->insn_idx + 1; |
Jiong Wang | e434b8c | 2018-12-07 12:16:18 -0500 | [diff] [blame] | 8096 | } else { |
| 8097 | mark_reg_unknown(env, regs, |
| 8098 | insn->dst_reg); |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 8099 | } |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 8100 | zext_32_to_64(dst_reg); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8101 | } |
| 8102 | } else { |
| 8103 | /* case: R = imm |
| 8104 | * remember the value we stored into this reg |
| 8105 | */ |
Arthur Fabre | fbeb160 | 2018-07-31 18:17:22 +0100 | [diff] [blame] | 8106 | /* clear any state __mark_reg_known doesn't set */ |
| 8107 | mark_reg_unknown(env, regs, insn->dst_reg); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 8108 | regs[insn->dst_reg].type = SCALAR_VALUE; |
Jann Horn | 95a762e | 2017-12-18 20:11:54 -0800 | [diff] [blame] | 8109 | if (BPF_CLASS(insn->code) == BPF_ALU64) { |
| 8110 | __mark_reg_known(regs + insn->dst_reg, |
| 8111 | insn->imm); |
| 8112 | } else { |
| 8113 | __mark_reg_known(regs + insn->dst_reg, |
| 8114 | (u32)insn->imm); |
| 8115 | } |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8116 | } |
| 8117 | |
| 8118 | } else if (opcode > BPF_END) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 8119 | verbose(env, "invalid BPF_ALU opcode %x\n", opcode); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8120 | return -EINVAL; |
| 8121 | |
| 8122 | } else { /* all other ALU ops: and, sub, xor, add, ... */ |
| 8123 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8124 | if (BPF_SRC(insn->code) == BPF_X) { |
| 8125 | if (insn->imm != 0 || insn->off != 0) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 8126 | verbose(env, "BPF_ALU uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8127 | return -EINVAL; |
| 8128 | } |
| 8129 | /* check src1 operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 8130 | err = check_reg_arg(env, insn->src_reg, SRC_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8131 | if (err) |
| 8132 | return err; |
| 8133 | } else { |
| 8134 | if (insn->src_reg != BPF_REG_0 || insn->off != 0) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 8135 | verbose(env, "BPF_ALU uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8136 | return -EINVAL; |
| 8137 | } |
| 8138 | } |
| 8139 | |
| 8140 | /* check src2 operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 8141 | err = check_reg_arg(env, insn->dst_reg, SRC_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8142 | if (err) |
| 8143 | return err; |
| 8144 | |
| 8145 | if ((opcode == BPF_MOD || opcode == BPF_DIV) && |
| 8146 | BPF_SRC(insn->code) == BPF_K && insn->imm == 0) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 8147 | verbose(env, "div by zero\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8148 | return -EINVAL; |
| 8149 | } |
| 8150 | |
Rabin Vincent | 229394e8 | 2016-01-12 20:17:08 +0100 | [diff] [blame] | 8151 | if ((opcode == BPF_LSH || opcode == BPF_RSH || |
| 8152 | opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) { |
| 8153 | int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32; |
| 8154 | |
| 8155 | if (insn->imm < 0 || insn->imm >= size) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 8156 | verbose(env, "invalid shift %d\n", insn->imm); |
Rabin Vincent | 229394e8 | 2016-01-12 20:17:08 +0100 | [diff] [blame] | 8157 | return -EINVAL; |
| 8158 | } |
| 8159 | } |
| 8160 | |
Alexei Starovoitov | 1a0dc1a | 2016-05-05 19:49:09 -0700 | [diff] [blame] | 8161 | /* check dest operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 8162 | err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); |
Alexei Starovoitov | 1a0dc1a | 2016-05-05 19:49:09 -0700 | [diff] [blame] | 8163 | if (err) |
| 8164 | return err; |
| 8165 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 8166 | return adjust_reg_min_max_vals(env, insn); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8167 | } |
| 8168 | |
| 8169 | return 0; |
| 8170 | } |
| 8171 | |
Paul Chaignon | c6a9efa | 2019-04-24 21:50:42 +0200 | [diff] [blame] | 8172 | static void __find_good_pkt_pointers(struct bpf_func_state *state, |
| 8173 | struct bpf_reg_state *dst_reg, |
Alexei Starovoitov | 6d94e74 | 2020-11-10 19:12:11 -0800 | [diff] [blame] | 8174 | enum bpf_reg_type type, int new_range) |
Paul Chaignon | c6a9efa | 2019-04-24 21:50:42 +0200 | [diff] [blame] | 8175 | { |
| 8176 | struct bpf_reg_state *reg; |
| 8177 | int i; |
| 8178 | |
| 8179 | for (i = 0; i < MAX_BPF_REG; i++) { |
| 8180 | reg = &state->regs[i]; |
| 8181 | if (reg->type == type && reg->id == dst_reg->id) |
| 8182 | /* keep the maximum range already checked */ |
| 8183 | reg->range = max(reg->range, new_range); |
| 8184 | } |
| 8185 | |
| 8186 | bpf_for_each_spilled_reg(i, state, reg) { |
| 8187 | if (!reg) |
| 8188 | continue; |
| 8189 | if (reg->type == type && reg->id == dst_reg->id) |
| 8190 | reg->range = max(reg->range, new_range); |
| 8191 | } |
| 8192 | } |
| 8193 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 8194 | static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 8195 | struct bpf_reg_state *dst_reg, |
David S. Miller | f8ddadc | 2017-10-22 13:36:53 +0100 | [diff] [blame] | 8196 | enum bpf_reg_type type, |
Daniel Borkmann | fb2a311 | 2017-10-21 02:34:21 +0200 | [diff] [blame] | 8197 | bool range_right_open) |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 8198 | { |
Alexei Starovoitov | 6d94e74 | 2020-11-10 19:12:11 -0800 | [diff] [blame] | 8199 | int new_range, i; |
Daniel Borkmann | 2d2be8c | 2016-09-08 01:03:42 +0200 | [diff] [blame] | 8200 | |
Daniel Borkmann | fb2a311 | 2017-10-21 02:34:21 +0200 | [diff] [blame] | 8201 | if (dst_reg->off < 0 || |
| 8202 | (dst_reg->off == 0 && range_right_open)) |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 8203 | /* This doesn't give us any range */ |
| 8204 | return; |
| 8205 | |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 8206 | if (dst_reg->umax_value > MAX_PACKET_OFF || |
| 8207 | dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF) |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 8208 | /* Risk of overflow. For instance, ptr + (1<<63) may be less |
| 8209 | * than pkt_end, but that's because it's also less than pkt. |
| 8210 | */ |
| 8211 | return; |
| 8212 | |
Daniel Borkmann | fb2a311 | 2017-10-21 02:34:21 +0200 | [diff] [blame] | 8213 | new_range = dst_reg->off; |
| 8214 | if (range_right_open) |
| 8215 | new_range--; |
| 8216 | |
| 8217 | /* Examples for register markings: |
Daniel Borkmann | 2d2be8c | 2016-09-08 01:03:42 +0200 | [diff] [blame] | 8218 | * |
Daniel Borkmann | fb2a311 | 2017-10-21 02:34:21 +0200 | [diff] [blame] | 8219 | * pkt_data in dst register: |
Daniel Borkmann | 2d2be8c | 2016-09-08 01:03:42 +0200 | [diff] [blame] | 8220 | * |
| 8221 | * r2 = r3; |
| 8222 | * r2 += 8; |
| 8223 | * if (r2 > pkt_end) goto <handle exception> |
| 8224 | * <access okay> |
| 8225 | * |
Daniel Borkmann | b4e432f | 2017-08-10 01:40:02 +0200 | [diff] [blame] | 8226 | * r2 = r3; |
| 8227 | * r2 += 8; |
| 8228 | * if (r2 < pkt_end) goto <access okay> |
| 8229 | * <handle exception> |
| 8230 | * |
Daniel Borkmann | 2d2be8c | 2016-09-08 01:03:42 +0200 | [diff] [blame] | 8231 | * Where: |
| 8232 | * r2 == dst_reg, pkt_end == src_reg |
| 8233 | * r2=pkt(id=n,off=8,r=0) |
| 8234 | * r3=pkt(id=n,off=0,r=0) |
| 8235 | * |
Daniel Borkmann | fb2a311 | 2017-10-21 02:34:21 +0200 | [diff] [blame] | 8236 | * pkt_data in src register: |
Daniel Borkmann | 2d2be8c | 2016-09-08 01:03:42 +0200 | [diff] [blame] | 8237 | * |
| 8238 | * r2 = r3; |
| 8239 | * r2 += 8; |
| 8240 | * if (pkt_end >= r2) goto <access okay> |
| 8241 | * <handle exception> |
| 8242 | * |
Daniel Borkmann | b4e432f | 2017-08-10 01:40:02 +0200 | [diff] [blame] | 8243 | * r2 = r3; |
| 8244 | * r2 += 8; |
| 8245 | * if (pkt_end <= r2) goto <handle exception> |
| 8246 | * <access okay> |
| 8247 | * |
Daniel Borkmann | 2d2be8c | 2016-09-08 01:03:42 +0200 | [diff] [blame] | 8248 | * Where: |
| 8249 | * pkt_end == dst_reg, r2 == src_reg |
| 8250 | * r2=pkt(id=n,off=8,r=0) |
| 8251 | * r3=pkt(id=n,off=0,r=0) |
| 8252 | * |
| 8253 | * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8) |
Daniel Borkmann | fb2a311 | 2017-10-21 02:34:21 +0200 | [diff] [blame] | 8254 | * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8) |
| 8255 | * and [r3, r3 + 8-1) respectively is safe to access depending on |
| 8256 | * the check. |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 8257 | */ |
Daniel Borkmann | 2d2be8c | 2016-09-08 01:03:42 +0200 | [diff] [blame] | 8258 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 8259 | /* If our ids match, then we must have the same max_value. And we |
| 8260 | * don't care about the other reg's fixed offset, since if it's too big |
| 8261 | * the range won't allow anything. |
| 8262 | * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16. |
| 8263 | */ |
Paul Chaignon | c6a9efa | 2019-04-24 21:50:42 +0200 | [diff] [blame] | 8264 | for (i = 0; i <= vstate->curframe; i++) |
| 8265 | __find_good_pkt_pointers(vstate->frame[i], dst_reg, type, |
| 8266 | new_range); |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 8267 | } |
| 8268 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 8269 | static int is_branch32_taken(struct bpf_reg_state *reg, u32 val, u8 opcode) |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 8270 | { |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 8271 | struct tnum subreg = tnum_subreg(reg->var_off); |
| 8272 | s32 sval = (s32)val; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 8273 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 8274 | switch (opcode) { |
| 8275 | case BPF_JEQ: |
| 8276 | if (tnum_is_const(subreg)) |
| 8277 | return !!tnum_equals_const(subreg, val); |
| 8278 | break; |
| 8279 | case BPF_JNE: |
| 8280 | if (tnum_is_const(subreg)) |
| 8281 | return !tnum_equals_const(subreg, val); |
| 8282 | break; |
| 8283 | case BPF_JSET: |
| 8284 | if ((~subreg.mask & subreg.value) & val) |
| 8285 | return 1; |
| 8286 | if (!((subreg.mask | subreg.value) & val)) |
| 8287 | return 0; |
| 8288 | break; |
| 8289 | case BPF_JGT: |
| 8290 | if (reg->u32_min_value > val) |
| 8291 | return 1; |
| 8292 | else if (reg->u32_max_value <= val) |
| 8293 | return 0; |
| 8294 | break; |
| 8295 | case BPF_JSGT: |
| 8296 | if (reg->s32_min_value > sval) |
| 8297 | return 1; |
Daniel Borkmann | ee114dd | 2021-02-05 17:20:14 +0100 | [diff] [blame] | 8298 | else if (reg->s32_max_value <= sval) |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 8299 | return 0; |
| 8300 | break; |
| 8301 | case BPF_JLT: |
| 8302 | if (reg->u32_max_value < val) |
| 8303 | return 1; |
| 8304 | else if (reg->u32_min_value >= val) |
| 8305 | return 0; |
| 8306 | break; |
| 8307 | case BPF_JSLT: |
| 8308 | if (reg->s32_max_value < sval) |
| 8309 | return 1; |
| 8310 | else if (reg->s32_min_value >= sval) |
| 8311 | return 0; |
| 8312 | break; |
| 8313 | case BPF_JGE: |
| 8314 | if (reg->u32_min_value >= val) |
| 8315 | return 1; |
| 8316 | else if (reg->u32_max_value < val) |
| 8317 | return 0; |
| 8318 | break; |
| 8319 | case BPF_JSGE: |
| 8320 | if (reg->s32_min_value >= sval) |
| 8321 | return 1; |
| 8322 | else if (reg->s32_max_value < sval) |
| 8323 | return 0; |
| 8324 | break; |
| 8325 | case BPF_JLE: |
| 8326 | if (reg->u32_max_value <= val) |
| 8327 | return 1; |
| 8328 | else if (reg->u32_min_value > val) |
| 8329 | return 0; |
| 8330 | break; |
| 8331 | case BPF_JSLE: |
| 8332 | if (reg->s32_max_value <= sval) |
| 8333 | return 1; |
| 8334 | else if (reg->s32_min_value > sval) |
| 8335 | return 0; |
| 8336 | break; |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 8337 | } |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 8338 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 8339 | return -1; |
| 8340 | } |
| 8341 | |
| 8342 | |
| 8343 | static int is_branch64_taken(struct bpf_reg_state *reg, u64 val, u8 opcode) |
| 8344 | { |
| 8345 | s64 sval = (s64)val; |
| 8346 | |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 8347 | switch (opcode) { |
| 8348 | case BPF_JEQ: |
| 8349 | if (tnum_is_const(reg->var_off)) |
| 8350 | return !!tnum_equals_const(reg->var_off, val); |
| 8351 | break; |
| 8352 | case BPF_JNE: |
| 8353 | if (tnum_is_const(reg->var_off)) |
| 8354 | return !tnum_equals_const(reg->var_off, val); |
| 8355 | break; |
Jakub Kicinski | 960ea05 | 2018-12-19 22:13:04 -0800 | [diff] [blame] | 8356 | case BPF_JSET: |
| 8357 | if ((~reg->var_off.mask & reg->var_off.value) & val) |
| 8358 | return 1; |
| 8359 | if (!((reg->var_off.mask | reg->var_off.value) & val)) |
| 8360 | return 0; |
| 8361 | break; |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 8362 | case BPF_JGT: |
| 8363 | if (reg->umin_value > val) |
| 8364 | return 1; |
| 8365 | else if (reg->umax_value <= val) |
| 8366 | return 0; |
| 8367 | break; |
| 8368 | case BPF_JSGT: |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 8369 | if (reg->smin_value > sval) |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 8370 | return 1; |
Daniel Borkmann | ee114dd | 2021-02-05 17:20:14 +0100 | [diff] [blame] | 8371 | else if (reg->smax_value <= sval) |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 8372 | return 0; |
| 8373 | break; |
| 8374 | case BPF_JLT: |
| 8375 | if (reg->umax_value < val) |
| 8376 | return 1; |
| 8377 | else if (reg->umin_value >= val) |
| 8378 | return 0; |
| 8379 | break; |
| 8380 | case BPF_JSLT: |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 8381 | if (reg->smax_value < sval) |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 8382 | return 1; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 8383 | else if (reg->smin_value >= sval) |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 8384 | return 0; |
| 8385 | break; |
| 8386 | case BPF_JGE: |
| 8387 | if (reg->umin_value >= val) |
| 8388 | return 1; |
| 8389 | else if (reg->umax_value < val) |
| 8390 | return 0; |
| 8391 | break; |
| 8392 | case BPF_JSGE: |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 8393 | if (reg->smin_value >= sval) |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 8394 | return 1; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 8395 | else if (reg->smax_value < sval) |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 8396 | return 0; |
| 8397 | break; |
| 8398 | case BPF_JLE: |
| 8399 | if (reg->umax_value <= val) |
| 8400 | return 1; |
| 8401 | else if (reg->umin_value > val) |
| 8402 | return 0; |
| 8403 | break; |
| 8404 | case BPF_JSLE: |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 8405 | if (reg->smax_value <= sval) |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 8406 | return 1; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 8407 | else if (reg->smin_value > sval) |
Alexei Starovoitov | 4f7b3e8 | 2018-12-03 22:46:05 -0800 | [diff] [blame] | 8408 | return 0; |
| 8409 | break; |
| 8410 | } |
| 8411 | |
| 8412 | return -1; |
| 8413 | } |
| 8414 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 8415 | /* compute branch direction of the expression "if (reg opcode val) goto target;" |
| 8416 | * and return: |
| 8417 | * 1 - branch will be taken and "goto target" will be executed |
| 8418 | * 0 - branch will not be taken and fall-through to next insn |
| 8419 | * -1 - unknown. Example: "if (reg < 5)" is unknown when register value |
| 8420 | * range [0,10] |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 8421 | */ |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 8422 | static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode, |
| 8423 | bool is_jmp32) |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 8424 | { |
John Fastabend | cac616d | 2020-05-21 13:07:26 -0700 | [diff] [blame] | 8425 | if (__is_pointer_value(false, reg)) { |
| 8426 | if (!reg_type_not_null(reg->type)) |
| 8427 | return -1; |
| 8428 | |
| 8429 | /* If pointer is valid tests against zero will fail so we can |
| 8430 | * use this to direct branch taken. |
| 8431 | */ |
| 8432 | if (val != 0) |
| 8433 | return -1; |
| 8434 | |
| 8435 | switch (opcode) { |
| 8436 | case BPF_JEQ: |
| 8437 | return 0; |
| 8438 | case BPF_JNE: |
| 8439 | return 1; |
| 8440 | default: |
| 8441 | return -1; |
| 8442 | } |
| 8443 | } |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 8444 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 8445 | if (is_jmp32) |
| 8446 | return is_branch32_taken(reg, val, opcode); |
| 8447 | return is_branch64_taken(reg, val, opcode); |
Jann Horn | 604dca5 | 2020-03-30 18:03:23 +0200 | [diff] [blame] | 8448 | } |
| 8449 | |
Alexei Starovoitov | 6d94e74 | 2020-11-10 19:12:11 -0800 | [diff] [blame] | 8450 | static int flip_opcode(u32 opcode) |
| 8451 | { |
| 8452 | /* How can we transform "a <op> b" into "b <op> a"? */ |
| 8453 | static const u8 opcode_flip[16] = { |
| 8454 | /* these stay the same */ |
| 8455 | [BPF_JEQ >> 4] = BPF_JEQ, |
| 8456 | [BPF_JNE >> 4] = BPF_JNE, |
| 8457 | [BPF_JSET >> 4] = BPF_JSET, |
| 8458 | /* these swap "lesser" and "greater" (L and G in the opcodes) */ |
| 8459 | [BPF_JGE >> 4] = BPF_JLE, |
| 8460 | [BPF_JGT >> 4] = BPF_JLT, |
| 8461 | [BPF_JLE >> 4] = BPF_JGE, |
| 8462 | [BPF_JLT >> 4] = BPF_JGT, |
| 8463 | [BPF_JSGE >> 4] = BPF_JSLE, |
| 8464 | [BPF_JSGT >> 4] = BPF_JSLT, |
| 8465 | [BPF_JSLE >> 4] = BPF_JSGE, |
| 8466 | [BPF_JSLT >> 4] = BPF_JSGT |
| 8467 | }; |
| 8468 | return opcode_flip[opcode >> 4]; |
| 8469 | } |
| 8470 | |
| 8471 | static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg, |
| 8472 | struct bpf_reg_state *src_reg, |
| 8473 | u8 opcode) |
| 8474 | { |
| 8475 | struct bpf_reg_state *pkt; |
| 8476 | |
| 8477 | if (src_reg->type == PTR_TO_PACKET_END) { |
| 8478 | pkt = dst_reg; |
| 8479 | } else if (dst_reg->type == PTR_TO_PACKET_END) { |
| 8480 | pkt = src_reg; |
| 8481 | opcode = flip_opcode(opcode); |
| 8482 | } else { |
| 8483 | return -1; |
| 8484 | } |
| 8485 | |
| 8486 | if (pkt->range >= 0) |
| 8487 | return -1; |
| 8488 | |
| 8489 | switch (opcode) { |
| 8490 | case BPF_JLE: |
| 8491 | /* pkt <= pkt_end */ |
| 8492 | fallthrough; |
| 8493 | case BPF_JGT: |
| 8494 | /* pkt > pkt_end */ |
| 8495 | if (pkt->range == BEYOND_PKT_END) |
| 8496 | /* pkt has at last one extra byte beyond pkt_end */ |
| 8497 | return opcode == BPF_JGT; |
| 8498 | break; |
| 8499 | case BPF_JLT: |
| 8500 | /* pkt < pkt_end */ |
| 8501 | fallthrough; |
| 8502 | case BPF_JGE: |
| 8503 | /* pkt >= pkt_end */ |
| 8504 | if (pkt->range == BEYOND_PKT_END || pkt->range == AT_PKT_END) |
| 8505 | return opcode == BPF_JGE; |
| 8506 | break; |
| 8507 | } |
| 8508 | return -1; |
| 8509 | } |
| 8510 | |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 8511 | /* Adjusts the register min/max values in the case that the dst_reg is the |
| 8512 | * variable register that we are working on, and src_reg is a constant or we're |
| 8513 | * simply doing a BPF_K check. |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 8514 | * In JEQ/JNE cases we also adjust the var_off values. |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 8515 | */ |
| 8516 | static void reg_set_min_max(struct bpf_reg_state *true_reg, |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 8517 | struct bpf_reg_state *false_reg, |
| 8518 | u64 val, u32 val32, |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 8519 | u8 opcode, bool is_jmp32) |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 8520 | { |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 8521 | struct tnum false_32off = tnum_subreg(false_reg->var_off); |
| 8522 | struct tnum false_64off = false_reg->var_off; |
| 8523 | struct tnum true_32off = tnum_subreg(true_reg->var_off); |
| 8524 | struct tnum true_64off = true_reg->var_off; |
| 8525 | s64 sval = (s64)val; |
| 8526 | s32 sval32 = (s32)val32; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 8527 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 8528 | /* If the dst_reg is a pointer, we can't learn anything about its |
| 8529 | * variable offset from the compare (unless src_reg were a pointer into |
| 8530 | * the same object, but we don't bother with that. |
| 8531 | * Since false_reg and true_reg have the same type by construction, we |
| 8532 | * only need to check one of them for pointerness. |
| 8533 | */ |
| 8534 | if (__is_pointer_value(false, false_reg)) |
| 8535 | return; |
Daniel Borkmann | 4cabc5b | 2017-07-21 00:00:21 +0200 | [diff] [blame] | 8536 | |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 8537 | switch (opcode) { |
| 8538 | case BPF_JEQ: |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 8539 | case BPF_JNE: |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 8540 | { |
| 8541 | struct bpf_reg_state *reg = |
| 8542 | opcode == BPF_JEQ ? true_reg : false_reg; |
| 8543 | |
Alexei Starovoitov | e688c3d | 2020-10-14 10:56:08 -0700 | [diff] [blame] | 8544 | /* JEQ/JNE comparison doesn't change the register equivalence. |
| 8545 | * r1 = r2; |
| 8546 | * if (r1 == 42) goto label; |
| 8547 | * ... |
| 8548 | * label: // here both r1 and r2 are known to be 42. |
| 8549 | * |
| 8550 | * Hence when marking register as known preserve it's ID. |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 8551 | */ |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 8552 | if (is_jmp32) |
| 8553 | __mark_reg32_known(reg, val32); |
| 8554 | else |
Alexei Starovoitov | e688c3d | 2020-10-14 10:56:08 -0700 | [diff] [blame] | 8555 | ___mark_reg_known(reg, val); |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 8556 | break; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 8557 | } |
Jakub Kicinski | 960ea05 | 2018-12-19 22:13:04 -0800 | [diff] [blame] | 8558 | case BPF_JSET: |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 8559 | if (is_jmp32) { |
| 8560 | false_32off = tnum_and(false_32off, tnum_const(~val32)); |
| 8561 | if (is_power_of_2(val32)) |
| 8562 | true_32off = tnum_or(true_32off, |
| 8563 | tnum_const(val32)); |
| 8564 | } else { |
| 8565 | false_64off = tnum_and(false_64off, tnum_const(~val)); |
| 8566 | if (is_power_of_2(val)) |
| 8567 | true_64off = tnum_or(true_64off, |
| 8568 | tnum_const(val)); |
| 8569 | } |
Jakub Kicinski | 960ea05 | 2018-12-19 22:13:04 -0800 | [diff] [blame] | 8570 | break; |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 8571 | case BPF_JGE: |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 8572 | case BPF_JGT: |
| 8573 | { |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 8574 | if (is_jmp32) { |
| 8575 | u32 false_umax = opcode == BPF_JGT ? val32 : val32 - 1; |
| 8576 | u32 true_umin = opcode == BPF_JGT ? val32 + 1 : val32; |
| 8577 | |
| 8578 | false_reg->u32_max_value = min(false_reg->u32_max_value, |
| 8579 | false_umax); |
| 8580 | true_reg->u32_min_value = max(true_reg->u32_min_value, |
| 8581 | true_umin); |
| 8582 | } else { |
| 8583 | u64 false_umax = opcode == BPF_JGT ? val : val - 1; |
| 8584 | u64 true_umin = opcode == BPF_JGT ? val + 1 : val; |
| 8585 | |
| 8586 | false_reg->umax_value = min(false_reg->umax_value, false_umax); |
| 8587 | true_reg->umin_value = max(true_reg->umin_value, true_umin); |
| 8588 | } |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 8589 | break; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 8590 | } |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 8591 | case BPF_JSGE: |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 8592 | case BPF_JSGT: |
| 8593 | { |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 8594 | if (is_jmp32) { |
| 8595 | s32 false_smax = opcode == BPF_JSGT ? sval32 : sval32 - 1; |
| 8596 | s32 true_smin = opcode == BPF_JSGT ? sval32 + 1 : sval32; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 8597 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 8598 | false_reg->s32_max_value = min(false_reg->s32_max_value, false_smax); |
| 8599 | true_reg->s32_min_value = max(true_reg->s32_min_value, true_smin); |
| 8600 | } else { |
| 8601 | s64 false_smax = opcode == BPF_JSGT ? sval : sval - 1; |
| 8602 | s64 true_smin = opcode == BPF_JSGT ? sval + 1 : sval; |
| 8603 | |
| 8604 | false_reg->smax_value = min(false_reg->smax_value, false_smax); |
| 8605 | true_reg->smin_value = max(true_reg->smin_value, true_smin); |
| 8606 | } |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 8607 | break; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 8608 | } |
Daniel Borkmann | b4e432f | 2017-08-10 01:40:02 +0200 | [diff] [blame] | 8609 | case BPF_JLE: |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 8610 | case BPF_JLT: |
| 8611 | { |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 8612 | if (is_jmp32) { |
| 8613 | u32 false_umin = opcode == BPF_JLT ? val32 : val32 + 1; |
| 8614 | u32 true_umax = opcode == BPF_JLT ? val32 - 1 : val32; |
| 8615 | |
| 8616 | false_reg->u32_min_value = max(false_reg->u32_min_value, |
| 8617 | false_umin); |
| 8618 | true_reg->u32_max_value = min(true_reg->u32_max_value, |
| 8619 | true_umax); |
| 8620 | } else { |
| 8621 | u64 false_umin = opcode == BPF_JLT ? val : val + 1; |
| 8622 | u64 true_umax = opcode == BPF_JLT ? val - 1 : val; |
| 8623 | |
| 8624 | false_reg->umin_value = max(false_reg->umin_value, false_umin); |
| 8625 | true_reg->umax_value = min(true_reg->umax_value, true_umax); |
| 8626 | } |
Daniel Borkmann | b4e432f | 2017-08-10 01:40:02 +0200 | [diff] [blame] | 8627 | break; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 8628 | } |
Daniel Borkmann | b4e432f | 2017-08-10 01:40:02 +0200 | [diff] [blame] | 8629 | case BPF_JSLE: |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 8630 | case BPF_JSLT: |
| 8631 | { |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 8632 | if (is_jmp32) { |
| 8633 | s32 false_smin = opcode == BPF_JSLT ? sval32 : sval32 + 1; |
| 8634 | s32 true_smax = opcode == BPF_JSLT ? sval32 - 1 : sval32; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 8635 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 8636 | false_reg->s32_min_value = max(false_reg->s32_min_value, false_smin); |
| 8637 | true_reg->s32_max_value = min(true_reg->s32_max_value, true_smax); |
| 8638 | } else { |
| 8639 | s64 false_smin = opcode == BPF_JSLT ? sval : sval + 1; |
| 8640 | s64 true_smax = opcode == BPF_JSLT ? sval - 1 : sval; |
| 8641 | |
| 8642 | false_reg->smin_value = max(false_reg->smin_value, false_smin); |
| 8643 | true_reg->smax_value = min(true_reg->smax_value, true_smax); |
| 8644 | } |
Daniel Borkmann | b4e432f | 2017-08-10 01:40:02 +0200 | [diff] [blame] | 8645 | break; |
Jiong Wang | a72dafa | 2019-01-26 12:26:00 -0500 | [diff] [blame] | 8646 | } |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 8647 | default: |
Jann Horn | 0fc31b1 | 2020-03-30 18:03:24 +0200 | [diff] [blame] | 8648 | return; |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 8649 | } |
| 8650 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 8651 | if (is_jmp32) { |
| 8652 | false_reg->var_off = tnum_or(tnum_clear_subreg(false_64off), |
| 8653 | tnum_subreg(false_32off)); |
| 8654 | true_reg->var_off = tnum_or(tnum_clear_subreg(true_64off), |
| 8655 | tnum_subreg(true_32off)); |
| 8656 | __reg_combine_32_into_64(false_reg); |
| 8657 | __reg_combine_32_into_64(true_reg); |
| 8658 | } else { |
| 8659 | false_reg->var_off = false_64off; |
| 8660 | true_reg->var_off = true_64off; |
| 8661 | __reg_combine_64_into_32(false_reg); |
| 8662 | __reg_combine_64_into_32(true_reg); |
| 8663 | } |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 8664 | } |
| 8665 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 8666 | /* Same as above, but for the case that dst_reg holds a constant and src_reg is |
| 8667 | * the variable reg. |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 8668 | */ |
| 8669 | static void reg_set_min_max_inv(struct bpf_reg_state *true_reg, |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 8670 | struct bpf_reg_state *false_reg, |
| 8671 | u64 val, u32 val32, |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 8672 | u8 opcode, bool is_jmp32) |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 8673 | { |
Alexei Starovoitov | 6d94e74 | 2020-11-10 19:12:11 -0800 | [diff] [blame] | 8674 | opcode = flip_opcode(opcode); |
Jann Horn | 0fc31b1 | 2020-03-30 18:03:24 +0200 | [diff] [blame] | 8675 | /* This uses zero as "not present in table"; luckily the zero opcode, |
| 8676 | * BPF_JA, can't get here. |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 8677 | */ |
Jann Horn | 0fc31b1 | 2020-03-30 18:03:24 +0200 | [diff] [blame] | 8678 | if (opcode) |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 8679 | reg_set_min_max(true_reg, false_reg, val, val32, opcode, is_jmp32); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 8680 | } |
| 8681 | |
| 8682 | /* Regs are known to be equal, so intersect their min/max/var_off */ |
| 8683 | static void __reg_combine_min_max(struct bpf_reg_state *src_reg, |
| 8684 | struct bpf_reg_state *dst_reg) |
| 8685 | { |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 8686 | src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value, |
| 8687 | dst_reg->umin_value); |
| 8688 | src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value, |
| 8689 | dst_reg->umax_value); |
| 8690 | src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value, |
| 8691 | dst_reg->smin_value); |
| 8692 | src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value, |
| 8693 | dst_reg->smax_value); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 8694 | src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off, |
| 8695 | dst_reg->var_off); |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 8696 | /* We might have learned new bounds from the var_off. */ |
| 8697 | __update_reg_bounds(src_reg); |
| 8698 | __update_reg_bounds(dst_reg); |
| 8699 | /* We might have learned something about the sign bit. */ |
| 8700 | __reg_deduce_bounds(src_reg); |
| 8701 | __reg_deduce_bounds(dst_reg); |
| 8702 | /* We might have learned some bits from the bounds. */ |
| 8703 | __reg_bound_offset(src_reg); |
| 8704 | __reg_bound_offset(dst_reg); |
| 8705 | /* Intersecting with the old var_off might have improved our bounds |
| 8706 | * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), |
| 8707 | * then new var_off is (0; 0x7f...fc) which improves our umax. |
| 8708 | */ |
| 8709 | __update_reg_bounds(src_reg); |
| 8710 | __update_reg_bounds(dst_reg); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 8711 | } |
| 8712 | |
| 8713 | static void reg_combine_min_max(struct bpf_reg_state *true_src, |
| 8714 | struct bpf_reg_state *true_dst, |
| 8715 | struct bpf_reg_state *false_src, |
| 8716 | struct bpf_reg_state *false_dst, |
| 8717 | u8 opcode) |
| 8718 | { |
| 8719 | switch (opcode) { |
| 8720 | case BPF_JEQ: |
| 8721 | __reg_combine_min_max(true_src, true_dst); |
| 8722 | break; |
| 8723 | case BPF_JNE: |
| 8724 | __reg_combine_min_max(false_src, false_dst); |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 8725 | break; |
Daniel Borkmann | 4cabc5b | 2017-07-21 00:00:21 +0200 | [diff] [blame] | 8726 | } |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 8727 | } |
| 8728 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 8729 | static void mark_ptr_or_null_reg(struct bpf_func_state *state, |
| 8730 | struct bpf_reg_state *reg, u32 id, |
Joe Stringer | 840b961 | 2018-10-02 13:35:32 -0700 | [diff] [blame] | 8731 | bool is_null) |
Thomas Graf | 57a09bf | 2016-10-18 19:51:19 +0200 | [diff] [blame] | 8732 | { |
Martin KaFai Lau | 93c230e | 2020-10-19 12:42:12 -0700 | [diff] [blame] | 8733 | if (reg_type_may_be_null(reg->type) && reg->id == id && |
| 8734 | !WARN_ON_ONCE(!reg->id)) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 8735 | /* Old offset (both fixed and variable parts) should |
| 8736 | * have been known-zero, because we don't allow pointer |
| 8737 | * arithmetic on pointers that might be NULL. |
| 8738 | */ |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 8739 | if (WARN_ON_ONCE(reg->smin_value || reg->smax_value || |
| 8740 | !tnum_equals_const(reg->var_off, 0) || |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 8741 | reg->off)) { |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 8742 | __mark_reg_known_zero(reg); |
| 8743 | reg->off = 0; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 8744 | } |
| 8745 | if (is_null) { |
| 8746 | reg->type = SCALAR_VALUE; |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 8747 | /* We don't need id and ref_obj_id from this point |
| 8748 | * onwards anymore, thus we should better reset it, |
| 8749 | * so that state pruning has chances to take effect. |
| 8750 | */ |
| 8751 | reg->id = 0; |
| 8752 | reg->ref_obj_id = 0; |
Dmitrii Banshchikov | 4ddb741 | 2021-02-13 00:56:40 +0400 | [diff] [blame] | 8753 | |
| 8754 | return; |
| 8755 | } |
| 8756 | |
| 8757 | mark_ptr_not_null_reg(reg); |
| 8758 | |
| 8759 | if (!reg_may_point_to_spin_lock(reg)) { |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 8760 | /* For not-NULL ptr, reg->ref_obj_id will be reset |
| 8761 | * in release_reg_references(). |
| 8762 | * |
| 8763 | * reg->id is still used by spin_lock ptr. Other |
| 8764 | * than spin_lock ptr type, reg->id can be reset. |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 8765 | */ |
| 8766 | reg->id = 0; |
| 8767 | } |
Thomas Graf | 57a09bf | 2016-10-18 19:51:19 +0200 | [diff] [blame] | 8768 | } |
| 8769 | } |
| 8770 | |
Paul Chaignon | c6a9efa | 2019-04-24 21:50:42 +0200 | [diff] [blame] | 8771 | static void __mark_ptr_or_null_regs(struct bpf_func_state *state, u32 id, |
| 8772 | bool is_null) |
| 8773 | { |
| 8774 | struct bpf_reg_state *reg; |
| 8775 | int i; |
| 8776 | |
| 8777 | for (i = 0; i < MAX_BPF_REG; i++) |
| 8778 | mark_ptr_or_null_reg(state, &state->regs[i], id, is_null); |
| 8779 | |
| 8780 | bpf_for_each_spilled_reg(i, state, reg) { |
| 8781 | if (!reg) |
| 8782 | continue; |
| 8783 | mark_ptr_or_null_reg(state, reg, id, is_null); |
| 8784 | } |
| 8785 | } |
| 8786 | |
Thomas Graf | 57a09bf | 2016-10-18 19:51:19 +0200 | [diff] [blame] | 8787 | /* The logic is similar to find_good_pkt_pointers(), both could eventually |
| 8788 | * be folded together at some point. |
| 8789 | */ |
Joe Stringer | 840b961 | 2018-10-02 13:35:32 -0700 | [diff] [blame] | 8790 | static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno, |
| 8791 | bool is_null) |
Thomas Graf | 57a09bf | 2016-10-18 19:51:19 +0200 | [diff] [blame] | 8792 | { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 8793 | struct bpf_func_state *state = vstate->frame[vstate->curframe]; |
Paul Chaignon | c6a9efa | 2019-04-24 21:50:42 +0200 | [diff] [blame] | 8794 | struct bpf_reg_state *regs = state->regs; |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 8795 | u32 ref_obj_id = regs[regno].ref_obj_id; |
Daniel Borkmann | a08dd0d | 2016-12-15 01:30:06 +0100 | [diff] [blame] | 8796 | u32 id = regs[regno].id; |
Paul Chaignon | c6a9efa | 2019-04-24 21:50:42 +0200 | [diff] [blame] | 8797 | int i; |
Thomas Graf | 57a09bf | 2016-10-18 19:51:19 +0200 | [diff] [blame] | 8798 | |
Martin KaFai Lau | 1b98658 | 2019-03-12 10:23:02 -0700 | [diff] [blame] | 8799 | if (ref_obj_id && ref_obj_id == id && is_null) |
| 8800 | /* regs[regno] is in the " == NULL" branch. |
| 8801 | * No one could have freed the reference state before |
| 8802 | * doing the NULL check. |
| 8803 | */ |
| 8804 | WARN_ON_ONCE(release_reference_state(state, id)); |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 8805 | |
Paul Chaignon | c6a9efa | 2019-04-24 21:50:42 +0200 | [diff] [blame] | 8806 | for (i = 0; i <= vstate->curframe; i++) |
| 8807 | __mark_ptr_or_null_regs(vstate->frame[i], id, is_null); |
Thomas Graf | 57a09bf | 2016-10-18 19:51:19 +0200 | [diff] [blame] | 8808 | } |
| 8809 | |
Daniel Borkmann | 5beca08 | 2017-11-01 23:58:10 +0100 | [diff] [blame] | 8810 | static bool try_match_pkt_pointers(const struct bpf_insn *insn, |
| 8811 | struct bpf_reg_state *dst_reg, |
| 8812 | struct bpf_reg_state *src_reg, |
| 8813 | struct bpf_verifier_state *this_branch, |
| 8814 | struct bpf_verifier_state *other_branch) |
| 8815 | { |
| 8816 | if (BPF_SRC(insn->code) != BPF_X) |
| 8817 | return false; |
| 8818 | |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 8819 | /* Pointers are always 64-bit. */ |
| 8820 | if (BPF_CLASS(insn->code) == BPF_JMP32) |
| 8821 | return false; |
| 8822 | |
Daniel Borkmann | 5beca08 | 2017-11-01 23:58:10 +0100 | [diff] [blame] | 8823 | switch (BPF_OP(insn->code)) { |
| 8824 | case BPF_JGT: |
| 8825 | if ((dst_reg->type == PTR_TO_PACKET && |
| 8826 | src_reg->type == PTR_TO_PACKET_END) || |
| 8827 | (dst_reg->type == PTR_TO_PACKET_META && |
| 8828 | reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { |
| 8829 | /* pkt_data' > pkt_end, pkt_meta' > pkt_data */ |
| 8830 | find_good_pkt_pointers(this_branch, dst_reg, |
| 8831 | dst_reg->type, false); |
Alexei Starovoitov | 6d94e74 | 2020-11-10 19:12:11 -0800 | [diff] [blame] | 8832 | mark_pkt_end(other_branch, insn->dst_reg, true); |
Daniel Borkmann | 5beca08 | 2017-11-01 23:58:10 +0100 | [diff] [blame] | 8833 | } else if ((dst_reg->type == PTR_TO_PACKET_END && |
| 8834 | src_reg->type == PTR_TO_PACKET) || |
| 8835 | (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && |
| 8836 | src_reg->type == PTR_TO_PACKET_META)) { |
| 8837 | /* pkt_end > pkt_data', pkt_data > pkt_meta' */ |
| 8838 | find_good_pkt_pointers(other_branch, src_reg, |
| 8839 | src_reg->type, true); |
Alexei Starovoitov | 6d94e74 | 2020-11-10 19:12:11 -0800 | [diff] [blame] | 8840 | mark_pkt_end(this_branch, insn->src_reg, false); |
Daniel Borkmann | 5beca08 | 2017-11-01 23:58:10 +0100 | [diff] [blame] | 8841 | } else { |
| 8842 | return false; |
| 8843 | } |
| 8844 | break; |
| 8845 | case BPF_JLT: |
| 8846 | if ((dst_reg->type == PTR_TO_PACKET && |
| 8847 | src_reg->type == PTR_TO_PACKET_END) || |
| 8848 | (dst_reg->type == PTR_TO_PACKET_META && |
| 8849 | reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { |
| 8850 | /* pkt_data' < pkt_end, pkt_meta' < pkt_data */ |
| 8851 | find_good_pkt_pointers(other_branch, dst_reg, |
| 8852 | dst_reg->type, true); |
Alexei Starovoitov | 6d94e74 | 2020-11-10 19:12:11 -0800 | [diff] [blame] | 8853 | mark_pkt_end(this_branch, insn->dst_reg, false); |
Daniel Borkmann | 5beca08 | 2017-11-01 23:58:10 +0100 | [diff] [blame] | 8854 | } else if ((dst_reg->type == PTR_TO_PACKET_END && |
| 8855 | src_reg->type == PTR_TO_PACKET) || |
| 8856 | (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && |
| 8857 | src_reg->type == PTR_TO_PACKET_META)) { |
| 8858 | /* pkt_end < pkt_data', pkt_data > pkt_meta' */ |
| 8859 | find_good_pkt_pointers(this_branch, src_reg, |
| 8860 | src_reg->type, false); |
Alexei Starovoitov | 6d94e74 | 2020-11-10 19:12:11 -0800 | [diff] [blame] | 8861 | mark_pkt_end(other_branch, insn->src_reg, true); |
Daniel Borkmann | 5beca08 | 2017-11-01 23:58:10 +0100 | [diff] [blame] | 8862 | } else { |
| 8863 | return false; |
| 8864 | } |
| 8865 | break; |
| 8866 | case BPF_JGE: |
| 8867 | if ((dst_reg->type == PTR_TO_PACKET && |
| 8868 | src_reg->type == PTR_TO_PACKET_END) || |
| 8869 | (dst_reg->type == PTR_TO_PACKET_META && |
| 8870 | reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { |
| 8871 | /* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */ |
| 8872 | find_good_pkt_pointers(this_branch, dst_reg, |
| 8873 | dst_reg->type, true); |
Alexei Starovoitov | 6d94e74 | 2020-11-10 19:12:11 -0800 | [diff] [blame] | 8874 | mark_pkt_end(other_branch, insn->dst_reg, false); |
Daniel Borkmann | 5beca08 | 2017-11-01 23:58:10 +0100 | [diff] [blame] | 8875 | } else if ((dst_reg->type == PTR_TO_PACKET_END && |
| 8876 | src_reg->type == PTR_TO_PACKET) || |
| 8877 | (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && |
| 8878 | src_reg->type == PTR_TO_PACKET_META)) { |
| 8879 | /* pkt_end >= pkt_data', pkt_data >= pkt_meta' */ |
| 8880 | find_good_pkt_pointers(other_branch, src_reg, |
| 8881 | src_reg->type, false); |
Alexei Starovoitov | 6d94e74 | 2020-11-10 19:12:11 -0800 | [diff] [blame] | 8882 | mark_pkt_end(this_branch, insn->src_reg, true); |
Daniel Borkmann | 5beca08 | 2017-11-01 23:58:10 +0100 | [diff] [blame] | 8883 | } else { |
| 8884 | return false; |
| 8885 | } |
| 8886 | break; |
| 8887 | case BPF_JLE: |
| 8888 | if ((dst_reg->type == PTR_TO_PACKET && |
| 8889 | src_reg->type == PTR_TO_PACKET_END) || |
| 8890 | (dst_reg->type == PTR_TO_PACKET_META && |
| 8891 | reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { |
| 8892 | /* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */ |
| 8893 | find_good_pkt_pointers(other_branch, dst_reg, |
| 8894 | dst_reg->type, false); |
Alexei Starovoitov | 6d94e74 | 2020-11-10 19:12:11 -0800 | [diff] [blame] | 8895 | mark_pkt_end(this_branch, insn->dst_reg, true); |
Daniel Borkmann | 5beca08 | 2017-11-01 23:58:10 +0100 | [diff] [blame] | 8896 | } else if ((dst_reg->type == PTR_TO_PACKET_END && |
| 8897 | src_reg->type == PTR_TO_PACKET) || |
| 8898 | (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && |
| 8899 | src_reg->type == PTR_TO_PACKET_META)) { |
| 8900 | /* pkt_end <= pkt_data', pkt_data <= pkt_meta' */ |
| 8901 | find_good_pkt_pointers(this_branch, src_reg, |
| 8902 | src_reg->type, true); |
Alexei Starovoitov | 6d94e74 | 2020-11-10 19:12:11 -0800 | [diff] [blame] | 8903 | mark_pkt_end(other_branch, insn->src_reg, false); |
Daniel Borkmann | 5beca08 | 2017-11-01 23:58:10 +0100 | [diff] [blame] | 8904 | } else { |
| 8905 | return false; |
| 8906 | } |
| 8907 | break; |
| 8908 | default: |
| 8909 | return false; |
| 8910 | } |
| 8911 | |
| 8912 | return true; |
| 8913 | } |
| 8914 | |
Alexei Starovoitov | 7574883 | 2020-10-08 18:12:37 -0700 | [diff] [blame] | 8915 | static void find_equal_scalars(struct bpf_verifier_state *vstate, |
| 8916 | struct bpf_reg_state *known_reg) |
| 8917 | { |
| 8918 | struct bpf_func_state *state; |
| 8919 | struct bpf_reg_state *reg; |
| 8920 | int i, j; |
| 8921 | |
| 8922 | for (i = 0; i <= vstate->curframe; i++) { |
| 8923 | state = vstate->frame[i]; |
| 8924 | for (j = 0; j < MAX_BPF_REG; j++) { |
| 8925 | reg = &state->regs[j]; |
| 8926 | if (reg->type == SCALAR_VALUE && reg->id == known_reg->id) |
| 8927 | *reg = *known_reg; |
| 8928 | } |
| 8929 | |
| 8930 | bpf_for_each_spilled_reg(j, state, reg) { |
| 8931 | if (!reg) |
| 8932 | continue; |
| 8933 | if (reg->type == SCALAR_VALUE && reg->id == known_reg->id) |
| 8934 | *reg = *known_reg; |
| 8935 | } |
| 8936 | } |
| 8937 | } |
| 8938 | |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 8939 | static int check_cond_jmp_op(struct bpf_verifier_env *env, |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8940 | struct bpf_insn *insn, int *insn_idx) |
| 8941 | { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 8942 | struct bpf_verifier_state *this_branch = env->cur_state; |
| 8943 | struct bpf_verifier_state *other_branch; |
| 8944 | struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs; |
Alexei Starovoitov | fb8d251 | 2019-06-15 12:12:19 -0700 | [diff] [blame] | 8945 | struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8946 | u8 opcode = BPF_OP(insn->code); |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 8947 | bool is_jmp32; |
Alexei Starovoitov | fb8d251 | 2019-06-15 12:12:19 -0700 | [diff] [blame] | 8948 | int pred = -1; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8949 | int err; |
| 8950 | |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 8951 | /* Only conditional jumps are expected to reach here. */ |
| 8952 | if (opcode == BPF_JA || opcode > BPF_JSLE) { |
| 8953 | verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8954 | return -EINVAL; |
| 8955 | } |
| 8956 | |
| 8957 | if (BPF_SRC(insn->code) == BPF_X) { |
| 8958 | if (insn->imm != 0) { |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 8959 | verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8960 | return -EINVAL; |
| 8961 | } |
| 8962 | |
| 8963 | /* check src1 operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 8964 | err = check_reg_arg(env, insn->src_reg, SRC_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8965 | if (err) |
| 8966 | return err; |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 8967 | |
| 8968 | if (is_pointer_value(env, insn->src_reg)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 8969 | verbose(env, "R%d pointer comparison prohibited\n", |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 8970 | insn->src_reg); |
| 8971 | return -EACCES; |
| 8972 | } |
Alexei Starovoitov | fb8d251 | 2019-06-15 12:12:19 -0700 | [diff] [blame] | 8973 | src_reg = ®s[insn->src_reg]; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8974 | } else { |
| 8975 | if (insn->src_reg != BPF_REG_0) { |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 8976 | verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8977 | return -EINVAL; |
| 8978 | } |
| 8979 | } |
| 8980 | |
| 8981 | /* check src2 operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 8982 | err = check_reg_arg(env, insn->dst_reg, SRC_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 8983 | if (err) |
| 8984 | return err; |
| 8985 | |
Alexei Starovoitov | 1a0dc1a | 2016-05-05 19:49:09 -0700 | [diff] [blame] | 8986 | dst_reg = ®s[insn->dst_reg]; |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 8987 | is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; |
Alexei Starovoitov | 1a0dc1a | 2016-05-05 19:49:09 -0700 | [diff] [blame] | 8988 | |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 8989 | if (BPF_SRC(insn->code) == BPF_K) { |
| 8990 | pred = is_branch_taken(dst_reg, insn->imm, opcode, is_jmp32); |
| 8991 | } else if (src_reg->type == SCALAR_VALUE && |
| 8992 | is_jmp32 && tnum_is_const(tnum_subreg(src_reg->var_off))) { |
| 8993 | pred = is_branch_taken(dst_reg, |
| 8994 | tnum_subreg(src_reg->var_off).value, |
| 8995 | opcode, |
| 8996 | is_jmp32); |
| 8997 | } else if (src_reg->type == SCALAR_VALUE && |
| 8998 | !is_jmp32 && tnum_is_const(src_reg->var_off)) { |
| 8999 | pred = is_branch_taken(dst_reg, |
| 9000 | src_reg->var_off.value, |
| 9001 | opcode, |
| 9002 | is_jmp32); |
Alexei Starovoitov | 6d94e74 | 2020-11-10 19:12:11 -0800 | [diff] [blame] | 9003 | } else if (reg_is_pkt_pointer_any(dst_reg) && |
| 9004 | reg_is_pkt_pointer_any(src_reg) && |
| 9005 | !is_jmp32) { |
| 9006 | pred = is_pkt_ptr_branch_taken(dst_reg, src_reg, opcode); |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 9007 | } |
| 9008 | |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 9009 | if (pred >= 0) { |
John Fastabend | cac616d | 2020-05-21 13:07:26 -0700 | [diff] [blame] | 9010 | /* If we get here with a dst_reg pointer type it is because |
| 9011 | * above is_branch_taken() special cased the 0 comparison. |
| 9012 | */ |
| 9013 | if (!__is_pointer_value(false, dst_reg)) |
| 9014 | err = mark_chain_precision(env, insn->dst_reg); |
Alexei Starovoitov | 6d94e74 | 2020-11-10 19:12:11 -0800 | [diff] [blame] | 9015 | if (BPF_SRC(insn->code) == BPF_X && !err && |
| 9016 | !__is_pointer_value(false, src_reg)) |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 9017 | err = mark_chain_precision(env, insn->src_reg); |
| 9018 | if (err) |
| 9019 | return err; |
| 9020 | } |
Daniel Borkmann | 9183671a | 2021-05-28 15:47:32 +0000 | [diff] [blame] | 9021 | |
Alexei Starovoitov | fb8d251 | 2019-06-15 12:12:19 -0700 | [diff] [blame] | 9022 | if (pred == 1) { |
Daniel Borkmann | 9183671a | 2021-05-28 15:47:32 +0000 | [diff] [blame] | 9023 | /* Only follow the goto, ignore fall-through. If needed, push |
| 9024 | * the fall-through branch for simulation under speculative |
| 9025 | * execution. |
| 9026 | */ |
| 9027 | if (!env->bypass_spec_v1 && |
| 9028 | !sanitize_speculative_path(env, insn, *insn_idx + 1, |
| 9029 | *insn_idx)) |
| 9030 | return -EFAULT; |
Alexei Starovoitov | fb8d251 | 2019-06-15 12:12:19 -0700 | [diff] [blame] | 9031 | *insn_idx += insn->off; |
| 9032 | return 0; |
| 9033 | } else if (pred == 0) { |
Daniel Borkmann | 9183671a | 2021-05-28 15:47:32 +0000 | [diff] [blame] | 9034 | /* Only follow the fall-through branch, since that's where the |
| 9035 | * program will go. If needed, push the goto branch for |
| 9036 | * simulation under speculative execution. |
Alexei Starovoitov | fb8d251 | 2019-06-15 12:12:19 -0700 | [diff] [blame] | 9037 | */ |
Daniel Borkmann | 9183671a | 2021-05-28 15:47:32 +0000 | [diff] [blame] | 9038 | if (!env->bypass_spec_v1 && |
| 9039 | !sanitize_speculative_path(env, insn, |
| 9040 | *insn_idx + insn->off + 1, |
| 9041 | *insn_idx)) |
| 9042 | return -EFAULT; |
Alexei Starovoitov | fb8d251 | 2019-06-15 12:12:19 -0700 | [diff] [blame] | 9043 | return 0; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 9044 | } |
| 9045 | |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 9046 | other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, |
| 9047 | false); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 9048 | if (!other_branch) |
| 9049 | return -EFAULT; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 9050 | other_branch_regs = other_branch->frame[other_branch->curframe]->regs; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 9051 | |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 9052 | /* detect if we are comparing against a constant value so we can adjust |
| 9053 | * our min/max values for our dst register. |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 9054 | * this is only legit if both are scalars (or pointers to the same |
| 9055 | * object, I suppose, but we don't support that right now), because |
| 9056 | * otherwise the different base pointers mean the offsets aren't |
| 9057 | * comparable. |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 9058 | */ |
| 9059 | if (BPF_SRC(insn->code) == BPF_X) { |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 9060 | struct bpf_reg_state *src_reg = ®s[insn->src_reg]; |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 9061 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 9062 | if (dst_reg->type == SCALAR_VALUE && |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 9063 | src_reg->type == SCALAR_VALUE) { |
| 9064 | if (tnum_is_const(src_reg->var_off) || |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 9065 | (is_jmp32 && |
| 9066 | tnum_is_const(tnum_subreg(src_reg->var_off)))) |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 9067 | reg_set_min_max(&other_branch_regs[insn->dst_reg], |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 9068 | dst_reg, |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 9069 | src_reg->var_off.value, |
| 9070 | tnum_subreg(src_reg->var_off).value, |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 9071 | opcode, is_jmp32); |
| 9072 | else if (tnum_is_const(dst_reg->var_off) || |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 9073 | (is_jmp32 && |
| 9074 | tnum_is_const(tnum_subreg(dst_reg->var_off)))) |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 9075 | reg_set_min_max_inv(&other_branch_regs[insn->src_reg], |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 9076 | src_reg, |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 9077 | dst_reg->var_off.value, |
| 9078 | tnum_subreg(dst_reg->var_off).value, |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 9079 | opcode, is_jmp32); |
| 9080 | else if (!is_jmp32 && |
| 9081 | (opcode == BPF_JEQ || opcode == BPF_JNE)) |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 9082 | /* Comparing for equality, we can combine knowledge */ |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 9083 | reg_combine_min_max(&other_branch_regs[insn->src_reg], |
| 9084 | &other_branch_regs[insn->dst_reg], |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 9085 | src_reg, dst_reg, opcode); |
Alexei Starovoitov | e688c3d | 2020-10-14 10:56:08 -0700 | [diff] [blame] | 9086 | if (src_reg->id && |
| 9087 | !WARN_ON_ONCE(src_reg->id != other_branch_regs[insn->src_reg].id)) { |
Alexei Starovoitov | 7574883 | 2020-10-08 18:12:37 -0700 | [diff] [blame] | 9088 | find_equal_scalars(this_branch, src_reg); |
| 9089 | find_equal_scalars(other_branch, &other_branch_regs[insn->src_reg]); |
| 9090 | } |
| 9091 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 9092 | } |
| 9093 | } else if (dst_reg->type == SCALAR_VALUE) { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 9094 | reg_set_min_max(&other_branch_regs[insn->dst_reg], |
John Fastabend | 3f50f13 | 2020-03-30 14:36:39 -0700 | [diff] [blame] | 9095 | dst_reg, insn->imm, (u32)insn->imm, |
| 9096 | opcode, is_jmp32); |
Josef Bacik | 4846113 | 2016-09-28 10:54:32 -0400 | [diff] [blame] | 9097 | } |
| 9098 | |
Alexei Starovoitov | e688c3d | 2020-10-14 10:56:08 -0700 | [diff] [blame] | 9099 | if (dst_reg->type == SCALAR_VALUE && dst_reg->id && |
| 9100 | !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) { |
Alexei Starovoitov | 7574883 | 2020-10-08 18:12:37 -0700 | [diff] [blame] | 9101 | find_equal_scalars(this_branch, dst_reg); |
| 9102 | find_equal_scalars(other_branch, &other_branch_regs[insn->dst_reg]); |
| 9103 | } |
| 9104 | |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 9105 | /* detect if R == 0 where R is returned from bpf_map_lookup_elem(). |
| 9106 | * NOTE: these optimizations below are related with pointer comparison |
| 9107 | * which will never be JMP32. |
| 9108 | */ |
| 9109 | if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K && |
Alexei Starovoitov | 1a0dc1a | 2016-05-05 19:49:09 -0700 | [diff] [blame] | 9110 | insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) && |
Joe Stringer | 840b961 | 2018-10-02 13:35:32 -0700 | [diff] [blame] | 9111 | reg_type_may_be_null(dst_reg->type)) { |
| 9112 | /* Mark all identical registers in each branch as either |
Thomas Graf | 57a09bf | 2016-10-18 19:51:19 +0200 | [diff] [blame] | 9113 | * safe or unknown depending R == 0 or R != 0 conditional. |
| 9114 | */ |
Joe Stringer | 840b961 | 2018-10-02 13:35:32 -0700 | [diff] [blame] | 9115 | mark_ptr_or_null_regs(this_branch, insn->dst_reg, |
| 9116 | opcode == BPF_JNE); |
| 9117 | mark_ptr_or_null_regs(other_branch, insn->dst_reg, |
| 9118 | opcode == BPF_JEQ); |
Daniel Borkmann | 5beca08 | 2017-11-01 23:58:10 +0100 | [diff] [blame] | 9119 | } else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], |
| 9120 | this_branch, other_branch) && |
| 9121 | is_pointer_value(env, insn->dst_reg)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 9122 | verbose(env, "R%d pointer comparison prohibited\n", |
| 9123 | insn->dst_reg); |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 9124 | return -EACCES; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 9125 | } |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 9126 | if (env->log.level & BPF_LOG_LEVEL) |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 9127 | print_verifier_state(env, this_branch->frame[this_branch->curframe]); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 9128 | return 0; |
| 9129 | } |
| 9130 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 9131 | /* verify BPF_LD_IMM64 instruction */ |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 9132 | static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 9133 | { |
Daniel Borkmann | d8eca5b | 2019-04-09 23:20:03 +0200 | [diff] [blame] | 9134 | struct bpf_insn_aux_data *aux = cur_aux(env); |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 9135 | struct bpf_reg_state *regs = cur_regs(env); |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 9136 | struct bpf_reg_state *dst_reg; |
Daniel Borkmann | d8eca5b | 2019-04-09 23:20:03 +0200 | [diff] [blame] | 9137 | struct bpf_map *map; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 9138 | int err; |
| 9139 | |
| 9140 | if (BPF_SIZE(insn->code) != BPF_DW) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 9141 | verbose(env, "invalid BPF_LD_IMM insn\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 9142 | return -EINVAL; |
| 9143 | } |
| 9144 | if (insn->off != 0) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 9145 | verbose(env, "BPF_LD_IMM64 uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 9146 | return -EINVAL; |
| 9147 | } |
| 9148 | |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 9149 | err = check_reg_arg(env, insn->dst_reg, DST_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 9150 | if (err) |
| 9151 | return err; |
| 9152 | |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 9153 | dst_reg = ®s[insn->dst_reg]; |
Jakub Kicinski | 6b17387 | 2016-09-21 11:43:59 +0100 | [diff] [blame] | 9154 | if (insn->src_reg == 0) { |
Jakub Kicinski | 6b17387 | 2016-09-21 11:43:59 +0100 | [diff] [blame] | 9155 | u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; |
| 9156 | |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 9157 | dst_reg->type = SCALAR_VALUE; |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 9158 | __mark_reg_known(®s[insn->dst_reg], imm); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 9159 | return 0; |
Jakub Kicinski | 6b17387 | 2016-09-21 11:43:59 +0100 | [diff] [blame] | 9160 | } |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 9161 | |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 9162 | if (insn->src_reg == BPF_PSEUDO_BTF_ID) { |
| 9163 | mark_reg_known_zero(env, regs, insn->dst_reg); |
| 9164 | |
| 9165 | dst_reg->type = aux->btf_var.reg_type; |
| 9166 | switch (dst_reg->type) { |
| 9167 | case PTR_TO_MEM: |
| 9168 | dst_reg->mem_size = aux->btf_var.mem_size; |
| 9169 | break; |
| 9170 | case PTR_TO_BTF_ID: |
Hao Luo | eaa6bcb | 2020-09-29 16:50:47 -0700 | [diff] [blame] | 9171 | case PTR_TO_PERCPU_BTF_ID: |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 9172 | dst_reg->btf = aux->btf_var.btf; |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 9173 | dst_reg->btf_id = aux->btf_var.btf_id; |
| 9174 | break; |
| 9175 | default: |
| 9176 | verbose(env, "bpf verifier is misconfigured\n"); |
| 9177 | return -EFAULT; |
| 9178 | } |
| 9179 | return 0; |
| 9180 | } |
| 9181 | |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 9182 | if (insn->src_reg == BPF_PSEUDO_FUNC) { |
| 9183 | struct bpf_prog_aux *aux = env->prog->aux; |
| 9184 | u32 subprogno = insn[1].imm; |
| 9185 | |
| 9186 | if (!aux->func_info) { |
| 9187 | verbose(env, "missing btf func_info\n"); |
| 9188 | return -EINVAL; |
| 9189 | } |
| 9190 | if (aux->func_info_aux[subprogno].linkage != BTF_FUNC_STATIC) { |
| 9191 | verbose(env, "callback function not static\n"); |
| 9192 | return -EINVAL; |
| 9193 | } |
| 9194 | |
| 9195 | dst_reg->type = PTR_TO_FUNC; |
| 9196 | dst_reg->subprogno = subprogno; |
| 9197 | return 0; |
| 9198 | } |
| 9199 | |
Daniel Borkmann | d8eca5b | 2019-04-09 23:20:03 +0200 | [diff] [blame] | 9200 | map = env->used_maps[aux->map_index]; |
| 9201 | mark_reg_known_zero(env, regs, insn->dst_reg); |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 9202 | dst_reg->map_ptr = map; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 9203 | |
Alexei Starovoitov | 387544b | 2021-05-13 17:36:10 -0700 | [diff] [blame] | 9204 | if (insn->src_reg == BPF_PSEUDO_MAP_VALUE || |
| 9205 | insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE) { |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 9206 | dst_reg->type = PTR_TO_MAP_VALUE; |
| 9207 | dst_reg->off = aux->map_off; |
Daniel Borkmann | d8eca5b | 2019-04-09 23:20:03 +0200 | [diff] [blame] | 9208 | if (map_value_has_spin_lock(map)) |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 9209 | dst_reg->id = ++env->id_gen; |
Alexei Starovoitov | 387544b | 2021-05-13 17:36:10 -0700 | [diff] [blame] | 9210 | } else if (insn->src_reg == BPF_PSEUDO_MAP_FD || |
| 9211 | insn->src_reg == BPF_PSEUDO_MAP_IDX) { |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 9212 | dst_reg->type = CONST_PTR_TO_MAP; |
Daniel Borkmann | d8eca5b | 2019-04-09 23:20:03 +0200 | [diff] [blame] | 9213 | } else { |
| 9214 | verbose(env, "bpf verifier is misconfigured\n"); |
| 9215 | return -EINVAL; |
| 9216 | } |
| 9217 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 9218 | return 0; |
| 9219 | } |
| 9220 | |
Daniel Borkmann | 96be432 | 2015-03-01 12:31:46 +0100 | [diff] [blame] | 9221 | static bool may_access_skb(enum bpf_prog_type type) |
| 9222 | { |
| 9223 | switch (type) { |
| 9224 | case BPF_PROG_TYPE_SOCKET_FILTER: |
| 9225 | case BPF_PROG_TYPE_SCHED_CLS: |
Daniel Borkmann | 94caee8c | 2015-03-20 15:11:11 +0100 | [diff] [blame] | 9226 | case BPF_PROG_TYPE_SCHED_ACT: |
Daniel Borkmann | 96be432 | 2015-03-01 12:31:46 +0100 | [diff] [blame] | 9227 | return true; |
| 9228 | default: |
| 9229 | return false; |
| 9230 | } |
| 9231 | } |
| 9232 | |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 9233 | /* verify safety of LD_ABS|LD_IND instructions: |
| 9234 | * - they can only appear in the programs where ctx == skb |
| 9235 | * - since they are wrappers of function calls, they scratch R1-R5 registers, |
| 9236 | * preserve R6-R9, and store return value into R0 |
| 9237 | * |
| 9238 | * Implicit input: |
| 9239 | * ctx == skb == R6 == CTX |
| 9240 | * |
| 9241 | * Explicit input: |
| 9242 | * SRC == any register |
| 9243 | * IMM == 32-bit immediate |
| 9244 | * |
| 9245 | * Output: |
| 9246 | * R0 - 8/16/32-bit skb data converted to cpu endianness |
| 9247 | */ |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 9248 | static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 9249 | { |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 9250 | struct bpf_reg_state *regs = cur_regs(env); |
Daniel Borkmann | 6d4f151 | 2020-01-06 22:51:57 +0100 | [diff] [blame] | 9251 | static const int ctx_reg = BPF_REG_6; |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 9252 | u8 mode = BPF_MODE(insn->code); |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 9253 | int i, err; |
| 9254 | |
Udip Pant | 7e40781 | 2020-08-25 16:20:00 -0700 | [diff] [blame] | 9255 | if (!may_access_skb(resolve_prog_type(env->prog))) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 9256 | verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n"); |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 9257 | return -EINVAL; |
| 9258 | } |
| 9259 | |
Daniel Borkmann | e0cea7c | 2018-05-04 01:08:14 +0200 | [diff] [blame] | 9260 | if (!env->ops->gen_ld_abs) { |
| 9261 | verbose(env, "bpf verifier is misconfigured\n"); |
| 9262 | return -EINVAL; |
| 9263 | } |
| 9264 | |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 9265 | if (insn->dst_reg != BPF_REG_0 || insn->off != 0 || |
Alexei Starovoitov | d82bccc | 2016-04-12 10:26:19 -0700 | [diff] [blame] | 9266 | BPF_SIZE(insn->code) == BPF_DW || |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 9267 | (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 9268 | verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n"); |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 9269 | return -EINVAL; |
| 9270 | } |
| 9271 | |
| 9272 | /* check whether implicit source operand (register R6) is readable */ |
Daniel Borkmann | 6d4f151 | 2020-01-06 22:51:57 +0100 | [diff] [blame] | 9273 | err = check_reg_arg(env, ctx_reg, SRC_OP); |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 9274 | if (err) |
| 9275 | return err; |
| 9276 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 9277 | /* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as |
| 9278 | * gen_ld_abs() may terminate the program at runtime, leading to |
| 9279 | * reference leak. |
| 9280 | */ |
| 9281 | err = check_reference_leak(env); |
| 9282 | if (err) { |
| 9283 | verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n"); |
| 9284 | return err; |
| 9285 | } |
| 9286 | |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 9287 | if (env->cur_state->active_spin_lock) { |
| 9288 | verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n"); |
| 9289 | return -EINVAL; |
| 9290 | } |
| 9291 | |
Daniel Borkmann | 6d4f151 | 2020-01-06 22:51:57 +0100 | [diff] [blame] | 9292 | if (regs[ctx_reg].type != PTR_TO_CTX) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 9293 | verbose(env, |
| 9294 | "at the time of BPF_LD_ABS|IND R6 != pointer to skb\n"); |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 9295 | return -EINVAL; |
| 9296 | } |
| 9297 | |
| 9298 | if (mode == BPF_IND) { |
| 9299 | /* check explicit source operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 9300 | err = check_reg_arg(env, insn->src_reg, SRC_OP); |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 9301 | if (err) |
| 9302 | return err; |
| 9303 | } |
| 9304 | |
Daniel Borkmann | 6d4f151 | 2020-01-06 22:51:57 +0100 | [diff] [blame] | 9305 | err = check_ctx_reg(env, ®s[ctx_reg], ctx_reg); |
| 9306 | if (err < 0) |
| 9307 | return err; |
| 9308 | |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 9309 | /* reset caller saved regs to unreadable */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 9310 | for (i = 0; i < CALLER_SAVED_REGS; i++) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 9311 | mark_reg_not_init(env, regs, caller_saved[i]); |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 9312 | check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); |
| 9313 | } |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 9314 | |
| 9315 | /* mark destination R0 register as readable, since it contains |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 9316 | * the value fetched from the packet. |
| 9317 | * Already marked as written above. |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 9318 | */ |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 9319 | mark_reg_unknown(env, regs, BPF_REG_0); |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 9320 | /* ld_abs load up to 32-bit skb data. */ |
| 9321 | regs[BPF_REG_0].subreg_def = env->insn_idx + 1; |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 9322 | return 0; |
| 9323 | } |
| 9324 | |
Alexei Starovoitov | 390ee7e | 2017-10-02 22:50:23 -0700 | [diff] [blame] | 9325 | static int check_return_code(struct bpf_verifier_env *env) |
| 9326 | { |
brakmo | 5cf1e91 | 2019-05-28 16:59:36 -0700 | [diff] [blame] | 9327 | struct tnum enforce_attach_type_range = tnum_unknown; |
Martin KaFai Lau | 27ae7997 | 2020-01-08 16:35:03 -0800 | [diff] [blame] | 9328 | const struct bpf_prog *prog = env->prog; |
Alexei Starovoitov | 390ee7e | 2017-10-02 22:50:23 -0700 | [diff] [blame] | 9329 | struct bpf_reg_state *reg; |
| 9330 | struct tnum range = tnum_range(0, 1); |
Udip Pant | 7e40781 | 2020-08-25 16:20:00 -0700 | [diff] [blame] | 9331 | enum bpf_prog_type prog_type = resolve_prog_type(env->prog); |
Martin KaFai Lau | 27ae7997 | 2020-01-08 16:35:03 -0800 | [diff] [blame] | 9332 | int err; |
Alexei Starovoitov | bfc6bb7 | 2021-07-14 17:54:14 -0700 | [diff] [blame] | 9333 | struct bpf_func_state *frame = env->cur_state->frame[0]; |
| 9334 | const bool is_subprog = frame->subprogno; |
Martin KaFai Lau | 27ae7997 | 2020-01-08 16:35:03 -0800 | [diff] [blame] | 9335 | |
KP Singh | 9e4e01d | 2020-03-29 01:43:52 +0100 | [diff] [blame] | 9336 | /* LSM and struct_ops func-ptr's return type could be "void" */ |
Dmitrii Banshchikov | f782e2c | 2020-11-13 17:17:56 +0000 | [diff] [blame] | 9337 | if (!is_subprog && |
| 9338 | (prog_type == BPF_PROG_TYPE_STRUCT_OPS || |
Udip Pant | 7e40781 | 2020-08-25 16:20:00 -0700 | [diff] [blame] | 9339 | prog_type == BPF_PROG_TYPE_LSM) && |
Martin KaFai Lau | 27ae7997 | 2020-01-08 16:35:03 -0800 | [diff] [blame] | 9340 | !prog->aux->attach_func_proto->type) |
| 9341 | return 0; |
| 9342 | |
Zhen Lei | 8fb33b6 | 2021-05-25 10:56:59 +0800 | [diff] [blame] | 9343 | /* eBPF calling convention is such that R0 is used |
Martin KaFai Lau | 27ae7997 | 2020-01-08 16:35:03 -0800 | [diff] [blame] | 9344 | * to return the value from eBPF program. |
| 9345 | * Make sure that it's readable at this time |
| 9346 | * of bpf_exit, which means that program wrote |
| 9347 | * something into it earlier |
| 9348 | */ |
| 9349 | err = check_reg_arg(env, BPF_REG_0, SRC_OP); |
| 9350 | if (err) |
| 9351 | return err; |
| 9352 | |
| 9353 | if (is_pointer_value(env, BPF_REG_0)) { |
| 9354 | verbose(env, "R0 leaks addr as return value\n"); |
| 9355 | return -EACCES; |
| 9356 | } |
Alexei Starovoitov | 390ee7e | 2017-10-02 22:50:23 -0700 | [diff] [blame] | 9357 | |
Dmitrii Banshchikov | f782e2c | 2020-11-13 17:17:56 +0000 | [diff] [blame] | 9358 | reg = cur_regs(env) + BPF_REG_0; |
Alexei Starovoitov | bfc6bb7 | 2021-07-14 17:54:14 -0700 | [diff] [blame] | 9359 | |
| 9360 | if (frame->in_async_callback_fn) { |
| 9361 | /* enforce return zero from async callbacks like timer */ |
| 9362 | if (reg->type != SCALAR_VALUE) { |
| 9363 | verbose(env, "In async callback the register R0 is not a known value (%s)\n", |
| 9364 | reg_type_str[reg->type]); |
| 9365 | return -EINVAL; |
| 9366 | } |
| 9367 | |
| 9368 | if (!tnum_in(tnum_const(0), reg->var_off)) { |
| 9369 | verbose_invalid_scalar(env, reg, &range, "async callback", "R0"); |
| 9370 | return -EINVAL; |
| 9371 | } |
| 9372 | return 0; |
| 9373 | } |
| 9374 | |
Dmitrii Banshchikov | f782e2c | 2020-11-13 17:17:56 +0000 | [diff] [blame] | 9375 | if (is_subprog) { |
| 9376 | if (reg->type != SCALAR_VALUE) { |
| 9377 | verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n", |
| 9378 | reg_type_str[reg->type]); |
| 9379 | return -EINVAL; |
| 9380 | } |
| 9381 | return 0; |
| 9382 | } |
| 9383 | |
Udip Pant | 7e40781 | 2020-08-25 16:20:00 -0700 | [diff] [blame] | 9384 | switch (prog_type) { |
Daniel Borkmann | 983695f | 2019-06-07 01:48:57 +0200 | [diff] [blame] | 9385 | case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: |
| 9386 | if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG || |
Daniel Borkmann | 1b66d25 | 2020-05-19 00:45:45 +0200 | [diff] [blame] | 9387 | env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG || |
| 9388 | env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME || |
| 9389 | env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME || |
| 9390 | env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME || |
| 9391 | env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME) |
Daniel Borkmann | 983695f | 2019-06-07 01:48:57 +0200 | [diff] [blame] | 9392 | range = tnum_range(1, 1); |
Stanislav Fomichev | 7724121 | 2021-01-27 11:31:39 -0800 | [diff] [blame] | 9393 | if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND || |
| 9394 | env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND) |
| 9395 | range = tnum_range(0, 3); |
Gustavo A. R. Silva | ed4ed40 | 2019-07-11 11:22:33 -0500 | [diff] [blame] | 9396 | break; |
Alexei Starovoitov | 390ee7e | 2017-10-02 22:50:23 -0700 | [diff] [blame] | 9397 | case BPF_PROG_TYPE_CGROUP_SKB: |
brakmo | 5cf1e91 | 2019-05-28 16:59:36 -0700 | [diff] [blame] | 9398 | if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) { |
| 9399 | range = tnum_range(0, 3); |
| 9400 | enforce_attach_type_range = tnum_range(2, 3); |
| 9401 | } |
Gustavo A. R. Silva | ed4ed40 | 2019-07-11 11:22:33 -0500 | [diff] [blame] | 9402 | break; |
Alexei Starovoitov | 390ee7e | 2017-10-02 22:50:23 -0700 | [diff] [blame] | 9403 | case BPF_PROG_TYPE_CGROUP_SOCK: |
| 9404 | case BPF_PROG_TYPE_SOCK_OPS: |
Roman Gushchin | ebc614f | 2017-11-05 08:15:32 -0500 | [diff] [blame] | 9405 | case BPF_PROG_TYPE_CGROUP_DEVICE: |
Andrey Ignatov | 7b146ce | 2019-02-27 12:59:24 -0800 | [diff] [blame] | 9406 | case BPF_PROG_TYPE_CGROUP_SYSCTL: |
Stanislav Fomichev | 0d01da6 | 2019-06-27 13:38:47 -0700 | [diff] [blame] | 9407 | case BPF_PROG_TYPE_CGROUP_SOCKOPT: |
Alexei Starovoitov | 390ee7e | 2017-10-02 22:50:23 -0700 | [diff] [blame] | 9408 | break; |
Alexei Starovoitov | 15ab09b | 2019-10-28 20:24:26 -0700 | [diff] [blame] | 9409 | case BPF_PROG_TYPE_RAW_TRACEPOINT: |
| 9410 | if (!env->prog->aux->attach_btf_id) |
| 9411 | return 0; |
| 9412 | range = tnum_const(0); |
| 9413 | break; |
Yonghong Song | 15d83c4 | 2020-05-09 10:59:00 -0700 | [diff] [blame] | 9414 | case BPF_PROG_TYPE_TRACING: |
Yonghong Song | e92888c7 | 2020-05-13 22:32:05 -0700 | [diff] [blame] | 9415 | switch (env->prog->expected_attach_type) { |
| 9416 | case BPF_TRACE_FENTRY: |
| 9417 | case BPF_TRACE_FEXIT: |
| 9418 | range = tnum_const(0); |
| 9419 | break; |
| 9420 | case BPF_TRACE_RAW_TP: |
| 9421 | case BPF_MODIFY_RETURN: |
Yonghong Song | 15d83c4 | 2020-05-09 10:59:00 -0700 | [diff] [blame] | 9422 | return 0; |
Daniel Borkmann | 2ec0616 | 2020-05-16 00:39:18 +0200 | [diff] [blame] | 9423 | case BPF_TRACE_ITER: |
| 9424 | break; |
Yonghong Song | e92888c7 | 2020-05-13 22:32:05 -0700 | [diff] [blame] | 9425 | default: |
| 9426 | return -ENOTSUPP; |
| 9427 | } |
Yonghong Song | 15d83c4 | 2020-05-09 10:59:00 -0700 | [diff] [blame] | 9428 | break; |
Jakub Sitnicki | e9ddbb7 | 2020-07-17 12:35:23 +0200 | [diff] [blame] | 9429 | case BPF_PROG_TYPE_SK_LOOKUP: |
| 9430 | range = tnum_range(SK_DROP, SK_PASS); |
| 9431 | break; |
Yonghong Song | e92888c7 | 2020-05-13 22:32:05 -0700 | [diff] [blame] | 9432 | case BPF_PROG_TYPE_EXT: |
| 9433 | /* freplace program can return anything as its return value |
| 9434 | * depends on the to-be-replaced kernel func or bpf program. |
| 9435 | */ |
Alexei Starovoitov | 390ee7e | 2017-10-02 22:50:23 -0700 | [diff] [blame] | 9436 | default: |
| 9437 | return 0; |
| 9438 | } |
| 9439 | |
Alexei Starovoitov | 390ee7e | 2017-10-02 22:50:23 -0700 | [diff] [blame] | 9440 | if (reg->type != SCALAR_VALUE) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 9441 | verbose(env, "At program exit the register R0 is not a known value (%s)\n", |
Alexei Starovoitov | 390ee7e | 2017-10-02 22:50:23 -0700 | [diff] [blame] | 9442 | reg_type_str[reg->type]); |
| 9443 | return -EINVAL; |
| 9444 | } |
| 9445 | |
| 9446 | if (!tnum_in(range, reg->var_off)) { |
Yonghong Song | bc2591d | 2021-02-26 12:49:22 -0800 | [diff] [blame] | 9447 | verbose_invalid_scalar(env, reg, &range, "program exit", "R0"); |
Alexei Starovoitov | 390ee7e | 2017-10-02 22:50:23 -0700 | [diff] [blame] | 9448 | return -EINVAL; |
| 9449 | } |
brakmo | 5cf1e91 | 2019-05-28 16:59:36 -0700 | [diff] [blame] | 9450 | |
| 9451 | if (!tnum_is_unknown(enforce_attach_type_range) && |
| 9452 | tnum_in(enforce_attach_type_range, reg->var_off)) |
| 9453 | env->prog->enforce_expected_attach_type = 1; |
Alexei Starovoitov | 390ee7e | 2017-10-02 22:50:23 -0700 | [diff] [blame] | 9454 | return 0; |
| 9455 | } |
| 9456 | |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9457 | /* non-recursive DFS pseudo code |
| 9458 | * 1 procedure DFS-iterative(G,v): |
| 9459 | * 2 label v as discovered |
| 9460 | * 3 let S be a stack |
| 9461 | * 4 S.push(v) |
| 9462 | * 5 while S is not empty |
| 9463 | * 6 t <- S.pop() |
| 9464 | * 7 if t is what we're looking for: |
| 9465 | * 8 return t |
| 9466 | * 9 for all edges e in G.adjacentEdges(t) do |
| 9467 | * 10 if edge e is already labelled |
| 9468 | * 11 continue with the next edge |
| 9469 | * 12 w <- G.adjacentVertex(t,e) |
| 9470 | * 13 if vertex w is not discovered and not explored |
| 9471 | * 14 label e as tree-edge |
| 9472 | * 15 label w as discovered |
| 9473 | * 16 S.push(w) |
| 9474 | * 17 continue at 5 |
| 9475 | * 18 else if vertex w is discovered |
| 9476 | * 19 label e as back-edge |
| 9477 | * 20 else |
| 9478 | * 21 // vertex w is explored |
| 9479 | * 22 label e as forward- or cross-edge |
| 9480 | * 23 label t as explored |
| 9481 | * 24 S.pop() |
| 9482 | * |
| 9483 | * convention: |
| 9484 | * 0x10 - discovered |
| 9485 | * 0x11 - discovered and fall-through edge labelled |
| 9486 | * 0x12 - discovered and fall-through and branch edges labelled |
| 9487 | * 0x20 - explored |
| 9488 | */ |
| 9489 | |
| 9490 | enum { |
| 9491 | DISCOVERED = 0x10, |
| 9492 | EXPLORED = 0x20, |
| 9493 | FALLTHROUGH = 1, |
| 9494 | BRANCH = 2, |
| 9495 | }; |
| 9496 | |
Alexei Starovoitov | dc2a4eb | 2019-05-21 20:17:07 -0700 | [diff] [blame] | 9497 | static u32 state_htab_size(struct bpf_verifier_env *env) |
| 9498 | { |
| 9499 | return env->prog->len; |
| 9500 | } |
| 9501 | |
Alexei Starovoitov | 5d83902 | 2019-05-21 20:17:05 -0700 | [diff] [blame] | 9502 | static struct bpf_verifier_state_list **explored_state( |
| 9503 | struct bpf_verifier_env *env, |
| 9504 | int idx) |
| 9505 | { |
Alexei Starovoitov | dc2a4eb | 2019-05-21 20:17:07 -0700 | [diff] [blame] | 9506 | struct bpf_verifier_state *cur = env->cur_state; |
| 9507 | struct bpf_func_state *state = cur->frame[cur->curframe]; |
| 9508 | |
| 9509 | return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)]; |
Alexei Starovoitov | 5d83902 | 2019-05-21 20:17:05 -0700 | [diff] [blame] | 9510 | } |
| 9511 | |
| 9512 | static void init_explored_state(struct bpf_verifier_env *env, int idx) |
| 9513 | { |
Alexei Starovoitov | a8f500a | 2019-05-21 20:17:06 -0700 | [diff] [blame] | 9514 | env->insn_aux_data[idx].prune_point = true; |
Alexei Starovoitov | 5d83902 | 2019-05-21 20:17:05 -0700 | [diff] [blame] | 9515 | } |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 9516 | |
Wedson Almeida Filho | 59e2e27 | 2020-11-21 01:55:09 +0000 | [diff] [blame] | 9517 | enum { |
| 9518 | DONE_EXPLORING = 0, |
| 9519 | KEEP_EXPLORING = 1, |
| 9520 | }; |
| 9521 | |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9522 | /* t, w, e - match pseudo-code above: |
| 9523 | * t - index of current instruction |
| 9524 | * w - next instruction |
| 9525 | * e - edge |
| 9526 | */ |
Alexei Starovoitov | 2589726 | 2019-06-15 12:12:20 -0700 | [diff] [blame] | 9527 | static int push_insn(int t, int w, int e, struct bpf_verifier_env *env, |
| 9528 | bool loop_ok) |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9529 | { |
Alexei Starovoitov | 7df737e | 2019-04-19 07:44:54 -0700 | [diff] [blame] | 9530 | int *insn_stack = env->cfg.insn_stack; |
| 9531 | int *insn_state = env->cfg.insn_state; |
| 9532 | |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9533 | if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH)) |
Wedson Almeida Filho | 59e2e27 | 2020-11-21 01:55:09 +0000 | [diff] [blame] | 9534 | return DONE_EXPLORING; |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9535 | |
| 9536 | if (e == BRANCH && insn_state[t] >= (DISCOVERED | BRANCH)) |
Wedson Almeida Filho | 59e2e27 | 2020-11-21 01:55:09 +0000 | [diff] [blame] | 9537 | return DONE_EXPLORING; |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9538 | |
| 9539 | if (w < 0 || w >= env->prog->len) { |
Martin KaFai Lau | d9762e8 | 2018-12-13 10:41:48 -0800 | [diff] [blame] | 9540 | verbose_linfo(env, t, "%d: ", t); |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 9541 | verbose(env, "jump out of range from insn %d to %d\n", t, w); |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9542 | return -EINVAL; |
| 9543 | } |
| 9544 | |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 9545 | if (e == BRANCH) |
| 9546 | /* mark branch target for state pruning */ |
Alexei Starovoitov | 5d83902 | 2019-05-21 20:17:05 -0700 | [diff] [blame] | 9547 | init_explored_state(env, w); |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 9548 | |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9549 | if (insn_state[w] == 0) { |
| 9550 | /* tree-edge */ |
| 9551 | insn_state[t] = DISCOVERED | e; |
| 9552 | insn_state[w] = DISCOVERED; |
Alexei Starovoitov | 7df737e | 2019-04-19 07:44:54 -0700 | [diff] [blame] | 9553 | if (env->cfg.cur_stack >= env->prog->len) |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9554 | return -E2BIG; |
Alexei Starovoitov | 7df737e | 2019-04-19 07:44:54 -0700 | [diff] [blame] | 9555 | insn_stack[env->cfg.cur_stack++] = w; |
Wedson Almeida Filho | 59e2e27 | 2020-11-21 01:55:09 +0000 | [diff] [blame] | 9556 | return KEEP_EXPLORING; |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9557 | } else if ((insn_state[w] & 0xF0) == DISCOVERED) { |
Alexei Starovoitov | 2c78ee8 | 2020-05-13 16:03:54 -0700 | [diff] [blame] | 9558 | if (loop_ok && env->bpf_capable) |
Wedson Almeida Filho | 59e2e27 | 2020-11-21 01:55:09 +0000 | [diff] [blame] | 9559 | return DONE_EXPLORING; |
Martin KaFai Lau | d9762e8 | 2018-12-13 10:41:48 -0800 | [diff] [blame] | 9560 | verbose_linfo(env, t, "%d: ", t); |
| 9561 | verbose_linfo(env, w, "%d: ", w); |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 9562 | verbose(env, "back-edge from insn %d to %d\n", t, w); |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9563 | return -EINVAL; |
| 9564 | } else if (insn_state[w] == EXPLORED) { |
| 9565 | /* forward- or cross-edge */ |
| 9566 | insn_state[t] = DISCOVERED | e; |
| 9567 | } else { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 9568 | verbose(env, "insn state internal bug\n"); |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9569 | return -EFAULT; |
| 9570 | } |
Wedson Almeida Filho | 59e2e27 | 2020-11-21 01:55:09 +0000 | [diff] [blame] | 9571 | return DONE_EXPLORING; |
| 9572 | } |
| 9573 | |
Yonghong Song | efdb22d | 2021-02-26 12:49:20 -0800 | [diff] [blame] | 9574 | static int visit_func_call_insn(int t, int insn_cnt, |
| 9575 | struct bpf_insn *insns, |
| 9576 | struct bpf_verifier_env *env, |
| 9577 | bool visit_callee) |
| 9578 | { |
| 9579 | int ret; |
| 9580 | |
| 9581 | ret = push_insn(t, t + 1, FALLTHROUGH, env, false); |
| 9582 | if (ret) |
| 9583 | return ret; |
| 9584 | |
| 9585 | if (t + 1 < insn_cnt) |
| 9586 | init_explored_state(env, t + 1); |
| 9587 | if (visit_callee) { |
| 9588 | init_explored_state(env, t); |
Alexei Starovoitov | 86fc6ee | 2021-07-14 17:54:13 -0700 | [diff] [blame] | 9589 | ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env, |
| 9590 | /* It's ok to allow recursion from CFG point of |
| 9591 | * view. __check_func_call() will do the actual |
| 9592 | * check. |
| 9593 | */ |
| 9594 | bpf_pseudo_func(insns + t)); |
Yonghong Song | efdb22d | 2021-02-26 12:49:20 -0800 | [diff] [blame] | 9595 | } |
| 9596 | return ret; |
| 9597 | } |
| 9598 | |
Wedson Almeida Filho | 59e2e27 | 2020-11-21 01:55:09 +0000 | [diff] [blame] | 9599 | /* Visits the instruction at index t and returns one of the following: |
| 9600 | * < 0 - an error occurred |
| 9601 | * DONE_EXPLORING - the instruction was fully explored |
| 9602 | * KEEP_EXPLORING - there is still work to be done before it is fully explored |
| 9603 | */ |
| 9604 | static int visit_insn(int t, int insn_cnt, struct bpf_verifier_env *env) |
| 9605 | { |
| 9606 | struct bpf_insn *insns = env->prog->insnsi; |
| 9607 | int ret; |
| 9608 | |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 9609 | if (bpf_pseudo_func(insns + t)) |
| 9610 | return visit_func_call_insn(t, insn_cnt, insns, env, true); |
| 9611 | |
Wedson Almeida Filho | 59e2e27 | 2020-11-21 01:55:09 +0000 | [diff] [blame] | 9612 | /* All non-branch instructions have a single fall-through edge. */ |
| 9613 | if (BPF_CLASS(insns[t].code) != BPF_JMP && |
| 9614 | BPF_CLASS(insns[t].code) != BPF_JMP32) |
| 9615 | return push_insn(t, t + 1, FALLTHROUGH, env, false); |
| 9616 | |
| 9617 | switch (BPF_OP(insns[t].code)) { |
| 9618 | case BPF_EXIT: |
| 9619 | return DONE_EXPLORING; |
| 9620 | |
| 9621 | case BPF_CALL: |
Alexei Starovoitov | bfc6bb7 | 2021-07-14 17:54:14 -0700 | [diff] [blame] | 9622 | if (insns[t].imm == BPF_FUNC_timer_set_callback) |
| 9623 | /* Mark this call insn to trigger is_state_visited() check |
| 9624 | * before call itself is processed by __check_func_call(). |
| 9625 | * Otherwise new async state will be pushed for further |
| 9626 | * exploration. |
| 9627 | */ |
| 9628 | init_explored_state(env, t); |
Yonghong Song | efdb22d | 2021-02-26 12:49:20 -0800 | [diff] [blame] | 9629 | return visit_func_call_insn(t, insn_cnt, insns, env, |
| 9630 | insns[t].src_reg == BPF_PSEUDO_CALL); |
Wedson Almeida Filho | 59e2e27 | 2020-11-21 01:55:09 +0000 | [diff] [blame] | 9631 | |
| 9632 | case BPF_JA: |
| 9633 | if (BPF_SRC(insns[t].code) != BPF_K) |
| 9634 | return -EINVAL; |
| 9635 | |
| 9636 | /* unconditional jump with single edge */ |
| 9637 | ret = push_insn(t, t + insns[t].off + 1, FALLTHROUGH, env, |
| 9638 | true); |
| 9639 | if (ret) |
| 9640 | return ret; |
| 9641 | |
| 9642 | /* unconditional jmp is not a good pruning point, |
| 9643 | * but it's marked, since backtracking needs |
| 9644 | * to record jmp history in is_state_visited(). |
| 9645 | */ |
| 9646 | init_explored_state(env, t + insns[t].off + 1); |
| 9647 | /* tell verifier to check for equivalent states |
| 9648 | * after every call and jump |
| 9649 | */ |
| 9650 | if (t + 1 < insn_cnt) |
| 9651 | init_explored_state(env, t + 1); |
| 9652 | |
| 9653 | return ret; |
| 9654 | |
| 9655 | default: |
| 9656 | /* conditional jump with two edges */ |
| 9657 | init_explored_state(env, t); |
| 9658 | ret = push_insn(t, t + 1, FALLTHROUGH, env, true); |
| 9659 | if (ret) |
| 9660 | return ret; |
| 9661 | |
| 9662 | return push_insn(t, t + insns[t].off + 1, BRANCH, env, true); |
| 9663 | } |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9664 | } |
| 9665 | |
| 9666 | /* non-recursive depth-first-search to detect loops in BPF program |
| 9667 | * loop == back-edge in directed graph |
| 9668 | */ |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 9669 | static int check_cfg(struct bpf_verifier_env *env) |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9670 | { |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9671 | int insn_cnt = env->prog->len; |
Alexei Starovoitov | 7df737e | 2019-04-19 07:44:54 -0700 | [diff] [blame] | 9672 | int *insn_stack, *insn_state; |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9673 | int ret = 0; |
Wedson Almeida Filho | 59e2e27 | 2020-11-21 01:55:09 +0000 | [diff] [blame] | 9674 | int i; |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9675 | |
Alexei Starovoitov | 7df737e | 2019-04-19 07:44:54 -0700 | [diff] [blame] | 9676 | insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9677 | if (!insn_state) |
| 9678 | return -ENOMEM; |
| 9679 | |
Alexei Starovoitov | 7df737e | 2019-04-19 07:44:54 -0700 | [diff] [blame] | 9680 | insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9681 | if (!insn_stack) { |
Alexei Starovoitov | 71dde68 | 2019-04-01 21:27:43 -0700 | [diff] [blame] | 9682 | kvfree(insn_state); |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9683 | return -ENOMEM; |
| 9684 | } |
| 9685 | |
| 9686 | insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */ |
| 9687 | insn_stack[0] = 0; /* 0 is the first instruction */ |
Alexei Starovoitov | 7df737e | 2019-04-19 07:44:54 -0700 | [diff] [blame] | 9688 | env->cfg.cur_stack = 1; |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9689 | |
Wedson Almeida Filho | 59e2e27 | 2020-11-21 01:55:09 +0000 | [diff] [blame] | 9690 | while (env->cfg.cur_stack > 0) { |
| 9691 | int t = insn_stack[env->cfg.cur_stack - 1]; |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9692 | |
Wedson Almeida Filho | 59e2e27 | 2020-11-21 01:55:09 +0000 | [diff] [blame] | 9693 | ret = visit_insn(t, insn_cnt, env); |
| 9694 | switch (ret) { |
| 9695 | case DONE_EXPLORING: |
| 9696 | insn_state[t] = EXPLORED; |
| 9697 | env->cfg.cur_stack--; |
| 9698 | break; |
| 9699 | case KEEP_EXPLORING: |
| 9700 | break; |
| 9701 | default: |
| 9702 | if (ret > 0) { |
| 9703 | verbose(env, "visit_insn internal bug\n"); |
| 9704 | ret = -EFAULT; |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 9705 | } |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9706 | goto err_free; |
Wedson Almeida Filho | 59e2e27 | 2020-11-21 01:55:09 +0000 | [diff] [blame] | 9707 | } |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9708 | } |
| 9709 | |
Wedson Almeida Filho | 59e2e27 | 2020-11-21 01:55:09 +0000 | [diff] [blame] | 9710 | if (env->cfg.cur_stack < 0) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 9711 | verbose(env, "pop stack internal bug\n"); |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9712 | ret = -EFAULT; |
| 9713 | goto err_free; |
| 9714 | } |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9715 | |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9716 | for (i = 0; i < insn_cnt; i++) { |
| 9717 | if (insn_state[i] != EXPLORED) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 9718 | verbose(env, "unreachable insn %d\n", i); |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9719 | ret = -EINVAL; |
| 9720 | goto err_free; |
| 9721 | } |
| 9722 | } |
| 9723 | ret = 0; /* cfg looks good */ |
| 9724 | |
| 9725 | err_free: |
Alexei Starovoitov | 71dde68 | 2019-04-01 21:27:43 -0700 | [diff] [blame] | 9726 | kvfree(insn_state); |
| 9727 | kvfree(insn_stack); |
Alexei Starovoitov | 7df737e | 2019-04-19 07:44:54 -0700 | [diff] [blame] | 9728 | env->cfg.insn_state = env->cfg.insn_stack = NULL; |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 9729 | return ret; |
| 9730 | } |
| 9731 | |
Alexei Starovoitov | 09b28d7 | 2020-09-17 19:09:18 -0700 | [diff] [blame] | 9732 | static int check_abnormal_return(struct bpf_verifier_env *env) |
| 9733 | { |
| 9734 | int i; |
| 9735 | |
| 9736 | for (i = 1; i < env->subprog_cnt; i++) { |
| 9737 | if (env->subprog_info[i].has_ld_abs) { |
| 9738 | verbose(env, "LD_ABS is not allowed in subprogs without BTF\n"); |
| 9739 | return -EINVAL; |
| 9740 | } |
| 9741 | if (env->subprog_info[i].has_tail_call) { |
| 9742 | verbose(env, "tail_call is not allowed in subprogs without BTF\n"); |
| 9743 | return -EINVAL; |
| 9744 | } |
| 9745 | } |
| 9746 | return 0; |
| 9747 | } |
| 9748 | |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 9749 | /* The minimum supported BTF func info size */ |
| 9750 | #define MIN_BPF_FUNCINFO_SIZE 8 |
| 9751 | #define MAX_FUNCINFO_REC_SIZE 252 |
| 9752 | |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 9753 | static int check_btf_func(struct bpf_verifier_env *env, |
| 9754 | const union bpf_attr *attr, |
Alexei Starovoitov | af2ac3e | 2021-05-13 17:36:05 -0700 | [diff] [blame] | 9755 | bpfptr_t uattr) |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 9756 | { |
Alexei Starovoitov | 09b28d7 | 2020-09-17 19:09:18 -0700 | [diff] [blame] | 9757 | const struct btf_type *type, *func_proto, *ret_type; |
Peter Oskolkov | d0b2818 | 2019-01-16 10:43:01 -0800 | [diff] [blame] | 9758 | u32 i, nfuncs, urec_size, min_size; |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 9759 | u32 krec_size = sizeof(struct bpf_func_info); |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 9760 | struct bpf_func_info *krecord; |
Alexei Starovoitov | 8c1b6e6 | 2019-11-14 10:57:16 -0800 | [diff] [blame] | 9761 | struct bpf_func_info_aux *info_aux = NULL; |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 9762 | struct bpf_prog *prog; |
| 9763 | const struct btf *btf; |
Alexei Starovoitov | af2ac3e | 2021-05-13 17:36:05 -0700 | [diff] [blame] | 9764 | bpfptr_t urecord; |
Peter Oskolkov | d0b2818 | 2019-01-16 10:43:01 -0800 | [diff] [blame] | 9765 | u32 prev_offset = 0; |
Alexei Starovoitov | 09b28d7 | 2020-09-17 19:09:18 -0700 | [diff] [blame] | 9766 | bool scalar_return; |
Dan Carpenter | e7ed83d | 2020-06-04 11:54:36 +0300 | [diff] [blame] | 9767 | int ret = -ENOMEM; |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 9768 | |
| 9769 | nfuncs = attr->func_info_cnt; |
Alexei Starovoitov | 09b28d7 | 2020-09-17 19:09:18 -0700 | [diff] [blame] | 9770 | if (!nfuncs) { |
| 9771 | if (check_abnormal_return(env)) |
| 9772 | return -EINVAL; |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 9773 | return 0; |
Alexei Starovoitov | 09b28d7 | 2020-09-17 19:09:18 -0700 | [diff] [blame] | 9774 | } |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 9775 | |
| 9776 | if (nfuncs != env->subprog_cnt) { |
| 9777 | verbose(env, "number of funcs in func_info doesn't match number of subprogs\n"); |
| 9778 | return -EINVAL; |
| 9779 | } |
| 9780 | |
| 9781 | urec_size = attr->func_info_rec_size; |
| 9782 | if (urec_size < MIN_BPF_FUNCINFO_SIZE || |
| 9783 | urec_size > MAX_FUNCINFO_REC_SIZE || |
| 9784 | urec_size % sizeof(u32)) { |
| 9785 | verbose(env, "invalid func info rec size %u\n", urec_size); |
| 9786 | return -EINVAL; |
| 9787 | } |
| 9788 | |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 9789 | prog = env->prog; |
| 9790 | btf = prog->aux->btf; |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 9791 | |
Alexei Starovoitov | af2ac3e | 2021-05-13 17:36:05 -0700 | [diff] [blame] | 9792 | urecord = make_bpfptr(attr->func_info, uattr.is_kernel); |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 9793 | min_size = min_t(u32, krec_size, urec_size); |
| 9794 | |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 9795 | krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN); |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 9796 | if (!krecord) |
| 9797 | return -ENOMEM; |
Alexei Starovoitov | 8c1b6e6 | 2019-11-14 10:57:16 -0800 | [diff] [blame] | 9798 | info_aux = kcalloc(nfuncs, sizeof(*info_aux), GFP_KERNEL | __GFP_NOWARN); |
| 9799 | if (!info_aux) |
| 9800 | goto err_free; |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 9801 | |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 9802 | for (i = 0; i < nfuncs; i++) { |
| 9803 | ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size); |
| 9804 | if (ret) { |
| 9805 | if (ret == -E2BIG) { |
| 9806 | verbose(env, "nonzero tailing record in func info"); |
| 9807 | /* set the size kernel expects so loader can zero |
| 9808 | * out the rest of the record. |
| 9809 | */ |
Alexei Starovoitov | af2ac3e | 2021-05-13 17:36:05 -0700 | [diff] [blame] | 9810 | if (copy_to_bpfptr_offset(uattr, |
| 9811 | offsetof(union bpf_attr, func_info_rec_size), |
| 9812 | &min_size, sizeof(min_size))) |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 9813 | ret = -EFAULT; |
| 9814 | } |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 9815 | goto err_free; |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 9816 | } |
| 9817 | |
Alexei Starovoitov | af2ac3e | 2021-05-13 17:36:05 -0700 | [diff] [blame] | 9818 | if (copy_from_bpfptr(&krecord[i], urecord, min_size)) { |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 9819 | ret = -EFAULT; |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 9820 | goto err_free; |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 9821 | } |
| 9822 | |
Martin KaFai Lau | d30d42e | 2018-12-05 17:35:44 -0800 | [diff] [blame] | 9823 | /* check insn_off */ |
Alexei Starovoitov | 09b28d7 | 2020-09-17 19:09:18 -0700 | [diff] [blame] | 9824 | ret = -EINVAL; |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 9825 | if (i == 0) { |
Martin KaFai Lau | d30d42e | 2018-12-05 17:35:44 -0800 | [diff] [blame] | 9826 | if (krecord[i].insn_off) { |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 9827 | verbose(env, |
Martin KaFai Lau | d30d42e | 2018-12-05 17:35:44 -0800 | [diff] [blame] | 9828 | "nonzero insn_off %u for the first func info record", |
| 9829 | krecord[i].insn_off); |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 9830 | goto err_free; |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 9831 | } |
Martin KaFai Lau | d30d42e | 2018-12-05 17:35:44 -0800 | [diff] [blame] | 9832 | } else if (krecord[i].insn_off <= prev_offset) { |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 9833 | verbose(env, |
| 9834 | "same or smaller insn offset (%u) than previous func info record (%u)", |
Martin KaFai Lau | d30d42e | 2018-12-05 17:35:44 -0800 | [diff] [blame] | 9835 | krecord[i].insn_off, prev_offset); |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 9836 | goto err_free; |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 9837 | } |
| 9838 | |
Martin KaFai Lau | d30d42e | 2018-12-05 17:35:44 -0800 | [diff] [blame] | 9839 | if (env->subprog_info[i].start != krecord[i].insn_off) { |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 9840 | verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 9841 | goto err_free; |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 9842 | } |
| 9843 | |
| 9844 | /* check type_id */ |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 9845 | type = btf_type_by_id(btf, krecord[i].type_id); |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 9846 | if (!type || !btf_type_is_func(type)) { |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 9847 | verbose(env, "invalid type id %d in func info", |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 9848 | krecord[i].type_id); |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 9849 | goto err_free; |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 9850 | } |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 9851 | info_aux[i].linkage = BTF_INFO_VLEN(type->info); |
Alexei Starovoitov | 09b28d7 | 2020-09-17 19:09:18 -0700 | [diff] [blame] | 9852 | |
| 9853 | func_proto = btf_type_by_id(btf, type->type); |
| 9854 | if (unlikely(!func_proto || !btf_type_is_func_proto(func_proto))) |
| 9855 | /* btf_func_check() already verified it during BTF load */ |
| 9856 | goto err_free; |
| 9857 | ret_type = btf_type_skip_modifiers(btf, func_proto->type, NULL); |
| 9858 | scalar_return = |
| 9859 | btf_type_is_small_int(ret_type) || btf_type_is_enum(ret_type); |
| 9860 | if (i && !scalar_return && env->subprog_info[i].has_ld_abs) { |
| 9861 | verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n"); |
| 9862 | goto err_free; |
| 9863 | } |
| 9864 | if (i && !scalar_return && env->subprog_info[i].has_tail_call) { |
| 9865 | verbose(env, "tail_call is only allowed in functions that return 'int'.\n"); |
| 9866 | goto err_free; |
| 9867 | } |
| 9868 | |
Martin KaFai Lau | d30d42e | 2018-12-05 17:35:44 -0800 | [diff] [blame] | 9869 | prev_offset = krecord[i].insn_off; |
Alexei Starovoitov | af2ac3e | 2021-05-13 17:36:05 -0700 | [diff] [blame] | 9870 | bpfptr_add(&urecord, urec_size); |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 9871 | } |
| 9872 | |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 9873 | prog->aux->func_info = krecord; |
| 9874 | prog->aux->func_info_cnt = nfuncs; |
Alexei Starovoitov | 8c1b6e6 | 2019-11-14 10:57:16 -0800 | [diff] [blame] | 9875 | prog->aux->func_info_aux = info_aux; |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 9876 | return 0; |
| 9877 | |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 9878 | err_free: |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 9879 | kvfree(krecord); |
Alexei Starovoitov | 8c1b6e6 | 2019-11-14 10:57:16 -0800 | [diff] [blame] | 9880 | kfree(info_aux); |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 9881 | return ret; |
| 9882 | } |
| 9883 | |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 9884 | static void adjust_btf_func(struct bpf_verifier_env *env) |
| 9885 | { |
Alexei Starovoitov | 8c1b6e6 | 2019-11-14 10:57:16 -0800 | [diff] [blame] | 9886 | struct bpf_prog_aux *aux = env->prog->aux; |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 9887 | int i; |
| 9888 | |
Alexei Starovoitov | 8c1b6e6 | 2019-11-14 10:57:16 -0800 | [diff] [blame] | 9889 | if (!aux->func_info) |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 9890 | return; |
| 9891 | |
| 9892 | for (i = 0; i < env->subprog_cnt; i++) |
Alexei Starovoitov | 8c1b6e6 | 2019-11-14 10:57:16 -0800 | [diff] [blame] | 9893 | aux->func_info[i].insn_off = env->subprog_info[i].start; |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 9894 | } |
| 9895 | |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 9896 | #define MIN_BPF_LINEINFO_SIZE (offsetof(struct bpf_line_info, line_col) + \ |
| 9897 | sizeof(((struct bpf_line_info *)(0))->line_col)) |
| 9898 | #define MAX_LINEINFO_REC_SIZE MAX_FUNCINFO_REC_SIZE |
| 9899 | |
| 9900 | static int check_btf_line(struct bpf_verifier_env *env, |
| 9901 | const union bpf_attr *attr, |
Alexei Starovoitov | af2ac3e | 2021-05-13 17:36:05 -0700 | [diff] [blame] | 9902 | bpfptr_t uattr) |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 9903 | { |
| 9904 | u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0; |
| 9905 | struct bpf_subprog_info *sub; |
| 9906 | struct bpf_line_info *linfo; |
| 9907 | struct bpf_prog *prog; |
| 9908 | const struct btf *btf; |
Alexei Starovoitov | af2ac3e | 2021-05-13 17:36:05 -0700 | [diff] [blame] | 9909 | bpfptr_t ulinfo; |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 9910 | int err; |
| 9911 | |
| 9912 | nr_linfo = attr->line_info_cnt; |
| 9913 | if (!nr_linfo) |
| 9914 | return 0; |
Bixuan Cui | 0e6491b | 2021-09-11 08:55:57 +0800 | [diff] [blame] | 9915 | if (nr_linfo > INT_MAX / sizeof(struct bpf_line_info)) |
| 9916 | return -EINVAL; |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 9917 | |
| 9918 | rec_size = attr->line_info_rec_size; |
| 9919 | if (rec_size < MIN_BPF_LINEINFO_SIZE || |
| 9920 | rec_size > MAX_LINEINFO_REC_SIZE || |
| 9921 | rec_size & (sizeof(u32) - 1)) |
| 9922 | return -EINVAL; |
| 9923 | |
| 9924 | /* Need to zero it in case the userspace may |
| 9925 | * pass in a smaller bpf_line_info object. |
| 9926 | */ |
| 9927 | linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info), |
| 9928 | GFP_KERNEL | __GFP_NOWARN); |
| 9929 | if (!linfo) |
| 9930 | return -ENOMEM; |
| 9931 | |
| 9932 | prog = env->prog; |
| 9933 | btf = prog->aux->btf; |
| 9934 | |
| 9935 | s = 0; |
| 9936 | sub = env->subprog_info; |
Alexei Starovoitov | af2ac3e | 2021-05-13 17:36:05 -0700 | [diff] [blame] | 9937 | ulinfo = make_bpfptr(attr->line_info, uattr.is_kernel); |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 9938 | expected_size = sizeof(struct bpf_line_info); |
| 9939 | ncopy = min_t(u32, expected_size, rec_size); |
| 9940 | for (i = 0; i < nr_linfo; i++) { |
| 9941 | err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size); |
| 9942 | if (err) { |
| 9943 | if (err == -E2BIG) { |
| 9944 | verbose(env, "nonzero tailing record in line_info"); |
Alexei Starovoitov | af2ac3e | 2021-05-13 17:36:05 -0700 | [diff] [blame] | 9945 | if (copy_to_bpfptr_offset(uattr, |
| 9946 | offsetof(union bpf_attr, line_info_rec_size), |
| 9947 | &expected_size, sizeof(expected_size))) |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 9948 | err = -EFAULT; |
| 9949 | } |
| 9950 | goto err_free; |
| 9951 | } |
| 9952 | |
Alexei Starovoitov | af2ac3e | 2021-05-13 17:36:05 -0700 | [diff] [blame] | 9953 | if (copy_from_bpfptr(&linfo[i], ulinfo, ncopy)) { |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 9954 | err = -EFAULT; |
| 9955 | goto err_free; |
| 9956 | } |
| 9957 | |
| 9958 | /* |
| 9959 | * Check insn_off to ensure |
| 9960 | * 1) strictly increasing AND |
| 9961 | * 2) bounded by prog->len |
| 9962 | * |
| 9963 | * The linfo[0].insn_off == 0 check logically falls into |
| 9964 | * the later "missing bpf_line_info for func..." case |
| 9965 | * because the first linfo[0].insn_off must be the |
| 9966 | * first sub also and the first sub must have |
| 9967 | * subprog_info[0].start == 0. |
| 9968 | */ |
| 9969 | if ((i && linfo[i].insn_off <= prev_offset) || |
| 9970 | linfo[i].insn_off >= prog->len) { |
| 9971 | verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n", |
| 9972 | i, linfo[i].insn_off, prev_offset, |
| 9973 | prog->len); |
| 9974 | err = -EINVAL; |
| 9975 | goto err_free; |
| 9976 | } |
| 9977 | |
Martin KaFai Lau | fdbaa0b | 2018-12-19 13:01:01 -0800 | [diff] [blame] | 9978 | if (!prog->insnsi[linfo[i].insn_off].code) { |
| 9979 | verbose(env, |
| 9980 | "Invalid insn code at line_info[%u].insn_off\n", |
| 9981 | i); |
| 9982 | err = -EINVAL; |
| 9983 | goto err_free; |
| 9984 | } |
| 9985 | |
Martin KaFai Lau | 23127b3 | 2018-12-13 10:41:46 -0800 | [diff] [blame] | 9986 | if (!btf_name_by_offset(btf, linfo[i].line_off) || |
| 9987 | !btf_name_by_offset(btf, linfo[i].file_name_off)) { |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 9988 | verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i); |
| 9989 | err = -EINVAL; |
| 9990 | goto err_free; |
| 9991 | } |
| 9992 | |
| 9993 | if (s != env->subprog_cnt) { |
| 9994 | if (linfo[i].insn_off == sub[s].start) { |
| 9995 | sub[s].linfo_idx = i; |
| 9996 | s++; |
| 9997 | } else if (sub[s].start < linfo[i].insn_off) { |
| 9998 | verbose(env, "missing bpf_line_info for func#%u\n", s); |
| 9999 | err = -EINVAL; |
| 10000 | goto err_free; |
| 10001 | } |
| 10002 | } |
| 10003 | |
| 10004 | prev_offset = linfo[i].insn_off; |
Alexei Starovoitov | af2ac3e | 2021-05-13 17:36:05 -0700 | [diff] [blame] | 10005 | bpfptr_add(&ulinfo, rec_size); |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 10006 | } |
| 10007 | |
| 10008 | if (s != env->subprog_cnt) { |
| 10009 | verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n", |
| 10010 | env->subprog_cnt - s, s); |
| 10011 | err = -EINVAL; |
| 10012 | goto err_free; |
| 10013 | } |
| 10014 | |
| 10015 | prog->aux->linfo = linfo; |
| 10016 | prog->aux->nr_linfo = nr_linfo; |
| 10017 | |
| 10018 | return 0; |
| 10019 | |
| 10020 | err_free: |
| 10021 | kvfree(linfo); |
| 10022 | return err; |
| 10023 | } |
| 10024 | |
| 10025 | static int check_btf_info(struct bpf_verifier_env *env, |
| 10026 | const union bpf_attr *attr, |
Alexei Starovoitov | af2ac3e | 2021-05-13 17:36:05 -0700 | [diff] [blame] | 10027 | bpfptr_t uattr) |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 10028 | { |
| 10029 | struct btf *btf; |
| 10030 | int err; |
| 10031 | |
Alexei Starovoitov | 09b28d7 | 2020-09-17 19:09:18 -0700 | [diff] [blame] | 10032 | if (!attr->func_info_cnt && !attr->line_info_cnt) { |
| 10033 | if (check_abnormal_return(env)) |
| 10034 | return -EINVAL; |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 10035 | return 0; |
Alexei Starovoitov | 09b28d7 | 2020-09-17 19:09:18 -0700 | [diff] [blame] | 10036 | } |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 10037 | |
| 10038 | btf = btf_get_by_fd(attr->prog_btf_fd); |
| 10039 | if (IS_ERR(btf)) |
| 10040 | return PTR_ERR(btf); |
Alexei Starovoitov | 350a5c4 | 2021-03-07 14:52:48 -0800 | [diff] [blame] | 10041 | if (btf_is_kernel(btf)) { |
| 10042 | btf_put(btf); |
| 10043 | return -EACCES; |
| 10044 | } |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 10045 | env->prog->aux->btf = btf; |
| 10046 | |
| 10047 | err = check_btf_func(env, attr, uattr); |
| 10048 | if (err) |
| 10049 | return err; |
| 10050 | |
| 10051 | err = check_btf_line(env, attr, uattr); |
| 10052 | if (err) |
| 10053 | return err; |
| 10054 | |
| 10055 | return 0; |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 10056 | } |
| 10057 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 10058 | /* check %cur's range satisfies %old's */ |
| 10059 | static bool range_within(struct bpf_reg_state *old, |
| 10060 | struct bpf_reg_state *cur) |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 10061 | { |
Edward Cree | b03c9f9 | 2017-08-07 15:26:36 +0100 | [diff] [blame] | 10062 | return old->umin_value <= cur->umin_value && |
| 10063 | old->umax_value >= cur->umax_value && |
| 10064 | old->smin_value <= cur->smin_value && |
Daniel Borkmann | fd67518 | 2021-02-05 20:48:21 +0100 | [diff] [blame] | 10065 | old->smax_value >= cur->smax_value && |
| 10066 | old->u32_min_value <= cur->u32_min_value && |
| 10067 | old->u32_max_value >= cur->u32_max_value && |
| 10068 | old->s32_min_value <= cur->s32_min_value && |
| 10069 | old->s32_max_value >= cur->s32_max_value; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 10070 | } |
| 10071 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 10072 | /* If in the old state two registers had the same id, then they need to have |
| 10073 | * the same id in the new state as well. But that id could be different from |
| 10074 | * the old state, so we need to track the mapping from old to new ids. |
| 10075 | * Once we have seen that, say, a reg with old id 5 had new id 9, any subsequent |
| 10076 | * regs with old id 5 must also have new id 9 for the new state to be safe. But |
| 10077 | * regs with a different old id could still have new id 9, we don't care about |
| 10078 | * that. |
| 10079 | * So we look through our idmap to see if this old id has been seen before. If |
| 10080 | * so, we require the new id to match; otherwise, we add the id pair to the map. |
| 10081 | */ |
Lorenz Bauer | c9e73e3 | 2021-04-29 14:46:56 +0100 | [diff] [blame] | 10082 | static bool check_ids(u32 old_id, u32 cur_id, struct bpf_id_pair *idmap) |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 10083 | { |
| 10084 | unsigned int i; |
| 10085 | |
Lorenz Bauer | c9e73e3 | 2021-04-29 14:46:56 +0100 | [diff] [blame] | 10086 | for (i = 0; i < BPF_ID_MAP_SIZE; i++) { |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 10087 | if (!idmap[i].old) { |
| 10088 | /* Reached an empty slot; haven't seen this id before */ |
| 10089 | idmap[i].old = old_id; |
| 10090 | idmap[i].cur = cur_id; |
| 10091 | return true; |
| 10092 | } |
| 10093 | if (idmap[i].old == old_id) |
| 10094 | return idmap[i].cur == cur_id; |
| 10095 | } |
| 10096 | /* We ran out of idmap slots, which should be impossible */ |
| 10097 | WARN_ON_ONCE(1); |
| 10098 | return false; |
| 10099 | } |
| 10100 | |
Alexei Starovoitov | 9242b5f | 2018-12-13 11:42:34 -0800 | [diff] [blame] | 10101 | static void clean_func_state(struct bpf_verifier_env *env, |
| 10102 | struct bpf_func_state *st) |
| 10103 | { |
| 10104 | enum bpf_reg_liveness live; |
| 10105 | int i, j; |
| 10106 | |
| 10107 | for (i = 0; i < BPF_REG_FP; i++) { |
| 10108 | live = st->regs[i].live; |
| 10109 | /* liveness must not touch this register anymore */ |
| 10110 | st->regs[i].live |= REG_LIVE_DONE; |
| 10111 | if (!(live & REG_LIVE_READ)) |
| 10112 | /* since the register is unused, clear its state |
| 10113 | * to make further comparison simpler |
| 10114 | */ |
Daniel Borkmann | f54c789 | 2019-12-22 23:37:40 +0100 | [diff] [blame] | 10115 | __mark_reg_not_init(env, &st->regs[i]); |
Alexei Starovoitov | 9242b5f | 2018-12-13 11:42:34 -0800 | [diff] [blame] | 10116 | } |
| 10117 | |
| 10118 | for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) { |
| 10119 | live = st->stack[i].spilled_ptr.live; |
| 10120 | /* liveness must not touch this stack slot anymore */ |
| 10121 | st->stack[i].spilled_ptr.live |= REG_LIVE_DONE; |
| 10122 | if (!(live & REG_LIVE_READ)) { |
Daniel Borkmann | f54c789 | 2019-12-22 23:37:40 +0100 | [diff] [blame] | 10123 | __mark_reg_not_init(env, &st->stack[i].spilled_ptr); |
Alexei Starovoitov | 9242b5f | 2018-12-13 11:42:34 -0800 | [diff] [blame] | 10124 | for (j = 0; j < BPF_REG_SIZE; j++) |
| 10125 | st->stack[i].slot_type[j] = STACK_INVALID; |
| 10126 | } |
| 10127 | } |
| 10128 | } |
| 10129 | |
| 10130 | static void clean_verifier_state(struct bpf_verifier_env *env, |
| 10131 | struct bpf_verifier_state *st) |
| 10132 | { |
| 10133 | int i; |
| 10134 | |
| 10135 | if (st->frame[0]->regs[0].live & REG_LIVE_DONE) |
| 10136 | /* all regs in this state in all frames were already marked */ |
| 10137 | return; |
| 10138 | |
| 10139 | for (i = 0; i <= st->curframe; i++) |
| 10140 | clean_func_state(env, st->frame[i]); |
| 10141 | } |
| 10142 | |
| 10143 | /* the parentage chains form a tree. |
| 10144 | * the verifier states are added to state lists at given insn and |
| 10145 | * pushed into state stack for future exploration. |
| 10146 | * when the verifier reaches bpf_exit insn some of the verifer states |
| 10147 | * stored in the state lists have their final liveness state already, |
| 10148 | * but a lot of states will get revised from liveness point of view when |
| 10149 | * the verifier explores other branches. |
| 10150 | * Example: |
| 10151 | * 1: r0 = 1 |
| 10152 | * 2: if r1 == 100 goto pc+1 |
| 10153 | * 3: r0 = 2 |
| 10154 | * 4: exit |
| 10155 | * when the verifier reaches exit insn the register r0 in the state list of |
| 10156 | * insn 2 will be seen as !REG_LIVE_READ. Then the verifier pops the other_branch |
| 10157 | * of insn 2 and goes exploring further. At the insn 4 it will walk the |
| 10158 | * parentage chain from insn 4 into insn 2 and will mark r0 as REG_LIVE_READ. |
| 10159 | * |
| 10160 | * Since the verifier pushes the branch states as it sees them while exploring |
| 10161 | * the program the condition of walking the branch instruction for the second |
| 10162 | * time means that all states below this branch were already explored and |
Zhen Lei | 8fb33b6 | 2021-05-25 10:56:59 +0800 | [diff] [blame] | 10163 | * their final liveness marks are already propagated. |
Alexei Starovoitov | 9242b5f | 2018-12-13 11:42:34 -0800 | [diff] [blame] | 10164 | * Hence when the verifier completes the search of state list in is_state_visited() |
| 10165 | * we can call this clean_live_states() function to mark all liveness states |
| 10166 | * as REG_LIVE_DONE to indicate that 'parent' pointers of 'struct bpf_reg_state' |
| 10167 | * will not be used. |
| 10168 | * This function also clears the registers and stack for states that !READ |
| 10169 | * to simplify state merging. |
| 10170 | * |
| 10171 | * Important note here that walking the same branch instruction in the callee |
| 10172 | * doesn't meant that the states are DONE. The verifier has to compare |
| 10173 | * the callsites |
| 10174 | */ |
| 10175 | static void clean_live_states(struct bpf_verifier_env *env, int insn, |
| 10176 | struct bpf_verifier_state *cur) |
| 10177 | { |
| 10178 | struct bpf_verifier_state_list *sl; |
| 10179 | int i; |
| 10180 | |
Alexei Starovoitov | 5d83902 | 2019-05-21 20:17:05 -0700 | [diff] [blame] | 10181 | sl = *explored_state(env, insn); |
Alexei Starovoitov | a8f500a | 2019-05-21 20:17:06 -0700 | [diff] [blame] | 10182 | while (sl) { |
Alexei Starovoitov | 2589726 | 2019-06-15 12:12:20 -0700 | [diff] [blame] | 10183 | if (sl->state.branches) |
| 10184 | goto next; |
Alexei Starovoitov | dc2a4eb | 2019-05-21 20:17:07 -0700 | [diff] [blame] | 10185 | if (sl->state.insn_idx != insn || |
| 10186 | sl->state.curframe != cur->curframe) |
Alexei Starovoitov | 9242b5f | 2018-12-13 11:42:34 -0800 | [diff] [blame] | 10187 | goto next; |
| 10188 | for (i = 0; i <= cur->curframe; i++) |
| 10189 | if (sl->state.frame[i]->callsite != cur->frame[i]->callsite) |
| 10190 | goto next; |
| 10191 | clean_verifier_state(env, &sl->state); |
| 10192 | next: |
| 10193 | sl = sl->next; |
| 10194 | } |
| 10195 | } |
| 10196 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 10197 | /* Returns true if (rold safe implies rcur safe) */ |
Daniel Borkmann | e042aa5 | 2021-07-16 09:18:21 +0000 | [diff] [blame] | 10198 | static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, |
| 10199 | struct bpf_reg_state *rcur, struct bpf_id_pair *idmap) |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 10200 | { |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 10201 | bool equal; |
| 10202 | |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 10203 | if (!(rold->live & REG_LIVE_READ)) |
| 10204 | /* explored state didn't use this */ |
| 10205 | return true; |
| 10206 | |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 10207 | equal = memcmp(rold, rcur, offsetof(struct bpf_reg_state, parent)) == 0; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 10208 | |
| 10209 | if (rold->type == PTR_TO_STACK) |
| 10210 | /* two stack pointers are equal only if they're pointing to |
| 10211 | * the same stack frame, since fp-8 in foo != fp-8 in bar |
| 10212 | */ |
| 10213 | return equal && rold->frameno == rcur->frameno; |
| 10214 | |
| 10215 | if (equal) |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 10216 | return true; |
| 10217 | |
| 10218 | if (rold->type == NOT_INIT) |
| 10219 | /* explored state can't have used this */ |
| 10220 | return true; |
| 10221 | if (rcur->type == NOT_INIT) |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 10222 | return false; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 10223 | switch (rold->type) { |
| 10224 | case SCALAR_VALUE: |
Daniel Borkmann | e042aa5 | 2021-07-16 09:18:21 +0000 | [diff] [blame] | 10225 | if (env->explore_alu_limits) |
| 10226 | return false; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 10227 | if (rcur->type == SCALAR_VALUE) { |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 10228 | if (!rold->precise && !rcur->precise) |
| 10229 | return true; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 10230 | /* new val must satisfy old val knowledge */ |
| 10231 | return range_within(rold, rcur) && |
| 10232 | tnum_in(rold->var_off, rcur->var_off); |
| 10233 | } else { |
Jann Horn | 179d1c5 | 2017-12-18 20:11:59 -0800 | [diff] [blame] | 10234 | /* We're trying to use a pointer in place of a scalar. |
| 10235 | * Even if the scalar was unbounded, this could lead to |
| 10236 | * pointer leaks because scalars are allowed to leak |
| 10237 | * while pointers are not. We could make this safe in |
| 10238 | * special cases if root is calling us, but it's |
| 10239 | * probably not worth the hassle. |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 10240 | */ |
Jann Horn | 179d1c5 | 2017-12-18 20:11:59 -0800 | [diff] [blame] | 10241 | return false; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 10242 | } |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 10243 | case PTR_TO_MAP_KEY: |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 10244 | case PTR_TO_MAP_VALUE: |
Edward Cree | 1b688a1 | 2017-08-23 15:10:50 +0100 | [diff] [blame] | 10245 | /* If the new min/max/var_off satisfy the old ones and |
| 10246 | * everything else matches, we are OK. |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 10247 | * 'id' is not compared, since it's only used for maps with |
| 10248 | * bpf_spin_lock inside map element and in such cases if |
| 10249 | * the rest of the prog is valid for one map element then |
| 10250 | * it's valid for all map elements regardless of the key |
| 10251 | * used in bpf_map_lookup() |
Edward Cree | 1b688a1 | 2017-08-23 15:10:50 +0100 | [diff] [blame] | 10252 | */ |
| 10253 | return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 && |
| 10254 | range_within(rold, rcur) && |
| 10255 | tnum_in(rold->var_off, rcur->var_off); |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 10256 | case PTR_TO_MAP_VALUE_OR_NULL: |
| 10257 | /* a PTR_TO_MAP_VALUE could be safe to use as a |
| 10258 | * PTR_TO_MAP_VALUE_OR_NULL into the same map. |
| 10259 | * However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL- |
| 10260 | * checked, doing so could have affected others with the same |
| 10261 | * id, and we can't check for that because we lost the id when |
| 10262 | * we converted to a PTR_TO_MAP_VALUE. |
| 10263 | */ |
| 10264 | if (rcur->type != PTR_TO_MAP_VALUE_OR_NULL) |
| 10265 | return false; |
| 10266 | if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id))) |
| 10267 | return false; |
| 10268 | /* Check our ids match any regs they're supposed to */ |
| 10269 | return check_ids(rold->id, rcur->id, idmap); |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 10270 | case PTR_TO_PACKET_META: |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 10271 | case PTR_TO_PACKET: |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 10272 | if (rcur->type != rold->type) |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 10273 | return false; |
| 10274 | /* We must have at least as much range as the old ptr |
| 10275 | * did, so that any accesses which were safe before are |
| 10276 | * still safe. This is true even if old range < old off, |
| 10277 | * since someone could have accessed through (ptr - k), or |
| 10278 | * even done ptr -= k in a register, to get a safe access. |
| 10279 | */ |
| 10280 | if (rold->range > rcur->range) |
| 10281 | return false; |
| 10282 | /* If the offsets don't match, we can't trust our alignment; |
| 10283 | * nor can we be sure that we won't fall out of range. |
| 10284 | */ |
| 10285 | if (rold->off != rcur->off) |
| 10286 | return false; |
| 10287 | /* id relations must be preserved */ |
| 10288 | if (rold->id && !check_ids(rold->id, rcur->id, idmap)) |
| 10289 | return false; |
| 10290 | /* new val must satisfy old val knowledge */ |
| 10291 | return range_within(rold, rcur) && |
| 10292 | tnum_in(rold->var_off, rcur->var_off); |
| 10293 | case PTR_TO_CTX: |
| 10294 | case CONST_PTR_TO_MAP: |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 10295 | case PTR_TO_PACKET_END: |
Petar Penkov | d58e468 | 2018-09-14 07:46:18 -0700 | [diff] [blame] | 10296 | case PTR_TO_FLOW_KEYS: |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 10297 | case PTR_TO_SOCKET: |
| 10298 | case PTR_TO_SOCKET_OR_NULL: |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 10299 | case PTR_TO_SOCK_COMMON: |
| 10300 | case PTR_TO_SOCK_COMMON_OR_NULL: |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 10301 | case PTR_TO_TCP_SOCK: |
| 10302 | case PTR_TO_TCP_SOCK_OR_NULL: |
Jonathan Lemon | fada7fd | 2019-06-06 13:59:40 -0700 | [diff] [blame] | 10303 | case PTR_TO_XDP_SOCK: |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 10304 | /* Only valid matches are exact, which memcmp() above |
| 10305 | * would have accepted |
| 10306 | */ |
| 10307 | default: |
| 10308 | /* Don't know what's going on, just say it's not safe */ |
| 10309 | return false; |
| 10310 | } |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 10311 | |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 10312 | /* Shouldn't get here; if we do, say it's not safe */ |
| 10313 | WARN_ON_ONCE(1); |
Alexei Starovoitov | 969bf05 | 2016-05-05 19:49:10 -0700 | [diff] [blame] | 10314 | return false; |
| 10315 | } |
| 10316 | |
Daniel Borkmann | e042aa5 | 2021-07-16 09:18:21 +0000 | [diff] [blame] | 10317 | static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, |
| 10318 | struct bpf_func_state *cur, struct bpf_id_pair *idmap) |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 10319 | { |
| 10320 | int i, spi; |
| 10321 | |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 10322 | /* walk slots of the explored stack and ignore any additional |
| 10323 | * slots in the current stack, since explored(safe) state |
| 10324 | * didn't use them |
| 10325 | */ |
| 10326 | for (i = 0; i < old->allocated_stack; i++) { |
| 10327 | spi = i / BPF_REG_SIZE; |
| 10328 | |
Alexei Starovoitov | b233920 | 2018-12-13 11:42:31 -0800 | [diff] [blame] | 10329 | if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ)) { |
| 10330 | i += BPF_REG_SIZE - 1; |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 10331 | /* explored state didn't use this */ |
Gianluca Borello | fd05e57 | 2017-12-23 10:09:55 +0000 | [diff] [blame] | 10332 | continue; |
Alexei Starovoitov | b233920 | 2018-12-13 11:42:31 -0800 | [diff] [blame] | 10333 | } |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 10334 | |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 10335 | if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID) |
| 10336 | continue; |
Alexei Starovoitov | 19e2dbb | 2018-12-13 11:42:33 -0800 | [diff] [blame] | 10337 | |
| 10338 | /* explored stack has more populated slots than current stack |
| 10339 | * and these slots were used |
| 10340 | */ |
| 10341 | if (i >= cur->allocated_stack) |
| 10342 | return false; |
| 10343 | |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 10344 | /* if old state was safe with misc data in the stack |
| 10345 | * it will be safe with zero-initialized stack. |
| 10346 | * The opposite is not true |
| 10347 | */ |
| 10348 | if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC && |
| 10349 | cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO) |
| 10350 | continue; |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 10351 | if (old->stack[spi].slot_type[i % BPF_REG_SIZE] != |
| 10352 | cur->stack[spi].slot_type[i % BPF_REG_SIZE]) |
| 10353 | /* Ex: old explored (safe) state has STACK_SPILL in |
Randy Dunlap | b8c1a30 | 2020-08-06 20:31:41 -0700 | [diff] [blame] | 10354 | * this stack slot, but current has STACK_MISC -> |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 10355 | * this verifier states are not equivalent, |
| 10356 | * return false to continue verification of this path |
| 10357 | */ |
| 10358 | return false; |
| 10359 | if (i % BPF_REG_SIZE) |
| 10360 | continue; |
| 10361 | if (old->stack[spi].slot_type[0] != STACK_SPILL) |
| 10362 | continue; |
Daniel Borkmann | e042aa5 | 2021-07-16 09:18:21 +0000 | [diff] [blame] | 10363 | if (!regsafe(env, &old->stack[spi].spilled_ptr, |
| 10364 | &cur->stack[spi].spilled_ptr, idmap)) |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 10365 | /* when explored and current stack slot are both storing |
| 10366 | * spilled registers, check that stored pointers types |
| 10367 | * are the same as well. |
| 10368 | * Ex: explored safe path could have stored |
| 10369 | * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -8} |
| 10370 | * but current path has stored: |
| 10371 | * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -16} |
| 10372 | * such verifier states are not equivalent. |
| 10373 | * return false to continue verification of this path |
| 10374 | */ |
| 10375 | return false; |
| 10376 | } |
| 10377 | return true; |
| 10378 | } |
| 10379 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 10380 | static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur) |
| 10381 | { |
| 10382 | if (old->acquired_refs != cur->acquired_refs) |
| 10383 | return false; |
| 10384 | return !memcmp(old->refs, cur->refs, |
| 10385 | sizeof(*old->refs) * old->acquired_refs); |
| 10386 | } |
| 10387 | |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 10388 | /* compare two verifier states |
| 10389 | * |
| 10390 | * all states stored in state_list are known to be valid, since |
| 10391 | * verifier reached 'bpf_exit' instruction through them |
| 10392 | * |
| 10393 | * this function is called when verifier exploring different branches of |
| 10394 | * execution popped from the state stack. If it sees an old state that has |
| 10395 | * more strict register state and more strict stack state then this execution |
| 10396 | * branch doesn't need to be explored further, since verifier already |
| 10397 | * concluded that more strict state leads to valid finish. |
| 10398 | * |
| 10399 | * Therefore two states are equivalent if register state is more conservative |
| 10400 | * and explored stack state is more conservative than the current one. |
| 10401 | * Example: |
| 10402 | * explored current |
| 10403 | * (slot1=INV slot2=MISC) == (slot1=MISC slot2=MISC) |
| 10404 | * (slot1=MISC slot2=MISC) != (slot1=INV slot2=MISC) |
| 10405 | * |
| 10406 | * In other words if current stack state (one being explored) has more |
| 10407 | * valid slots than old one that already passed validation, it means |
| 10408 | * the verifier can stop exploring and conclude that current state is valid too |
| 10409 | * |
| 10410 | * Similarly with registers. If explored state has register type as invalid |
| 10411 | * whereas register type in current state is meaningful, it means that |
| 10412 | * the current state will reach 'bpf_exit' instruction safely |
| 10413 | */ |
Lorenz Bauer | c9e73e3 | 2021-04-29 14:46:56 +0100 | [diff] [blame] | 10414 | static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old, |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 10415 | struct bpf_func_state *cur) |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 10416 | { |
| 10417 | int i; |
| 10418 | |
Lorenz Bauer | c9e73e3 | 2021-04-29 14:46:56 +0100 | [diff] [blame] | 10419 | memset(env->idmap_scratch, 0, sizeof(env->idmap_scratch)); |
| 10420 | for (i = 0; i < MAX_BPF_REG; i++) |
Daniel Borkmann | e042aa5 | 2021-07-16 09:18:21 +0000 | [diff] [blame] | 10421 | if (!regsafe(env, &old->regs[i], &cur->regs[i], |
| 10422 | env->idmap_scratch)) |
Lorenz Bauer | c9e73e3 | 2021-04-29 14:46:56 +0100 | [diff] [blame] | 10423 | return false; |
| 10424 | |
Daniel Borkmann | e042aa5 | 2021-07-16 09:18:21 +0000 | [diff] [blame] | 10425 | if (!stacksafe(env, old, cur, env->idmap_scratch)) |
Alexei Starovoitov | 1a0dc1a | 2016-05-05 19:49:09 -0700 | [diff] [blame] | 10426 | return false; |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 10427 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 10428 | if (!refsafe(old, cur)) |
Lorenz Bauer | c9e73e3 | 2021-04-29 14:46:56 +0100 | [diff] [blame] | 10429 | return false; |
| 10430 | |
| 10431 | return true; |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 10432 | } |
| 10433 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 10434 | static bool states_equal(struct bpf_verifier_env *env, |
| 10435 | struct bpf_verifier_state *old, |
| 10436 | struct bpf_verifier_state *cur) |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 10437 | { |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 10438 | int i; |
| 10439 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 10440 | if (old->curframe != cur->curframe) |
| 10441 | return false; |
| 10442 | |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 10443 | /* Verification state from speculative execution simulation |
| 10444 | * must never prune a non-speculative execution one. |
| 10445 | */ |
| 10446 | if (old->speculative && !cur->speculative) |
| 10447 | return false; |
| 10448 | |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 10449 | if (old->active_spin_lock != cur->active_spin_lock) |
| 10450 | return false; |
| 10451 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 10452 | /* for states to be equal callsites have to be the same |
| 10453 | * and all frame states need to be equivalent |
| 10454 | */ |
| 10455 | for (i = 0; i <= old->curframe; i++) { |
| 10456 | if (old->frame[i]->callsite != cur->frame[i]->callsite) |
| 10457 | return false; |
Lorenz Bauer | c9e73e3 | 2021-04-29 14:46:56 +0100 | [diff] [blame] | 10458 | if (!func_states_equal(env, old->frame[i], cur->frame[i])) |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 10459 | return false; |
| 10460 | } |
| 10461 | return true; |
| 10462 | } |
| 10463 | |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 10464 | /* Return 0 if no propagation happened. Return negative error code if error |
| 10465 | * happened. Otherwise, return the propagated bit. |
| 10466 | */ |
Jiong Wang | 55e7f3b | 2019-04-12 22:59:36 +0100 | [diff] [blame] | 10467 | static int propagate_liveness_reg(struct bpf_verifier_env *env, |
| 10468 | struct bpf_reg_state *reg, |
| 10469 | struct bpf_reg_state *parent_reg) |
| 10470 | { |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 10471 | u8 parent_flag = parent_reg->live & REG_LIVE_READ; |
| 10472 | u8 flag = reg->live & REG_LIVE_READ; |
Jiong Wang | 55e7f3b | 2019-04-12 22:59:36 +0100 | [diff] [blame] | 10473 | int err; |
| 10474 | |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 10475 | /* When comes here, read flags of PARENT_REG or REG could be any of |
| 10476 | * REG_LIVE_READ64, REG_LIVE_READ32, REG_LIVE_NONE. There is no need |
| 10477 | * of propagation if PARENT_REG has strongest REG_LIVE_READ64. |
| 10478 | */ |
| 10479 | if (parent_flag == REG_LIVE_READ64 || |
| 10480 | /* Or if there is no read flag from REG. */ |
| 10481 | !flag || |
| 10482 | /* Or if the read flag from REG is the same as PARENT_REG. */ |
| 10483 | parent_flag == flag) |
Jiong Wang | 55e7f3b | 2019-04-12 22:59:36 +0100 | [diff] [blame] | 10484 | return 0; |
| 10485 | |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 10486 | err = mark_reg_read(env, reg, parent_reg, flag); |
Jiong Wang | 55e7f3b | 2019-04-12 22:59:36 +0100 | [diff] [blame] | 10487 | if (err) |
| 10488 | return err; |
| 10489 | |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 10490 | return flag; |
Jiong Wang | 55e7f3b | 2019-04-12 22:59:36 +0100 | [diff] [blame] | 10491 | } |
| 10492 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 10493 | /* A write screens off any subsequent reads; but write marks come from the |
| 10494 | * straight-line code between a state and its parent. When we arrive at an |
| 10495 | * equivalent state (jump target or such) we didn't arrive by the straight-line |
| 10496 | * code, so read marks in the state must propagate to the parent regardless |
| 10497 | * of the state's write marks. That's what 'parent == state->parent' comparison |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 10498 | * in mark_reg_read() is for. |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 10499 | */ |
| 10500 | static int propagate_liveness(struct bpf_verifier_env *env, |
| 10501 | const struct bpf_verifier_state *vstate, |
| 10502 | struct bpf_verifier_state *vparent) |
| 10503 | { |
Jiong Wang | 3f8cafa | 2019-04-12 22:59:35 +0100 | [diff] [blame] | 10504 | struct bpf_reg_state *state_reg, *parent_reg; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 10505 | struct bpf_func_state *state, *parent; |
Jiong Wang | 3f8cafa | 2019-04-12 22:59:35 +0100 | [diff] [blame] | 10506 | int i, frame, err = 0; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 10507 | |
| 10508 | if (vparent->curframe != vstate->curframe) { |
| 10509 | WARN(1, "propagate_live: parent frame %d current frame %d\n", |
| 10510 | vparent->curframe, vstate->curframe); |
| 10511 | return -EFAULT; |
| 10512 | } |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 10513 | /* Propagate read liveness of registers... */ |
| 10514 | BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG); |
Jakub Kicinski | 83d1631 | 2019-03-21 14:34:36 -0700 | [diff] [blame] | 10515 | for (frame = 0; frame <= vstate->curframe; frame++) { |
Jiong Wang | 3f8cafa | 2019-04-12 22:59:35 +0100 | [diff] [blame] | 10516 | parent = vparent->frame[frame]; |
| 10517 | state = vstate->frame[frame]; |
| 10518 | parent_reg = parent->regs; |
| 10519 | state_reg = state->regs; |
Jakub Kicinski | 83d1631 | 2019-03-21 14:34:36 -0700 | [diff] [blame] | 10520 | /* We don't need to worry about FP liveness, it's read-only */ |
| 10521 | for (i = frame < vstate->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) { |
Jiong Wang | 55e7f3b | 2019-04-12 22:59:36 +0100 | [diff] [blame] | 10522 | err = propagate_liveness_reg(env, &state_reg[i], |
| 10523 | &parent_reg[i]); |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 10524 | if (err < 0) |
Jiong Wang | 3f8cafa | 2019-04-12 22:59:35 +0100 | [diff] [blame] | 10525 | return err; |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 10526 | if (err == REG_LIVE_READ64) |
| 10527 | mark_insn_zext(env, &parent_reg[i]); |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 10528 | } |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 10529 | |
Jiong Wang | 1b04aee | 2019-04-12 22:59:34 +0100 | [diff] [blame] | 10530 | /* Propagate stack slots. */ |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 10531 | for (i = 0; i < state->allocated_stack / BPF_REG_SIZE && |
| 10532 | i < parent->allocated_stack / BPF_REG_SIZE; i++) { |
Jiong Wang | 3f8cafa | 2019-04-12 22:59:35 +0100 | [diff] [blame] | 10533 | parent_reg = &parent->stack[i].spilled_ptr; |
| 10534 | state_reg = &state->stack[i].spilled_ptr; |
Jiong Wang | 55e7f3b | 2019-04-12 22:59:36 +0100 | [diff] [blame] | 10535 | err = propagate_liveness_reg(env, state_reg, |
| 10536 | parent_reg); |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 10537 | if (err < 0) |
Jiong Wang | 3f8cafa | 2019-04-12 22:59:35 +0100 | [diff] [blame] | 10538 | return err; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 10539 | } |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 10540 | } |
Jiong Wang | 5327ed3 | 2019-05-24 23:25:12 +0100 | [diff] [blame] | 10541 | return 0; |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 10542 | } |
| 10543 | |
Alexei Starovoitov | a3ce685 | 2019-06-28 09:24:09 -0700 | [diff] [blame] | 10544 | /* find precise scalars in the previous equivalent state and |
| 10545 | * propagate them into the current state |
| 10546 | */ |
| 10547 | static int propagate_precision(struct bpf_verifier_env *env, |
| 10548 | const struct bpf_verifier_state *old) |
| 10549 | { |
| 10550 | struct bpf_reg_state *state_reg; |
| 10551 | struct bpf_func_state *state; |
| 10552 | int i, err = 0; |
| 10553 | |
| 10554 | state = old->frame[old->curframe]; |
| 10555 | state_reg = state->regs; |
| 10556 | for (i = 0; i < BPF_REG_FP; i++, state_reg++) { |
| 10557 | if (state_reg->type != SCALAR_VALUE || |
| 10558 | !state_reg->precise) |
| 10559 | continue; |
| 10560 | if (env->log.level & BPF_LOG_LEVEL2) |
| 10561 | verbose(env, "propagating r%d\n", i); |
| 10562 | err = mark_chain_precision(env, i); |
| 10563 | if (err < 0) |
| 10564 | return err; |
| 10565 | } |
| 10566 | |
| 10567 | for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { |
| 10568 | if (state->stack[i].slot_type[0] != STACK_SPILL) |
| 10569 | continue; |
| 10570 | state_reg = &state->stack[i].spilled_ptr; |
| 10571 | if (state_reg->type != SCALAR_VALUE || |
| 10572 | !state_reg->precise) |
| 10573 | continue; |
| 10574 | if (env->log.level & BPF_LOG_LEVEL2) |
| 10575 | verbose(env, "propagating fp%d\n", |
| 10576 | (-i - 1) * BPF_REG_SIZE); |
| 10577 | err = mark_chain_precision_stack(env, i); |
| 10578 | if (err < 0) |
| 10579 | return err; |
| 10580 | } |
| 10581 | return 0; |
| 10582 | } |
| 10583 | |
Alexei Starovoitov | 2589726 | 2019-06-15 12:12:20 -0700 | [diff] [blame] | 10584 | static bool states_maybe_looping(struct bpf_verifier_state *old, |
| 10585 | struct bpf_verifier_state *cur) |
| 10586 | { |
| 10587 | struct bpf_func_state *fold, *fcur; |
| 10588 | int i, fr = cur->curframe; |
| 10589 | |
| 10590 | if (old->curframe != fr) |
| 10591 | return false; |
| 10592 | |
| 10593 | fold = old->frame[fr]; |
| 10594 | fcur = cur->frame[fr]; |
| 10595 | for (i = 0; i < MAX_BPF_REG; i++) |
| 10596 | if (memcmp(&fold->regs[i], &fcur->regs[i], |
| 10597 | offsetof(struct bpf_reg_state, parent))) |
| 10598 | return false; |
| 10599 | return true; |
| 10600 | } |
| 10601 | |
| 10602 | |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 10603 | static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 10604 | { |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 10605 | struct bpf_verifier_state_list *new_sl; |
Alexei Starovoitov | 9f4686c | 2019-04-01 21:27:41 -0700 | [diff] [blame] | 10606 | struct bpf_verifier_state_list *sl, **pprev; |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 10607 | struct bpf_verifier_state *cur = env->cur_state, *new; |
Alexei Starovoitov | ceefbc9 | 2018-12-03 22:46:06 -0800 | [diff] [blame] | 10608 | int i, j, err, states_cnt = 0; |
Alexei Starovoitov | 10d274e | 2019-08-22 22:52:12 -0700 | [diff] [blame] | 10609 | bool add_new_state = env->test_state_freq ? true : false; |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 10610 | |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 10611 | cur->last_insn_idx = env->prev_insn_idx; |
Alexei Starovoitov | a8f500a | 2019-05-21 20:17:06 -0700 | [diff] [blame] | 10612 | if (!env->insn_aux_data[insn_idx].prune_point) |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 10613 | /* this 'insn_idx' instruction wasn't marked, so we will not |
| 10614 | * be doing state search here |
| 10615 | */ |
| 10616 | return 0; |
| 10617 | |
Alexei Starovoitov | 2589726 | 2019-06-15 12:12:20 -0700 | [diff] [blame] | 10618 | /* bpf progs typically have pruning point every 4 instructions |
| 10619 | * http://vger.kernel.org/bpfconf2019.html#session-1 |
| 10620 | * Do not add new state for future pruning if the verifier hasn't seen |
| 10621 | * at least 2 jumps and at least 8 instructions. |
| 10622 | * This heuristics helps decrease 'total_states' and 'peak_states' metric. |
| 10623 | * In tests that amounts to up to 50% reduction into total verifier |
| 10624 | * memory consumption and 20% verifier time speedup. |
| 10625 | */ |
| 10626 | if (env->jmps_processed - env->prev_jmps_processed >= 2 && |
| 10627 | env->insn_processed - env->prev_insn_processed >= 8) |
| 10628 | add_new_state = true; |
| 10629 | |
Alexei Starovoitov | a8f500a | 2019-05-21 20:17:06 -0700 | [diff] [blame] | 10630 | pprev = explored_state(env, insn_idx); |
| 10631 | sl = *pprev; |
| 10632 | |
Alexei Starovoitov | 9242b5f | 2018-12-13 11:42:34 -0800 | [diff] [blame] | 10633 | clean_live_states(env, insn_idx, cur); |
| 10634 | |
Alexei Starovoitov | a8f500a | 2019-05-21 20:17:06 -0700 | [diff] [blame] | 10635 | while (sl) { |
Alexei Starovoitov | dc2a4eb | 2019-05-21 20:17:07 -0700 | [diff] [blame] | 10636 | states_cnt++; |
| 10637 | if (sl->state.insn_idx != insn_idx) |
| 10638 | goto next; |
Alexei Starovoitov | bfc6bb7 | 2021-07-14 17:54:14 -0700 | [diff] [blame] | 10639 | |
Alexei Starovoitov | 2589726 | 2019-06-15 12:12:20 -0700 | [diff] [blame] | 10640 | if (sl->state.branches) { |
Alexei Starovoitov | bfc6bb7 | 2021-07-14 17:54:14 -0700 | [diff] [blame] | 10641 | struct bpf_func_state *frame = sl->state.frame[sl->state.curframe]; |
| 10642 | |
| 10643 | if (frame->in_async_callback_fn && |
| 10644 | frame->async_entry_cnt != cur->frame[cur->curframe]->async_entry_cnt) { |
| 10645 | /* Different async_entry_cnt means that the verifier is |
| 10646 | * processing another entry into async callback. |
| 10647 | * Seeing the same state is not an indication of infinite |
| 10648 | * loop or infinite recursion. |
| 10649 | * But finding the same state doesn't mean that it's safe |
| 10650 | * to stop processing the current state. The previous state |
| 10651 | * hasn't yet reached bpf_exit, since state.branches > 0. |
| 10652 | * Checking in_async_callback_fn alone is not enough either. |
| 10653 | * Since the verifier still needs to catch infinite loops |
| 10654 | * inside async callbacks. |
| 10655 | */ |
| 10656 | } else if (states_maybe_looping(&sl->state, cur) && |
| 10657 | states_equal(env, &sl->state, cur)) { |
Alexei Starovoitov | 2589726 | 2019-06-15 12:12:20 -0700 | [diff] [blame] | 10658 | verbose_linfo(env, insn_idx, "; "); |
| 10659 | verbose(env, "infinite loop detected at insn %d\n", insn_idx); |
| 10660 | return -EINVAL; |
| 10661 | } |
| 10662 | /* if the verifier is processing a loop, avoid adding new state |
| 10663 | * too often, since different loop iterations have distinct |
| 10664 | * states and may not help future pruning. |
| 10665 | * This threshold shouldn't be too low to make sure that |
| 10666 | * a loop with large bound will be rejected quickly. |
| 10667 | * The most abusive loop will be: |
| 10668 | * r1 += 1 |
| 10669 | * if r1 < 1000000 goto pc-2 |
| 10670 | * 1M insn_procssed limit / 100 == 10k peak states. |
| 10671 | * This threshold shouldn't be too high either, since states |
| 10672 | * at the end of the loop are likely to be useful in pruning. |
| 10673 | */ |
| 10674 | if (env->jmps_processed - env->prev_jmps_processed < 20 && |
| 10675 | env->insn_processed - env->prev_insn_processed < 100) |
| 10676 | add_new_state = false; |
| 10677 | goto miss; |
| 10678 | } |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 10679 | if (states_equal(env, &sl->state, cur)) { |
Alexei Starovoitov | 9f4686c | 2019-04-01 21:27:41 -0700 | [diff] [blame] | 10680 | sl->hit_cnt++; |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 10681 | /* reached equivalent register/stack state, |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 10682 | * prune the search. |
| 10683 | * Registers read by the continuation are read by us. |
Edward Cree | 8e9cd9c | 2017-08-23 15:11:21 +0100 | [diff] [blame] | 10684 | * If we have any write marks in env->cur_state, they |
| 10685 | * will prevent corresponding reads in the continuation |
| 10686 | * from reaching our parent (an explored_state). Our |
| 10687 | * own state will get the read marks recorded, but |
| 10688 | * they'll be immediately forgotten as we're pruning |
| 10689 | * this state and will pop a new one. |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 10690 | */ |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 10691 | err = propagate_liveness(env, &sl->state, cur); |
Alexei Starovoitov | a3ce685 | 2019-06-28 09:24:09 -0700 | [diff] [blame] | 10692 | |
| 10693 | /* if previous state reached the exit with precision and |
| 10694 | * current state is equivalent to it (except precsion marks) |
| 10695 | * the precision needs to be propagated back in |
| 10696 | * the current state. |
| 10697 | */ |
| 10698 | err = err ? : push_jmp_history(env, cur); |
| 10699 | err = err ? : propagate_precision(env, &sl->state); |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 10700 | if (err) |
| 10701 | return err; |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 10702 | return 1; |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 10703 | } |
Alexei Starovoitov | 2589726 | 2019-06-15 12:12:20 -0700 | [diff] [blame] | 10704 | miss: |
| 10705 | /* when new state is not going to be added do not increase miss count. |
| 10706 | * Otherwise several loop iterations will remove the state |
| 10707 | * recorded earlier. The goal of these heuristics is to have |
| 10708 | * states from some iterations of the loop (some in the beginning |
| 10709 | * and some at the end) to help pruning. |
| 10710 | */ |
| 10711 | if (add_new_state) |
| 10712 | sl->miss_cnt++; |
Alexei Starovoitov | 9f4686c | 2019-04-01 21:27:41 -0700 | [diff] [blame] | 10713 | /* heuristic to determine whether this state is beneficial |
| 10714 | * to keep checking from state equivalence point of view. |
| 10715 | * Higher numbers increase max_states_per_insn and verification time, |
| 10716 | * but do not meaningfully decrease insn_processed. |
| 10717 | */ |
| 10718 | if (sl->miss_cnt > sl->hit_cnt * 3 + 3) { |
| 10719 | /* the state is unlikely to be useful. Remove it to |
| 10720 | * speed up verification |
| 10721 | */ |
| 10722 | *pprev = sl->next; |
| 10723 | if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE) { |
Alexei Starovoitov | 2589726 | 2019-06-15 12:12:20 -0700 | [diff] [blame] | 10724 | u32 br = sl->state.branches; |
| 10725 | |
| 10726 | WARN_ONCE(br, |
| 10727 | "BUG live_done but branches_to_explore %d\n", |
| 10728 | br); |
Alexei Starovoitov | 9f4686c | 2019-04-01 21:27:41 -0700 | [diff] [blame] | 10729 | free_verifier_state(&sl->state, false); |
| 10730 | kfree(sl); |
| 10731 | env->peak_states--; |
| 10732 | } else { |
| 10733 | /* cannot free this state, since parentage chain may |
| 10734 | * walk it later. Add it for free_list instead to |
| 10735 | * be freed at the end of verification |
| 10736 | */ |
| 10737 | sl->next = env->free_list; |
| 10738 | env->free_list = sl; |
| 10739 | } |
| 10740 | sl = *pprev; |
| 10741 | continue; |
| 10742 | } |
Alexei Starovoitov | dc2a4eb | 2019-05-21 20:17:07 -0700 | [diff] [blame] | 10743 | next: |
Alexei Starovoitov | 9f4686c | 2019-04-01 21:27:41 -0700 | [diff] [blame] | 10744 | pprev = &sl->next; |
| 10745 | sl = *pprev; |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 10746 | } |
| 10747 | |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 10748 | if (env->max_states_per_insn < states_cnt) |
| 10749 | env->max_states_per_insn = states_cnt; |
| 10750 | |
Alexei Starovoitov | 2c78ee8 | 2020-05-13 16:03:54 -0700 | [diff] [blame] | 10751 | if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES) |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 10752 | return push_jmp_history(env, cur); |
Alexei Starovoitov | ceefbc9 | 2018-12-03 22:46:06 -0800 | [diff] [blame] | 10753 | |
Alexei Starovoitov | 2589726 | 2019-06-15 12:12:20 -0700 | [diff] [blame] | 10754 | if (!add_new_state) |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 10755 | return push_jmp_history(env, cur); |
Alexei Starovoitov | 2589726 | 2019-06-15 12:12:20 -0700 | [diff] [blame] | 10756 | |
| 10757 | /* There were no equivalent states, remember the current one. |
| 10758 | * Technically the current state is not proven to be safe yet, |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 10759 | * but it will either reach outer most bpf_exit (which means it's safe) |
Alexei Starovoitov | 2589726 | 2019-06-15 12:12:20 -0700 | [diff] [blame] | 10760 | * or it will be rejected. When there are no loops the verifier won't be |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 10761 | * seeing this tuple (frame[0].callsite, frame[1].callsite, .. insn_idx) |
Alexei Starovoitov | 2589726 | 2019-06-15 12:12:20 -0700 | [diff] [blame] | 10762 | * again on the way to bpf_exit. |
| 10763 | * When looping the sl->state.branches will be > 0 and this state |
| 10764 | * will not be considered for equivalence until branches == 0. |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 10765 | */ |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 10766 | new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL); |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 10767 | if (!new_sl) |
| 10768 | return -ENOMEM; |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 10769 | env->total_states++; |
| 10770 | env->peak_states++; |
Alexei Starovoitov | 2589726 | 2019-06-15 12:12:20 -0700 | [diff] [blame] | 10771 | env->prev_jmps_processed = env->jmps_processed; |
| 10772 | env->prev_insn_processed = env->insn_processed; |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 10773 | |
| 10774 | /* add new state to the head of linked list */ |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 10775 | new = &new_sl->state; |
| 10776 | err = copy_verifier_state(new, cur); |
Alexei Starovoitov | 1969db4 | 2017-11-01 00:08:04 -0700 | [diff] [blame] | 10777 | if (err) { |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 10778 | free_verifier_state(new, false); |
Alexei Starovoitov | 1969db4 | 2017-11-01 00:08:04 -0700 | [diff] [blame] | 10779 | kfree(new_sl); |
| 10780 | return err; |
| 10781 | } |
Alexei Starovoitov | dc2a4eb | 2019-05-21 20:17:07 -0700 | [diff] [blame] | 10782 | new->insn_idx = insn_idx; |
Alexei Starovoitov | 2589726 | 2019-06-15 12:12:20 -0700 | [diff] [blame] | 10783 | WARN_ONCE(new->branches != 1, |
| 10784 | "BUG is_state_visited:branches_to_explore=%d insn %d\n", new->branches, insn_idx); |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 10785 | |
Alexei Starovoitov | 2589726 | 2019-06-15 12:12:20 -0700 | [diff] [blame] | 10786 | cur->parent = new; |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 10787 | cur->first_insn_idx = insn_idx; |
| 10788 | clear_jmp_history(cur); |
Alexei Starovoitov | 5d83902 | 2019-05-21 20:17:05 -0700 | [diff] [blame] | 10789 | new_sl->next = *explored_state(env, insn_idx); |
| 10790 | *explored_state(env, insn_idx) = new_sl; |
Jakub Kicinski | 7640ead | 2018-12-12 16:29:07 -0800 | [diff] [blame] | 10791 | /* connect new state to parentage chain. Current frame needs all |
| 10792 | * registers connected. Only r6 - r9 of the callers are alive (pushed |
| 10793 | * to the stack implicitly by JITs) so in callers' frames connect just |
| 10794 | * r6 - r9 as an optimization. Callers will have r1 - r5 connected to |
| 10795 | * the state of the call instruction (with WRITTEN set), and r0 comes |
| 10796 | * from callee with its full parentage chain, anyway. |
| 10797 | */ |
Edward Cree | 8e9cd9c | 2017-08-23 15:11:21 +0100 | [diff] [blame] | 10798 | /* clear write marks in current state: the writes we did are not writes |
| 10799 | * our child did, so they don't screen off its reads from us. |
| 10800 | * (There are no read marks in current state, because reads always mark |
| 10801 | * their parent and current state never has children yet. Only |
| 10802 | * explored_states can get read marks.) |
| 10803 | */ |
Alexei Starovoitov | eea1c22 | 2019-06-15 12:12:21 -0700 | [diff] [blame] | 10804 | for (j = 0; j <= cur->curframe; j++) { |
| 10805 | for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) |
| 10806 | cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i]; |
| 10807 | for (i = 0; i < BPF_REG_FP; i++) |
| 10808 | cur->frame[j]->regs[i].live = REG_LIVE_NONE; |
| 10809 | } |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 10810 | |
| 10811 | /* all stack frames are accessible from callee, clear them all */ |
| 10812 | for (j = 0; j <= cur->curframe; j++) { |
| 10813 | struct bpf_func_state *frame = cur->frame[j]; |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 10814 | struct bpf_func_state *newframe = new->frame[j]; |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 10815 | |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 10816 | for (i = 0; i < frame->allocated_stack / BPF_REG_SIZE; i++) { |
Alexei Starovoitov | cc2b14d | 2017-12-14 17:55:08 -0800 | [diff] [blame] | 10817 | frame->stack[i].spilled_ptr.live = REG_LIVE_NONE; |
Edward Cree | 679c782 | 2018-08-22 20:02:19 +0100 | [diff] [blame] | 10818 | frame->stack[i].spilled_ptr.parent = |
| 10819 | &newframe->stack[i].spilled_ptr; |
| 10820 | } |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 10821 | } |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 10822 | return 0; |
| 10823 | } |
| 10824 | |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 10825 | /* Return true if it's OK to have the same insn return a different type. */ |
| 10826 | static bool reg_type_mismatch_ok(enum bpf_reg_type type) |
| 10827 | { |
| 10828 | switch (type) { |
| 10829 | case PTR_TO_CTX: |
| 10830 | case PTR_TO_SOCKET: |
| 10831 | case PTR_TO_SOCKET_OR_NULL: |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 10832 | case PTR_TO_SOCK_COMMON: |
| 10833 | case PTR_TO_SOCK_COMMON_OR_NULL: |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 10834 | case PTR_TO_TCP_SOCK: |
| 10835 | case PTR_TO_TCP_SOCK_OR_NULL: |
Jonathan Lemon | fada7fd | 2019-06-06 13:59:40 -0700 | [diff] [blame] | 10836 | case PTR_TO_XDP_SOCK: |
Alexei Starovoitov | 2a02759 | 2019-10-15 20:25:02 -0700 | [diff] [blame] | 10837 | case PTR_TO_BTF_ID: |
Yonghong Song | b121b34 | 2020-05-09 10:59:12 -0700 | [diff] [blame] | 10838 | case PTR_TO_BTF_ID_OR_NULL: |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 10839 | return false; |
| 10840 | default: |
| 10841 | return true; |
| 10842 | } |
| 10843 | } |
| 10844 | |
| 10845 | /* If an instruction was previously used with particular pointer types, then we |
| 10846 | * need to be careful to avoid cases such as the below, where it may be ok |
| 10847 | * for one branch accessing the pointer, but not ok for the other branch: |
| 10848 | * |
| 10849 | * R1 = sock_ptr |
| 10850 | * goto X; |
| 10851 | * ... |
| 10852 | * R1 = some_other_valid_ptr; |
| 10853 | * goto X; |
| 10854 | * ... |
| 10855 | * R2 = *(u32 *)(R1 + 0); |
| 10856 | */ |
| 10857 | static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev) |
| 10858 | { |
| 10859 | return src != prev && (!reg_type_mismatch_ok(src) || |
| 10860 | !reg_type_mismatch_ok(prev)); |
| 10861 | } |
| 10862 | |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 10863 | static int do_check(struct bpf_verifier_env *env) |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 10864 | { |
Andrii Nakryiko | 6f8a57c | 2020-04-23 12:58:50 -0700 | [diff] [blame] | 10865 | bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 10866 | struct bpf_verifier_state *state = env->cur_state; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 10867 | struct bpf_insn *insns = env->prog->insnsi; |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 10868 | struct bpf_reg_state *regs; |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 10869 | int insn_cnt = env->prog->len; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 10870 | bool do_print_state = false; |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 10871 | int prev_insn_idx = -1; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 10872 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 10873 | for (;;) { |
| 10874 | struct bpf_insn *insn; |
| 10875 | u8 class; |
| 10876 | int err; |
| 10877 | |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 10878 | env->prev_insn_idx = prev_insn_idx; |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 10879 | if (env->insn_idx >= insn_cnt) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 10880 | verbose(env, "invalid insn idx %d insn_cnt %d\n", |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 10881 | env->insn_idx, insn_cnt); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 10882 | return -EFAULT; |
| 10883 | } |
| 10884 | |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 10885 | insn = &insns[env->insn_idx]; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 10886 | class = BPF_CLASS(insn->code); |
| 10887 | |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 10888 | if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 10889 | verbose(env, |
| 10890 | "BPF program is too large. Processed %d insn\n", |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 10891 | env->insn_processed); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 10892 | return -E2BIG; |
| 10893 | } |
| 10894 | |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 10895 | err = is_state_visited(env, env->insn_idx); |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 10896 | if (err < 0) |
| 10897 | return err; |
| 10898 | if (err == 1) { |
| 10899 | /* found equivalent state, can prune the search */ |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 10900 | if (env->log.level & BPF_LOG_LEVEL) { |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 10901 | if (do_print_state) |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 10902 | verbose(env, "\nfrom %d to %d%s: safe\n", |
| 10903 | env->prev_insn_idx, env->insn_idx, |
| 10904 | env->cur_state->speculative ? |
| 10905 | " (speculative execution)" : ""); |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 10906 | else |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 10907 | verbose(env, "%d: safe\n", env->insn_idx); |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 10908 | } |
| 10909 | goto process_bpf_exit; |
| 10910 | } |
| 10911 | |
Alexei Starovoitov | c349480 | 2018-12-03 22:46:04 -0800 | [diff] [blame] | 10912 | if (signal_pending(current)) |
| 10913 | return -EAGAIN; |
| 10914 | |
Daniel Borkmann | 3c2ce60 | 2017-05-18 03:00:06 +0200 | [diff] [blame] | 10915 | if (need_resched()) |
| 10916 | cond_resched(); |
| 10917 | |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 10918 | if (env->log.level & BPF_LOG_LEVEL2 || |
| 10919 | (env->log.level & BPF_LOG_LEVEL && do_print_state)) { |
| 10920 | if (env->log.level & BPF_LOG_LEVEL2) |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 10921 | verbose(env, "%d:", env->insn_idx); |
David S. Miller | c5fc969 | 2017-05-10 11:25:17 -0700 | [diff] [blame] | 10922 | else |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 10923 | verbose(env, "\nfrom %d to %d%s:", |
| 10924 | env->prev_insn_idx, env->insn_idx, |
| 10925 | env->cur_state->speculative ? |
| 10926 | " (speculative execution)" : ""); |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 10927 | print_verifier_state(env, state->frame[state->curframe]); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 10928 | do_print_state = false; |
| 10929 | } |
| 10930 | |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 10931 | if (env->log.level & BPF_LOG_LEVEL) { |
Daniel Borkmann | 7105e82 | 2017-12-20 13:42:57 +0100 | [diff] [blame] | 10932 | const struct bpf_insn_cbs cbs = { |
Martin KaFai Lau | e6ac245 | 2021-03-24 18:51:42 -0700 | [diff] [blame] | 10933 | .cb_call = disasm_kfunc_name, |
Daniel Borkmann | 7105e82 | 2017-12-20 13:42:57 +0100 | [diff] [blame] | 10934 | .cb_print = verbose, |
Jiri Olsa | abe0884 | 2018-03-23 11:41:28 +0100 | [diff] [blame] | 10935 | .private_data = env, |
Daniel Borkmann | 7105e82 | 2017-12-20 13:42:57 +0100 | [diff] [blame] | 10936 | }; |
| 10937 | |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 10938 | verbose_linfo(env, env->insn_idx, "; "); |
| 10939 | verbose(env, "%d: ", env->insn_idx); |
Jiri Olsa | abe0884 | 2018-03-23 11:41:28 +0100 | [diff] [blame] | 10940 | print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 10941 | } |
| 10942 | |
Jakub Kicinski | cae1927 | 2017-12-27 18:39:05 -0800 | [diff] [blame] | 10943 | if (bpf_prog_is_dev_bound(env->prog->aux)) { |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 10944 | err = bpf_prog_offload_verify_insn(env, env->insn_idx, |
| 10945 | env->prev_insn_idx); |
Jakub Kicinski | cae1927 | 2017-12-27 18:39:05 -0800 | [diff] [blame] | 10946 | if (err) |
| 10947 | return err; |
| 10948 | } |
Jakub Kicinski | 13a27df | 2016-09-21 11:43:58 +0100 | [diff] [blame] | 10949 | |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 10950 | regs = cur_regs(env); |
Daniel Borkmann | fe9a5ca | 2021-05-28 13:47:27 +0000 | [diff] [blame] | 10951 | sanitize_mark_insn_seen(env); |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 10952 | prev_insn_idx = env->insn_idx; |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 10953 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 10954 | if (class == BPF_ALU || class == BPF_ALU64) { |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 10955 | err = check_alu_op(env, insn); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 10956 | if (err) |
| 10957 | return err; |
| 10958 | |
| 10959 | } else if (class == BPF_LDX) { |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 10960 | enum bpf_reg_type *prev_src_type, src_reg_type; |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 10961 | |
| 10962 | /* check for reserved fields is already done */ |
| 10963 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 10964 | /* check src operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 10965 | err = check_reg_arg(env, insn->src_reg, SRC_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 10966 | if (err) |
| 10967 | return err; |
| 10968 | |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 10969 | err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 10970 | if (err) |
| 10971 | return err; |
| 10972 | |
Alexei Starovoitov | 725f9dc | 2015-04-15 16:19:33 -0700 | [diff] [blame] | 10973 | src_reg_type = regs[insn->src_reg].type; |
| 10974 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 10975 | /* check that memory (src_reg + off) is readable, |
| 10976 | * the state of dst_reg will be updated by this func |
| 10977 | */ |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 10978 | err = check_mem_access(env, env->insn_idx, insn->src_reg, |
| 10979 | insn->off, BPF_SIZE(insn->code), |
| 10980 | BPF_READ, insn->dst_reg, false); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 10981 | if (err) |
| 10982 | return err; |
| 10983 | |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 10984 | prev_src_type = &env->insn_aux_data[env->insn_idx].ptr_type; |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 10985 | |
| 10986 | if (*prev_src_type == NOT_INIT) { |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 10987 | /* saw a valid insn |
| 10988 | * dst_reg = *(u32 *)(src_reg + off) |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 10989 | * save type to validate intersecting paths |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 10990 | */ |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 10991 | *prev_src_type = src_reg_type; |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 10992 | |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 10993 | } else if (reg_type_mismatch(src_reg_type, *prev_src_type)) { |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 10994 | /* ABuser program is trying to use the same insn |
| 10995 | * dst_reg = *(u32*) (src_reg + off) |
| 10996 | * with different pointer types: |
| 10997 | * src_reg == ctx in one branch and |
| 10998 | * src_reg == stack|map in some other branch. |
| 10999 | * Reject it. |
| 11000 | */ |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 11001 | verbose(env, "same insn cannot be used with different pointers\n"); |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 11002 | return -EINVAL; |
| 11003 | } |
| 11004 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11005 | } else if (class == BPF_STX) { |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 11006 | enum bpf_reg_type *prev_dst_type, dst_reg_type; |
Alexei Starovoitov | d691f9e | 2015-06-04 10:11:54 -0700 | [diff] [blame] | 11007 | |
Brendan Jackman | 91c960b | 2021-01-14 18:17:44 +0000 | [diff] [blame] | 11008 | if (BPF_MODE(insn->code) == BPF_ATOMIC) { |
| 11009 | err = check_atomic(env, env->insn_idx, insn); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11010 | if (err) |
| 11011 | return err; |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 11012 | env->insn_idx++; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11013 | continue; |
| 11014 | } |
| 11015 | |
Brendan Jackman | 5ca419f | 2021-01-14 18:17:46 +0000 | [diff] [blame] | 11016 | if (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0) { |
| 11017 | verbose(env, "BPF_STX uses reserved fields\n"); |
| 11018 | return -EINVAL; |
| 11019 | } |
| 11020 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11021 | /* check src1 operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 11022 | err = check_reg_arg(env, insn->src_reg, SRC_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11023 | if (err) |
| 11024 | return err; |
| 11025 | /* check src2 operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 11026 | err = check_reg_arg(env, insn->dst_reg, SRC_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11027 | if (err) |
| 11028 | return err; |
| 11029 | |
Alexei Starovoitov | d691f9e | 2015-06-04 10:11:54 -0700 | [diff] [blame] | 11030 | dst_reg_type = regs[insn->dst_reg].type; |
| 11031 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11032 | /* check that memory (dst_reg + off) is writeable */ |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 11033 | err = check_mem_access(env, env->insn_idx, insn->dst_reg, |
| 11034 | insn->off, BPF_SIZE(insn->code), |
| 11035 | BPF_WRITE, insn->src_reg, false); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11036 | if (err) |
| 11037 | return err; |
| 11038 | |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 11039 | prev_dst_type = &env->insn_aux_data[env->insn_idx].ptr_type; |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 11040 | |
| 11041 | if (*prev_dst_type == NOT_INIT) { |
| 11042 | *prev_dst_type = dst_reg_type; |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 11043 | } else if (reg_type_mismatch(dst_reg_type, *prev_dst_type)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 11044 | verbose(env, "same insn cannot be used with different pointers\n"); |
Alexei Starovoitov | d691f9e | 2015-06-04 10:11:54 -0700 | [diff] [blame] | 11045 | return -EINVAL; |
| 11046 | } |
| 11047 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11048 | } else if (class == BPF_ST) { |
| 11049 | if (BPF_MODE(insn->code) != BPF_MEM || |
| 11050 | insn->src_reg != BPF_REG_0) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 11051 | verbose(env, "BPF_ST uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11052 | return -EINVAL; |
| 11053 | } |
| 11054 | /* check src operand */ |
Edward Cree | dc503a8 | 2017-08-15 20:34:35 +0100 | [diff] [blame] | 11055 | err = check_reg_arg(env, insn->dst_reg, SRC_OP); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11056 | if (err) |
| 11057 | return err; |
| 11058 | |
Daniel Borkmann | f37a8cb | 2018-01-16 23:30:10 +0100 | [diff] [blame] | 11059 | if (is_ctx_reg(env, insn->dst_reg)) { |
Joe Stringer | 9d2be44 | 2018-10-02 13:35:31 -0700 | [diff] [blame] | 11060 | verbose(env, "BPF_ST stores into R%d %s is not allowed\n", |
Daniel Borkmann | 2a159c6 | 2018-10-21 02:09:24 +0200 | [diff] [blame] | 11061 | insn->dst_reg, |
| 11062 | reg_type_str[reg_state(env, insn->dst_reg)->type]); |
Daniel Borkmann | f37a8cb | 2018-01-16 23:30:10 +0100 | [diff] [blame] | 11063 | return -EACCES; |
| 11064 | } |
| 11065 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11066 | /* check that memory (dst_reg + off) is writeable */ |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 11067 | err = check_mem_access(env, env->insn_idx, insn->dst_reg, |
| 11068 | insn->off, BPF_SIZE(insn->code), |
| 11069 | BPF_WRITE, -1, false); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11070 | if (err) |
| 11071 | return err; |
| 11072 | |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 11073 | } else if (class == BPF_JMP || class == BPF_JMP32) { |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11074 | u8 opcode = BPF_OP(insn->code); |
| 11075 | |
Alexei Starovoitov | 2589726 | 2019-06-15 12:12:20 -0700 | [diff] [blame] | 11076 | env->jmps_processed++; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11077 | if (opcode == BPF_CALL) { |
| 11078 | if (BPF_SRC(insn->code) != BPF_K || |
| 11079 | insn->off != 0 || |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 11080 | (insn->src_reg != BPF_REG_0 && |
Martin KaFai Lau | e6ac245 | 2021-03-24 18:51:42 -0700 | [diff] [blame] | 11081 | insn->src_reg != BPF_PSEUDO_CALL && |
| 11082 | insn->src_reg != BPF_PSEUDO_KFUNC_CALL) || |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 11083 | insn->dst_reg != BPF_REG_0 || |
| 11084 | class == BPF_JMP32) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 11085 | verbose(env, "BPF_CALL uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11086 | return -EINVAL; |
| 11087 | } |
| 11088 | |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 11089 | if (env->cur_state->active_spin_lock && |
| 11090 | (insn->src_reg == BPF_PSEUDO_CALL || |
| 11091 | insn->imm != BPF_FUNC_spin_unlock)) { |
| 11092 | verbose(env, "function calls are not allowed while holding a lock\n"); |
| 11093 | return -EINVAL; |
| 11094 | } |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 11095 | if (insn->src_reg == BPF_PSEUDO_CALL) |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 11096 | err = check_func_call(env, insn, &env->insn_idx); |
Martin KaFai Lau | e6ac245 | 2021-03-24 18:51:42 -0700 | [diff] [blame] | 11097 | else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) |
| 11098 | err = check_kfunc_call(env, insn); |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 11099 | else |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 11100 | err = check_helper_call(env, insn, &env->insn_idx); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11101 | if (err) |
| 11102 | return err; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11103 | } else if (opcode == BPF_JA) { |
| 11104 | if (BPF_SRC(insn->code) != BPF_K || |
| 11105 | insn->imm != 0 || |
| 11106 | insn->src_reg != BPF_REG_0 || |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 11107 | insn->dst_reg != BPF_REG_0 || |
| 11108 | class == BPF_JMP32) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 11109 | verbose(env, "BPF_JA uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11110 | return -EINVAL; |
| 11111 | } |
| 11112 | |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 11113 | env->insn_idx += insn->off + 1; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11114 | continue; |
| 11115 | |
| 11116 | } else if (opcode == BPF_EXIT) { |
| 11117 | if (BPF_SRC(insn->code) != BPF_K || |
| 11118 | insn->imm != 0 || |
| 11119 | insn->src_reg != BPF_REG_0 || |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 11120 | insn->dst_reg != BPF_REG_0 || |
| 11121 | class == BPF_JMP32) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 11122 | verbose(env, "BPF_EXIT uses reserved fields\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11123 | return -EINVAL; |
| 11124 | } |
| 11125 | |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 11126 | if (env->cur_state->active_spin_lock) { |
| 11127 | verbose(env, "bpf_spin_unlock is missing\n"); |
| 11128 | return -EINVAL; |
| 11129 | } |
| 11130 | |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 11131 | if (state->curframe) { |
| 11132 | /* exit from nested function */ |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 11133 | err = prepare_func_exit(env, &env->insn_idx); |
Alexei Starovoitov | f4d7e40 | 2017-12-14 17:55:06 -0800 | [diff] [blame] | 11134 | if (err) |
| 11135 | return err; |
| 11136 | do_print_state = true; |
| 11137 | continue; |
| 11138 | } |
| 11139 | |
Joe Stringer | fd978bf7 | 2018-10-02 13:35:35 -0700 | [diff] [blame] | 11140 | err = check_reference_leak(env); |
| 11141 | if (err) |
| 11142 | return err; |
| 11143 | |
Alexei Starovoitov | 390ee7e | 2017-10-02 22:50:23 -0700 | [diff] [blame] | 11144 | err = check_return_code(env); |
| 11145 | if (err) |
| 11146 | return err; |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 11147 | process_bpf_exit: |
Alexei Starovoitov | 2589726 | 2019-06-15 12:12:20 -0700 | [diff] [blame] | 11148 | update_branch_counts(env, env->cur_state); |
Alexei Starovoitov | b5dc016 | 2019-06-15 12:12:25 -0700 | [diff] [blame] | 11149 | err = pop_stack(env, &prev_insn_idx, |
Andrii Nakryiko | 6f8a57c | 2020-04-23 12:58:50 -0700 | [diff] [blame] | 11150 | &env->insn_idx, pop_log); |
Alexei Starovoitov | 638f5b9 | 2017-10-31 18:16:05 -0700 | [diff] [blame] | 11151 | if (err < 0) { |
| 11152 | if (err != -ENOENT) |
| 11153 | return err; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11154 | break; |
| 11155 | } else { |
| 11156 | do_print_state = true; |
| 11157 | continue; |
| 11158 | } |
| 11159 | } else { |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 11160 | err = check_cond_jmp_op(env, insn, &env->insn_idx); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11161 | if (err) |
| 11162 | return err; |
| 11163 | } |
| 11164 | } else if (class == BPF_LD) { |
| 11165 | u8 mode = BPF_MODE(insn->code); |
| 11166 | |
| 11167 | if (mode == BPF_ABS || mode == BPF_IND) { |
Alexei Starovoitov | ddd872b | 2014-12-01 15:06:34 -0800 | [diff] [blame] | 11168 | err = check_ld_abs(env, insn); |
| 11169 | if (err) |
| 11170 | return err; |
| 11171 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11172 | } else if (mode == BPF_IMM) { |
| 11173 | err = check_ld_imm(env, insn); |
| 11174 | if (err) |
| 11175 | return err; |
| 11176 | |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 11177 | env->insn_idx++; |
Daniel Borkmann | fe9a5ca | 2021-05-28 13:47:27 +0000 | [diff] [blame] | 11178 | sanitize_mark_insn_seen(env); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11179 | } else { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 11180 | verbose(env, "invalid BPF_LD mode\n"); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11181 | return -EINVAL; |
| 11182 | } |
| 11183 | } else { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 11184 | verbose(env, "unknown insn class %d\n", class); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11185 | return -EINVAL; |
| 11186 | } |
| 11187 | |
Daniel Borkmann | c08435e | 2019-01-03 00:58:27 +0100 | [diff] [blame] | 11188 | env->insn_idx++; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 11189 | } |
| 11190 | |
| 11191 | return 0; |
| 11192 | } |
| 11193 | |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 11194 | static int find_btf_percpu_datasec(struct btf *btf) |
| 11195 | { |
| 11196 | const struct btf_type *t; |
| 11197 | const char *tname; |
| 11198 | int i, n; |
| 11199 | |
| 11200 | /* |
| 11201 | * Both vmlinux and module each have their own ".data..percpu" |
| 11202 | * DATASECs in BTF. So for module's case, we need to skip vmlinux BTF |
| 11203 | * types to look at only module's own BTF types. |
| 11204 | */ |
| 11205 | n = btf_nr_types(btf); |
| 11206 | if (btf_is_module(btf)) |
| 11207 | i = btf_nr_types(btf_vmlinux); |
| 11208 | else |
| 11209 | i = 1; |
| 11210 | |
| 11211 | for(; i < n; i++) { |
| 11212 | t = btf_type_by_id(btf, i); |
| 11213 | if (BTF_INFO_KIND(t->info) != BTF_KIND_DATASEC) |
| 11214 | continue; |
| 11215 | |
| 11216 | tname = btf_name_by_offset(btf, t->name_off); |
| 11217 | if (!strcmp(tname, ".data..percpu")) |
| 11218 | return i; |
| 11219 | } |
| 11220 | |
| 11221 | return -ENOENT; |
| 11222 | } |
| 11223 | |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 11224 | /* replace pseudo btf_id with kernel symbol address */ |
| 11225 | static int check_pseudo_btf_id(struct bpf_verifier_env *env, |
| 11226 | struct bpf_insn *insn, |
| 11227 | struct bpf_insn_aux_data *aux) |
| 11228 | { |
Hao Luo | eaa6bcb | 2020-09-29 16:50:47 -0700 | [diff] [blame] | 11229 | const struct btf_var_secinfo *vsi; |
| 11230 | const struct btf_type *datasec; |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 11231 | struct btf_mod_pair *btf_mod; |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 11232 | const struct btf_type *t; |
| 11233 | const char *sym_name; |
Hao Luo | eaa6bcb | 2020-09-29 16:50:47 -0700 | [diff] [blame] | 11234 | bool percpu = false; |
Kaixu Xia | f16e631 | 2020-11-11 13:03:46 +0800 | [diff] [blame] | 11235 | u32 type, id = insn->imm; |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 11236 | struct btf *btf; |
Kaixu Xia | f16e631 | 2020-11-11 13:03:46 +0800 | [diff] [blame] | 11237 | s32 datasec_id; |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 11238 | u64 addr; |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 11239 | int i, btf_fd, err; |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 11240 | |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 11241 | btf_fd = insn[1].imm; |
| 11242 | if (btf_fd) { |
| 11243 | btf = btf_get_by_fd(btf_fd); |
| 11244 | if (IS_ERR(btf)) { |
| 11245 | verbose(env, "invalid module BTF object FD specified.\n"); |
| 11246 | return -EINVAL; |
| 11247 | } |
| 11248 | } else { |
| 11249 | if (!btf_vmlinux) { |
| 11250 | verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"); |
| 11251 | return -EINVAL; |
| 11252 | } |
| 11253 | btf = btf_vmlinux; |
| 11254 | btf_get(btf); |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 11255 | } |
| 11256 | |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 11257 | t = btf_type_by_id(btf, id); |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 11258 | if (!t) { |
| 11259 | verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id); |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 11260 | err = -ENOENT; |
| 11261 | goto err_put; |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 11262 | } |
| 11263 | |
| 11264 | if (!btf_type_is_var(t)) { |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 11265 | verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR.\n", id); |
| 11266 | err = -EINVAL; |
| 11267 | goto err_put; |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 11268 | } |
| 11269 | |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 11270 | sym_name = btf_name_by_offset(btf, t->name_off); |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 11271 | addr = kallsyms_lookup_name(sym_name); |
| 11272 | if (!addr) { |
| 11273 | verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n", |
| 11274 | sym_name); |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 11275 | err = -ENOENT; |
| 11276 | goto err_put; |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 11277 | } |
| 11278 | |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 11279 | datasec_id = find_btf_percpu_datasec(btf); |
Hao Luo | eaa6bcb | 2020-09-29 16:50:47 -0700 | [diff] [blame] | 11280 | if (datasec_id > 0) { |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 11281 | datasec = btf_type_by_id(btf, datasec_id); |
Hao Luo | eaa6bcb | 2020-09-29 16:50:47 -0700 | [diff] [blame] | 11282 | for_each_vsi(i, datasec, vsi) { |
| 11283 | if (vsi->type == id) { |
| 11284 | percpu = true; |
| 11285 | break; |
| 11286 | } |
| 11287 | } |
| 11288 | } |
| 11289 | |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 11290 | insn[0].imm = (u32)addr; |
| 11291 | insn[1].imm = addr >> 32; |
| 11292 | |
| 11293 | type = t->type; |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 11294 | t = btf_type_skip_modifiers(btf, type, NULL); |
Hao Luo | eaa6bcb | 2020-09-29 16:50:47 -0700 | [diff] [blame] | 11295 | if (percpu) { |
| 11296 | aux->btf_var.reg_type = PTR_TO_PERCPU_BTF_ID; |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 11297 | aux->btf_var.btf = btf; |
Hao Luo | eaa6bcb | 2020-09-29 16:50:47 -0700 | [diff] [blame] | 11298 | aux->btf_var.btf_id = type; |
| 11299 | } else if (!btf_type_is_struct(t)) { |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 11300 | const struct btf_type *ret; |
| 11301 | const char *tname; |
| 11302 | u32 tsize; |
| 11303 | |
| 11304 | /* resolve the type size of ksym. */ |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 11305 | ret = btf_resolve_size(btf, t, &tsize); |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 11306 | if (IS_ERR(ret)) { |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 11307 | tname = btf_name_by_offset(btf, t->name_off); |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 11308 | verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n", |
| 11309 | tname, PTR_ERR(ret)); |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 11310 | err = -EINVAL; |
| 11311 | goto err_put; |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 11312 | } |
| 11313 | aux->btf_var.reg_type = PTR_TO_MEM; |
| 11314 | aux->btf_var.mem_size = tsize; |
| 11315 | } else { |
| 11316 | aux->btf_var.reg_type = PTR_TO_BTF_ID; |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 11317 | aux->btf_var.btf = btf; |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 11318 | aux->btf_var.btf_id = type; |
| 11319 | } |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 11320 | |
| 11321 | /* check whether we recorded this BTF (and maybe module) already */ |
| 11322 | for (i = 0; i < env->used_btf_cnt; i++) { |
| 11323 | if (env->used_btfs[i].btf == btf) { |
| 11324 | btf_put(btf); |
| 11325 | return 0; |
| 11326 | } |
| 11327 | } |
| 11328 | |
| 11329 | if (env->used_btf_cnt >= MAX_USED_BTFS) { |
| 11330 | err = -E2BIG; |
| 11331 | goto err_put; |
| 11332 | } |
| 11333 | |
| 11334 | btf_mod = &env->used_btfs[env->used_btf_cnt]; |
| 11335 | btf_mod->btf = btf; |
| 11336 | btf_mod->module = NULL; |
| 11337 | |
| 11338 | /* if we reference variables from kernel module, bump its refcount */ |
| 11339 | if (btf_is_module(btf)) { |
| 11340 | btf_mod->module = btf_try_get_module(btf); |
| 11341 | if (!btf_mod->module) { |
| 11342 | err = -ENXIO; |
| 11343 | goto err_put; |
| 11344 | } |
| 11345 | } |
| 11346 | |
| 11347 | env->used_btf_cnt++; |
| 11348 | |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 11349 | return 0; |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 11350 | err_put: |
| 11351 | btf_put(btf); |
| 11352 | return err; |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 11353 | } |
| 11354 | |
Martin KaFai Lau | 56f668d | 2017-03-22 10:00:33 -0700 | [diff] [blame] | 11355 | static int check_map_prealloc(struct bpf_map *map) |
| 11356 | { |
| 11357 | return (map->map_type != BPF_MAP_TYPE_HASH && |
Martin KaFai Lau | bcc6b1b | 2017-03-22 10:00:34 -0700 | [diff] [blame] | 11358 | map->map_type != BPF_MAP_TYPE_PERCPU_HASH && |
| 11359 | map->map_type != BPF_MAP_TYPE_HASH_OF_MAPS) || |
Martin KaFai Lau | 56f668d | 2017-03-22 10:00:33 -0700 | [diff] [blame] | 11360 | !(map->map_flags & BPF_F_NO_PREALLOC); |
| 11361 | } |
| 11362 | |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 11363 | static bool is_tracing_prog_type(enum bpf_prog_type type) |
| 11364 | { |
| 11365 | switch (type) { |
| 11366 | case BPF_PROG_TYPE_KPROBE: |
| 11367 | case BPF_PROG_TYPE_TRACEPOINT: |
| 11368 | case BPF_PROG_TYPE_PERF_EVENT: |
| 11369 | case BPF_PROG_TYPE_RAW_TRACEPOINT: |
| 11370 | return true; |
| 11371 | default: |
| 11372 | return false; |
| 11373 | } |
| 11374 | } |
| 11375 | |
Thomas Gleixner | 94dacdb | 2020-02-24 15:01:32 +0100 | [diff] [blame] | 11376 | static bool is_preallocated_map(struct bpf_map *map) |
| 11377 | { |
| 11378 | if (!check_map_prealloc(map)) |
| 11379 | return false; |
| 11380 | if (map->inner_map_meta && !check_map_prealloc(map->inner_map_meta)) |
| 11381 | return false; |
| 11382 | return true; |
| 11383 | } |
| 11384 | |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 11385 | static int check_map_prog_compatibility(struct bpf_verifier_env *env, |
| 11386 | struct bpf_map *map, |
Alexei Starovoitov | fdc15d3 | 2016-09-01 18:37:23 -0700 | [diff] [blame] | 11387 | struct bpf_prog *prog) |
| 11388 | |
| 11389 | { |
Udip Pant | 7e40781 | 2020-08-25 16:20:00 -0700 | [diff] [blame] | 11390 | enum bpf_prog_type prog_type = resolve_prog_type(prog); |
Thomas Gleixner | 94dacdb | 2020-02-24 15:01:32 +0100 | [diff] [blame] | 11391 | /* |
| 11392 | * Validate that trace type programs use preallocated hash maps. |
| 11393 | * |
| 11394 | * For programs attached to PERF events this is mandatory as the |
| 11395 | * perf NMI can hit any arbitrary code sequence. |
| 11396 | * |
| 11397 | * All other trace types using preallocated hash maps are unsafe as |
| 11398 | * well because tracepoint or kprobes can be inside locked regions |
| 11399 | * of the memory allocator or at a place where a recursion into the |
| 11400 | * memory allocator would see inconsistent state. |
| 11401 | * |
Thomas Gleixner | 2ed905c | 2020-02-24 15:01:33 +0100 | [diff] [blame] | 11402 | * On RT enabled kernels run-time allocation of all trace type |
| 11403 | * programs is strictly prohibited due to lock type constraints. On |
| 11404 | * !RT kernels it is allowed for backwards compatibility reasons for |
| 11405 | * now, but warnings are emitted so developers are made aware of |
| 11406 | * the unsafety and can fix their programs before this is enforced. |
Martin KaFai Lau | 56f668d | 2017-03-22 10:00:33 -0700 | [diff] [blame] | 11407 | */ |
Udip Pant | 7e40781 | 2020-08-25 16:20:00 -0700 | [diff] [blame] | 11408 | if (is_tracing_prog_type(prog_type) && !is_preallocated_map(map)) { |
| 11409 | if (prog_type == BPF_PROG_TYPE_PERF_EVENT) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 11410 | verbose(env, "perf_event programs can only use preallocated hash map\n"); |
Martin KaFai Lau | 56f668d | 2017-03-22 10:00:33 -0700 | [diff] [blame] | 11411 | return -EINVAL; |
| 11412 | } |
Thomas Gleixner | 2ed905c | 2020-02-24 15:01:33 +0100 | [diff] [blame] | 11413 | if (IS_ENABLED(CONFIG_PREEMPT_RT)) { |
| 11414 | verbose(env, "trace type programs can only use preallocated hash map\n"); |
| 11415 | return -EINVAL; |
| 11416 | } |
Thomas Gleixner | 94dacdb | 2020-02-24 15:01:32 +0100 | [diff] [blame] | 11417 | WARN_ONCE(1, "trace type BPF program uses run-time allocation\n"); |
| 11418 | verbose(env, "trace type programs with run-time allocated hash maps are unsafe. Switch to preallocated hash maps.\n"); |
Alexei Starovoitov | fdc15d3 | 2016-09-01 18:37:23 -0700 | [diff] [blame] | 11419 | } |
Jakub Kicinski | a388457 | 2018-01-11 20:29:09 -0800 | [diff] [blame] | 11420 | |
KP Singh | 9e7a4d9 | 2020-11-06 10:37:39 +0000 | [diff] [blame] | 11421 | if (map_value_has_spin_lock(map)) { |
| 11422 | if (prog_type == BPF_PROG_TYPE_SOCKET_FILTER) { |
| 11423 | verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n"); |
| 11424 | return -EINVAL; |
| 11425 | } |
| 11426 | |
| 11427 | if (is_tracing_prog_type(prog_type)) { |
| 11428 | verbose(env, "tracing progs cannot use bpf_spin_lock yet\n"); |
| 11429 | return -EINVAL; |
| 11430 | } |
| 11431 | |
| 11432 | if (prog->aux->sleepable) { |
| 11433 | verbose(env, "sleepable progs cannot use bpf_spin_lock yet\n"); |
| 11434 | return -EINVAL; |
| 11435 | } |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 11436 | } |
| 11437 | |
Jakub Kicinski | a388457 | 2018-01-11 20:29:09 -0800 | [diff] [blame] | 11438 | if ((bpf_prog_is_dev_bound(prog->aux) || bpf_map_is_dev_bound(map)) && |
Jakub Kicinski | 0972826 | 2018-07-17 10:53:23 -0700 | [diff] [blame] | 11439 | !bpf_offload_prog_map_match(prog, map)) { |
Jakub Kicinski | a388457 | 2018-01-11 20:29:09 -0800 | [diff] [blame] | 11440 | verbose(env, "offload device mismatch between prog and map\n"); |
| 11441 | return -EINVAL; |
| 11442 | } |
| 11443 | |
Martin KaFai Lau | 85d33df | 2020-01-08 16:35:05 -0800 | [diff] [blame] | 11444 | if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { |
| 11445 | verbose(env, "bpf_struct_ops map cannot be used in prog\n"); |
| 11446 | return -EINVAL; |
| 11447 | } |
| 11448 | |
Alexei Starovoitov | 1e6c62a | 2020-08-27 15:01:11 -0700 | [diff] [blame] | 11449 | if (prog->aux->sleepable) |
| 11450 | switch (map->map_type) { |
| 11451 | case BPF_MAP_TYPE_HASH: |
| 11452 | case BPF_MAP_TYPE_LRU_HASH: |
| 11453 | case BPF_MAP_TYPE_ARRAY: |
Alexei Starovoitov | 638e4b8 | 2021-02-09 19:36:33 -0800 | [diff] [blame] | 11454 | case BPF_MAP_TYPE_PERCPU_HASH: |
| 11455 | case BPF_MAP_TYPE_PERCPU_ARRAY: |
| 11456 | case BPF_MAP_TYPE_LRU_PERCPU_HASH: |
| 11457 | case BPF_MAP_TYPE_ARRAY_OF_MAPS: |
| 11458 | case BPF_MAP_TYPE_HASH_OF_MAPS: |
Alexei Starovoitov | 1e6c62a | 2020-08-27 15:01:11 -0700 | [diff] [blame] | 11459 | if (!is_preallocated_map(map)) { |
| 11460 | verbose(env, |
Alexei Starovoitov | 638e4b8 | 2021-02-09 19:36:33 -0800 | [diff] [blame] | 11461 | "Sleepable programs can only use preallocated maps\n"); |
Alexei Starovoitov | 1e6c62a | 2020-08-27 15:01:11 -0700 | [diff] [blame] | 11462 | return -EINVAL; |
| 11463 | } |
| 11464 | break; |
KP Singh | ba90c2c | 2021-02-04 19:36:21 +0000 | [diff] [blame] | 11465 | case BPF_MAP_TYPE_RINGBUF: |
| 11466 | break; |
Alexei Starovoitov | 1e6c62a | 2020-08-27 15:01:11 -0700 | [diff] [blame] | 11467 | default: |
| 11468 | verbose(env, |
KP Singh | ba90c2c | 2021-02-04 19:36:21 +0000 | [diff] [blame] | 11469 | "Sleepable programs can only use array, hash, and ringbuf maps\n"); |
Alexei Starovoitov | 1e6c62a | 2020-08-27 15:01:11 -0700 | [diff] [blame] | 11470 | return -EINVAL; |
| 11471 | } |
| 11472 | |
Alexei Starovoitov | fdc15d3 | 2016-09-01 18:37:23 -0700 | [diff] [blame] | 11473 | return 0; |
| 11474 | } |
| 11475 | |
Roman Gushchin | b741f16 | 2018-09-28 14:45:43 +0000 | [diff] [blame] | 11476 | static bool bpf_map_is_cgroup_storage(struct bpf_map *map) |
| 11477 | { |
| 11478 | return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE || |
| 11479 | map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE); |
| 11480 | } |
| 11481 | |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 11482 | /* find and rewrite pseudo imm in ld_imm64 instructions: |
| 11483 | * |
| 11484 | * 1. if it accesses map FD, replace it with actual map pointer. |
| 11485 | * 2. if it accesses btf_id of a VAR, replace it with pointer to the var. |
| 11486 | * |
| 11487 | * NOTE: btf_vmlinux is required for converting pseudo btf_id. |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 11488 | */ |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 11489 | static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 11490 | { |
| 11491 | struct bpf_insn *insn = env->prog->insnsi; |
| 11492 | int insn_cnt = env->prog->len; |
Alexei Starovoitov | fdc15d3 | 2016-09-01 18:37:23 -0700 | [diff] [blame] | 11493 | int i, j, err; |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 11494 | |
Daniel Borkmann | f1f7714 | 2017-01-13 23:38:15 +0100 | [diff] [blame] | 11495 | err = bpf_prog_calc_tag(env->prog); |
Daniel Borkmann | aafe6ae | 2016-12-18 01:52:57 +0100 | [diff] [blame] | 11496 | if (err) |
| 11497 | return err; |
| 11498 | |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 11499 | for (i = 0; i < insn_cnt; i++, insn++) { |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 11500 | if (BPF_CLASS(insn->code) == BPF_LDX && |
Alexei Starovoitov | d691f9e | 2015-06-04 10:11:54 -0700 | [diff] [blame] | 11501 | (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 11502 | verbose(env, "BPF_LDX uses reserved fields\n"); |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 11503 | return -EINVAL; |
| 11504 | } |
| 11505 | |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 11506 | if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) { |
Daniel Borkmann | d8eca5b | 2019-04-09 23:20:03 +0200 | [diff] [blame] | 11507 | struct bpf_insn_aux_data *aux; |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 11508 | struct bpf_map *map; |
| 11509 | struct fd f; |
Daniel Borkmann | d8eca5b | 2019-04-09 23:20:03 +0200 | [diff] [blame] | 11510 | u64 addr; |
Alexei Starovoitov | 387544b | 2021-05-13 17:36:10 -0700 | [diff] [blame] | 11511 | u32 fd; |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 11512 | |
| 11513 | if (i == insn_cnt - 1 || insn[1].code != 0 || |
| 11514 | insn[1].dst_reg != 0 || insn[1].src_reg != 0 || |
| 11515 | insn[1].off != 0) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 11516 | verbose(env, "invalid bpf_ld_imm64 insn\n"); |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 11517 | return -EINVAL; |
| 11518 | } |
| 11519 | |
Daniel Borkmann | d8eca5b | 2019-04-09 23:20:03 +0200 | [diff] [blame] | 11520 | if (insn[0].src_reg == 0) |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 11521 | /* valid generic load 64-bit imm */ |
| 11522 | goto next_insn; |
| 11523 | |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 11524 | if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) { |
| 11525 | aux = &env->insn_aux_data[i]; |
| 11526 | err = check_pseudo_btf_id(env, insn, aux); |
| 11527 | if (err) |
| 11528 | return err; |
| 11529 | goto next_insn; |
| 11530 | } |
| 11531 | |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 11532 | if (insn[0].src_reg == BPF_PSEUDO_FUNC) { |
| 11533 | aux = &env->insn_aux_data[i]; |
| 11534 | aux->ptr_type = PTR_TO_FUNC; |
| 11535 | goto next_insn; |
| 11536 | } |
| 11537 | |
Daniel Borkmann | d8eca5b | 2019-04-09 23:20:03 +0200 | [diff] [blame] | 11538 | /* In final convert_pseudo_ld_imm64() step, this is |
| 11539 | * converted into regular 64-bit imm load insn. |
| 11540 | */ |
Alexei Starovoitov | 387544b | 2021-05-13 17:36:10 -0700 | [diff] [blame] | 11541 | switch (insn[0].src_reg) { |
| 11542 | case BPF_PSEUDO_MAP_VALUE: |
| 11543 | case BPF_PSEUDO_MAP_IDX_VALUE: |
| 11544 | break; |
| 11545 | case BPF_PSEUDO_MAP_FD: |
| 11546 | case BPF_PSEUDO_MAP_IDX: |
| 11547 | if (insn[1].imm == 0) |
| 11548 | break; |
| 11549 | fallthrough; |
| 11550 | default: |
| 11551 | verbose(env, "unrecognized bpf_ld_imm64 insn\n"); |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 11552 | return -EINVAL; |
| 11553 | } |
| 11554 | |
Alexei Starovoitov | 387544b | 2021-05-13 17:36:10 -0700 | [diff] [blame] | 11555 | switch (insn[0].src_reg) { |
| 11556 | case BPF_PSEUDO_MAP_IDX_VALUE: |
| 11557 | case BPF_PSEUDO_MAP_IDX: |
| 11558 | if (bpfptr_is_null(env->fd_array)) { |
| 11559 | verbose(env, "fd_idx without fd_array is invalid\n"); |
| 11560 | return -EPROTO; |
| 11561 | } |
| 11562 | if (copy_from_bpfptr_offset(&fd, env->fd_array, |
| 11563 | insn[0].imm * sizeof(fd), |
| 11564 | sizeof(fd))) |
| 11565 | return -EFAULT; |
| 11566 | break; |
| 11567 | default: |
| 11568 | fd = insn[0].imm; |
| 11569 | break; |
| 11570 | } |
| 11571 | |
| 11572 | f = fdget(fd); |
Daniel Borkmann | c210129 | 2015-10-29 14:58:07 +0100 | [diff] [blame] | 11573 | map = __bpf_map_get(f); |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 11574 | if (IS_ERR(map)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 11575 | verbose(env, "fd %d is not pointing to valid bpf_map\n", |
Daniel Borkmann | 2018239 | 2019-03-04 21:08:53 +0100 | [diff] [blame] | 11576 | insn[0].imm); |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 11577 | return PTR_ERR(map); |
| 11578 | } |
| 11579 | |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 11580 | err = check_map_prog_compatibility(env, map, env->prog); |
Alexei Starovoitov | fdc15d3 | 2016-09-01 18:37:23 -0700 | [diff] [blame] | 11581 | if (err) { |
| 11582 | fdput(f); |
| 11583 | return err; |
| 11584 | } |
| 11585 | |
Daniel Borkmann | d8eca5b | 2019-04-09 23:20:03 +0200 | [diff] [blame] | 11586 | aux = &env->insn_aux_data[i]; |
Alexei Starovoitov | 387544b | 2021-05-13 17:36:10 -0700 | [diff] [blame] | 11587 | if (insn[0].src_reg == BPF_PSEUDO_MAP_FD || |
| 11588 | insn[0].src_reg == BPF_PSEUDO_MAP_IDX) { |
Daniel Borkmann | d8eca5b | 2019-04-09 23:20:03 +0200 | [diff] [blame] | 11589 | addr = (unsigned long)map; |
| 11590 | } else { |
| 11591 | u32 off = insn[1].imm; |
| 11592 | |
| 11593 | if (off >= BPF_MAX_VAR_OFF) { |
| 11594 | verbose(env, "direct value offset of %u is not allowed\n", off); |
| 11595 | fdput(f); |
| 11596 | return -EINVAL; |
| 11597 | } |
| 11598 | |
| 11599 | if (!map->ops->map_direct_value_addr) { |
| 11600 | verbose(env, "no direct value access support for this map type\n"); |
| 11601 | fdput(f); |
| 11602 | return -EINVAL; |
| 11603 | } |
| 11604 | |
| 11605 | err = map->ops->map_direct_value_addr(map, &addr, off); |
| 11606 | if (err) { |
| 11607 | verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n", |
| 11608 | map->value_size, off); |
| 11609 | fdput(f); |
| 11610 | return err; |
| 11611 | } |
| 11612 | |
| 11613 | aux->map_off = off; |
| 11614 | addr += off; |
| 11615 | } |
| 11616 | |
| 11617 | insn[0].imm = (u32)addr; |
| 11618 | insn[1].imm = addr >> 32; |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 11619 | |
| 11620 | /* check whether we recorded this map already */ |
Daniel Borkmann | d8eca5b | 2019-04-09 23:20:03 +0200 | [diff] [blame] | 11621 | for (j = 0; j < env->used_map_cnt; j++) { |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 11622 | if (env->used_maps[j] == map) { |
Daniel Borkmann | d8eca5b | 2019-04-09 23:20:03 +0200 | [diff] [blame] | 11623 | aux->map_index = j; |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 11624 | fdput(f); |
| 11625 | goto next_insn; |
| 11626 | } |
Daniel Borkmann | d8eca5b | 2019-04-09 23:20:03 +0200 | [diff] [blame] | 11627 | } |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 11628 | |
| 11629 | if (env->used_map_cnt >= MAX_USED_MAPS) { |
| 11630 | fdput(f); |
| 11631 | return -E2BIG; |
| 11632 | } |
| 11633 | |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 11634 | /* hold the map. If the program is rejected by verifier, |
| 11635 | * the map will be released by release_maps() or it |
| 11636 | * will be used by the valid program until it's unloaded |
Jakub Kicinski | ab7f5bf | 2018-05-03 18:37:17 -0700 | [diff] [blame] | 11637 | * and all maps are released in free_used_maps() |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 11638 | */ |
Andrii Nakryiko | 1e0bd5a | 2019-11-17 09:28:02 -0800 | [diff] [blame] | 11639 | bpf_map_inc(map); |
Daniel Borkmann | d8eca5b | 2019-04-09 23:20:03 +0200 | [diff] [blame] | 11640 | |
| 11641 | aux->map_index = env->used_map_cnt; |
Alexei Starovoitov | 92117d8 | 2016-04-27 18:56:20 -0700 | [diff] [blame] | 11642 | env->used_maps[env->used_map_cnt++] = map; |
| 11643 | |
Roman Gushchin | b741f16 | 2018-09-28 14:45:43 +0000 | [diff] [blame] | 11644 | if (bpf_map_is_cgroup_storage(map) && |
Daniel Borkmann | e473042 | 2019-12-17 13:28:16 +0100 | [diff] [blame] | 11645 | bpf_cgroup_storage_assign(env->prog->aux, map)) { |
Roman Gushchin | b741f16 | 2018-09-28 14:45:43 +0000 | [diff] [blame] | 11646 | verbose(env, "only one cgroup storage of each type is allowed\n"); |
Roman Gushchin | de9cbba | 2018-08-02 14:27:18 -0700 | [diff] [blame] | 11647 | fdput(f); |
| 11648 | return -EBUSY; |
| 11649 | } |
| 11650 | |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 11651 | fdput(f); |
| 11652 | next_insn: |
| 11653 | insn++; |
| 11654 | i++; |
Daniel Borkmann | 5e581da | 2018-01-26 23:33:38 +0100 | [diff] [blame] | 11655 | continue; |
| 11656 | } |
| 11657 | |
| 11658 | /* Basic sanity check before we invest more work here. */ |
| 11659 | if (!bpf_opcode_in_insntable(insn->code)) { |
| 11660 | verbose(env, "unknown opcode %02x\n", insn->code); |
| 11661 | return -EINVAL; |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 11662 | } |
| 11663 | } |
| 11664 | |
| 11665 | /* now all pseudo BPF_LD_IMM64 instructions load valid |
| 11666 | * 'struct bpf_map *' into a register instead of user map_fd. |
| 11667 | * These pointers will be used later by verifier to validate map access. |
| 11668 | */ |
| 11669 | return 0; |
| 11670 | } |
| 11671 | |
| 11672 | /* drop refcnt of maps used by the rejected program */ |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 11673 | static void release_maps(struct bpf_verifier_env *env) |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 11674 | { |
Daniel Borkmann | a2ea074 | 2019-12-16 17:49:00 +0100 | [diff] [blame] | 11675 | __bpf_free_used_maps(env->prog->aux, env->used_maps, |
| 11676 | env->used_map_cnt); |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 11677 | } |
| 11678 | |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 11679 | /* drop refcnt of maps used by the rejected program */ |
| 11680 | static void release_btfs(struct bpf_verifier_env *env) |
| 11681 | { |
| 11682 | __bpf_free_used_btfs(env->prog->aux, env->used_btfs, |
| 11683 | env->used_btf_cnt); |
| 11684 | } |
| 11685 | |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 11686 | /* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */ |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 11687 | static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env) |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 11688 | { |
| 11689 | struct bpf_insn *insn = env->prog->insnsi; |
| 11690 | int insn_cnt = env->prog->len; |
| 11691 | int i; |
| 11692 | |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 11693 | for (i = 0; i < insn_cnt; i++, insn++) { |
| 11694 | if (insn->code != (BPF_LD | BPF_IMM | BPF_DW)) |
| 11695 | continue; |
| 11696 | if (insn->src_reg == BPF_PSEUDO_FUNC) |
| 11697 | continue; |
| 11698 | insn->src_reg = 0; |
| 11699 | } |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 11700 | } |
| 11701 | |
Alexei Starovoitov | 8041902 | 2017-03-15 18:26:41 -0700 | [diff] [blame] | 11702 | /* single env->prog->insni[off] instruction was replaced with the range |
| 11703 | * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying |
| 11704 | * [0, off) and [off, end) to new locations, so the patched range stays zero |
| 11705 | */ |
He Fengqing | 75f0fc7 | 2021-07-14 10:18:15 +0000 | [diff] [blame] | 11706 | static void adjust_insn_aux_data(struct bpf_verifier_env *env, |
| 11707 | struct bpf_insn_aux_data *new_data, |
| 11708 | struct bpf_prog *new_prog, u32 off, u32 cnt) |
Alexei Starovoitov | 8041902 | 2017-03-15 18:26:41 -0700 | [diff] [blame] | 11709 | { |
He Fengqing | 75f0fc7 | 2021-07-14 10:18:15 +0000 | [diff] [blame] | 11710 | struct bpf_insn_aux_data *old_data = env->insn_aux_data; |
Jiong Wang | b325fbc | 2019-05-24 23:25:13 +0100 | [diff] [blame] | 11711 | struct bpf_insn *insn = new_prog->insnsi; |
Daniel Borkmann | d203b0f | 2021-05-28 13:03:30 +0000 | [diff] [blame] | 11712 | u32 old_seen = old_data[off].seen; |
Jiong Wang | b325fbc | 2019-05-24 23:25:13 +0100 | [diff] [blame] | 11713 | u32 prog_len; |
Alexei Starovoitov | c131187 | 2017-11-22 16:42:05 -0800 | [diff] [blame] | 11714 | int i; |
Alexei Starovoitov | 8041902 | 2017-03-15 18:26:41 -0700 | [diff] [blame] | 11715 | |
Jiong Wang | b325fbc | 2019-05-24 23:25:13 +0100 | [diff] [blame] | 11716 | /* aux info at OFF always needs adjustment, no matter fast path |
| 11717 | * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the |
| 11718 | * original insn at old prog. |
| 11719 | */ |
| 11720 | old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1); |
| 11721 | |
Alexei Starovoitov | 8041902 | 2017-03-15 18:26:41 -0700 | [diff] [blame] | 11722 | if (cnt == 1) |
He Fengqing | 75f0fc7 | 2021-07-14 10:18:15 +0000 | [diff] [blame] | 11723 | return; |
Jiong Wang | b325fbc | 2019-05-24 23:25:13 +0100 | [diff] [blame] | 11724 | prog_len = new_prog->len; |
He Fengqing | 75f0fc7 | 2021-07-14 10:18:15 +0000 | [diff] [blame] | 11725 | |
Alexei Starovoitov | 8041902 | 2017-03-15 18:26:41 -0700 | [diff] [blame] | 11726 | memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off); |
| 11727 | memcpy(new_data + off + cnt - 1, old_data + off, |
| 11728 | sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); |
Jiong Wang | b325fbc | 2019-05-24 23:25:13 +0100 | [diff] [blame] | 11729 | for (i = off; i < off + cnt - 1; i++) { |
Daniel Borkmann | d203b0f | 2021-05-28 13:03:30 +0000 | [diff] [blame] | 11730 | /* Expand insni[off]'s seen count to the patched range. */ |
| 11731 | new_data[i].seen = old_seen; |
Jiong Wang | b325fbc | 2019-05-24 23:25:13 +0100 | [diff] [blame] | 11732 | new_data[i].zext_dst = insn_has_def32(env, insn + i); |
| 11733 | } |
Alexei Starovoitov | 8041902 | 2017-03-15 18:26:41 -0700 | [diff] [blame] | 11734 | env->insn_aux_data = new_data; |
| 11735 | vfree(old_data); |
Alexei Starovoitov | 8041902 | 2017-03-15 18:26:41 -0700 | [diff] [blame] | 11736 | } |
| 11737 | |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 11738 | static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) |
| 11739 | { |
| 11740 | int i; |
| 11741 | |
| 11742 | if (len == 1) |
| 11743 | return; |
Jiong Wang | 4cb3d99 | 2018-05-02 16:17:19 -0400 | [diff] [blame] | 11744 | /* NOTE: fake 'exit' subprog should be updated as well. */ |
| 11745 | for (i = 0; i <= env->subprog_cnt; i++) { |
Edward Cree | afd5942 | 2018-11-16 12:00:07 +0000 | [diff] [blame] | 11746 | if (env->subprog_info[i].start <= off) |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 11747 | continue; |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 11748 | env->subprog_info[i].start += len - 1; |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 11749 | } |
| 11750 | } |
| 11751 | |
John Fastabend | 7506d21 | 2021-06-16 15:55:00 -0700 | [diff] [blame] | 11752 | static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) |
Maciej Fijalkowski | a748c69 | 2020-09-16 23:10:05 +0200 | [diff] [blame] | 11753 | { |
| 11754 | struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; |
| 11755 | int i, sz = prog->aux->size_poke_tab; |
| 11756 | struct bpf_jit_poke_descriptor *desc; |
| 11757 | |
| 11758 | for (i = 0; i < sz; i++) { |
| 11759 | desc = &tab[i]; |
John Fastabend | 7506d21 | 2021-06-16 15:55:00 -0700 | [diff] [blame] | 11760 | if (desc->insn_idx <= off) |
| 11761 | continue; |
Maciej Fijalkowski | a748c69 | 2020-09-16 23:10:05 +0200 | [diff] [blame] | 11762 | desc->insn_idx += len - 1; |
| 11763 | } |
| 11764 | } |
| 11765 | |
Alexei Starovoitov | 8041902 | 2017-03-15 18:26:41 -0700 | [diff] [blame] | 11766 | static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, |
| 11767 | const struct bpf_insn *patch, u32 len) |
| 11768 | { |
| 11769 | struct bpf_prog *new_prog; |
He Fengqing | 75f0fc7 | 2021-07-14 10:18:15 +0000 | [diff] [blame] | 11770 | struct bpf_insn_aux_data *new_data = NULL; |
| 11771 | |
| 11772 | if (len > 1) { |
| 11773 | new_data = vzalloc(array_size(env->prog->len + len - 1, |
| 11774 | sizeof(struct bpf_insn_aux_data))); |
| 11775 | if (!new_data) |
| 11776 | return NULL; |
| 11777 | } |
Alexei Starovoitov | 8041902 | 2017-03-15 18:26:41 -0700 | [diff] [blame] | 11778 | |
| 11779 | new_prog = bpf_patch_insn_single(env->prog, off, patch, len); |
Alexei Starovoitov | 4f73379 | 2019-04-01 21:27:44 -0700 | [diff] [blame] | 11780 | if (IS_ERR(new_prog)) { |
| 11781 | if (PTR_ERR(new_prog) == -ERANGE) |
| 11782 | verbose(env, |
| 11783 | "insn %d cannot be patched due to 16-bit range\n", |
| 11784 | env->insn_aux_data[off].orig_idx); |
He Fengqing | 75f0fc7 | 2021-07-14 10:18:15 +0000 | [diff] [blame] | 11785 | vfree(new_data); |
Alexei Starovoitov | 8041902 | 2017-03-15 18:26:41 -0700 | [diff] [blame] | 11786 | return NULL; |
Alexei Starovoitov | 4f73379 | 2019-04-01 21:27:44 -0700 | [diff] [blame] | 11787 | } |
He Fengqing | 75f0fc7 | 2021-07-14 10:18:15 +0000 | [diff] [blame] | 11788 | adjust_insn_aux_data(env, new_data, new_prog, off, len); |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 11789 | adjust_subprog_starts(env, off, len); |
John Fastabend | 7506d21 | 2021-06-16 15:55:00 -0700 | [diff] [blame] | 11790 | adjust_poke_descs(new_prog, off, len); |
Alexei Starovoitov | 8041902 | 2017-03-15 18:26:41 -0700 | [diff] [blame] | 11791 | return new_prog; |
| 11792 | } |
| 11793 | |
Jakub Kicinski | 52875a0 | 2019-01-22 22:45:20 -0800 | [diff] [blame] | 11794 | static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, |
| 11795 | u32 off, u32 cnt) |
| 11796 | { |
| 11797 | int i, j; |
| 11798 | |
| 11799 | /* find first prog starting at or after off (first to remove) */ |
| 11800 | for (i = 0; i < env->subprog_cnt; i++) |
| 11801 | if (env->subprog_info[i].start >= off) |
| 11802 | break; |
| 11803 | /* find first prog starting at or after off + cnt (first to stay) */ |
| 11804 | for (j = i; j < env->subprog_cnt; j++) |
| 11805 | if (env->subprog_info[j].start >= off + cnt) |
| 11806 | break; |
| 11807 | /* if j doesn't start exactly at off + cnt, we are just removing |
| 11808 | * the front of previous prog |
| 11809 | */ |
| 11810 | if (env->subprog_info[j].start != off + cnt) |
| 11811 | j--; |
| 11812 | |
| 11813 | if (j > i) { |
| 11814 | struct bpf_prog_aux *aux = env->prog->aux; |
| 11815 | int move; |
| 11816 | |
| 11817 | /* move fake 'exit' subprog as well */ |
| 11818 | move = env->subprog_cnt + 1 - j; |
| 11819 | |
| 11820 | memmove(env->subprog_info + i, |
| 11821 | env->subprog_info + j, |
| 11822 | sizeof(*env->subprog_info) * move); |
| 11823 | env->subprog_cnt -= j - i; |
| 11824 | |
| 11825 | /* remove func_info */ |
| 11826 | if (aux->func_info) { |
| 11827 | move = aux->func_info_cnt - j; |
| 11828 | |
| 11829 | memmove(aux->func_info + i, |
| 11830 | aux->func_info + j, |
| 11831 | sizeof(*aux->func_info) * move); |
| 11832 | aux->func_info_cnt -= j - i; |
| 11833 | /* func_info->insn_off is set after all code rewrites, |
| 11834 | * in adjust_btf_func() - no need to adjust |
| 11835 | */ |
| 11836 | } |
| 11837 | } else { |
| 11838 | /* convert i from "first prog to remove" to "first to adjust" */ |
| 11839 | if (env->subprog_info[i].start == off) |
| 11840 | i++; |
| 11841 | } |
| 11842 | |
| 11843 | /* update fake 'exit' subprog as well */ |
| 11844 | for (; i <= env->subprog_cnt; i++) |
| 11845 | env->subprog_info[i].start -= cnt; |
| 11846 | |
| 11847 | return 0; |
| 11848 | } |
| 11849 | |
| 11850 | static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, |
| 11851 | u32 cnt) |
| 11852 | { |
| 11853 | struct bpf_prog *prog = env->prog; |
| 11854 | u32 i, l_off, l_cnt, nr_linfo; |
| 11855 | struct bpf_line_info *linfo; |
| 11856 | |
| 11857 | nr_linfo = prog->aux->nr_linfo; |
| 11858 | if (!nr_linfo) |
| 11859 | return 0; |
| 11860 | |
| 11861 | linfo = prog->aux->linfo; |
| 11862 | |
| 11863 | /* find first line info to remove, count lines to be removed */ |
| 11864 | for (i = 0; i < nr_linfo; i++) |
| 11865 | if (linfo[i].insn_off >= off) |
| 11866 | break; |
| 11867 | |
| 11868 | l_off = i; |
| 11869 | l_cnt = 0; |
| 11870 | for (; i < nr_linfo; i++) |
| 11871 | if (linfo[i].insn_off < off + cnt) |
| 11872 | l_cnt++; |
| 11873 | else |
| 11874 | break; |
| 11875 | |
| 11876 | /* First live insn doesn't match first live linfo, it needs to "inherit" |
| 11877 | * last removed linfo. prog is already modified, so prog->len == off |
| 11878 | * means no live instructions after (tail of the program was removed). |
| 11879 | */ |
| 11880 | if (prog->len != off && l_cnt && |
| 11881 | (i == nr_linfo || linfo[i].insn_off != off + cnt)) { |
| 11882 | l_cnt--; |
| 11883 | linfo[--i].insn_off = off + cnt; |
| 11884 | } |
| 11885 | |
| 11886 | /* remove the line info which refer to the removed instructions */ |
| 11887 | if (l_cnt) { |
| 11888 | memmove(linfo + l_off, linfo + i, |
| 11889 | sizeof(*linfo) * (nr_linfo - i)); |
| 11890 | |
| 11891 | prog->aux->nr_linfo -= l_cnt; |
| 11892 | nr_linfo = prog->aux->nr_linfo; |
| 11893 | } |
| 11894 | |
| 11895 | /* pull all linfo[i].insn_off >= off + cnt in by cnt */ |
| 11896 | for (i = l_off; i < nr_linfo; i++) |
| 11897 | linfo[i].insn_off -= cnt; |
| 11898 | |
| 11899 | /* fix up all subprogs (incl. 'exit') which start >= off */ |
| 11900 | for (i = 0; i <= env->subprog_cnt; i++) |
| 11901 | if (env->subprog_info[i].linfo_idx > l_off) { |
| 11902 | /* program may have started in the removed region but |
| 11903 | * may not be fully removed |
| 11904 | */ |
| 11905 | if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) |
| 11906 | env->subprog_info[i].linfo_idx -= l_cnt; |
| 11907 | else |
| 11908 | env->subprog_info[i].linfo_idx = l_off; |
| 11909 | } |
| 11910 | |
| 11911 | return 0; |
| 11912 | } |
| 11913 | |
| 11914 | static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) |
| 11915 | { |
| 11916 | struct bpf_insn_aux_data *aux_data = env->insn_aux_data; |
| 11917 | unsigned int orig_prog_len = env->prog->len; |
| 11918 | int err; |
| 11919 | |
Jakub Kicinski | 08ca90a | 2019-01-22 22:45:24 -0800 | [diff] [blame] | 11920 | if (bpf_prog_is_dev_bound(env->prog->aux)) |
| 11921 | bpf_prog_offload_remove_insns(env, off, cnt); |
| 11922 | |
Jakub Kicinski | 52875a0 | 2019-01-22 22:45:20 -0800 | [diff] [blame] | 11923 | err = bpf_remove_insns(env->prog, off, cnt); |
| 11924 | if (err) |
| 11925 | return err; |
| 11926 | |
| 11927 | err = adjust_subprog_starts_after_remove(env, off, cnt); |
| 11928 | if (err) |
| 11929 | return err; |
| 11930 | |
| 11931 | err = bpf_adj_linfo_after_remove(env, off, cnt); |
| 11932 | if (err) |
| 11933 | return err; |
| 11934 | |
| 11935 | memmove(aux_data + off, aux_data + off + cnt, |
| 11936 | sizeof(*aux_data) * (orig_prog_len - off - cnt)); |
| 11937 | |
| 11938 | return 0; |
| 11939 | } |
| 11940 | |
Daniel Borkmann | 2a5418a | 2018-01-26 23:33:37 +0100 | [diff] [blame] | 11941 | /* The verifier does more data flow analysis than llvm and will not |
| 11942 | * explore branches that are dead at run time. Malicious programs can |
| 11943 | * have dead code too. Therefore replace all dead at-run-time code |
| 11944 | * with 'ja -1'. |
| 11945 | * |
| 11946 | * Just nops are not optimal, e.g. if they would sit at the end of the |
| 11947 | * program and through another bug we would manage to jump there, then |
| 11948 | * we'd execute beyond program memory otherwise. Returning exception |
| 11949 | * code also wouldn't work since we can have subprogs where the dead |
| 11950 | * code could be located. |
Alexei Starovoitov | c131187 | 2017-11-22 16:42:05 -0800 | [diff] [blame] | 11951 | */ |
| 11952 | static void sanitize_dead_code(struct bpf_verifier_env *env) |
| 11953 | { |
| 11954 | struct bpf_insn_aux_data *aux_data = env->insn_aux_data; |
Daniel Borkmann | 2a5418a | 2018-01-26 23:33:37 +0100 | [diff] [blame] | 11955 | struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); |
Alexei Starovoitov | c131187 | 2017-11-22 16:42:05 -0800 | [diff] [blame] | 11956 | struct bpf_insn *insn = env->prog->insnsi; |
| 11957 | const int insn_cnt = env->prog->len; |
| 11958 | int i; |
| 11959 | |
| 11960 | for (i = 0; i < insn_cnt; i++) { |
| 11961 | if (aux_data[i].seen) |
| 11962 | continue; |
Daniel Borkmann | 2a5418a | 2018-01-26 23:33:37 +0100 | [diff] [blame] | 11963 | memcpy(insn + i, &trap, sizeof(trap)); |
Ilya Leoshkevich | 45c709f | 2021-08-12 17:18:10 +0200 | [diff] [blame] | 11964 | aux_data[i].zext_dst = false; |
Alexei Starovoitov | c131187 | 2017-11-22 16:42:05 -0800 | [diff] [blame] | 11965 | } |
| 11966 | } |
| 11967 | |
Jakub Kicinski | e2ae4ca | 2019-01-22 22:45:19 -0800 | [diff] [blame] | 11968 | static bool insn_is_cond_jump(u8 code) |
| 11969 | { |
| 11970 | u8 op; |
| 11971 | |
Jiong Wang | 092ed09 | 2019-01-26 12:26:01 -0500 | [diff] [blame] | 11972 | if (BPF_CLASS(code) == BPF_JMP32) |
| 11973 | return true; |
| 11974 | |
Jakub Kicinski | e2ae4ca | 2019-01-22 22:45:19 -0800 | [diff] [blame] | 11975 | if (BPF_CLASS(code) != BPF_JMP) |
| 11976 | return false; |
| 11977 | |
| 11978 | op = BPF_OP(code); |
| 11979 | return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL; |
| 11980 | } |
| 11981 | |
| 11982 | static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) |
| 11983 | { |
| 11984 | struct bpf_insn_aux_data *aux_data = env->insn_aux_data; |
| 11985 | struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); |
| 11986 | struct bpf_insn *insn = env->prog->insnsi; |
| 11987 | const int insn_cnt = env->prog->len; |
| 11988 | int i; |
| 11989 | |
| 11990 | for (i = 0; i < insn_cnt; i++, insn++) { |
| 11991 | if (!insn_is_cond_jump(insn->code)) |
| 11992 | continue; |
| 11993 | |
| 11994 | if (!aux_data[i + 1].seen) |
| 11995 | ja.off = insn->off; |
| 11996 | else if (!aux_data[i + 1 + insn->off].seen) |
| 11997 | ja.off = 0; |
| 11998 | else |
| 11999 | continue; |
| 12000 | |
Jakub Kicinski | 08ca90a | 2019-01-22 22:45:24 -0800 | [diff] [blame] | 12001 | if (bpf_prog_is_dev_bound(env->prog->aux)) |
| 12002 | bpf_prog_offload_replace_insn(env, i, &ja); |
| 12003 | |
Jakub Kicinski | e2ae4ca | 2019-01-22 22:45:19 -0800 | [diff] [blame] | 12004 | memcpy(insn, &ja, sizeof(ja)); |
| 12005 | } |
| 12006 | } |
| 12007 | |
Jakub Kicinski | 52875a0 | 2019-01-22 22:45:20 -0800 | [diff] [blame] | 12008 | static int opt_remove_dead_code(struct bpf_verifier_env *env) |
| 12009 | { |
| 12010 | struct bpf_insn_aux_data *aux_data = env->insn_aux_data; |
| 12011 | int insn_cnt = env->prog->len; |
| 12012 | int i, err; |
| 12013 | |
| 12014 | for (i = 0; i < insn_cnt; i++) { |
| 12015 | int j; |
| 12016 | |
| 12017 | j = 0; |
| 12018 | while (i + j < insn_cnt && !aux_data[i + j].seen) |
| 12019 | j++; |
| 12020 | if (!j) |
| 12021 | continue; |
| 12022 | |
| 12023 | err = verifier_remove_insns(env, i, j); |
| 12024 | if (err) |
| 12025 | return err; |
| 12026 | insn_cnt = env->prog->len; |
| 12027 | } |
| 12028 | |
| 12029 | return 0; |
| 12030 | } |
| 12031 | |
Jakub Kicinski | a1b14ab | 2019-01-22 22:45:21 -0800 | [diff] [blame] | 12032 | static int opt_remove_nops(struct bpf_verifier_env *env) |
| 12033 | { |
| 12034 | const struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); |
| 12035 | struct bpf_insn *insn = env->prog->insnsi; |
| 12036 | int insn_cnt = env->prog->len; |
| 12037 | int i, err; |
| 12038 | |
| 12039 | for (i = 0; i < insn_cnt; i++) { |
| 12040 | if (memcmp(&insn[i], &ja, sizeof(ja))) |
| 12041 | continue; |
| 12042 | |
| 12043 | err = verifier_remove_insns(env, i, 1); |
| 12044 | if (err) |
| 12045 | return err; |
| 12046 | insn_cnt--; |
| 12047 | i--; |
| 12048 | } |
| 12049 | |
| 12050 | return 0; |
| 12051 | } |
| 12052 | |
Jiong Wang | d6c2308 | 2019-05-24 23:25:18 +0100 | [diff] [blame] | 12053 | static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, |
| 12054 | const union bpf_attr *attr) |
Jiong Wang | a4b1d3c | 2019-05-24 23:25:15 +0100 | [diff] [blame] | 12055 | { |
Jiong Wang | d6c2308 | 2019-05-24 23:25:18 +0100 | [diff] [blame] | 12056 | struct bpf_insn *patch, zext_patch[2], rnd_hi32_patch[4]; |
Jiong Wang | a4b1d3c | 2019-05-24 23:25:15 +0100 | [diff] [blame] | 12057 | struct bpf_insn_aux_data *aux = env->insn_aux_data; |
Jiong Wang | d6c2308 | 2019-05-24 23:25:18 +0100 | [diff] [blame] | 12058 | int i, patch_len, delta = 0, len = env->prog->len; |
Jiong Wang | a4b1d3c | 2019-05-24 23:25:15 +0100 | [diff] [blame] | 12059 | struct bpf_insn *insns = env->prog->insnsi; |
Jiong Wang | a4b1d3c | 2019-05-24 23:25:15 +0100 | [diff] [blame] | 12060 | struct bpf_prog *new_prog; |
Jiong Wang | d6c2308 | 2019-05-24 23:25:18 +0100 | [diff] [blame] | 12061 | bool rnd_hi32; |
Jiong Wang | a4b1d3c | 2019-05-24 23:25:15 +0100 | [diff] [blame] | 12062 | |
Jiong Wang | d6c2308 | 2019-05-24 23:25:18 +0100 | [diff] [blame] | 12063 | rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32; |
Jiong Wang | a4b1d3c | 2019-05-24 23:25:15 +0100 | [diff] [blame] | 12064 | zext_patch[1] = BPF_ZEXT_REG(0); |
Jiong Wang | d6c2308 | 2019-05-24 23:25:18 +0100 | [diff] [blame] | 12065 | rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0); |
| 12066 | rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32); |
| 12067 | rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX); |
Jiong Wang | a4b1d3c | 2019-05-24 23:25:15 +0100 | [diff] [blame] | 12068 | for (i = 0; i < len; i++) { |
| 12069 | int adj_idx = i + delta; |
| 12070 | struct bpf_insn insn; |
Ilya Leoshkevich | 83a2881 | 2021-03-01 16:40:19 +0100 | [diff] [blame] | 12071 | int load_reg; |
Jiong Wang | a4b1d3c | 2019-05-24 23:25:15 +0100 | [diff] [blame] | 12072 | |
Jiong Wang | d6c2308 | 2019-05-24 23:25:18 +0100 | [diff] [blame] | 12073 | insn = insns[adj_idx]; |
Ilya Leoshkevich | 83a2881 | 2021-03-01 16:40:19 +0100 | [diff] [blame] | 12074 | load_reg = insn_def_regno(&insn); |
Jiong Wang | d6c2308 | 2019-05-24 23:25:18 +0100 | [diff] [blame] | 12075 | if (!aux[adj_idx].zext_dst) { |
| 12076 | u8 code, class; |
| 12077 | u32 imm_rnd; |
| 12078 | |
| 12079 | if (!rnd_hi32) |
| 12080 | continue; |
| 12081 | |
| 12082 | code = insn.code; |
| 12083 | class = BPF_CLASS(code); |
Ilya Leoshkevich | 83a2881 | 2021-03-01 16:40:19 +0100 | [diff] [blame] | 12084 | if (load_reg == -1) |
Jiong Wang | d6c2308 | 2019-05-24 23:25:18 +0100 | [diff] [blame] | 12085 | continue; |
| 12086 | |
| 12087 | /* NOTE: arg "reg" (the fourth one) is only used for |
Ilya Leoshkevich | 83a2881 | 2021-03-01 16:40:19 +0100 | [diff] [blame] | 12088 | * BPF_STX + SRC_OP, so it is safe to pass NULL |
| 12089 | * here. |
Jiong Wang | d6c2308 | 2019-05-24 23:25:18 +0100 | [diff] [blame] | 12090 | */ |
Ilya Leoshkevich | 83a2881 | 2021-03-01 16:40:19 +0100 | [diff] [blame] | 12091 | if (is_reg64(env, &insn, load_reg, NULL, DST_OP)) { |
Jiong Wang | d6c2308 | 2019-05-24 23:25:18 +0100 | [diff] [blame] | 12092 | if (class == BPF_LD && |
| 12093 | BPF_MODE(code) == BPF_IMM) |
| 12094 | i++; |
| 12095 | continue; |
| 12096 | } |
| 12097 | |
| 12098 | /* ctx load could be transformed into wider load. */ |
| 12099 | if (class == BPF_LDX && |
| 12100 | aux[adj_idx].ptr_type == PTR_TO_CTX) |
| 12101 | continue; |
| 12102 | |
| 12103 | imm_rnd = get_random_int(); |
| 12104 | rnd_hi32_patch[0] = insn; |
| 12105 | rnd_hi32_patch[1].imm = imm_rnd; |
Ilya Leoshkevich | 83a2881 | 2021-03-01 16:40:19 +0100 | [diff] [blame] | 12106 | rnd_hi32_patch[3].dst_reg = load_reg; |
Jiong Wang | d6c2308 | 2019-05-24 23:25:18 +0100 | [diff] [blame] | 12107 | patch = rnd_hi32_patch; |
| 12108 | patch_len = 4; |
| 12109 | goto apply_patch_buffer; |
| 12110 | } |
| 12111 | |
Brendan Jackman | 3949186 | 2021-03-04 18:56:46 -0800 | [diff] [blame] | 12112 | /* Add in an zero-extend instruction if a) the JIT has requested |
| 12113 | * it or b) it's a CMPXCHG. |
| 12114 | * |
| 12115 | * The latter is because: BPF_CMPXCHG always loads a value into |
| 12116 | * R0, therefore always zero-extends. However some archs' |
| 12117 | * equivalent instruction only does this load when the |
| 12118 | * comparison is successful. This detail of CMPXCHG is |
| 12119 | * orthogonal to the general zero-extension behaviour of the |
| 12120 | * CPU, so it's treated independently of bpf_jit_needs_zext. |
| 12121 | */ |
| 12122 | if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn)) |
Jiong Wang | a4b1d3c | 2019-05-24 23:25:15 +0100 | [diff] [blame] | 12123 | continue; |
| 12124 | |
Ilya Leoshkevich | 83a2881 | 2021-03-01 16:40:19 +0100 | [diff] [blame] | 12125 | if (WARN_ON(load_reg == -1)) { |
| 12126 | verbose(env, "verifier bug. zext_dst is set, but no reg is defined\n"); |
| 12127 | return -EFAULT; |
Ilya Leoshkevich | b2e37a7 | 2021-02-10 21:45:02 +0100 | [diff] [blame] | 12128 | } |
| 12129 | |
Jiong Wang | a4b1d3c | 2019-05-24 23:25:15 +0100 | [diff] [blame] | 12130 | zext_patch[0] = insn; |
Ilya Leoshkevich | b2e37a7 | 2021-02-10 21:45:02 +0100 | [diff] [blame] | 12131 | zext_patch[1].dst_reg = load_reg; |
| 12132 | zext_patch[1].src_reg = load_reg; |
Jiong Wang | d6c2308 | 2019-05-24 23:25:18 +0100 | [diff] [blame] | 12133 | patch = zext_patch; |
| 12134 | patch_len = 2; |
| 12135 | apply_patch_buffer: |
| 12136 | new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); |
Jiong Wang | a4b1d3c | 2019-05-24 23:25:15 +0100 | [diff] [blame] | 12137 | if (!new_prog) |
| 12138 | return -ENOMEM; |
| 12139 | env->prog = new_prog; |
| 12140 | insns = new_prog->insnsi; |
| 12141 | aux = env->insn_aux_data; |
Jiong Wang | d6c2308 | 2019-05-24 23:25:18 +0100 | [diff] [blame] | 12142 | delta += patch_len - 1; |
Jiong Wang | a4b1d3c | 2019-05-24 23:25:15 +0100 | [diff] [blame] | 12143 | } |
| 12144 | |
| 12145 | return 0; |
| 12146 | } |
| 12147 | |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 12148 | /* convert load instructions that access fields of a context type into a |
| 12149 | * sequence of instructions that access fields of the underlying structure: |
| 12150 | * struct __sk_buff -> struct sk_buff |
| 12151 | * struct bpf_sock_ops -> struct sock |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 12152 | */ |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 12153 | static int convert_ctx_accesses(struct bpf_verifier_env *env) |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 12154 | { |
Jakub Kicinski | 00176a3 | 2017-10-16 16:40:54 -0700 | [diff] [blame] | 12155 | const struct bpf_verifier_ops *ops = env->ops; |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 12156 | int i, cnt, size, ctx_field_size, delta = 0; |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 12157 | const int insn_cnt = env->prog->len; |
Daniel Borkmann | 36bbef5 | 2016-09-20 00:26:13 +0200 | [diff] [blame] | 12158 | struct bpf_insn insn_buf[16], *insn; |
Andrey Ignatov | 46f53a6 | 2018-11-10 22:15:13 -0800 | [diff] [blame] | 12159 | u32 target_size, size_default, off; |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 12160 | struct bpf_prog *new_prog; |
Alexei Starovoitov | d691f9e | 2015-06-04 10:11:54 -0700 | [diff] [blame] | 12161 | enum bpf_access_type type; |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 12162 | bool is_narrower_load; |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 12163 | |
Daniel Borkmann | b09928b | 2018-10-24 22:05:49 +0200 | [diff] [blame] | 12164 | if (ops->gen_prologue || env->seen_direct_write) { |
| 12165 | if (!ops->gen_prologue) { |
| 12166 | verbose(env, "bpf verifier is misconfigured\n"); |
| 12167 | return -EINVAL; |
| 12168 | } |
Daniel Borkmann | 36bbef5 | 2016-09-20 00:26:13 +0200 | [diff] [blame] | 12169 | cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, |
| 12170 | env->prog); |
| 12171 | if (cnt >= ARRAY_SIZE(insn_buf)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 12172 | verbose(env, "bpf verifier is misconfigured\n"); |
Daniel Borkmann | 36bbef5 | 2016-09-20 00:26:13 +0200 | [diff] [blame] | 12173 | return -EINVAL; |
| 12174 | } else if (cnt) { |
Alexei Starovoitov | 8041902 | 2017-03-15 18:26:41 -0700 | [diff] [blame] | 12175 | new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); |
Daniel Borkmann | 36bbef5 | 2016-09-20 00:26:13 +0200 | [diff] [blame] | 12176 | if (!new_prog) |
| 12177 | return -ENOMEM; |
Alexei Starovoitov | 8041902 | 2017-03-15 18:26:41 -0700 | [diff] [blame] | 12178 | |
Daniel Borkmann | 36bbef5 | 2016-09-20 00:26:13 +0200 | [diff] [blame] | 12179 | env->prog = new_prog; |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 12180 | delta += cnt - 1; |
Daniel Borkmann | 36bbef5 | 2016-09-20 00:26:13 +0200 | [diff] [blame] | 12181 | } |
| 12182 | } |
| 12183 | |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 12184 | if (bpf_prog_is_dev_bound(env->prog->aux)) |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 12185 | return 0; |
| 12186 | |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 12187 | insn = env->prog->insnsi + delta; |
Daniel Borkmann | 36bbef5 | 2016-09-20 00:26:13 +0200 | [diff] [blame] | 12188 | |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 12189 | for (i = 0; i < insn_cnt; i++, insn++) { |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 12190 | bpf_convert_ctx_access_t convert_ctx_access; |
Daniel Borkmann | 2039f26 | 2021-07-13 08:18:31 +0000 | [diff] [blame] | 12191 | bool ctx_access; |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 12192 | |
Daniel Borkmann | 62c7989 | 2017-01-12 11:51:33 +0100 | [diff] [blame] | 12193 | if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) || |
| 12194 | insn->code == (BPF_LDX | BPF_MEM | BPF_H) || |
| 12195 | insn->code == (BPF_LDX | BPF_MEM | BPF_W) || |
Daniel Borkmann | 2039f26 | 2021-07-13 08:18:31 +0000 | [diff] [blame] | 12196 | insn->code == (BPF_LDX | BPF_MEM | BPF_DW)) { |
Alexei Starovoitov | d691f9e | 2015-06-04 10:11:54 -0700 | [diff] [blame] | 12197 | type = BPF_READ; |
Daniel Borkmann | 2039f26 | 2021-07-13 08:18:31 +0000 | [diff] [blame] | 12198 | ctx_access = true; |
| 12199 | } else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) || |
| 12200 | insn->code == (BPF_STX | BPF_MEM | BPF_H) || |
| 12201 | insn->code == (BPF_STX | BPF_MEM | BPF_W) || |
| 12202 | insn->code == (BPF_STX | BPF_MEM | BPF_DW) || |
| 12203 | insn->code == (BPF_ST | BPF_MEM | BPF_B) || |
| 12204 | insn->code == (BPF_ST | BPF_MEM | BPF_H) || |
| 12205 | insn->code == (BPF_ST | BPF_MEM | BPF_W) || |
| 12206 | insn->code == (BPF_ST | BPF_MEM | BPF_DW)) { |
Alexei Starovoitov | d691f9e | 2015-06-04 10:11:54 -0700 | [diff] [blame] | 12207 | type = BPF_WRITE; |
Daniel Borkmann | 2039f26 | 2021-07-13 08:18:31 +0000 | [diff] [blame] | 12208 | ctx_access = BPF_CLASS(insn->code) == BPF_STX; |
| 12209 | } else { |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 12210 | continue; |
Daniel Borkmann | 2039f26 | 2021-07-13 08:18:31 +0000 | [diff] [blame] | 12211 | } |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 12212 | |
Alexei Starovoitov | af86ca4 | 2018-05-15 09:27:05 -0700 | [diff] [blame] | 12213 | if (type == BPF_WRITE && |
Daniel Borkmann | 2039f26 | 2021-07-13 08:18:31 +0000 | [diff] [blame] | 12214 | env->insn_aux_data[i + delta].sanitize_stack_spill) { |
Alexei Starovoitov | af86ca4 | 2018-05-15 09:27:05 -0700 | [diff] [blame] | 12215 | struct bpf_insn patch[] = { |
Alexei Starovoitov | af86ca4 | 2018-05-15 09:27:05 -0700 | [diff] [blame] | 12216 | *insn, |
Daniel Borkmann | 2039f26 | 2021-07-13 08:18:31 +0000 | [diff] [blame] | 12217 | BPF_ST_NOSPEC(), |
Alexei Starovoitov | af86ca4 | 2018-05-15 09:27:05 -0700 | [diff] [blame] | 12218 | }; |
| 12219 | |
| 12220 | cnt = ARRAY_SIZE(patch); |
| 12221 | new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt); |
| 12222 | if (!new_prog) |
| 12223 | return -ENOMEM; |
| 12224 | |
| 12225 | delta += cnt - 1; |
| 12226 | env->prog = new_prog; |
| 12227 | insn = new_prog->insnsi + i + delta; |
| 12228 | continue; |
| 12229 | } |
| 12230 | |
Daniel Borkmann | 2039f26 | 2021-07-13 08:18:31 +0000 | [diff] [blame] | 12231 | if (!ctx_access) |
| 12232 | continue; |
| 12233 | |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 12234 | switch (env->insn_aux_data[i + delta].ptr_type) { |
| 12235 | case PTR_TO_CTX: |
| 12236 | if (!ops->convert_ctx_access) |
| 12237 | continue; |
| 12238 | convert_ctx_access = ops->convert_ctx_access; |
| 12239 | break; |
| 12240 | case PTR_TO_SOCKET: |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 12241 | case PTR_TO_SOCK_COMMON: |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 12242 | convert_ctx_access = bpf_sock_convert_ctx_access; |
| 12243 | break; |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 12244 | case PTR_TO_TCP_SOCK: |
| 12245 | convert_ctx_access = bpf_tcp_sock_convert_ctx_access; |
| 12246 | break; |
Jonathan Lemon | fada7fd | 2019-06-06 13:59:40 -0700 | [diff] [blame] | 12247 | case PTR_TO_XDP_SOCK: |
| 12248 | convert_ctx_access = bpf_xdp_sock_convert_ctx_access; |
| 12249 | break; |
Alexei Starovoitov | 2a02759 | 2019-10-15 20:25:02 -0700 | [diff] [blame] | 12250 | case PTR_TO_BTF_ID: |
Martin KaFai Lau | 27ae7997 | 2020-01-08 16:35:03 -0800 | [diff] [blame] | 12251 | if (type == BPF_READ) { |
| 12252 | insn->code = BPF_LDX | BPF_PROBE_MEM | |
| 12253 | BPF_SIZE((insn)->code); |
| 12254 | env->prog->aux->num_exentries++; |
Udip Pant | 7e40781 | 2020-08-25 16:20:00 -0700 | [diff] [blame] | 12255 | } else if (resolve_prog_type(env->prog) != BPF_PROG_TYPE_STRUCT_OPS) { |
Alexei Starovoitov | 2a02759 | 2019-10-15 20:25:02 -0700 | [diff] [blame] | 12256 | verbose(env, "Writes through BTF pointers are not allowed\n"); |
| 12257 | return -EINVAL; |
| 12258 | } |
Alexei Starovoitov | 2a02759 | 2019-10-15 20:25:02 -0700 | [diff] [blame] | 12259 | continue; |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 12260 | default: |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 12261 | continue; |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 12262 | } |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 12263 | |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 12264 | ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 12265 | size = BPF_LDST_BYTES(insn); |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 12266 | |
| 12267 | /* If the read access is a narrower load of the field, |
| 12268 | * convert to a 4/8-byte load, to minimum program type specific |
| 12269 | * convert_ctx_access changes. If conversion is successful, |
| 12270 | * we will apply proper mask to the result. |
| 12271 | */ |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 12272 | is_narrower_load = size < ctx_field_size; |
Andrey Ignatov | 46f53a6 | 2018-11-10 22:15:13 -0800 | [diff] [blame] | 12273 | size_default = bpf_ctx_off_adjust_machine(ctx_field_size); |
| 12274 | off = insn->off; |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 12275 | if (is_narrower_load) { |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 12276 | u8 size_code; |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 12277 | |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 12278 | if (type == BPF_WRITE) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 12279 | verbose(env, "bpf verifier narrow ctx access misconfigured\n"); |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 12280 | return -EINVAL; |
| 12281 | } |
| 12282 | |
| 12283 | size_code = BPF_H; |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 12284 | if (ctx_field_size == 4) |
| 12285 | size_code = BPF_W; |
| 12286 | else if (ctx_field_size == 8) |
| 12287 | size_code = BPF_DW; |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 12288 | |
Daniel Borkmann | bc23105 | 2018-06-02 23:06:39 +0200 | [diff] [blame] | 12289 | insn->off = off & ~(size_default - 1); |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 12290 | insn->code = BPF_LDX | BPF_MEM | size_code; |
| 12291 | } |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 12292 | |
| 12293 | target_size = 0; |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 12294 | cnt = convert_ctx_access(type, insn, insn_buf, env->prog, |
| 12295 | &target_size); |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 12296 | if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf) || |
| 12297 | (ctx_field_size && !target_size)) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 12298 | verbose(env, "bpf verifier is misconfigured\n"); |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 12299 | return -EINVAL; |
| 12300 | } |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 12301 | |
| 12302 | if (is_narrower_load && size < target_size) { |
Ilya Leoshkevich | d895a0f | 2019-08-16 12:53:00 +0200 | [diff] [blame] | 12303 | u8 shift = bpf_ctx_narrow_access_offset( |
| 12304 | off, size, size_default) * 8; |
Andrey Ignatov | d7af7e4 | 2021-08-20 09:39:35 -0700 | [diff] [blame] | 12305 | if (shift && cnt + 1 >= ARRAY_SIZE(insn_buf)) { |
| 12306 | verbose(env, "bpf verifier narrow ctx load misconfigured\n"); |
| 12307 | return -EINVAL; |
| 12308 | } |
Andrey Ignatov | 46f53a6 | 2018-11-10 22:15:13 -0800 | [diff] [blame] | 12309 | if (ctx_field_size <= 4) { |
| 12310 | if (shift) |
| 12311 | insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH, |
| 12312 | insn->dst_reg, |
| 12313 | shift); |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 12314 | insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 12315 | (1 << size * 8) - 1); |
Andrey Ignatov | 46f53a6 | 2018-11-10 22:15:13 -0800 | [diff] [blame] | 12316 | } else { |
| 12317 | if (shift) |
| 12318 | insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH, |
| 12319 | insn->dst_reg, |
| 12320 | shift); |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 12321 | insn_buf[cnt++] = BPF_ALU64_IMM(BPF_AND, insn->dst_reg, |
Krzesimir Nowak | e2f7fc0 | 2019-05-08 18:08:58 +0200 | [diff] [blame] | 12322 | (1ULL << size * 8) - 1); |
Andrey Ignatov | 46f53a6 | 2018-11-10 22:15:13 -0800 | [diff] [blame] | 12323 | } |
Yonghong Song | 31fd858 | 2017-06-13 15:52:13 -0700 | [diff] [blame] | 12324 | } |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 12325 | |
Alexei Starovoitov | 8041902 | 2017-03-15 18:26:41 -0700 | [diff] [blame] | 12326 | new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 12327 | if (!new_prog) |
| 12328 | return -ENOMEM; |
| 12329 | |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 12330 | delta += cnt - 1; |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 12331 | |
| 12332 | /* keep walking new program and skip insns we just inserted */ |
| 12333 | env->prog = new_prog; |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 12334 | insn = new_prog->insnsi + i + delta; |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 12335 | } |
| 12336 | |
| 12337 | return 0; |
| 12338 | } |
| 12339 | |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12340 | static int jit_subprogs(struct bpf_verifier_env *env) |
| 12341 | { |
| 12342 | struct bpf_prog *prog = env->prog, **func, *tmp; |
| 12343 | int i, j, subprog_start, subprog_end = 0, len, subprog; |
Maciej Fijalkowski | a748c69 | 2020-09-16 23:10:05 +0200 | [diff] [blame] | 12344 | struct bpf_map *map_ptr; |
Daniel Borkmann | 7105e82 | 2017-12-20 13:42:57 +0100 | [diff] [blame] | 12345 | struct bpf_insn *insn; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12346 | void *old_bpf_func; |
Yonghong Song | c4c0bdc | 2020-06-23 17:10:54 -0700 | [diff] [blame] | 12347 | int err, num_exentries; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12348 | |
Jiong Wang | f910cef | 2018-05-02 16:17:17 -0400 | [diff] [blame] | 12349 | if (env->subprog_cnt <= 1) |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12350 | return 0; |
| 12351 | |
Daniel Borkmann | 7105e82 | 2017-12-20 13:42:57 +0100 | [diff] [blame] | 12352 | for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 12353 | if (bpf_pseudo_func(insn)) { |
| 12354 | env->insn_aux_data[i].call_imm = insn->imm; |
| 12355 | /* subprog is encoded in insn[1].imm */ |
| 12356 | continue; |
| 12357 | } |
| 12358 | |
Yonghong Song | 23a2d70 | 2021-02-04 15:48:27 -0800 | [diff] [blame] | 12359 | if (!bpf_pseudo_call(insn)) |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12360 | continue; |
Daniel Borkmann | c7a8978 | 2018-07-12 21:44:28 +0200 | [diff] [blame] | 12361 | /* Upon error here we cannot fall back to interpreter but |
| 12362 | * need a hard reject of the program. Thus -EFAULT is |
| 12363 | * propagated in any case. |
| 12364 | */ |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12365 | subprog = find_subprog(env, i + insn->imm + 1); |
| 12366 | if (subprog < 0) { |
| 12367 | WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", |
| 12368 | i + insn->imm + 1); |
| 12369 | return -EFAULT; |
| 12370 | } |
| 12371 | /* temporarily remember subprog id inside insn instead of |
| 12372 | * aux_data, since next loop will split up all insns into funcs |
| 12373 | */ |
Jiong Wang | f910cef | 2018-05-02 16:17:17 -0400 | [diff] [blame] | 12374 | insn->off = subprog; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12375 | /* remember original imm in case JIT fails and fallback |
| 12376 | * to interpreter will be needed |
| 12377 | */ |
| 12378 | env->insn_aux_data[i].call_imm = insn->imm; |
| 12379 | /* point imm to __bpf_call_base+1 from JITs point of view */ |
| 12380 | insn->imm = 1; |
| 12381 | } |
| 12382 | |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 12383 | err = bpf_prog_alloc_jited_linfo(prog); |
| 12384 | if (err) |
| 12385 | goto out_undo_insn; |
| 12386 | |
| 12387 | err = -ENOMEM; |
Kees Cook | 6396bb2 | 2018-06-12 14:03:40 -0700 | [diff] [blame] | 12388 | func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL); |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12389 | if (!func) |
Daniel Borkmann | c7a8978 | 2018-07-12 21:44:28 +0200 | [diff] [blame] | 12390 | goto out_undo_insn; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12391 | |
Jiong Wang | f910cef | 2018-05-02 16:17:17 -0400 | [diff] [blame] | 12392 | for (i = 0; i < env->subprog_cnt; i++) { |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12393 | subprog_start = subprog_end; |
Jiong Wang | 4cb3d99 | 2018-05-02 16:17:19 -0400 | [diff] [blame] | 12394 | subprog_end = env->subprog_info[i + 1].start; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12395 | |
| 12396 | len = subprog_end - subprog_start; |
Andrii Nakryiko | fb7dd8b | 2021-08-15 00:05:54 -0700 | [diff] [blame] | 12397 | /* bpf_prog_run() doesn't call subprogs directly, |
Alexei Starovoitov | 492ecee | 2019-02-25 14:28:39 -0800 | [diff] [blame] | 12398 | * hence main prog stats include the runtime of subprogs. |
| 12399 | * subprogs don't have IDs and not reachable via prog_get_next_id |
Alexei Starovoitov | 700d479 | 2021-02-09 19:36:26 -0800 | [diff] [blame] | 12400 | * func[i]->stats will never be accessed and stays NULL |
Alexei Starovoitov | 492ecee | 2019-02-25 14:28:39 -0800 | [diff] [blame] | 12401 | */ |
| 12402 | func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER); |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12403 | if (!func[i]) |
| 12404 | goto out_free; |
| 12405 | memcpy(func[i]->insnsi, &prog->insnsi[subprog_start], |
| 12406 | len * sizeof(struct bpf_insn)); |
Daniel Borkmann | 4f74d80 | 2017-12-20 13:42:56 +0100 | [diff] [blame] | 12407 | func[i]->type = prog->type; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12408 | func[i]->len = len; |
Daniel Borkmann | 4f74d80 | 2017-12-20 13:42:56 +0100 | [diff] [blame] | 12409 | if (bpf_prog_calc_tag(func[i])) |
| 12410 | goto out_free; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12411 | func[i]->is_func = 1; |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 12412 | func[i]->aux->func_idx = i; |
John Fastabend | f263a81 | 2021-07-07 15:38:47 -0700 | [diff] [blame] | 12413 | /* Below members will be freed only at prog->aux */ |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 12414 | func[i]->aux->btf = prog->aux->btf; |
| 12415 | func[i]->aux->func_info = prog->aux->func_info; |
John Fastabend | f263a81 | 2021-07-07 15:38:47 -0700 | [diff] [blame] | 12416 | func[i]->aux->poke_tab = prog->aux->poke_tab; |
| 12417 | func[i]->aux->size_poke_tab = prog->aux->size_poke_tab; |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 12418 | |
Maciej Fijalkowski | a748c69 | 2020-09-16 23:10:05 +0200 | [diff] [blame] | 12419 | for (j = 0; j < prog->aux->size_poke_tab; j++) { |
John Fastabend | f263a81 | 2021-07-07 15:38:47 -0700 | [diff] [blame] | 12420 | struct bpf_jit_poke_descriptor *poke; |
Maciej Fijalkowski | a748c69 | 2020-09-16 23:10:05 +0200 | [diff] [blame] | 12421 | |
John Fastabend | f263a81 | 2021-07-07 15:38:47 -0700 | [diff] [blame] | 12422 | poke = &prog->aux->poke_tab[j]; |
| 12423 | if (poke->insn_idx < subprog_end && |
| 12424 | poke->insn_idx >= subprog_start) |
| 12425 | poke->aux = func[i]->aux; |
Maciej Fijalkowski | a748c69 | 2020-09-16 23:10:05 +0200 | [diff] [blame] | 12426 | } |
| 12427 | |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12428 | /* Use bpf_prog_F_tag to indicate functions in stack traces. |
| 12429 | * Long term would need debug info to populate names |
| 12430 | */ |
| 12431 | func[i]->aux->name[0] = 'F'; |
Jiong Wang | 9c8105b | 2018-05-02 16:17:18 -0400 | [diff] [blame] | 12432 | func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12433 | func[i]->jit_requested = 1; |
Martin KaFai Lau | e6ac245 | 2021-03-24 18:51:42 -0700 | [diff] [blame] | 12434 | func[i]->aux->kfunc_tab = prog->aux->kfunc_tab; |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 12435 | func[i]->aux->linfo = prog->aux->linfo; |
| 12436 | func[i]->aux->nr_linfo = prog->aux->nr_linfo; |
| 12437 | func[i]->aux->jited_linfo = prog->aux->jited_linfo; |
| 12438 | func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; |
Yonghong Song | c4c0bdc | 2020-06-23 17:10:54 -0700 | [diff] [blame] | 12439 | num_exentries = 0; |
| 12440 | insn = func[i]->insnsi; |
| 12441 | for (j = 0; j < func[i]->len; j++, insn++) { |
| 12442 | if (BPF_CLASS(insn->code) == BPF_LDX && |
| 12443 | BPF_MODE(insn->code) == BPF_PROBE_MEM) |
| 12444 | num_exentries++; |
| 12445 | } |
| 12446 | func[i]->aux->num_exentries = num_exentries; |
Maciej Fijalkowski | ebf7d1f | 2020-09-16 23:10:08 +0200 | [diff] [blame] | 12447 | func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12448 | func[i] = bpf_int_jit_compile(func[i]); |
| 12449 | if (!func[i]->jited) { |
| 12450 | err = -ENOTSUPP; |
| 12451 | goto out_free; |
| 12452 | } |
| 12453 | cond_resched(); |
| 12454 | } |
Maciej Fijalkowski | a748c69 | 2020-09-16 23:10:05 +0200 | [diff] [blame] | 12455 | |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12456 | /* at this point all bpf functions were successfully JITed |
| 12457 | * now populate all bpf_calls with correct addresses and |
| 12458 | * run last pass of JIT |
| 12459 | */ |
Jiong Wang | f910cef | 2018-05-02 16:17:17 -0400 | [diff] [blame] | 12460 | for (i = 0; i < env->subprog_cnt; i++) { |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12461 | insn = func[i]->insnsi; |
| 12462 | for (j = 0; j < func[i]->len; j++, insn++) { |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 12463 | if (bpf_pseudo_func(insn)) { |
| 12464 | subprog = insn[1].imm; |
| 12465 | insn[0].imm = (u32)(long)func[subprog]->bpf_func; |
| 12466 | insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32; |
| 12467 | continue; |
| 12468 | } |
Yonghong Song | 23a2d70 | 2021-02-04 15:48:27 -0800 | [diff] [blame] | 12469 | if (!bpf_pseudo_call(insn)) |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12470 | continue; |
| 12471 | subprog = insn->off; |
Prashant Bhole | 0d306c3 | 2019-04-16 18:13:01 +0900 | [diff] [blame] | 12472 | insn->imm = BPF_CAST_CALL(func[subprog]->bpf_func) - |
| 12473 | __bpf_call_base; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12474 | } |
Sandipan Das | 2162fed | 2018-05-24 12:26:45 +0530 | [diff] [blame] | 12475 | |
| 12476 | /* we use the aux data to keep a list of the start addresses |
| 12477 | * of the JITed images for each function in the program |
| 12478 | * |
| 12479 | * for some architectures, such as powerpc64, the imm field |
| 12480 | * might not be large enough to hold the offset of the start |
| 12481 | * address of the callee's JITed image from __bpf_call_base |
| 12482 | * |
| 12483 | * in such cases, we can lookup the start address of a callee |
| 12484 | * by using its subprog id, available from the off field of |
| 12485 | * the call instruction, as an index for this list |
| 12486 | */ |
| 12487 | func[i]->aux->func = func; |
| 12488 | func[i]->aux->func_cnt = env->subprog_cnt; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12489 | } |
Jiong Wang | f910cef | 2018-05-02 16:17:17 -0400 | [diff] [blame] | 12490 | for (i = 0; i < env->subprog_cnt; i++) { |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12491 | old_bpf_func = func[i]->bpf_func; |
| 12492 | tmp = bpf_int_jit_compile(func[i]); |
| 12493 | if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) { |
| 12494 | verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); |
Daniel Borkmann | c7a8978 | 2018-07-12 21:44:28 +0200 | [diff] [blame] | 12495 | err = -ENOTSUPP; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12496 | goto out_free; |
| 12497 | } |
| 12498 | cond_resched(); |
| 12499 | } |
| 12500 | |
| 12501 | /* finally lock prog and jit images for all functions and |
| 12502 | * populate kallsysm |
| 12503 | */ |
Jiong Wang | f910cef | 2018-05-02 16:17:17 -0400 | [diff] [blame] | 12504 | for (i = 0; i < env->subprog_cnt; i++) { |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12505 | bpf_prog_lock_ro(func[i]); |
| 12506 | bpf_prog_kallsyms_add(func[i]); |
| 12507 | } |
Daniel Borkmann | 7105e82 | 2017-12-20 13:42:57 +0100 | [diff] [blame] | 12508 | |
| 12509 | /* Last step: make now unused interpreter insns from main |
| 12510 | * prog consistent for later dump requests, so they can |
| 12511 | * later look the same as if they were interpreted only. |
| 12512 | */ |
| 12513 | for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 12514 | if (bpf_pseudo_func(insn)) { |
| 12515 | insn[0].imm = env->insn_aux_data[i].call_imm; |
| 12516 | insn[1].imm = find_subprog(env, i + insn[0].imm + 1); |
| 12517 | continue; |
| 12518 | } |
Yonghong Song | 23a2d70 | 2021-02-04 15:48:27 -0800 | [diff] [blame] | 12519 | if (!bpf_pseudo_call(insn)) |
Daniel Borkmann | 7105e82 | 2017-12-20 13:42:57 +0100 | [diff] [blame] | 12520 | continue; |
| 12521 | insn->off = env->insn_aux_data[i].call_imm; |
| 12522 | subprog = find_subprog(env, i + insn->off + 1); |
Sandipan Das | dbecd73 | 2018-05-24 12:26:48 +0530 | [diff] [blame] | 12523 | insn->imm = subprog; |
Daniel Borkmann | 7105e82 | 2017-12-20 13:42:57 +0100 | [diff] [blame] | 12524 | } |
| 12525 | |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12526 | prog->jited = 1; |
| 12527 | prog->bpf_func = func[0]->bpf_func; |
| 12528 | prog->aux->func = func; |
Jiong Wang | f910cef | 2018-05-02 16:17:17 -0400 | [diff] [blame] | 12529 | prog->aux->func_cnt = env->subprog_cnt; |
Martin KaFai Lau | e16301f | 2021-03-24 18:51:30 -0700 | [diff] [blame] | 12530 | bpf_prog_jit_attempt_done(prog); |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12531 | return 0; |
| 12532 | out_free: |
John Fastabend | f263a81 | 2021-07-07 15:38:47 -0700 | [diff] [blame] | 12533 | /* We failed JIT'ing, so at this point we need to unregister poke |
| 12534 | * descriptors from subprogs, so that kernel is not attempting to |
| 12535 | * patch it anymore as we're freeing the subprog JIT memory. |
| 12536 | */ |
| 12537 | for (i = 0; i < prog->aux->size_poke_tab; i++) { |
| 12538 | map_ptr = prog->aux->poke_tab[i].tail_call.map; |
| 12539 | map_ptr->ops->map_poke_untrack(map_ptr, prog->aux); |
| 12540 | } |
| 12541 | /* At this point we're guaranteed that poke descriptors are not |
| 12542 | * live anymore. We can just unlink its descriptor table as it's |
| 12543 | * released with the main prog. |
| 12544 | */ |
Maciej Fijalkowski | a748c69 | 2020-09-16 23:10:05 +0200 | [diff] [blame] | 12545 | for (i = 0; i < env->subprog_cnt; i++) { |
| 12546 | if (!func[i]) |
| 12547 | continue; |
John Fastabend | f263a81 | 2021-07-07 15:38:47 -0700 | [diff] [blame] | 12548 | func[i]->aux->poke_tab = NULL; |
Maciej Fijalkowski | a748c69 | 2020-09-16 23:10:05 +0200 | [diff] [blame] | 12549 | bpf_jit_free(func[i]); |
| 12550 | } |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12551 | kfree(func); |
Daniel Borkmann | c7a8978 | 2018-07-12 21:44:28 +0200 | [diff] [blame] | 12552 | out_undo_insn: |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12553 | /* cleanup main prog to be interpreted */ |
| 12554 | prog->jit_requested = 0; |
| 12555 | for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { |
Yonghong Song | 23a2d70 | 2021-02-04 15:48:27 -0800 | [diff] [blame] | 12556 | if (!bpf_pseudo_call(insn)) |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12557 | continue; |
| 12558 | insn->off = 0; |
| 12559 | insn->imm = env->insn_aux_data[i].call_imm; |
| 12560 | } |
Martin KaFai Lau | e16301f | 2021-03-24 18:51:30 -0700 | [diff] [blame] | 12561 | bpf_prog_jit_attempt_done(prog); |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12562 | return err; |
| 12563 | } |
| 12564 | |
Alexei Starovoitov | 1ea47e0 | 2017-12-14 17:55:13 -0800 | [diff] [blame] | 12565 | static int fixup_call_args(struct bpf_verifier_env *env) |
| 12566 | { |
David S. Miller | 19d28fb | 2018-01-11 21:27:54 -0500 | [diff] [blame] | 12567 | #ifndef CONFIG_BPF_JIT_ALWAYS_ON |
Alexei Starovoitov | 1ea47e0 | 2017-12-14 17:55:13 -0800 | [diff] [blame] | 12568 | struct bpf_prog *prog = env->prog; |
| 12569 | struct bpf_insn *insn = prog->insnsi; |
Martin KaFai Lau | e6ac245 | 2021-03-24 18:51:42 -0700 | [diff] [blame] | 12570 | bool has_kfunc_call = bpf_prog_has_kfunc_call(prog); |
Alexei Starovoitov | 1ea47e0 | 2017-12-14 17:55:13 -0800 | [diff] [blame] | 12571 | int i, depth; |
David S. Miller | 19d28fb | 2018-01-11 21:27:54 -0500 | [diff] [blame] | 12572 | #endif |
Quentin Monnet | e4052d0 | 2018-10-07 12:56:58 +0100 | [diff] [blame] | 12573 | int err = 0; |
Alexei Starovoitov | 1ea47e0 | 2017-12-14 17:55:13 -0800 | [diff] [blame] | 12574 | |
Quentin Monnet | e4052d0 | 2018-10-07 12:56:58 +0100 | [diff] [blame] | 12575 | if (env->prog->jit_requested && |
| 12576 | !bpf_prog_is_dev_bound(env->prog->aux)) { |
David S. Miller | 19d28fb | 2018-01-11 21:27:54 -0500 | [diff] [blame] | 12577 | err = jit_subprogs(env); |
| 12578 | if (err == 0) |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 12579 | return 0; |
Daniel Borkmann | c7a8978 | 2018-07-12 21:44:28 +0200 | [diff] [blame] | 12580 | if (err == -EFAULT) |
| 12581 | return err; |
David S. Miller | 19d28fb | 2018-01-11 21:27:54 -0500 | [diff] [blame] | 12582 | } |
| 12583 | #ifndef CONFIG_BPF_JIT_ALWAYS_ON |
Martin KaFai Lau | e6ac245 | 2021-03-24 18:51:42 -0700 | [diff] [blame] | 12584 | if (has_kfunc_call) { |
| 12585 | verbose(env, "calling kernel functions are not allowed in non-JITed programs\n"); |
| 12586 | return -EINVAL; |
| 12587 | } |
Maciej Fijalkowski | e411901 | 2020-09-16 23:10:09 +0200 | [diff] [blame] | 12588 | if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { |
| 12589 | /* When JIT fails the progs with bpf2bpf calls and tail_calls |
| 12590 | * have to be rejected, since interpreter doesn't support them yet. |
| 12591 | */ |
| 12592 | verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); |
| 12593 | return -EINVAL; |
| 12594 | } |
Alexei Starovoitov | 1ea47e0 | 2017-12-14 17:55:13 -0800 | [diff] [blame] | 12595 | for (i = 0; i < prog->len; i++, insn++) { |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 12596 | if (bpf_pseudo_func(insn)) { |
| 12597 | /* When JIT fails the progs with callback calls |
| 12598 | * have to be rejected, since interpreter doesn't support them yet. |
| 12599 | */ |
| 12600 | verbose(env, "callbacks are not allowed in non-JITed programs\n"); |
| 12601 | return -EINVAL; |
| 12602 | } |
| 12603 | |
Yonghong Song | 23a2d70 | 2021-02-04 15:48:27 -0800 | [diff] [blame] | 12604 | if (!bpf_pseudo_call(insn)) |
Alexei Starovoitov | 1ea47e0 | 2017-12-14 17:55:13 -0800 | [diff] [blame] | 12605 | continue; |
| 12606 | depth = get_callee_stack_depth(env, insn, i); |
| 12607 | if (depth < 0) |
| 12608 | return depth; |
| 12609 | bpf_patch_call_args(insn, depth); |
| 12610 | } |
David S. Miller | 19d28fb | 2018-01-11 21:27:54 -0500 | [diff] [blame] | 12611 | err = 0; |
| 12612 | #endif |
| 12613 | return err; |
Alexei Starovoitov | 1ea47e0 | 2017-12-14 17:55:13 -0800 | [diff] [blame] | 12614 | } |
| 12615 | |
Martin KaFai Lau | e6ac245 | 2021-03-24 18:51:42 -0700 | [diff] [blame] | 12616 | static int fixup_kfunc_call(struct bpf_verifier_env *env, |
| 12617 | struct bpf_insn *insn) |
| 12618 | { |
| 12619 | const struct bpf_kfunc_desc *desc; |
| 12620 | |
| 12621 | /* insn->imm has the btf func_id. Replace it with |
| 12622 | * an address (relative to __bpf_base_call). |
| 12623 | */ |
| 12624 | desc = find_kfunc_desc(env->prog, insn->imm); |
| 12625 | if (!desc) { |
| 12626 | verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n", |
| 12627 | insn->imm); |
| 12628 | return -EFAULT; |
| 12629 | } |
| 12630 | |
| 12631 | insn->imm = desc->imm; |
| 12632 | |
| 12633 | return 0; |
| 12634 | } |
| 12635 | |
Brendan Jackman | e6ac593 | 2021-02-17 10:45:09 +0000 | [diff] [blame] | 12636 | /* Do various post-verification rewrites in a single program pass. |
| 12637 | * These rewrites simplify JIT and interpreter implementations. |
Alexei Starovoitov | e245c5c6 | 2017-03-15 18:26:39 -0700 | [diff] [blame] | 12638 | */ |
Brendan Jackman | e6ac593 | 2021-02-17 10:45:09 +0000 | [diff] [blame] | 12639 | static int do_misc_fixups(struct bpf_verifier_env *env) |
Alexei Starovoitov | e245c5c6 | 2017-03-15 18:26:39 -0700 | [diff] [blame] | 12640 | { |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 12641 | struct bpf_prog *prog = env->prog; |
Daniel Borkmann | d2e4c1e | 2019-11-22 21:07:59 +0100 | [diff] [blame] | 12642 | bool expect_blinding = bpf_jit_blinding_enabled(prog); |
Jiri Olsa | 9b99edc | 2021-07-14 11:43:55 +0200 | [diff] [blame] | 12643 | enum bpf_prog_type prog_type = resolve_prog_type(prog); |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 12644 | struct bpf_insn *insn = prog->insnsi; |
Alexei Starovoitov | e245c5c6 | 2017-03-15 18:26:39 -0700 | [diff] [blame] | 12645 | const struct bpf_func_proto *fn; |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 12646 | const int insn_cnt = prog->len; |
Daniel Borkmann | 09772d9 | 2018-06-02 23:06:35 +0200 | [diff] [blame] | 12647 | const struct bpf_map_ops *ops; |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 12648 | struct bpf_insn_aux_data *aux; |
Alexei Starovoitov | 81ed18a | 2017-03-15 18:26:42 -0700 | [diff] [blame] | 12649 | struct bpf_insn insn_buf[16]; |
| 12650 | struct bpf_prog *new_prog; |
| 12651 | struct bpf_map *map_ptr; |
Daniel Borkmann | d2e4c1e | 2019-11-22 21:07:59 +0100 | [diff] [blame] | 12652 | int i, ret, cnt, delta = 0; |
Alexei Starovoitov | e245c5c6 | 2017-03-15 18:26:39 -0700 | [diff] [blame] | 12653 | |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 12654 | for (i = 0; i < insn_cnt; i++, insn++) { |
Brendan Jackman | e6ac593 | 2021-02-17 10:45:09 +0000 | [diff] [blame] | 12655 | /* Make divide-by-zero exceptions impossible. */ |
Daniel Borkmann | f6b1b3b | 2018-01-26 23:33:39 +0100 | [diff] [blame] | 12656 | if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) || |
| 12657 | insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || |
| 12658 | insn->code == (BPF_ALU | BPF_MOD | BPF_X) || |
Alexei Starovoitov | 68fda45 | 2018-01-12 18:59:52 -0800 | [diff] [blame] | 12659 | insn->code == (BPF_ALU | BPF_DIV | BPF_X)) { |
Daniel Borkmann | f6b1b3b | 2018-01-26 23:33:39 +0100 | [diff] [blame] | 12660 | bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; |
Daniel Borkmann | e88b2c6 | 2021-02-09 18:46:10 +0000 | [diff] [blame] | 12661 | bool isdiv = BPF_OP(insn->code) == BPF_DIV; |
| 12662 | struct bpf_insn *patchlet; |
| 12663 | struct bpf_insn chk_and_div[] = { |
Daniel Borkmann | 9b00f1b | 2021-02-10 14:14:42 +0100 | [diff] [blame] | 12664 | /* [R,W]x div 0 -> 0 */ |
Daniel Borkmann | e88b2c6 | 2021-02-09 18:46:10 +0000 | [diff] [blame] | 12665 | BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | |
| 12666 | BPF_JNE | BPF_K, insn->src_reg, |
| 12667 | 0, 2, 0), |
Daniel Borkmann | f6b1b3b | 2018-01-26 23:33:39 +0100 | [diff] [blame] | 12668 | BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg), |
| 12669 | BPF_JMP_IMM(BPF_JA, 0, 0, 1), |
| 12670 | *insn, |
| 12671 | }; |
Daniel Borkmann | e88b2c6 | 2021-02-09 18:46:10 +0000 | [diff] [blame] | 12672 | struct bpf_insn chk_and_mod[] = { |
Daniel Borkmann | 9b00f1b | 2021-02-10 14:14:42 +0100 | [diff] [blame] | 12673 | /* [R,W]x mod 0 -> [R,W]x */ |
Daniel Borkmann | e88b2c6 | 2021-02-09 18:46:10 +0000 | [diff] [blame] | 12674 | BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | |
| 12675 | BPF_JEQ | BPF_K, insn->src_reg, |
Daniel Borkmann | 9b00f1b | 2021-02-10 14:14:42 +0100 | [diff] [blame] | 12676 | 0, 1 + (is64 ? 0 : 1), 0), |
Daniel Borkmann | f6b1b3b | 2018-01-26 23:33:39 +0100 | [diff] [blame] | 12677 | *insn, |
Daniel Borkmann | 9b00f1b | 2021-02-10 14:14:42 +0100 | [diff] [blame] | 12678 | BPF_JMP_IMM(BPF_JA, 0, 0, 1), |
| 12679 | BPF_MOV32_REG(insn->dst_reg, insn->dst_reg), |
Daniel Borkmann | f6b1b3b | 2018-01-26 23:33:39 +0100 | [diff] [blame] | 12680 | }; |
Daniel Borkmann | f6b1b3b | 2018-01-26 23:33:39 +0100 | [diff] [blame] | 12681 | |
Daniel Borkmann | e88b2c6 | 2021-02-09 18:46:10 +0000 | [diff] [blame] | 12682 | patchlet = isdiv ? chk_and_div : chk_and_mod; |
| 12683 | cnt = isdiv ? ARRAY_SIZE(chk_and_div) : |
Daniel Borkmann | 9b00f1b | 2021-02-10 14:14:42 +0100 | [diff] [blame] | 12684 | ARRAY_SIZE(chk_and_mod) - (is64 ? 2 : 0); |
Daniel Borkmann | f6b1b3b | 2018-01-26 23:33:39 +0100 | [diff] [blame] | 12685 | |
| 12686 | new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt); |
Alexei Starovoitov | 68fda45 | 2018-01-12 18:59:52 -0800 | [diff] [blame] | 12687 | if (!new_prog) |
| 12688 | return -ENOMEM; |
| 12689 | |
| 12690 | delta += cnt - 1; |
| 12691 | env->prog = prog = new_prog; |
| 12692 | insn = new_prog->insnsi + i + delta; |
| 12693 | continue; |
| 12694 | } |
| 12695 | |
Brendan Jackman | e6ac593 | 2021-02-17 10:45:09 +0000 | [diff] [blame] | 12696 | /* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */ |
Daniel Borkmann | e0cea7c | 2018-05-04 01:08:14 +0200 | [diff] [blame] | 12697 | if (BPF_CLASS(insn->code) == BPF_LD && |
| 12698 | (BPF_MODE(insn->code) == BPF_ABS || |
| 12699 | BPF_MODE(insn->code) == BPF_IND)) { |
| 12700 | cnt = env->ops->gen_ld_abs(insn, insn_buf); |
| 12701 | if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) { |
| 12702 | verbose(env, "bpf verifier is misconfigured\n"); |
| 12703 | return -EINVAL; |
| 12704 | } |
| 12705 | |
| 12706 | new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); |
| 12707 | if (!new_prog) |
| 12708 | return -ENOMEM; |
| 12709 | |
| 12710 | delta += cnt - 1; |
| 12711 | env->prog = prog = new_prog; |
| 12712 | insn = new_prog->insnsi + i + delta; |
| 12713 | continue; |
| 12714 | } |
| 12715 | |
Brendan Jackman | e6ac593 | 2021-02-17 10:45:09 +0000 | [diff] [blame] | 12716 | /* Rewrite pointer arithmetic to mitigate speculation attacks. */ |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 12717 | if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) || |
| 12718 | insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) { |
| 12719 | const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X; |
| 12720 | const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X; |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 12721 | struct bpf_insn *patch = &insn_buf[0]; |
Daniel Borkmann | 801c605 | 2021-04-29 15:19:37 +0000 | [diff] [blame] | 12722 | bool issrc, isneg, isimm; |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 12723 | u32 off_reg; |
| 12724 | |
| 12725 | aux = &env->insn_aux_data[i + delta]; |
Daniel Borkmann | 3612af7 | 2019-03-01 22:05:29 +0100 | [diff] [blame] | 12726 | if (!aux->alu_state || |
| 12727 | aux->alu_state == BPF_ALU_NON_POINTER) |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 12728 | continue; |
| 12729 | |
| 12730 | isneg = aux->alu_state & BPF_ALU_NEG_VALUE; |
| 12731 | issrc = (aux->alu_state & BPF_ALU_SANITIZE) == |
| 12732 | BPF_ALU_SANITIZE_SRC; |
Daniel Borkmann | 801c605 | 2021-04-29 15:19:37 +0000 | [diff] [blame] | 12733 | isimm = aux->alu_state & BPF_ALU_IMMEDIATE; |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 12734 | |
| 12735 | off_reg = issrc ? insn->src_reg : insn->dst_reg; |
Daniel Borkmann | 801c605 | 2021-04-29 15:19:37 +0000 | [diff] [blame] | 12736 | if (isimm) { |
| 12737 | *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); |
| 12738 | } else { |
| 12739 | if (isneg) |
| 12740 | *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); |
| 12741 | *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); |
| 12742 | *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg); |
| 12743 | *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg); |
| 12744 | *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0); |
| 12745 | *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63); |
| 12746 | *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg); |
| 12747 | } |
Daniel Borkmann | b9b34dd | 2021-04-30 16:21:46 +0200 | [diff] [blame] | 12748 | if (!issrc) |
| 12749 | *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg); |
| 12750 | insn->src_reg = BPF_REG_AX; |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 12751 | if (isneg) |
| 12752 | insn->code = insn->code == code_add ? |
| 12753 | code_sub : code_add; |
| 12754 | *patch++ = *insn; |
Daniel Borkmann | 801c605 | 2021-04-29 15:19:37 +0000 | [diff] [blame] | 12755 | if (issrc && isneg && !isimm) |
Daniel Borkmann | 979d63d | 2019-01-03 00:58:34 +0100 | [diff] [blame] | 12756 | *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); |
| 12757 | cnt = patch - insn_buf; |
| 12758 | |
| 12759 | new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); |
| 12760 | if (!new_prog) |
| 12761 | return -ENOMEM; |
| 12762 | |
| 12763 | delta += cnt - 1; |
| 12764 | env->prog = prog = new_prog; |
| 12765 | insn = new_prog->insnsi + i + delta; |
| 12766 | continue; |
| 12767 | } |
| 12768 | |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 12769 | if (insn->code != (BPF_JMP | BPF_CALL)) |
| 12770 | continue; |
Alexei Starovoitov | cc8b0b9 | 2017-12-14 17:55:05 -0800 | [diff] [blame] | 12771 | if (insn->src_reg == BPF_PSEUDO_CALL) |
| 12772 | continue; |
Martin KaFai Lau | e6ac245 | 2021-03-24 18:51:42 -0700 | [diff] [blame] | 12773 | if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { |
| 12774 | ret = fixup_kfunc_call(env, insn); |
| 12775 | if (ret) |
| 12776 | return ret; |
| 12777 | continue; |
| 12778 | } |
Alexei Starovoitov | e245c5c6 | 2017-03-15 18:26:39 -0700 | [diff] [blame] | 12779 | |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 12780 | if (insn->imm == BPF_FUNC_get_route_realm) |
| 12781 | prog->dst_needed = 1; |
| 12782 | if (insn->imm == BPF_FUNC_get_prandom_u32) |
| 12783 | bpf_user_rnd_init_once(); |
Josef Bacik | 9802d86 | 2017-12-11 11:36:48 -0500 | [diff] [blame] | 12784 | if (insn->imm == BPF_FUNC_override_return) |
| 12785 | prog->kprobe_override = 1; |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 12786 | if (insn->imm == BPF_FUNC_tail_call) { |
David S. Miller | 7b9f6da | 2017-04-20 10:35:33 -0400 | [diff] [blame] | 12787 | /* If we tail call into other programs, we |
| 12788 | * cannot make any assumptions since they can |
| 12789 | * be replaced dynamically during runtime in |
| 12790 | * the program array. |
| 12791 | */ |
| 12792 | prog->cb_access = 1; |
Maciej Fijalkowski | e411901 | 2020-09-16 23:10:09 +0200 | [diff] [blame] | 12793 | if (!allow_tail_call_in_subprogs(env)) |
| 12794 | prog->aux->stack_depth = MAX_BPF_STACK; |
| 12795 | prog->aux->max_pkt_offset = MAX_PACKET_OFF; |
David S. Miller | 7b9f6da | 2017-04-20 10:35:33 -0400 | [diff] [blame] | 12796 | |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 12797 | /* mark bpf_tail_call as different opcode to avoid |
Zhen Lei | 8fb33b6 | 2021-05-25 10:56:59 +0800 | [diff] [blame] | 12798 | * conditional branch in the interpreter for every normal |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 12799 | * call and to prevent accidental JITing by JIT compiler |
| 12800 | * that doesn't support bpf_tail_call yet |
Alexei Starovoitov | e245c5c6 | 2017-03-15 18:26:39 -0700 | [diff] [blame] | 12801 | */ |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 12802 | insn->imm = 0; |
Alexei Starovoitov | 71189fa | 2017-05-30 13:31:27 -0700 | [diff] [blame] | 12803 | insn->code = BPF_JMP | BPF_TAIL_CALL; |
Alexei Starovoitov | b215739 | 2018-01-07 17:33:02 -0800 | [diff] [blame] | 12804 | |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 12805 | aux = &env->insn_aux_data[i + delta]; |
Alexei Starovoitov | 2c78ee8 | 2020-05-13 16:03:54 -0700 | [diff] [blame] | 12806 | if (env->bpf_capable && !expect_blinding && |
Daniel Borkmann | cc52d91 | 2019-12-19 22:19:50 +0100 | [diff] [blame] | 12807 | prog->jit_requested && |
Daniel Borkmann | d2e4c1e | 2019-11-22 21:07:59 +0100 | [diff] [blame] | 12808 | !bpf_map_key_poisoned(aux) && |
| 12809 | !bpf_map_ptr_poisoned(aux) && |
| 12810 | !bpf_map_ptr_unpriv(aux)) { |
| 12811 | struct bpf_jit_poke_descriptor desc = { |
| 12812 | .reason = BPF_POKE_REASON_TAIL_CALL, |
| 12813 | .tail_call.map = BPF_MAP_PTR(aux->map_ptr_state), |
| 12814 | .tail_call.key = bpf_map_key_immediate(aux), |
Maciej Fijalkowski | a748c69 | 2020-09-16 23:10:05 +0200 | [diff] [blame] | 12815 | .insn_idx = i + delta, |
Daniel Borkmann | d2e4c1e | 2019-11-22 21:07:59 +0100 | [diff] [blame] | 12816 | }; |
| 12817 | |
| 12818 | ret = bpf_jit_add_poke_descriptor(prog, &desc); |
| 12819 | if (ret < 0) { |
| 12820 | verbose(env, "adding tail call poke descriptor failed\n"); |
| 12821 | return ret; |
| 12822 | } |
| 12823 | |
| 12824 | insn->imm = ret + 1; |
| 12825 | continue; |
| 12826 | } |
| 12827 | |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 12828 | if (!bpf_map_ptr_unpriv(aux)) |
| 12829 | continue; |
| 12830 | |
Alexei Starovoitov | b215739 | 2018-01-07 17:33:02 -0800 | [diff] [blame] | 12831 | /* instead of changing every JIT dealing with tail_call |
| 12832 | * emit two extra insns: |
| 12833 | * if (index >= max_entries) goto out; |
| 12834 | * index &= array->index_mask; |
| 12835 | * to avoid out-of-bounds cpu speculation |
| 12836 | */ |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 12837 | if (bpf_map_ptr_poisoned(aux)) { |
Colin Ian King | 4095034 | 2018-01-10 09:20:54 +0000 | [diff] [blame] | 12838 | verbose(env, "tail_call abusing map_ptr\n"); |
Alexei Starovoitov | b215739 | 2018-01-07 17:33:02 -0800 | [diff] [blame] | 12839 | return -EINVAL; |
| 12840 | } |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 12841 | |
Daniel Borkmann | d2e4c1e | 2019-11-22 21:07:59 +0100 | [diff] [blame] | 12842 | map_ptr = BPF_MAP_PTR(aux->map_ptr_state); |
Alexei Starovoitov | b215739 | 2018-01-07 17:33:02 -0800 | [diff] [blame] | 12843 | insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3, |
| 12844 | map_ptr->max_entries, 2); |
| 12845 | insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3, |
| 12846 | container_of(map_ptr, |
| 12847 | struct bpf_array, |
| 12848 | map)->index_mask); |
| 12849 | insn_buf[2] = *insn; |
| 12850 | cnt = 3; |
| 12851 | new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); |
| 12852 | if (!new_prog) |
| 12853 | return -ENOMEM; |
| 12854 | |
| 12855 | delta += cnt - 1; |
| 12856 | env->prog = prog = new_prog; |
| 12857 | insn = new_prog->insnsi + i + delta; |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 12858 | continue; |
Alexei Starovoitov | e245c5c6 | 2017-03-15 18:26:39 -0700 | [diff] [blame] | 12859 | } |
Alexei Starovoitov | e245c5c6 | 2017-03-15 18:26:39 -0700 | [diff] [blame] | 12860 | |
Alexei Starovoitov | b00628b | 2021-07-14 17:54:09 -0700 | [diff] [blame] | 12861 | if (insn->imm == BPF_FUNC_timer_set_callback) { |
| 12862 | /* The verifier will process callback_fn as many times as necessary |
| 12863 | * with different maps and the register states prepared by |
| 12864 | * set_timer_callback_state will be accurate. |
| 12865 | * |
| 12866 | * The following use case is valid: |
| 12867 | * map1 is shared by prog1, prog2, prog3. |
| 12868 | * prog1 calls bpf_timer_init for some map1 elements |
| 12869 | * prog2 calls bpf_timer_set_callback for some map1 elements. |
| 12870 | * Those that were not bpf_timer_init-ed will return -EINVAL. |
| 12871 | * prog3 calls bpf_timer_start for some map1 elements. |
| 12872 | * Those that were not both bpf_timer_init-ed and |
| 12873 | * bpf_timer_set_callback-ed will return -EINVAL. |
| 12874 | */ |
| 12875 | struct bpf_insn ld_addrs[2] = { |
| 12876 | BPF_LD_IMM64(BPF_REG_3, (long)prog->aux), |
| 12877 | }; |
| 12878 | |
| 12879 | insn_buf[0] = ld_addrs[0]; |
| 12880 | insn_buf[1] = ld_addrs[1]; |
| 12881 | insn_buf[2] = *insn; |
| 12882 | cnt = 3; |
| 12883 | |
| 12884 | new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); |
| 12885 | if (!new_prog) |
| 12886 | return -ENOMEM; |
| 12887 | |
| 12888 | delta += cnt - 1; |
| 12889 | env->prog = prog = new_prog; |
| 12890 | insn = new_prog->insnsi + i + delta; |
| 12891 | goto patch_call_imm; |
| 12892 | } |
| 12893 | |
Daniel Borkmann | 89c6307 | 2017-08-19 03:12:45 +0200 | [diff] [blame] | 12894 | /* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup |
Daniel Borkmann | 09772d9 | 2018-06-02 23:06:35 +0200 | [diff] [blame] | 12895 | * and other inlining handlers are currently limited to 64 bit |
| 12896 | * only. |
Daniel Borkmann | 89c6307 | 2017-08-19 03:12:45 +0200 | [diff] [blame] | 12897 | */ |
Alexei Starovoitov | 60b58afc | 2017-12-14 17:55:14 -0800 | [diff] [blame] | 12898 | if (prog->jit_requested && BITS_PER_LONG == 64 && |
Daniel Borkmann | 09772d9 | 2018-06-02 23:06:35 +0200 | [diff] [blame] | 12899 | (insn->imm == BPF_FUNC_map_lookup_elem || |
| 12900 | insn->imm == BPF_FUNC_map_update_elem || |
Daniel Borkmann | 84430d4 | 2018-10-21 02:09:27 +0200 | [diff] [blame] | 12901 | insn->imm == BPF_FUNC_map_delete_elem || |
| 12902 | insn->imm == BPF_FUNC_map_push_elem || |
| 12903 | insn->imm == BPF_FUNC_map_pop_elem || |
Björn Töpel | e6a4750 | 2021-03-08 12:29:06 +0100 | [diff] [blame] | 12904 | insn->imm == BPF_FUNC_map_peek_elem || |
| 12905 | insn->imm == BPF_FUNC_redirect_map)) { |
Daniel Borkmann | c93552c | 2018-05-24 02:32:53 +0200 | [diff] [blame] | 12906 | aux = &env->insn_aux_data[i + delta]; |
| 12907 | if (bpf_map_ptr_poisoned(aux)) |
| 12908 | goto patch_call_imm; |
| 12909 | |
Daniel Borkmann | d2e4c1e | 2019-11-22 21:07:59 +0100 | [diff] [blame] | 12910 | map_ptr = BPF_MAP_PTR(aux->map_ptr_state); |
Daniel Borkmann | 09772d9 | 2018-06-02 23:06:35 +0200 | [diff] [blame] | 12911 | ops = map_ptr->ops; |
| 12912 | if (insn->imm == BPF_FUNC_map_lookup_elem && |
| 12913 | ops->map_gen_lookup) { |
| 12914 | cnt = ops->map_gen_lookup(map_ptr, insn_buf); |
Daniel Borkmann | 4a8f87e | 2020-10-11 01:40:03 +0200 | [diff] [blame] | 12915 | if (cnt == -EOPNOTSUPP) |
| 12916 | goto patch_map_ops_generic; |
| 12917 | if (cnt <= 0 || cnt >= ARRAY_SIZE(insn_buf)) { |
Daniel Borkmann | 09772d9 | 2018-06-02 23:06:35 +0200 | [diff] [blame] | 12918 | verbose(env, "bpf verifier is misconfigured\n"); |
| 12919 | return -EINVAL; |
| 12920 | } |
Alexei Starovoitov | 81ed18a | 2017-03-15 18:26:42 -0700 | [diff] [blame] | 12921 | |
Daniel Borkmann | 09772d9 | 2018-06-02 23:06:35 +0200 | [diff] [blame] | 12922 | new_prog = bpf_patch_insn_data(env, i + delta, |
| 12923 | insn_buf, cnt); |
| 12924 | if (!new_prog) |
| 12925 | return -ENOMEM; |
| 12926 | |
| 12927 | delta += cnt - 1; |
| 12928 | env->prog = prog = new_prog; |
| 12929 | insn = new_prog->insnsi + i + delta; |
| 12930 | continue; |
Alexei Starovoitov | 81ed18a | 2017-03-15 18:26:42 -0700 | [diff] [blame] | 12931 | } |
| 12932 | |
Daniel Borkmann | 09772d9 | 2018-06-02 23:06:35 +0200 | [diff] [blame] | 12933 | BUILD_BUG_ON(!__same_type(ops->map_lookup_elem, |
| 12934 | (void *(*)(struct bpf_map *map, void *key))NULL)); |
| 12935 | BUILD_BUG_ON(!__same_type(ops->map_delete_elem, |
| 12936 | (int (*)(struct bpf_map *map, void *key))NULL)); |
| 12937 | BUILD_BUG_ON(!__same_type(ops->map_update_elem, |
| 12938 | (int (*)(struct bpf_map *map, void *key, void *value, |
| 12939 | u64 flags))NULL)); |
Daniel Borkmann | 84430d4 | 2018-10-21 02:09:27 +0200 | [diff] [blame] | 12940 | BUILD_BUG_ON(!__same_type(ops->map_push_elem, |
| 12941 | (int (*)(struct bpf_map *map, void *value, |
| 12942 | u64 flags))NULL)); |
| 12943 | BUILD_BUG_ON(!__same_type(ops->map_pop_elem, |
| 12944 | (int (*)(struct bpf_map *map, void *value))NULL)); |
| 12945 | BUILD_BUG_ON(!__same_type(ops->map_peek_elem, |
| 12946 | (int (*)(struct bpf_map *map, void *value))NULL)); |
Björn Töpel | e6a4750 | 2021-03-08 12:29:06 +0100 | [diff] [blame] | 12947 | BUILD_BUG_ON(!__same_type(ops->map_redirect, |
| 12948 | (int (*)(struct bpf_map *map, u32 ifindex, u64 flags))NULL)); |
| 12949 | |
Daniel Borkmann | 4a8f87e | 2020-10-11 01:40:03 +0200 | [diff] [blame] | 12950 | patch_map_ops_generic: |
Daniel Borkmann | 09772d9 | 2018-06-02 23:06:35 +0200 | [diff] [blame] | 12951 | switch (insn->imm) { |
| 12952 | case BPF_FUNC_map_lookup_elem: |
| 12953 | insn->imm = BPF_CAST_CALL(ops->map_lookup_elem) - |
| 12954 | __bpf_call_base; |
| 12955 | continue; |
| 12956 | case BPF_FUNC_map_update_elem: |
| 12957 | insn->imm = BPF_CAST_CALL(ops->map_update_elem) - |
| 12958 | __bpf_call_base; |
| 12959 | continue; |
| 12960 | case BPF_FUNC_map_delete_elem: |
| 12961 | insn->imm = BPF_CAST_CALL(ops->map_delete_elem) - |
| 12962 | __bpf_call_base; |
| 12963 | continue; |
Daniel Borkmann | 84430d4 | 2018-10-21 02:09:27 +0200 | [diff] [blame] | 12964 | case BPF_FUNC_map_push_elem: |
| 12965 | insn->imm = BPF_CAST_CALL(ops->map_push_elem) - |
| 12966 | __bpf_call_base; |
| 12967 | continue; |
| 12968 | case BPF_FUNC_map_pop_elem: |
| 12969 | insn->imm = BPF_CAST_CALL(ops->map_pop_elem) - |
| 12970 | __bpf_call_base; |
| 12971 | continue; |
| 12972 | case BPF_FUNC_map_peek_elem: |
| 12973 | insn->imm = BPF_CAST_CALL(ops->map_peek_elem) - |
| 12974 | __bpf_call_base; |
| 12975 | continue; |
Björn Töpel | e6a4750 | 2021-03-08 12:29:06 +0100 | [diff] [blame] | 12976 | case BPF_FUNC_redirect_map: |
| 12977 | insn->imm = BPF_CAST_CALL(ops->map_redirect) - |
| 12978 | __bpf_call_base; |
| 12979 | continue; |
Daniel Borkmann | 09772d9 | 2018-06-02 23:06:35 +0200 | [diff] [blame] | 12980 | } |
Alexei Starovoitov | 81ed18a | 2017-03-15 18:26:42 -0700 | [diff] [blame] | 12981 | |
Daniel Borkmann | 09772d9 | 2018-06-02 23:06:35 +0200 | [diff] [blame] | 12982 | goto patch_call_imm; |
Alexei Starovoitov | 81ed18a | 2017-03-15 18:26:42 -0700 | [diff] [blame] | 12983 | } |
| 12984 | |
Brendan Jackman | e6ac593 | 2021-02-17 10:45:09 +0000 | [diff] [blame] | 12985 | /* Implement bpf_jiffies64 inline. */ |
Martin KaFai Lau | 5576b99 | 2020-01-22 15:36:46 -0800 | [diff] [blame] | 12986 | if (prog->jit_requested && BITS_PER_LONG == 64 && |
| 12987 | insn->imm == BPF_FUNC_jiffies64) { |
| 12988 | struct bpf_insn ld_jiffies_addr[2] = { |
| 12989 | BPF_LD_IMM64(BPF_REG_0, |
| 12990 | (unsigned long)&jiffies), |
| 12991 | }; |
| 12992 | |
| 12993 | insn_buf[0] = ld_jiffies_addr[0]; |
| 12994 | insn_buf[1] = ld_jiffies_addr[1]; |
| 12995 | insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, |
| 12996 | BPF_REG_0, 0); |
| 12997 | cnt = 3; |
| 12998 | |
| 12999 | new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, |
| 13000 | cnt); |
| 13001 | if (!new_prog) |
| 13002 | return -ENOMEM; |
| 13003 | |
| 13004 | delta += cnt - 1; |
| 13005 | env->prog = prog = new_prog; |
| 13006 | insn = new_prog->insnsi + i + delta; |
| 13007 | continue; |
| 13008 | } |
| 13009 | |
Jiri Olsa | 9b99edc | 2021-07-14 11:43:55 +0200 | [diff] [blame] | 13010 | /* Implement bpf_get_func_ip inline. */ |
| 13011 | if (prog_type == BPF_PROG_TYPE_TRACING && |
| 13012 | insn->imm == BPF_FUNC_get_func_ip) { |
| 13013 | /* Load IP address from ctx - 8 */ |
| 13014 | insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); |
| 13015 | |
| 13016 | new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); |
| 13017 | if (!new_prog) |
| 13018 | return -ENOMEM; |
| 13019 | |
| 13020 | env->prog = prog = new_prog; |
| 13021 | insn = new_prog->insnsi + i + delta; |
| 13022 | continue; |
| 13023 | } |
| 13024 | |
Alexei Starovoitov | 81ed18a | 2017-03-15 18:26:42 -0700 | [diff] [blame] | 13025 | patch_call_imm: |
Andrey Ignatov | 5e43f89 | 2018-03-30 15:08:00 -0700 | [diff] [blame] | 13026 | fn = env->ops->get_func_proto(insn->imm, env->prog); |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 13027 | /* all functions that have prototype and verifier allowed |
| 13028 | * programs to call them, must be real in-kernel functions |
| 13029 | */ |
| 13030 | if (!fn->func) { |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 13031 | verbose(env, |
| 13032 | "kernel subsystem misconfigured func %s#%d\n", |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 13033 | func_id_name(insn->imm), insn->imm); |
| 13034 | return -EFAULT; |
| 13035 | } |
| 13036 | insn->imm = fn->func - __bpf_call_base; |
| 13037 | } |
| 13038 | |
Daniel Borkmann | d2e4c1e | 2019-11-22 21:07:59 +0100 | [diff] [blame] | 13039 | /* Since poke tab is now finalized, publish aux to tracker. */ |
| 13040 | for (i = 0; i < prog->aux->size_poke_tab; i++) { |
| 13041 | map_ptr = prog->aux->poke_tab[i].tail_call.map; |
| 13042 | if (!map_ptr->ops->map_poke_track || |
| 13043 | !map_ptr->ops->map_poke_untrack || |
| 13044 | !map_ptr->ops->map_poke_run) { |
| 13045 | verbose(env, "bpf verifier is misconfigured\n"); |
| 13046 | return -EINVAL; |
| 13047 | } |
| 13048 | |
| 13049 | ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux); |
| 13050 | if (ret < 0) { |
| 13051 | verbose(env, "tracking tail call prog failed\n"); |
| 13052 | return ret; |
| 13053 | } |
| 13054 | } |
| 13055 | |
Martin KaFai Lau | e6ac245 | 2021-03-24 18:51:42 -0700 | [diff] [blame] | 13056 | sort_kfunc_descs_by_imm(env->prog); |
| 13057 | |
Alexei Starovoitov | 79741b3 | 2017-03-15 18:26:40 -0700 | [diff] [blame] | 13058 | return 0; |
| 13059 | } |
Alexei Starovoitov | e245c5c6 | 2017-03-15 18:26:39 -0700 | [diff] [blame] | 13060 | |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 13061 | static void free_states(struct bpf_verifier_env *env) |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 13062 | { |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 13063 | struct bpf_verifier_state_list *sl, *sln; |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 13064 | int i; |
| 13065 | |
Alexei Starovoitov | 9f4686c | 2019-04-01 21:27:41 -0700 | [diff] [blame] | 13066 | sl = env->free_list; |
| 13067 | while (sl) { |
| 13068 | sln = sl->next; |
| 13069 | free_verifier_state(&sl->state, false); |
| 13070 | kfree(sl); |
| 13071 | sl = sln; |
| 13072 | } |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 13073 | env->free_list = NULL; |
Alexei Starovoitov | 9f4686c | 2019-04-01 21:27:41 -0700 | [diff] [blame] | 13074 | |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 13075 | if (!env->explored_states) |
| 13076 | return; |
| 13077 | |
Alexei Starovoitov | dc2a4eb | 2019-05-21 20:17:07 -0700 | [diff] [blame] | 13078 | for (i = 0; i < state_htab_size(env); i++) { |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 13079 | sl = env->explored_states[i]; |
| 13080 | |
Alexei Starovoitov | a8f500a | 2019-05-21 20:17:06 -0700 | [diff] [blame] | 13081 | while (sl) { |
| 13082 | sln = sl->next; |
| 13083 | free_verifier_state(&sl->state, false); |
| 13084 | kfree(sl); |
| 13085 | sl = sln; |
| 13086 | } |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 13087 | env->explored_states[i] = NULL; |
| 13088 | } |
| 13089 | } |
| 13090 | |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 13091 | static int do_check_common(struct bpf_verifier_env *env, int subprog) |
| 13092 | { |
Andrii Nakryiko | 6f8a57c | 2020-04-23 12:58:50 -0700 | [diff] [blame] | 13093 | bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 13094 | struct bpf_verifier_state *state; |
| 13095 | struct bpf_reg_state *regs; |
| 13096 | int ret, i; |
| 13097 | |
| 13098 | env->prev_linfo = NULL; |
| 13099 | env->pass_cnt++; |
| 13100 | |
| 13101 | state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL); |
| 13102 | if (!state) |
| 13103 | return -ENOMEM; |
| 13104 | state->curframe = 0; |
| 13105 | state->speculative = false; |
| 13106 | state->branches = 1; |
| 13107 | state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL); |
| 13108 | if (!state->frame[0]) { |
| 13109 | kfree(state); |
| 13110 | return -ENOMEM; |
| 13111 | } |
| 13112 | env->cur_state = state; |
| 13113 | init_func_state(env, state->frame[0], |
| 13114 | BPF_MAIN_FUNC /* callsite */, |
| 13115 | 0 /* frameno */, |
| 13116 | subprog); |
| 13117 | |
| 13118 | regs = state->frame[state->curframe]->regs; |
Alexei Starovoitov | be8704f | 2020-01-20 16:53:46 -0800 | [diff] [blame] | 13119 | if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 13120 | ret = btf_prepare_func_args(env, subprog, regs); |
| 13121 | if (ret) |
| 13122 | goto out; |
| 13123 | for (i = BPF_REG_1; i <= BPF_REG_5; i++) { |
| 13124 | if (regs[i].type == PTR_TO_CTX) |
| 13125 | mark_reg_known_zero(env, regs, i); |
| 13126 | else if (regs[i].type == SCALAR_VALUE) |
| 13127 | mark_reg_unknown(env, regs, i); |
Dmitrii Banshchikov | e5069b9c | 2021-02-13 00:56:41 +0400 | [diff] [blame] | 13128 | else if (regs[i].type == PTR_TO_MEM_OR_NULL) { |
| 13129 | const u32 mem_size = regs[i].mem_size; |
| 13130 | |
| 13131 | mark_reg_known_zero(env, regs, i); |
| 13132 | regs[i].mem_size = mem_size; |
| 13133 | regs[i].id = ++env->id_gen; |
| 13134 | } |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 13135 | } |
| 13136 | } else { |
| 13137 | /* 1st arg to a function */ |
| 13138 | regs[BPF_REG_1].type = PTR_TO_CTX; |
| 13139 | mark_reg_known_zero(env, regs, BPF_REG_1); |
Martin KaFai Lau | 34747c4 | 2021-03-24 18:51:36 -0700 | [diff] [blame] | 13140 | ret = btf_check_subprog_arg_match(env, subprog, regs); |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 13141 | if (ret == -EFAULT) |
| 13142 | /* unlikely verifier bug. abort. |
| 13143 | * ret == 0 and ret < 0 are sadly acceptable for |
| 13144 | * main() function due to backward compatibility. |
| 13145 | * Like socket filter program may be written as: |
| 13146 | * int bpf_prog(struct pt_regs *ctx) |
| 13147 | * and never dereference that ctx in the program. |
| 13148 | * 'struct pt_regs' is a type mismatch for socket |
| 13149 | * filter that should be using 'struct __sk_buff'. |
| 13150 | */ |
| 13151 | goto out; |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 13152 | } |
| 13153 | |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 13154 | ret = do_check(env); |
| 13155 | out: |
Alexei Starovoitov | f59bbfc | 2020-01-21 18:41:38 -0800 | [diff] [blame] | 13156 | /* check for NULL is necessary, since cur_state can be freed inside |
| 13157 | * do_check() under memory pressure. |
| 13158 | */ |
| 13159 | if (env->cur_state) { |
| 13160 | free_verifier_state(env->cur_state, true); |
| 13161 | env->cur_state = NULL; |
| 13162 | } |
Andrii Nakryiko | 6f8a57c | 2020-04-23 12:58:50 -0700 | [diff] [blame] | 13163 | while (!pop_stack(env, NULL, NULL, false)); |
| 13164 | if (!ret && pop_log) |
| 13165 | bpf_vlog_reset(&env->log, 0); |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 13166 | free_states(env); |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 13167 | return ret; |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 13168 | } |
| 13169 | |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 13170 | /* Verify all global functions in a BPF program one by one based on their BTF. |
| 13171 | * All global functions must pass verification. Otherwise the whole program is rejected. |
| 13172 | * Consider: |
| 13173 | * int bar(int); |
| 13174 | * int foo(int f) |
| 13175 | * { |
| 13176 | * return bar(f); |
| 13177 | * } |
| 13178 | * int bar(int b) |
| 13179 | * { |
| 13180 | * ... |
| 13181 | * } |
| 13182 | * foo() will be verified first for R1=any_scalar_value. During verification it |
| 13183 | * will be assumed that bar() already verified successfully and call to bar() |
| 13184 | * from foo() will be checked for type match only. Later bar() will be verified |
| 13185 | * independently to check that it's safe for R1=any_scalar_value. |
| 13186 | */ |
| 13187 | static int do_check_subprogs(struct bpf_verifier_env *env) |
| 13188 | { |
| 13189 | struct bpf_prog_aux *aux = env->prog->aux; |
| 13190 | int i, ret; |
| 13191 | |
| 13192 | if (!aux->func_info) |
| 13193 | return 0; |
| 13194 | |
| 13195 | for (i = 1; i < env->subprog_cnt; i++) { |
| 13196 | if (aux->func_info_aux[i].linkage != BTF_FUNC_GLOBAL) |
| 13197 | continue; |
| 13198 | env->insn_idx = env->subprog_info[i].start; |
| 13199 | WARN_ON_ONCE(env->insn_idx == 0); |
| 13200 | ret = do_check_common(env, i); |
| 13201 | if (ret) { |
| 13202 | return ret; |
| 13203 | } else if (env->log.level & BPF_LOG_LEVEL) { |
| 13204 | verbose(env, |
| 13205 | "Func#%d is safe for any args that match its prototype\n", |
| 13206 | i); |
| 13207 | } |
| 13208 | } |
| 13209 | return 0; |
| 13210 | } |
| 13211 | |
| 13212 | static int do_check_main(struct bpf_verifier_env *env) |
| 13213 | { |
| 13214 | int ret; |
| 13215 | |
| 13216 | env->insn_idx = 0; |
| 13217 | ret = do_check_common(env, 0); |
| 13218 | if (!ret) |
| 13219 | env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; |
| 13220 | return ret; |
| 13221 | } |
| 13222 | |
| 13223 | |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 13224 | static void print_verification_stats(struct bpf_verifier_env *env) |
| 13225 | { |
| 13226 | int i; |
| 13227 | |
| 13228 | if (env->log.level & BPF_LOG_STATS) { |
| 13229 | verbose(env, "verification time %lld usec\n", |
| 13230 | div_u64(env->verification_time, 1000)); |
| 13231 | verbose(env, "stack depth "); |
| 13232 | for (i = 0; i < env->subprog_cnt; i++) { |
| 13233 | u32 depth = env->subprog_info[i].stack_depth; |
| 13234 | |
| 13235 | verbose(env, "%d", depth); |
| 13236 | if (i + 1 < env->subprog_cnt) |
| 13237 | verbose(env, "+"); |
| 13238 | } |
| 13239 | verbose(env, "\n"); |
| 13240 | } |
| 13241 | verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " |
| 13242 | "total_states %d peak_states %d mark_read %d\n", |
| 13243 | env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS, |
| 13244 | env->max_states_per_insn, env->total_states, |
| 13245 | env->peak_states, env->longest_mark_read_walk); |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 13246 | } |
| 13247 | |
Martin KaFai Lau | 27ae7997 | 2020-01-08 16:35:03 -0800 | [diff] [blame] | 13248 | static int check_struct_ops_btf_id(struct bpf_verifier_env *env) |
| 13249 | { |
| 13250 | const struct btf_type *t, *func_proto; |
| 13251 | const struct bpf_struct_ops *st_ops; |
| 13252 | const struct btf_member *member; |
| 13253 | struct bpf_prog *prog = env->prog; |
| 13254 | u32 btf_id, member_idx; |
| 13255 | const char *mname; |
| 13256 | |
Toke Høiland-Jørgensen | 12aa8a9 | 2021-03-26 11:03:13 +0100 | [diff] [blame] | 13257 | if (!prog->gpl_compatible) { |
| 13258 | verbose(env, "struct ops programs must have a GPL compatible license\n"); |
| 13259 | return -EINVAL; |
| 13260 | } |
| 13261 | |
Martin KaFai Lau | 27ae7997 | 2020-01-08 16:35:03 -0800 | [diff] [blame] | 13262 | btf_id = prog->aux->attach_btf_id; |
| 13263 | st_ops = bpf_struct_ops_find(btf_id); |
| 13264 | if (!st_ops) { |
| 13265 | verbose(env, "attach_btf_id %u is not a supported struct\n", |
| 13266 | btf_id); |
| 13267 | return -ENOTSUPP; |
| 13268 | } |
| 13269 | |
| 13270 | t = st_ops->type; |
| 13271 | member_idx = prog->expected_attach_type; |
| 13272 | if (member_idx >= btf_type_vlen(t)) { |
| 13273 | verbose(env, "attach to invalid member idx %u of struct %s\n", |
| 13274 | member_idx, st_ops->name); |
| 13275 | return -EINVAL; |
| 13276 | } |
| 13277 | |
| 13278 | member = &btf_type_member(t)[member_idx]; |
| 13279 | mname = btf_name_by_offset(btf_vmlinux, member->name_off); |
| 13280 | func_proto = btf_type_resolve_func_ptr(btf_vmlinux, member->type, |
| 13281 | NULL); |
| 13282 | if (!func_proto) { |
| 13283 | verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n", |
| 13284 | mname, member_idx, st_ops->name); |
| 13285 | return -EINVAL; |
| 13286 | } |
| 13287 | |
| 13288 | if (st_ops->check_member) { |
| 13289 | int err = st_ops->check_member(t, member); |
| 13290 | |
| 13291 | if (err) { |
| 13292 | verbose(env, "attach to unsupported member %s of struct %s\n", |
| 13293 | mname, st_ops->name); |
| 13294 | return err; |
| 13295 | } |
| 13296 | } |
| 13297 | |
| 13298 | prog->aux->attach_func_proto = func_proto; |
| 13299 | prog->aux->attach_func_name = mname; |
| 13300 | env->ops = st_ops->verifier_ops; |
| 13301 | |
| 13302 | return 0; |
| 13303 | } |
KP Singh | 6ba43b7 | 2020-03-04 20:18:50 +0100 | [diff] [blame] | 13304 | #define SECURITY_PREFIX "security_" |
| 13305 | |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 13306 | static int check_attach_modify_return(unsigned long addr, const char *func_name) |
KP Singh | 6ba43b7 | 2020-03-04 20:18:50 +0100 | [diff] [blame] | 13307 | { |
KP Singh | 6919175 | 2020-03-05 21:49:55 +0100 | [diff] [blame] | 13308 | if (within_error_injection_list(addr) || |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 13309 | !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1)) |
KP Singh | 6ba43b7 | 2020-03-04 20:18:50 +0100 | [diff] [blame] | 13310 | return 0; |
KP Singh | 6ba43b7 | 2020-03-04 20:18:50 +0100 | [diff] [blame] | 13311 | |
KP Singh | 6ba43b7 | 2020-03-04 20:18:50 +0100 | [diff] [blame] | 13312 | return -EINVAL; |
| 13313 | } |
Martin KaFai Lau | 27ae7997 | 2020-01-08 16:35:03 -0800 | [diff] [blame] | 13314 | |
Alexei Starovoitov | 1e6c62a | 2020-08-27 15:01:11 -0700 | [diff] [blame] | 13315 | /* list of non-sleepable functions that are otherwise on |
| 13316 | * ALLOW_ERROR_INJECTION list |
| 13317 | */ |
| 13318 | BTF_SET_START(btf_non_sleepable_error_inject) |
| 13319 | /* Three functions below can be called from sleepable and non-sleepable context. |
| 13320 | * Assume non-sleepable from bpf safety point of view. |
| 13321 | */ |
| 13322 | BTF_ID(func, __add_to_page_cache_locked) |
| 13323 | BTF_ID(func, should_fail_alloc_page) |
| 13324 | BTF_ID(func, should_failslab) |
| 13325 | BTF_SET_END(btf_non_sleepable_error_inject) |
| 13326 | |
| 13327 | static int check_non_sleepable_error_inject(u32 btf_id) |
| 13328 | { |
| 13329 | return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id); |
| 13330 | } |
| 13331 | |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 13332 | int bpf_check_attach_target(struct bpf_verifier_log *log, |
| 13333 | const struct bpf_prog *prog, |
| 13334 | const struct bpf_prog *tgt_prog, |
| 13335 | u32 btf_id, |
| 13336 | struct bpf_attach_target_info *tgt_info) |
Martin KaFai Lau | 3820729 | 2019-10-24 17:18:11 -0700 | [diff] [blame] | 13337 | { |
Alexei Starovoitov | be8704f | 2020-01-20 16:53:46 -0800 | [diff] [blame] | 13338 | bool prog_extension = prog->type == BPF_PROG_TYPE_EXT; |
Alexei Starovoitov | f1b9509 | 2019-10-30 15:32:11 -0700 | [diff] [blame] | 13339 | const char prefix[] = "btf_trace_"; |
Alexei Starovoitov | 5b92a28 | 2019-11-14 10:57:17 -0800 | [diff] [blame] | 13340 | int ret = 0, subprog = -1, i; |
Martin KaFai Lau | 3820729 | 2019-10-24 17:18:11 -0700 | [diff] [blame] | 13341 | const struct btf_type *t; |
Alexei Starovoitov | 5b92a28 | 2019-11-14 10:57:17 -0800 | [diff] [blame] | 13342 | bool conservative = true; |
Martin KaFai Lau | 3820729 | 2019-10-24 17:18:11 -0700 | [diff] [blame] | 13343 | const char *tname; |
Alexei Starovoitov | 5b92a28 | 2019-11-14 10:57:17 -0800 | [diff] [blame] | 13344 | struct btf *btf; |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 13345 | long addr = 0; |
Martin KaFai Lau | 3820729 | 2019-10-24 17:18:11 -0700 | [diff] [blame] | 13346 | |
Alexei Starovoitov | f1b9509 | 2019-10-30 15:32:11 -0700 | [diff] [blame] | 13347 | if (!btf_id) { |
Toke Høiland-Jørgensen | efc6815 | 2020-09-25 23:25:01 +0200 | [diff] [blame] | 13348 | bpf_log(log, "Tracing programs must provide btf_id\n"); |
Alexei Starovoitov | f1b9509 | 2019-10-30 15:32:11 -0700 | [diff] [blame] | 13349 | return -EINVAL; |
| 13350 | } |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 13351 | btf = tgt_prog ? tgt_prog->aux->btf : prog->aux->attach_btf; |
Alexei Starovoitov | 5b92a28 | 2019-11-14 10:57:17 -0800 | [diff] [blame] | 13352 | if (!btf) { |
Toke Høiland-Jørgensen | efc6815 | 2020-09-25 23:25:01 +0200 | [diff] [blame] | 13353 | bpf_log(log, |
Alexei Starovoitov | 5b92a28 | 2019-11-14 10:57:17 -0800 | [diff] [blame] | 13354 | "FENTRY/FEXIT program can only be attached to another program annotated with BTF\n"); |
| 13355 | return -EINVAL; |
| 13356 | } |
| 13357 | t = btf_type_by_id(btf, btf_id); |
Alexei Starovoitov | f1b9509 | 2019-10-30 15:32:11 -0700 | [diff] [blame] | 13358 | if (!t) { |
Toke Høiland-Jørgensen | efc6815 | 2020-09-25 23:25:01 +0200 | [diff] [blame] | 13359 | bpf_log(log, "attach_btf_id %u is invalid\n", btf_id); |
Alexei Starovoitov | f1b9509 | 2019-10-30 15:32:11 -0700 | [diff] [blame] | 13360 | return -EINVAL; |
| 13361 | } |
Alexei Starovoitov | 5b92a28 | 2019-11-14 10:57:17 -0800 | [diff] [blame] | 13362 | tname = btf_name_by_offset(btf, t->name_off); |
Alexei Starovoitov | f1b9509 | 2019-10-30 15:32:11 -0700 | [diff] [blame] | 13363 | if (!tname) { |
Toke Høiland-Jørgensen | efc6815 | 2020-09-25 23:25:01 +0200 | [diff] [blame] | 13364 | bpf_log(log, "attach_btf_id %u doesn't have a name\n", btf_id); |
Alexei Starovoitov | f1b9509 | 2019-10-30 15:32:11 -0700 | [diff] [blame] | 13365 | return -EINVAL; |
| 13366 | } |
Alexei Starovoitov | 5b92a28 | 2019-11-14 10:57:17 -0800 | [diff] [blame] | 13367 | if (tgt_prog) { |
| 13368 | struct bpf_prog_aux *aux = tgt_prog->aux; |
| 13369 | |
| 13370 | for (i = 0; i < aux->func_info_cnt; i++) |
| 13371 | if (aux->func_info[i].type_id == btf_id) { |
| 13372 | subprog = i; |
| 13373 | break; |
| 13374 | } |
| 13375 | if (subprog == -1) { |
Toke Høiland-Jørgensen | efc6815 | 2020-09-25 23:25:01 +0200 | [diff] [blame] | 13376 | bpf_log(log, "Subprog %s doesn't exist\n", tname); |
Alexei Starovoitov | 5b92a28 | 2019-11-14 10:57:17 -0800 | [diff] [blame] | 13377 | return -EINVAL; |
| 13378 | } |
| 13379 | conservative = aux->func_info_aux[subprog].unreliable; |
Alexei Starovoitov | be8704f | 2020-01-20 16:53:46 -0800 | [diff] [blame] | 13380 | if (prog_extension) { |
| 13381 | if (conservative) { |
Toke Høiland-Jørgensen | efc6815 | 2020-09-25 23:25:01 +0200 | [diff] [blame] | 13382 | bpf_log(log, |
Alexei Starovoitov | be8704f | 2020-01-20 16:53:46 -0800 | [diff] [blame] | 13383 | "Cannot replace static functions\n"); |
| 13384 | return -EINVAL; |
| 13385 | } |
| 13386 | if (!prog->jit_requested) { |
Toke Høiland-Jørgensen | efc6815 | 2020-09-25 23:25:01 +0200 | [diff] [blame] | 13387 | bpf_log(log, |
Alexei Starovoitov | be8704f | 2020-01-20 16:53:46 -0800 | [diff] [blame] | 13388 | "Extension programs should be JITed\n"); |
| 13389 | return -EINVAL; |
| 13390 | } |
Alexei Starovoitov | be8704f | 2020-01-20 16:53:46 -0800 | [diff] [blame] | 13391 | } |
| 13392 | if (!tgt_prog->jited) { |
Toke Høiland-Jørgensen | efc6815 | 2020-09-25 23:25:01 +0200 | [diff] [blame] | 13393 | bpf_log(log, "Can attach to only JITed progs\n"); |
Alexei Starovoitov | be8704f | 2020-01-20 16:53:46 -0800 | [diff] [blame] | 13394 | return -EINVAL; |
| 13395 | } |
| 13396 | if (tgt_prog->type == prog->type) { |
| 13397 | /* Cannot fentry/fexit another fentry/fexit program. |
| 13398 | * Cannot attach program extension to another extension. |
| 13399 | * It's ok to attach fentry/fexit to extension program. |
| 13400 | */ |
Toke Høiland-Jørgensen | efc6815 | 2020-09-25 23:25:01 +0200 | [diff] [blame] | 13401 | bpf_log(log, "Cannot recursively attach\n"); |
Alexei Starovoitov | be8704f | 2020-01-20 16:53:46 -0800 | [diff] [blame] | 13402 | return -EINVAL; |
| 13403 | } |
| 13404 | if (tgt_prog->type == BPF_PROG_TYPE_TRACING && |
| 13405 | prog_extension && |
| 13406 | (tgt_prog->expected_attach_type == BPF_TRACE_FENTRY || |
| 13407 | tgt_prog->expected_attach_type == BPF_TRACE_FEXIT)) { |
| 13408 | /* Program extensions can extend all program types |
| 13409 | * except fentry/fexit. The reason is the following. |
| 13410 | * The fentry/fexit programs are used for performance |
| 13411 | * analysis, stats and can be attached to any program |
| 13412 | * type except themselves. When extension program is |
| 13413 | * replacing XDP function it is necessary to allow |
| 13414 | * performance analysis of all functions. Both original |
| 13415 | * XDP program and its program extension. Hence |
| 13416 | * attaching fentry/fexit to BPF_PROG_TYPE_EXT is |
| 13417 | * allowed. If extending of fentry/fexit was allowed it |
| 13418 | * would be possible to create long call chain |
| 13419 | * fentry->extension->fentry->extension beyond |
| 13420 | * reasonable stack size. Hence extending fentry is not |
| 13421 | * allowed. |
| 13422 | */ |
Toke Høiland-Jørgensen | efc6815 | 2020-09-25 23:25:01 +0200 | [diff] [blame] | 13423 | bpf_log(log, "Cannot extend fentry/fexit\n"); |
Alexei Starovoitov | be8704f | 2020-01-20 16:53:46 -0800 | [diff] [blame] | 13424 | return -EINVAL; |
| 13425 | } |
Alexei Starovoitov | 5b92a28 | 2019-11-14 10:57:17 -0800 | [diff] [blame] | 13426 | } else { |
Alexei Starovoitov | be8704f | 2020-01-20 16:53:46 -0800 | [diff] [blame] | 13427 | if (prog_extension) { |
Toke Høiland-Jørgensen | efc6815 | 2020-09-25 23:25:01 +0200 | [diff] [blame] | 13428 | bpf_log(log, "Cannot replace kernel functions\n"); |
Alexei Starovoitov | be8704f | 2020-01-20 16:53:46 -0800 | [diff] [blame] | 13429 | return -EINVAL; |
| 13430 | } |
Alexei Starovoitov | 5b92a28 | 2019-11-14 10:57:17 -0800 | [diff] [blame] | 13431 | } |
Alexei Starovoitov | f1b9509 | 2019-10-30 15:32:11 -0700 | [diff] [blame] | 13432 | |
| 13433 | switch (prog->expected_attach_type) { |
| 13434 | case BPF_TRACE_RAW_TP: |
Alexei Starovoitov | 5b92a28 | 2019-11-14 10:57:17 -0800 | [diff] [blame] | 13435 | if (tgt_prog) { |
Toke Høiland-Jørgensen | efc6815 | 2020-09-25 23:25:01 +0200 | [diff] [blame] | 13436 | bpf_log(log, |
Alexei Starovoitov | 5b92a28 | 2019-11-14 10:57:17 -0800 | [diff] [blame] | 13437 | "Only FENTRY/FEXIT progs are attachable to another BPF prog\n"); |
| 13438 | return -EINVAL; |
| 13439 | } |
Martin KaFai Lau | 3820729 | 2019-10-24 17:18:11 -0700 | [diff] [blame] | 13440 | if (!btf_type_is_typedef(t)) { |
Toke Høiland-Jørgensen | efc6815 | 2020-09-25 23:25:01 +0200 | [diff] [blame] | 13441 | bpf_log(log, "attach_btf_id %u is not a typedef\n", |
Martin KaFai Lau | 3820729 | 2019-10-24 17:18:11 -0700 | [diff] [blame] | 13442 | btf_id); |
| 13443 | return -EINVAL; |
| 13444 | } |
Alexei Starovoitov | f1b9509 | 2019-10-30 15:32:11 -0700 | [diff] [blame] | 13445 | if (strncmp(prefix, tname, sizeof(prefix) - 1)) { |
Toke Høiland-Jørgensen | efc6815 | 2020-09-25 23:25:01 +0200 | [diff] [blame] | 13446 | bpf_log(log, "attach_btf_id %u points to wrong type name %s\n", |
Martin KaFai Lau | 3820729 | 2019-10-24 17:18:11 -0700 | [diff] [blame] | 13447 | btf_id, tname); |
| 13448 | return -EINVAL; |
| 13449 | } |
| 13450 | tname += sizeof(prefix) - 1; |
Alexei Starovoitov | 5b92a28 | 2019-11-14 10:57:17 -0800 | [diff] [blame] | 13451 | t = btf_type_by_id(btf, t->type); |
Martin KaFai Lau | 3820729 | 2019-10-24 17:18:11 -0700 | [diff] [blame] | 13452 | if (!btf_type_is_ptr(t)) |
| 13453 | /* should never happen in valid vmlinux build */ |
| 13454 | return -EINVAL; |
Alexei Starovoitov | 5b92a28 | 2019-11-14 10:57:17 -0800 | [diff] [blame] | 13455 | t = btf_type_by_id(btf, t->type); |
Martin KaFai Lau | 3820729 | 2019-10-24 17:18:11 -0700 | [diff] [blame] | 13456 | if (!btf_type_is_func_proto(t)) |
| 13457 | /* should never happen in valid vmlinux build */ |
| 13458 | return -EINVAL; |
| 13459 | |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 13460 | break; |
Yonghong Song | 15d83c4 | 2020-05-09 10:59:00 -0700 | [diff] [blame] | 13461 | case BPF_TRACE_ITER: |
| 13462 | if (!btf_type_is_func(t)) { |
Toke Høiland-Jørgensen | efc6815 | 2020-09-25 23:25:01 +0200 | [diff] [blame] | 13463 | bpf_log(log, "attach_btf_id %u is not a function\n", |
Yonghong Song | 15d83c4 | 2020-05-09 10:59:00 -0700 | [diff] [blame] | 13464 | btf_id); |
| 13465 | return -EINVAL; |
| 13466 | } |
| 13467 | t = btf_type_by_id(btf, t->type); |
| 13468 | if (!btf_type_is_func_proto(t)) |
| 13469 | return -EINVAL; |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 13470 | ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); |
| 13471 | if (ret) |
| 13472 | return ret; |
| 13473 | break; |
Alexei Starovoitov | be8704f | 2020-01-20 16:53:46 -0800 | [diff] [blame] | 13474 | default: |
| 13475 | if (!prog_extension) |
| 13476 | return -EINVAL; |
Gustavo A. R. Silva | df561f66 | 2020-08-23 17:36:59 -0500 | [diff] [blame] | 13477 | fallthrough; |
KP Singh | ae24082 | 2020-03-04 20:18:49 +0100 | [diff] [blame] | 13478 | case BPF_MODIFY_RETURN: |
KP Singh | 9e4e01d | 2020-03-29 01:43:52 +0100 | [diff] [blame] | 13479 | case BPF_LSM_MAC: |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 13480 | case BPF_TRACE_FENTRY: |
| 13481 | case BPF_TRACE_FEXIT: |
| 13482 | if (!btf_type_is_func(t)) { |
Toke Høiland-Jørgensen | efc6815 | 2020-09-25 23:25:01 +0200 | [diff] [blame] | 13483 | bpf_log(log, "attach_btf_id %u is not a function\n", |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 13484 | btf_id); |
| 13485 | return -EINVAL; |
| 13486 | } |
Alexei Starovoitov | be8704f | 2020-01-20 16:53:46 -0800 | [diff] [blame] | 13487 | if (prog_extension && |
Toke Høiland-Jørgensen | efc6815 | 2020-09-25 23:25:01 +0200 | [diff] [blame] | 13488 | btf_check_type_match(log, prog, btf, t)) |
Alexei Starovoitov | be8704f | 2020-01-20 16:53:46 -0800 | [diff] [blame] | 13489 | return -EINVAL; |
Alexei Starovoitov | 5b92a28 | 2019-11-14 10:57:17 -0800 | [diff] [blame] | 13490 | t = btf_type_by_id(btf, t->type); |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 13491 | if (!btf_type_is_func_proto(t)) |
| 13492 | return -EINVAL; |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 13493 | |
Toke Høiland-Jørgensen | 4a1e7c0 | 2020-09-29 14:45:51 +0200 | [diff] [blame] | 13494 | if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) && |
| 13495 | (!tgt_prog || prog->aux->saved_dst_prog_type != tgt_prog->type || |
| 13496 | prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type)) |
| 13497 | return -EINVAL; |
| 13498 | |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 13499 | if (tgt_prog && conservative) |
Alexei Starovoitov | 5b92a28 | 2019-11-14 10:57:17 -0800 | [diff] [blame] | 13500 | t = NULL; |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 13501 | |
| 13502 | ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 13503 | if (ret < 0) |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 13504 | return ret; |
| 13505 | |
Alexei Starovoitov | 5b92a28 | 2019-11-14 10:57:17 -0800 | [diff] [blame] | 13506 | if (tgt_prog) { |
Yonghong Song | e9eeec5 | 2019-12-04 17:06:06 -0800 | [diff] [blame] | 13507 | if (subprog == 0) |
| 13508 | addr = (long) tgt_prog->bpf_func; |
| 13509 | else |
| 13510 | addr = (long) tgt_prog->aux->func[subprog]->bpf_func; |
Alexei Starovoitov | 5b92a28 | 2019-11-14 10:57:17 -0800 | [diff] [blame] | 13511 | } else { |
| 13512 | addr = kallsyms_lookup_name(tname); |
| 13513 | if (!addr) { |
Toke Høiland-Jørgensen | efc6815 | 2020-09-25 23:25:01 +0200 | [diff] [blame] | 13514 | bpf_log(log, |
Alexei Starovoitov | 5b92a28 | 2019-11-14 10:57:17 -0800 | [diff] [blame] | 13515 | "The address of function %s cannot be found\n", |
| 13516 | tname); |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 13517 | return -ENOENT; |
Alexei Starovoitov | 5b92a28 | 2019-11-14 10:57:17 -0800 | [diff] [blame] | 13518 | } |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 13519 | } |
Alexei Starovoitov | 18644ce | 2020-05-28 21:38:36 -0700 | [diff] [blame] | 13520 | |
Alexei Starovoitov | 1e6c62a | 2020-08-27 15:01:11 -0700 | [diff] [blame] | 13521 | if (prog->aux->sleepable) { |
| 13522 | ret = -EINVAL; |
| 13523 | switch (prog->type) { |
| 13524 | case BPF_PROG_TYPE_TRACING: |
| 13525 | /* fentry/fexit/fmod_ret progs can be sleepable only if they are |
| 13526 | * attached to ALLOW_ERROR_INJECTION and are not in denylist. |
| 13527 | */ |
| 13528 | if (!check_non_sleepable_error_inject(btf_id) && |
| 13529 | within_error_injection_list(addr)) |
| 13530 | ret = 0; |
| 13531 | break; |
| 13532 | case BPF_PROG_TYPE_LSM: |
| 13533 | /* LSM progs check that they are attached to bpf_lsm_*() funcs. |
| 13534 | * Only some of them are sleepable. |
| 13535 | */ |
KP Singh | 423f161 | 2020-11-13 00:59:29 +0000 | [diff] [blame] | 13536 | if (bpf_lsm_is_sleepable_hook(btf_id)) |
Alexei Starovoitov | 1e6c62a | 2020-08-27 15:01:11 -0700 | [diff] [blame] | 13537 | ret = 0; |
| 13538 | break; |
| 13539 | default: |
| 13540 | break; |
| 13541 | } |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 13542 | if (ret) { |
| 13543 | bpf_log(log, "%s is not sleepable\n", tname); |
| 13544 | return ret; |
| 13545 | } |
Alexei Starovoitov | 1e6c62a | 2020-08-27 15:01:11 -0700 | [diff] [blame] | 13546 | } else if (prog->expected_attach_type == BPF_MODIFY_RETURN) { |
Toke Høiland-Jørgensen | 1af9270 | 2020-09-25 23:25:00 +0200 | [diff] [blame] | 13547 | if (tgt_prog) { |
Toke Høiland-Jørgensen | efc6815 | 2020-09-25 23:25:01 +0200 | [diff] [blame] | 13548 | bpf_log(log, "can't modify return codes of BPF programs\n"); |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 13549 | return -EINVAL; |
Toke Høiland-Jørgensen | 1af9270 | 2020-09-25 23:25:00 +0200 | [diff] [blame] | 13550 | } |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 13551 | ret = check_attach_modify_return(addr, tname); |
| 13552 | if (ret) { |
| 13553 | bpf_log(log, "%s() is not modifiable\n", tname); |
| 13554 | return ret; |
| 13555 | } |
Alexei Starovoitov | 18644ce | 2020-05-28 21:38:36 -0700 | [diff] [blame] | 13556 | } |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 13557 | |
| 13558 | break; |
Martin KaFai Lau | 3820729 | 2019-10-24 17:18:11 -0700 | [diff] [blame] | 13559 | } |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 13560 | tgt_info->tgt_addr = addr; |
| 13561 | tgt_info->tgt_name = tname; |
| 13562 | tgt_info->tgt_type = t; |
| 13563 | return 0; |
| 13564 | } |
| 13565 | |
Jiri Olsa | 35e3815 | 2021-04-29 13:47:12 +0200 | [diff] [blame] | 13566 | BTF_SET_START(btf_id_deny) |
| 13567 | BTF_ID_UNUSED |
| 13568 | #ifdef CONFIG_SMP |
| 13569 | BTF_ID(func, migrate_disable) |
| 13570 | BTF_ID(func, migrate_enable) |
| 13571 | #endif |
| 13572 | #if !defined CONFIG_PREEMPT_RCU && !defined CONFIG_TINY_RCU |
| 13573 | BTF_ID(func, rcu_read_unlock_strict) |
| 13574 | #endif |
| 13575 | BTF_SET_END(btf_id_deny) |
| 13576 | |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 13577 | static int check_attach_btf_id(struct bpf_verifier_env *env) |
| 13578 | { |
| 13579 | struct bpf_prog *prog = env->prog; |
Toke Høiland-Jørgensen | 3aac1ea | 2020-09-29 14:45:50 +0200 | [diff] [blame] | 13580 | struct bpf_prog *tgt_prog = prog->aux->dst_prog; |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 13581 | struct bpf_attach_target_info tgt_info = {}; |
| 13582 | u32 btf_id = prog->aux->attach_btf_id; |
| 13583 | struct bpf_trampoline *tr; |
| 13584 | int ret; |
| 13585 | u64 key; |
| 13586 | |
Alexei Starovoitov | 79a7f8b | 2021-05-13 17:36:03 -0700 | [diff] [blame] | 13587 | if (prog->type == BPF_PROG_TYPE_SYSCALL) { |
| 13588 | if (prog->aux->sleepable) |
| 13589 | /* attach_btf_id checked to be zero already */ |
| 13590 | return 0; |
| 13591 | verbose(env, "Syscall programs can only be sleepable\n"); |
| 13592 | return -EINVAL; |
| 13593 | } |
| 13594 | |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 13595 | if (prog->aux->sleepable && prog->type != BPF_PROG_TYPE_TRACING && |
| 13596 | prog->type != BPF_PROG_TYPE_LSM) { |
| 13597 | verbose(env, "Only fentry/fexit/fmod_ret and lsm programs can be sleepable\n"); |
| 13598 | return -EINVAL; |
| 13599 | } |
| 13600 | |
| 13601 | if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) |
| 13602 | return check_struct_ops_btf_id(env); |
| 13603 | |
| 13604 | if (prog->type != BPF_PROG_TYPE_TRACING && |
| 13605 | prog->type != BPF_PROG_TYPE_LSM && |
| 13606 | prog->type != BPF_PROG_TYPE_EXT) |
| 13607 | return 0; |
| 13608 | |
| 13609 | ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info); |
| 13610 | if (ret) |
| 13611 | return ret; |
| 13612 | |
| 13613 | if (tgt_prog && prog->type == BPF_PROG_TYPE_EXT) { |
Toke Høiland-Jørgensen | 3aac1ea | 2020-09-29 14:45:50 +0200 | [diff] [blame] | 13614 | /* to make freplace equivalent to their targets, they need to |
| 13615 | * inherit env->ops and expected_attach_type for the rest of the |
| 13616 | * verification |
| 13617 | */ |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 13618 | env->ops = bpf_verifier_ops[tgt_prog->type]; |
| 13619 | prog->expected_attach_type = tgt_prog->expected_attach_type; |
| 13620 | } |
| 13621 | |
| 13622 | /* store info about the attachment target that will be used later */ |
| 13623 | prog->aux->attach_func_proto = tgt_info.tgt_type; |
| 13624 | prog->aux->attach_func_name = tgt_info.tgt_name; |
| 13625 | |
Toke Høiland-Jørgensen | 4a1e7c0 | 2020-09-29 14:45:51 +0200 | [diff] [blame] | 13626 | if (tgt_prog) { |
| 13627 | prog->aux->saved_dst_prog_type = tgt_prog->type; |
| 13628 | prog->aux->saved_dst_attach_type = tgt_prog->expected_attach_type; |
| 13629 | } |
| 13630 | |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 13631 | if (prog->expected_attach_type == BPF_TRACE_RAW_TP) { |
| 13632 | prog->aux->attach_btf_trace = true; |
| 13633 | return 0; |
| 13634 | } else if (prog->expected_attach_type == BPF_TRACE_ITER) { |
| 13635 | if (!bpf_iter_prog_supported(prog)) |
| 13636 | return -EINVAL; |
| 13637 | return 0; |
| 13638 | } |
| 13639 | |
| 13640 | if (prog->type == BPF_PROG_TYPE_LSM) { |
| 13641 | ret = bpf_lsm_verify_prog(&env->log, prog); |
| 13642 | if (ret < 0) |
| 13643 | return ret; |
Jiri Olsa | 35e3815 | 2021-04-29 13:47:12 +0200 | [diff] [blame] | 13644 | } else if (prog->type == BPF_PROG_TYPE_TRACING && |
| 13645 | btf_id_set_contains(&btf_id_deny, btf_id)) { |
| 13646 | return -EINVAL; |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 13647 | } |
| 13648 | |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 13649 | key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id); |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 13650 | tr = bpf_trampoline_get(key, &tgt_info); |
| 13651 | if (!tr) |
| 13652 | return -ENOMEM; |
| 13653 | |
Toke Høiland-Jørgensen | 3aac1ea | 2020-09-29 14:45:50 +0200 | [diff] [blame] | 13654 | prog->aux->dst_trampoline = tr; |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 13655 | return 0; |
Martin KaFai Lau | 3820729 | 2019-10-24 17:18:11 -0700 | [diff] [blame] | 13656 | } |
| 13657 | |
Alan Maguire | 76654e6 | 2020-09-28 12:31:03 +0100 | [diff] [blame] | 13658 | struct btf *bpf_get_btf_vmlinux(void) |
| 13659 | { |
| 13660 | if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { |
| 13661 | mutex_lock(&bpf_verifier_lock); |
| 13662 | if (!btf_vmlinux) |
| 13663 | btf_vmlinux = btf_parse_vmlinux(); |
| 13664 | mutex_unlock(&bpf_verifier_lock); |
| 13665 | } |
| 13666 | return btf_vmlinux; |
| 13667 | } |
| 13668 | |
Alexei Starovoitov | af2ac3e | 2021-05-13 17:36:05 -0700 | [diff] [blame] | 13669 | int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr) |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 13670 | { |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 13671 | u64 start_time = ktime_get_ns(); |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 13672 | struct bpf_verifier_env *env; |
Martin KaFai Lau | b9193c1 | 2018-03-24 11:44:22 -0700 | [diff] [blame] | 13673 | struct bpf_verifier_log *log; |
Jakub Kicinski | 9e4c24e | 2019-01-22 22:45:23 -0800 | [diff] [blame] | 13674 | int i, len, ret = -EINVAL; |
Jakub Kicinski | e2ae4ca | 2019-01-22 22:45:19 -0800 | [diff] [blame] | 13675 | bool is_priv; |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 13676 | |
Arnd Bergmann | eba0c92 | 2017-11-02 12:05:52 +0100 | [diff] [blame] | 13677 | /* no program is valid */ |
| 13678 | if (ARRAY_SIZE(bpf_verifier_ops) == 0) |
| 13679 | return -EINVAL; |
| 13680 | |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 13681 | /* 'struct bpf_verifier_env' can be global, but since it's not small, |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 13682 | * allocate/free it every time bpf_check() is called |
| 13683 | */ |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 13684 | env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL); |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 13685 | if (!env) |
| 13686 | return -ENOMEM; |
Jakub Kicinski | 61bd521 | 2017-10-09 10:30:11 -0700 | [diff] [blame] | 13687 | log = &env->log; |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 13688 | |
Jakub Kicinski | 9e4c24e | 2019-01-22 22:45:23 -0800 | [diff] [blame] | 13689 | len = (*prog)->len; |
Kees Cook | fad953c | 2018-06-12 14:27:37 -0700 | [diff] [blame] | 13690 | env->insn_aux_data = |
Jakub Kicinski | 9e4c24e | 2019-01-22 22:45:23 -0800 | [diff] [blame] | 13691 | vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len)); |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 13692 | ret = -ENOMEM; |
| 13693 | if (!env->insn_aux_data) |
| 13694 | goto err_free_env; |
Jakub Kicinski | 9e4c24e | 2019-01-22 22:45:23 -0800 | [diff] [blame] | 13695 | for (i = 0; i < len; i++) |
| 13696 | env->insn_aux_data[i].orig_idx = i; |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 13697 | env->prog = *prog; |
Jakub Kicinski | 00176a3 | 2017-10-16 16:40:54 -0700 | [diff] [blame] | 13698 | env->ops = bpf_verifier_ops[env->prog->type]; |
Alexei Starovoitov | 387544b | 2021-05-13 17:36:10 -0700 | [diff] [blame] | 13699 | env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); |
Alexei Starovoitov | 2c78ee8 | 2020-05-13 16:03:54 -0700 | [diff] [blame] | 13700 | is_priv = bpf_capable(); |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 13701 | |
Alan Maguire | 76654e6 | 2020-09-28 12:31:03 +0100 | [diff] [blame] | 13702 | bpf_get_btf_vmlinux(); |
Alexei Starovoitov | 8580ac9 | 2019-10-15 20:24:57 -0700 | [diff] [blame] | 13703 | |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 13704 | /* grab the mutex to protect few globals used by verifier */ |
Alexei Starovoitov | 45a73c1 | 2019-04-19 07:44:55 -0700 | [diff] [blame] | 13705 | if (!is_priv) |
| 13706 | mutex_lock(&bpf_verifier_lock); |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 13707 | |
| 13708 | if (attr->log_level || attr->log_buf || attr->log_size) { |
| 13709 | /* user requested verbose verifier output |
| 13710 | * and supplied buffer to store the verification trace |
| 13711 | */ |
Jakub Kicinski | e7bf824 | 2017-10-09 10:30:10 -0700 | [diff] [blame] | 13712 | log->level = attr->log_level; |
| 13713 | log->ubuf = (char __user *) (unsigned long) attr->log_buf; |
| 13714 | log->len_total = attr->log_size; |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 13715 | |
| 13716 | ret = -EINVAL; |
Jakub Kicinski | e7bf824 | 2017-10-09 10:30:10 -0700 | [diff] [blame] | 13717 | /* log attributes have to be sane */ |
Alexei Starovoitov | 7a9f5c6 | 2019-04-01 21:27:46 -0700 | [diff] [blame] | 13718 | if (log->len_total < 128 || log->len_total > UINT_MAX >> 2 || |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 13719 | !log->level || !log->ubuf || log->level & ~BPF_LOG_MASK) |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 13720 | goto err_unlock; |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 13721 | } |
Daniel Borkmann | 1ad2f58 | 2017-05-25 01:05:05 +0200 | [diff] [blame] | 13722 | |
Alexei Starovoitov | 8580ac9 | 2019-10-15 20:24:57 -0700 | [diff] [blame] | 13723 | if (IS_ERR(btf_vmlinux)) { |
| 13724 | /* Either gcc or pahole or kernel are broken. */ |
| 13725 | verbose(env, "in-kernel BTF is malformed\n"); |
| 13726 | ret = PTR_ERR(btf_vmlinux); |
Martin KaFai Lau | 3820729 | 2019-10-24 17:18:11 -0700 | [diff] [blame] | 13727 | goto skip_full_check; |
Alexei Starovoitov | 8580ac9 | 2019-10-15 20:24:57 -0700 | [diff] [blame] | 13728 | } |
| 13729 | |
Daniel Borkmann | 1ad2f58 | 2017-05-25 01:05:05 +0200 | [diff] [blame] | 13730 | env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT); |
| 13731 | if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) |
David S. Miller | e07b98d | 2017-05-10 11:38:07 -0700 | [diff] [blame] | 13732 | env->strict_alignment = true; |
David Miller | e9ee9ef | 2018-11-30 21:08:14 -0800 | [diff] [blame] | 13733 | if (attr->prog_flags & BPF_F_ANY_ALIGNMENT) |
| 13734 | env->strict_alignment = false; |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 13735 | |
Alexei Starovoitov | 2c78ee8 | 2020-05-13 16:03:54 -0700 | [diff] [blame] | 13736 | env->allow_ptr_leaks = bpf_allow_ptr_leaks(); |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 13737 | env->allow_uninit_stack = bpf_allow_uninit_stack(); |
Andrey Ignatov | 41c48f3 | 2020-06-19 14:11:43 -0700 | [diff] [blame] | 13738 | env->allow_ptr_to_map_access = bpf_allow_ptr_to_map_access(); |
Alexei Starovoitov | 2c78ee8 | 2020-05-13 16:03:54 -0700 | [diff] [blame] | 13739 | env->bypass_spec_v1 = bpf_bypass_spec_v1(); |
| 13740 | env->bypass_spec_v4 = bpf_bypass_spec_v4(); |
| 13741 | env->bpf_capable = bpf_capable(); |
Jakub Kicinski | e2ae4ca | 2019-01-22 22:45:19 -0800 | [diff] [blame] | 13742 | |
Alexei Starovoitov | 10d274e | 2019-08-22 22:52:12 -0700 | [diff] [blame] | 13743 | if (is_priv) |
| 13744 | env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; |
| 13745 | |
Alexei Starovoitov | dc2a4eb | 2019-05-21 20:17:07 -0700 | [diff] [blame] | 13746 | env->explored_states = kvcalloc(state_htab_size(env), |
Jakub Kicinski | 58e2af8b | 2016-09-21 11:43:57 +0100 | [diff] [blame] | 13747 | sizeof(struct bpf_verifier_state_list *), |
Alexei Starovoitov | f1bca82 | 2014-09-29 18:50:01 -0700 | [diff] [blame] | 13748 | GFP_USER); |
| 13749 | ret = -ENOMEM; |
| 13750 | if (!env->explored_states) |
| 13751 | goto skip_full_check; |
| 13752 | |
Martin KaFai Lau | e6ac245 | 2021-03-24 18:51:42 -0700 | [diff] [blame] | 13753 | ret = add_subprog_and_kfunc(env); |
| 13754 | if (ret < 0) |
| 13755 | goto skip_full_check; |
| 13756 | |
Martin KaFai Lau | d9762e8 | 2018-12-13 10:41:48 -0800 | [diff] [blame] | 13757 | ret = check_subprogs(env); |
Alexei Starovoitov | 475fb78 | 2014-09-26 00:17:05 -0700 | [diff] [blame] | 13758 | if (ret < 0) |
| 13759 | goto skip_full_check; |
| 13760 | |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 13761 | ret = check_btf_info(env, attr, uattr); |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 13762 | if (ret < 0) |
| 13763 | goto skip_full_check; |
| 13764 | |
Alexei Starovoitov | be8704f | 2020-01-20 16:53:46 -0800 | [diff] [blame] | 13765 | ret = check_attach_btf_id(env); |
| 13766 | if (ret) |
| 13767 | goto skip_full_check; |
| 13768 | |
Hao Luo | 4976b71 | 2020-09-29 16:50:44 -0700 | [diff] [blame] | 13769 | ret = resolve_pseudo_ldimm64(env); |
| 13770 | if (ret < 0) |
| 13771 | goto skip_full_check; |
| 13772 | |
Yinjun Zhang | ceb1167 | 2021-05-20 10:58:34 +0200 | [diff] [blame] | 13773 | if (bpf_prog_is_dev_bound(env->prog->aux)) { |
| 13774 | ret = bpf_prog_offload_verifier_prep(env->prog); |
| 13775 | if (ret) |
| 13776 | goto skip_full_check; |
| 13777 | } |
| 13778 | |
Martin KaFai Lau | d9762e8 | 2018-12-13 10:41:48 -0800 | [diff] [blame] | 13779 | ret = check_cfg(env); |
| 13780 | if (ret < 0) |
| 13781 | goto skip_full_check; |
| 13782 | |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 13783 | ret = do_check_subprogs(env); |
| 13784 | ret = ret ?: do_check_main(env); |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 13785 | |
Quentin Monnet | c941ce9 | 2018-10-07 12:56:47 +0100 | [diff] [blame] | 13786 | if (ret == 0 && bpf_prog_is_dev_bound(env->prog->aux)) |
| 13787 | ret = bpf_prog_offload_finalize(env); |
| 13788 | |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 13789 | skip_full_check: |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 13790 | kvfree(env->explored_states); |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 13791 | |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 13792 | if (ret == 0) |
Alexei Starovoitov | 70a87ff | 2017-12-25 13:15:40 -0800 | [diff] [blame] | 13793 | ret = check_max_stack_depth(env); |
| 13794 | |
Jakub Kicinski | 9b38c40 | 2018-12-19 22:13:06 -0800 | [diff] [blame] | 13795 | /* instruction rewrites happen after this point */ |
Jakub Kicinski | e2ae4ca | 2019-01-22 22:45:19 -0800 | [diff] [blame] | 13796 | if (is_priv) { |
| 13797 | if (ret == 0) |
| 13798 | opt_hard_wire_dead_code_branches(env); |
Jakub Kicinski | 52875a0 | 2019-01-22 22:45:20 -0800 | [diff] [blame] | 13799 | if (ret == 0) |
| 13800 | ret = opt_remove_dead_code(env); |
Jakub Kicinski | a1b14ab | 2019-01-22 22:45:21 -0800 | [diff] [blame] | 13801 | if (ret == 0) |
| 13802 | ret = opt_remove_nops(env); |
Jakub Kicinski | 52875a0 | 2019-01-22 22:45:20 -0800 | [diff] [blame] | 13803 | } else { |
| 13804 | if (ret == 0) |
| 13805 | sanitize_dead_code(env); |
Jakub Kicinski | e2ae4ca | 2019-01-22 22:45:19 -0800 | [diff] [blame] | 13806 | } |
| 13807 | |
Jakub Kicinski | 9b38c40 | 2018-12-19 22:13:06 -0800 | [diff] [blame] | 13808 | if (ret == 0) |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 13809 | /* program is valid, convert *(u32*)(ctx + off) accesses */ |
| 13810 | ret = convert_ctx_accesses(env); |
| 13811 | |
Alexei Starovoitov | e245c5c6 | 2017-03-15 18:26:39 -0700 | [diff] [blame] | 13812 | if (ret == 0) |
Brendan Jackman | e6ac593 | 2021-02-17 10:45:09 +0000 | [diff] [blame] | 13813 | ret = do_misc_fixups(env); |
Alexei Starovoitov | e245c5c6 | 2017-03-15 18:26:39 -0700 | [diff] [blame] | 13814 | |
Jiong Wang | a4b1d3c | 2019-05-24 23:25:15 +0100 | [diff] [blame] | 13815 | /* do 32-bit optimization after insn patching has done so those patched |
| 13816 | * insns could be handled correctly. |
| 13817 | */ |
Jiong Wang | d6c2308 | 2019-05-24 23:25:18 +0100 | [diff] [blame] | 13818 | if (ret == 0 && !bpf_prog_is_dev_bound(env->prog->aux)) { |
| 13819 | ret = opt_subreg_zext_lo32_rnd_hi32(env, attr); |
| 13820 | env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret |
| 13821 | : false; |
Jiong Wang | a4b1d3c | 2019-05-24 23:25:15 +0100 | [diff] [blame] | 13822 | } |
| 13823 | |
Alexei Starovoitov | 1ea47e0 | 2017-12-14 17:55:13 -0800 | [diff] [blame] | 13824 | if (ret == 0) |
| 13825 | ret = fixup_call_args(env); |
| 13826 | |
Alexei Starovoitov | 06ee711 | 2019-04-01 21:27:40 -0700 | [diff] [blame] | 13827 | env->verification_time = ktime_get_ns() - start_time; |
| 13828 | print_verification_stats(env); |
| 13829 | |
Jakub Kicinski | a2a7d57 | 2017-10-09 10:30:15 -0700 | [diff] [blame] | 13830 | if (log->level && bpf_verifier_log_full(log)) |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 13831 | ret = -ENOSPC; |
Jakub Kicinski | a2a7d57 | 2017-10-09 10:30:15 -0700 | [diff] [blame] | 13832 | if (log->level && !log->ubuf) { |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 13833 | ret = -EFAULT; |
Jakub Kicinski | a2a7d57 | 2017-10-09 10:30:15 -0700 | [diff] [blame] | 13834 | goto err_release_maps; |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 13835 | } |
| 13836 | |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 13837 | if (ret) |
| 13838 | goto err_release_maps; |
| 13839 | |
| 13840 | if (env->used_map_cnt) { |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 13841 | /* if program passed verifier, update used_maps in bpf_prog_info */ |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 13842 | env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt, |
| 13843 | sizeof(env->used_maps[0]), |
| 13844 | GFP_KERNEL); |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 13845 | |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 13846 | if (!env->prog->aux->used_maps) { |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 13847 | ret = -ENOMEM; |
Jakub Kicinski | a2a7d57 | 2017-10-09 10:30:15 -0700 | [diff] [blame] | 13848 | goto err_release_maps; |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 13849 | } |
| 13850 | |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 13851 | memcpy(env->prog->aux->used_maps, env->used_maps, |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 13852 | sizeof(env->used_maps[0]) * env->used_map_cnt); |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 13853 | env->prog->aux->used_map_cnt = env->used_map_cnt; |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 13854 | } |
| 13855 | if (env->used_btf_cnt) { |
| 13856 | /* if program passed verifier, update used_btfs in bpf_prog_aux */ |
| 13857 | env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt, |
| 13858 | sizeof(env->used_btfs[0]), |
| 13859 | GFP_KERNEL); |
| 13860 | if (!env->prog->aux->used_btfs) { |
| 13861 | ret = -ENOMEM; |
| 13862 | goto err_release_maps; |
| 13863 | } |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 13864 | |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 13865 | memcpy(env->prog->aux->used_btfs, env->used_btfs, |
| 13866 | sizeof(env->used_btfs[0]) * env->used_btf_cnt); |
| 13867 | env->prog->aux->used_btf_cnt = env->used_btf_cnt; |
| 13868 | } |
| 13869 | if (env->used_map_cnt || env->used_btf_cnt) { |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 13870 | /* program is valid. Convert pseudo bpf_ld_imm64 into generic |
| 13871 | * bpf_ld_imm64 instructions |
| 13872 | */ |
| 13873 | convert_pseudo_ld_imm64(env); |
| 13874 | } |
Alexei Starovoitov | cbd3570 | 2014-09-26 00:17:03 -0700 | [diff] [blame] | 13875 | |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 13876 | adjust_btf_func(env); |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 13877 | |
Jakub Kicinski | a2a7d57 | 2017-10-09 10:30:15 -0700 | [diff] [blame] | 13878 | err_release_maps: |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 13879 | if (!env->prog->aux->used_maps) |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 13880 | /* if we didn't copy map pointers into bpf_prog_info, release |
Jakub Kicinski | ab7f5bf | 2018-05-03 18:37:17 -0700 | [diff] [blame] | 13881 | * them now. Otherwise free_used_maps() will release them. |
Alexei Starovoitov | 0246e64 | 2014-09-26 00:17:04 -0700 | [diff] [blame] | 13882 | */ |
| 13883 | release_maps(env); |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 13884 | if (!env->prog->aux->used_btfs) |
| 13885 | release_btfs(env); |
Toke Høiland-Jørgensen | 03f87c0 | 2020-04-24 15:34:27 +0200 | [diff] [blame] | 13886 | |
| 13887 | /* extension progs temporarily inherit the attach_type of their targets |
| 13888 | for verification purposes, so set it back to zero before returning |
| 13889 | */ |
| 13890 | if (env->prog->type == BPF_PROG_TYPE_EXT) |
| 13891 | env->prog->expected_attach_type = 0; |
| 13892 | |
Alexei Starovoitov | 9bac3d6 | 2015-03-13 11:57:42 -0700 | [diff] [blame] | 13893 | *prog = env->prog; |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 13894 | err_unlock: |
Alexei Starovoitov | 45a73c1 | 2019-04-19 07:44:55 -0700 | [diff] [blame] | 13895 | if (!is_priv) |
| 13896 | mutex_unlock(&bpf_verifier_lock); |
Jakub Kicinski | 3df126f | 2016-09-21 11:43:56 +0100 | [diff] [blame] | 13897 | vfree(env->insn_aux_data); |
| 13898 | err_free_env: |
| 13899 | kfree(env); |
Alexei Starovoitov | 51580e7 | 2014-09-26 00:17:02 -0700 | [diff] [blame] | 13900 | return ret; |
| 13901 | } |