blob: e729c9e587a076a57d8dc652481da8423a58d2f4 [file] [log] [blame]
Steven Rostedt (VMware)179a0cc2018-08-16 11:20:54 -04001// SPDX-License-Identifier: GPL-2.0
Alexei Starovoitov25415172015-03-25 12:49:20 -07002/* Copyright (c) 2011-2015 PLUMgrid, http://plumgrid.com
Alexei Starovoitov0515e592016-09-01 18:37:22 -07003 * Copyright (c) 2016 Facebook
Alexei Starovoitov25415172015-03-25 12:49:20 -07004 */
5#include <linux/kernel.h>
6#include <linux/types.h>
7#include <linux/slab.h>
8#include <linux/bpf.h>
Alexei Starovoitov0515e592016-09-01 18:37:22 -07009#include <linux/bpf_perf_event.h>
Alexei Starovoitov25415172015-03-25 12:49:20 -070010#include <linux/filter.h>
11#include <linux/uaccess.h>
Alexei Starovoitov9c959c82015-03-25 12:49:22 -070012#include <linux/ctype.h>
Josef Bacik9802d862017-12-11 11:36:48 -050013#include <linux/kprobes.h>
Yonghong Song41bdc4b2018-05-24 11:21:09 -070014#include <linux/syscalls.h>
Masami Hiramatsu540adea2018-01-13 02:55:03 +090015#include <linux/error-injection.h>
Josef Bacik9802d862017-12-11 11:36:48 -050016
Nadav Amitc7b6f292019-04-25 17:11:43 -070017#include <asm/tlb.h>
18
Josef Bacik9802d862017-12-11 11:36:48 -050019#include "trace_probe.h"
Alexei Starovoitov25415172015-03-25 12:49:20 -070020#include "trace.h"
21
Stanislav Fomicheve672db02019-05-28 14:14:44 -070022#define bpf_event_rcu_dereference(p) \
23 rcu_dereference_protected(p, lockdep_is_held(&bpf_event_mutex))
24
Matt Mullinsa38d1102018-12-12 16:42:37 -080025#ifdef CONFIG_MODULES
26struct bpf_trace_module {
27 struct module *module;
28 struct list_head list;
29};
30
31static LIST_HEAD(bpf_trace_modules);
32static DEFINE_MUTEX(bpf_module_mutex);
33
34static struct bpf_raw_event_map *bpf_get_raw_tracepoint_module(const char *name)
35{
36 struct bpf_raw_event_map *btp, *ret = NULL;
37 struct bpf_trace_module *btm;
38 unsigned int i;
39
40 mutex_lock(&bpf_module_mutex);
41 list_for_each_entry(btm, &bpf_trace_modules, list) {
42 for (i = 0; i < btm->module->num_bpf_raw_events; ++i) {
43 btp = &btm->module->bpf_raw_events[i];
44 if (!strcmp(btp->tp->name, name)) {
45 if (try_module_get(btm->module))
46 ret = btp;
47 goto out;
48 }
49 }
50 }
51out:
52 mutex_unlock(&bpf_module_mutex);
53 return ret;
54}
55#else
56static struct bpf_raw_event_map *bpf_get_raw_tracepoint_module(const char *name)
57{
58 return NULL;
59}
60#endif /* CONFIG_MODULES */
61
Gianluca Borello035226b2017-10-26 01:47:42 +000062u64 bpf_get_stackid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
Yonghong Songc195651e2018-04-28 22:28:08 -070063u64 bpf_get_stack(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
Gianluca Borello035226b2017-10-26 01:47:42 +000064
Alexei Starovoitov25415172015-03-25 12:49:20 -070065/**
66 * trace_call_bpf - invoke BPF program
Yonghong Songe87c6bc2017-10-23 23:53:08 -070067 * @call: tracepoint event
Alexei Starovoitov25415172015-03-25 12:49:20 -070068 * @ctx: opaque context pointer
69 *
70 * kprobe handlers execute BPF programs via this helper.
71 * Can be used from static tracepoints in the future.
72 *
73 * Return: BPF programs always return an integer which is interpreted by
74 * kprobe handler as:
75 * 0 - return from kprobe (event is filtered out)
76 * 1 - store kprobe event into ring buffer
77 * Other values are reserved and currently alias to 1
78 */
Yonghong Songe87c6bc2017-10-23 23:53:08 -070079unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx)
Alexei Starovoitov25415172015-03-25 12:49:20 -070080{
81 unsigned int ret;
82
83 if (in_nmi()) /* not supported yet */
84 return 1;
85
Thomas Gleixnerb0a81b92020-02-24 15:01:37 +010086 cant_sleep();
Alexei Starovoitov25415172015-03-25 12:49:20 -070087
88 if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) {
89 /*
90 * since some bpf program is already running on this cpu,
91 * don't call into another bpf program (same or different)
92 * and don't send kprobe event into ring-buffer,
93 * so return zero here
94 */
95 ret = 0;
96 goto out;
97 }
98
Yonghong Songe87c6bc2017-10-23 23:53:08 -070099 /*
100 * Instead of moving rcu_read_lock/rcu_dereference/rcu_read_unlock
101 * to all call sites, we did a bpf_prog_array_valid() there to check
102 * whether call->prog_array is empty or not, which is
103 * a heurisitc to speed up execution.
104 *
105 * If bpf_prog_array_valid() fetched prog_array was
106 * non-NULL, we go into trace_call_bpf() and do the actual
107 * proper rcu_dereference() under RCU lock.
108 * If it turns out that prog_array is NULL then, we bail out.
109 * For the opposite, if the bpf_prog_array_valid() fetched pointer
110 * was NULL, you'll skip the prog_array with the risk of missing
111 * out of events when it was updated in between this and the
112 * rcu_dereference() which is accepted risk.
113 */
114 ret = BPF_PROG_RUN_ARRAY_CHECK(call->prog_array, ctx, BPF_PROG_RUN);
Alexei Starovoitov25415172015-03-25 12:49:20 -0700115
116 out:
117 __this_cpu_dec(bpf_prog_active);
Alexei Starovoitov25415172015-03-25 12:49:20 -0700118
119 return ret;
120}
Alexei Starovoitov25415172015-03-25 12:49:20 -0700121
Josef Bacik9802d862017-12-11 11:36:48 -0500122#ifdef CONFIG_BPF_KPROBE_OVERRIDE
123BPF_CALL_2(bpf_override_return, struct pt_regs *, regs, unsigned long, rc)
124{
Josef Bacik9802d862017-12-11 11:36:48 -0500125 regs_set_return_value(regs, rc);
Masami Hiramatsu540adea2018-01-13 02:55:03 +0900126 override_function_with_return(regs);
Josef Bacik9802d862017-12-11 11:36:48 -0500127 return 0;
128}
129
130static const struct bpf_func_proto bpf_override_return_proto = {
131 .func = bpf_override_return,
132 .gpl_only = true,
133 .ret_type = RET_INTEGER,
134 .arg1_type = ARG_PTR_TO_CTX,
135 .arg2_type = ARG_ANYTHING,
136};
137#endif
138
Christoph Hellwig8d92db52020-06-08 21:34:40 -0700139static __always_inline int
140bpf_probe_read_user_common(void *dst, u32 size, const void __user *unsafe_ptr)
141{
142 int ret;
143
144 ret = probe_user_read(dst, unsafe_ptr, size);
145 if (unlikely(ret < 0))
146 memset(dst, 0, size);
147 return ret;
148}
149
Daniel Borkmann6ae08ae2019-11-02 00:17:59 +0100150BPF_CALL_3(bpf_probe_read_user, void *, dst, u32, size,
151 const void __user *, unsafe_ptr)
Alexei Starovoitov25415172015-03-25 12:49:20 -0700152{
Christoph Hellwig8d92db52020-06-08 21:34:40 -0700153 return bpf_probe_read_user_common(dst, size, unsafe_ptr);
Alexei Starovoitov25415172015-03-25 12:49:20 -0700154}
155
John Fastabendf4703782020-05-24 09:50:55 -0700156const struct bpf_func_proto bpf_probe_read_user_proto = {
Daniel Borkmann6ae08ae2019-11-02 00:17:59 +0100157 .func = bpf_probe_read_user,
158 .gpl_only = true,
159 .ret_type = RET_INTEGER,
160 .arg1_type = ARG_PTR_TO_UNINIT_MEM,
161 .arg2_type = ARG_CONST_SIZE_OR_ZERO,
162 .arg3_type = ARG_ANYTHING,
163};
164
Christoph Hellwig8d92db52020-06-08 21:34:40 -0700165static __always_inline int
166bpf_probe_read_user_str_common(void *dst, u32 size,
167 const void __user *unsafe_ptr)
168{
169 int ret;
170
171 ret = strncpy_from_user_nofault(dst, unsafe_ptr, size);
172 if (unlikely(ret < 0))
173 memset(dst, 0, size);
174 return ret;
175}
176
Daniel Borkmann6ae08ae2019-11-02 00:17:59 +0100177BPF_CALL_3(bpf_probe_read_user_str, void *, dst, u32, size,
178 const void __user *, unsafe_ptr)
179{
Christoph Hellwig8d92db52020-06-08 21:34:40 -0700180 return bpf_probe_read_user_str_common(dst, size, unsafe_ptr);
Daniel Borkmann6ae08ae2019-11-02 00:17:59 +0100181}
182
John Fastabendf4703782020-05-24 09:50:55 -0700183const struct bpf_func_proto bpf_probe_read_user_str_proto = {
Daniel Borkmann6ae08ae2019-11-02 00:17:59 +0100184 .func = bpf_probe_read_user_str,
185 .gpl_only = true,
186 .ret_type = RET_INTEGER,
187 .arg1_type = ARG_PTR_TO_UNINIT_MEM,
188 .arg2_type = ARG_CONST_SIZE_OR_ZERO,
189 .arg3_type = ARG_ANYTHING,
190};
191
192static __always_inline int
Christoph Hellwig8d92db52020-06-08 21:34:40 -0700193bpf_probe_read_kernel_common(void *dst, u32 size, const void *unsafe_ptr)
Daniel Borkmann6ae08ae2019-11-02 00:17:59 +0100194{
195 int ret = security_locked_down(LOCKDOWN_BPF_READ);
196
197 if (unlikely(ret < 0))
Christoph Hellwig8d92db52020-06-08 21:34:40 -0700198 goto fail;
Christoph Hellwig98a23602020-06-08 21:34:50 -0700199 ret = probe_kernel_read(dst, unsafe_ptr, size);
Daniel Borkmann6ae08ae2019-11-02 00:17:59 +0100200 if (unlikely(ret < 0))
Christoph Hellwig8d92db52020-06-08 21:34:40 -0700201 goto fail;
202 return ret;
203fail:
204 memset(dst, 0, size);
Daniel Borkmann6ae08ae2019-11-02 00:17:59 +0100205 return ret;
206}
207
208BPF_CALL_3(bpf_probe_read_kernel, void *, dst, u32, size,
209 const void *, unsafe_ptr)
210{
Christoph Hellwig8d92db52020-06-08 21:34:40 -0700211 return bpf_probe_read_kernel_common(dst, size, unsafe_ptr);
Daniel Borkmann6ae08ae2019-11-02 00:17:59 +0100212}
213
John Fastabendf4703782020-05-24 09:50:55 -0700214const struct bpf_func_proto bpf_probe_read_kernel_proto = {
Daniel Borkmann6ae08ae2019-11-02 00:17:59 +0100215 .func = bpf_probe_read_kernel,
216 .gpl_only = true,
217 .ret_type = RET_INTEGER,
218 .arg1_type = ARG_PTR_TO_UNINIT_MEM,
219 .arg2_type = ARG_CONST_SIZE_OR_ZERO,
220 .arg3_type = ARG_ANYTHING,
221};
222
Daniel Borkmann6ae08ae2019-11-02 00:17:59 +0100223static __always_inline int
Christoph Hellwig8d92db52020-06-08 21:34:40 -0700224bpf_probe_read_kernel_str_common(void *dst, u32 size, const void *unsafe_ptr)
Daniel Borkmann6ae08ae2019-11-02 00:17:59 +0100225{
226 int ret = security_locked_down(LOCKDOWN_BPF_READ);
227
228 if (unlikely(ret < 0))
Christoph Hellwig8d92db52020-06-08 21:34:40 -0700229 goto fail;
230
Daniel Borkmann6ae08ae2019-11-02 00:17:59 +0100231 /*
Christoph Hellwig8d92db52020-06-08 21:34:40 -0700232 * The strncpy_from_kernel_nofault() call will likely not fill the
233 * entire buffer, but that's okay in this circumstance as we're probing
Daniel Borkmann6ae08ae2019-11-02 00:17:59 +0100234 * arbitrary memory anyway similar to bpf_probe_read_*() and might
235 * as well probe the stack. Thus, memory is explicitly cleared
236 * only in error case, so that improper users ignoring return
237 * code altogether don't copy garbage; otherwise length of string
238 * is returned that can be used for bpf_perf_event_output() et al.
239 */
Christoph Hellwig8d92db52020-06-08 21:34:40 -0700240 ret = strncpy_from_kernel_nofault(dst, unsafe_ptr, size);
Daniel Borkmann6ae08ae2019-11-02 00:17:59 +0100241 if (unlikely(ret < 0))
Christoph Hellwig8d92db52020-06-08 21:34:40 -0700242 goto fail;
243
244 return 0;
245fail:
246 memset(dst, 0, size);
Daniel Borkmann6ae08ae2019-11-02 00:17:59 +0100247 return ret;
248}
249
250BPF_CALL_3(bpf_probe_read_kernel_str, void *, dst, u32, size,
251 const void *, unsafe_ptr)
252{
Christoph Hellwig8d92db52020-06-08 21:34:40 -0700253 return bpf_probe_read_kernel_str_common(dst, size, unsafe_ptr);
Daniel Borkmann6ae08ae2019-11-02 00:17:59 +0100254}
255
John Fastabendf4703782020-05-24 09:50:55 -0700256const struct bpf_func_proto bpf_probe_read_kernel_str_proto = {
Daniel Borkmann6ae08ae2019-11-02 00:17:59 +0100257 .func = bpf_probe_read_kernel_str,
258 .gpl_only = true,
259 .ret_type = RET_INTEGER,
260 .arg1_type = ARG_PTR_TO_UNINIT_MEM,
261 .arg2_type = ARG_CONST_SIZE_OR_ZERO,
262 .arg3_type = ARG_ANYTHING,
263};
264
Christoph Hellwig8d92db52020-06-08 21:34:40 -0700265#ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
266BPF_CALL_3(bpf_probe_read_compat, void *, dst, u32, size,
267 const void *, unsafe_ptr)
268{
269 if ((unsigned long)unsafe_ptr < TASK_SIZE) {
270 return bpf_probe_read_user_common(dst, size,
271 (__force void __user *)unsafe_ptr);
272 }
273 return bpf_probe_read_kernel_common(dst, size, unsafe_ptr);
274}
275
276static const struct bpf_func_proto bpf_probe_read_compat_proto = {
277 .func = bpf_probe_read_compat,
278 .gpl_only = true,
279 .ret_type = RET_INTEGER,
280 .arg1_type = ARG_PTR_TO_UNINIT_MEM,
281 .arg2_type = ARG_CONST_SIZE_OR_ZERO,
282 .arg3_type = ARG_ANYTHING,
283};
284
Daniel Borkmann6ae08ae2019-11-02 00:17:59 +0100285BPF_CALL_3(bpf_probe_read_compat_str, void *, dst, u32, size,
286 const void *, unsafe_ptr)
287{
Christoph Hellwig8d92db52020-06-08 21:34:40 -0700288 if ((unsigned long)unsafe_ptr < TASK_SIZE) {
289 return bpf_probe_read_user_str_common(dst, size,
290 (__force void __user *)unsafe_ptr);
291 }
292 return bpf_probe_read_kernel_str_common(dst, size, unsafe_ptr);
Daniel Borkmann6ae08ae2019-11-02 00:17:59 +0100293}
294
295static const struct bpf_func_proto bpf_probe_read_compat_str_proto = {
296 .func = bpf_probe_read_compat_str,
Alexei Starovoitov25415172015-03-25 12:49:20 -0700297 .gpl_only = true,
298 .ret_type = RET_INTEGER,
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -0800299 .arg1_type = ARG_PTR_TO_UNINIT_MEM,
Yonghong Song9c019e22017-11-12 14:49:10 -0800300 .arg2_type = ARG_CONST_SIZE_OR_ZERO,
Alexei Starovoitov25415172015-03-25 12:49:20 -0700301 .arg3_type = ARG_ANYTHING,
302};
Christoph Hellwig8d92db52020-06-08 21:34:40 -0700303#endif /* CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE */
Alexei Starovoitov25415172015-03-25 12:49:20 -0700304
Daniel Borkmanneb1b6682019-11-02 00:17:58 +0100305BPF_CALL_3(bpf_probe_write_user, void __user *, unsafe_ptr, const void *, src,
Daniel Borkmannf3694e02016-09-09 02:45:31 +0200306 u32, size)
Sargun Dhillon96ae5222016-07-25 05:54:46 -0700307{
Sargun Dhillon96ae5222016-07-25 05:54:46 -0700308 /*
309 * Ensure we're in user context which is safe for the helper to
310 * run. This helper has no business in a kthread.
311 *
312 * access_ok() should prevent writing to non-user memory, but in
313 * some situations (nommu, temporary switch, etc) access_ok() does
314 * not provide enough validation, hence the check on KERNEL_DS.
Nadav Amitc7b6f292019-04-25 17:11:43 -0700315 *
316 * nmi_uaccess_okay() ensures the probe is not run in an interim
317 * state, when the task or mm are switched. This is specifically
318 * required to prevent the use of temporary mm.
Sargun Dhillon96ae5222016-07-25 05:54:46 -0700319 */
320
321 if (unlikely(in_interrupt() ||
322 current->flags & (PF_KTHREAD | PF_EXITING)))
323 return -EPERM;
Al Virodb68ce12017-03-20 21:08:07 -0400324 if (unlikely(uaccess_kernel()))
Sargun Dhillon96ae5222016-07-25 05:54:46 -0700325 return -EPERM;
Nadav Amitc7b6f292019-04-25 17:11:43 -0700326 if (unlikely(!nmi_uaccess_okay()))
327 return -EPERM;
Sargun Dhillon96ae5222016-07-25 05:54:46 -0700328
Daniel Borkmanneb1b6682019-11-02 00:17:58 +0100329 return probe_user_write(unsafe_ptr, src, size);
Sargun Dhillon96ae5222016-07-25 05:54:46 -0700330}
331
332static const struct bpf_func_proto bpf_probe_write_user_proto = {
333 .func = bpf_probe_write_user,
334 .gpl_only = true,
335 .ret_type = RET_INTEGER,
336 .arg1_type = ARG_ANYTHING,
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -0800337 .arg2_type = ARG_PTR_TO_MEM,
338 .arg3_type = ARG_CONST_SIZE,
Sargun Dhillon96ae5222016-07-25 05:54:46 -0700339};
340
341static const struct bpf_func_proto *bpf_get_probe_write_proto(void)
342{
Alexei Starovoitov2c78ee82020-05-13 16:03:54 -0700343 if (!capable(CAP_SYS_ADMIN))
344 return NULL;
345
Sargun Dhillon96ae5222016-07-25 05:54:46 -0700346 pr_warn_ratelimited("%s[%d] is installing a program with bpf_probe_write_user helper that may corrupt user memory!",
347 current->comm, task_pid_nr(current));
348
349 return &bpf_probe_write_user_proto;
350}
351
Christoph Hellwigd7b29772020-06-08 21:34:30 -0700352static void bpf_trace_copy_string(char *buf, void *unsafe_ptr, char fmt_ptype,
353 size_t bufsz)
354{
355 void __user *user_ptr = (__force void __user *)unsafe_ptr;
356
357 buf[0] = 0;
358
359 switch (fmt_ptype) {
360 case 's':
361#ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
Christoph Hellwigaec6ce52020-06-08 21:34:33 -0700362 if ((unsigned long)unsafe_ptr < TASK_SIZE) {
363 strncpy_from_user_nofault(buf, user_ptr, bufsz);
364 break;
365 }
366 fallthrough;
Christoph Hellwigd7b29772020-06-08 21:34:30 -0700367#endif
368 case 'k':
369 strncpy_from_kernel_nofault(buf, unsafe_ptr, bufsz);
370 break;
371 case 'u':
372 strncpy_from_user_nofault(buf, user_ptr, bufsz);
373 break;
374 }
375}
376
Alexei Starovoitov9c959c82015-03-25 12:49:22 -0700377/*
John Fastabend7bda4b42017-07-02 02:13:29 +0200378 * Only limited trace_printk() conversion specifiers allowed:
Daniel Borkmannb2a52122020-05-15 12:11:18 +0200379 * %d %i %u %x %ld %li %lu %lx %lld %lli %llu %llx %p %pks %pus %s
Alexei Starovoitov9c959c82015-03-25 12:49:22 -0700380 */
Daniel Borkmannf3694e02016-09-09 02:45:31 +0200381BPF_CALL_5(bpf_trace_printk, char *, fmt, u32, fmt_size, u64, arg1,
382 u64, arg2, u64, arg3)
Alexei Starovoitov9c959c82015-03-25 12:49:22 -0700383{
Daniel Borkmannb2a52122020-05-15 12:11:18 +0200384 int i, mod[3] = {}, fmt_cnt = 0;
385 char buf[64], fmt_ptype;
386 void *unsafe_ptr = NULL;
Alexei Starovoitov8d3b7dc2015-08-28 15:56:23 -0700387 bool str_seen = false;
Alexei Starovoitov9c959c82015-03-25 12:49:22 -0700388
389 /*
390 * bpf_check()->check_func_arg()->check_stack_boundary()
391 * guarantees that fmt points to bpf program stack,
392 * fmt_size bytes of it were initialized and fmt_size > 0
393 */
394 if (fmt[--fmt_size] != 0)
395 return -EINVAL;
396
397 /* check format string for allowed specifiers */
398 for (i = 0; i < fmt_size; i++) {
399 if ((!isprint(fmt[i]) && !isspace(fmt[i])) || !isascii(fmt[i]))
400 return -EINVAL;
401
402 if (fmt[i] != '%')
403 continue;
404
405 if (fmt_cnt >= 3)
406 return -EINVAL;
407
408 /* fmt[i] != 0 && fmt[last] == 0, so we can access fmt[i + 1] */
409 i++;
410 if (fmt[i] == 'l') {
411 mod[fmt_cnt]++;
412 i++;
Daniel Borkmannb2a52122020-05-15 12:11:18 +0200413 } else if (fmt[i] == 'p') {
Alexei Starovoitov9c959c82015-03-25 12:49:22 -0700414 mod[fmt_cnt]++;
Daniel Borkmannb2a52122020-05-15 12:11:18 +0200415 if ((fmt[i + 1] == 'k' ||
416 fmt[i + 1] == 'u') &&
417 fmt[i + 2] == 's') {
418 fmt_ptype = fmt[i + 1];
419 i += 2;
420 goto fmt_str;
421 }
422
Martynas Pumputis1efb6ee2018-11-23 17:43:26 +0100423 /* disallow any further format extensions */
424 if (fmt[i + 1] != 0 &&
425 !isspace(fmt[i + 1]) &&
426 !ispunct(fmt[i + 1]))
Alexei Starovoitov9c959c82015-03-25 12:49:22 -0700427 return -EINVAL;
Alexei Starovoitov8d3b7dc2015-08-28 15:56:23 -0700428
Daniel Borkmannb2a52122020-05-15 12:11:18 +0200429 goto fmt_next;
430 } else if (fmt[i] == 's') {
431 mod[fmt_cnt]++;
432 fmt_ptype = fmt[i];
433fmt_str:
434 if (str_seen)
435 /* allow only one '%s' per fmt string */
436 return -EINVAL;
437 str_seen = true;
438
439 if (fmt[i + 1] != 0 &&
440 !isspace(fmt[i + 1]) &&
441 !ispunct(fmt[i + 1]))
442 return -EINVAL;
443
444 switch (fmt_cnt) {
445 case 0:
446 unsafe_ptr = (void *)(long)arg1;
447 arg1 = (long)buf;
448 break;
449 case 1:
450 unsafe_ptr = (void *)(long)arg2;
451 arg2 = (long)buf;
452 break;
453 case 2:
454 unsafe_ptr = (void *)(long)arg3;
455 arg3 = (long)buf;
456 break;
Alexei Starovoitov8d3b7dc2015-08-28 15:56:23 -0700457 }
Daniel Borkmannb2a52122020-05-15 12:11:18 +0200458
Christoph Hellwigd7b29772020-06-08 21:34:30 -0700459 bpf_trace_copy_string(buf, unsafe_ptr, fmt_ptype,
460 sizeof(buf));
Daniel Borkmannb2a52122020-05-15 12:11:18 +0200461 goto fmt_next;
Alexei Starovoitov9c959c82015-03-25 12:49:22 -0700462 }
463
464 if (fmt[i] == 'l') {
465 mod[fmt_cnt]++;
466 i++;
467 }
468
John Fastabend7bda4b42017-07-02 02:13:29 +0200469 if (fmt[i] != 'i' && fmt[i] != 'd' &&
470 fmt[i] != 'u' && fmt[i] != 'x')
Alexei Starovoitov9c959c82015-03-25 12:49:22 -0700471 return -EINVAL;
Daniel Borkmannb2a52122020-05-15 12:11:18 +0200472fmt_next:
Alexei Starovoitov9c959c82015-03-25 12:49:22 -0700473 fmt_cnt++;
474 }
475
Daniel Borkmann88a5c692017-08-16 01:45:33 +0200476/* Horrid workaround for getting va_list handling working with different
477 * argument type combinations generically for 32 and 64 bit archs.
478 */
479#define __BPF_TP_EMIT() __BPF_ARG3_TP()
480#define __BPF_TP(...) \
Yonghong Songeefa864a2018-01-17 09:19:32 -0800481 __trace_printk(0 /* Fake ip */, \
Daniel Borkmann88a5c692017-08-16 01:45:33 +0200482 fmt, ##__VA_ARGS__)
483
484#define __BPF_ARG1_TP(...) \
485 ((mod[0] == 2 || (mod[0] == 1 && __BITS_PER_LONG == 64)) \
486 ? __BPF_TP(arg1, ##__VA_ARGS__) \
487 : ((mod[0] == 1 || (mod[0] == 0 && __BITS_PER_LONG == 32)) \
488 ? __BPF_TP((long)arg1, ##__VA_ARGS__) \
489 : __BPF_TP((u32)arg1, ##__VA_ARGS__)))
490
491#define __BPF_ARG2_TP(...) \
492 ((mod[1] == 2 || (mod[1] == 1 && __BITS_PER_LONG == 64)) \
493 ? __BPF_ARG1_TP(arg2, ##__VA_ARGS__) \
494 : ((mod[1] == 1 || (mod[1] == 0 && __BITS_PER_LONG == 32)) \
495 ? __BPF_ARG1_TP((long)arg2, ##__VA_ARGS__) \
496 : __BPF_ARG1_TP((u32)arg2, ##__VA_ARGS__)))
497
498#define __BPF_ARG3_TP(...) \
499 ((mod[2] == 2 || (mod[2] == 1 && __BITS_PER_LONG == 64)) \
500 ? __BPF_ARG2_TP(arg3, ##__VA_ARGS__) \
501 : ((mod[2] == 1 || (mod[2] == 0 && __BITS_PER_LONG == 32)) \
502 ? __BPF_ARG2_TP((long)arg3, ##__VA_ARGS__) \
503 : __BPF_ARG2_TP((u32)arg3, ##__VA_ARGS__)))
504
505 return __BPF_TP_EMIT();
Alexei Starovoitov9c959c82015-03-25 12:49:22 -0700506}
507
508static const struct bpf_func_proto bpf_trace_printk_proto = {
509 .func = bpf_trace_printk,
510 .gpl_only = true,
511 .ret_type = RET_INTEGER,
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -0800512 .arg1_type = ARG_PTR_TO_MEM,
513 .arg2_type = ARG_CONST_SIZE,
Alexei Starovoitov9c959c82015-03-25 12:49:22 -0700514};
515
Alexei Starovoitov0756ea32015-06-12 19:39:13 -0700516const struct bpf_func_proto *bpf_get_trace_printk_proto(void)
517{
518 /*
519 * this program might be calling bpf_trace_printk,
520 * so allocate per-cpu printk buffers
521 */
522 trace_printk_init_buffers();
523
524 return &bpf_trace_printk_proto;
525}
526
Yonghong Song492e6392020-05-09 10:59:14 -0700527#define MAX_SEQ_PRINTF_VARARGS 12
528#define MAX_SEQ_PRINTF_MAX_MEMCPY 6
529#define MAX_SEQ_PRINTF_STR_LEN 128
530
531struct bpf_seq_printf_buf {
532 char buf[MAX_SEQ_PRINTF_MAX_MEMCPY][MAX_SEQ_PRINTF_STR_LEN];
533};
534static DEFINE_PER_CPU(struct bpf_seq_printf_buf, bpf_seq_printf_buf);
535static DEFINE_PER_CPU(int, bpf_seq_printf_buf_used);
536
537BPF_CALL_5(bpf_seq_printf, struct seq_file *, m, char *, fmt, u32, fmt_size,
538 const void *, data, u32, data_len)
539{
540 int err = -EINVAL, fmt_cnt = 0, memcpy_cnt = 0;
541 int i, buf_used, copy_size, num_args;
542 u64 params[MAX_SEQ_PRINTF_VARARGS];
543 struct bpf_seq_printf_buf *bufs;
544 const u64 *args = data;
545
546 buf_used = this_cpu_inc_return(bpf_seq_printf_buf_used);
547 if (WARN_ON_ONCE(buf_used > 1)) {
548 err = -EBUSY;
549 goto out;
550 }
551
552 bufs = this_cpu_ptr(&bpf_seq_printf_buf);
553
554 /*
555 * bpf_check()->check_func_arg()->check_stack_boundary()
556 * guarantees that fmt points to bpf program stack,
557 * fmt_size bytes of it were initialized and fmt_size > 0
558 */
559 if (fmt[--fmt_size] != 0)
560 goto out;
561
562 if (data_len & 7)
563 goto out;
564
565 for (i = 0; i < fmt_size; i++) {
566 if (fmt[i] == '%') {
567 if (fmt[i + 1] == '%')
568 i++;
569 else if (!data || !data_len)
570 goto out;
571 }
572 }
573
574 num_args = data_len / 8;
575
576 /* check format string for allowed specifiers */
577 for (i = 0; i < fmt_size; i++) {
578 /* only printable ascii for now. */
579 if ((!isprint(fmt[i]) && !isspace(fmt[i])) || !isascii(fmt[i])) {
580 err = -EINVAL;
581 goto out;
582 }
583
584 if (fmt[i] != '%')
585 continue;
586
587 if (fmt[i + 1] == '%') {
588 i++;
589 continue;
590 }
591
592 if (fmt_cnt >= MAX_SEQ_PRINTF_VARARGS) {
593 err = -E2BIG;
594 goto out;
595 }
596
597 if (fmt_cnt >= num_args) {
598 err = -EINVAL;
599 goto out;
600 }
601
602 /* fmt[i] != 0 && fmt[last] == 0, so we can access fmt[i + 1] */
603 i++;
604
605 /* skip optional "[0 +-][num]" width formating field */
606 while (fmt[i] == '0' || fmt[i] == '+' || fmt[i] == '-' ||
607 fmt[i] == ' ')
608 i++;
609 if (fmt[i] >= '1' && fmt[i] <= '9') {
610 i++;
611 while (fmt[i] >= '0' && fmt[i] <= '9')
612 i++;
613 }
614
615 if (fmt[i] == 's') {
Andrew Morton19c8d8a2020-06-08 21:34:37 -0700616 void *unsafe_ptr;
617
Yonghong Song492e6392020-05-09 10:59:14 -0700618 /* try our best to copy */
619 if (memcpy_cnt >= MAX_SEQ_PRINTF_MAX_MEMCPY) {
620 err = -E2BIG;
621 goto out;
622 }
623
Andrew Morton19c8d8a2020-06-08 21:34:37 -0700624 unsafe_ptr = (void *)(long)args[fmt_cnt];
625 err = strncpy_from_kernel_nofault(bufs->buf[memcpy_cnt],
626 unsafe_ptr, MAX_SEQ_PRINTF_STR_LEN);
Yonghong Song492e6392020-05-09 10:59:14 -0700627 if (err < 0)
628 bufs->buf[memcpy_cnt][0] = '\0';
629 params[fmt_cnt] = (u64)(long)bufs->buf[memcpy_cnt];
630
631 fmt_cnt++;
632 memcpy_cnt++;
633 continue;
634 }
635
636 if (fmt[i] == 'p') {
637 if (fmt[i + 1] == 0 ||
638 fmt[i + 1] == 'K' ||
639 fmt[i + 1] == 'x') {
640 /* just kernel pointers */
641 params[fmt_cnt] = args[fmt_cnt];
642 fmt_cnt++;
643 continue;
644 }
645
646 /* only support "%pI4", "%pi4", "%pI6" and "%pi6". */
647 if (fmt[i + 1] != 'i' && fmt[i + 1] != 'I') {
648 err = -EINVAL;
649 goto out;
650 }
651 if (fmt[i + 2] != '4' && fmt[i + 2] != '6') {
652 err = -EINVAL;
653 goto out;
654 }
655
656 if (memcpy_cnt >= MAX_SEQ_PRINTF_MAX_MEMCPY) {
657 err = -E2BIG;
658 goto out;
659 }
660
661
662 copy_size = (fmt[i + 2] == '4') ? 4 : 16;
663
664 err = probe_kernel_read(bufs->buf[memcpy_cnt],
665 (void *) (long) args[fmt_cnt],
666 copy_size);
667 if (err < 0)
668 memset(bufs->buf[memcpy_cnt], 0, copy_size);
669 params[fmt_cnt] = (u64)(long)bufs->buf[memcpy_cnt];
670
671 i += 2;
672 fmt_cnt++;
673 memcpy_cnt++;
674 continue;
675 }
676
677 if (fmt[i] == 'l') {
678 i++;
679 if (fmt[i] == 'l')
680 i++;
681 }
682
683 if (fmt[i] != 'i' && fmt[i] != 'd' &&
684 fmt[i] != 'u' && fmt[i] != 'x') {
685 err = -EINVAL;
686 goto out;
687 }
688
689 params[fmt_cnt] = args[fmt_cnt];
690 fmt_cnt++;
691 }
692
693 /* Maximumly we can have MAX_SEQ_PRINTF_VARARGS parameter, just give
694 * all of them to seq_printf().
695 */
696 seq_printf(m, fmt, params[0], params[1], params[2], params[3],
697 params[4], params[5], params[6], params[7], params[8],
698 params[9], params[10], params[11]);
699
700 err = seq_has_overflowed(m) ? -EOVERFLOW : 0;
701out:
702 this_cpu_dec(bpf_seq_printf_buf_used);
703 return err;
704}
705
706static int bpf_seq_printf_btf_ids[5];
707static const struct bpf_func_proto bpf_seq_printf_proto = {
708 .func = bpf_seq_printf,
709 .gpl_only = true,
710 .ret_type = RET_INTEGER,
711 .arg1_type = ARG_PTR_TO_BTF_ID,
712 .arg2_type = ARG_PTR_TO_MEM,
713 .arg3_type = ARG_CONST_SIZE,
714 .arg4_type = ARG_PTR_TO_MEM_OR_NULL,
715 .arg5_type = ARG_CONST_SIZE_OR_ZERO,
716 .btf_id = bpf_seq_printf_btf_ids,
717};
718
719BPF_CALL_3(bpf_seq_write, struct seq_file *, m, const void *, data, u32, len)
720{
721 return seq_write(m, data, len) ? -EOVERFLOW : 0;
722}
723
724static int bpf_seq_write_btf_ids[5];
725static const struct bpf_func_proto bpf_seq_write_proto = {
726 .func = bpf_seq_write,
727 .gpl_only = true,
728 .ret_type = RET_INTEGER,
729 .arg1_type = ARG_PTR_TO_BTF_ID,
730 .arg2_type = ARG_PTR_TO_MEM,
731 .arg3_type = ARG_CONST_SIZE_OR_ZERO,
732 .btf_id = bpf_seq_write_btf_ids,
733};
734
Yonghong Song908432c2017-10-05 09:19:20 -0700735static __always_inline int
736get_map_perf_counter(struct bpf_map *map, u64 flags,
737 u64 *value, u64 *enabled, u64 *running)
Kaixu Xia35578d72015-08-06 07:02:35 +0000738{
Kaixu Xia35578d72015-08-06 07:02:35 +0000739 struct bpf_array *array = container_of(map, struct bpf_array, map);
Daniel Borkmann6816a7f2016-06-28 12:18:25 +0200740 unsigned int cpu = smp_processor_id();
741 u64 index = flags & BPF_F_INDEX_MASK;
Daniel Borkmann3b1efb12016-06-15 22:47:14 +0200742 struct bpf_event_entry *ee;
Kaixu Xia35578d72015-08-06 07:02:35 +0000743
Daniel Borkmann6816a7f2016-06-28 12:18:25 +0200744 if (unlikely(flags & ~(BPF_F_INDEX_MASK)))
745 return -EINVAL;
746 if (index == BPF_F_CURRENT_CPU)
747 index = cpu;
Kaixu Xia35578d72015-08-06 07:02:35 +0000748 if (unlikely(index >= array->map.max_entries))
749 return -E2BIG;
750
Daniel Borkmann3b1efb12016-06-15 22:47:14 +0200751 ee = READ_ONCE(array->ptrs[index]);
Daniel Borkmann1ca1cc92016-06-28 12:18:23 +0200752 if (!ee)
Kaixu Xia35578d72015-08-06 07:02:35 +0000753 return -ENOENT;
754
Yonghong Song908432c2017-10-05 09:19:20 -0700755 return perf_event_read_local(ee->event, value, enabled, running);
756}
757
758BPF_CALL_2(bpf_perf_event_read, struct bpf_map *, map, u64, flags)
759{
760 u64 value = 0;
761 int err;
762
763 err = get_map_perf_counter(map, flags, &value, NULL, NULL);
Kaixu Xia35578d72015-08-06 07:02:35 +0000764 /*
Alexei Starovoitovf91840a2017-06-02 21:03:52 -0700765 * this api is ugly since we miss [-22..-2] range of valid
766 * counter values, but that's uapi
Kaixu Xia35578d72015-08-06 07:02:35 +0000767 */
Alexei Starovoitovf91840a2017-06-02 21:03:52 -0700768 if (err)
769 return err;
770 return value;
Kaixu Xia35578d72015-08-06 07:02:35 +0000771}
772
Alexei Starovoitov62544ce2015-10-22 17:10:14 -0700773static const struct bpf_func_proto bpf_perf_event_read_proto = {
Kaixu Xia35578d72015-08-06 07:02:35 +0000774 .func = bpf_perf_event_read,
Alexei Starovoitov1075ef52015-10-23 14:58:19 -0700775 .gpl_only = true,
Kaixu Xia35578d72015-08-06 07:02:35 +0000776 .ret_type = RET_INTEGER,
777 .arg1_type = ARG_CONST_MAP_PTR,
778 .arg2_type = ARG_ANYTHING,
779};
780
Yonghong Song908432c2017-10-05 09:19:20 -0700781BPF_CALL_4(bpf_perf_event_read_value, struct bpf_map *, map, u64, flags,
782 struct bpf_perf_event_value *, buf, u32, size)
783{
784 int err = -EINVAL;
785
786 if (unlikely(size != sizeof(struct bpf_perf_event_value)))
787 goto clear;
788 err = get_map_perf_counter(map, flags, &buf->counter, &buf->enabled,
789 &buf->running);
790 if (unlikely(err))
791 goto clear;
792 return 0;
793clear:
794 memset(buf, 0, size);
795 return err;
796}
797
798static const struct bpf_func_proto bpf_perf_event_read_value_proto = {
799 .func = bpf_perf_event_read_value,
800 .gpl_only = true,
801 .ret_type = RET_INTEGER,
802 .arg1_type = ARG_CONST_MAP_PTR,
803 .arg2_type = ARG_ANYTHING,
804 .arg3_type = ARG_PTR_TO_UNINIT_MEM,
805 .arg4_type = ARG_CONST_SIZE,
806};
807
Daniel Borkmann8e7a3922016-07-14 18:08:04 +0200808static __always_inline u64
809__bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,
Daniel Borkmann283ca522017-12-12 02:25:30 +0100810 u64 flags, struct perf_sample_data *sd)
Alexei Starovoitova43eec32015-10-20 20:02:34 -0700811{
Alexei Starovoitova43eec32015-10-20 20:02:34 -0700812 struct bpf_array *array = container_of(map, struct bpf_array, map);
Daniel Borkmannd7931332016-06-28 12:18:24 +0200813 unsigned int cpu = smp_processor_id();
Daniel Borkmann1e337592016-04-18 21:01:23 +0200814 u64 index = flags & BPF_F_INDEX_MASK;
Daniel Borkmann3b1efb12016-06-15 22:47:14 +0200815 struct bpf_event_entry *ee;
Alexei Starovoitova43eec32015-10-20 20:02:34 -0700816 struct perf_event *event;
Alexei Starovoitova43eec32015-10-20 20:02:34 -0700817
Daniel Borkmann1e337592016-04-18 21:01:23 +0200818 if (index == BPF_F_CURRENT_CPU)
Daniel Borkmannd7931332016-06-28 12:18:24 +0200819 index = cpu;
Alexei Starovoitova43eec32015-10-20 20:02:34 -0700820 if (unlikely(index >= array->map.max_entries))
821 return -E2BIG;
822
Daniel Borkmann3b1efb12016-06-15 22:47:14 +0200823 ee = READ_ONCE(array->ptrs[index]);
Daniel Borkmann1ca1cc92016-06-28 12:18:23 +0200824 if (!ee)
Alexei Starovoitova43eec32015-10-20 20:02:34 -0700825 return -ENOENT;
826
Daniel Borkmann3b1efb12016-06-15 22:47:14 +0200827 event = ee->event;
Alexei Starovoitova43eec32015-10-20 20:02:34 -0700828 if (unlikely(event->attr.type != PERF_TYPE_SOFTWARE ||
829 event->attr.config != PERF_COUNT_SW_BPF_OUTPUT))
830 return -EINVAL;
831
Daniel Borkmannd7931332016-06-28 12:18:24 +0200832 if (unlikely(event->oncpu != cpu))
Alexei Starovoitova43eec32015-10-20 20:02:34 -0700833 return -EOPNOTSUPP;
834
Arnaldo Carvalho de Melo56201962019-01-11 13:20:20 -0300835 return perf_event_output(event, sd, regs);
Alexei Starovoitova43eec32015-10-20 20:02:34 -0700836}
837
Matt Mullins9594dc32019-06-11 14:53:04 -0700838/*
839 * Support executing tracepoints in normal, irq, and nmi context that each call
840 * bpf_perf_event_output
841 */
842struct bpf_trace_sample_data {
843 struct perf_sample_data sds[3];
844};
845
846static DEFINE_PER_CPU(struct bpf_trace_sample_data, bpf_trace_sds);
847static DEFINE_PER_CPU(int, bpf_trace_nest_level);
Daniel Borkmannf3694e02016-09-09 02:45:31 +0200848BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map,
849 u64, flags, void *, data, u64, size)
Daniel Borkmann8e7a3922016-07-14 18:08:04 +0200850{
Matt Mullins9594dc32019-06-11 14:53:04 -0700851 struct bpf_trace_sample_data *sds = this_cpu_ptr(&bpf_trace_sds);
852 int nest_level = this_cpu_inc_return(bpf_trace_nest_level);
Daniel Borkmann8e7a3922016-07-14 18:08:04 +0200853 struct perf_raw_record raw = {
854 .frag = {
855 .size = size,
856 .data = data,
857 },
858 };
Matt Mullins9594dc32019-06-11 14:53:04 -0700859 struct perf_sample_data *sd;
860 int err;
Daniel Borkmann8e7a3922016-07-14 18:08:04 +0200861
Matt Mullins9594dc32019-06-11 14:53:04 -0700862 if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(sds->sds))) {
863 err = -EBUSY;
864 goto out;
865 }
866
867 sd = &sds->sds[nest_level - 1];
868
869 if (unlikely(flags & ~(BPF_F_INDEX_MASK))) {
870 err = -EINVAL;
871 goto out;
872 }
Daniel Borkmann8e7a3922016-07-14 18:08:04 +0200873
Daniel Borkmann283ca522017-12-12 02:25:30 +0100874 perf_sample_data_init(sd, 0, 0);
875 sd->raw = &raw;
876
Matt Mullins9594dc32019-06-11 14:53:04 -0700877 err = __bpf_perf_event_output(regs, map, flags, sd);
878
879out:
880 this_cpu_dec(bpf_trace_nest_level);
881 return err;
Daniel Borkmann8e7a3922016-07-14 18:08:04 +0200882}
883
Alexei Starovoitova43eec32015-10-20 20:02:34 -0700884static const struct bpf_func_proto bpf_perf_event_output_proto = {
885 .func = bpf_perf_event_output,
Alexei Starovoitov1075ef52015-10-23 14:58:19 -0700886 .gpl_only = true,
Alexei Starovoitova43eec32015-10-20 20:02:34 -0700887 .ret_type = RET_INTEGER,
888 .arg1_type = ARG_PTR_TO_CTX,
889 .arg2_type = ARG_CONST_MAP_PTR,
890 .arg3_type = ARG_ANYTHING,
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -0800891 .arg4_type = ARG_PTR_TO_MEM,
Gianluca Borelloa60dd352017-11-22 18:32:56 +0000892 .arg5_type = ARG_CONST_SIZE_OR_ZERO,
Alexei Starovoitova43eec32015-10-20 20:02:34 -0700893};
894
Allan Zhang768fb612019-09-25 16:43:12 -0700895static DEFINE_PER_CPU(int, bpf_event_output_nest_level);
896struct bpf_nested_pt_regs {
897 struct pt_regs regs[3];
898};
899static DEFINE_PER_CPU(struct bpf_nested_pt_regs, bpf_pt_regs);
900static DEFINE_PER_CPU(struct bpf_trace_sample_data, bpf_misc_sds);
Daniel Borkmannbd570ff2016-04-18 21:01:24 +0200901
Daniel Borkmann555c8a82016-07-14 18:08:05 +0200902u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
903 void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy)
Daniel Borkmannbd570ff2016-04-18 21:01:24 +0200904{
Allan Zhang768fb612019-09-25 16:43:12 -0700905 int nest_level = this_cpu_inc_return(bpf_event_output_nest_level);
Daniel Borkmann555c8a82016-07-14 18:08:05 +0200906 struct perf_raw_frag frag = {
907 .copy = ctx_copy,
908 .size = ctx_size,
909 .data = ctx,
910 };
911 struct perf_raw_record raw = {
912 .frag = {
Andrew Morton183fc152016-07-18 15:50:58 -0700913 {
914 .next = ctx_size ? &frag : NULL,
915 },
Daniel Borkmann555c8a82016-07-14 18:08:05 +0200916 .size = meta_size,
917 .data = meta,
918 },
919 };
Allan Zhang768fb612019-09-25 16:43:12 -0700920 struct perf_sample_data *sd;
921 struct pt_regs *regs;
922 u64 ret;
923
924 if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(bpf_misc_sds.sds))) {
925 ret = -EBUSY;
926 goto out;
927 }
928 sd = this_cpu_ptr(&bpf_misc_sds.sds[nest_level - 1]);
929 regs = this_cpu_ptr(&bpf_pt_regs.regs[nest_level - 1]);
Daniel Borkmannbd570ff2016-04-18 21:01:24 +0200930
931 perf_fetch_caller_regs(regs);
Daniel Borkmann283ca522017-12-12 02:25:30 +0100932 perf_sample_data_init(sd, 0, 0);
933 sd->raw = &raw;
Daniel Borkmannbd570ff2016-04-18 21:01:24 +0200934
Allan Zhang768fb612019-09-25 16:43:12 -0700935 ret = __bpf_perf_event_output(regs, map, flags, sd);
936out:
937 this_cpu_dec(bpf_event_output_nest_level);
938 return ret;
Daniel Borkmannbd570ff2016-04-18 21:01:24 +0200939}
940
Daniel Borkmannf3694e02016-09-09 02:45:31 +0200941BPF_CALL_0(bpf_get_current_task)
Alexei Starovoitov606274c2016-07-06 22:38:36 -0700942{
943 return (long) current;
944}
945
John Fastabendf4703782020-05-24 09:50:55 -0700946const struct bpf_func_proto bpf_get_current_task_proto = {
Alexei Starovoitov606274c2016-07-06 22:38:36 -0700947 .func = bpf_get_current_task,
948 .gpl_only = true,
949 .ret_type = RET_INTEGER,
950};
951
Daniel Borkmannf3694e02016-09-09 02:45:31 +0200952BPF_CALL_2(bpf_current_task_under_cgroup, struct bpf_map *, map, u32, idx)
Sargun Dhillon60d20f92016-08-12 08:56:52 -0700953{
Sargun Dhillon60d20f92016-08-12 08:56:52 -0700954 struct bpf_array *array = container_of(map, struct bpf_array, map);
955 struct cgroup *cgrp;
Sargun Dhillon60d20f92016-08-12 08:56:52 -0700956
Sargun Dhillon60d20f92016-08-12 08:56:52 -0700957 if (unlikely(idx >= array->map.max_entries))
958 return -E2BIG;
959
960 cgrp = READ_ONCE(array->ptrs[idx]);
961 if (unlikely(!cgrp))
962 return -EAGAIN;
963
964 return task_under_cgroup_hierarchy(current, cgrp);
965}
966
967static const struct bpf_func_proto bpf_current_task_under_cgroup_proto = {
968 .func = bpf_current_task_under_cgroup,
969 .gpl_only = false,
970 .ret_type = RET_INTEGER,
971 .arg1_type = ARG_CONST_MAP_PTR,
972 .arg2_type = ARG_ANYTHING,
973};
974
Yonghong Song8b401f92019-05-23 14:47:45 -0700975struct send_signal_irq_work {
976 struct irq_work irq_work;
977 struct task_struct *task;
978 u32 sig;
Yonghong Song84829412020-01-14 19:50:02 -0800979 enum pid_type type;
Yonghong Song8b401f92019-05-23 14:47:45 -0700980};
981
982static DEFINE_PER_CPU(struct send_signal_irq_work, send_signal_work);
983
984static void do_bpf_send_signal(struct irq_work *entry)
985{
986 struct send_signal_irq_work *work;
987
988 work = container_of(entry, struct send_signal_irq_work, irq_work);
Yonghong Song84829412020-01-14 19:50:02 -0800989 group_send_sig_info(work->sig, SEND_SIG_PRIV, work->task, work->type);
Yonghong Song8b401f92019-05-23 14:47:45 -0700990}
991
Yonghong Song84829412020-01-14 19:50:02 -0800992static int bpf_send_signal_common(u32 sig, enum pid_type type)
Yonghong Song8b401f92019-05-23 14:47:45 -0700993{
994 struct send_signal_irq_work *work = NULL;
995
996 /* Similar to bpf_probe_write_user, task needs to be
997 * in a sound condition and kernel memory access be
998 * permitted in order to send signal to the current
999 * task.
1000 */
1001 if (unlikely(current->flags & (PF_KTHREAD | PF_EXITING)))
1002 return -EPERM;
1003 if (unlikely(uaccess_kernel()))
1004 return -EPERM;
1005 if (unlikely(!nmi_uaccess_okay()))
1006 return -EPERM;
1007
Yonghong Song1bc78962020-03-04 11:11:04 -08001008 if (irqs_disabled()) {
Yonghong Songe1afb7022019-05-25 11:57:53 -07001009 /* Do an early check on signal validity. Otherwise,
1010 * the error is lost in deferred irq_work.
1011 */
1012 if (unlikely(!valid_signal(sig)))
1013 return -EINVAL;
1014
Yonghong Song8b401f92019-05-23 14:47:45 -07001015 work = this_cpu_ptr(&send_signal_work);
Frederic Weisbecker153bedb2019-11-08 17:08:55 +01001016 if (atomic_read(&work->irq_work.flags) & IRQ_WORK_BUSY)
Yonghong Song8b401f92019-05-23 14:47:45 -07001017 return -EBUSY;
1018
1019 /* Add the current task, which is the target of sending signal,
1020 * to the irq_work. The current task may change when queued
1021 * irq works get executed.
1022 */
1023 work->task = current;
1024 work->sig = sig;
Yonghong Song84829412020-01-14 19:50:02 -08001025 work->type = type;
Yonghong Song8b401f92019-05-23 14:47:45 -07001026 irq_work_queue(&work->irq_work);
1027 return 0;
1028 }
1029
Yonghong Song84829412020-01-14 19:50:02 -08001030 return group_send_sig_info(sig, SEND_SIG_PRIV, current, type);
1031}
1032
1033BPF_CALL_1(bpf_send_signal, u32, sig)
1034{
1035 return bpf_send_signal_common(sig, PIDTYPE_TGID);
Yonghong Song8b401f92019-05-23 14:47:45 -07001036}
1037
1038static const struct bpf_func_proto bpf_send_signal_proto = {
1039 .func = bpf_send_signal,
1040 .gpl_only = false,
1041 .ret_type = RET_INTEGER,
1042 .arg1_type = ARG_ANYTHING,
1043};
1044
Yonghong Song84829412020-01-14 19:50:02 -08001045BPF_CALL_1(bpf_send_signal_thread, u32, sig)
1046{
1047 return bpf_send_signal_common(sig, PIDTYPE_PID);
1048}
1049
1050static const struct bpf_func_proto bpf_send_signal_thread_proto = {
1051 .func = bpf_send_signal_thread,
1052 .gpl_only = false,
1053 .ret_type = RET_INTEGER,
1054 .arg1_type = ARG_ANYTHING,
1055};
1056
KP Singhfc611f42020-03-29 01:43:49 +01001057const struct bpf_func_proto *
1058bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
Alexei Starovoitov25415172015-03-25 12:49:20 -07001059{
1060 switch (func_id) {
1061 case BPF_FUNC_map_lookup_elem:
1062 return &bpf_map_lookup_elem_proto;
1063 case BPF_FUNC_map_update_elem:
1064 return &bpf_map_update_elem_proto;
1065 case BPF_FUNC_map_delete_elem:
1066 return &bpf_map_delete_elem_proto;
Alban Crequy02a8c812019-04-14 18:58:46 +02001067 case BPF_FUNC_map_push_elem:
1068 return &bpf_map_push_elem_proto;
1069 case BPF_FUNC_map_pop_elem:
1070 return &bpf_map_pop_elem_proto;
1071 case BPF_FUNC_map_peek_elem:
1072 return &bpf_map_peek_elem_proto;
Alexei Starovoitovd9847d32015-03-25 12:49:21 -07001073 case BPF_FUNC_ktime_get_ns:
1074 return &bpf_ktime_get_ns_proto;
Maciej Żenczykowski71d19212020-04-26 09:15:25 -07001075 case BPF_FUNC_ktime_get_boot_ns:
1076 return &bpf_ktime_get_boot_ns_proto;
Alexei Starovoitov04fd61ab2015-05-19 16:59:03 -07001077 case BPF_FUNC_tail_call:
1078 return &bpf_tail_call_proto;
Alexei Starovoitovffeedaf2015-06-12 19:39:12 -07001079 case BPF_FUNC_get_current_pid_tgid:
1080 return &bpf_get_current_pid_tgid_proto;
Alexei Starovoitov606274c2016-07-06 22:38:36 -07001081 case BPF_FUNC_get_current_task:
1082 return &bpf_get_current_task_proto;
Alexei Starovoitovffeedaf2015-06-12 19:39:12 -07001083 case BPF_FUNC_get_current_uid_gid:
1084 return &bpf_get_current_uid_gid_proto;
1085 case BPF_FUNC_get_current_comm:
1086 return &bpf_get_current_comm_proto;
Alexei Starovoitov9c959c82015-03-25 12:49:22 -07001087 case BPF_FUNC_trace_printk:
Alexei Starovoitov0756ea32015-06-12 19:39:13 -07001088 return bpf_get_trace_printk_proto();
Alexei Starovoitovab1973d2015-06-12 19:39:14 -07001089 case BPF_FUNC_get_smp_processor_id:
1090 return &bpf_get_smp_processor_id_proto;
Daniel Borkmann2d0e30c2016-10-21 12:46:33 +02001091 case BPF_FUNC_get_numa_node_id:
1092 return &bpf_get_numa_node_id_proto;
Kaixu Xia35578d72015-08-06 07:02:35 +00001093 case BPF_FUNC_perf_event_read:
1094 return &bpf_perf_event_read_proto;
Sargun Dhillon96ae5222016-07-25 05:54:46 -07001095 case BPF_FUNC_probe_write_user:
1096 return bpf_get_probe_write_proto();
Sargun Dhillon60d20f92016-08-12 08:56:52 -07001097 case BPF_FUNC_current_task_under_cgroup:
1098 return &bpf_current_task_under_cgroup_proto;
Alexei Starovoitov8937bd82016-08-11 18:17:18 -07001099 case BPF_FUNC_get_prandom_u32:
1100 return &bpf_get_prandom_u32_proto;
Daniel Borkmann6ae08ae2019-11-02 00:17:59 +01001101 case BPF_FUNC_probe_read_user:
1102 return &bpf_probe_read_user_proto;
1103 case BPF_FUNC_probe_read_kernel:
1104 return &bpf_probe_read_kernel_proto;
Daniel Borkmann6ae08ae2019-11-02 00:17:59 +01001105 case BPF_FUNC_probe_read_user_str:
1106 return &bpf_probe_read_user_str_proto;
1107 case BPF_FUNC_probe_read_kernel_str:
1108 return &bpf_probe_read_kernel_str_proto;
Daniel Borkmann0ebeea82020-05-15 12:11:16 +02001109#ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
1110 case BPF_FUNC_probe_read:
1111 return &bpf_probe_read_compat_proto;
Gianluca Borelloa5e8c072017-01-18 17:55:49 +00001112 case BPF_FUNC_probe_read_str:
Daniel Borkmann6ae08ae2019-11-02 00:17:59 +01001113 return &bpf_probe_read_compat_str_proto;
Daniel Borkmann0ebeea82020-05-15 12:11:16 +02001114#endif
Yonghong Song34ea38c2018-06-04 08:53:41 -07001115#ifdef CONFIG_CGROUPS
Yonghong Songbf6fa2c82018-06-03 15:59:41 -07001116 case BPF_FUNC_get_current_cgroup_id:
1117 return &bpf_get_current_cgroup_id_proto;
Yonghong Song34ea38c2018-06-04 08:53:41 -07001118#endif
Yonghong Song8b401f92019-05-23 14:47:45 -07001119 case BPF_FUNC_send_signal:
1120 return &bpf_send_signal_proto;
Yonghong Song84829412020-01-14 19:50:02 -08001121 case BPF_FUNC_send_signal_thread:
1122 return &bpf_send_signal_thread_proto;
Song Liub80b0332020-02-14 15:41:46 -08001123 case BPF_FUNC_perf_event_read_value:
1124 return &bpf_perf_event_read_value_proto;
Carlos Neirab4490c52020-03-04 17:41:56 -03001125 case BPF_FUNC_get_ns_current_pid_tgid:
1126 return &bpf_get_ns_current_pid_tgid_proto;
Andrii Nakryiko457f4432020-05-29 00:54:20 -07001127 case BPF_FUNC_ringbuf_output:
1128 return &bpf_ringbuf_output_proto;
1129 case BPF_FUNC_ringbuf_reserve:
1130 return &bpf_ringbuf_reserve_proto;
1131 case BPF_FUNC_ringbuf_submit:
1132 return &bpf_ringbuf_submit_proto;
1133 case BPF_FUNC_ringbuf_discard:
1134 return &bpf_ringbuf_discard_proto;
1135 case BPF_FUNC_ringbuf_query:
1136 return &bpf_ringbuf_query_proto;
Alexei Starovoitov9fd82b612016-04-06 18:43:26 -07001137 default:
1138 return NULL;
1139 }
1140}
1141
Andrey Ignatov5e43f892018-03-30 15:08:00 -07001142static const struct bpf_func_proto *
1143kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
Alexei Starovoitov9fd82b612016-04-06 18:43:26 -07001144{
1145 switch (func_id) {
Alexei Starovoitova43eec32015-10-20 20:02:34 -07001146 case BPF_FUNC_perf_event_output:
1147 return &bpf_perf_event_output_proto;
Alexei Starovoitovd5a3b1f2016-02-17 19:58:58 -08001148 case BPF_FUNC_get_stackid:
1149 return &bpf_get_stackid_proto;
Yonghong Songc195651e2018-04-28 22:28:08 -07001150 case BPF_FUNC_get_stack:
1151 return &bpf_get_stack_proto;
Josef Bacik9802d862017-12-11 11:36:48 -05001152#ifdef CONFIG_BPF_KPROBE_OVERRIDE
1153 case BPF_FUNC_override_return:
1154 return &bpf_override_return_proto;
1155#endif
Alexei Starovoitov25415172015-03-25 12:49:20 -07001156 default:
KP Singhfc611f42020-03-29 01:43:49 +01001157 return bpf_tracing_func_proto(func_id, prog);
Alexei Starovoitov25415172015-03-25 12:49:20 -07001158 }
1159}
1160
1161/* bpf+kprobe programs can access fields of 'struct pt_regs' */
Alexei Starovoitov19de99f2016-06-15 18:25:38 -07001162static bool kprobe_prog_is_valid_access(int off, int size, enum bpf_access_type type,
Andrey Ignatov5e43f892018-03-30 15:08:00 -07001163 const struct bpf_prog *prog,
Yonghong Song23994632017-06-22 15:07:39 -07001164 struct bpf_insn_access_aux *info)
Alexei Starovoitov25415172015-03-25 12:49:20 -07001165{
Alexei Starovoitov25415172015-03-25 12:49:20 -07001166 if (off < 0 || off >= sizeof(struct pt_regs))
1167 return false;
Alexei Starovoitov25415172015-03-25 12:49:20 -07001168 if (type != BPF_READ)
1169 return false;
Alexei Starovoitov25415172015-03-25 12:49:20 -07001170 if (off % size != 0)
1171 return false;
Daniel Borkmann2d071c62017-01-15 01:34:25 +01001172 /*
1173 * Assertion for 32 bit to make sure last 8 byte access
1174 * (BPF_DW) to the last 4 byte member is disallowed.
1175 */
1176 if (off + size > sizeof(struct pt_regs))
1177 return false;
1178
Alexei Starovoitov25415172015-03-25 12:49:20 -07001179 return true;
1180}
1181
Jakub Kicinski7de16e32017-10-16 16:40:53 -07001182const struct bpf_verifier_ops kprobe_verifier_ops = {
Alexei Starovoitov25415172015-03-25 12:49:20 -07001183 .get_func_proto = kprobe_prog_func_proto,
1184 .is_valid_access = kprobe_prog_is_valid_access,
1185};
1186
Jakub Kicinski7de16e32017-10-16 16:40:53 -07001187const struct bpf_prog_ops kprobe_prog_ops = {
1188};
1189
Daniel Borkmannf3694e02016-09-09 02:45:31 +02001190BPF_CALL_5(bpf_perf_event_output_tp, void *, tp_buff, struct bpf_map *, map,
1191 u64, flags, void *, data, u64, size)
Alexei Starovoitov9940d672016-04-06 18:43:27 -07001192{
Daniel Borkmannf3694e02016-09-09 02:45:31 +02001193 struct pt_regs *regs = *(struct pt_regs **)tp_buff;
1194
Alexei Starovoitov9940d672016-04-06 18:43:27 -07001195 /*
1196 * r1 points to perf tracepoint buffer where first 8 bytes are hidden
1197 * from bpf program and contain a pointer to 'struct pt_regs'. Fetch it
Daniel Borkmannf3694e02016-09-09 02:45:31 +02001198 * from there and call the same bpf_perf_event_output() helper inline.
Alexei Starovoitov9940d672016-04-06 18:43:27 -07001199 */
Daniel Borkmannf3694e02016-09-09 02:45:31 +02001200 return ____bpf_perf_event_output(regs, map, flags, data, size);
Alexei Starovoitov9940d672016-04-06 18:43:27 -07001201}
1202
1203static const struct bpf_func_proto bpf_perf_event_output_proto_tp = {
1204 .func = bpf_perf_event_output_tp,
1205 .gpl_only = true,
1206 .ret_type = RET_INTEGER,
1207 .arg1_type = ARG_PTR_TO_CTX,
1208 .arg2_type = ARG_CONST_MAP_PTR,
1209 .arg3_type = ARG_ANYTHING,
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -08001210 .arg4_type = ARG_PTR_TO_MEM,
Gianluca Borelloa60dd352017-11-22 18:32:56 +00001211 .arg5_type = ARG_CONST_SIZE_OR_ZERO,
Alexei Starovoitov9940d672016-04-06 18:43:27 -07001212};
1213
Daniel Borkmannf3694e02016-09-09 02:45:31 +02001214BPF_CALL_3(bpf_get_stackid_tp, void *, tp_buff, struct bpf_map *, map,
1215 u64, flags)
Alexei Starovoitov9940d672016-04-06 18:43:27 -07001216{
Daniel Borkmannf3694e02016-09-09 02:45:31 +02001217 struct pt_regs *regs = *(struct pt_regs **)tp_buff;
Alexei Starovoitov9940d672016-04-06 18:43:27 -07001218
Daniel Borkmannf3694e02016-09-09 02:45:31 +02001219 /*
1220 * Same comment as in bpf_perf_event_output_tp(), only that this time
1221 * the other helper's function body cannot be inlined due to being
1222 * external, thus we need to call raw helper function.
1223 */
1224 return bpf_get_stackid((unsigned long) regs, (unsigned long) map,
1225 flags, 0, 0);
Alexei Starovoitov9940d672016-04-06 18:43:27 -07001226}
1227
1228static const struct bpf_func_proto bpf_get_stackid_proto_tp = {
1229 .func = bpf_get_stackid_tp,
1230 .gpl_only = true,
1231 .ret_type = RET_INTEGER,
1232 .arg1_type = ARG_PTR_TO_CTX,
1233 .arg2_type = ARG_CONST_MAP_PTR,
1234 .arg3_type = ARG_ANYTHING,
1235};
1236
Yonghong Songc195651e2018-04-28 22:28:08 -07001237BPF_CALL_4(bpf_get_stack_tp, void *, tp_buff, void *, buf, u32, size,
1238 u64, flags)
1239{
1240 struct pt_regs *regs = *(struct pt_regs **)tp_buff;
1241
1242 return bpf_get_stack((unsigned long) regs, (unsigned long) buf,
1243 (unsigned long) size, flags, 0);
1244}
1245
1246static const struct bpf_func_proto bpf_get_stack_proto_tp = {
1247 .func = bpf_get_stack_tp,
1248 .gpl_only = true,
1249 .ret_type = RET_INTEGER,
1250 .arg1_type = ARG_PTR_TO_CTX,
1251 .arg2_type = ARG_PTR_TO_UNINIT_MEM,
1252 .arg3_type = ARG_CONST_SIZE_OR_ZERO,
1253 .arg4_type = ARG_ANYTHING,
1254};
1255
Andrey Ignatov5e43f892018-03-30 15:08:00 -07001256static const struct bpf_func_proto *
1257tp_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
Alexei Starovoitov9fd82b612016-04-06 18:43:26 -07001258{
1259 switch (func_id) {
1260 case BPF_FUNC_perf_event_output:
Alexei Starovoitov9940d672016-04-06 18:43:27 -07001261 return &bpf_perf_event_output_proto_tp;
Alexei Starovoitov9fd82b612016-04-06 18:43:26 -07001262 case BPF_FUNC_get_stackid:
Alexei Starovoitov9940d672016-04-06 18:43:27 -07001263 return &bpf_get_stackid_proto_tp;
Yonghong Songc195651e2018-04-28 22:28:08 -07001264 case BPF_FUNC_get_stack:
1265 return &bpf_get_stack_proto_tp;
Alexei Starovoitov9fd82b612016-04-06 18:43:26 -07001266 default:
KP Singhfc611f42020-03-29 01:43:49 +01001267 return bpf_tracing_func_proto(func_id, prog);
Alexei Starovoitov9fd82b612016-04-06 18:43:26 -07001268 }
1269}
1270
Alexei Starovoitov19de99f2016-06-15 18:25:38 -07001271static bool tp_prog_is_valid_access(int off, int size, enum bpf_access_type type,
Andrey Ignatov5e43f892018-03-30 15:08:00 -07001272 const struct bpf_prog *prog,
Yonghong Song23994632017-06-22 15:07:39 -07001273 struct bpf_insn_access_aux *info)
Alexei Starovoitov9fd82b612016-04-06 18:43:26 -07001274{
1275 if (off < sizeof(void *) || off >= PERF_MAX_TRACE_SIZE)
1276 return false;
1277 if (type != BPF_READ)
1278 return false;
1279 if (off % size != 0)
1280 return false;
Daniel Borkmann2d071c62017-01-15 01:34:25 +01001281
1282 BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(__u64));
Alexei Starovoitov9fd82b612016-04-06 18:43:26 -07001283 return true;
1284}
1285
Jakub Kicinski7de16e32017-10-16 16:40:53 -07001286const struct bpf_verifier_ops tracepoint_verifier_ops = {
Alexei Starovoitov9fd82b612016-04-06 18:43:26 -07001287 .get_func_proto = tp_prog_func_proto,
1288 .is_valid_access = tp_prog_is_valid_access,
1289};
1290
Jakub Kicinski7de16e32017-10-16 16:40:53 -07001291const struct bpf_prog_ops tracepoint_prog_ops = {
1292};
1293
Yonghong Songf005afe2018-03-20 11:19:17 -07001294BPF_CALL_3(bpf_perf_prog_read_value, struct bpf_perf_event_data_kern *, ctx,
1295 struct bpf_perf_event_value *, buf, u32, size)
1296{
1297 int err = -EINVAL;
1298
1299 if (unlikely(size != sizeof(struct bpf_perf_event_value)))
1300 goto clear;
1301 err = perf_event_read_local(ctx->event, &buf->counter, &buf->enabled,
1302 &buf->running);
1303 if (unlikely(err))
1304 goto clear;
1305 return 0;
1306clear:
1307 memset(buf, 0, size);
1308 return err;
1309}
1310
1311static const struct bpf_func_proto bpf_perf_prog_read_value_proto = {
1312 .func = bpf_perf_prog_read_value,
1313 .gpl_only = true,
1314 .ret_type = RET_INTEGER,
1315 .arg1_type = ARG_PTR_TO_CTX,
1316 .arg2_type = ARG_PTR_TO_UNINIT_MEM,
1317 .arg3_type = ARG_CONST_SIZE,
1318};
1319
Daniel Xufff7b642020-02-17 19:04:31 -08001320BPF_CALL_4(bpf_read_branch_records, struct bpf_perf_event_data_kern *, ctx,
1321 void *, buf, u32, size, u64, flags)
1322{
1323#ifndef CONFIG_X86
1324 return -ENOENT;
1325#else
1326 static const u32 br_entry_size = sizeof(struct perf_branch_entry);
1327 struct perf_branch_stack *br_stack = ctx->data->br_stack;
1328 u32 to_copy;
1329
1330 if (unlikely(flags & ~BPF_F_GET_BRANCH_RECORDS_SIZE))
1331 return -EINVAL;
1332
1333 if (unlikely(!br_stack))
1334 return -EINVAL;
1335
1336 if (flags & BPF_F_GET_BRANCH_RECORDS_SIZE)
1337 return br_stack->nr * br_entry_size;
1338
1339 if (!buf || (size % br_entry_size != 0))
1340 return -EINVAL;
1341
1342 to_copy = min_t(u32, br_stack->nr * br_entry_size, size);
1343 memcpy(buf, br_stack->entries, to_copy);
1344
1345 return to_copy;
1346#endif
1347}
1348
1349static const struct bpf_func_proto bpf_read_branch_records_proto = {
1350 .func = bpf_read_branch_records,
1351 .gpl_only = true,
1352 .ret_type = RET_INTEGER,
1353 .arg1_type = ARG_PTR_TO_CTX,
1354 .arg2_type = ARG_PTR_TO_MEM_OR_NULL,
1355 .arg3_type = ARG_CONST_SIZE_OR_ZERO,
1356 .arg4_type = ARG_ANYTHING,
1357};
1358
Andrey Ignatov5e43f892018-03-30 15:08:00 -07001359static const struct bpf_func_proto *
1360pe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
Yonghong Songf005afe2018-03-20 11:19:17 -07001361{
1362 switch (func_id) {
1363 case BPF_FUNC_perf_event_output:
1364 return &bpf_perf_event_output_proto_tp;
1365 case BPF_FUNC_get_stackid:
1366 return &bpf_get_stackid_proto_tp;
Yonghong Songc195651e2018-04-28 22:28:08 -07001367 case BPF_FUNC_get_stack:
1368 return &bpf_get_stack_proto_tp;
Yonghong Songf005afe2018-03-20 11:19:17 -07001369 case BPF_FUNC_perf_prog_read_value:
1370 return &bpf_perf_prog_read_value_proto;
Daniel Xufff7b642020-02-17 19:04:31 -08001371 case BPF_FUNC_read_branch_records:
1372 return &bpf_read_branch_records_proto;
Yonghong Songf005afe2018-03-20 11:19:17 -07001373 default:
KP Singhfc611f42020-03-29 01:43:49 +01001374 return bpf_tracing_func_proto(func_id, prog);
Yonghong Songf005afe2018-03-20 11:19:17 -07001375 }
1376}
1377
Alexei Starovoitovc4f66992018-03-28 12:05:37 -07001378/*
1379 * bpf_raw_tp_regs are separate from bpf_pt_regs used from skb/xdp
1380 * to avoid potential recursive reuse issue when/if tracepoints are added
Matt Mullins9594dc32019-06-11 14:53:04 -07001381 * inside bpf_*_event_output, bpf_get_stackid and/or bpf_get_stack.
1382 *
1383 * Since raw tracepoints run despite bpf_prog_active, support concurrent usage
1384 * in normal, irq, and nmi context.
Alexei Starovoitovc4f66992018-03-28 12:05:37 -07001385 */
Matt Mullins9594dc32019-06-11 14:53:04 -07001386struct bpf_raw_tp_regs {
1387 struct pt_regs regs[3];
1388};
1389static DEFINE_PER_CPU(struct bpf_raw_tp_regs, bpf_raw_tp_regs);
1390static DEFINE_PER_CPU(int, bpf_raw_tp_nest_level);
1391static struct pt_regs *get_bpf_raw_tp_regs(void)
1392{
1393 struct bpf_raw_tp_regs *tp_regs = this_cpu_ptr(&bpf_raw_tp_regs);
1394 int nest_level = this_cpu_inc_return(bpf_raw_tp_nest_level);
1395
1396 if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(tp_regs->regs))) {
1397 this_cpu_dec(bpf_raw_tp_nest_level);
1398 return ERR_PTR(-EBUSY);
1399 }
1400
1401 return &tp_regs->regs[nest_level - 1];
1402}
1403
1404static void put_bpf_raw_tp_regs(void)
1405{
1406 this_cpu_dec(bpf_raw_tp_nest_level);
1407}
1408
Alexei Starovoitovc4f66992018-03-28 12:05:37 -07001409BPF_CALL_5(bpf_perf_event_output_raw_tp, struct bpf_raw_tracepoint_args *, args,
1410 struct bpf_map *, map, u64, flags, void *, data, u64, size)
1411{
Matt Mullins9594dc32019-06-11 14:53:04 -07001412 struct pt_regs *regs = get_bpf_raw_tp_regs();
1413 int ret;
1414
1415 if (IS_ERR(regs))
1416 return PTR_ERR(regs);
Alexei Starovoitovc4f66992018-03-28 12:05:37 -07001417
1418 perf_fetch_caller_regs(regs);
Matt Mullins9594dc32019-06-11 14:53:04 -07001419 ret = ____bpf_perf_event_output(regs, map, flags, data, size);
1420
1421 put_bpf_raw_tp_regs();
1422 return ret;
Alexei Starovoitovc4f66992018-03-28 12:05:37 -07001423}
1424
1425static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = {
1426 .func = bpf_perf_event_output_raw_tp,
1427 .gpl_only = true,
1428 .ret_type = RET_INTEGER,
1429 .arg1_type = ARG_PTR_TO_CTX,
1430 .arg2_type = ARG_CONST_MAP_PTR,
1431 .arg3_type = ARG_ANYTHING,
1432 .arg4_type = ARG_PTR_TO_MEM,
1433 .arg5_type = ARG_CONST_SIZE_OR_ZERO,
1434};
1435
Alexei Starovoitova7658e12019-10-15 20:25:04 -07001436extern const struct bpf_func_proto bpf_skb_output_proto;
Eelco Chaudrond831ee82020-03-06 08:59:23 +00001437extern const struct bpf_func_proto bpf_xdp_output_proto;
Alexei Starovoitova7658e12019-10-15 20:25:04 -07001438
Alexei Starovoitovc4f66992018-03-28 12:05:37 -07001439BPF_CALL_3(bpf_get_stackid_raw_tp, struct bpf_raw_tracepoint_args *, args,
1440 struct bpf_map *, map, u64, flags)
1441{
Matt Mullins9594dc32019-06-11 14:53:04 -07001442 struct pt_regs *regs = get_bpf_raw_tp_regs();
1443 int ret;
1444
1445 if (IS_ERR(regs))
1446 return PTR_ERR(regs);
Alexei Starovoitovc4f66992018-03-28 12:05:37 -07001447
1448 perf_fetch_caller_regs(regs);
1449 /* similar to bpf_perf_event_output_tp, but pt_regs fetched differently */
Matt Mullins9594dc32019-06-11 14:53:04 -07001450 ret = bpf_get_stackid((unsigned long) regs, (unsigned long) map,
1451 flags, 0, 0);
1452 put_bpf_raw_tp_regs();
1453 return ret;
Alexei Starovoitovc4f66992018-03-28 12:05:37 -07001454}
1455
1456static const struct bpf_func_proto bpf_get_stackid_proto_raw_tp = {
1457 .func = bpf_get_stackid_raw_tp,
1458 .gpl_only = true,
1459 .ret_type = RET_INTEGER,
1460 .arg1_type = ARG_PTR_TO_CTX,
1461 .arg2_type = ARG_CONST_MAP_PTR,
1462 .arg3_type = ARG_ANYTHING,
1463};
1464
Yonghong Songc195651e2018-04-28 22:28:08 -07001465BPF_CALL_4(bpf_get_stack_raw_tp, struct bpf_raw_tracepoint_args *, args,
1466 void *, buf, u32, size, u64, flags)
1467{
Matt Mullins9594dc32019-06-11 14:53:04 -07001468 struct pt_regs *regs = get_bpf_raw_tp_regs();
1469 int ret;
1470
1471 if (IS_ERR(regs))
1472 return PTR_ERR(regs);
Yonghong Songc195651e2018-04-28 22:28:08 -07001473
1474 perf_fetch_caller_regs(regs);
Matt Mullins9594dc32019-06-11 14:53:04 -07001475 ret = bpf_get_stack((unsigned long) regs, (unsigned long) buf,
1476 (unsigned long) size, flags, 0);
1477 put_bpf_raw_tp_regs();
1478 return ret;
Yonghong Songc195651e2018-04-28 22:28:08 -07001479}
1480
1481static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = {
1482 .func = bpf_get_stack_raw_tp,
1483 .gpl_only = true,
1484 .ret_type = RET_INTEGER,
1485 .arg1_type = ARG_PTR_TO_CTX,
1486 .arg2_type = ARG_PTR_TO_MEM,
1487 .arg3_type = ARG_CONST_SIZE_OR_ZERO,
1488 .arg4_type = ARG_ANYTHING,
1489};
1490
Andrey Ignatov5e43f892018-03-30 15:08:00 -07001491static const struct bpf_func_proto *
1492raw_tp_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
Alexei Starovoitovc4f66992018-03-28 12:05:37 -07001493{
1494 switch (func_id) {
1495 case BPF_FUNC_perf_event_output:
1496 return &bpf_perf_event_output_proto_raw_tp;
1497 case BPF_FUNC_get_stackid:
1498 return &bpf_get_stackid_proto_raw_tp;
Yonghong Songc195651e2018-04-28 22:28:08 -07001499 case BPF_FUNC_get_stack:
1500 return &bpf_get_stack_proto_raw_tp;
Alexei Starovoitovc4f66992018-03-28 12:05:37 -07001501 default:
KP Singhfc611f42020-03-29 01:43:49 +01001502 return bpf_tracing_func_proto(func_id, prog);
Alexei Starovoitovc4f66992018-03-28 12:05:37 -07001503 }
1504}
1505
Jiri Olsa958a3f22020-05-31 17:42:55 +02001506const struct bpf_func_proto *
Alexei Starovoitovf1b95092019-10-30 15:32:11 -07001507tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1508{
1509 switch (func_id) {
1510#ifdef CONFIG_NET
1511 case BPF_FUNC_skb_output:
1512 return &bpf_skb_output_proto;
Eelco Chaudrond831ee82020-03-06 08:59:23 +00001513 case BPF_FUNC_xdp_output:
1514 return &bpf_xdp_output_proto;
Alexei Starovoitovf1b95092019-10-30 15:32:11 -07001515#endif
Yonghong Song492e6392020-05-09 10:59:14 -07001516 case BPF_FUNC_seq_printf:
1517 return prog->expected_attach_type == BPF_TRACE_ITER ?
1518 &bpf_seq_printf_proto :
1519 NULL;
1520 case BPF_FUNC_seq_write:
1521 return prog->expected_attach_type == BPF_TRACE_ITER ?
1522 &bpf_seq_write_proto :
1523 NULL;
Alexei Starovoitovf1b95092019-10-30 15:32:11 -07001524 default:
1525 return raw_tp_prog_func_proto(func_id, prog);
1526 }
1527}
1528
Alexei Starovoitovc4f66992018-03-28 12:05:37 -07001529static bool raw_tp_prog_is_valid_access(int off, int size,
1530 enum bpf_access_type type,
Andrey Ignatov5e43f892018-03-30 15:08:00 -07001531 const struct bpf_prog *prog,
Alexei Starovoitovc4f66992018-03-28 12:05:37 -07001532 struct bpf_insn_access_aux *info)
1533{
Alexei Starovoitovf1b95092019-10-30 15:32:11 -07001534 if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS)
Alexei Starovoitovc4f66992018-03-28 12:05:37 -07001535 return false;
1536 if (type != BPF_READ)
1537 return false;
1538 if (off % size != 0)
1539 return false;
Alexei Starovoitovf1b95092019-10-30 15:32:11 -07001540 return true;
1541}
1542
1543static bool tracing_prog_is_valid_access(int off, int size,
1544 enum bpf_access_type type,
1545 const struct bpf_prog *prog,
1546 struct bpf_insn_access_aux *info)
1547{
1548 if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS)
1549 return false;
1550 if (type != BPF_READ)
1551 return false;
1552 if (off % size != 0)
1553 return false;
Alexei Starovoitov9e15db62019-10-15 20:25:00 -07001554 return btf_ctx_access(off, size, type, prog, info);
Alexei Starovoitovc4f66992018-03-28 12:05:37 -07001555}
1556
KP Singh3e7c67d2020-03-05 23:01:27 +01001557int __weak bpf_prog_test_run_tracing(struct bpf_prog *prog,
1558 const union bpf_attr *kattr,
1559 union bpf_attr __user *uattr)
1560{
1561 return -ENOTSUPP;
1562}
1563
Alexei Starovoitovc4f66992018-03-28 12:05:37 -07001564const struct bpf_verifier_ops raw_tracepoint_verifier_ops = {
1565 .get_func_proto = raw_tp_prog_func_proto,
1566 .is_valid_access = raw_tp_prog_is_valid_access,
1567};
1568
1569const struct bpf_prog_ops raw_tracepoint_prog_ops = {
1570};
1571
Alexei Starovoitovf1b95092019-10-30 15:32:11 -07001572const struct bpf_verifier_ops tracing_verifier_ops = {
1573 .get_func_proto = tracing_prog_func_proto,
1574 .is_valid_access = tracing_prog_is_valid_access,
1575};
1576
1577const struct bpf_prog_ops tracing_prog_ops = {
KP Singhda00d2f2020-03-04 20:18:52 +01001578 .test_run = bpf_prog_test_run_tracing,
Alexei Starovoitovf1b95092019-10-30 15:32:11 -07001579};
1580
Matt Mullins9df1c282019-04-26 11:49:47 -07001581static bool raw_tp_writable_prog_is_valid_access(int off, int size,
1582 enum bpf_access_type type,
1583 const struct bpf_prog *prog,
1584 struct bpf_insn_access_aux *info)
1585{
1586 if (off == 0) {
1587 if (size != sizeof(u64) || type != BPF_READ)
1588 return false;
1589 info->reg_type = PTR_TO_TP_BUFFER;
1590 }
1591 return raw_tp_prog_is_valid_access(off, size, type, prog, info);
1592}
1593
1594const struct bpf_verifier_ops raw_tracepoint_writable_verifier_ops = {
1595 .get_func_proto = raw_tp_prog_func_proto,
1596 .is_valid_access = raw_tp_writable_prog_is_valid_access,
1597};
1598
1599const struct bpf_prog_ops raw_tracepoint_writable_prog_ops = {
1600};
1601
Alexei Starovoitov0515e592016-09-01 18:37:22 -07001602static bool pe_prog_is_valid_access(int off, int size, enum bpf_access_type type,
Andrey Ignatov5e43f892018-03-30 15:08:00 -07001603 const struct bpf_prog *prog,
Yonghong Song23994632017-06-22 15:07:39 -07001604 struct bpf_insn_access_aux *info)
Alexei Starovoitov0515e592016-09-01 18:37:22 -07001605{
Teng Qin95da0cd2018-03-06 10:55:01 -08001606 const int size_u64 = sizeof(u64);
Yonghong Song31fd8582017-06-13 15:52:13 -07001607
Alexei Starovoitov0515e592016-09-01 18:37:22 -07001608 if (off < 0 || off >= sizeof(struct bpf_perf_event_data))
1609 return false;
1610 if (type != BPF_READ)
1611 return false;
Daniel Borkmannbc231052018-06-02 23:06:39 +02001612 if (off % size != 0) {
1613 if (sizeof(unsigned long) != 4)
1614 return false;
1615 if (size != 8)
1616 return false;
1617 if (off % size != 4)
1618 return false;
1619 }
Yonghong Song31fd8582017-06-13 15:52:13 -07001620
Daniel Borkmannf96da092017-07-02 02:13:27 +02001621 switch (off) {
1622 case bpf_ctx_range(struct bpf_perf_event_data, sample_period):
Teng Qin95da0cd2018-03-06 10:55:01 -08001623 bpf_ctx_record_field_size(info, size_u64);
1624 if (!bpf_ctx_narrow_access_ok(off, size, size_u64))
1625 return false;
1626 break;
1627 case bpf_ctx_range(struct bpf_perf_event_data, addr):
1628 bpf_ctx_record_field_size(info, size_u64);
1629 if (!bpf_ctx_narrow_access_ok(off, size, size_u64))
Yonghong Song23994632017-06-22 15:07:39 -07001630 return false;
Daniel Borkmannf96da092017-07-02 02:13:27 +02001631 break;
1632 default:
Alexei Starovoitov0515e592016-09-01 18:37:22 -07001633 if (size != sizeof(long))
1634 return false;
1635 }
Daniel Borkmannf96da092017-07-02 02:13:27 +02001636
Alexei Starovoitov0515e592016-09-01 18:37:22 -07001637 return true;
1638}
1639
Daniel Borkmann6b8cc1d2017-01-12 11:51:32 +01001640static u32 pe_prog_convert_ctx_access(enum bpf_access_type type,
1641 const struct bpf_insn *si,
Alexei Starovoitov0515e592016-09-01 18:37:22 -07001642 struct bpf_insn *insn_buf,
Daniel Borkmannf96da092017-07-02 02:13:27 +02001643 struct bpf_prog *prog, u32 *target_size)
Alexei Starovoitov0515e592016-09-01 18:37:22 -07001644{
1645 struct bpf_insn *insn = insn_buf;
1646
Daniel Borkmann6b8cc1d2017-01-12 11:51:32 +01001647 switch (si->off) {
Alexei Starovoitov0515e592016-09-01 18:37:22 -07001648 case offsetof(struct bpf_perf_event_data, sample_period):
Daniel Borkmannf035a512016-09-09 02:45:29 +02001649 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern,
Daniel Borkmann6b8cc1d2017-01-12 11:51:32 +01001650 data), si->dst_reg, si->src_reg,
Alexei Starovoitov0515e592016-09-01 18:37:22 -07001651 offsetof(struct bpf_perf_event_data_kern, data));
Daniel Borkmann6b8cc1d2017-01-12 11:51:32 +01001652 *insn++ = BPF_LDX_MEM(BPF_DW, si->dst_reg, si->dst_reg,
Daniel Borkmannf96da092017-07-02 02:13:27 +02001653 bpf_target_off(struct perf_sample_data, period, 8,
1654 target_size));
Alexei Starovoitov0515e592016-09-01 18:37:22 -07001655 break;
Teng Qin95da0cd2018-03-06 10:55:01 -08001656 case offsetof(struct bpf_perf_event_data, addr):
1657 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern,
1658 data), si->dst_reg, si->src_reg,
1659 offsetof(struct bpf_perf_event_data_kern, data));
1660 *insn++ = BPF_LDX_MEM(BPF_DW, si->dst_reg, si->dst_reg,
1661 bpf_target_off(struct perf_sample_data, addr, 8,
1662 target_size));
1663 break;
Alexei Starovoitov0515e592016-09-01 18:37:22 -07001664 default:
Daniel Borkmannf035a512016-09-09 02:45:29 +02001665 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern,
Daniel Borkmann6b8cc1d2017-01-12 11:51:32 +01001666 regs), si->dst_reg, si->src_reg,
Alexei Starovoitov0515e592016-09-01 18:37:22 -07001667 offsetof(struct bpf_perf_event_data_kern, regs));
Daniel Borkmann6b8cc1d2017-01-12 11:51:32 +01001668 *insn++ = BPF_LDX_MEM(BPF_SIZEOF(long), si->dst_reg, si->dst_reg,
1669 si->off);
Alexei Starovoitov0515e592016-09-01 18:37:22 -07001670 break;
1671 }
1672
1673 return insn - insn_buf;
1674}
1675
Jakub Kicinski7de16e32017-10-16 16:40:53 -07001676const struct bpf_verifier_ops perf_event_verifier_ops = {
Yonghong Songf005afe2018-03-20 11:19:17 -07001677 .get_func_proto = pe_prog_func_proto,
Alexei Starovoitov0515e592016-09-01 18:37:22 -07001678 .is_valid_access = pe_prog_is_valid_access,
1679 .convert_ctx_access = pe_prog_convert_ctx_access,
1680};
Jakub Kicinski7de16e32017-10-16 16:40:53 -07001681
1682const struct bpf_prog_ops perf_event_prog_ops = {
1683};
Yonghong Songe87c6bc2017-10-23 23:53:08 -07001684
1685static DEFINE_MUTEX(bpf_event_mutex);
1686
Yonghong Songc8c088b2017-11-30 13:47:54 -08001687#define BPF_TRACE_MAX_PROGS 64
1688
Yonghong Songe87c6bc2017-10-23 23:53:08 -07001689int perf_event_attach_bpf_prog(struct perf_event *event,
1690 struct bpf_prog *prog)
1691{
Stanislav Fomicheve672db02019-05-28 14:14:44 -07001692 struct bpf_prog_array *old_array;
Yonghong Songe87c6bc2017-10-23 23:53:08 -07001693 struct bpf_prog_array *new_array;
1694 int ret = -EEXIST;
1695
Josef Bacik9802d862017-12-11 11:36:48 -05001696 /*
Masami Hiramatsub4da3342018-01-13 02:54:04 +09001697 * Kprobe override only works if they are on the function entry,
1698 * and only if they are on the opt-in list.
Josef Bacik9802d862017-12-11 11:36:48 -05001699 */
1700 if (prog->kprobe_override &&
Masami Hiramatsub4da3342018-01-13 02:54:04 +09001701 (!trace_kprobe_on_func_entry(event->tp_event) ||
Josef Bacik9802d862017-12-11 11:36:48 -05001702 !trace_kprobe_error_injectable(event->tp_event)))
1703 return -EINVAL;
1704
Yonghong Songe87c6bc2017-10-23 23:53:08 -07001705 mutex_lock(&bpf_event_mutex);
1706
1707 if (event->prog)
Yonghong Song07c41a22017-10-30 13:50:22 -07001708 goto unlock;
Yonghong Songe87c6bc2017-10-23 23:53:08 -07001709
Stanislav Fomicheve672db02019-05-28 14:14:44 -07001710 old_array = bpf_event_rcu_dereference(event->tp_event->prog_array);
Yonghong Songc8c088b2017-11-30 13:47:54 -08001711 if (old_array &&
1712 bpf_prog_array_length(old_array) >= BPF_TRACE_MAX_PROGS) {
1713 ret = -E2BIG;
1714 goto unlock;
1715 }
1716
Yonghong Songe87c6bc2017-10-23 23:53:08 -07001717 ret = bpf_prog_array_copy(old_array, NULL, prog, &new_array);
1718 if (ret < 0)
Yonghong Song07c41a22017-10-30 13:50:22 -07001719 goto unlock;
Yonghong Songe87c6bc2017-10-23 23:53:08 -07001720
1721 /* set the new array to event->tp_event and set event->prog */
1722 event->prog = prog;
1723 rcu_assign_pointer(event->tp_event->prog_array, new_array);
1724 bpf_prog_array_free(old_array);
1725
Yonghong Song07c41a22017-10-30 13:50:22 -07001726unlock:
Yonghong Songe87c6bc2017-10-23 23:53:08 -07001727 mutex_unlock(&bpf_event_mutex);
1728 return ret;
1729}
1730
1731void perf_event_detach_bpf_prog(struct perf_event *event)
1732{
Stanislav Fomicheve672db02019-05-28 14:14:44 -07001733 struct bpf_prog_array *old_array;
Yonghong Songe87c6bc2017-10-23 23:53:08 -07001734 struct bpf_prog_array *new_array;
1735 int ret;
1736
1737 mutex_lock(&bpf_event_mutex);
1738
1739 if (!event->prog)
Yonghong Song07c41a22017-10-30 13:50:22 -07001740 goto unlock;
Yonghong Songe87c6bc2017-10-23 23:53:08 -07001741
Stanislav Fomicheve672db02019-05-28 14:14:44 -07001742 old_array = bpf_event_rcu_dereference(event->tp_event->prog_array);
Yonghong Songe87c6bc2017-10-23 23:53:08 -07001743 ret = bpf_prog_array_copy(old_array, event->prog, NULL, &new_array);
Sean Young170a7e32018-05-27 12:24:08 +01001744 if (ret == -ENOENT)
1745 goto unlock;
Yonghong Songe87c6bc2017-10-23 23:53:08 -07001746 if (ret < 0) {
1747 bpf_prog_array_delete_safe(old_array, event->prog);
1748 } else {
1749 rcu_assign_pointer(event->tp_event->prog_array, new_array);
1750 bpf_prog_array_free(old_array);
1751 }
1752
1753 bpf_prog_put(event->prog);
1754 event->prog = NULL;
1755
Yonghong Song07c41a22017-10-30 13:50:22 -07001756unlock:
Yonghong Songe87c6bc2017-10-23 23:53:08 -07001757 mutex_unlock(&bpf_event_mutex);
1758}
Yonghong Songf371b302017-12-11 11:39:02 -08001759
Yonghong Songf4e22982017-12-13 10:35:37 -08001760int perf_event_query_prog_array(struct perf_event *event, void __user *info)
Yonghong Songf371b302017-12-11 11:39:02 -08001761{
1762 struct perf_event_query_bpf __user *uquery = info;
1763 struct perf_event_query_bpf query = {};
Stanislav Fomicheve672db02019-05-28 14:14:44 -07001764 struct bpf_prog_array *progs;
Yonghong Song3a38bb92018-04-10 09:37:32 -07001765 u32 *ids, prog_cnt, ids_len;
Yonghong Songf371b302017-12-11 11:39:02 -08001766 int ret;
1767
Alexey Budankov031258d2020-04-02 11:48:54 +03001768 if (!perfmon_capable())
Yonghong Songf371b302017-12-11 11:39:02 -08001769 return -EPERM;
1770 if (event->attr.type != PERF_TYPE_TRACEPOINT)
1771 return -EINVAL;
1772 if (copy_from_user(&query, uquery, sizeof(query)))
1773 return -EFAULT;
Yonghong Song3a38bb92018-04-10 09:37:32 -07001774
1775 ids_len = query.ids_len;
1776 if (ids_len > BPF_TRACE_MAX_PROGS)
Daniel Borkmann9c481b92018-02-14 15:31:00 +01001777 return -E2BIG;
Yonghong Song3a38bb92018-04-10 09:37:32 -07001778 ids = kcalloc(ids_len, sizeof(u32), GFP_USER | __GFP_NOWARN);
1779 if (!ids)
1780 return -ENOMEM;
1781 /*
1782 * The above kcalloc returns ZERO_SIZE_PTR when ids_len = 0, which
1783 * is required when user only wants to check for uquery->prog_cnt.
1784 * There is no need to check for it since the case is handled
1785 * gracefully in bpf_prog_array_copy_info.
1786 */
Yonghong Songf371b302017-12-11 11:39:02 -08001787
1788 mutex_lock(&bpf_event_mutex);
Stanislav Fomicheve672db02019-05-28 14:14:44 -07001789 progs = bpf_event_rcu_dereference(event->tp_event->prog_array);
1790 ret = bpf_prog_array_copy_info(progs, ids, ids_len, &prog_cnt);
Yonghong Songf371b302017-12-11 11:39:02 -08001791 mutex_unlock(&bpf_event_mutex);
1792
Yonghong Song3a38bb92018-04-10 09:37:32 -07001793 if (copy_to_user(&uquery->prog_cnt, &prog_cnt, sizeof(prog_cnt)) ||
1794 copy_to_user(uquery->ids, ids, ids_len * sizeof(u32)))
1795 ret = -EFAULT;
1796
1797 kfree(ids);
Yonghong Songf371b302017-12-11 11:39:02 -08001798 return ret;
1799}
Alexei Starovoitovc4f66992018-03-28 12:05:37 -07001800
1801extern struct bpf_raw_event_map __start__bpf_raw_tp[];
1802extern struct bpf_raw_event_map __stop__bpf_raw_tp[];
1803
Matt Mullinsa38d1102018-12-12 16:42:37 -08001804struct bpf_raw_event_map *bpf_get_raw_tracepoint(const char *name)
Alexei Starovoitovc4f66992018-03-28 12:05:37 -07001805{
1806 struct bpf_raw_event_map *btp = __start__bpf_raw_tp;
1807
1808 for (; btp < __stop__bpf_raw_tp; btp++) {
1809 if (!strcmp(btp->tp->name, name))
1810 return btp;
1811 }
Matt Mullinsa38d1102018-12-12 16:42:37 -08001812
1813 return bpf_get_raw_tracepoint_module(name);
1814}
1815
1816void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp)
1817{
1818 struct module *mod = __module_address((unsigned long)btp);
1819
1820 if (mod)
1821 module_put(mod);
Alexei Starovoitovc4f66992018-03-28 12:05:37 -07001822}
1823
1824static __always_inline
1825void __bpf_trace_run(struct bpf_prog *prog, u64 *args)
1826{
Thomas Gleixnerf03efe42020-02-24 15:01:35 +01001827 cant_sleep();
Alexei Starovoitovc4f66992018-03-28 12:05:37 -07001828 rcu_read_lock();
Alexei Starovoitovc4f66992018-03-28 12:05:37 -07001829 (void) BPF_PROG_RUN(prog, args);
Alexei Starovoitovc4f66992018-03-28 12:05:37 -07001830 rcu_read_unlock();
1831}
1832
1833#define UNPACK(...) __VA_ARGS__
1834#define REPEAT_1(FN, DL, X, ...) FN(X)
1835#define REPEAT_2(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_1(FN, DL, __VA_ARGS__)
1836#define REPEAT_3(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_2(FN, DL, __VA_ARGS__)
1837#define REPEAT_4(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_3(FN, DL, __VA_ARGS__)
1838#define REPEAT_5(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_4(FN, DL, __VA_ARGS__)
1839#define REPEAT_6(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_5(FN, DL, __VA_ARGS__)
1840#define REPEAT_7(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_6(FN, DL, __VA_ARGS__)
1841#define REPEAT_8(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_7(FN, DL, __VA_ARGS__)
1842#define REPEAT_9(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_8(FN, DL, __VA_ARGS__)
1843#define REPEAT_10(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_9(FN, DL, __VA_ARGS__)
1844#define REPEAT_11(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_10(FN, DL, __VA_ARGS__)
1845#define REPEAT_12(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_11(FN, DL, __VA_ARGS__)
1846#define REPEAT(X, FN, DL, ...) REPEAT_##X(FN, DL, __VA_ARGS__)
1847
1848#define SARG(X) u64 arg##X
1849#define COPY(X) args[X] = arg##X
1850
1851#define __DL_COM (,)
1852#define __DL_SEM (;)
1853
1854#define __SEQ_0_11 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11
1855
1856#define BPF_TRACE_DEFN_x(x) \
1857 void bpf_trace_run##x(struct bpf_prog *prog, \
1858 REPEAT(x, SARG, __DL_COM, __SEQ_0_11)) \
1859 { \
1860 u64 args[x]; \
1861 REPEAT(x, COPY, __DL_SEM, __SEQ_0_11); \
1862 __bpf_trace_run(prog, args); \
1863 } \
1864 EXPORT_SYMBOL_GPL(bpf_trace_run##x)
1865BPF_TRACE_DEFN_x(1);
1866BPF_TRACE_DEFN_x(2);
1867BPF_TRACE_DEFN_x(3);
1868BPF_TRACE_DEFN_x(4);
1869BPF_TRACE_DEFN_x(5);
1870BPF_TRACE_DEFN_x(6);
1871BPF_TRACE_DEFN_x(7);
1872BPF_TRACE_DEFN_x(8);
1873BPF_TRACE_DEFN_x(9);
1874BPF_TRACE_DEFN_x(10);
1875BPF_TRACE_DEFN_x(11);
1876BPF_TRACE_DEFN_x(12);
1877
1878static int __bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *prog)
1879{
1880 struct tracepoint *tp = btp->tp;
1881
1882 /*
1883 * check that program doesn't access arguments beyond what's
1884 * available in this tracepoint
1885 */
1886 if (prog->aux->max_ctx_offset > btp->num_args * sizeof(u64))
1887 return -EINVAL;
1888
Matt Mullins9df1c282019-04-26 11:49:47 -07001889 if (prog->aux->max_tp_access > btp->writable_size)
1890 return -EINVAL;
1891
Alexei Starovoitovc4f66992018-03-28 12:05:37 -07001892 return tracepoint_probe_register(tp, (void *)btp->bpf_func, prog);
1893}
1894
1895int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *prog)
1896{
Alexei Starovoitove16ec342019-01-30 18:12:44 -08001897 return __bpf_probe_register(btp, prog);
Alexei Starovoitovc4f66992018-03-28 12:05:37 -07001898}
1899
1900int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_prog *prog)
1901{
Alexei Starovoitove16ec342019-01-30 18:12:44 -08001902 return tracepoint_probe_unregister(btp->tp, (void *)btp->bpf_func, prog);
Alexei Starovoitovc4f66992018-03-28 12:05:37 -07001903}
Yonghong Song41bdc4b2018-05-24 11:21:09 -07001904
1905int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id,
1906 u32 *fd_type, const char **buf,
1907 u64 *probe_offset, u64 *probe_addr)
1908{
1909 bool is_tracepoint, is_syscall_tp;
1910 struct bpf_prog *prog;
1911 int flags, err = 0;
1912
1913 prog = event->prog;
1914 if (!prog)
1915 return -ENOENT;
1916
1917 /* not supporting BPF_PROG_TYPE_PERF_EVENT yet */
1918 if (prog->type == BPF_PROG_TYPE_PERF_EVENT)
1919 return -EOPNOTSUPP;
1920
1921 *prog_id = prog->aux->id;
1922 flags = event->tp_event->flags;
1923 is_tracepoint = flags & TRACE_EVENT_FL_TRACEPOINT;
1924 is_syscall_tp = is_syscall_trace_event(event->tp_event);
1925
1926 if (is_tracepoint || is_syscall_tp) {
1927 *buf = is_tracepoint ? event->tp_event->tp->name
1928 : event->tp_event->name;
1929 *fd_type = BPF_FD_TYPE_TRACEPOINT;
1930 *probe_offset = 0x0;
1931 *probe_addr = 0x0;
1932 } else {
1933 /* kprobe/uprobe */
1934 err = -EOPNOTSUPP;
1935#ifdef CONFIG_KPROBE_EVENTS
1936 if (flags & TRACE_EVENT_FL_KPROBE)
1937 err = bpf_get_kprobe_info(event, fd_type, buf,
1938 probe_offset, probe_addr,
1939 event->attr.type == PERF_TYPE_TRACEPOINT);
1940#endif
1941#ifdef CONFIG_UPROBE_EVENTS
1942 if (flags & TRACE_EVENT_FL_UPROBE)
1943 err = bpf_get_uprobe_info(event, fd_type, buf,
1944 probe_offset,
1945 event->attr.type == PERF_TYPE_TRACEPOINT);
1946#endif
1947 }
1948
1949 return err;
1950}
Matt Mullinsa38d1102018-12-12 16:42:37 -08001951
Yonghong Song9db1ff02019-06-25 17:35:03 -07001952static int __init send_signal_irq_work_init(void)
1953{
1954 int cpu;
1955 struct send_signal_irq_work *work;
1956
1957 for_each_possible_cpu(cpu) {
1958 work = per_cpu_ptr(&send_signal_work, cpu);
1959 init_irq_work(&work->irq_work, do_bpf_send_signal);
1960 }
1961 return 0;
1962}
1963
1964subsys_initcall(send_signal_irq_work_init);
1965
Matt Mullinsa38d1102018-12-12 16:42:37 -08001966#ifdef CONFIG_MODULES
Stanislav Fomichev390e99c2019-05-13 12:04:36 -07001967static int bpf_event_notify(struct notifier_block *nb, unsigned long op,
1968 void *module)
Matt Mullinsa38d1102018-12-12 16:42:37 -08001969{
1970 struct bpf_trace_module *btm, *tmp;
1971 struct module *mod = module;
1972
1973 if (mod->num_bpf_raw_events == 0 ||
1974 (op != MODULE_STATE_COMING && op != MODULE_STATE_GOING))
1975 return 0;
1976
1977 mutex_lock(&bpf_module_mutex);
1978
1979 switch (op) {
1980 case MODULE_STATE_COMING:
1981 btm = kzalloc(sizeof(*btm), GFP_KERNEL);
1982 if (btm) {
1983 btm->module = module;
1984 list_add(&btm->list, &bpf_trace_modules);
1985 }
1986 break;
1987 case MODULE_STATE_GOING:
1988 list_for_each_entry_safe(btm, tmp, &bpf_trace_modules, list) {
1989 if (btm->module == module) {
1990 list_del(&btm->list);
1991 kfree(btm);
1992 break;
1993 }
1994 }
1995 break;
1996 }
1997
1998 mutex_unlock(&bpf_module_mutex);
1999
2000 return 0;
2001}
2002
2003static struct notifier_block bpf_module_nb = {
2004 .notifier_call = bpf_event_notify,
2005};
2006
Stanislav Fomichev390e99c2019-05-13 12:04:36 -07002007static int __init bpf_event_init(void)
Matt Mullinsa38d1102018-12-12 16:42:37 -08002008{
2009 register_module_notifier(&bpf_module_nb);
2010 return 0;
2011}
2012
2013fs_initcall(bpf_event_init);
2014#endif /* CONFIG_MODULES */