blob: 8892b45b436873068148b99b64f7c93759d45f88 [file] [log] [blame]
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +02001/*
2 * Infrastructure for profiling code inserted by 'gcc -pg'.
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally ported from the -rt patch by:
8 * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code in the latency_tracer, that is:
11 *
12 * Copyright (C) 2004-2006 Ingo Molnar
Nadia Yvette Chambers6d49e352012-12-06 10:39:54 +010013 * Copyright (C) 2004 Nadia Yvette Chambers
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +020014 */
15
Steven Rostedt3d083392008-05-12 21:20:42 +020016#include <linux/stop_machine.h>
17#include <linux/clocksource.h>
18#include <linux/kallsyms.h>
Steven Rostedt5072c592008-05-12 21:20:43 +020019#include <linux/seq_file.h>
Frederic Weisbecker4a2b8dd2009-01-14 13:33:27 -080020#include <linux/suspend.h>
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -050021#include <linux/tracefs.h>
Steven Rostedt3d083392008-05-12 21:20:42 +020022#include <linux/hardirq.h>
Ingo Molnar2d8b8202008-02-23 16:55:50 +010023#include <linux/kthread.h>
Steven Rostedt5072c592008-05-12 21:20:43 +020024#include <linux/uaccess.h>
Steven Rostedt5855fea2011-12-16 19:27:42 -050025#include <linux/bsearch.h>
Paul Gortmaker56d82e02011-05-26 17:53:52 -040026#include <linux/module.h>
Ingo Molnar2d8b8202008-02-23 16:55:50 +010027#include <linux/ftrace.h>
Steven Rostedtb0fc4942008-05-12 21:20:43 +020028#include <linux/sysctl.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090029#include <linux/slab.h>
Steven Rostedt5072c592008-05-12 21:20:43 +020030#include <linux/ctype.h>
Steven Rostedt68950612011-12-16 17:06:45 -050031#include <linux/sort.h>
Steven Rostedt3d083392008-05-12 21:20:42 +020032#include <linux/list.h>
Steven Rostedt59df055f2009-02-14 15:29:06 -050033#include <linux/hash.h>
Paul E. McKenney3f379b02010-03-05 15:03:25 -080034#include <linux/rcupdate.h>
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +020035
Steven Rostedtad8d75f2009-04-14 19:39:12 -040036#include <trace/events/sched.h>
Steven Rostedt8aef2d22009-03-24 01:10:15 -040037
Steven Rostedt2af15d62009-05-28 13:37:24 -040038#include <asm/setup.h>
Abhishek Sagar395a59d2008-06-21 23:47:27 +053039
Steven Rostedt0706f1c2009-03-23 23:12:58 -040040#include "trace_output.h"
Steven Rostedtbac429f2009-03-20 12:50:56 -040041#include "trace_stat.h"
Steven Rostedt3d083392008-05-12 21:20:42 +020042
Steven Rostedt6912896e2008-10-23 09:33:03 -040043#define FTRACE_WARN_ON(cond) \
Steven Rostedt0778d9a2011-04-29 10:36:31 -040044 ({ \
45 int ___r = cond; \
46 if (WARN_ON(___r)) \
Steven Rostedt6912896e2008-10-23 09:33:03 -040047 ftrace_kill(); \
Steven Rostedt0778d9a2011-04-29 10:36:31 -040048 ___r; \
49 })
Steven Rostedt6912896e2008-10-23 09:33:03 -040050
51#define FTRACE_WARN_ON_ONCE(cond) \
Steven Rostedt0778d9a2011-04-29 10:36:31 -040052 ({ \
53 int ___r = cond; \
54 if (WARN_ON_ONCE(___r)) \
Steven Rostedt6912896e2008-10-23 09:33:03 -040055 ftrace_kill(); \
Steven Rostedt0778d9a2011-04-29 10:36:31 -040056 ___r; \
57 })
Steven Rostedt6912896e2008-10-23 09:33:03 -040058
Steven Rostedt8fc0c702009-02-16 15:28:00 -050059/* hash bits for specific function selection */
60#define FTRACE_HASH_BITS 7
61#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
Steven Rostedt33dc9b12011-05-02 17:34:47 -040062#define FTRACE_HASH_DEFAULT_BITS 10
63#define FTRACE_HASH_MAX_BITS 12
Steven Rostedt8fc0c702009-02-16 15:28:00 -050064
Steven Rostedt (Red Hat)4104d322014-01-10 17:01:58 -050065#define FL_GLOBAL_CONTROL_MASK (FTRACE_OPS_FL_CONTROL)
Jiri Olsae2484912012-02-15 15:51:48 +010066
Masami Hiramatsuf04f24fb2013-05-09 14:44:17 +090067#ifdef CONFIG_DYNAMIC_FTRACE
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -040068#define INIT_OPS_HASH(opsname) \
69 .func_hash = &opsname.local_hash, \
70 .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
Steven Rostedt (Red Hat)5f151b22014-08-15 17:18:46 -040071#define ASSIGN_OPS_HASH(opsname, val) \
72 .func_hash = val, \
73 .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
Masami Hiramatsuf04f24fb2013-05-09 14:44:17 +090074#else
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -040075#define INIT_OPS_HASH(opsname)
Steven Rostedt (Red Hat)5f151b22014-08-15 17:18:46 -040076#define ASSIGN_OPS_HASH(opsname, val)
Masami Hiramatsuf04f24fb2013-05-09 14:44:17 +090077#endif
78
Steven Rostedt2f5f6ad2011-08-08 16:57:47 -040079static struct ftrace_ops ftrace_list_end __read_mostly = {
80 .func = ftrace_stub,
Steven Rostedt (Red Hat)395b97a2013-03-27 09:31:28 -040081 .flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB,
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -040082 INIT_OPS_HASH(ftrace_list_end)
Steven Rostedt2f5f6ad2011-08-08 16:57:47 -040083};
84
Steven Rostedt4eebcc82008-05-12 21:20:48 +020085/* ftrace_enabled is a method to turn ftrace on or off */
86int ftrace_enabled __read_mostly;
Steven Rostedtd61f82d2008-05-12 21:20:43 +020087static int last_ftrace_enabled;
Steven Rostedtb0fc4942008-05-12 21:20:43 +020088
Steven Rostedt2f5f6ad2011-08-08 16:57:47 -040089/* Current function tracing op */
90struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;
Steven Rostedt (Red Hat)405e1d82013-11-08 14:17:30 -050091/* What to set function_trace_op to */
92static struct ftrace_ops *set_function_trace_op;
Steven Rostedt60a7ecf2008-11-05 16:05:44 -050093
jolsa@redhat.com756d17e2009-10-13 16:33:52 -040094/* List for set_ftrace_pid's pids. */
95LIST_HEAD(ftrace_pids);
96struct ftrace_pid {
97 struct list_head list;
98 struct pid *pid;
99};
100
Steven Rostedt (Red Hat)e3eea142015-07-24 10:38:12 -0400101static bool ftrace_pids_enabled(void)
102{
103 return !list_empty(&ftrace_pids);
104}
105
106static void ftrace_update_trampoline(struct ftrace_ops *ops);
107
Steven Rostedt4eebcc82008-05-12 21:20:48 +0200108/*
109 * ftrace_disabled is set when an anomaly is discovered.
110 * ftrace_disabled is much stronger than ftrace_enabled.
111 */
112static int ftrace_disabled __read_mostly;
113
Steven Rostedt52baf112009-02-14 01:15:39 -0500114static DEFINE_MUTEX(ftrace_lock);
Steven Rostedtb0fc4942008-05-12 21:20:43 +0200115
Jiri Olsae2484912012-02-15 15:51:48 +0100116static struct ftrace_ops *ftrace_control_list __read_mostly = &ftrace_list_end;
Steven Rostedtb8489142011-05-04 09:27:52 -0400117static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end;
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200118ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
Steven Rostedt2b499382011-05-03 22:49:52 -0400119static struct ftrace_ops global_ops;
Jiri Olsae2484912012-02-15 15:51:48 +0100120static struct ftrace_ops control_ops;
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200121
Steven Rostedt (Red Hat)f1ff6342014-07-22 20:16:57 -0400122static void ftrace_ops_recurs_func(unsigned long ip, unsigned long parent_ip,
123 struct ftrace_ops *op, struct pt_regs *regs);
124
Steven Rostedt2f5f6ad2011-08-08 16:57:47 -0400125#if ARCH_SUPPORTS_FTRACE_OPS
126static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
Steven Rostedta1e2e312011-08-09 12:50:46 -0400127 struct ftrace_ops *op, struct pt_regs *regs);
Steven Rostedt2f5f6ad2011-08-08 16:57:47 -0400128#else
129/* See comment below, where ftrace_ops_list_func is defined */
130static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip);
131#define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops)
132#endif
Steven Rostedtb8489142011-05-04 09:27:52 -0400133
Steven Rostedt0a016402012-11-02 17:03:03 -0400134/*
135 * Traverse the ftrace_global_list, invoking all entries. The reason that we
Steven Rostedt1bb539c2013-05-28 14:38:43 -0400136 * can use rcu_dereference_raw_notrace() is that elements removed from this list
Steven Rostedt0a016402012-11-02 17:03:03 -0400137 * are simply leaked, so there is no need to interact with a grace-period
Steven Rostedt1bb539c2013-05-28 14:38:43 -0400138 * mechanism. The rcu_dereference_raw_notrace() calls are needed to handle
Steven Rostedt0a016402012-11-02 17:03:03 -0400139 * concurrent insertions into the ftrace_global_list.
140 *
141 * Silly Alpha and silly pointer-speculation compiler optimizations!
142 */
143#define do_for_each_ftrace_op(op, list) \
Steven Rostedt1bb539c2013-05-28 14:38:43 -0400144 op = rcu_dereference_raw_notrace(list); \
Steven Rostedt0a016402012-11-02 17:03:03 -0400145 do
146
147/*
148 * Optimized for just a single item in the list (as that is the normal case).
149 */
150#define while_for_each_ftrace_op(op) \
Steven Rostedt1bb539c2013-05-28 14:38:43 -0400151 while (likely(op = rcu_dereference_raw_notrace((op)->next)) && \
Steven Rostedt0a016402012-11-02 17:03:03 -0400152 unlikely((op) != &ftrace_list_end))
153
Masami Hiramatsuf04f24fb2013-05-09 14:44:17 +0900154static inline void ftrace_ops_init(struct ftrace_ops *ops)
155{
156#ifdef CONFIG_DYNAMIC_FTRACE
157 if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) {
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -0400158 mutex_init(&ops->local_hash.regex_lock);
159 ops->func_hash = &ops->local_hash;
Masami Hiramatsuf04f24fb2013-05-09 14:44:17 +0900160 ops->flags |= FTRACE_OPS_FL_INITIALIZED;
161 }
162#endif
163}
164
Steven Rostedtea701f12012-07-20 13:08:05 -0400165/**
166 * ftrace_nr_registered_ops - return number of ops registered
167 *
168 * Returns the number of ftrace_ops registered and tracing functions
169 */
170int ftrace_nr_registered_ops(void)
171{
172 struct ftrace_ops *ops;
173 int cnt = 0;
174
175 mutex_lock(&ftrace_lock);
176
177 for (ops = ftrace_ops_list;
178 ops != &ftrace_list_end; ops = ops->next)
179 cnt++;
180
181 mutex_unlock(&ftrace_lock);
182
183 return cnt;
184}
185
Steven Rostedt2f5f6ad2011-08-08 16:57:47 -0400186static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,
Steven Rostedta1e2e312011-08-09 12:50:46 -0400187 struct ftrace_ops *op, struct pt_regs *regs)
Steven Rostedtdf4fc312008-11-26 00:16:23 -0500188{
Steven Rostedt0ef8cde2008-12-03 15:36:58 -0500189 if (!test_tsk_trace_trace(current))
Steven Rostedtdf4fc312008-11-26 00:16:23 -0500190 return;
191
Steven Rostedt (Red Hat)e3eea142015-07-24 10:38:12 -0400192 op->saved_func(ip, parent_ip, op, regs);
Steven Rostedtdf4fc312008-11-26 00:16:23 -0500193}
194
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200195/**
Steven Rostedt3d083392008-05-12 21:20:42 +0200196 * clear_ftrace_function - reset the ftrace function
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200197 *
Steven Rostedt3d083392008-05-12 21:20:42 +0200198 * This NULLs the ftrace function and in essence stops
199 * tracing. There may be lag
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200200 */
Steven Rostedt3d083392008-05-12 21:20:42 +0200201void clear_ftrace_function(void)
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200202{
Steven Rostedt3d083392008-05-12 21:20:42 +0200203 ftrace_trace_function = ftrace_stub;
204}
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200205
Jiri Olsae2484912012-02-15 15:51:48 +0100206static void control_ops_disable_all(struct ftrace_ops *ops)
207{
208 int cpu;
209
210 for_each_possible_cpu(cpu)
211 *per_cpu_ptr(ops->disabled, cpu) = 1;
212}
213
214static int control_ops_alloc(struct ftrace_ops *ops)
215{
216 int __percpu *disabled;
217
218 disabled = alloc_percpu(int);
219 if (!disabled)
220 return -ENOMEM;
221
222 ops->disabled = disabled;
223 control_ops_disable_all(ops);
224 return 0;
225}
226
Steven Rostedt (Red Hat)405e1d82013-11-08 14:17:30 -0500227static void ftrace_sync(struct work_struct *work)
228{
229 /*
230 * This function is just a stub to implement a hard force
231 * of synchronize_sched(). This requires synchronizing
232 * tasks even in userspace and idle.
233 *
234 * Yes, function tracing is rude.
235 */
236}
237
238static void ftrace_sync_ipi(void *data)
239{
240 /* Probably not needed, but do it anyway */
241 smp_rmb();
242}
243
Steven Rostedt (Red Hat)23a8e842014-01-13 10:30:23 -0500244#ifdef CONFIG_FUNCTION_GRAPH_TRACER
245static void update_function_graph_func(void);
Steven Rostedt (Red Hat)55577202015-09-29 19:06:50 -0400246
247/* Both enabled by default (can be cleared by function_graph tracer flags */
248static bool fgraph_sleep_time = true;
249static bool fgraph_graph_time = true;
250
Steven Rostedt (Red Hat)23a8e842014-01-13 10:30:23 -0500251#else
252static inline void update_function_graph_func(void) { }
253#endif
254
Steven Rostedt (Red Hat)00ccbf22015-02-19 15:56:14 +0100255
256static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops)
257{
258 /*
259 * If this is a dynamic ops or we force list func,
260 * then it needs to call the list anyway.
261 */
262 if (ops->flags & FTRACE_OPS_FL_DYNAMIC || FTRACE_FORCE_LIST_FUNC)
263 return ftrace_ops_list_func;
264
265 return ftrace_ops_get_func(ops);
266}
267
Steven Rostedt2b499382011-05-03 22:49:52 -0400268static void update_ftrace_function(void)
269{
270 ftrace_func_t func;
271
Steven Rostedtcdbe61b2011-05-05 21:14:55 -0400272 /*
Steven Rostedt (Red Hat)f7aad4e2014-09-10 10:42:46 -0400273 * Prepare the ftrace_ops that the arch callback will use.
274 * If there's only one ftrace_ops registered, the ftrace_ops_list
275 * will point to the ops we want.
276 */
277 set_function_trace_op = ftrace_ops_list;
278
279 /* If there's no ftrace_ops registered, just call the stub function */
280 if (ftrace_ops_list == &ftrace_list_end) {
281 func = ftrace_stub;
282
283 /*
Steven Rostedtcdbe61b2011-05-05 21:14:55 -0400284 * If we are at the end of the list and this ops is
Steven Rostedt47409742012-07-20 11:04:44 -0400285 * recursion safe and not dynamic and the arch supports passing ops,
286 * then have the mcount trampoline call the function directly.
Steven Rostedtcdbe61b2011-05-05 21:14:55 -0400287 */
Steven Rostedt (Red Hat)f7aad4e2014-09-10 10:42:46 -0400288 } else if (ftrace_ops_list->next == &ftrace_list_end) {
Steven Rostedt (Red Hat)00ccbf22015-02-19 15:56:14 +0100289 func = ftrace_ops_get_list_func(ftrace_ops_list);
Steven Rostedt (Red Hat)f7aad4e2014-09-10 10:42:46 -0400290
Steven Rostedt2f5f6ad2011-08-08 16:57:47 -0400291 } else {
292 /* Just use the default ftrace_ops */
Steven Rostedt (Red Hat)405e1d82013-11-08 14:17:30 -0500293 set_function_trace_op = &ftrace_list_end;
Steven Rostedtb8489142011-05-04 09:27:52 -0400294 func = ftrace_ops_list_func;
Steven Rostedt2f5f6ad2011-08-08 16:57:47 -0400295 }
Steven Rostedt2b499382011-05-03 22:49:52 -0400296
Steven Rostedt (Red Hat)5f8bf2d22014-07-15 11:05:12 -0400297 update_function_graph_func();
298
Steven Rostedt (Red Hat)405e1d82013-11-08 14:17:30 -0500299 /* If there's no change, then do nothing more here */
300 if (ftrace_trace_function == func)
301 return;
302
303 /*
304 * If we are using the list function, it doesn't care
305 * about the function_trace_ops.
306 */
307 if (func == ftrace_ops_list_func) {
308 ftrace_trace_function = func;
309 /*
310 * Don't even bother setting function_trace_ops,
311 * it would be racy to do so anyway.
312 */
313 return;
314 }
315
316#ifndef CONFIG_DYNAMIC_FTRACE
317 /*
318 * For static tracing, we need to be a bit more careful.
319 * The function change takes affect immediately. Thus,
320 * we need to coorditate the setting of the function_trace_ops
321 * with the setting of the ftrace_trace_function.
322 *
323 * Set the function to the list ops, which will call the
324 * function we want, albeit indirectly, but it handles the
325 * ftrace_ops and doesn't depend on function_trace_op.
326 */
327 ftrace_trace_function = ftrace_ops_list_func;
328 /*
329 * Make sure all CPUs see this. Yes this is slow, but static
330 * tracing is slow and nasty to have enabled.
331 */
332 schedule_on_each_cpu(ftrace_sync);
333 /* Now all cpus are using the list ops. */
334 function_trace_op = set_function_trace_op;
335 /* Make sure the function_trace_op is visible on all CPUs */
336 smp_wmb();
337 /* Nasty way to force a rmb on all cpus */
338 smp_call_function(ftrace_sync_ipi, NULL, 1);
339 /* OK, we are all set to update the ftrace_trace_function now! */
340#endif /* !CONFIG_DYNAMIC_FTRACE */
341
Steven Rostedt491d0dc2011-04-27 21:43:36 -0400342 ftrace_trace_function = func;
Steven Rostedt491d0dc2011-04-27 21:43:36 -0400343}
344
Jiaxing Wang7eea4fc2014-04-20 23:10:43 +0800345int using_ftrace_ops_list_func(void)
346{
347 return ftrace_trace_function == ftrace_ops_list_func;
348}
349
Steven Rostedt2b499382011-05-03 22:49:52 -0400350static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
Steven Rostedt3d083392008-05-12 21:20:42 +0200351{
Steven Rostedt2b499382011-05-03 22:49:52 -0400352 ops->next = *list;
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200353 /*
Steven Rostedtb8489142011-05-04 09:27:52 -0400354 * We are entering ops into the list but another
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200355 * CPU might be walking that list. We need to make sure
356 * the ops->next pointer is valid before another CPU sees
Steven Rostedtb8489142011-05-04 09:27:52 -0400357 * the ops pointer included into the list.
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200358 */
Steven Rostedt2b499382011-05-03 22:49:52 -0400359 rcu_assign_pointer(*list, ops);
360}
Steven Rostedt3d083392008-05-12 21:20:42 +0200361
Steven Rostedt2b499382011-05-03 22:49:52 -0400362static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
363{
364 struct ftrace_ops **p;
365
366 /*
367 * If we are removing the last function, then simply point
368 * to the ftrace_stub.
369 */
370 if (*list == ops && ops->next == &ftrace_list_end) {
371 *list = &ftrace_list_end;
372 return 0;
373 }
374
375 for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
376 if (*p == ops)
377 break;
378
379 if (*p != ops)
380 return -1;
381
382 *p = (*p)->next;
383 return 0;
384}
385
Jiri Olsae2484912012-02-15 15:51:48 +0100386static void add_ftrace_list_ops(struct ftrace_ops **list,
387 struct ftrace_ops *main_ops,
388 struct ftrace_ops *ops)
389{
390 int first = *list == &ftrace_list_end;
391 add_ftrace_ops(list, ops);
392 if (first)
393 add_ftrace_ops(&ftrace_ops_list, main_ops);
394}
395
396static int remove_ftrace_list_ops(struct ftrace_ops **list,
397 struct ftrace_ops *main_ops,
398 struct ftrace_ops *ops)
399{
400 int ret = remove_ftrace_ops(list, ops);
401 if (!ret && *list == &ftrace_list_end)
402 ret = remove_ftrace_ops(&ftrace_ops_list, main_ops);
403 return ret;
404}
405
Steven Rostedt (Red Hat)f3bea492014-07-02 23:23:31 -0400406static void ftrace_update_trampoline(struct ftrace_ops *ops);
407
Steven Rostedt2b499382011-05-03 22:49:52 -0400408static int __register_ftrace_function(struct ftrace_ops *ops)
409{
Steven Rostedt (Red Hat)591dffd2014-01-10 16:17:45 -0500410 if (ops->flags & FTRACE_OPS_FL_DELETED)
411 return -EINVAL;
412
Steven Rostedtb8489142011-05-04 09:27:52 -0400413 if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
414 return -EBUSY;
415
Masami Hiramatsu06aeaae2012-09-28 17:15:17 +0900416#ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS
Steven Rostedt08f6fba2012-04-30 16:20:23 -0400417 /*
418 * If the ftrace_ops specifies SAVE_REGS, then it only can be used
419 * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set.
420 * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant.
421 */
422 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&
423 !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))
424 return -EINVAL;
425
426 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)
427 ops->flags |= FTRACE_OPS_FL_SAVE_REGS;
428#endif
429
Steven Rostedtcdbe61b2011-05-05 21:14:55 -0400430 if (!core_kernel_data((unsigned long)ops))
431 ops->flags |= FTRACE_OPS_FL_DYNAMIC;
432
Steven Rostedt (Red Hat)4104d322014-01-10 17:01:58 -0500433 if (ops->flags & FTRACE_OPS_FL_CONTROL) {
Jiri Olsae2484912012-02-15 15:51:48 +0100434 if (control_ops_alloc(ops))
435 return -ENOMEM;
436 add_ftrace_list_ops(&ftrace_control_list, &control_ops, ops);
Steven Rostedt (Red Hat)fe578ba2014-11-13 19:40:56 -0500437 /* The control_ops needs the trampoline update */
438 ops = &control_ops;
Steven Rostedtb8489142011-05-04 09:27:52 -0400439 } else
440 add_ftrace_ops(&ftrace_ops_list, ops);
441
Steven Rostedt (Red Hat)e3eea142015-07-24 10:38:12 -0400442 /* Always save the function, and reset at unregistering */
443 ops->saved_func = ops->func;
444
445 if (ops->flags & FTRACE_OPS_FL_PID && ftrace_pids_enabled())
446 ops->func = ftrace_pid_func;
447
Steven Rostedt (Red Hat)f3bea492014-07-02 23:23:31 -0400448 ftrace_update_trampoline(ops);
449
Steven Rostedt491d0dc2011-04-27 21:43:36 -0400450 if (ftrace_enabled)
451 update_ftrace_function();
Steven Rostedt3d083392008-05-12 21:20:42 +0200452
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200453 return 0;
454}
455
Ingo Molnare309b412008-05-12 21:20:51 +0200456static int __unregister_ftrace_function(struct ftrace_ops *ops)
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200457{
Steven Rostedt2b499382011-05-03 22:49:52 -0400458 int ret;
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200459
Steven Rostedtb8489142011-05-04 09:27:52 -0400460 if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
461 return -EBUSY;
462
Steven Rostedt (Red Hat)4104d322014-01-10 17:01:58 -0500463 if (ops->flags & FTRACE_OPS_FL_CONTROL) {
Jiri Olsae2484912012-02-15 15:51:48 +0100464 ret = remove_ftrace_list_ops(&ftrace_control_list,
465 &control_ops, ops);
Steven Rostedtb8489142011-05-04 09:27:52 -0400466 } else
467 ret = remove_ftrace_ops(&ftrace_ops_list, ops);
468
Steven Rostedt2b499382011-05-03 22:49:52 -0400469 if (ret < 0)
470 return ret;
Steven Rostedtb8489142011-05-04 09:27:52 -0400471
Steven Rostedt491d0dc2011-04-27 21:43:36 -0400472 if (ftrace_enabled)
473 update_ftrace_function();
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200474
Steven Rostedt (Red Hat)e3eea142015-07-24 10:38:12 -0400475 ops->func = ops->saved_func;
476
Steven Rostedte6ea44e2009-02-14 01:42:44 -0500477 return 0;
Steven Rostedt3d083392008-05-12 21:20:42 +0200478}
479
Steven Rostedtdf4fc312008-11-26 00:16:23 -0500480static void ftrace_update_pid_func(void)
481{
Steven Rostedt (Red Hat)e3eea142015-07-24 10:38:12 -0400482 bool enabled = ftrace_pids_enabled();
483 struct ftrace_ops *op;
484
Steven Rostedt491d0dc2011-04-27 21:43:36 -0400485 /* Only do something if we are tracing something */
Steven Rostedtdf4fc312008-11-26 00:16:23 -0500486 if (ftrace_trace_function == ftrace_stub)
KOSAKI Motohiro10dd3eb2009-03-06 15:29:04 +0900487 return;
Steven Rostedtdf4fc312008-11-26 00:16:23 -0500488
Steven Rostedt (Red Hat)e3eea142015-07-24 10:38:12 -0400489 do_for_each_ftrace_op(op, ftrace_ops_list) {
490 if (op->flags & FTRACE_OPS_FL_PID) {
491 op->func = enabled ? ftrace_pid_func :
492 op->saved_func;
493 ftrace_update_trampoline(op);
494 }
495 } while_for_each_ftrace_op(op);
496
Steven Rostedt491d0dc2011-04-27 21:43:36 -0400497 update_ftrace_function();
Steven Rostedtdf4fc312008-11-26 00:16:23 -0500498}
499
Steven Rostedt493762f2009-03-23 17:12:36 -0400500#ifdef CONFIG_FUNCTION_PROFILER
501struct ftrace_profile {
502 struct hlist_node node;
503 unsigned long ip;
504 unsigned long counter;
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400505#ifdef CONFIG_FUNCTION_GRAPH_TRACER
506 unsigned long long time;
Chase Douglase330b3b2010-04-26 14:02:05 -0400507 unsigned long long time_squared;
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400508#endif
Steven Rostedt493762f2009-03-23 17:12:36 -0400509};
510
511struct ftrace_profile_page {
512 struct ftrace_profile_page *next;
513 unsigned long index;
514 struct ftrace_profile records[];
515};
516
Steven Rostedtcafb1682009-03-24 20:50:39 -0400517struct ftrace_profile_stat {
518 atomic_t disabled;
519 struct hlist_head *hash;
520 struct ftrace_profile_page *pages;
521 struct ftrace_profile_page *start;
522 struct tracer_stat stat;
523};
524
Steven Rostedt493762f2009-03-23 17:12:36 -0400525#define PROFILE_RECORDS_SIZE \
526 (PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
527
528#define PROFILES_PER_PAGE \
529 (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
530
Steven Rostedtfb9fb012009-03-25 13:26:41 -0400531static int ftrace_profile_enabled __read_mostly;
532
533/* ftrace_profile_lock - synchronize the enable and disable of the profiler */
Steven Rostedt493762f2009-03-23 17:12:36 -0400534static DEFINE_MUTEX(ftrace_profile_lock);
535
Steven Rostedtcafb1682009-03-24 20:50:39 -0400536static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
Steven Rostedt493762f2009-03-23 17:12:36 -0400537
Namhyung Kim20079eb2013-04-10 08:55:50 +0900538#define FTRACE_PROFILE_HASH_BITS 10
539#define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS)
Steven Rostedt493762f2009-03-23 17:12:36 -0400540
Steven Rostedt493762f2009-03-23 17:12:36 -0400541static void *
542function_stat_next(void *v, int idx)
543{
544 struct ftrace_profile *rec = v;
545 struct ftrace_profile_page *pg;
546
547 pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
548
549 again:
Li Zefan0296e422009-06-26 11:15:37 +0800550 if (idx != 0)
551 rec++;
552
Steven Rostedt493762f2009-03-23 17:12:36 -0400553 if ((void *)rec >= (void *)&pg->records[pg->index]) {
554 pg = pg->next;
555 if (!pg)
556 return NULL;
557 rec = &pg->records[0];
558 if (!rec->counter)
559 goto again;
560 }
561
562 return rec;
563}
564
565static void *function_stat_start(struct tracer_stat *trace)
566{
Steven Rostedtcafb1682009-03-24 20:50:39 -0400567 struct ftrace_profile_stat *stat =
568 container_of(trace, struct ftrace_profile_stat, stat);
569
570 if (!stat || !stat->start)
571 return NULL;
572
573 return function_stat_next(&stat->start->records[0], 0);
Steven Rostedt493762f2009-03-23 17:12:36 -0400574}
575
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400576#ifdef CONFIG_FUNCTION_GRAPH_TRACER
577/* function graph compares on total time */
578static int function_stat_cmp(void *p1, void *p2)
579{
580 struct ftrace_profile *a = p1;
581 struct ftrace_profile *b = p2;
582
583 if (a->time < b->time)
584 return -1;
585 if (a->time > b->time)
586 return 1;
587 else
588 return 0;
589}
590#else
591/* not function graph compares against hits */
Steven Rostedt493762f2009-03-23 17:12:36 -0400592static int function_stat_cmp(void *p1, void *p2)
593{
594 struct ftrace_profile *a = p1;
595 struct ftrace_profile *b = p2;
596
597 if (a->counter < b->counter)
598 return -1;
599 if (a->counter > b->counter)
600 return 1;
601 else
602 return 0;
603}
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400604#endif
Steven Rostedt493762f2009-03-23 17:12:36 -0400605
606static int function_stat_headers(struct seq_file *m)
607{
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400608#ifdef CONFIG_FUNCTION_GRAPH_TRACER
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +0100609 seq_puts(m, " Function "
610 "Hit Time Avg s^2\n"
611 " -------- "
612 "--- ---- --- ---\n");
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400613#else
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +0100614 seq_puts(m, " Function Hit\n"
615 " -------- ---\n");
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400616#endif
Steven Rostedt493762f2009-03-23 17:12:36 -0400617 return 0;
618}
619
620static int function_stat_show(struct seq_file *m, void *v)
621{
622 struct ftrace_profile *rec = v;
623 char str[KSYM_SYMBOL_LEN];
Li Zefan3aaba202010-08-23 16:50:12 +0800624 int ret = 0;
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400625#ifdef CONFIG_FUNCTION_GRAPH_TRACER
Steven Rostedt34886c82009-03-25 21:00:47 -0400626 static struct trace_seq s;
627 unsigned long long avg;
Chase Douglase330b3b2010-04-26 14:02:05 -0400628 unsigned long long stddev;
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400629#endif
Li Zefan3aaba202010-08-23 16:50:12 +0800630 mutex_lock(&ftrace_profile_lock);
631
632 /* we raced with function_profile_reset() */
633 if (unlikely(rec->counter == 0)) {
634 ret = -EBUSY;
635 goto out;
636 }
Steven Rostedt493762f2009-03-23 17:12:36 -0400637
Umesh Tiwari8e436ca2015-06-22 16:58:08 +0530638#ifdef CONFIG_FUNCTION_GRAPH_TRACER
639 avg = rec->time;
640 do_div(avg, rec->counter);
641 if (tracing_thresh && (avg < tracing_thresh))
642 goto out;
643#endif
644
Steven Rostedt493762f2009-03-23 17:12:36 -0400645 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400646 seq_printf(m, " %-30.30s %10lu", str, rec->counter);
Steven Rostedt493762f2009-03-23 17:12:36 -0400647
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400648#ifdef CONFIG_FUNCTION_GRAPH_TRACER
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +0100649 seq_puts(m, " ");
Steven Rostedt34886c82009-03-25 21:00:47 -0400650
Chase Douglase330b3b2010-04-26 14:02:05 -0400651 /* Sample standard deviation (s^2) */
652 if (rec->counter <= 1)
653 stddev = 0;
654 else {
Juri Lelli52d85d72013-06-12 12:03:18 +0200655 /*
656 * Apply Welford's method:
657 * s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2)
658 */
659 stddev = rec->counter * rec->time_squared -
660 rec->time * rec->time;
661
Chase Douglase330b3b2010-04-26 14:02:05 -0400662 /*
663 * Divide only 1000 for ns^2 -> us^2 conversion.
664 * trace_print_graph_duration will divide 1000 again.
665 */
Juri Lelli52d85d72013-06-12 12:03:18 +0200666 do_div(stddev, rec->counter * (rec->counter - 1) * 1000);
Chase Douglase330b3b2010-04-26 14:02:05 -0400667 }
668
Steven Rostedt34886c82009-03-25 21:00:47 -0400669 trace_seq_init(&s);
670 trace_print_graph_duration(rec->time, &s);
671 trace_seq_puts(&s, " ");
672 trace_print_graph_duration(avg, &s);
Chase Douglase330b3b2010-04-26 14:02:05 -0400673 trace_seq_puts(&s, " ");
674 trace_print_graph_duration(stddev, &s);
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400675 trace_print_seq(m, &s);
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400676#endif
677 seq_putc(m, '\n');
Li Zefan3aaba202010-08-23 16:50:12 +0800678out:
679 mutex_unlock(&ftrace_profile_lock);
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400680
Li Zefan3aaba202010-08-23 16:50:12 +0800681 return ret;
Steven Rostedt493762f2009-03-23 17:12:36 -0400682}
683
Steven Rostedtcafb1682009-03-24 20:50:39 -0400684static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
Steven Rostedt493762f2009-03-23 17:12:36 -0400685{
686 struct ftrace_profile_page *pg;
687
Steven Rostedtcafb1682009-03-24 20:50:39 -0400688 pg = stat->pages = stat->start;
Steven Rostedt493762f2009-03-23 17:12:36 -0400689
690 while (pg) {
691 memset(pg->records, 0, PROFILE_RECORDS_SIZE);
692 pg->index = 0;
693 pg = pg->next;
694 }
695
Steven Rostedtcafb1682009-03-24 20:50:39 -0400696 memset(stat->hash, 0,
Steven Rostedt493762f2009-03-23 17:12:36 -0400697 FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
698}
699
Steven Rostedtcafb1682009-03-24 20:50:39 -0400700int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
Steven Rostedt493762f2009-03-23 17:12:36 -0400701{
702 struct ftrace_profile_page *pg;
Steven Rostedt318e0a72009-03-25 20:06:34 -0400703 int functions;
704 int pages;
Steven Rostedt493762f2009-03-23 17:12:36 -0400705 int i;
706
707 /* If we already allocated, do nothing */
Steven Rostedtcafb1682009-03-24 20:50:39 -0400708 if (stat->pages)
Steven Rostedt493762f2009-03-23 17:12:36 -0400709 return 0;
710
Steven Rostedtcafb1682009-03-24 20:50:39 -0400711 stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
712 if (!stat->pages)
Steven Rostedt493762f2009-03-23 17:12:36 -0400713 return -ENOMEM;
714
Steven Rostedt318e0a72009-03-25 20:06:34 -0400715#ifdef CONFIG_DYNAMIC_FTRACE
716 functions = ftrace_update_tot_cnt;
717#else
718 /*
719 * We do not know the number of functions that exist because
720 * dynamic tracing is what counts them. With past experience
721 * we have around 20K functions. That should be more than enough.
722 * It is highly unlikely we will execute every function in
723 * the kernel.
724 */
725 functions = 20000;
726#endif
727
Steven Rostedtcafb1682009-03-24 20:50:39 -0400728 pg = stat->start = stat->pages;
Steven Rostedt493762f2009-03-23 17:12:36 -0400729
Steven Rostedt318e0a72009-03-25 20:06:34 -0400730 pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
731
Namhyung Kim39e30cd2013-04-01 21:46:24 +0900732 for (i = 1; i < pages; i++) {
Steven Rostedt493762f2009-03-23 17:12:36 -0400733 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
Steven Rostedt493762f2009-03-23 17:12:36 -0400734 if (!pg->next)
Steven Rostedt318e0a72009-03-25 20:06:34 -0400735 goto out_free;
Steven Rostedt493762f2009-03-23 17:12:36 -0400736 pg = pg->next;
737 }
738
739 return 0;
Steven Rostedt318e0a72009-03-25 20:06:34 -0400740
741 out_free:
742 pg = stat->start;
743 while (pg) {
744 unsigned long tmp = (unsigned long)pg;
745
746 pg = pg->next;
747 free_page(tmp);
748 }
749
Steven Rostedt318e0a72009-03-25 20:06:34 -0400750 stat->pages = NULL;
751 stat->start = NULL;
752
753 return -ENOMEM;
Steven Rostedt493762f2009-03-23 17:12:36 -0400754}
755
Steven Rostedtcafb1682009-03-24 20:50:39 -0400756static int ftrace_profile_init_cpu(int cpu)
Steven Rostedt493762f2009-03-23 17:12:36 -0400757{
Steven Rostedtcafb1682009-03-24 20:50:39 -0400758 struct ftrace_profile_stat *stat;
Steven Rostedt493762f2009-03-23 17:12:36 -0400759 int size;
760
Steven Rostedtcafb1682009-03-24 20:50:39 -0400761 stat = &per_cpu(ftrace_profile_stats, cpu);
762
763 if (stat->hash) {
Steven Rostedt493762f2009-03-23 17:12:36 -0400764 /* If the profile is already created, simply reset it */
Steven Rostedtcafb1682009-03-24 20:50:39 -0400765 ftrace_profile_reset(stat);
Steven Rostedt493762f2009-03-23 17:12:36 -0400766 return 0;
767 }
768
769 /*
770 * We are profiling all functions, but usually only a few thousand
771 * functions are hit. We'll make a hash of 1024 items.
772 */
773 size = FTRACE_PROFILE_HASH_SIZE;
774
Steven Rostedtcafb1682009-03-24 20:50:39 -0400775 stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
Steven Rostedt493762f2009-03-23 17:12:36 -0400776
Steven Rostedtcafb1682009-03-24 20:50:39 -0400777 if (!stat->hash)
Steven Rostedt493762f2009-03-23 17:12:36 -0400778 return -ENOMEM;
779
Steven Rostedt318e0a72009-03-25 20:06:34 -0400780 /* Preallocate the function profiling pages */
Steven Rostedtcafb1682009-03-24 20:50:39 -0400781 if (ftrace_profile_pages_init(stat) < 0) {
782 kfree(stat->hash);
783 stat->hash = NULL;
Steven Rostedt493762f2009-03-23 17:12:36 -0400784 return -ENOMEM;
785 }
786
787 return 0;
788}
789
Steven Rostedtcafb1682009-03-24 20:50:39 -0400790static int ftrace_profile_init(void)
791{
792 int cpu;
793 int ret = 0;
794
Miao Xiec4602c12013-12-16 15:20:01 +0800795 for_each_possible_cpu(cpu) {
Steven Rostedtcafb1682009-03-24 20:50:39 -0400796 ret = ftrace_profile_init_cpu(cpu);
797 if (ret)
798 break;
799 }
800
801 return ret;
802}
803
Steven Rostedt493762f2009-03-23 17:12:36 -0400804/* interrupts must be disabled */
Steven Rostedtcafb1682009-03-24 20:50:39 -0400805static struct ftrace_profile *
806ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
Steven Rostedt493762f2009-03-23 17:12:36 -0400807{
808 struct ftrace_profile *rec;
809 struct hlist_head *hhd;
Steven Rostedt493762f2009-03-23 17:12:36 -0400810 unsigned long key;
811
Namhyung Kim20079eb2013-04-10 08:55:50 +0900812 key = hash_long(ip, FTRACE_PROFILE_HASH_BITS);
Steven Rostedtcafb1682009-03-24 20:50:39 -0400813 hhd = &stat->hash[key];
Steven Rostedt493762f2009-03-23 17:12:36 -0400814
815 if (hlist_empty(hhd))
816 return NULL;
817
Steven Rostedt1bb539c2013-05-28 14:38:43 -0400818 hlist_for_each_entry_rcu_notrace(rec, hhd, node) {
Steven Rostedt493762f2009-03-23 17:12:36 -0400819 if (rec->ip == ip)
820 return rec;
821 }
822
823 return NULL;
824}
825
Steven Rostedtcafb1682009-03-24 20:50:39 -0400826static void ftrace_add_profile(struct ftrace_profile_stat *stat,
827 struct ftrace_profile *rec)
Steven Rostedt493762f2009-03-23 17:12:36 -0400828{
829 unsigned long key;
830
Namhyung Kim20079eb2013-04-10 08:55:50 +0900831 key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS);
Steven Rostedtcafb1682009-03-24 20:50:39 -0400832 hlist_add_head_rcu(&rec->node, &stat->hash[key]);
Steven Rostedt493762f2009-03-23 17:12:36 -0400833}
834
Steven Rostedt318e0a72009-03-25 20:06:34 -0400835/*
836 * The memory is already allocated, this simply finds a new record to use.
837 */
Steven Rostedt493762f2009-03-23 17:12:36 -0400838static struct ftrace_profile *
Steven Rostedt318e0a72009-03-25 20:06:34 -0400839ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
Steven Rostedt493762f2009-03-23 17:12:36 -0400840{
841 struct ftrace_profile *rec = NULL;
842
Steven Rostedt318e0a72009-03-25 20:06:34 -0400843 /* prevent recursion (from NMIs) */
Steven Rostedtcafb1682009-03-24 20:50:39 -0400844 if (atomic_inc_return(&stat->disabled) != 1)
Steven Rostedt493762f2009-03-23 17:12:36 -0400845 goto out;
846
Steven Rostedt493762f2009-03-23 17:12:36 -0400847 /*
Steven Rostedt318e0a72009-03-25 20:06:34 -0400848 * Try to find the function again since an NMI
849 * could have added it
Steven Rostedt493762f2009-03-23 17:12:36 -0400850 */
Steven Rostedtcafb1682009-03-24 20:50:39 -0400851 rec = ftrace_find_profiled_func(stat, ip);
Steven Rostedt493762f2009-03-23 17:12:36 -0400852 if (rec)
Steven Rostedtcafb1682009-03-24 20:50:39 -0400853 goto out;
Steven Rostedt493762f2009-03-23 17:12:36 -0400854
Steven Rostedtcafb1682009-03-24 20:50:39 -0400855 if (stat->pages->index == PROFILES_PER_PAGE) {
856 if (!stat->pages->next)
857 goto out;
858 stat->pages = stat->pages->next;
Steven Rostedt493762f2009-03-23 17:12:36 -0400859 }
860
Steven Rostedtcafb1682009-03-24 20:50:39 -0400861 rec = &stat->pages->records[stat->pages->index++];
Steven Rostedt493762f2009-03-23 17:12:36 -0400862 rec->ip = ip;
Steven Rostedtcafb1682009-03-24 20:50:39 -0400863 ftrace_add_profile(stat, rec);
Steven Rostedt493762f2009-03-23 17:12:36 -0400864
Steven Rostedt493762f2009-03-23 17:12:36 -0400865 out:
Steven Rostedtcafb1682009-03-24 20:50:39 -0400866 atomic_dec(&stat->disabled);
Steven Rostedt493762f2009-03-23 17:12:36 -0400867
868 return rec;
869}
870
Steven Rostedt493762f2009-03-23 17:12:36 -0400871static void
Steven Rostedt2f5f6ad2011-08-08 16:57:47 -0400872function_profile_call(unsigned long ip, unsigned long parent_ip,
Steven Rostedta1e2e312011-08-09 12:50:46 -0400873 struct ftrace_ops *ops, struct pt_regs *regs)
Steven Rostedt493762f2009-03-23 17:12:36 -0400874{
Steven Rostedtcafb1682009-03-24 20:50:39 -0400875 struct ftrace_profile_stat *stat;
Steven Rostedt493762f2009-03-23 17:12:36 -0400876 struct ftrace_profile *rec;
877 unsigned long flags;
Steven Rostedt493762f2009-03-23 17:12:36 -0400878
879 if (!ftrace_profile_enabled)
880 return;
881
Steven Rostedt493762f2009-03-23 17:12:36 -0400882 local_irq_save(flags);
Steven Rostedtcafb1682009-03-24 20:50:39 -0400883
Christoph Lameterbdffd892014-04-29 14:17:40 -0500884 stat = this_cpu_ptr(&ftrace_profile_stats);
Steven Rostedt0f6ce3d2009-06-01 21:51:28 -0400885 if (!stat->hash || !ftrace_profile_enabled)
Steven Rostedtcafb1682009-03-24 20:50:39 -0400886 goto out;
887
888 rec = ftrace_find_profiled_func(stat, ip);
Steven Rostedt493762f2009-03-23 17:12:36 -0400889 if (!rec) {
Steven Rostedt318e0a72009-03-25 20:06:34 -0400890 rec = ftrace_profile_alloc(stat, ip);
Steven Rostedt493762f2009-03-23 17:12:36 -0400891 if (!rec)
892 goto out;
893 }
894
895 rec->counter++;
896 out:
897 local_irq_restore(flags);
898}
899
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400900#ifdef CONFIG_FUNCTION_GRAPH_TRACER
901static int profile_graph_entry(struct ftrace_graph_ent *trace)
902{
Steven Rostedta1e2e312011-08-09 12:50:46 -0400903 function_profile_call(trace->func, 0, NULL, NULL);
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400904 return 1;
905}
906
907static void profile_graph_return(struct ftrace_graph_ret *trace)
908{
Steven Rostedtcafb1682009-03-24 20:50:39 -0400909 struct ftrace_profile_stat *stat;
Steven Rostedta2a16d62009-03-24 23:17:58 -0400910 unsigned long long calltime;
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400911 struct ftrace_profile *rec;
Steven Rostedtcafb1682009-03-24 20:50:39 -0400912 unsigned long flags;
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400913
914 local_irq_save(flags);
Christoph Lameterbdffd892014-04-29 14:17:40 -0500915 stat = this_cpu_ptr(&ftrace_profile_stats);
Steven Rostedt0f6ce3d2009-06-01 21:51:28 -0400916 if (!stat->hash || !ftrace_profile_enabled)
Steven Rostedtcafb1682009-03-24 20:50:39 -0400917 goto out;
918
Steven Rostedt37e44bc2010-04-27 21:04:24 -0400919 /* If the calltime was zero'd ignore it */
920 if (!trace->calltime)
921 goto out;
922
Steven Rostedta2a16d62009-03-24 23:17:58 -0400923 calltime = trace->rettime - trace->calltime;
924
Steven Rostedt (Red Hat)55577202015-09-29 19:06:50 -0400925 if (!fgraph_graph_time) {
Steven Rostedta2a16d62009-03-24 23:17:58 -0400926 int index;
927
928 index = trace->depth;
929
930 /* Append this call time to the parent time to subtract */
931 if (index)
932 current->ret_stack[index - 1].subtime += calltime;
933
934 if (current->ret_stack[index].subtime < calltime)
935 calltime -= current->ret_stack[index].subtime;
936 else
937 calltime = 0;
938 }
939
Steven Rostedtcafb1682009-03-24 20:50:39 -0400940 rec = ftrace_find_profiled_func(stat, trace->func);
Chase Douglase330b3b2010-04-26 14:02:05 -0400941 if (rec) {
Steven Rostedta2a16d62009-03-24 23:17:58 -0400942 rec->time += calltime;
Chase Douglase330b3b2010-04-26 14:02:05 -0400943 rec->time_squared += calltime * calltime;
944 }
Steven Rostedta2a16d62009-03-24 23:17:58 -0400945
Steven Rostedtcafb1682009-03-24 20:50:39 -0400946 out:
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400947 local_irq_restore(flags);
948}
949
950static int register_ftrace_profiler(void)
951{
952 return register_ftrace_graph(&profile_graph_return,
953 &profile_graph_entry);
954}
955
956static void unregister_ftrace_profiler(void)
957{
958 unregister_ftrace_graph();
959}
960#else
Paul McQuadebd38c0e2011-05-31 20:51:55 +0100961static struct ftrace_ops ftrace_profile_ops __read_mostly = {
Steven Rostedtfb9fb012009-03-25 13:26:41 -0400962 .func = function_profile_call,
Masami Hiramatsuf04f24fb2013-05-09 14:44:17 +0900963 .flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -0400964 INIT_OPS_HASH(ftrace_profile_ops)
Steven Rostedt493762f2009-03-23 17:12:36 -0400965};
966
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400967static int register_ftrace_profiler(void)
968{
969 return register_ftrace_function(&ftrace_profile_ops);
970}
971
972static void unregister_ftrace_profiler(void)
973{
974 unregister_ftrace_function(&ftrace_profile_ops);
975}
976#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
977
Steven Rostedt493762f2009-03-23 17:12:36 -0400978static ssize_t
979ftrace_profile_write(struct file *filp, const char __user *ubuf,
980 size_t cnt, loff_t *ppos)
981{
982 unsigned long val;
Steven Rostedt493762f2009-03-23 17:12:36 -0400983 int ret;
984
Peter Huewe22fe9b52011-06-07 21:58:27 +0200985 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
986 if (ret)
Steven Rostedt493762f2009-03-23 17:12:36 -0400987 return ret;
988
989 val = !!val;
990
991 mutex_lock(&ftrace_profile_lock);
992 if (ftrace_profile_enabled ^ val) {
993 if (val) {
994 ret = ftrace_profile_init();
995 if (ret < 0) {
996 cnt = ret;
997 goto out;
998 }
999
Steven Rostedt0706f1c2009-03-23 23:12:58 -04001000 ret = register_ftrace_profiler();
1001 if (ret < 0) {
1002 cnt = ret;
1003 goto out;
1004 }
Steven Rostedt493762f2009-03-23 17:12:36 -04001005 ftrace_profile_enabled = 1;
1006 } else {
1007 ftrace_profile_enabled = 0;
Steven Rostedt0f6ce3d2009-06-01 21:51:28 -04001008 /*
1009 * unregister_ftrace_profiler calls stop_machine
1010 * so this acts like an synchronize_sched.
1011 */
Steven Rostedt0706f1c2009-03-23 23:12:58 -04001012 unregister_ftrace_profiler();
Steven Rostedt493762f2009-03-23 17:12:36 -04001013 }
1014 }
1015 out:
1016 mutex_unlock(&ftrace_profile_lock);
1017
Jiri Olsacf8517c2009-10-23 19:36:16 -04001018 *ppos += cnt;
Steven Rostedt493762f2009-03-23 17:12:36 -04001019
1020 return cnt;
1021}
1022
1023static ssize_t
1024ftrace_profile_read(struct file *filp, char __user *ubuf,
1025 size_t cnt, loff_t *ppos)
1026{
Steven Rostedtfb9fb012009-03-25 13:26:41 -04001027 char buf[64]; /* big enough to hold a number */
Steven Rostedt493762f2009-03-23 17:12:36 -04001028 int r;
1029
1030 r = sprintf(buf, "%u\n", ftrace_profile_enabled);
1031 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
1032}
1033
1034static const struct file_operations ftrace_profile_fops = {
1035 .open = tracing_open_generic,
1036 .read = ftrace_profile_read,
1037 .write = ftrace_profile_write,
Arnd Bergmann6038f372010-08-15 18:52:59 +02001038 .llseek = default_llseek,
Steven Rostedt493762f2009-03-23 17:12:36 -04001039};
1040
Steven Rostedtcafb1682009-03-24 20:50:39 -04001041/* used to initialize the real stat files */
1042static struct tracer_stat function_stats __initdata = {
Steven Rostedtfb9fb012009-03-25 13:26:41 -04001043 .name = "functions",
1044 .stat_start = function_stat_start,
1045 .stat_next = function_stat_next,
1046 .stat_cmp = function_stat_cmp,
1047 .stat_headers = function_stat_headers,
1048 .stat_show = function_stat_show
Steven Rostedtcafb1682009-03-24 20:50:39 -04001049};
1050
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05001051static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
Steven Rostedt493762f2009-03-23 17:12:36 -04001052{
Steven Rostedtcafb1682009-03-24 20:50:39 -04001053 struct ftrace_profile_stat *stat;
Steven Rostedt493762f2009-03-23 17:12:36 -04001054 struct dentry *entry;
Steven Rostedtcafb1682009-03-24 20:50:39 -04001055 char *name;
Steven Rostedt493762f2009-03-23 17:12:36 -04001056 int ret;
Steven Rostedtcafb1682009-03-24 20:50:39 -04001057 int cpu;
Steven Rostedt493762f2009-03-23 17:12:36 -04001058
Steven Rostedtcafb1682009-03-24 20:50:39 -04001059 for_each_possible_cpu(cpu) {
1060 stat = &per_cpu(ftrace_profile_stats, cpu);
1061
1062 /* allocate enough for function name + cpu number */
1063 name = kmalloc(32, GFP_KERNEL);
1064 if (!name) {
1065 /*
1066 * The files created are permanent, if something happens
1067 * we still do not free memory.
1068 */
Steven Rostedtcafb1682009-03-24 20:50:39 -04001069 WARN(1,
1070 "Could not allocate stat file for cpu %d\n",
1071 cpu);
1072 return;
1073 }
1074 stat->stat = function_stats;
1075 snprintf(name, 32, "function%d", cpu);
1076 stat->stat.name = name;
1077 ret = register_stat_tracer(&stat->stat);
1078 if (ret) {
1079 WARN(1,
1080 "Could not register function stat for cpu %d\n",
1081 cpu);
1082 kfree(name);
1083 return;
1084 }
Steven Rostedt493762f2009-03-23 17:12:36 -04001085 }
1086
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05001087 entry = tracefs_create_file("function_profile_enabled", 0644,
Steven Rostedt493762f2009-03-23 17:12:36 -04001088 d_tracer, NULL, &ftrace_profile_fops);
1089 if (!entry)
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05001090 pr_warning("Could not create tracefs "
Steven Rostedt493762f2009-03-23 17:12:36 -04001091 "'function_profile_enabled' entry\n");
1092}
1093
1094#else /* CONFIG_FUNCTION_PROFILER */
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05001095static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
Steven Rostedt493762f2009-03-23 17:12:36 -04001096{
1097}
1098#endif /* CONFIG_FUNCTION_PROFILER */
1099
Ingo Molnar73d3fd92009-02-17 11:48:18 +01001100static struct pid * const ftrace_swapper_pid = &init_struct_pid;
1101
Pratyush Anand1619dc32015-03-06 23:58:06 +05301102#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1103static int ftrace_graph_active;
1104#else
1105# define ftrace_graph_active 0
1106#endif
1107
Steven Rostedt3d083392008-05-12 21:20:42 +02001108#ifdef CONFIG_DYNAMIC_FTRACE
Ingo Molnar73d3fd92009-02-17 11:48:18 +01001109
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04001110static struct ftrace_ops *removed_ops;
1111
Steven Rostedt (Red Hat)e1effa02014-08-05 17:19:38 -04001112/*
1113 * Set when doing a global update, like enabling all recs or disabling them.
1114 * It is not set when just updating a single ftrace_ops.
1115 */
1116static bool update_all_ops;
1117
Steven Rostedt99ecdc42008-08-15 21:40:05 -04001118#ifndef CONFIG_FTRACE_MCOUNT_RECORD
Steven Rostedtcb7be3b2008-10-23 09:33:05 -04001119# error Dynamic ftrace depends on MCOUNT_RECORD
Steven Rostedt99ecdc42008-08-15 21:40:05 -04001120#endif
1121
Steven Rostedt8fc0c702009-02-16 15:28:00 -05001122static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
1123
Steven Rostedtb6887d72009-02-17 12:32:04 -05001124struct ftrace_func_probe {
Steven Rostedt8fc0c702009-02-16 15:28:00 -05001125 struct hlist_node node;
Steven Rostedtb6887d72009-02-17 12:32:04 -05001126 struct ftrace_probe_ops *ops;
Steven Rostedt8fc0c702009-02-16 15:28:00 -05001127 unsigned long flags;
1128 unsigned long ip;
1129 void *data;
Steven Rostedt (Red Hat)7818b382013-03-13 12:42:58 -04001130 struct list_head free_list;
Steven Rostedt8fc0c702009-02-16 15:28:00 -05001131};
1132
Steven Rostedtb448c4e2011-04-29 15:12:32 -04001133struct ftrace_func_entry {
1134 struct hlist_node hlist;
1135 unsigned long ip;
1136};
1137
1138struct ftrace_hash {
1139 unsigned long size_bits;
1140 struct hlist_head *buckets;
1141 unsigned long count;
Steven Rostedt07fd5512011-05-05 18:03:47 -04001142 struct rcu_head rcu;
Steven Rostedtb448c4e2011-04-29 15:12:32 -04001143};
1144
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001145/*
1146 * We make these constant because no one should touch them,
1147 * but they are used as the default "empty hash", to avoid allocating
1148 * it all the time. These are in a read only section such that if
1149 * anyone does try to modify it, it will cause an exception.
1150 */
1151static const struct hlist_head empty_buckets[1];
1152static const struct ftrace_hash empty_hash = {
1153 .buckets = (struct hlist_head *)empty_buckets,
Steven Rostedtb448c4e2011-04-29 15:12:32 -04001154};
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001155#define EMPTY_HASH ((struct ftrace_hash *)&empty_hash)
Steven Rostedt5072c592008-05-12 21:20:43 +02001156
Steven Rostedt2b499382011-05-03 22:49:52 -04001157static struct ftrace_ops global_ops = {
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04001158 .func = ftrace_stub,
1159 .local_hash.notrace_hash = EMPTY_HASH,
1160 .local_hash.filter_hash = EMPTY_HASH,
1161 INIT_OPS_HASH(global_ops)
1162 .flags = FTRACE_OPS_FL_RECURSION_SAFE |
Steven Rostedt (Red Hat)e3eea142015-07-24 10:38:12 -04001163 FTRACE_OPS_FL_INITIALIZED |
1164 FTRACE_OPS_FL_PID,
Steven Rostedtf45948e2011-05-02 12:29:25 -04001165};
1166
Steven Rostedt (Red Hat)aec0be22014-11-18 21:14:11 -05001167/*
1168 * This is used by __kernel_text_address() to return true if the
Steven Rostedt (Red Hat)0af26492014-11-20 10:05:36 -05001169 * address is on a dynamically allocated trampoline that would
Steven Rostedt (Red Hat)aec0be22014-11-18 21:14:11 -05001170 * not return true for either core_kernel_text() or
1171 * is_module_text_address().
1172 */
1173bool is_ftrace_trampoline(unsigned long addr)
1174{
1175 struct ftrace_ops *op;
1176 bool ret = false;
1177
1178 /*
1179 * Some of the ops may be dynamically allocated,
1180 * they are freed after a synchronize_sched().
1181 */
1182 preempt_disable_notrace();
1183
1184 do_for_each_ftrace_op(op, ftrace_ops_list) {
1185 /*
1186 * This is to check for dynamically allocated trampolines.
1187 * Trampolines that are in kernel text will have
1188 * core_kernel_text() return true.
1189 */
1190 if (op->trampoline && op->trampoline_size)
1191 if (addr >= op->trampoline &&
1192 addr < op->trampoline + op->trampoline_size) {
1193 ret = true;
1194 goto out;
1195 }
1196 } while_for_each_ftrace_op(op);
1197
1198 out:
1199 preempt_enable_notrace();
1200
1201 return ret;
1202}
1203
Steven Rostedt3c1720f2008-05-12 21:20:43 +02001204struct ftrace_page {
1205 struct ftrace_page *next;
Steven Rostedta7900872011-12-16 16:23:44 -05001206 struct dyn_ftrace *records;
Steven Rostedt431aa3f2009-01-06 12:43:01 -05001207 int index;
Steven Rostedta7900872011-12-16 16:23:44 -05001208 int size;
David Milleraa5e5ce2008-05-13 22:06:56 -07001209};
Steven Rostedt3c1720f2008-05-12 21:20:43 +02001210
Steven Rostedta7900872011-12-16 16:23:44 -05001211#define ENTRY_SIZE sizeof(struct dyn_ftrace)
1212#define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
Steven Rostedt3c1720f2008-05-12 21:20:43 +02001213
1214/* estimate from running different kernels */
1215#define NR_TO_INIT 10000
1216
1217static struct ftrace_page *ftrace_pages_start;
1218static struct ftrace_page *ftrace_pages;
1219
Steven Rostedt (Red Hat)68f40962014-05-01 12:44:50 -04001220static bool __always_inline ftrace_hash_empty(struct ftrace_hash *hash)
Steven Rostedt06a51d92011-12-19 19:07:36 -05001221{
1222 return !hash || !hash->count;
1223}
1224
Steven Rostedtb448c4e2011-04-29 15:12:32 -04001225static struct ftrace_func_entry *
1226ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
1227{
1228 unsigned long key;
1229 struct ftrace_func_entry *entry;
1230 struct hlist_head *hhd;
Steven Rostedtb448c4e2011-04-29 15:12:32 -04001231
Steven Rostedt06a51d92011-12-19 19:07:36 -05001232 if (ftrace_hash_empty(hash))
Steven Rostedtb448c4e2011-04-29 15:12:32 -04001233 return NULL;
1234
1235 if (hash->size_bits > 0)
1236 key = hash_long(ip, hash->size_bits);
1237 else
1238 key = 0;
1239
1240 hhd = &hash->buckets[key];
1241
Steven Rostedt1bb539c2013-05-28 14:38:43 -04001242 hlist_for_each_entry_rcu_notrace(entry, hhd, hlist) {
Steven Rostedtb448c4e2011-04-29 15:12:32 -04001243 if (entry->ip == ip)
1244 return entry;
1245 }
1246 return NULL;
1247}
1248
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001249static void __add_hash_entry(struct ftrace_hash *hash,
1250 struct ftrace_func_entry *entry)
1251{
1252 struct hlist_head *hhd;
1253 unsigned long key;
1254
1255 if (hash->size_bits)
1256 key = hash_long(entry->ip, hash->size_bits);
1257 else
1258 key = 0;
1259
1260 hhd = &hash->buckets[key];
1261 hlist_add_head(&entry->hlist, hhd);
1262 hash->count++;
1263}
1264
Steven Rostedtb448c4e2011-04-29 15:12:32 -04001265static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
1266{
1267 struct ftrace_func_entry *entry;
Steven Rostedtb448c4e2011-04-29 15:12:32 -04001268
1269 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1270 if (!entry)
1271 return -ENOMEM;
1272
Steven Rostedtb448c4e2011-04-29 15:12:32 -04001273 entry->ip = ip;
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001274 __add_hash_entry(hash, entry);
Steven Rostedtb448c4e2011-04-29 15:12:32 -04001275
1276 return 0;
1277}
1278
1279static void
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001280free_hash_entry(struct ftrace_hash *hash,
1281 struct ftrace_func_entry *entry)
1282{
1283 hlist_del(&entry->hlist);
1284 kfree(entry);
1285 hash->count--;
1286}
1287
1288static void
Steven Rostedtb448c4e2011-04-29 15:12:32 -04001289remove_hash_entry(struct ftrace_hash *hash,
1290 struct ftrace_func_entry *entry)
1291{
1292 hlist_del(&entry->hlist);
Steven Rostedtb448c4e2011-04-29 15:12:32 -04001293 hash->count--;
1294}
1295
1296static void ftrace_hash_clear(struct ftrace_hash *hash)
1297{
1298 struct hlist_head *hhd;
Sasha Levinb67bfe02013-02-27 17:06:00 -08001299 struct hlist_node *tn;
Steven Rostedtb448c4e2011-04-29 15:12:32 -04001300 struct ftrace_func_entry *entry;
1301 int size = 1 << hash->size_bits;
1302 int i;
1303
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001304 if (!hash->count)
1305 return;
1306
Steven Rostedtb448c4e2011-04-29 15:12:32 -04001307 for (i = 0; i < size; i++) {
1308 hhd = &hash->buckets[i];
Sasha Levinb67bfe02013-02-27 17:06:00 -08001309 hlist_for_each_entry_safe(entry, tn, hhd, hlist)
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001310 free_hash_entry(hash, entry);
Steven Rostedtb448c4e2011-04-29 15:12:32 -04001311 }
1312 FTRACE_WARN_ON(hash->count);
1313}
1314
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001315static void free_ftrace_hash(struct ftrace_hash *hash)
1316{
1317 if (!hash || hash == EMPTY_HASH)
1318 return;
1319 ftrace_hash_clear(hash);
1320 kfree(hash->buckets);
1321 kfree(hash);
1322}
1323
Steven Rostedt07fd5512011-05-05 18:03:47 -04001324static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
1325{
1326 struct ftrace_hash *hash;
1327
1328 hash = container_of(rcu, struct ftrace_hash, rcu);
1329 free_ftrace_hash(hash);
1330}
1331
1332static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
1333{
1334 if (!hash || hash == EMPTY_HASH)
1335 return;
1336 call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);
1337}
1338
Jiri Olsa5500fa52012-02-15 15:51:54 +01001339void ftrace_free_filter(struct ftrace_ops *ops)
1340{
Masami Hiramatsuf04f24fb2013-05-09 14:44:17 +09001341 ftrace_ops_init(ops);
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04001342 free_ftrace_hash(ops->func_hash->filter_hash);
1343 free_ftrace_hash(ops->func_hash->notrace_hash);
Jiri Olsa5500fa52012-02-15 15:51:54 +01001344}
1345
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001346static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
1347{
1348 struct ftrace_hash *hash;
1349 int size;
1350
1351 hash = kzalloc(sizeof(*hash), GFP_KERNEL);
1352 if (!hash)
1353 return NULL;
1354
1355 size = 1 << size_bits;
Thomas Meyer47b0edc2011-11-29 22:08:00 +01001356 hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL);
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001357
1358 if (!hash->buckets) {
1359 kfree(hash);
1360 return NULL;
1361 }
1362
1363 hash->size_bits = size_bits;
1364
1365 return hash;
1366}
1367
1368static struct ftrace_hash *
1369alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
1370{
1371 struct ftrace_func_entry *entry;
1372 struct ftrace_hash *new_hash;
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001373 int size;
1374 int ret;
1375 int i;
1376
1377 new_hash = alloc_ftrace_hash(size_bits);
1378 if (!new_hash)
1379 return NULL;
1380
1381 /* Empty hash? */
Steven Rostedt06a51d92011-12-19 19:07:36 -05001382 if (ftrace_hash_empty(hash))
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001383 return new_hash;
1384
1385 size = 1 << hash->size_bits;
1386 for (i = 0; i < size; i++) {
Sasha Levinb67bfe02013-02-27 17:06:00 -08001387 hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001388 ret = add_hash_entry(new_hash, entry->ip);
1389 if (ret < 0)
1390 goto free_hash;
1391 }
1392 }
1393
1394 FTRACE_WARN_ON(new_hash->count != hash->count);
1395
1396 return new_hash;
1397
1398 free_hash:
1399 free_ftrace_hash(new_hash);
1400 return NULL;
1401}
1402
Steven Rostedt41fb61c2011-07-13 15:03:44 -04001403static void
Steven Rostedt (Red Hat)84261912014-08-18 13:21:08 -04001404ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, int filter_hash);
Steven Rostedt41fb61c2011-07-13 15:03:44 -04001405static void
Steven Rostedt (Red Hat)84261912014-08-18 13:21:08 -04001406ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, int filter_hash);
Steven Rostedt41fb61c2011-07-13 15:03:44 -04001407
Masami Hiramatsuf8b8be82014-11-21 05:25:16 -05001408static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
1409 struct ftrace_hash *new_hash);
1410
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001411static int
Steven Rostedt41fb61c2011-07-13 15:03:44 -04001412ftrace_hash_move(struct ftrace_ops *ops, int enable,
1413 struct ftrace_hash **dst, struct ftrace_hash *src)
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001414{
1415 struct ftrace_func_entry *entry;
Sasha Levinb67bfe02013-02-27 17:06:00 -08001416 struct hlist_node *tn;
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001417 struct hlist_head *hhd;
Steven Rostedt07fd5512011-05-05 18:03:47 -04001418 struct ftrace_hash *new_hash;
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001419 int size = src->count;
1420 int bits = 0;
Masami Hiramatsuf8b8be82014-11-21 05:25:16 -05001421 int ret;
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001422 int i;
1423
Masami Hiramatsuf8b8be82014-11-21 05:25:16 -05001424 /* Reject setting notrace hash on IPMODIFY ftrace_ops */
1425 if (ops->flags & FTRACE_OPS_FL_IPMODIFY && !enable)
1426 return -EINVAL;
1427
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001428 /*
1429 * If the new source is empty, just free dst and assign it
1430 * the empty_hash.
1431 */
1432 if (!src->count) {
Masami Hiramatsu5c27c772014-06-17 11:04:42 +00001433 new_hash = EMPTY_HASH;
1434 goto update;
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001435 }
1436
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001437 /*
1438 * Make the hash size about 1/2 the # found
1439 */
1440 for (size /= 2; size; size >>= 1)
1441 bits++;
1442
1443 /* Don't allocate too much */
1444 if (bits > FTRACE_HASH_MAX_BITS)
1445 bits = FTRACE_HASH_MAX_BITS;
1446
Steven Rostedt07fd5512011-05-05 18:03:47 -04001447 new_hash = alloc_ftrace_hash(bits);
1448 if (!new_hash)
Masami Hiramatsu5c27c772014-06-17 11:04:42 +00001449 return -ENOMEM;
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001450
1451 size = 1 << src->size_bits;
1452 for (i = 0; i < size; i++) {
1453 hhd = &src->buckets[i];
Sasha Levinb67bfe02013-02-27 17:06:00 -08001454 hlist_for_each_entry_safe(entry, tn, hhd, hlist) {
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001455 remove_hash_entry(src, entry);
Steven Rostedt07fd5512011-05-05 18:03:47 -04001456 __add_hash_entry(new_hash, entry);
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001457 }
1458 }
1459
Masami Hiramatsu5c27c772014-06-17 11:04:42 +00001460update:
Masami Hiramatsuf8b8be82014-11-21 05:25:16 -05001461 /* Make sure this can be applied if it is IPMODIFY ftrace_ops */
1462 if (enable) {
1463 /* IPMODIFY should be updated only when filter_hash updating */
1464 ret = ftrace_hash_ipmodify_update(ops, new_hash);
1465 if (ret < 0) {
1466 free_ftrace_hash(new_hash);
1467 return ret;
1468 }
1469 }
1470
Masami Hiramatsu5c27c772014-06-17 11:04:42 +00001471 /*
1472 * Remove the current set, update the hash and add
1473 * them back.
1474 */
Steven Rostedt (Red Hat)84261912014-08-18 13:21:08 -04001475 ftrace_hash_rec_disable_modify(ops, enable);
Masami Hiramatsu5c27c772014-06-17 11:04:42 +00001476
Steven Rostedt07fd5512011-05-05 18:03:47 -04001477 rcu_assign_pointer(*dst, new_hash);
Steven Rostedt07fd5512011-05-05 18:03:47 -04001478
Steven Rostedt (Red Hat)84261912014-08-18 13:21:08 -04001479 ftrace_hash_rec_enable_modify(ops, enable);
Steven Rostedt41fb61c2011-07-13 15:03:44 -04001480
Masami Hiramatsu5c27c772014-06-17 11:04:42 +00001481 return 0;
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001482}
1483
Steven Rostedt (Red Hat)fef5aee2014-07-24 12:25:47 -04001484static bool hash_contains_ip(unsigned long ip,
1485 struct ftrace_ops_hash *hash)
1486{
1487 /*
1488 * The function record is a match if it exists in the filter
1489 * hash and not in the notrace hash. Note, an emty hash is
1490 * considered a match for the filter hash, but an empty
1491 * notrace hash is considered not in the notrace hash.
1492 */
1493 return (ftrace_hash_empty(hash->filter_hash) ||
1494 ftrace_lookup_ip(hash->filter_hash, ip)) &&
1495 (ftrace_hash_empty(hash->notrace_hash) ||
1496 !ftrace_lookup_ip(hash->notrace_hash, ip));
1497}
1498
Steven Rostedt265c8312009-02-13 12:43:56 -05001499/*
Steven Rostedtb8489142011-05-04 09:27:52 -04001500 * Test the hashes for this ops to see if we want to call
1501 * the ops->func or not.
1502 *
1503 * It's a match if the ip is in the ops->filter_hash or
1504 * the filter_hash does not exist or is empty,
1505 * AND
1506 * the ip is not in the ops->notrace_hash.
Steven Rostedtcdbe61b2011-05-05 21:14:55 -04001507 *
1508 * This needs to be called with preemption disabled as
1509 * the hashes are freed with call_rcu_sched().
Steven Rostedtb8489142011-05-04 09:27:52 -04001510 */
1511static int
Steven Rostedt (Red Hat)195a8af2013-07-23 22:06:15 -04001512ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
Steven Rostedtb8489142011-05-04 09:27:52 -04001513{
Steven Rostedt (Red Hat)fef5aee2014-07-24 12:25:47 -04001514 struct ftrace_ops_hash hash;
Steven Rostedtb8489142011-05-04 09:27:52 -04001515 int ret;
1516
Steven Rostedt (Red Hat)195a8af2013-07-23 22:06:15 -04001517#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
1518 /*
1519 * There's a small race when adding ops that the ftrace handler
1520 * that wants regs, may be called without them. We can not
1521 * allow that handler to be called if regs is NULL.
1522 */
1523 if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS))
1524 return 0;
1525#endif
1526
Steven Rostedt (Red Hat)fef5aee2014-07-24 12:25:47 -04001527 hash.filter_hash = rcu_dereference_raw_notrace(ops->func_hash->filter_hash);
1528 hash.notrace_hash = rcu_dereference_raw_notrace(ops->func_hash->notrace_hash);
Steven Rostedtb8489142011-05-04 09:27:52 -04001529
Steven Rostedt (Red Hat)fef5aee2014-07-24 12:25:47 -04001530 if (hash_contains_ip(ip, &hash))
Steven Rostedtb8489142011-05-04 09:27:52 -04001531 ret = 1;
1532 else
1533 ret = 0;
Steven Rostedtb8489142011-05-04 09:27:52 -04001534
1535 return ret;
1536}
1537
1538/*
Steven Rostedt265c8312009-02-13 12:43:56 -05001539 * This is a double for. Do not use 'break' to break out of the loop,
1540 * you must use a goto.
1541 */
1542#define do_for_each_ftrace_rec(pg, rec) \
1543 for (pg = ftrace_pages_start; pg; pg = pg->next) { \
1544 int _____i; \
1545 for (_____i = 0; _____i < pg->index; _____i++) { \
1546 rec = &pg->records[_____i];
1547
1548#define while_for_each_ftrace_rec() \
1549 } \
1550 }
Abhishek Sagarecea6562008-06-21 23:47:53 +05301551
Steven Rostedt5855fea2011-12-16 19:27:42 -05001552
1553static int ftrace_cmp_recs(const void *a, const void *b)
1554{
Steven Rostedta650e022012-04-25 13:48:13 -04001555 const struct dyn_ftrace *key = a;
1556 const struct dyn_ftrace *rec = b;
Steven Rostedt5855fea2011-12-16 19:27:42 -05001557
Steven Rostedta650e022012-04-25 13:48:13 -04001558 if (key->flags < rec->ip)
Steven Rostedt5855fea2011-12-16 19:27:42 -05001559 return -1;
Steven Rostedta650e022012-04-25 13:48:13 -04001560 if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)
1561 return 1;
1562 return 0;
1563}
1564
Steven Rostedtf0cf9732012-04-25 14:39:54 -04001565static unsigned long ftrace_location_range(unsigned long start, unsigned long end)
Steven Rostedta650e022012-04-25 13:48:13 -04001566{
1567 struct ftrace_page *pg;
1568 struct dyn_ftrace *rec;
1569 struct dyn_ftrace key;
1570
1571 key.ip = start;
1572 key.flags = end; /* overload flags, as it is unsigned long */
1573
1574 for (pg = ftrace_pages_start; pg; pg = pg->next) {
1575 if (end < pg->records[0].ip ||
1576 start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
1577 continue;
1578 rec = bsearch(&key, pg->records, pg->index,
1579 sizeof(struct dyn_ftrace),
1580 ftrace_cmp_recs);
1581 if (rec)
Steven Rostedtf0cf9732012-04-25 14:39:54 -04001582 return rec->ip;
Steven Rostedta650e022012-04-25 13:48:13 -04001583 }
1584
Steven Rostedt5855fea2011-12-16 19:27:42 -05001585 return 0;
1586}
1587
Steven Rostedtc88fd862011-08-16 09:53:39 -04001588/**
1589 * ftrace_location - return true if the ip giving is a traced location
1590 * @ip: the instruction pointer to check
1591 *
Steven Rostedtf0cf9732012-04-25 14:39:54 -04001592 * Returns rec->ip if @ip given is a pointer to a ftrace location.
Steven Rostedtc88fd862011-08-16 09:53:39 -04001593 * That is, the instruction that is either a NOP or call to
1594 * the function tracer. It checks the ftrace internal tables to
1595 * determine if the address belongs or not.
1596 */
Steven Rostedtf0cf9732012-04-25 14:39:54 -04001597unsigned long ftrace_location(unsigned long ip)
Steven Rostedtc88fd862011-08-16 09:53:39 -04001598{
Steven Rostedta650e022012-04-25 13:48:13 -04001599 return ftrace_location_range(ip, ip);
1600}
Steven Rostedtc88fd862011-08-16 09:53:39 -04001601
Steven Rostedta650e022012-04-25 13:48:13 -04001602/**
1603 * ftrace_text_reserved - return true if range contains an ftrace location
1604 * @start: start of range to search
1605 * @end: end of range to search (inclusive). @end points to the last byte to check.
1606 *
1607 * Returns 1 if @start and @end contains a ftrace location.
1608 * That is, the instruction that is either a NOP or call to
1609 * the function tracer. It checks the ftrace internal tables to
1610 * determine if the address belongs or not.
1611 */
Sasha Levind88471c2013-01-09 18:09:20 -05001612int ftrace_text_reserved(const void *start, const void *end)
Steven Rostedta650e022012-04-25 13:48:13 -04001613{
Steven Rostedtf0cf9732012-04-25 14:39:54 -04001614 unsigned long ret;
1615
1616 ret = ftrace_location_range((unsigned long)start,
1617 (unsigned long)end);
1618
1619 return (int)!!ret;
Steven Rostedtc88fd862011-08-16 09:53:39 -04001620}
1621
Steven Rostedt (Red Hat)4fbb48c2014-04-30 22:35:48 -04001622/* Test if ops registered to this rec needs regs */
1623static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec)
1624{
1625 struct ftrace_ops *ops;
1626 bool keep_regs = false;
1627
1628 for (ops = ftrace_ops_list;
1629 ops != &ftrace_list_end; ops = ops->next) {
1630 /* pass rec in as regs to have non-NULL val */
1631 if (ftrace_ops_test(ops, rec->ip, rec)) {
1632 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
1633 keep_regs = true;
1634 break;
1635 }
1636 }
1637 }
1638
1639 return keep_regs;
1640}
1641
Steven Rostedted926f92011-05-03 13:25:24 -04001642static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
1643 int filter_hash,
1644 bool inc)
1645{
1646 struct ftrace_hash *hash;
1647 struct ftrace_hash *other_hash;
1648 struct ftrace_page *pg;
1649 struct dyn_ftrace *rec;
1650 int count = 0;
1651 int all = 0;
1652
1653 /* Only update if the ops has been registered */
1654 if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1655 return;
1656
1657 /*
1658 * In the filter_hash case:
1659 * If the count is zero, we update all records.
1660 * Otherwise we just update the items in the hash.
1661 *
1662 * In the notrace_hash case:
1663 * We enable the update in the hash.
1664 * As disabling notrace means enabling the tracing,
1665 * and enabling notrace means disabling, the inc variable
1666 * gets inversed.
1667 */
1668 if (filter_hash) {
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04001669 hash = ops->func_hash->filter_hash;
1670 other_hash = ops->func_hash->notrace_hash;
Steven Rostedt06a51d92011-12-19 19:07:36 -05001671 if (ftrace_hash_empty(hash))
Steven Rostedted926f92011-05-03 13:25:24 -04001672 all = 1;
1673 } else {
1674 inc = !inc;
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04001675 hash = ops->func_hash->notrace_hash;
1676 other_hash = ops->func_hash->filter_hash;
Steven Rostedted926f92011-05-03 13:25:24 -04001677 /*
1678 * If the notrace hash has no items,
1679 * then there's nothing to do.
1680 */
Steven Rostedt06a51d92011-12-19 19:07:36 -05001681 if (ftrace_hash_empty(hash))
Steven Rostedted926f92011-05-03 13:25:24 -04001682 return;
1683 }
1684
1685 do_for_each_ftrace_rec(pg, rec) {
1686 int in_other_hash = 0;
1687 int in_hash = 0;
1688 int match = 0;
1689
1690 if (all) {
1691 /*
1692 * Only the filter_hash affects all records.
1693 * Update if the record is not in the notrace hash.
1694 */
Steven Rostedtb8489142011-05-04 09:27:52 -04001695 if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
Steven Rostedted926f92011-05-03 13:25:24 -04001696 match = 1;
1697 } else {
Steven Rostedt06a51d92011-12-19 19:07:36 -05001698 in_hash = !!ftrace_lookup_ip(hash, rec->ip);
1699 in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
Steven Rostedted926f92011-05-03 13:25:24 -04001700
1701 /*
Steven Rostedt (Red Hat)19eab4a2014-05-07 15:06:14 -04001702 * If filter_hash is set, we want to match all functions
1703 * that are in the hash but not in the other hash.
Steven Rostedted926f92011-05-03 13:25:24 -04001704 *
Steven Rostedt (Red Hat)19eab4a2014-05-07 15:06:14 -04001705 * If filter_hash is not set, then we are decrementing.
1706 * That means we match anything that is in the hash
1707 * and also in the other_hash. That is, we need to turn
1708 * off functions in the other hash because they are disabled
1709 * by this hash.
Steven Rostedted926f92011-05-03 13:25:24 -04001710 */
1711 if (filter_hash && in_hash && !in_other_hash)
1712 match = 1;
1713 else if (!filter_hash && in_hash &&
Steven Rostedt06a51d92011-12-19 19:07:36 -05001714 (in_other_hash || ftrace_hash_empty(other_hash)))
Steven Rostedted926f92011-05-03 13:25:24 -04001715 match = 1;
1716 }
1717 if (!match)
1718 continue;
1719
1720 if (inc) {
1721 rec->flags++;
Steven Rostedt (Red Hat)0376bde2014-05-07 13:46:45 -04001722 if (FTRACE_WARN_ON(ftrace_rec_count(rec) == FTRACE_REF_MAX))
Steven Rostedted926f92011-05-03 13:25:24 -04001723 return;
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04001724
1725 /*
1726 * If there's only a single callback registered to a
1727 * function, and the ops has a trampoline registered
1728 * for it, then we can call it directly.
1729 */
Steven Rostedt (Red Hat)fef5aee2014-07-24 12:25:47 -04001730 if (ftrace_rec_count(rec) == 1 && ops->trampoline)
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04001731 rec->flags |= FTRACE_FL_TRAMP;
Steven Rostedt (Red Hat)fef5aee2014-07-24 12:25:47 -04001732 else
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04001733 /*
1734 * If we are adding another function callback
1735 * to this function, and the previous had a
Steven Rostedt (Red Hat)bce0b6c2014-08-20 23:57:04 -04001736 * custom trampoline in use, then we need to go
1737 * back to the default trampoline.
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04001738 */
Steven Rostedt (Red Hat)fef5aee2014-07-24 12:25:47 -04001739 rec->flags &= ~FTRACE_FL_TRAMP;
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04001740
Steven Rostedt08f6fba2012-04-30 16:20:23 -04001741 /*
1742 * If any ops wants regs saved for this function
1743 * then all ops will get saved regs.
1744 */
1745 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
1746 rec->flags |= FTRACE_FL_REGS;
Steven Rostedted926f92011-05-03 13:25:24 -04001747 } else {
Steven Rostedt (Red Hat)0376bde2014-05-07 13:46:45 -04001748 if (FTRACE_WARN_ON(ftrace_rec_count(rec) == 0))
Steven Rostedted926f92011-05-03 13:25:24 -04001749 return;
1750 rec->flags--;
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04001751
Steven Rostedt (Red Hat)4fbb48c2014-04-30 22:35:48 -04001752 /*
1753 * If the rec had REGS enabled and the ops that is
1754 * being removed had REGS set, then see if there is
1755 * still any ops for this record that wants regs.
1756 * If not, we can stop recording them.
1757 */
Steven Rostedt (Red Hat)0376bde2014-05-07 13:46:45 -04001758 if (ftrace_rec_count(rec) > 0 &&
Steven Rostedt (Red Hat)4fbb48c2014-04-30 22:35:48 -04001759 rec->flags & FTRACE_FL_REGS &&
1760 ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
1761 if (!test_rec_ops_needs_regs(rec))
1762 rec->flags &= ~FTRACE_FL_REGS;
1763 }
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04001764
1765 /*
Steven Rostedt (Red Hat)fef5aee2014-07-24 12:25:47 -04001766 * If the rec had TRAMP enabled, then it needs to
1767 * be cleared. As TRAMP can only be enabled iff
1768 * there is only a single ops attached to it.
1769 * In otherwords, always disable it on decrementing.
1770 * In the future, we may set it if rec count is
1771 * decremented to one, and the ops that is left
1772 * has a trampoline.
1773 */
1774 rec->flags &= ~FTRACE_FL_TRAMP;
1775
1776 /*
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04001777 * flags will be cleared in ftrace_check_record()
1778 * if rec count is zero.
1779 */
Steven Rostedted926f92011-05-03 13:25:24 -04001780 }
1781 count++;
1782 /* Shortcut, if we handled all records, we are done. */
1783 if (!all && count == hash->count)
1784 return;
1785 } while_for_each_ftrace_rec();
1786}
1787
1788static void ftrace_hash_rec_disable(struct ftrace_ops *ops,
1789 int filter_hash)
1790{
1791 __ftrace_hash_rec_update(ops, filter_hash, 0);
1792}
1793
1794static void ftrace_hash_rec_enable(struct ftrace_ops *ops,
1795 int filter_hash)
1796{
1797 __ftrace_hash_rec_update(ops, filter_hash, 1);
1798}
1799
Steven Rostedt (Red Hat)84261912014-08-18 13:21:08 -04001800static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops,
1801 int filter_hash, int inc)
1802{
1803 struct ftrace_ops *op;
1804
1805 __ftrace_hash_rec_update(ops, filter_hash, inc);
1806
1807 if (ops->func_hash != &global_ops.local_hash)
1808 return;
1809
1810 /*
1811 * If the ops shares the global_ops hash, then we need to update
1812 * all ops that are enabled and use this hash.
1813 */
1814 do_for_each_ftrace_op(op, ftrace_ops_list) {
1815 /* Already done */
1816 if (op == ops)
1817 continue;
1818 if (op->func_hash == &global_ops.local_hash)
1819 __ftrace_hash_rec_update(op, filter_hash, inc);
1820 } while_for_each_ftrace_op(op);
1821}
1822
1823static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops,
1824 int filter_hash)
1825{
1826 ftrace_hash_rec_update_modify(ops, filter_hash, 0);
1827}
1828
1829static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops,
1830 int filter_hash)
1831{
1832 ftrace_hash_rec_update_modify(ops, filter_hash, 1);
1833}
1834
Masami Hiramatsuf8b8be82014-11-21 05:25:16 -05001835/*
1836 * Try to update IPMODIFY flag on each ftrace_rec. Return 0 if it is OK
1837 * or no-needed to update, -EBUSY if it detects a conflict of the flag
1838 * on a ftrace_rec, and -EINVAL if the new_hash tries to trace all recs.
1839 * Note that old_hash and new_hash has below meanings
1840 * - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected)
1841 * - If the hash is EMPTY_HASH, it hits nothing
1842 * - Anything else hits the recs which match the hash entries.
1843 */
1844static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops,
1845 struct ftrace_hash *old_hash,
1846 struct ftrace_hash *new_hash)
1847{
1848 struct ftrace_page *pg;
1849 struct dyn_ftrace *rec, *end = NULL;
1850 int in_old, in_new;
1851
1852 /* Only update if the ops has been registered */
1853 if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1854 return 0;
1855
1856 if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY))
1857 return 0;
1858
1859 /*
1860 * Since the IPMODIFY is a very address sensitive action, we do not
1861 * allow ftrace_ops to set all functions to new hash.
1862 */
1863 if (!new_hash || !old_hash)
1864 return -EINVAL;
1865
1866 /* Update rec->flags */
1867 do_for_each_ftrace_rec(pg, rec) {
1868 /* We need to update only differences of filter_hash */
1869 in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
1870 in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
1871 if (in_old == in_new)
1872 continue;
1873
1874 if (in_new) {
1875 /* New entries must ensure no others are using it */
1876 if (rec->flags & FTRACE_FL_IPMODIFY)
1877 goto rollback;
1878 rec->flags |= FTRACE_FL_IPMODIFY;
1879 } else /* Removed entry */
1880 rec->flags &= ~FTRACE_FL_IPMODIFY;
1881 } while_for_each_ftrace_rec();
1882
1883 return 0;
1884
1885rollback:
1886 end = rec;
1887
1888 /* Roll back what we did above */
1889 do_for_each_ftrace_rec(pg, rec) {
1890 if (rec == end)
1891 goto err_out;
1892
1893 in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
1894 in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
1895 if (in_old == in_new)
1896 continue;
1897
1898 if (in_new)
1899 rec->flags &= ~FTRACE_FL_IPMODIFY;
1900 else
1901 rec->flags |= FTRACE_FL_IPMODIFY;
1902 } while_for_each_ftrace_rec();
1903
1904err_out:
1905 return -EBUSY;
1906}
1907
1908static int ftrace_hash_ipmodify_enable(struct ftrace_ops *ops)
1909{
1910 struct ftrace_hash *hash = ops->func_hash->filter_hash;
1911
1912 if (ftrace_hash_empty(hash))
1913 hash = NULL;
1914
1915 return __ftrace_hash_update_ipmodify(ops, EMPTY_HASH, hash);
1916}
1917
1918/* Disabling always succeeds */
1919static void ftrace_hash_ipmodify_disable(struct ftrace_ops *ops)
1920{
1921 struct ftrace_hash *hash = ops->func_hash->filter_hash;
1922
1923 if (ftrace_hash_empty(hash))
1924 hash = NULL;
1925
1926 __ftrace_hash_update_ipmodify(ops, hash, EMPTY_HASH);
1927}
1928
1929static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
1930 struct ftrace_hash *new_hash)
1931{
1932 struct ftrace_hash *old_hash = ops->func_hash->filter_hash;
1933
1934 if (ftrace_hash_empty(old_hash))
1935 old_hash = NULL;
1936
1937 if (ftrace_hash_empty(new_hash))
1938 new_hash = NULL;
1939
1940 return __ftrace_hash_update_ipmodify(ops, old_hash, new_hash);
1941}
1942
Steven Rostedt05736a42008-09-22 14:55:47 -07001943static void print_ip_ins(const char *fmt, unsigned char *p)
1944{
1945 int i;
1946
1947 printk(KERN_CONT "%s", fmt);
1948
1949 for (i = 0; i < MCOUNT_INSN_SIZE; i++)
1950 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
1951}
1952
Steven Rostedt (Red Hat)4fd32792014-10-24 17:56:04 -04001953static struct ftrace_ops *
1954ftrace_find_tramp_ops_any(struct dyn_ftrace *rec);
1955
Steven Rostedtc88fd862011-08-16 09:53:39 -04001956/**
1957 * ftrace_bug - report and shutdown function tracer
1958 * @failed: The failed type (EFAULT, EINVAL, EPERM)
Steven Rostedt (Red Hat)4fd32792014-10-24 17:56:04 -04001959 * @rec: The record that failed
Steven Rostedtc88fd862011-08-16 09:53:39 -04001960 *
1961 * The arch code that enables or disables the function tracing
1962 * can call ftrace_bug() when it has detected a problem in
1963 * modifying the code. @failed should be one of either:
1964 * EFAULT - if the problem happens on reading the @ip address
1965 * EINVAL - if what is read at @ip is not what was expected
1966 * EPERM - if the problem happens on writting to the @ip address
1967 */
Steven Rostedt (Red Hat)4fd32792014-10-24 17:56:04 -04001968void ftrace_bug(int failed, struct dyn_ftrace *rec)
Steven Rostedtb17e8a32008-11-14 16:21:19 -08001969{
Steven Rostedt (Red Hat)4fd32792014-10-24 17:56:04 -04001970 unsigned long ip = rec ? rec->ip : 0;
1971
Steven Rostedtb17e8a32008-11-14 16:21:19 -08001972 switch (failed) {
1973 case -EFAULT:
1974 FTRACE_WARN_ON_ONCE(1);
1975 pr_info("ftrace faulted on modifying ");
1976 print_ip_sym(ip);
1977 break;
1978 case -EINVAL:
1979 FTRACE_WARN_ON_ONCE(1);
1980 pr_info("ftrace failed to modify ");
1981 print_ip_sym(ip);
Steven Rostedtb17e8a32008-11-14 16:21:19 -08001982 print_ip_ins(" actual: ", (unsigned char *)ip);
Steven Rostedt (Red Hat)4fd32792014-10-24 17:56:04 -04001983 pr_cont("\n");
Steven Rostedtb17e8a32008-11-14 16:21:19 -08001984 break;
1985 case -EPERM:
1986 FTRACE_WARN_ON_ONCE(1);
1987 pr_info("ftrace faulted on writing ");
1988 print_ip_sym(ip);
1989 break;
1990 default:
1991 FTRACE_WARN_ON_ONCE(1);
1992 pr_info("ftrace faulted on unknown error ");
1993 print_ip_sym(ip);
1994 }
Steven Rostedt (Red Hat)4fd32792014-10-24 17:56:04 -04001995 if (rec) {
1996 struct ftrace_ops *ops = NULL;
1997
1998 pr_info("ftrace record flags: %lx\n", rec->flags);
1999 pr_cont(" (%ld)%s", ftrace_rec_count(rec),
2000 rec->flags & FTRACE_FL_REGS ? " R" : " ");
2001 if (rec->flags & FTRACE_FL_TRAMP_EN) {
2002 ops = ftrace_find_tramp_ops_any(rec);
2003 if (ops)
2004 pr_cont("\ttramp: %pS",
2005 (void *)ops->trampoline);
2006 else
2007 pr_cont("\ttramp: ERROR!");
2008
2009 }
2010 ip = ftrace_get_addr_curr(rec);
2011 pr_cont(" expected tramp: %lx\n", ip);
2012 }
Steven Rostedtb17e8a32008-11-14 16:21:19 -08002013}
2014
Steven Rostedtc88fd862011-08-16 09:53:39 -04002015static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update)
Steven Rostedt5072c592008-05-12 21:20:43 +02002016{
Xiao Guangrong64fbcd12009-07-15 12:32:15 +08002017 unsigned long flag = 0UL;
Frederic Weisbeckere7d37372008-11-16 06:02:06 +01002018
Steven Rostedt982c3502008-11-15 16:31:41 -05002019 /*
Jiri Olsa30fb6aa2011-12-05 18:22:48 +01002020 * If we are updating calls:
Steven Rostedt982c3502008-11-15 16:31:41 -05002021 *
Steven Rostedted926f92011-05-03 13:25:24 -04002022 * If the record has a ref count, then we need to enable it
2023 * because someone is using it.
Steven Rostedt982c3502008-11-15 16:31:41 -05002024 *
Steven Rostedted926f92011-05-03 13:25:24 -04002025 * Otherwise we make sure its disabled.
2026 *
Jiri Olsa30fb6aa2011-12-05 18:22:48 +01002027 * If we are disabling calls, then disable all records that
Steven Rostedted926f92011-05-03 13:25:24 -04002028 * are enabled.
Steven Rostedt982c3502008-11-15 16:31:41 -05002029 */
Steven Rostedt (Red Hat)0376bde2014-05-07 13:46:45 -04002030 if (enable && ftrace_rec_count(rec))
Steven Rostedted926f92011-05-03 13:25:24 -04002031 flag = FTRACE_FL_ENABLED;
Steven Rostedt5072c592008-05-12 21:20:43 +02002032
Steven Rostedt08f6fba2012-04-30 16:20:23 -04002033 /*
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04002034 * If enabling and the REGS flag does not match the REGS_EN, or
2035 * the TRAMP flag doesn't match the TRAMP_EN, then do not ignore
2036 * this record. Set flags to fail the compare against ENABLED.
Steven Rostedt08f6fba2012-04-30 16:20:23 -04002037 */
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04002038 if (flag) {
2039 if (!(rec->flags & FTRACE_FL_REGS) !=
2040 !(rec->flags & FTRACE_FL_REGS_EN))
2041 flag |= FTRACE_FL_REGS;
2042
2043 if (!(rec->flags & FTRACE_FL_TRAMP) !=
2044 !(rec->flags & FTRACE_FL_TRAMP_EN))
2045 flag |= FTRACE_FL_TRAMP;
2046 }
Steven Rostedt08f6fba2012-04-30 16:20:23 -04002047
Xiao Guangrong64fbcd12009-07-15 12:32:15 +08002048 /* If the state of this record hasn't changed, then do nothing */
2049 if ((rec->flags & FTRACE_FL_ENABLED) == flag)
Steven Rostedtc88fd862011-08-16 09:53:39 -04002050 return FTRACE_UPDATE_IGNORE;
Xiao Guangrong64fbcd12009-07-15 12:32:15 +08002051
2052 if (flag) {
Steven Rostedt08f6fba2012-04-30 16:20:23 -04002053 /* Save off if rec is being enabled (for return value) */
2054 flag ^= rec->flags & FTRACE_FL_ENABLED;
2055
2056 if (update) {
Steven Rostedtc88fd862011-08-16 09:53:39 -04002057 rec->flags |= FTRACE_FL_ENABLED;
Steven Rostedt08f6fba2012-04-30 16:20:23 -04002058 if (flag & FTRACE_FL_REGS) {
2059 if (rec->flags & FTRACE_FL_REGS)
2060 rec->flags |= FTRACE_FL_REGS_EN;
2061 else
2062 rec->flags &= ~FTRACE_FL_REGS_EN;
2063 }
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04002064 if (flag & FTRACE_FL_TRAMP) {
2065 if (rec->flags & FTRACE_FL_TRAMP)
2066 rec->flags |= FTRACE_FL_TRAMP_EN;
2067 else
2068 rec->flags &= ~FTRACE_FL_TRAMP_EN;
2069 }
Steven Rostedt08f6fba2012-04-30 16:20:23 -04002070 }
2071
2072 /*
2073 * If this record is being updated from a nop, then
2074 * return UPDATE_MAKE_CALL.
Steven Rostedt08f6fba2012-04-30 16:20:23 -04002075 * Otherwise,
2076 * return UPDATE_MODIFY_CALL to tell the caller to convert
Steven Rostedt (Red Hat)f1b2f2b2014-05-07 16:09:49 -04002077 * from the save regs, to a non-save regs function or
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04002078 * vice versa, or from a trampoline call.
Steven Rostedt08f6fba2012-04-30 16:20:23 -04002079 */
2080 if (flag & FTRACE_FL_ENABLED)
2081 return FTRACE_UPDATE_MAKE_CALL;
Steven Rostedt (Red Hat)f1b2f2b2014-05-07 16:09:49 -04002082
2083 return FTRACE_UPDATE_MODIFY_CALL;
Xiao Guangrong64fbcd12009-07-15 12:32:15 +08002084 }
2085
Steven Rostedt08f6fba2012-04-30 16:20:23 -04002086 if (update) {
2087 /* If there's no more users, clear all flags */
Steven Rostedt (Red Hat)0376bde2014-05-07 13:46:45 -04002088 if (!ftrace_rec_count(rec))
Steven Rostedt08f6fba2012-04-30 16:20:23 -04002089 rec->flags = 0;
2090 else
Steven Rostedt (Red Hat)b24d4432015-03-04 23:10:28 -05002091 /*
2092 * Just disable the record, but keep the ops TRAMP
2093 * and REGS states. The _EN flags must be disabled though.
2094 */
2095 rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN |
2096 FTRACE_FL_REGS_EN);
Steven Rostedt08f6fba2012-04-30 16:20:23 -04002097 }
Steven Rostedtc88fd862011-08-16 09:53:39 -04002098
2099 return FTRACE_UPDATE_MAKE_NOP;
2100}
2101
2102/**
2103 * ftrace_update_record, set a record that now is tracing or not
2104 * @rec: the record to update
2105 * @enable: set to 1 if the record is tracing, zero to force disable
2106 *
2107 * The records that represent all functions that can be traced need
2108 * to be updated when tracing has been enabled.
2109 */
2110int ftrace_update_record(struct dyn_ftrace *rec, int enable)
2111{
2112 return ftrace_check_record(rec, enable, 1);
2113}
2114
2115/**
2116 * ftrace_test_record, check if the record has been enabled or not
2117 * @rec: the record to test
2118 * @enable: set to 1 to check if enabled, 0 if it is disabled
2119 *
2120 * The arch code may need to test if a record is already set to
2121 * tracing to determine how to modify the function code that it
2122 * represents.
2123 */
2124int ftrace_test_record(struct dyn_ftrace *rec, int enable)
2125{
2126 return ftrace_check_record(rec, enable, 0);
2127}
2128
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04002129static struct ftrace_ops *
Steven Rostedt (Red Hat)5fecaa02014-07-24 16:00:31 -04002130ftrace_find_tramp_ops_any(struct dyn_ftrace *rec)
2131{
2132 struct ftrace_ops *op;
Steven Rostedt (Red Hat)fef5aee2014-07-24 12:25:47 -04002133 unsigned long ip = rec->ip;
Steven Rostedt (Red Hat)5fecaa02014-07-24 16:00:31 -04002134
2135 do_for_each_ftrace_op(op, ftrace_ops_list) {
2136
2137 if (!op->trampoline)
2138 continue;
2139
Steven Rostedt (Red Hat)fef5aee2014-07-24 12:25:47 -04002140 if (hash_contains_ip(ip, op->func_hash))
Steven Rostedt (Red Hat)5fecaa02014-07-24 16:00:31 -04002141 return op;
2142 } while_for_each_ftrace_op(op);
2143
2144 return NULL;
2145}
2146
2147static struct ftrace_ops *
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04002148ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec)
2149{
2150 struct ftrace_ops *op;
Steven Rostedt (Red Hat)fef5aee2014-07-24 12:25:47 -04002151 unsigned long ip = rec->ip;
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04002152
Steven Rostedt (Red Hat)fef5aee2014-07-24 12:25:47 -04002153 /*
2154 * Need to check removed ops first.
2155 * If they are being removed, and this rec has a tramp,
2156 * and this rec is in the ops list, then it would be the
2157 * one with the tramp.
2158 */
2159 if (removed_ops) {
2160 if (hash_contains_ip(ip, &removed_ops->old_hash))
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04002161 return removed_ops;
2162 }
2163
Steven Rostedt (Red Hat)fef5aee2014-07-24 12:25:47 -04002164 /*
2165 * Need to find the current trampoline for a rec.
2166 * Now, a trampoline is only attached to a rec if there
2167 * was a single 'ops' attached to it. But this can be called
2168 * when we are adding another op to the rec or removing the
2169 * current one. Thus, if the op is being added, we can
2170 * ignore it because it hasn't attached itself to the rec
Steven Rostedt (Red Hat)4fc40902014-10-24 14:48:35 -04002171 * yet.
2172 *
2173 * If an ops is being modified (hooking to different functions)
2174 * then we don't care about the new functions that are being
2175 * added, just the old ones (that are probably being removed).
2176 *
2177 * If we are adding an ops to a function that already is using
2178 * a trampoline, it needs to be removed (trampolines are only
2179 * for single ops connected), then an ops that is not being
2180 * modified also needs to be checked.
Steven Rostedt (Red Hat)fef5aee2014-07-24 12:25:47 -04002181 */
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04002182 do_for_each_ftrace_op(op, ftrace_ops_list) {
Steven Rostedt (Red Hat)fef5aee2014-07-24 12:25:47 -04002183
2184 if (!op->trampoline)
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04002185 continue;
2186
Steven Rostedt (Red Hat)fef5aee2014-07-24 12:25:47 -04002187 /*
2188 * If the ops is being added, it hasn't gotten to
2189 * the point to be removed from this tree yet.
2190 */
2191 if (op->flags & FTRACE_OPS_FL_ADDING)
2192 continue;
2193
Steven Rostedt (Red Hat)fef5aee2014-07-24 12:25:47 -04002194
Steven Rostedt (Red Hat)4fc40902014-10-24 14:48:35 -04002195 /*
2196 * If the ops is being modified and is in the old
2197 * hash, then it is probably being removed from this
2198 * function.
2199 */
Steven Rostedt (Red Hat)fef5aee2014-07-24 12:25:47 -04002200 if ((op->flags & FTRACE_OPS_FL_MODIFYING) &&
2201 hash_contains_ip(ip, &op->old_hash))
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04002202 return op;
Steven Rostedt (Red Hat)4fc40902014-10-24 14:48:35 -04002203 /*
2204 * If the ops is not being added or modified, and it's
2205 * in its normal filter hash, then this must be the one
2206 * we want!
2207 */
2208 if (!(op->flags & FTRACE_OPS_FL_MODIFYING) &&
2209 hash_contains_ip(ip, op->func_hash))
2210 return op;
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04002211
2212 } while_for_each_ftrace_op(op);
2213
2214 return NULL;
2215}
2216
2217static struct ftrace_ops *
2218ftrace_find_tramp_ops_new(struct dyn_ftrace *rec)
2219{
2220 struct ftrace_ops *op;
Steven Rostedt (Red Hat)fef5aee2014-07-24 12:25:47 -04002221 unsigned long ip = rec->ip;
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04002222
2223 do_for_each_ftrace_op(op, ftrace_ops_list) {
2224 /* pass rec in as regs to have non-NULL val */
Steven Rostedt (Red Hat)fef5aee2014-07-24 12:25:47 -04002225 if (hash_contains_ip(ip, op->func_hash))
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04002226 return op;
2227 } while_for_each_ftrace_op(op);
2228
2229 return NULL;
2230}
2231
Steven Rostedt (Red Hat)7413af12014-05-06 21:34:14 -04002232/**
2233 * ftrace_get_addr_new - Get the call address to set to
2234 * @rec: The ftrace record descriptor
2235 *
2236 * If the record has the FTRACE_FL_REGS set, that means that it
2237 * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS
2238 * is not not set, then it wants to convert to the normal callback.
2239 *
2240 * Returns the address of the trampoline to set to
2241 */
2242unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec)
2243{
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04002244 struct ftrace_ops *ops;
2245
2246 /* Trampolines take precedence over regs */
2247 if (rec->flags & FTRACE_FL_TRAMP) {
2248 ops = ftrace_find_tramp_ops_new(rec);
2249 if (FTRACE_WARN_ON(!ops || !ops->trampoline)) {
Steven Rostedt (Red Hat)bce0b6c2014-08-20 23:57:04 -04002250 pr_warn("Bad trampoline accounting at: %p (%pS) (%lx)\n",
2251 (void *)rec->ip, (void *)rec->ip, rec->flags);
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04002252 /* Ftrace is shutting down, return anything */
2253 return (unsigned long)FTRACE_ADDR;
2254 }
2255 return ops->trampoline;
2256 }
2257
Steven Rostedt (Red Hat)7413af12014-05-06 21:34:14 -04002258 if (rec->flags & FTRACE_FL_REGS)
2259 return (unsigned long)FTRACE_REGS_ADDR;
2260 else
2261 return (unsigned long)FTRACE_ADDR;
2262}
2263
2264/**
2265 * ftrace_get_addr_curr - Get the call address that is already there
2266 * @rec: The ftrace record descriptor
2267 *
2268 * The FTRACE_FL_REGS_EN is set when the record already points to
2269 * a function that saves all the regs. Basically the '_EN' version
2270 * represents the current state of the function.
2271 *
2272 * Returns the address of the trampoline that is currently being called
2273 */
2274unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec)
2275{
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04002276 struct ftrace_ops *ops;
2277
2278 /* Trampolines take precedence over regs */
2279 if (rec->flags & FTRACE_FL_TRAMP_EN) {
2280 ops = ftrace_find_tramp_ops_curr(rec);
2281 if (FTRACE_WARN_ON(!ops)) {
2282 pr_warning("Bad trampoline accounting at: %p (%pS)\n",
2283 (void *)rec->ip, (void *)rec->ip);
2284 /* Ftrace is shutting down, return anything */
2285 return (unsigned long)FTRACE_ADDR;
2286 }
2287 return ops->trampoline;
2288 }
2289
Steven Rostedt (Red Hat)7413af12014-05-06 21:34:14 -04002290 if (rec->flags & FTRACE_FL_REGS_EN)
2291 return (unsigned long)FTRACE_REGS_ADDR;
2292 else
2293 return (unsigned long)FTRACE_ADDR;
2294}
2295
Steven Rostedtc88fd862011-08-16 09:53:39 -04002296static int
2297__ftrace_replace_code(struct dyn_ftrace *rec, int enable)
2298{
Steven Rostedt08f6fba2012-04-30 16:20:23 -04002299 unsigned long ftrace_old_addr;
Steven Rostedtc88fd862011-08-16 09:53:39 -04002300 unsigned long ftrace_addr;
2301 int ret;
2302
Steven Rostedt (Red Hat)7c0868e2014-05-08 07:01:21 -04002303 ftrace_addr = ftrace_get_addr_new(rec);
Steven Rostedtc88fd862011-08-16 09:53:39 -04002304
Steven Rostedt (Red Hat)7c0868e2014-05-08 07:01:21 -04002305 /* This needs to be done before we call ftrace_update_record */
2306 ftrace_old_addr = ftrace_get_addr_curr(rec);
2307
2308 ret = ftrace_update_record(rec, enable);
Steven Rostedt08f6fba2012-04-30 16:20:23 -04002309
Steven Rostedtc88fd862011-08-16 09:53:39 -04002310 switch (ret) {
2311 case FTRACE_UPDATE_IGNORE:
2312 return 0;
2313
2314 case FTRACE_UPDATE_MAKE_CALL:
2315 return ftrace_make_call(rec, ftrace_addr);
2316
2317 case FTRACE_UPDATE_MAKE_NOP:
Steven Rostedt (Red Hat)39b55522014-08-17 20:59:10 -04002318 return ftrace_make_nop(NULL, rec, ftrace_old_addr);
Steven Rostedt08f6fba2012-04-30 16:20:23 -04002319
Steven Rostedt08f6fba2012-04-30 16:20:23 -04002320 case FTRACE_UPDATE_MODIFY_CALL:
Steven Rostedt08f6fba2012-04-30 16:20:23 -04002321 return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);
Steven Rostedtc88fd862011-08-16 09:53:39 -04002322 }
2323
2324 return -1; /* unknow ftrace bug */
Steven Rostedt5072c592008-05-12 21:20:43 +02002325}
2326
Steven Rostedte4f5d542012-04-27 09:13:18 -04002327void __weak ftrace_replace_code(int enable)
Steven Rostedt3c1720f2008-05-12 21:20:43 +02002328{
Steven Rostedt3c1720f2008-05-12 21:20:43 +02002329 struct dyn_ftrace *rec;
2330 struct ftrace_page *pg;
Steven Rostedt6a24a242009-02-17 11:20:26 -05002331 int failed;
Steven Rostedt3c1720f2008-05-12 21:20:43 +02002332
Steven Rostedt45a4a232011-04-21 23:16:46 -04002333 if (unlikely(ftrace_disabled))
2334 return;
2335
Steven Rostedt265c8312009-02-13 12:43:56 -05002336 do_for_each_ftrace_rec(pg, rec) {
Steven Rostedte4f5d542012-04-27 09:13:18 -04002337 failed = __ftrace_replace_code(rec, enable);
Zhaoleifa9d13c2009-03-13 17:16:34 +08002338 if (failed) {
Steven Rostedt (Red Hat)4fd32792014-10-24 17:56:04 -04002339 ftrace_bug(failed, rec);
Steven Rostedt3279ba32009-10-07 16:57:56 -04002340 /* Stop processing */
2341 return;
Steven Rostedt265c8312009-02-13 12:43:56 -05002342 }
2343 } while_for_each_ftrace_rec();
Steven Rostedt3c1720f2008-05-12 21:20:43 +02002344}
2345
Steven Rostedtc88fd862011-08-16 09:53:39 -04002346struct ftrace_rec_iter {
2347 struct ftrace_page *pg;
2348 int index;
2349};
2350
2351/**
2352 * ftrace_rec_iter_start, start up iterating over traced functions
2353 *
2354 * Returns an iterator handle that is used to iterate over all
2355 * the records that represent address locations where functions
2356 * are traced.
2357 *
2358 * May return NULL if no records are available.
2359 */
2360struct ftrace_rec_iter *ftrace_rec_iter_start(void)
2361{
2362 /*
2363 * We only use a single iterator.
2364 * Protected by the ftrace_lock mutex.
2365 */
2366 static struct ftrace_rec_iter ftrace_rec_iter;
2367 struct ftrace_rec_iter *iter = &ftrace_rec_iter;
2368
2369 iter->pg = ftrace_pages_start;
2370 iter->index = 0;
2371
2372 /* Could have empty pages */
2373 while (iter->pg && !iter->pg->index)
2374 iter->pg = iter->pg->next;
2375
2376 if (!iter->pg)
2377 return NULL;
2378
2379 return iter;
2380}
2381
2382/**
2383 * ftrace_rec_iter_next, get the next record to process.
2384 * @iter: The handle to the iterator.
2385 *
2386 * Returns the next iterator after the given iterator @iter.
2387 */
2388struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter)
2389{
2390 iter->index++;
2391
2392 if (iter->index >= iter->pg->index) {
2393 iter->pg = iter->pg->next;
2394 iter->index = 0;
2395
2396 /* Could have empty pages */
2397 while (iter->pg && !iter->pg->index)
2398 iter->pg = iter->pg->next;
2399 }
2400
2401 if (!iter->pg)
2402 return NULL;
2403
2404 return iter;
2405}
2406
2407/**
2408 * ftrace_rec_iter_record, get the record at the iterator location
2409 * @iter: The current iterator location
2410 *
2411 * Returns the record that the current @iter is at.
2412 */
2413struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter)
2414{
2415 return &iter->pg->records[iter->index];
2416}
2417
Abhishek Sagar492a7ea52008-05-25 00:10:04 +05302418static int
Steven Rostedt31e88902008-11-14 16:21:19 -08002419ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
Steven Rostedt3c1720f2008-05-12 21:20:43 +02002420{
Steven Rostedt593eb8a2008-10-23 09:32:59 -04002421 int ret;
Steven Rostedt3c1720f2008-05-12 21:20:43 +02002422
Steven Rostedt45a4a232011-04-21 23:16:46 -04002423 if (unlikely(ftrace_disabled))
2424 return 0;
2425
Shaohua Li25aac9d2009-01-09 11:29:40 +08002426 ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
Steven Rostedt593eb8a2008-10-23 09:32:59 -04002427 if (ret) {
Steven Rostedt (Red Hat)4fd32792014-10-24 17:56:04 -04002428 ftrace_bug(ret, rec);
Abhishek Sagar492a7ea52008-05-25 00:10:04 +05302429 return 0;
Steven Rostedt37ad50842008-05-12 21:20:48 +02002430 }
Abhishek Sagar492a7ea52008-05-25 00:10:04 +05302431 return 1;
Steven Rostedt3c1720f2008-05-12 21:20:43 +02002432}
2433
Steven Rostedt000ab692009-02-17 13:35:06 -05002434/*
2435 * archs can override this function if they must do something
2436 * before the modifying code is performed.
2437 */
2438int __weak ftrace_arch_code_modify_prepare(void)
2439{
2440 return 0;
2441}
2442
2443/*
2444 * archs can override this function if they must do something
2445 * after the modifying code is performed.
2446 */
2447int __weak ftrace_arch_code_modify_post_process(void)
2448{
2449 return 0;
2450}
2451
Steven Rostedt8ed3e2c2012-04-26 14:59:43 -04002452void ftrace_modify_all_code(int command)
2453{
Steven Rostedt (Red Hat)59338f72013-08-31 01:04:07 -04002454 int update = command & FTRACE_UPDATE_TRACE_FUNC;
Petr Mladekcd21067f2014-02-24 17:12:21 +01002455 int err = 0;
Steven Rostedt (Red Hat)59338f72013-08-31 01:04:07 -04002456
2457 /*
2458 * If the ftrace_caller calls a ftrace_ops func directly,
2459 * we need to make sure that it only traces functions it
2460 * expects to trace. When doing the switch of functions,
2461 * we need to update to the ftrace_ops_list_func first
2462 * before the transition between old and new calls are set,
2463 * as the ftrace_ops_list_func will check the ops hashes
2464 * to make sure the ops are having the right functions
2465 * traced.
2466 */
Petr Mladekcd21067f2014-02-24 17:12:21 +01002467 if (update) {
2468 err = ftrace_update_ftrace_func(ftrace_ops_list_func);
2469 if (FTRACE_WARN_ON(err))
2470 return;
2471 }
Steven Rostedt (Red Hat)59338f72013-08-31 01:04:07 -04002472
Steven Rostedt8ed3e2c2012-04-26 14:59:43 -04002473 if (command & FTRACE_UPDATE_CALLS)
2474 ftrace_replace_code(1);
2475 else if (command & FTRACE_DISABLE_CALLS)
2476 ftrace_replace_code(0);
2477
Steven Rostedt (Red Hat)405e1d82013-11-08 14:17:30 -05002478 if (update && ftrace_trace_function != ftrace_ops_list_func) {
2479 function_trace_op = set_function_trace_op;
2480 smp_wmb();
2481 /* If irqs are disabled, we are in stop machine */
2482 if (!irqs_disabled())
2483 smp_call_function(ftrace_sync_ipi, NULL, 1);
Petr Mladekcd21067f2014-02-24 17:12:21 +01002484 err = ftrace_update_ftrace_func(ftrace_trace_function);
2485 if (FTRACE_WARN_ON(err))
2486 return;
Steven Rostedt (Red Hat)405e1d82013-11-08 14:17:30 -05002487 }
Steven Rostedt8ed3e2c2012-04-26 14:59:43 -04002488
2489 if (command & FTRACE_START_FUNC_RET)
Petr Mladekcd21067f2014-02-24 17:12:21 +01002490 err = ftrace_enable_ftrace_graph_caller();
Steven Rostedt8ed3e2c2012-04-26 14:59:43 -04002491 else if (command & FTRACE_STOP_FUNC_RET)
Petr Mladekcd21067f2014-02-24 17:12:21 +01002492 err = ftrace_disable_ftrace_graph_caller();
2493 FTRACE_WARN_ON(err);
Steven Rostedt8ed3e2c2012-04-26 14:59:43 -04002494}
2495
Ingo Molnare309b412008-05-12 21:20:51 +02002496static int __ftrace_modify_code(void *data)
Steven Rostedt3d083392008-05-12 21:20:42 +02002497{
Steven Rostedtd61f82d2008-05-12 21:20:43 +02002498 int *command = data;
2499
Steven Rostedt8ed3e2c2012-04-26 14:59:43 -04002500 ftrace_modify_all_code(*command);
Steven Rostedt5a45cfe2008-11-26 00:16:24 -05002501
Steven Rostedtc88fd862011-08-16 09:53:39 -04002502 return 0;
2503}
2504
2505/**
2506 * ftrace_run_stop_machine, go back to the stop machine method
2507 * @command: The command to tell ftrace what to do
2508 *
2509 * If an arch needs to fall back to the stop machine method, the
2510 * it can call this function.
2511 */
2512void ftrace_run_stop_machine(int command)
2513{
2514 stop_machine(__ftrace_modify_code, &command, NULL);
2515}
2516
2517/**
2518 * arch_ftrace_update_code, modify the code to trace or not trace
2519 * @command: The command that needs to be done
2520 *
2521 * Archs can override this function if it does not need to
2522 * run stop_machine() to modify code.
2523 */
2524void __weak arch_ftrace_update_code(int command)
2525{
2526 ftrace_run_stop_machine(command);
2527}
2528
2529static void ftrace_run_update_code(int command)
2530{
2531 int ret;
2532
2533 ret = ftrace_arch_code_modify_prepare();
2534 FTRACE_WARN_ON(ret);
2535 if (ret)
2536 return;
Steven Rostedtc88fd862011-08-16 09:53:39 -04002537
2538 /*
2539 * By default we use stop_machine() to modify the code.
2540 * But archs can do what ever they want as long as it
2541 * is safe. The stop_machine() is the safest, but also
2542 * produces the most overhead.
2543 */
2544 arch_ftrace_update_code(command);
2545
Steven Rostedt000ab692009-02-17 13:35:06 -05002546 ret = ftrace_arch_code_modify_post_process();
2547 FTRACE_WARN_ON(ret);
Steven Rostedt3d083392008-05-12 21:20:42 +02002548}
2549
Steven Rostedt (Red Hat)8252ecf2014-10-24 14:56:01 -04002550static void ftrace_run_modify_code(struct ftrace_ops *ops, int command,
Steven Rostedt (Red Hat)7485058e2015-01-13 14:03:38 -05002551 struct ftrace_ops_hash *old_hash)
Steven Rostedt (Red Hat)e1effa02014-08-05 17:19:38 -04002552{
2553 ops->flags |= FTRACE_OPS_FL_MODIFYING;
Steven Rostedt (Red Hat)7485058e2015-01-13 14:03:38 -05002554 ops->old_hash.filter_hash = old_hash->filter_hash;
2555 ops->old_hash.notrace_hash = old_hash->notrace_hash;
Steven Rostedt (Red Hat)e1effa02014-08-05 17:19:38 -04002556 ftrace_run_update_code(command);
Steven Rostedt (Red Hat)8252ecf2014-10-24 14:56:01 -04002557 ops->old_hash.filter_hash = NULL;
Steven Rostedt (Red Hat)7485058e2015-01-13 14:03:38 -05002558 ops->old_hash.notrace_hash = NULL;
Steven Rostedt (Red Hat)e1effa02014-08-05 17:19:38 -04002559 ops->flags &= ~FTRACE_OPS_FL_MODIFYING;
2560}
2561
Steven Rostedtd61f82d2008-05-12 21:20:43 +02002562static ftrace_func_t saved_ftrace_func;
Steven Rostedt60a7ecf2008-11-05 16:05:44 -05002563static int ftrace_start_up;
Steven Rostedtdf4fc312008-11-26 00:16:23 -05002564
Steven Rostedt (Red Hat)12cce592014-07-03 15:48:16 -04002565void __weak arch_ftrace_trampoline_free(struct ftrace_ops *ops)
2566{
2567}
2568
Jiri Slabydb0fbad2014-03-10 21:42:11 +01002569static void control_ops_free(struct ftrace_ops *ops)
2570{
2571 free_percpu(ops->disabled);
2572}
2573
Steven Rostedtdf4fc312008-11-26 00:16:23 -05002574static void ftrace_startup_enable(int command)
2575{
2576 if (saved_ftrace_func != ftrace_trace_function) {
2577 saved_ftrace_func = ftrace_trace_function;
2578 command |= FTRACE_UPDATE_TRACE_FUNC;
2579 }
2580
2581 if (!command || !ftrace_enabled)
2582 return;
2583
2584 ftrace_run_update_code(command);
2585}
Steven Rostedtd61f82d2008-05-12 21:20:43 +02002586
Steven Rostedt (Red Hat)e1effa02014-08-05 17:19:38 -04002587static void ftrace_startup_all(int command)
2588{
2589 update_all_ops = true;
2590 ftrace_startup_enable(command);
2591 update_all_ops = false;
2592}
2593
Steven Rostedta1cd6172011-05-23 15:24:25 -04002594static int ftrace_startup(struct ftrace_ops *ops, int command)
Steven Rostedt3d083392008-05-12 21:20:42 +02002595{
Steven Rostedt (Red Hat)8a56d772013-11-25 20:59:46 -05002596 int ret;
Steven Rostedtb8489142011-05-04 09:27:52 -04002597
Steven Rostedt4eebcc82008-05-12 21:20:48 +02002598 if (unlikely(ftrace_disabled))
Steven Rostedta1cd6172011-05-23 15:24:25 -04002599 return -ENODEV;
Steven Rostedt4eebcc82008-05-12 21:20:48 +02002600
Steven Rostedt (Red Hat)8a56d772013-11-25 20:59:46 -05002601 ret = __register_ftrace_function(ops);
2602 if (ret)
2603 return ret;
2604
Steven Rostedt60a7ecf2008-11-05 16:05:44 -05002605 ftrace_start_up++;
Jiri Olsa30fb6aa2011-12-05 18:22:48 +01002606 command |= FTRACE_UPDATE_CALLS;
Steven Rostedt3d083392008-05-12 21:20:42 +02002607
Steven Rostedt (Red Hat)e1effa02014-08-05 17:19:38 -04002608 /*
2609 * Note that ftrace probes uses this to start up
2610 * and modify functions it will probe. But we still
2611 * set the ADDING flag for modification, as probes
2612 * do not have trampolines. If they add them in the
2613 * future, then the probes will need to distinguish
2614 * between adding and updating probes.
2615 */
2616 ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING;
Steven Rostedt (Red Hat)66209a52014-05-06 21:57:49 -04002617
Masami Hiramatsuf8b8be82014-11-21 05:25:16 -05002618 ret = ftrace_hash_ipmodify_enable(ops);
2619 if (ret < 0) {
2620 /* Rollback registration process */
2621 __unregister_ftrace_function(ops);
2622 ftrace_start_up--;
2623 ops->flags &= ~FTRACE_OPS_FL_ENABLED;
2624 return ret;
2625 }
2626
Steven Rostedt (Red Hat)66209a52014-05-06 21:57:49 -04002627 ftrace_hash_rec_enable(ops, 1);
Steven Rostedted926f92011-05-03 13:25:24 -04002628
Steven Rostedtdf4fc312008-11-26 00:16:23 -05002629 ftrace_startup_enable(command);
Steven Rostedta1cd6172011-05-23 15:24:25 -04002630
Steven Rostedt (Red Hat)e1effa02014-08-05 17:19:38 -04002631 ops->flags &= ~FTRACE_OPS_FL_ADDING;
2632
Steven Rostedta1cd6172011-05-23 15:24:25 -04002633 return 0;
Steven Rostedt3d083392008-05-12 21:20:42 +02002634}
2635
Steven Rostedt (Red Hat)8a56d772013-11-25 20:59:46 -05002636static int ftrace_shutdown(struct ftrace_ops *ops, int command)
Steven Rostedt3d083392008-05-12 21:20:42 +02002637{
Steven Rostedt (Red Hat)8a56d772013-11-25 20:59:46 -05002638 int ret;
Steven Rostedtb8489142011-05-04 09:27:52 -04002639
Steven Rostedt4eebcc82008-05-12 21:20:48 +02002640 if (unlikely(ftrace_disabled))
Steven Rostedt (Red Hat)8a56d772013-11-25 20:59:46 -05002641 return -ENODEV;
2642
2643 ret = __unregister_ftrace_function(ops);
2644 if (ret)
2645 return ret;
Steven Rostedt4eebcc82008-05-12 21:20:48 +02002646
Steven Rostedt60a7ecf2008-11-05 16:05:44 -05002647 ftrace_start_up--;
Frederic Weisbecker9ea1a152009-06-20 06:52:21 +02002648 /*
2649 * Just warn in case of unbalance, no need to kill ftrace, it's not
2650 * critical but the ftrace_call callers may be never nopped again after
2651 * further ftrace uses.
2652 */
2653 WARN_ON_ONCE(ftrace_start_up < 0);
2654
Masami Hiramatsuf8b8be82014-11-21 05:25:16 -05002655 /* Disabling ipmodify never fails */
2656 ftrace_hash_ipmodify_disable(ops);
Steven Rostedt (Red Hat)66209a52014-05-06 21:57:49 -04002657 ftrace_hash_rec_disable(ops, 1);
Steven Rostedtb8489142011-05-04 09:27:52 -04002658
Namhyung Kima737e6d2014-06-12 23:56:12 +09002659 ops->flags &= ~FTRACE_OPS_FL_ENABLED;
Steven Rostedtb8489142011-05-04 09:27:52 -04002660
Jiri Olsa30fb6aa2011-12-05 18:22:48 +01002661 command |= FTRACE_UPDATE_CALLS;
Steven Rostedtd61f82d2008-05-12 21:20:43 +02002662
2663 if (saved_ftrace_func != ftrace_trace_function) {
2664 saved_ftrace_func = ftrace_trace_function;
2665 command |= FTRACE_UPDATE_TRACE_FUNC;
2666 }
2667
Steven Rostedt (Red Hat)a4c35ed22014-01-13 12:56:21 -05002668 if (!command || !ftrace_enabled) {
2669 /*
2670 * If these are control ops, they still need their
2671 * per_cpu field freed. Since, function tracing is
2672 * not currently active, we can just free them
2673 * without synchronizing all CPUs.
2674 */
2675 if (ops->flags & FTRACE_OPS_FL_CONTROL)
2676 control_ops_free(ops);
Steven Rostedt (Red Hat)8a56d772013-11-25 20:59:46 -05002677 return 0;
Steven Rostedt (Red Hat)a4c35ed22014-01-13 12:56:21 -05002678 }
Steven Rostedt3d083392008-05-12 21:20:42 +02002679
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04002680 /*
2681 * If the ops uses a trampoline, then it needs to be
2682 * tested first on update.
2683 */
Steven Rostedt (Red Hat)e1effa02014-08-05 17:19:38 -04002684 ops->flags |= FTRACE_OPS_FL_REMOVING;
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04002685 removed_ops = ops;
2686
Steven Rostedt (Red Hat)fef5aee2014-07-24 12:25:47 -04002687 /* The trampoline logic checks the old hashes */
2688 ops->old_hash.filter_hash = ops->func_hash->filter_hash;
2689 ops->old_hash.notrace_hash = ops->func_hash->notrace_hash;
2690
Steven Rostedtd61f82d2008-05-12 21:20:43 +02002691 ftrace_run_update_code(command);
Steven Rostedt (Red Hat)a4c35ed22014-01-13 12:56:21 -05002692
Steven Rostedt (Red Hat)84bde622014-09-12 14:21:13 -04002693 /*
2694 * If there's no more ops registered with ftrace, run a
2695 * sanity check to make sure all rec flags are cleared.
2696 */
2697 if (ftrace_ops_list == &ftrace_list_end) {
2698 struct ftrace_page *pg;
2699 struct dyn_ftrace *rec;
2700
2701 do_for_each_ftrace_rec(pg, rec) {
2702 if (FTRACE_WARN_ON_ONCE(rec->flags))
2703 pr_warn(" %pS flags:%lx\n",
2704 (void *)rec->ip, rec->flags);
2705 } while_for_each_ftrace_rec();
2706 }
2707
Steven Rostedt (Red Hat)fef5aee2014-07-24 12:25:47 -04002708 ops->old_hash.filter_hash = NULL;
2709 ops->old_hash.notrace_hash = NULL;
2710
2711 removed_ops = NULL;
Steven Rostedt (Red Hat)e1effa02014-08-05 17:19:38 -04002712 ops->flags &= ~FTRACE_OPS_FL_REMOVING;
Steven Rostedt (Red Hat)79922b82014-05-06 21:56:17 -04002713
Steven Rostedt (Red Hat)a4c35ed22014-01-13 12:56:21 -05002714 /*
2715 * Dynamic ops may be freed, we must make sure that all
2716 * callers are done before leaving this function.
2717 * The same goes for freeing the per_cpu data of the control
2718 * ops.
2719 *
2720 * Again, normal synchronize_sched() is not good enough.
2721 * We need to do a hard force of sched synchronization.
2722 * This is because we use preempt_disable() to do RCU, but
2723 * the function tracers can be called where RCU is not watching
2724 * (like before user_exit()). We can not rely on the RCU
2725 * infrastructure to do the synchronization, thus we must do it
2726 * ourselves.
2727 */
2728 if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_CONTROL)) {
2729 schedule_on_each_cpu(ftrace_sync);
2730
Steven Rostedt (Red Hat)12cce592014-07-03 15:48:16 -04002731 arch_ftrace_trampoline_free(ops);
2732
Steven Rostedt (Red Hat)a4c35ed22014-01-13 12:56:21 -05002733 if (ops->flags & FTRACE_OPS_FL_CONTROL)
2734 control_ops_free(ops);
2735 }
2736
Steven Rostedt (Red Hat)8a56d772013-11-25 20:59:46 -05002737 return 0;
Steven Rostedt3d083392008-05-12 21:20:42 +02002738}
2739
Ingo Molnare309b412008-05-12 21:20:51 +02002740static void ftrace_startup_sysctl(void)
Steven Rostedtb0fc4942008-05-12 21:20:43 +02002741{
Pratyush Anand1619dc32015-03-06 23:58:06 +05302742 int command;
2743
Steven Rostedt4eebcc82008-05-12 21:20:48 +02002744 if (unlikely(ftrace_disabled))
2745 return;
2746
Steven Rostedtd61f82d2008-05-12 21:20:43 +02002747 /* Force update next time */
2748 saved_ftrace_func = NULL;
Steven Rostedt60a7ecf2008-11-05 16:05:44 -05002749 /* ftrace_start_up is true if we want ftrace running */
Pratyush Anand1619dc32015-03-06 23:58:06 +05302750 if (ftrace_start_up) {
2751 command = FTRACE_UPDATE_CALLS;
2752 if (ftrace_graph_active)
2753 command |= FTRACE_START_FUNC_RET;
Steven Rostedt (Red Hat)524a3862015-03-06 19:55:13 -05002754 ftrace_startup_enable(command);
Pratyush Anand1619dc32015-03-06 23:58:06 +05302755 }
Steven Rostedtb0fc4942008-05-12 21:20:43 +02002756}
2757
Ingo Molnare309b412008-05-12 21:20:51 +02002758static void ftrace_shutdown_sysctl(void)
Steven Rostedtb0fc4942008-05-12 21:20:43 +02002759{
Pratyush Anand1619dc32015-03-06 23:58:06 +05302760 int command;
2761
Steven Rostedt4eebcc82008-05-12 21:20:48 +02002762 if (unlikely(ftrace_disabled))
2763 return;
2764
Steven Rostedt60a7ecf2008-11-05 16:05:44 -05002765 /* ftrace_start_up is true if ftrace is running */
Pratyush Anand1619dc32015-03-06 23:58:06 +05302766 if (ftrace_start_up) {
2767 command = FTRACE_DISABLE_CALLS;
2768 if (ftrace_graph_active)
2769 command |= FTRACE_STOP_FUNC_RET;
2770 ftrace_run_update_code(command);
2771 }
Steven Rostedtb0fc4942008-05-12 21:20:43 +02002772}
2773
Steven Rostedt3d083392008-05-12 21:20:42 +02002774static cycle_t ftrace_update_time;
Steven Rostedt3d083392008-05-12 21:20:42 +02002775unsigned long ftrace_update_tot_cnt;
2776
Steven Rostedt (Red Hat)8c4f3c32013-07-30 00:04:32 -04002777static inline int ops_traces_mod(struct ftrace_ops *ops)
Steven Rostedtf7bc8b62011-07-14 23:02:27 -04002778{
Steven Rostedt (Red Hat)8c4f3c32013-07-30 00:04:32 -04002779 /*
2780 * Filter_hash being empty will default to trace module.
2781 * But notrace hash requires a test of individual module functions.
2782 */
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04002783 return ftrace_hash_empty(ops->func_hash->filter_hash) &&
2784 ftrace_hash_empty(ops->func_hash->notrace_hash);
Steven Rostedt (Red Hat)8c4f3c32013-07-30 00:04:32 -04002785}
Steven Rostedtf7bc8b62011-07-14 23:02:27 -04002786
Steven Rostedt (Red Hat)8c4f3c32013-07-30 00:04:32 -04002787/*
2788 * Check if the current ops references the record.
2789 *
2790 * If the ops traces all functions, then it was already accounted for.
2791 * If the ops does not trace the current record function, skip it.
2792 * If the ops ignores the function via notrace filter, skip it.
2793 */
2794static inline bool
2795ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec)
2796{
2797 /* If ops isn't enabled, ignore it */
2798 if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
2799 return 0;
2800
2801 /* If ops traces all mods, we already accounted for it */
2802 if (ops_traces_mod(ops))
2803 return 0;
2804
2805 /* The function must be in the filter */
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04002806 if (!ftrace_hash_empty(ops->func_hash->filter_hash) &&
2807 !ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip))
Steven Rostedt (Red Hat)8c4f3c32013-07-30 00:04:32 -04002808 return 0;
2809
2810 /* If in notrace hash, we ignore it too */
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04002811 if (ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip))
Steven Rostedt (Red Hat)8c4f3c32013-07-30 00:04:32 -04002812 return 0;
2813
2814 return 1;
2815}
2816
2817static int referenced_filters(struct dyn_ftrace *rec)
2818{
2819 struct ftrace_ops *ops;
2820 int cnt = 0;
2821
2822 for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) {
2823 if (ops_references_rec(ops, rec))
2824 cnt++;
2825 }
2826
2827 return cnt;
Steven Rostedtf7bc8b62011-07-14 23:02:27 -04002828}
2829
Jiri Slaby1dc43cf2014-02-24 19:59:56 +01002830static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs)
Steven Rostedt3d083392008-05-12 21:20:42 +02002831{
Steven Rostedt85ae32a2011-12-16 16:30:31 -05002832 struct ftrace_page *pg;
Lai Jiangshane94142a2009-03-13 17:51:27 +08002833 struct dyn_ftrace *p;
Abhishek Sagarf22f9a82008-06-21 23:50:29 +05302834 cycle_t start, stop;
Jiri Slaby1dc43cf2014-02-24 19:59:56 +01002835 unsigned long update_cnt = 0;
Steven Rostedtf7bc8b62011-07-14 23:02:27 -04002836 unsigned long ref = 0;
Steven Rostedt (Red Hat)8c4f3c32013-07-30 00:04:32 -04002837 bool test = false;
Steven Rostedt85ae32a2011-12-16 16:30:31 -05002838 int i;
Steven Rostedtf7bc8b62011-07-14 23:02:27 -04002839
2840 /*
2841 * When adding a module, we need to check if tracers are
2842 * currently enabled and if they are set to trace all functions.
2843 * If they are, we need to enable the module functions as well
2844 * as update the reference counts for those function records.
2845 */
2846 if (mod) {
2847 struct ftrace_ops *ops;
2848
2849 for (ops = ftrace_ops_list;
2850 ops != &ftrace_list_end; ops = ops->next) {
Steven Rostedt (Red Hat)8c4f3c32013-07-30 00:04:32 -04002851 if (ops->flags & FTRACE_OPS_FL_ENABLED) {
2852 if (ops_traces_mod(ops))
2853 ref++;
2854 else
2855 test = true;
2856 }
Steven Rostedtf7bc8b62011-07-14 23:02:27 -04002857 }
2858 }
Steven Rostedt3d083392008-05-12 21:20:42 +02002859
Ingo Molnar750ed1a2008-05-12 21:20:46 +02002860 start = ftrace_now(raw_smp_processor_id());
Steven Rostedt3d083392008-05-12 21:20:42 +02002861
Jiri Slaby1dc43cf2014-02-24 19:59:56 +01002862 for (pg = new_pgs; pg; pg = pg->next) {
Abhishek Sagarf22f9a82008-06-21 23:50:29 +05302863
Steven Rostedt85ae32a2011-12-16 16:30:31 -05002864 for (i = 0; i < pg->index; i++) {
Steven Rostedt (Red Hat)8c4f3c32013-07-30 00:04:32 -04002865 int cnt = ref;
2866
Steven Rostedt85ae32a2011-12-16 16:30:31 -05002867 /* If something went wrong, bail without enabling anything */
2868 if (unlikely(ftrace_disabled))
2869 return -1;
Steven Rostedt3d083392008-05-12 21:20:42 +02002870
Steven Rostedt85ae32a2011-12-16 16:30:31 -05002871 p = &pg->records[i];
Steven Rostedt (Red Hat)8c4f3c32013-07-30 00:04:32 -04002872 if (test)
2873 cnt += referenced_filters(p);
2874 p->flags = cnt;
Abhishek Sagar0eb96702008-06-01 21:47:30 +05302875
Steven Rostedt85ae32a2011-12-16 16:30:31 -05002876 /*
2877 * Do the initial record conversion from mcount jump
2878 * to the NOP instructions.
2879 */
2880 if (!ftrace_code_disable(mod, p))
2881 break;
Jiri Olsa5cb084b2009-10-13 16:33:53 -04002882
Jiri Slaby1dc43cf2014-02-24 19:59:56 +01002883 update_cnt++;
Jiri Olsa5cb084b2009-10-13 16:33:53 -04002884
Steven Rostedt85ae32a2011-12-16 16:30:31 -05002885 /*
2886 * If the tracing is enabled, go ahead and enable the record.
2887 *
2888 * The reason not to enable the record immediatelly is the
2889 * inherent check of ftrace_make_nop/ftrace_make_call for
2890 * correct previous instructions. Making first the NOP
2891 * conversion puts the module to the correct state, thus
2892 * passing the ftrace_make_call check.
2893 */
Steven Rostedt (Red Hat)8c4f3c32013-07-30 00:04:32 -04002894 if (ftrace_start_up && cnt) {
Steven Rostedt85ae32a2011-12-16 16:30:31 -05002895 int failed = __ftrace_replace_code(p, 1);
2896 if (failed)
Steven Rostedt (Red Hat)4fd32792014-10-24 17:56:04 -04002897 ftrace_bug(failed, p);
Steven Rostedt85ae32a2011-12-16 16:30:31 -05002898 }
Jiri Olsa5cb084b2009-10-13 16:33:53 -04002899 }
Steven Rostedt3d083392008-05-12 21:20:42 +02002900 }
2901
Ingo Molnar750ed1a2008-05-12 21:20:46 +02002902 stop = ftrace_now(raw_smp_processor_id());
Steven Rostedt3d083392008-05-12 21:20:42 +02002903 ftrace_update_time = stop - start;
Jiri Slaby1dc43cf2014-02-24 19:59:56 +01002904 ftrace_update_tot_cnt += update_cnt;
Steven Rostedt3d083392008-05-12 21:20:42 +02002905
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +02002906 return 0;
2907}
2908
Steven Rostedta7900872011-12-16 16:23:44 -05002909static int ftrace_allocate_records(struct ftrace_page *pg, int count)
Steven Rostedt3c1720f2008-05-12 21:20:43 +02002910{
Steven Rostedta7900872011-12-16 16:23:44 -05002911 int order;
Steven Rostedt3c1720f2008-05-12 21:20:43 +02002912 int cnt;
Steven Rostedt3c1720f2008-05-12 21:20:43 +02002913
Steven Rostedta7900872011-12-16 16:23:44 -05002914 if (WARN_ON(!count))
2915 return -EINVAL;
2916
2917 order = get_count_order(DIV_ROUND_UP(count, ENTRIES_PER_PAGE));
Steven Rostedt3c1720f2008-05-12 21:20:43 +02002918
2919 /*
Steven Rostedta7900872011-12-16 16:23:44 -05002920 * We want to fill as much as possible. No more than a page
2921 * may be empty.
Steven Rostedt3c1720f2008-05-12 21:20:43 +02002922 */
Steven Rostedta7900872011-12-16 16:23:44 -05002923 while ((PAGE_SIZE << order) / ENTRY_SIZE >= count + ENTRIES_PER_PAGE)
2924 order--;
Steven Rostedt3c1720f2008-05-12 21:20:43 +02002925
Steven Rostedta7900872011-12-16 16:23:44 -05002926 again:
2927 pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
2928
2929 if (!pg->records) {
2930 /* if we can't allocate this size, try something smaller */
2931 if (!order)
2932 return -ENOMEM;
2933 order >>= 1;
2934 goto again;
2935 }
2936
2937 cnt = (PAGE_SIZE << order) / ENTRY_SIZE;
2938 pg->size = cnt;
2939
2940 if (cnt > count)
2941 cnt = count;
2942
2943 return cnt;
2944}
2945
2946static struct ftrace_page *
2947ftrace_allocate_pages(unsigned long num_to_init)
2948{
2949 struct ftrace_page *start_pg;
2950 struct ftrace_page *pg;
2951 int order;
2952 int cnt;
2953
2954 if (!num_to_init)
2955 return 0;
2956
2957 start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);
2958 if (!pg)
2959 return NULL;
2960
2961 /*
2962 * Try to allocate as much as possible in one continues
2963 * location that fills in all of the space. We want to
2964 * waste as little space as possible.
2965 */
2966 for (;;) {
2967 cnt = ftrace_allocate_records(pg, num_to_init);
2968 if (cnt < 0)
2969 goto free_pages;
2970
2971 num_to_init -= cnt;
2972 if (!num_to_init)
2973 break;
2974
2975 pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);
2976 if (!pg->next)
2977 goto free_pages;
2978
2979 pg = pg->next;
2980 }
2981
2982 return start_pg;
2983
2984 free_pages:
Namhyung Kim1f61be002014-06-11 17:06:53 +09002985 pg = start_pg;
2986 while (pg) {
Steven Rostedta7900872011-12-16 16:23:44 -05002987 order = get_count_order(pg->size / ENTRIES_PER_PAGE);
2988 free_pages((unsigned long)pg->records, order);
2989 start_pg = pg->next;
2990 kfree(pg);
2991 pg = start_pg;
2992 }
2993 pr_info("ftrace: FAILED to allocate memory for functions\n");
2994 return NULL;
2995}
2996
Steven Rostedt5072c592008-05-12 21:20:43 +02002997#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
2998
2999struct ftrace_iterator {
Steven Rostedt98c4fd02010-09-10 11:47:43 -04003000 loff_t pos;
Steven Rostedt4aeb6962010-09-09 10:00:28 -04003001 loff_t func_pos;
3002 struct ftrace_page *pg;
3003 struct dyn_ftrace *func;
3004 struct ftrace_func_probe *probe;
3005 struct trace_parser parser;
Steven Rostedt1cf41dd72011-04-29 20:59:51 -04003006 struct ftrace_hash *hash;
Steven Rostedt33dc9b12011-05-02 17:34:47 -04003007 struct ftrace_ops *ops;
Steven Rostedt4aeb6962010-09-09 10:00:28 -04003008 int hidx;
3009 int idx;
3010 unsigned flags;
Steven Rostedt5072c592008-05-12 21:20:43 +02003011};
3012
Ingo Molnare309b412008-05-12 21:20:51 +02003013static void *
Steven Rostedt4aeb6962010-09-09 10:00:28 -04003014t_hash_next(struct seq_file *m, loff_t *pos)
Steven Rostedt8fc0c702009-02-16 15:28:00 -05003015{
3016 struct ftrace_iterator *iter = m->private;
Steven Rostedt4aeb6962010-09-09 10:00:28 -04003017 struct hlist_node *hnd = NULL;
Steven Rostedt8fc0c702009-02-16 15:28:00 -05003018 struct hlist_head *hhd;
3019
Steven Rostedt8fc0c702009-02-16 15:28:00 -05003020 (*pos)++;
Steven Rostedt98c4fd02010-09-10 11:47:43 -04003021 iter->pos = *pos;
Steven Rostedt8fc0c702009-02-16 15:28:00 -05003022
Steven Rostedt4aeb6962010-09-09 10:00:28 -04003023 if (iter->probe)
3024 hnd = &iter->probe->node;
Steven Rostedt8fc0c702009-02-16 15:28:00 -05003025 retry:
3026 if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
3027 return NULL;
3028
3029 hhd = &ftrace_func_hash[iter->hidx];
3030
3031 if (hlist_empty(hhd)) {
3032 iter->hidx++;
3033 hnd = NULL;
3034 goto retry;
3035 }
3036
3037 if (!hnd)
3038 hnd = hhd->first;
3039 else {
3040 hnd = hnd->next;
3041 if (!hnd) {
3042 iter->hidx++;
3043 goto retry;
3044 }
3045 }
3046
Steven Rostedt4aeb6962010-09-09 10:00:28 -04003047 if (WARN_ON_ONCE(!hnd))
3048 return NULL;
3049
3050 iter->probe = hlist_entry(hnd, struct ftrace_func_probe, node);
3051
3052 return iter;
Steven Rostedt8fc0c702009-02-16 15:28:00 -05003053}
3054
3055static void *t_hash_start(struct seq_file *m, loff_t *pos)
3056{
3057 struct ftrace_iterator *iter = m->private;
3058 void *p = NULL;
Li Zefand82d6242009-06-24 09:54:54 +08003059 loff_t l;
3060
Steven Rostedt69a30832011-12-19 15:21:16 -05003061 if (!(iter->flags & FTRACE_ITER_DO_HASH))
3062 return NULL;
3063
Steven Rostedt2bccfff2010-09-09 08:43:22 -04003064 if (iter->func_pos > *pos)
3065 return NULL;
Steven Rostedt8fc0c702009-02-16 15:28:00 -05003066
Li Zefand82d6242009-06-24 09:54:54 +08003067 iter->hidx = 0;
Steven Rostedt2bccfff2010-09-09 08:43:22 -04003068 for (l = 0; l <= (*pos - iter->func_pos); ) {
Steven Rostedt4aeb6962010-09-09 10:00:28 -04003069 p = t_hash_next(m, &l);
Li Zefand82d6242009-06-24 09:54:54 +08003070 if (!p)
3071 break;
3072 }
Steven Rostedt4aeb6962010-09-09 10:00:28 -04003073 if (!p)
3074 return NULL;
3075
Steven Rostedt98c4fd02010-09-10 11:47:43 -04003076 /* Only set this if we have an item */
3077 iter->flags |= FTRACE_ITER_HASH;
3078
Steven Rostedt4aeb6962010-09-09 10:00:28 -04003079 return iter;
Steven Rostedt8fc0c702009-02-16 15:28:00 -05003080}
3081
Steven Rostedt4aeb6962010-09-09 10:00:28 -04003082static int
3083t_hash_show(struct seq_file *m, struct ftrace_iterator *iter)
Steven Rostedt8fc0c702009-02-16 15:28:00 -05003084{
Steven Rostedtb6887d72009-02-17 12:32:04 -05003085 struct ftrace_func_probe *rec;
Steven Rostedt8fc0c702009-02-16 15:28:00 -05003086
Steven Rostedt4aeb6962010-09-09 10:00:28 -04003087 rec = iter->probe;
3088 if (WARN_ON_ONCE(!rec))
3089 return -EIO;
Steven Rostedt8fc0c702009-02-16 15:28:00 -05003090
Steven Rostedt809dcf22009-02-16 23:06:01 -05003091 if (rec->ops->print)
3092 return rec->ops->print(m, rec->ip, rec->ops, rec->data);
3093
Steven Rostedtb375a112009-09-17 00:05:58 -04003094 seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
Steven Rostedt8fc0c702009-02-16 15:28:00 -05003095
3096 if (rec->data)
3097 seq_printf(m, ":%p", rec->data);
3098 seq_putc(m, '\n');
3099
3100 return 0;
3101}
3102
3103static void *
Steven Rostedt5072c592008-05-12 21:20:43 +02003104t_next(struct seq_file *m, void *v, loff_t *pos)
3105{
3106 struct ftrace_iterator *iter = m->private;
Steven Rostedtfc13cb02011-12-19 14:41:25 -05003107 struct ftrace_ops *ops = iter->ops;
Steven Rostedt5072c592008-05-12 21:20:43 +02003108 struct dyn_ftrace *rec = NULL;
3109
Steven Rostedt45a4a232011-04-21 23:16:46 -04003110 if (unlikely(ftrace_disabled))
3111 return NULL;
3112
Steven Rostedt8fc0c702009-02-16 15:28:00 -05003113 if (iter->flags & FTRACE_ITER_HASH)
Steven Rostedt4aeb6962010-09-09 10:00:28 -04003114 return t_hash_next(m, pos);
Steven Rostedt8fc0c702009-02-16 15:28:00 -05003115
Steven Rostedt5072c592008-05-12 21:20:43 +02003116 (*pos)++;
Jiri Olsa1106b692011-02-16 17:35:34 +01003117 iter->pos = iter->func_pos = *pos;
Steven Rostedt5072c592008-05-12 21:20:43 +02003118
Steven Rostedt0c75a3e2009-02-16 11:21:52 -05003119 if (iter->flags & FTRACE_ITER_PRINTALL)
Steven Rostedt57c072c2010-09-14 11:21:11 -04003120 return t_hash_start(m, pos);
Steven Rostedt0c75a3e2009-02-16 11:21:52 -05003121
Steven Rostedt5072c592008-05-12 21:20:43 +02003122 retry:
3123 if (iter->idx >= iter->pg->index) {
3124 if (iter->pg->next) {
3125 iter->pg = iter->pg->next;
3126 iter->idx = 0;
3127 goto retry;
3128 }
3129 } else {
3130 rec = &iter->pg->records[iter->idx++];
Steven Rostedt320823092011-12-16 14:42:37 -05003131 if (((iter->flags & FTRACE_ITER_FILTER) &&
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04003132 !(ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip))) ||
Steven Rostedt0183fb1c2008-11-07 22:36:02 -05003133
Steven Rostedt41c52c02008-05-22 11:46:33 -04003134 ((iter->flags & FTRACE_ITER_NOTRACE) &&
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04003135 !ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip)) ||
Steven Rostedt647bcd02011-05-03 14:39:21 -04003136
3137 ((iter->flags & FTRACE_ITER_ENABLED) &&
Steven Rostedt (Red Hat)23ea9c42013-05-09 19:31:48 -04003138 !(rec->flags & FTRACE_FL_ENABLED))) {
Steven Rostedt647bcd02011-05-03 14:39:21 -04003139
Steven Rostedt5072c592008-05-12 21:20:43 +02003140 rec = NULL;
3141 goto retry;
3142 }
3143 }
3144
Steven Rostedt4aeb6962010-09-09 10:00:28 -04003145 if (!rec)
Steven Rostedt57c072c2010-09-14 11:21:11 -04003146 return t_hash_start(m, pos);
Steven Rostedt4aeb6962010-09-09 10:00:28 -04003147
3148 iter->func = rec;
3149
3150 return iter;
Steven Rostedt5072c592008-05-12 21:20:43 +02003151}
3152
Steven Rostedt98c4fd02010-09-10 11:47:43 -04003153static void reset_iter_read(struct ftrace_iterator *iter)
3154{
3155 iter->pos = 0;
3156 iter->func_pos = 0;
Dan Carpenter70f77b3f2012-06-09 19:10:27 +03003157 iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_HASH);
Steven Rostedt5072c592008-05-12 21:20:43 +02003158}
3159
3160static void *t_start(struct seq_file *m, loff_t *pos)
3161{
3162 struct ftrace_iterator *iter = m->private;
Steven Rostedtfc13cb02011-12-19 14:41:25 -05003163 struct ftrace_ops *ops = iter->ops;
Steven Rostedt5072c592008-05-12 21:20:43 +02003164 void *p = NULL;
Li Zefan694ce0a2009-06-24 09:54:19 +08003165 loff_t l;
Steven Rostedt5072c592008-05-12 21:20:43 +02003166
Steven Rostedt8fc0c702009-02-16 15:28:00 -05003167 mutex_lock(&ftrace_lock);
Steven Rostedt45a4a232011-04-21 23:16:46 -04003168
3169 if (unlikely(ftrace_disabled))
3170 return NULL;
3171
Steven Rostedt0c75a3e2009-02-16 11:21:52 -05003172 /*
Steven Rostedt98c4fd02010-09-10 11:47:43 -04003173 * If an lseek was done, then reset and start from beginning.
3174 */
3175 if (*pos < iter->pos)
3176 reset_iter_read(iter);
3177
3178 /*
Steven Rostedt0c75a3e2009-02-16 11:21:52 -05003179 * For set_ftrace_filter reading, if we have the filter
3180 * off, we can short cut and just print out that all
3181 * functions are enabled.
3182 */
Namhyung Kim8c006cf2014-06-13 16:24:06 +09003183 if ((iter->flags & FTRACE_ITER_FILTER &&
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04003184 ftrace_hash_empty(ops->func_hash->filter_hash)) ||
Namhyung Kim8c006cf2014-06-13 16:24:06 +09003185 (iter->flags & FTRACE_ITER_NOTRACE &&
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04003186 ftrace_hash_empty(ops->func_hash->notrace_hash))) {
Steven Rostedt0c75a3e2009-02-16 11:21:52 -05003187 if (*pos > 0)
Steven Rostedt8fc0c702009-02-16 15:28:00 -05003188 return t_hash_start(m, pos);
Steven Rostedt0c75a3e2009-02-16 11:21:52 -05003189 iter->flags |= FTRACE_ITER_PRINTALL;
Chris Wrightdf091622010-09-09 16:34:59 -07003190 /* reset in case of seek/pread */
3191 iter->flags &= ~FTRACE_ITER_HASH;
Steven Rostedt0c75a3e2009-02-16 11:21:52 -05003192 return iter;
3193 }
3194
Steven Rostedt8fc0c702009-02-16 15:28:00 -05003195 if (iter->flags & FTRACE_ITER_HASH)
3196 return t_hash_start(m, pos);
3197
Steven Rostedt98c4fd02010-09-10 11:47:43 -04003198 /*
3199 * Unfortunately, we need to restart at ftrace_pages_start
3200 * every time we let go of the ftrace_mutex. This is because
3201 * those pointers can change without the lock.
3202 */
Li Zefan694ce0a2009-06-24 09:54:19 +08003203 iter->pg = ftrace_pages_start;
3204 iter->idx = 0;
3205 for (l = 0; l <= *pos; ) {
3206 p = t_next(m, p, &l);
3207 if (!p)
3208 break;
Liming Wang50cdaf02008-11-28 12:13:21 +08003209 }
walimis5821e1b2008-11-15 15:19:06 +08003210
Steven Rostedt69a30832011-12-19 15:21:16 -05003211 if (!p)
3212 return t_hash_start(m, pos);
Steven Rostedt4aeb6962010-09-09 10:00:28 -04003213
3214 return iter;
Steven Rostedt5072c592008-05-12 21:20:43 +02003215}
3216
3217static void t_stop(struct seq_file *m, void *p)
3218{
Steven Rostedt8fc0c702009-02-16 15:28:00 -05003219 mutex_unlock(&ftrace_lock);
Steven Rostedt5072c592008-05-12 21:20:43 +02003220}
3221
Steven Rostedt (Red Hat)15d5b022014-07-03 14:51:36 -04003222void * __weak
3223arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
3224{
3225 return NULL;
3226}
3227
3228static void add_trampoline_func(struct seq_file *m, struct ftrace_ops *ops,
3229 struct dyn_ftrace *rec)
3230{
3231 void *ptr;
3232
3233 ptr = arch_ftrace_trampoline_func(ops, rec);
3234 if (ptr)
3235 seq_printf(m, " ->%pS", ptr);
3236}
3237
Steven Rostedt5072c592008-05-12 21:20:43 +02003238static int t_show(struct seq_file *m, void *v)
3239{
Steven Rostedt0c75a3e2009-02-16 11:21:52 -05003240 struct ftrace_iterator *iter = m->private;
Steven Rostedt4aeb6962010-09-09 10:00:28 -04003241 struct dyn_ftrace *rec;
Steven Rostedt5072c592008-05-12 21:20:43 +02003242
Steven Rostedt8fc0c702009-02-16 15:28:00 -05003243 if (iter->flags & FTRACE_ITER_HASH)
Steven Rostedt4aeb6962010-09-09 10:00:28 -04003244 return t_hash_show(m, iter);
Steven Rostedt8fc0c702009-02-16 15:28:00 -05003245
Steven Rostedt0c75a3e2009-02-16 11:21:52 -05003246 if (iter->flags & FTRACE_ITER_PRINTALL) {
Namhyung Kim8c006cf2014-06-13 16:24:06 +09003247 if (iter->flags & FTRACE_ITER_NOTRACE)
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01003248 seq_puts(m, "#### no functions disabled ####\n");
Namhyung Kim8c006cf2014-06-13 16:24:06 +09003249 else
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01003250 seq_puts(m, "#### all functions enabled ####\n");
Steven Rostedt0c75a3e2009-02-16 11:21:52 -05003251 return 0;
3252 }
3253
Steven Rostedt4aeb6962010-09-09 10:00:28 -04003254 rec = iter->func;
3255
Steven Rostedt5072c592008-05-12 21:20:43 +02003256 if (!rec)
3257 return 0;
3258
Steven Rostedt647bcd02011-05-03 14:39:21 -04003259 seq_printf(m, "%ps", (void *)rec->ip);
Steven Rostedt (Red Hat)9674b2f2014-05-09 16:54:59 -04003260 if (iter->flags & FTRACE_ITER_ENABLED) {
Steven Rostedt (Red Hat)15d5b022014-07-03 14:51:36 -04003261 struct ftrace_ops *ops = NULL;
3262
Masami Hiramatsuf8b8be82014-11-21 05:25:16 -05003263 seq_printf(m, " (%ld)%s%s",
Steven Rostedt (Red Hat)0376bde2014-05-07 13:46:45 -04003264 ftrace_rec_count(rec),
Masami Hiramatsuf8b8be82014-11-21 05:25:16 -05003265 rec->flags & FTRACE_FL_REGS ? " R" : " ",
3266 rec->flags & FTRACE_FL_IPMODIFY ? " I" : " ");
Steven Rostedt (Red Hat)9674b2f2014-05-09 16:54:59 -04003267 if (rec->flags & FTRACE_FL_TRAMP_EN) {
Steven Rostedt (Red Hat)5fecaa02014-07-24 16:00:31 -04003268 ops = ftrace_find_tramp_ops_any(rec);
Steven Rostedt (Red Hat)fef5aee2014-07-24 12:25:47 -04003269 if (ops)
Steven Rostedt (Red Hat)9674b2f2014-05-09 16:54:59 -04003270 seq_printf(m, "\ttramp: %pS",
3271 (void *)ops->trampoline);
3272 else
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01003273 seq_puts(m, "\ttramp: ERROR!");
Steven Rostedt (Red Hat)15d5b022014-07-03 14:51:36 -04003274
Steven Rostedt (Red Hat)9674b2f2014-05-09 16:54:59 -04003275 }
Steven Rostedt (Red Hat)15d5b022014-07-03 14:51:36 -04003276 add_trampoline_func(m, ops, rec);
Steven Rostedt (Red Hat)9674b2f2014-05-09 16:54:59 -04003277 }
3278
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01003279 seq_putc(m, '\n');
Steven Rostedt5072c592008-05-12 21:20:43 +02003280
3281 return 0;
3282}
3283
James Morris88e9d342009-09-22 16:43:43 -07003284static const struct seq_operations show_ftrace_seq_ops = {
Steven Rostedt5072c592008-05-12 21:20:43 +02003285 .start = t_start,
3286 .next = t_next,
3287 .stop = t_stop,
3288 .show = t_show,
3289};
3290
Ingo Molnare309b412008-05-12 21:20:51 +02003291static int
Steven Rostedt5072c592008-05-12 21:20:43 +02003292ftrace_avail_open(struct inode *inode, struct file *file)
3293{
3294 struct ftrace_iterator *iter;
Steven Rostedt5072c592008-05-12 21:20:43 +02003295
Steven Rostedt4eebcc82008-05-12 21:20:48 +02003296 if (unlikely(ftrace_disabled))
3297 return -ENODEV;
3298
Jiri Olsa50e18b92012-04-25 10:23:39 +02003299 iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
3300 if (iter) {
3301 iter->pg = ftrace_pages_start;
3302 iter->ops = &global_ops;
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003303 }
Steven Rostedt5072c592008-05-12 21:20:43 +02003304
Jiri Olsa50e18b92012-04-25 10:23:39 +02003305 return iter ? 0 : -ENOMEM;
Steven Rostedt5072c592008-05-12 21:20:43 +02003306}
3307
Steven Rostedt647bcd02011-05-03 14:39:21 -04003308static int
3309ftrace_enabled_open(struct inode *inode, struct file *file)
3310{
3311 struct ftrace_iterator *iter;
Steven Rostedt647bcd02011-05-03 14:39:21 -04003312
Jiri Olsa50e18b92012-04-25 10:23:39 +02003313 iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
3314 if (iter) {
3315 iter->pg = ftrace_pages_start;
3316 iter->flags = FTRACE_ITER_ENABLED;
3317 iter->ops = &global_ops;
Steven Rostedt647bcd02011-05-03 14:39:21 -04003318 }
3319
Jiri Olsa50e18b92012-04-25 10:23:39 +02003320 return iter ? 0 : -ENOMEM;
Steven Rostedt647bcd02011-05-03 14:39:21 -04003321}
3322
Steven Rostedtfc13cb02011-12-19 14:41:25 -05003323/**
3324 * ftrace_regex_open - initialize function tracer filter files
3325 * @ops: The ftrace_ops that hold the hash filters
3326 * @flag: The type of filter to process
3327 * @inode: The inode, usually passed in to your open routine
3328 * @file: The file, usually passed in to your open routine
3329 *
3330 * ftrace_regex_open() initializes the filter files for the
3331 * @ops. Depending on @flag it may process the filter hash or
3332 * the notrace hash of @ops. With this called from the open
3333 * routine, you can use ftrace_filter_write() for the write
3334 * routine if @flag has FTRACE_ITER_FILTER set, or
3335 * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set.
Steven Rostedt (Red Hat)098c8792013-12-21 17:39:40 -05003336 * tracing_lseek() should be used as the lseek routine, and
Steven Rostedtfc13cb02011-12-19 14:41:25 -05003337 * release must call ftrace_regex_release().
3338 */
3339int
Steven Rostedtf45948e2011-05-02 12:29:25 -04003340ftrace_regex_open(struct ftrace_ops *ops, int flag,
Steven Rostedt1cf41dd72011-04-29 20:59:51 -04003341 struct inode *inode, struct file *file)
Steven Rostedt5072c592008-05-12 21:20:43 +02003342{
3343 struct ftrace_iterator *iter;
Steven Rostedtf45948e2011-05-02 12:29:25 -04003344 struct ftrace_hash *hash;
Steven Rostedt5072c592008-05-12 21:20:43 +02003345 int ret = 0;
3346
Masami Hiramatsuf04f24fb2013-05-09 14:44:17 +09003347 ftrace_ops_init(ops);
3348
Steven Rostedt4eebcc82008-05-12 21:20:48 +02003349 if (unlikely(ftrace_disabled))
3350 return -ENODEV;
3351
Steven Rostedt5072c592008-05-12 21:20:43 +02003352 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3353 if (!iter)
3354 return -ENOMEM;
3355
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02003356 if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
3357 kfree(iter);
3358 return -ENOMEM;
3359 }
3360
Masami Hiramatsu3f2367b2013-05-09 14:44:21 +09003361 iter->ops = ops;
3362 iter->flags = flag;
3363
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04003364 mutex_lock(&ops->func_hash->regex_lock);
Masami Hiramatsu3f2367b2013-05-09 14:44:21 +09003365
Steven Rostedtf45948e2011-05-02 12:29:25 -04003366 if (flag & FTRACE_ITER_NOTRACE)
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04003367 hash = ops->func_hash->notrace_hash;
Steven Rostedtf45948e2011-05-02 12:29:25 -04003368 else
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04003369 hash = ops->func_hash->filter_hash;
Steven Rostedtf45948e2011-05-02 12:29:25 -04003370
Steven Rostedt33dc9b12011-05-02 17:34:47 -04003371 if (file->f_mode & FMODE_WRITE) {
Namhyung Kimef2fbe12014-06-11 17:06:54 +09003372 const int size_bits = FTRACE_HASH_DEFAULT_BITS;
3373
3374 if (file->f_flags & O_TRUNC)
3375 iter->hash = alloc_ftrace_hash(size_bits);
3376 else
3377 iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash);
3378
Steven Rostedt33dc9b12011-05-02 17:34:47 -04003379 if (!iter->hash) {
3380 trace_parser_put(&iter->parser);
3381 kfree(iter);
Masami Hiramatsu3f2367b2013-05-09 14:44:21 +09003382 ret = -ENOMEM;
3383 goto out_unlock;
Steven Rostedt33dc9b12011-05-02 17:34:47 -04003384 }
3385 }
Steven Rostedt1cf41dd72011-04-29 20:59:51 -04003386
Steven Rostedt5072c592008-05-12 21:20:43 +02003387 if (file->f_mode & FMODE_READ) {
3388 iter->pg = ftrace_pages_start;
Steven Rostedt5072c592008-05-12 21:20:43 +02003389
3390 ret = seq_open(file, &show_ftrace_seq_ops);
3391 if (!ret) {
3392 struct seq_file *m = file->private_data;
3393 m->private = iter;
Li Zefan79fe2492009-09-22 13:54:28 +08003394 } else {
Steven Rostedt33dc9b12011-05-02 17:34:47 -04003395 /* Failed */
3396 free_ftrace_hash(iter->hash);
Li Zefan79fe2492009-09-22 13:54:28 +08003397 trace_parser_put(&iter->parser);
Steven Rostedt5072c592008-05-12 21:20:43 +02003398 kfree(iter);
Li Zefan79fe2492009-09-22 13:54:28 +08003399 }
Steven Rostedt5072c592008-05-12 21:20:43 +02003400 } else
3401 file->private_data = iter;
Masami Hiramatsu3f2367b2013-05-09 14:44:21 +09003402
3403 out_unlock:
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04003404 mutex_unlock(&ops->func_hash->regex_lock);
Steven Rostedt5072c592008-05-12 21:20:43 +02003405
3406 return ret;
3407}
3408
Steven Rostedt41c52c02008-05-22 11:46:33 -04003409static int
3410ftrace_filter_open(struct inode *inode, struct file *file)
3411{
Steven Rostedt (Red Hat)e3b3e2e2013-11-11 23:07:14 -05003412 struct ftrace_ops *ops = inode->i_private;
3413
3414 return ftrace_regex_open(ops,
Steven Rostedt69a30832011-12-19 15:21:16 -05003415 FTRACE_ITER_FILTER | FTRACE_ITER_DO_HASH,
3416 inode, file);
Steven Rostedt41c52c02008-05-22 11:46:33 -04003417}
3418
3419static int
3420ftrace_notrace_open(struct inode *inode, struct file *file)
3421{
Steven Rostedt (Red Hat)e3b3e2e2013-11-11 23:07:14 -05003422 struct ftrace_ops *ops = inode->i_private;
3423
3424 return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE,
Steven Rostedt1cf41dd72011-04-29 20:59:51 -04003425 inode, file);
Steven Rostedt41c52c02008-05-22 11:46:33 -04003426}
3427
Steven Rostedt64e7c442009-02-13 17:08:48 -05003428static int ftrace_match(char *str, char *regex, int len, int type)
Steven Rostedt9f4801e2009-02-13 15:56:43 -05003429{
Steven Rostedt9f4801e2009-02-13 15:56:43 -05003430 int matched = 0;
Li Zefan751e9982010-01-14 10:53:02 +08003431 int slen;
Steven Rostedt9f4801e2009-02-13 15:56:43 -05003432
Steven Rostedt9f4801e2009-02-13 15:56:43 -05003433 switch (type) {
3434 case MATCH_FULL:
3435 if (strcmp(str, regex) == 0)
3436 matched = 1;
3437 break;
3438 case MATCH_FRONT_ONLY:
3439 if (strncmp(str, regex, len) == 0)
3440 matched = 1;
3441 break;
3442 case MATCH_MIDDLE_ONLY:
3443 if (strstr(str, regex))
3444 matched = 1;
3445 break;
3446 case MATCH_END_ONLY:
Li Zefan751e9982010-01-14 10:53:02 +08003447 slen = strlen(str);
3448 if (slen >= len && memcmp(str + slen - len, regex, len) == 0)
Steven Rostedt9f4801e2009-02-13 15:56:43 -05003449 matched = 1;
3450 break;
3451 }
3452
3453 return matched;
3454}
3455
Steven Rostedtb448c4e2011-04-29 15:12:32 -04003456static int
Steven Rostedt1cf41dd72011-04-29 20:59:51 -04003457enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int not)
Steven Rostedt996e87b2011-04-26 16:11:03 -04003458{
Steven Rostedtb448c4e2011-04-29 15:12:32 -04003459 struct ftrace_func_entry *entry;
Steven Rostedtb448c4e2011-04-29 15:12:32 -04003460 int ret = 0;
3461
Steven Rostedt1cf41dd72011-04-29 20:59:51 -04003462 entry = ftrace_lookup_ip(hash, rec->ip);
3463 if (not) {
3464 /* Do nothing if it doesn't exist */
3465 if (!entry)
3466 return 0;
3467
Steven Rostedt33dc9b12011-05-02 17:34:47 -04003468 free_hash_entry(hash, entry);
Steven Rostedtb448c4e2011-04-29 15:12:32 -04003469 } else {
Steven Rostedt1cf41dd72011-04-29 20:59:51 -04003470 /* Do nothing if it exists */
3471 if (entry)
3472 return 0;
Steven Rostedtb448c4e2011-04-29 15:12:32 -04003473
Steven Rostedt1cf41dd72011-04-29 20:59:51 -04003474 ret = add_hash_entry(hash, rec->ip);
Steven Rostedtb448c4e2011-04-29 15:12:32 -04003475 }
3476 return ret;
Steven Rostedt996e87b2011-04-26 16:11:03 -04003477}
3478
Steven Rostedt64e7c442009-02-13 17:08:48 -05003479static int
Steven Rostedtb9df92d2011-04-28 20:32:08 -04003480ftrace_match_record(struct dyn_ftrace *rec, char *mod,
3481 char *regex, int len, int type)
Steven Rostedt64e7c442009-02-13 17:08:48 -05003482{
3483 char str[KSYM_SYMBOL_LEN];
Steven Rostedtb9df92d2011-04-28 20:32:08 -04003484 char *modname;
Steven Rostedt64e7c442009-02-13 17:08:48 -05003485
Steven Rostedtb9df92d2011-04-28 20:32:08 -04003486 kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
3487
3488 if (mod) {
3489 /* module lookup requires matching the module */
3490 if (!modname || strcmp(modname, mod))
3491 return 0;
3492
3493 /* blank search means to match all funcs in the mod */
3494 if (!len)
3495 return 1;
3496 }
3497
Steven Rostedt64e7c442009-02-13 17:08:48 -05003498 return ftrace_match(str, regex, len, type);
3499}
3500
Steven Rostedt1cf41dd72011-04-29 20:59:51 -04003501static int
3502match_records(struct ftrace_hash *hash, char *buff,
3503 int len, char *mod, int not)
Steven Rostedt9f4801e2009-02-13 15:56:43 -05003504{
Steven Rostedtb9df92d2011-04-28 20:32:08 -04003505 unsigned search_len = 0;
Steven Rostedt9f4801e2009-02-13 15:56:43 -05003506 struct ftrace_page *pg;
3507 struct dyn_ftrace *rec;
Steven Rostedtb9df92d2011-04-28 20:32:08 -04003508 int type = MATCH_FULL;
3509 char *search = buff;
Li Zefan311d16d2009-12-08 11:15:11 +08003510 int found = 0;
Steven Rostedtb448c4e2011-04-29 15:12:32 -04003511 int ret;
Steven Rostedt9f4801e2009-02-13 15:56:43 -05003512
Steven Rostedtb9df92d2011-04-28 20:32:08 -04003513 if (len) {
3514 type = filter_parse_regex(buff, len, &search, &not);
3515 search_len = strlen(search);
3516 }
Steven Rostedt9f4801e2009-02-13 15:56:43 -05003517
Steven Rostedt52baf112009-02-14 01:15:39 -05003518 mutex_lock(&ftrace_lock);
Steven Rostedtb9df92d2011-04-28 20:32:08 -04003519
3520 if (unlikely(ftrace_disabled))
3521 goto out_unlock;
3522
Steven Rostedt265c8312009-02-13 12:43:56 -05003523 do_for_each_ftrace_rec(pg, rec) {
Steven Rostedtb9df92d2011-04-28 20:32:08 -04003524 if (ftrace_match_record(rec, mod, search, search_len, type)) {
Steven Rostedt1cf41dd72011-04-29 20:59:51 -04003525 ret = enter_record(hash, rec, not);
Steven Rostedtb448c4e2011-04-29 15:12:32 -04003526 if (ret < 0) {
3527 found = ret;
3528 goto out_unlock;
3529 }
Li Zefan311d16d2009-12-08 11:15:11 +08003530 found = 1;
Steven Rostedt265c8312009-02-13 12:43:56 -05003531 }
3532 } while_for_each_ftrace_rec();
Steven Rostedtb9df92d2011-04-28 20:32:08 -04003533 out_unlock:
Steven Rostedt52baf112009-02-14 01:15:39 -05003534 mutex_unlock(&ftrace_lock);
Li Zefan311d16d2009-12-08 11:15:11 +08003535
3536 return found;
Steven Rostedt5072c592008-05-12 21:20:43 +02003537}
3538
Steven Rostedt64e7c442009-02-13 17:08:48 -05003539static int
Steven Rostedt1cf41dd72011-04-29 20:59:51 -04003540ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
Steven Rostedt64e7c442009-02-13 17:08:48 -05003541{
Steven Rostedt1cf41dd72011-04-29 20:59:51 -04003542 return match_records(hash, buff, len, NULL, 0);
Steven Rostedt64e7c442009-02-13 17:08:48 -05003543}
3544
Steven Rostedt1cf41dd72011-04-29 20:59:51 -04003545static int
3546ftrace_match_module_records(struct ftrace_hash *hash, char *buff, char *mod)
Steven Rostedt64e7c442009-02-13 17:08:48 -05003547{
Steven Rostedt64e7c442009-02-13 17:08:48 -05003548 int not = 0;
Steven Rostedt6a24a242009-02-17 11:20:26 -05003549
Steven Rostedt64e7c442009-02-13 17:08:48 -05003550 /* blank or '*' mean the same */
3551 if (strcmp(buff, "*") == 0)
3552 buff[0] = 0;
3553
3554 /* handle the case of 'dont filter this module' */
3555 if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
3556 buff[0] = 0;
3557 not = 1;
3558 }
3559
Steven Rostedt1cf41dd72011-04-29 20:59:51 -04003560 return match_records(hash, buff, strlen(buff), mod, not);
Steven Rostedt64e7c442009-02-13 17:08:48 -05003561}
3562
Steven Rostedtf6180772009-02-14 00:40:25 -05003563/*
3564 * We register the module command as a template to show others how
3565 * to register the a command as well.
3566 */
3567
3568static int
Steven Rostedt43dd61c2011-07-07 11:09:22 -04003569ftrace_mod_callback(struct ftrace_hash *hash,
3570 char *func, char *cmd, char *param, int enable)
Steven Rostedtf6180772009-02-14 00:40:25 -05003571{
Dmitry Safonov5e3949f2015-09-29 19:46:12 +03003572 int ret;
Steven Rostedtf6180772009-02-14 00:40:25 -05003573
3574 /*
3575 * cmd == 'mod' because we only registered this func
3576 * for the 'mod' ftrace_func_command.
3577 * But if you register one func with multiple commands,
3578 * you can tell which command was used by the cmd
3579 * parameter.
3580 */
3581
3582 /* we must have a module name */
Dmitry Safonov5e3949f2015-09-29 19:46:12 +03003583 if (!param || !strlen(param))
3584 return -EINVAL;
Steven Rostedtf6180772009-02-14 00:40:25 -05003585
Dmitry Safonov5e3949f2015-09-29 19:46:12 +03003586 ret = ftrace_match_module_records(hash, func, param);
Steven Rostedtb448c4e2011-04-29 15:12:32 -04003587 if (!ret)
Dmitry Safonov5e3949f2015-09-29 19:46:12 +03003588 return -EINVAL;
Steven Rostedtb448c4e2011-04-29 15:12:32 -04003589 if (ret < 0)
3590 return ret;
3591
3592 return 0;
Steven Rostedtf6180772009-02-14 00:40:25 -05003593}
3594
3595static struct ftrace_func_command ftrace_mod_cmd = {
3596 .name = "mod",
3597 .func = ftrace_mod_callback,
3598};
3599
3600static int __init ftrace_mod_cmd_init(void)
3601{
3602 return register_ftrace_command(&ftrace_mod_cmd);
3603}
Steven Rostedt6f415672012-10-05 12:13:07 -04003604core_initcall(ftrace_mod_cmd_init);
Steven Rostedtf6180772009-02-14 00:40:25 -05003605
Steven Rostedt2f5f6ad2011-08-08 16:57:47 -04003606static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
Steven Rostedta1e2e312011-08-09 12:50:46 -04003607 struct ftrace_ops *op, struct pt_regs *pt_regs)
Steven Rostedt59df055f2009-02-14 15:29:06 -05003608{
Steven Rostedtb6887d72009-02-17 12:32:04 -05003609 struct ftrace_func_probe *entry;
Steven Rostedt59df055f2009-02-14 15:29:06 -05003610 struct hlist_head *hhd;
Steven Rostedt59df055f2009-02-14 15:29:06 -05003611 unsigned long key;
Steven Rostedt59df055f2009-02-14 15:29:06 -05003612
3613 key = hash_long(ip, FTRACE_HASH_BITS);
3614
3615 hhd = &ftrace_func_hash[key];
3616
3617 if (hlist_empty(hhd))
3618 return;
3619
3620 /*
3621 * Disable preemption for these calls to prevent a RCU grace
3622 * period. This syncs the hash iteration and freeing of items
3623 * on the hash. rcu_read_lock is too dangerous here.
3624 */
Steven Rostedt5168ae52010-06-03 09:36:50 -04003625 preempt_disable_notrace();
Steven Rostedt1bb539c2013-05-28 14:38:43 -04003626 hlist_for_each_entry_rcu_notrace(entry, hhd, node) {
Steven Rostedt59df055f2009-02-14 15:29:06 -05003627 if (entry->ip == ip)
3628 entry->ops->func(ip, parent_ip, &entry->data);
3629 }
Steven Rostedt5168ae52010-06-03 09:36:50 -04003630 preempt_enable_notrace();
Steven Rostedt59df055f2009-02-14 15:29:06 -05003631}
3632
Steven Rostedtb6887d72009-02-17 12:32:04 -05003633static struct ftrace_ops trace_probe_ops __read_mostly =
Steven Rostedt59df055f2009-02-14 15:29:06 -05003634{
Steven Rostedtfb9fb012009-03-25 13:26:41 -04003635 .func = function_trace_probe_call,
Masami Hiramatsuf04f24fb2013-05-09 14:44:17 +09003636 .flags = FTRACE_OPS_FL_INITIALIZED,
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04003637 INIT_OPS_HASH(trace_probe_ops)
Steven Rostedt59df055f2009-02-14 15:29:06 -05003638};
3639
Steven Rostedtb6887d72009-02-17 12:32:04 -05003640static int ftrace_probe_registered;
Steven Rostedt59df055f2009-02-14 15:29:06 -05003641
Steven Rostedt (Red Hat)7485058e2015-01-13 14:03:38 -05003642static void __enable_ftrace_function_probe(struct ftrace_ops_hash *old_hash)
Steven Rostedt59df055f2009-02-14 15:29:06 -05003643{
Steven Rostedtb8489142011-05-04 09:27:52 -04003644 int ret;
Steven Rostedt59df055f2009-02-14 15:29:06 -05003645 int i;
3646
Steven Rostedt (Red Hat)19dd6032013-05-09 19:37:36 -04003647 if (ftrace_probe_registered) {
3648 /* still need to update the function call sites */
3649 if (ftrace_enabled)
Steven Rostedt (Red Hat)8252ecf2014-10-24 14:56:01 -04003650 ftrace_run_modify_code(&trace_probe_ops, FTRACE_UPDATE_CALLS,
3651 old_hash);
Steven Rostedt59df055f2009-02-14 15:29:06 -05003652 return;
Steven Rostedt (Red Hat)19dd6032013-05-09 19:37:36 -04003653 }
Steven Rostedt59df055f2009-02-14 15:29:06 -05003654
3655 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
3656 struct hlist_head *hhd = &ftrace_func_hash[i];
3657 if (hhd->first)
3658 break;
3659 }
3660 /* Nothing registered? */
3661 if (i == FTRACE_FUNC_HASHSIZE)
3662 return;
3663
Steven Rostedt (Red Hat)8a56d772013-11-25 20:59:46 -05003664 ret = ftrace_startup(&trace_probe_ops, 0);
Steven Rostedtb8489142011-05-04 09:27:52 -04003665
Steven Rostedtb6887d72009-02-17 12:32:04 -05003666 ftrace_probe_registered = 1;
Steven Rostedt59df055f2009-02-14 15:29:06 -05003667}
3668
Steven Rostedtb6887d72009-02-17 12:32:04 -05003669static void __disable_ftrace_function_probe(void)
Steven Rostedt59df055f2009-02-14 15:29:06 -05003670{
3671 int i;
3672
Steven Rostedtb6887d72009-02-17 12:32:04 -05003673 if (!ftrace_probe_registered)
Steven Rostedt59df055f2009-02-14 15:29:06 -05003674 return;
3675
3676 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
3677 struct hlist_head *hhd = &ftrace_func_hash[i];
3678 if (hhd->first)
3679 return;
3680 }
3681
3682 /* no more funcs left */
Steven Rostedt (Red Hat)8a56d772013-11-25 20:59:46 -05003683 ftrace_shutdown(&trace_probe_ops, 0);
Steven Rostedtb8489142011-05-04 09:27:52 -04003684
Steven Rostedtb6887d72009-02-17 12:32:04 -05003685 ftrace_probe_registered = 0;
Steven Rostedt59df055f2009-02-14 15:29:06 -05003686}
3687
3688
Steven Rostedt (Red Hat)7818b382013-03-13 12:42:58 -04003689static void ftrace_free_entry(struct ftrace_func_probe *entry)
Steven Rostedt59df055f2009-02-14 15:29:06 -05003690{
Steven Rostedt59df055f2009-02-14 15:29:06 -05003691 if (entry->ops->free)
Steven Rostedt (Red Hat)e67efb92013-03-12 15:07:59 -04003692 entry->ops->free(entry->ops, entry->ip, &entry->data);
Steven Rostedt59df055f2009-02-14 15:29:06 -05003693 kfree(entry);
3694}
3695
Steven Rostedt59df055f2009-02-14 15:29:06 -05003696int
Steven Rostedtb6887d72009-02-17 12:32:04 -05003697register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
Steven Rostedt59df055f2009-02-14 15:29:06 -05003698 void *data)
3699{
Steven Rostedt (Red Hat)7485058e2015-01-13 14:03:38 -05003700 struct ftrace_ops_hash old_hash_ops;
Steven Rostedtb6887d72009-02-17 12:32:04 -05003701 struct ftrace_func_probe *entry;
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04003702 struct ftrace_hash **orig_hash = &trace_probe_ops.func_hash->filter_hash;
Steven Rostedt (Red Hat)3296fc42014-07-24 15:33:41 -04003703 struct ftrace_hash *old_hash = *orig_hash;
Steven Rostedt (Red Hat)e1df4cb2013-03-12 10:09:42 -04003704 struct ftrace_hash *hash;
Steven Rostedt59df055f2009-02-14 15:29:06 -05003705 struct ftrace_page *pg;
3706 struct dyn_ftrace *rec;
Steven Rostedt59df055f2009-02-14 15:29:06 -05003707 int type, len, not;
Steven Rostedt6a24a242009-02-17 11:20:26 -05003708 unsigned long key;
Steven Rostedt59df055f2009-02-14 15:29:06 -05003709 int count = 0;
3710 char *search;
Steven Rostedt (Red Hat)e1df4cb2013-03-12 10:09:42 -04003711 int ret;
Steven Rostedt59df055f2009-02-14 15:29:06 -05003712
Frederic Weisbecker3f6fe062009-09-24 21:31:51 +02003713 type = filter_parse_regex(glob, strlen(glob), &search, &not);
Steven Rostedt59df055f2009-02-14 15:29:06 -05003714 len = strlen(search);
3715
Steven Rostedtb6887d72009-02-17 12:32:04 -05003716 /* we do not support '!' for function probes */
Steven Rostedt59df055f2009-02-14 15:29:06 -05003717 if (WARN_ON(not))
3718 return -EINVAL;
3719
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04003720 mutex_lock(&trace_probe_ops.func_hash->regex_lock);
Steven Rostedt59df055f2009-02-14 15:29:06 -05003721
Steven Rostedt (Red Hat)7485058e2015-01-13 14:03:38 -05003722 old_hash_ops.filter_hash = old_hash;
3723 /* Probes only have filters */
3724 old_hash_ops.notrace_hash = NULL;
3725
Steven Rostedt (Red Hat)3296fc42014-07-24 15:33:41 -04003726 hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
Steven Rostedt (Red Hat)e1df4cb2013-03-12 10:09:42 -04003727 if (!hash) {
3728 count = -ENOMEM;
Steven Rostedt (Red Hat)5ae0bf52013-05-09 18:20:37 -04003729 goto out;
Steven Rostedt (Red Hat)e1df4cb2013-03-12 10:09:42 -04003730 }
3731
3732 if (unlikely(ftrace_disabled)) {
3733 count = -ENODEV;
Steven Rostedt (Red Hat)5ae0bf52013-05-09 18:20:37 -04003734 goto out;
Steven Rostedt (Red Hat)e1df4cb2013-03-12 10:09:42 -04003735 }
Steven Rostedt45a4a232011-04-21 23:16:46 -04003736
Steven Rostedt (Red Hat)5ae0bf52013-05-09 18:20:37 -04003737 mutex_lock(&ftrace_lock);
3738
Steven Rostedt59df055f2009-02-14 15:29:06 -05003739 do_for_each_ftrace_rec(pg, rec) {
3740
Steven Rostedtb9df92d2011-04-28 20:32:08 -04003741 if (!ftrace_match_record(rec, NULL, search, len, type))
Steven Rostedt59df055f2009-02-14 15:29:06 -05003742 continue;
3743
3744 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
3745 if (!entry) {
Steven Rostedtb6887d72009-02-17 12:32:04 -05003746 /* If we did not process any, then return error */
Steven Rostedt59df055f2009-02-14 15:29:06 -05003747 if (!count)
3748 count = -ENOMEM;
3749 goto out_unlock;
3750 }
3751
3752 count++;
3753
3754 entry->data = data;
3755
3756 /*
3757 * The caller might want to do something special
3758 * for each function we find. We call the callback
3759 * to give the caller an opportunity to do so.
3760 */
Steven Rostedt (Red Hat)e67efb92013-03-12 15:07:59 -04003761 if (ops->init) {
3762 if (ops->init(ops, rec->ip, &entry->data) < 0) {
Steven Rostedt59df055f2009-02-14 15:29:06 -05003763 /* caller does not like this func */
3764 kfree(entry);
3765 continue;
3766 }
3767 }
3768
Steven Rostedt (Red Hat)e1df4cb2013-03-12 10:09:42 -04003769 ret = enter_record(hash, rec, 0);
3770 if (ret < 0) {
3771 kfree(entry);
3772 count = ret;
3773 goto out_unlock;
3774 }
3775
Steven Rostedt59df055f2009-02-14 15:29:06 -05003776 entry->ops = ops;
3777 entry->ip = rec->ip;
3778
3779 key = hash_long(entry->ip, FTRACE_HASH_BITS);
3780 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
3781
3782 } while_for_each_ftrace_rec();
Steven Rostedt (Red Hat)e1df4cb2013-03-12 10:09:42 -04003783
3784 ret = ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);
Steven Rostedt (Red Hat)8252ecf2014-10-24 14:56:01 -04003785
Steven Rostedt (Red Hat)7485058e2015-01-13 14:03:38 -05003786 __enable_ftrace_function_probe(&old_hash_ops);
Steven Rostedt (Red Hat)8252ecf2014-10-24 14:56:01 -04003787
Steven Rostedt (Red Hat)3296fc42014-07-24 15:33:41 -04003788 if (!ret)
3789 free_ftrace_hash_rcu(old_hash);
3790 else
Steven Rostedt (Red Hat)e1df4cb2013-03-12 10:09:42 -04003791 count = ret;
3792
Steven Rostedt59df055f2009-02-14 15:29:06 -05003793 out_unlock:
Steven Rostedt (Red Hat)5ae0bf52013-05-09 18:20:37 -04003794 mutex_unlock(&ftrace_lock);
3795 out:
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04003796 mutex_unlock(&trace_probe_ops.func_hash->regex_lock);
Steven Rostedt (Red Hat)e1df4cb2013-03-12 10:09:42 -04003797 free_ftrace_hash(hash);
Steven Rostedt59df055f2009-02-14 15:29:06 -05003798
3799 return count;
3800}
3801
3802enum {
Steven Rostedtb6887d72009-02-17 12:32:04 -05003803 PROBE_TEST_FUNC = 1,
3804 PROBE_TEST_DATA = 2
Steven Rostedt59df055f2009-02-14 15:29:06 -05003805};
3806
3807static void
Steven Rostedtb6887d72009-02-17 12:32:04 -05003808__unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
Steven Rostedt59df055f2009-02-14 15:29:06 -05003809 void *data, int flags)
3810{
Steven Rostedt (Red Hat)e1df4cb2013-03-12 10:09:42 -04003811 struct ftrace_func_entry *rec_entry;
Steven Rostedtb6887d72009-02-17 12:32:04 -05003812 struct ftrace_func_probe *entry;
Steven Rostedt (Red Hat)7818b382013-03-13 12:42:58 -04003813 struct ftrace_func_probe *p;
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04003814 struct ftrace_hash **orig_hash = &trace_probe_ops.func_hash->filter_hash;
Steven Rostedt (Red Hat)3296fc42014-07-24 15:33:41 -04003815 struct ftrace_hash *old_hash = *orig_hash;
Steven Rostedt (Red Hat)7818b382013-03-13 12:42:58 -04003816 struct list_head free_list;
Steven Rostedt (Red Hat)e1df4cb2013-03-12 10:09:42 -04003817 struct ftrace_hash *hash;
Sasha Levinb67bfe02013-02-27 17:06:00 -08003818 struct hlist_node *tmp;
Steven Rostedt59df055f2009-02-14 15:29:06 -05003819 char str[KSYM_SYMBOL_LEN];
3820 int type = MATCH_FULL;
3821 int i, len = 0;
3822 char *search;
Steven Rostedt (Red Hat)3296fc42014-07-24 15:33:41 -04003823 int ret;
Steven Rostedt59df055f2009-02-14 15:29:06 -05003824
Atsushi Tsujib36461d2009-09-15 19:06:30 +09003825 if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
Steven Rostedt59df055f2009-02-14 15:29:06 -05003826 glob = NULL;
Atsushi Tsujib36461d2009-09-15 19:06:30 +09003827 else if (glob) {
Steven Rostedt59df055f2009-02-14 15:29:06 -05003828 int not;
3829
Frederic Weisbecker3f6fe062009-09-24 21:31:51 +02003830 type = filter_parse_regex(glob, strlen(glob), &search, &not);
Steven Rostedt59df055f2009-02-14 15:29:06 -05003831 len = strlen(search);
3832
Steven Rostedtb6887d72009-02-17 12:32:04 -05003833 /* we do not support '!' for function probes */
Steven Rostedt59df055f2009-02-14 15:29:06 -05003834 if (WARN_ON(not))
3835 return;
3836 }
3837
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04003838 mutex_lock(&trace_probe_ops.func_hash->regex_lock);
Steven Rostedt (Red Hat)e1df4cb2013-03-12 10:09:42 -04003839
3840 hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
3841 if (!hash)
3842 /* Hmm, should report this somehow */
3843 goto out_unlock;
3844
Steven Rostedt (Red Hat)7818b382013-03-13 12:42:58 -04003845 INIT_LIST_HEAD(&free_list);
3846
Steven Rostedt59df055f2009-02-14 15:29:06 -05003847 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
3848 struct hlist_head *hhd = &ftrace_func_hash[i];
3849
Sasha Levinb67bfe02013-02-27 17:06:00 -08003850 hlist_for_each_entry_safe(entry, tmp, hhd, node) {
Steven Rostedt59df055f2009-02-14 15:29:06 -05003851
3852 /* break up if statements for readability */
Steven Rostedtb6887d72009-02-17 12:32:04 -05003853 if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
Steven Rostedt59df055f2009-02-14 15:29:06 -05003854 continue;
3855
Steven Rostedtb6887d72009-02-17 12:32:04 -05003856 if ((flags & PROBE_TEST_DATA) && entry->data != data)
Steven Rostedt59df055f2009-02-14 15:29:06 -05003857 continue;
3858
3859 /* do this last, since it is the most expensive */
3860 if (glob) {
3861 kallsyms_lookup(entry->ip, NULL, NULL,
3862 NULL, str);
3863 if (!ftrace_match(str, glob, len, type))
3864 continue;
3865 }
3866
Steven Rostedt (Red Hat)e1df4cb2013-03-12 10:09:42 -04003867 rec_entry = ftrace_lookup_ip(hash, entry->ip);
3868 /* It is possible more than one entry had this ip */
3869 if (rec_entry)
3870 free_hash_entry(hash, rec_entry);
3871
Steven Rostedt (Red Hat)740466b2013-03-13 11:15:19 -04003872 hlist_del_rcu(&entry->node);
Steven Rostedt (Red Hat)7818b382013-03-13 12:42:58 -04003873 list_add(&entry->free_list, &free_list);
Steven Rostedt59df055f2009-02-14 15:29:06 -05003874 }
3875 }
Masami Hiramatsu3f2367b2013-05-09 14:44:21 +09003876 mutex_lock(&ftrace_lock);
Steven Rostedtb6887d72009-02-17 12:32:04 -05003877 __disable_ftrace_function_probe();
Steven Rostedt (Red Hat)e1df4cb2013-03-12 10:09:42 -04003878 /*
3879 * Remove after the disable is called. Otherwise, if the last
3880 * probe is removed, a null hash means *all enabled*.
3881 */
Steven Rostedt (Red Hat)3296fc42014-07-24 15:33:41 -04003882 ret = ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);
Steven Rostedt (Red Hat)7818b382013-03-13 12:42:58 -04003883 synchronize_sched();
Steven Rostedt (Red Hat)3296fc42014-07-24 15:33:41 -04003884 if (!ret)
3885 free_ftrace_hash_rcu(old_hash);
3886
Steven Rostedt (Red Hat)7818b382013-03-13 12:42:58 -04003887 list_for_each_entry_safe(entry, p, &free_list, free_list) {
3888 list_del(&entry->free_list);
3889 ftrace_free_entry(entry);
3890 }
Masami Hiramatsu3f2367b2013-05-09 14:44:21 +09003891 mutex_unlock(&ftrace_lock);
Steven Rostedt (Red Hat)7818b382013-03-13 12:42:58 -04003892
Steven Rostedt (Red Hat)e1df4cb2013-03-12 10:09:42 -04003893 out_unlock:
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04003894 mutex_unlock(&trace_probe_ops.func_hash->regex_lock);
Steven Rostedt (Red Hat)e1df4cb2013-03-12 10:09:42 -04003895 free_ftrace_hash(hash);
Steven Rostedt59df055f2009-02-14 15:29:06 -05003896}
3897
3898void
Steven Rostedtb6887d72009-02-17 12:32:04 -05003899unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
Steven Rostedt59df055f2009-02-14 15:29:06 -05003900 void *data)
3901{
Steven Rostedtb6887d72009-02-17 12:32:04 -05003902 __unregister_ftrace_function_probe(glob, ops, data,
3903 PROBE_TEST_FUNC | PROBE_TEST_DATA);
Steven Rostedt59df055f2009-02-14 15:29:06 -05003904}
3905
3906void
Steven Rostedtb6887d72009-02-17 12:32:04 -05003907unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
Steven Rostedt59df055f2009-02-14 15:29:06 -05003908{
Steven Rostedtb6887d72009-02-17 12:32:04 -05003909 __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
Steven Rostedt59df055f2009-02-14 15:29:06 -05003910}
3911
Steven Rostedtb6887d72009-02-17 12:32:04 -05003912void unregister_ftrace_function_probe_all(char *glob)
Steven Rostedt59df055f2009-02-14 15:29:06 -05003913{
Steven Rostedtb6887d72009-02-17 12:32:04 -05003914 __unregister_ftrace_function_probe(glob, NULL, NULL, 0);
Steven Rostedt59df055f2009-02-14 15:29:06 -05003915}
3916
Steven Rostedtf6180772009-02-14 00:40:25 -05003917static LIST_HEAD(ftrace_commands);
3918static DEFINE_MUTEX(ftrace_cmd_mutex);
3919
Tom Zanussi38de93a2013-10-24 08:34:18 -05003920/*
3921 * Currently we only register ftrace commands from __init, so mark this
3922 * __init too.
3923 */
3924__init int register_ftrace_command(struct ftrace_func_command *cmd)
Steven Rostedtf6180772009-02-14 00:40:25 -05003925{
3926 struct ftrace_func_command *p;
3927 int ret = 0;
3928
3929 mutex_lock(&ftrace_cmd_mutex);
3930 list_for_each_entry(p, &ftrace_commands, list) {
3931 if (strcmp(cmd->name, p->name) == 0) {
3932 ret = -EBUSY;
3933 goto out_unlock;
3934 }
3935 }
3936 list_add(&cmd->list, &ftrace_commands);
3937 out_unlock:
3938 mutex_unlock(&ftrace_cmd_mutex);
3939
3940 return ret;
3941}
3942
Tom Zanussi38de93a2013-10-24 08:34:18 -05003943/*
3944 * Currently we only unregister ftrace commands from __init, so mark
3945 * this __init too.
3946 */
3947__init int unregister_ftrace_command(struct ftrace_func_command *cmd)
Steven Rostedtf6180772009-02-14 00:40:25 -05003948{
3949 struct ftrace_func_command *p, *n;
3950 int ret = -ENODEV;
3951
3952 mutex_lock(&ftrace_cmd_mutex);
3953 list_for_each_entry_safe(p, n, &ftrace_commands, list) {
3954 if (strcmp(cmd->name, p->name) == 0) {
3955 ret = 0;
3956 list_del_init(&p->list);
3957 goto out_unlock;
3958 }
3959 }
3960 out_unlock:
3961 mutex_unlock(&ftrace_cmd_mutex);
3962
3963 return ret;
3964}
3965
Steven Rostedt33dc9b12011-05-02 17:34:47 -04003966static int ftrace_process_regex(struct ftrace_hash *hash,
3967 char *buff, int len, int enable)
Steven Rostedt64e7c442009-02-13 17:08:48 -05003968{
Steven Rostedtf6180772009-02-14 00:40:25 -05003969 char *func, *command, *next = buff;
Steven Rostedt6a24a242009-02-17 11:20:26 -05003970 struct ftrace_func_command *p;
GuoWen Li0aff1c02011-06-01 19:18:47 +08003971 int ret = -EINVAL;
Steven Rostedt64e7c442009-02-13 17:08:48 -05003972
3973 func = strsep(&next, ":");
3974
3975 if (!next) {
Steven Rostedt1cf41dd72011-04-29 20:59:51 -04003976 ret = ftrace_match_records(hash, func, len);
Steven Rostedtb448c4e2011-04-29 15:12:32 -04003977 if (!ret)
3978 ret = -EINVAL;
3979 if (ret < 0)
3980 return ret;
3981 return 0;
Steven Rostedt64e7c442009-02-13 17:08:48 -05003982 }
3983
Steven Rostedtf6180772009-02-14 00:40:25 -05003984 /* command found */
Steven Rostedt64e7c442009-02-13 17:08:48 -05003985
3986 command = strsep(&next, ":");
3987
Steven Rostedtf6180772009-02-14 00:40:25 -05003988 mutex_lock(&ftrace_cmd_mutex);
3989 list_for_each_entry(p, &ftrace_commands, list) {
3990 if (strcmp(p->name, command) == 0) {
Steven Rostedt43dd61c2011-07-07 11:09:22 -04003991 ret = p->func(hash, func, command, next, enable);
Steven Rostedtf6180772009-02-14 00:40:25 -05003992 goto out_unlock;
3993 }
Steven Rostedt64e7c442009-02-13 17:08:48 -05003994 }
Steven Rostedtf6180772009-02-14 00:40:25 -05003995 out_unlock:
3996 mutex_unlock(&ftrace_cmd_mutex);
Steven Rostedt64e7c442009-02-13 17:08:48 -05003997
Steven Rostedtf6180772009-02-14 00:40:25 -05003998 return ret;
Steven Rostedt64e7c442009-02-13 17:08:48 -05003999}
4000
Ingo Molnare309b412008-05-12 21:20:51 +02004001static ssize_t
Steven Rostedt41c52c02008-05-22 11:46:33 -04004002ftrace_regex_write(struct file *file, const char __user *ubuf,
4003 size_t cnt, loff_t *ppos, int enable)
Steven Rostedt5072c592008-05-12 21:20:43 +02004004{
4005 struct ftrace_iterator *iter;
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02004006 struct trace_parser *parser;
4007 ssize_t ret, read;
Steven Rostedt5072c592008-05-12 21:20:43 +02004008
Li Zefan4ba79782009-09-22 13:52:20 +08004009 if (!cnt)
Steven Rostedt5072c592008-05-12 21:20:43 +02004010 return 0;
4011
Steven Rostedt5072c592008-05-12 21:20:43 +02004012 if (file->f_mode & FMODE_READ) {
4013 struct seq_file *m = file->private_data;
4014 iter = m->private;
4015 } else
4016 iter = file->private_data;
4017
Masami Hiramatsuf04f24fb2013-05-09 14:44:17 +09004018 if (unlikely(ftrace_disabled))
Masami Hiramatsu3f2367b2013-05-09 14:44:21 +09004019 return -ENODEV;
4020
4021 /* iter->hash is a local copy, so we don't need regex_lock */
Masami Hiramatsuf04f24fb2013-05-09 14:44:17 +09004022
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02004023 parser = &iter->parser;
4024 read = trace_get_user(parser, ubuf, cnt, ppos);
Steven Rostedt5072c592008-05-12 21:20:43 +02004025
Li Zefan4ba79782009-09-22 13:52:20 +08004026 if (read >= 0 && trace_parser_loaded(parser) &&
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02004027 !trace_parser_cont(parser)) {
Steven Rostedt33dc9b12011-05-02 17:34:47 -04004028 ret = ftrace_process_regex(iter->hash, parser->buffer,
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02004029 parser->idx, enable);
Li Zefan313254a2009-12-08 11:15:30 +08004030 trace_parser_clear(parser);
Steven Rostedt (Red Hat)7c088b52013-05-09 11:35:12 -04004031 if (ret < 0)
Masami Hiramatsu3f2367b2013-05-09 14:44:21 +09004032 goto out;
Steven Rostedt5072c592008-05-12 21:20:43 +02004033 }
4034
Steven Rostedt5072c592008-05-12 21:20:43 +02004035 ret = read;
Masami Hiramatsu3f2367b2013-05-09 14:44:21 +09004036 out:
Steven Rostedt5072c592008-05-12 21:20:43 +02004037 return ret;
4038}
4039
Steven Rostedtfc13cb02011-12-19 14:41:25 -05004040ssize_t
Steven Rostedt41c52c02008-05-22 11:46:33 -04004041ftrace_filter_write(struct file *file, const char __user *ubuf,
4042 size_t cnt, loff_t *ppos)
4043{
4044 return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
4045}
4046
Steven Rostedtfc13cb02011-12-19 14:41:25 -05004047ssize_t
Steven Rostedt41c52c02008-05-22 11:46:33 -04004048ftrace_notrace_write(struct file *file, const char __user *ubuf,
4049 size_t cnt, loff_t *ppos)
4050{
4051 return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
4052}
4053
Steven Rostedt33dc9b12011-05-02 17:34:47 -04004054static int
Masami Hiramatsu647664e2012-06-05 19:28:08 +09004055ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)
4056{
4057 struct ftrace_func_entry *entry;
4058
4059 if (!ftrace_location(ip))
4060 return -EINVAL;
4061
4062 if (remove) {
4063 entry = ftrace_lookup_ip(hash, ip);
4064 if (!entry)
4065 return -ENOENT;
4066 free_hash_entry(hash, entry);
4067 return 0;
4068 }
4069
4070 return add_hash_entry(hash, ip);
4071}
4072
Steven Rostedt (Red Hat)8252ecf2014-10-24 14:56:01 -04004073static void ftrace_ops_update_code(struct ftrace_ops *ops,
Steven Rostedt (Red Hat)7485058e2015-01-13 14:03:38 -05004074 struct ftrace_ops_hash *old_hash)
Steven Rostedt (Red Hat)1c80c432013-07-25 20:22:00 -04004075{
Steven Rostedt (Red Hat)8f86f832015-01-13 11:20:43 -05004076 struct ftrace_ops *op;
4077
4078 if (!ftrace_enabled)
4079 return;
4080
4081 if (ops->flags & FTRACE_OPS_FL_ENABLED) {
Steven Rostedt (Red Hat)8252ecf2014-10-24 14:56:01 -04004082 ftrace_run_modify_code(ops, FTRACE_UPDATE_CALLS, old_hash);
Steven Rostedt (Red Hat)8f86f832015-01-13 11:20:43 -05004083 return;
4084 }
4085
4086 /*
4087 * If this is the shared global_ops filter, then we need to
4088 * check if there is another ops that shares it, is enabled.
4089 * If so, we still need to run the modify code.
4090 */
4091 if (ops->func_hash != &global_ops.local_hash)
4092 return;
4093
4094 do_for_each_ftrace_op(op, ftrace_ops_list) {
4095 if (op->func_hash == &global_ops.local_hash &&
4096 op->flags & FTRACE_OPS_FL_ENABLED) {
4097 ftrace_run_modify_code(op, FTRACE_UPDATE_CALLS, old_hash);
4098 /* Only need to do this once */
4099 return;
4100 }
4101 } while_for_each_ftrace_op(op);
Steven Rostedt (Red Hat)1c80c432013-07-25 20:22:00 -04004102}
4103
Masami Hiramatsu647664e2012-06-05 19:28:08 +09004104static int
4105ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len,
4106 unsigned long ip, int remove, int reset, int enable)
Steven Rostedt41c52c02008-05-22 11:46:33 -04004107{
Steven Rostedt33dc9b12011-05-02 17:34:47 -04004108 struct ftrace_hash **orig_hash;
Steven Rostedt (Red Hat)7485058e2015-01-13 14:03:38 -05004109 struct ftrace_ops_hash old_hash_ops;
Steven Rostedt (Red Hat)3296fc42014-07-24 15:33:41 -04004110 struct ftrace_hash *old_hash;
Steven Rostedtf45948e2011-05-02 12:29:25 -04004111 struct ftrace_hash *hash;
Steven Rostedt33dc9b12011-05-02 17:34:47 -04004112 int ret;
Steven Rostedtf45948e2011-05-02 12:29:25 -04004113
Steven Rostedt41c52c02008-05-22 11:46:33 -04004114 if (unlikely(ftrace_disabled))
Steven Rostedt33dc9b12011-05-02 17:34:47 -04004115 return -ENODEV;
Steven Rostedt41c52c02008-05-22 11:46:33 -04004116
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04004117 mutex_lock(&ops->func_hash->regex_lock);
Masami Hiramatsu3f2367b2013-05-09 14:44:21 +09004118
Steven Rostedtf45948e2011-05-02 12:29:25 -04004119 if (enable)
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04004120 orig_hash = &ops->func_hash->filter_hash;
Steven Rostedtf45948e2011-05-02 12:29:25 -04004121 else
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04004122 orig_hash = &ops->func_hash->notrace_hash;
Steven Rostedt33dc9b12011-05-02 17:34:47 -04004123
Wang Nanb972cc52014-07-15 08:40:20 +08004124 if (reset)
4125 hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
4126 else
4127 hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
4128
Masami Hiramatsu3f2367b2013-05-09 14:44:21 +09004129 if (!hash) {
4130 ret = -ENOMEM;
4131 goto out_regex_unlock;
4132 }
Steven Rostedtf45948e2011-05-02 12:29:25 -04004133
Jiri Olsaac483c42012-01-02 10:04:14 +01004134 if (buf && !ftrace_match_records(hash, buf, len)) {
4135 ret = -EINVAL;
4136 goto out_regex_unlock;
4137 }
Masami Hiramatsu647664e2012-06-05 19:28:08 +09004138 if (ip) {
4139 ret = ftrace_match_addr(hash, ip, remove);
4140 if (ret < 0)
4141 goto out_regex_unlock;
4142 }
Steven Rostedt33dc9b12011-05-02 17:34:47 -04004143
4144 mutex_lock(&ftrace_lock);
Steven Rostedt (Red Hat)3296fc42014-07-24 15:33:41 -04004145 old_hash = *orig_hash;
Steven Rostedt (Red Hat)7485058e2015-01-13 14:03:38 -05004146 old_hash_ops.filter_hash = ops->func_hash->filter_hash;
4147 old_hash_ops.notrace_hash = ops->func_hash->notrace_hash;
Steven Rostedt41fb61c2011-07-13 15:03:44 -04004148 ret = ftrace_hash_move(ops, enable, orig_hash, hash);
Steven Rostedt (Red Hat)3296fc42014-07-24 15:33:41 -04004149 if (!ret) {
Steven Rostedt (Red Hat)7485058e2015-01-13 14:03:38 -05004150 ftrace_ops_update_code(ops, &old_hash_ops);
Steven Rostedt (Red Hat)3296fc42014-07-24 15:33:41 -04004151 free_ftrace_hash_rcu(old_hash);
4152 }
Steven Rostedt33dc9b12011-05-02 17:34:47 -04004153 mutex_unlock(&ftrace_lock);
4154
Jiri Olsaac483c42012-01-02 10:04:14 +01004155 out_regex_unlock:
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04004156 mutex_unlock(&ops->func_hash->regex_lock);
Steven Rostedt33dc9b12011-05-02 17:34:47 -04004157
4158 free_ftrace_hash(hash);
4159 return ret;
Steven Rostedt41c52c02008-05-22 11:46:33 -04004160}
4161
Masami Hiramatsu647664e2012-06-05 19:28:08 +09004162static int
4163ftrace_set_addr(struct ftrace_ops *ops, unsigned long ip, int remove,
4164 int reset, int enable)
4165{
4166 return ftrace_set_hash(ops, 0, 0, ip, remove, reset, enable);
4167}
4168
4169/**
4170 * ftrace_set_filter_ip - set a function to filter on in ftrace by address
4171 * @ops - the ops to set the filter with
4172 * @ip - the address to add to or remove from the filter.
4173 * @remove - non zero to remove the ip from the filter
4174 * @reset - non zero to reset all filters before applying this filter.
4175 *
4176 * Filters denote which functions should be enabled when tracing is enabled
4177 * If @ip is NULL, it failes to update filter.
4178 */
4179int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
4180 int remove, int reset)
4181{
Masami Hiramatsuf04f24fb2013-05-09 14:44:17 +09004182 ftrace_ops_init(ops);
Masami Hiramatsu647664e2012-06-05 19:28:08 +09004183 return ftrace_set_addr(ops, ip, remove, reset, 1);
4184}
4185EXPORT_SYMBOL_GPL(ftrace_set_filter_ip);
4186
4187static int
4188ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
4189 int reset, int enable)
4190{
4191 return ftrace_set_hash(ops, buf, len, 0, 0, reset, enable);
4192}
4193
Steven Rostedt77a2b372008-05-12 21:20:45 +02004194/**
4195 * ftrace_set_filter - set a function to filter on in ftrace
Steven Rostedt936e0742011-05-05 22:54:01 -04004196 * @ops - the ops to set the filter with
Steven Rostedt77a2b372008-05-12 21:20:45 +02004197 * @buf - the string that holds the function filter text.
4198 * @len - the length of the string.
4199 * @reset - non zero to reset all filters before applying this filter.
4200 *
4201 * Filters denote which functions should be enabled when tracing is enabled.
4202 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
4203 */
Jiri Olsaac483c42012-01-02 10:04:14 +01004204int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
Steven Rostedt936e0742011-05-05 22:54:01 -04004205 int len, int reset)
Steven Rostedt77a2b372008-05-12 21:20:45 +02004206{
Masami Hiramatsuf04f24fb2013-05-09 14:44:17 +09004207 ftrace_ops_init(ops);
Jiri Olsaac483c42012-01-02 10:04:14 +01004208 return ftrace_set_regex(ops, buf, len, reset, 1);
Steven Rostedt41c52c02008-05-22 11:46:33 -04004209}
Steven Rostedt936e0742011-05-05 22:54:01 -04004210EXPORT_SYMBOL_GPL(ftrace_set_filter);
Steven Rostedt4eebcc82008-05-12 21:20:48 +02004211
Steven Rostedt41c52c02008-05-22 11:46:33 -04004212/**
4213 * ftrace_set_notrace - set a function to not trace in ftrace
Steven Rostedt936e0742011-05-05 22:54:01 -04004214 * @ops - the ops to set the notrace filter with
Steven Rostedt41c52c02008-05-22 11:46:33 -04004215 * @buf - the string that holds the function notrace text.
4216 * @len - the length of the string.
4217 * @reset - non zero to reset all filters before applying this filter.
4218 *
4219 * Notrace Filters denote which functions should not be enabled when tracing
4220 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
4221 * for tracing.
4222 */
Jiri Olsaac483c42012-01-02 10:04:14 +01004223int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
Steven Rostedt936e0742011-05-05 22:54:01 -04004224 int len, int reset)
4225{
Masami Hiramatsuf04f24fb2013-05-09 14:44:17 +09004226 ftrace_ops_init(ops);
Jiri Olsaac483c42012-01-02 10:04:14 +01004227 return ftrace_set_regex(ops, buf, len, reset, 0);
Steven Rostedt936e0742011-05-05 22:54:01 -04004228}
4229EXPORT_SYMBOL_GPL(ftrace_set_notrace);
4230/**
Jiaxing Wang8d1b0652014-04-20 23:10:44 +08004231 * ftrace_set_global_filter - set a function to filter on with global tracers
Steven Rostedt936e0742011-05-05 22:54:01 -04004232 * @buf - the string that holds the function filter text.
4233 * @len - the length of the string.
4234 * @reset - non zero to reset all filters before applying this filter.
4235 *
4236 * Filters denote which functions should be enabled when tracing is enabled.
4237 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
4238 */
4239void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
4240{
4241 ftrace_set_regex(&global_ops, buf, len, reset, 1);
4242}
4243EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
4244
4245/**
Jiaxing Wang8d1b0652014-04-20 23:10:44 +08004246 * ftrace_set_global_notrace - set a function to not trace with global tracers
Steven Rostedt936e0742011-05-05 22:54:01 -04004247 * @buf - the string that holds the function notrace text.
4248 * @len - the length of the string.
4249 * @reset - non zero to reset all filters before applying this filter.
4250 *
4251 * Notrace Filters denote which functions should not be enabled when tracing
4252 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
4253 * for tracing.
4254 */
4255void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
Steven Rostedt41c52c02008-05-22 11:46:33 -04004256{
Steven Rostedtf45948e2011-05-02 12:29:25 -04004257 ftrace_set_regex(&global_ops, buf, len, reset, 0);
Steven Rostedt77a2b372008-05-12 21:20:45 +02004258}
Steven Rostedt936e0742011-05-05 22:54:01 -04004259EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
Steven Rostedt77a2b372008-05-12 21:20:45 +02004260
Steven Rostedt2af15d62009-05-28 13:37:24 -04004261/*
4262 * command line interface to allow users to set filters on boot up.
4263 */
4264#define FTRACE_FILTER_SIZE COMMAND_LINE_SIZE
4265static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
4266static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
4267
Steven Rostedt (Red Hat)f1ed7c72013-06-27 22:18:06 -04004268/* Used by function selftest to not test if filter is set */
4269bool ftrace_filter_param __initdata;
4270
Steven Rostedt2af15d62009-05-28 13:37:24 -04004271static int __init set_ftrace_notrace(char *str)
4272{
Steven Rostedt (Red Hat)f1ed7c72013-06-27 22:18:06 -04004273 ftrace_filter_param = true;
Chen Gang75761cc2013-04-08 12:12:39 +08004274 strlcpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
Steven Rostedt2af15d62009-05-28 13:37:24 -04004275 return 1;
4276}
4277__setup("ftrace_notrace=", set_ftrace_notrace);
4278
4279static int __init set_ftrace_filter(char *str)
4280{
Steven Rostedt (Red Hat)f1ed7c72013-06-27 22:18:06 -04004281 ftrace_filter_param = true;
Chen Gang75761cc2013-04-08 12:12:39 +08004282 strlcpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
Steven Rostedt2af15d62009-05-28 13:37:24 -04004283 return 1;
4284}
4285__setup("ftrace_filter=", set_ftrace_filter);
4286
Stefan Assmann369bc182009-10-12 22:17:21 +02004287#ifdef CONFIG_FUNCTION_GRAPH_TRACER
Lai Jiangshanf6060f42009-11-05 11:16:17 +08004288static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
Namhyung Kim0d7d9a12014-06-13 01:23:50 +09004289static char ftrace_graph_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
Namhyung Kimfaf982a2013-10-14 17:24:24 +09004290static int ftrace_set_func(unsigned long *array, int *idx, int size, char *buffer);
Steven Rostedt801c29f2010-03-05 20:02:19 -05004291
Steven Rostedt (Red Hat)f3bea492014-07-02 23:23:31 -04004292static unsigned long save_global_trampoline;
4293static unsigned long save_global_flags;
4294
Stefan Assmann369bc182009-10-12 22:17:21 +02004295static int __init set_graph_function(char *str)
4296{
Frederic Weisbecker06f43d62009-10-14 20:43:39 +02004297 strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
Stefan Assmann369bc182009-10-12 22:17:21 +02004298 return 1;
4299}
4300__setup("ftrace_graph_filter=", set_graph_function);
4301
Namhyung Kim0d7d9a12014-06-13 01:23:50 +09004302static int __init set_graph_notrace_function(char *str)
4303{
4304 strlcpy(ftrace_graph_notrace_buf, str, FTRACE_FILTER_SIZE);
4305 return 1;
4306}
4307__setup("ftrace_graph_notrace=", set_graph_notrace_function);
4308
4309static void __init set_ftrace_early_graph(char *buf, int enable)
Stefan Assmann369bc182009-10-12 22:17:21 +02004310{
4311 int ret;
4312 char *func;
Namhyung Kim0d7d9a12014-06-13 01:23:50 +09004313 unsigned long *table = ftrace_graph_funcs;
4314 int *count = &ftrace_graph_count;
4315
4316 if (!enable) {
4317 table = ftrace_graph_notrace_funcs;
4318 count = &ftrace_graph_notrace_count;
4319 }
Stefan Assmann369bc182009-10-12 22:17:21 +02004320
4321 while (buf) {
4322 func = strsep(&buf, ",");
4323 /* we allow only one expression at a time */
Namhyung Kim0d7d9a12014-06-13 01:23:50 +09004324 ret = ftrace_set_func(table, count, FTRACE_GRAPH_MAX_FUNCS, func);
Stefan Assmann369bc182009-10-12 22:17:21 +02004325 if (ret)
4326 printk(KERN_DEBUG "ftrace: function %s not "
4327 "traceable\n", func);
4328 }
4329}
4330#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
4331
Steven Rostedt2a85a372011-12-19 21:57:44 -05004332void __init
4333ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable)
Steven Rostedt2af15d62009-05-28 13:37:24 -04004334{
4335 char *func;
4336
Masami Hiramatsuf04f24fb2013-05-09 14:44:17 +09004337 ftrace_ops_init(ops);
4338
Steven Rostedt2af15d62009-05-28 13:37:24 -04004339 while (buf) {
4340 func = strsep(&buf, ",");
Steven Rostedtf45948e2011-05-02 12:29:25 -04004341 ftrace_set_regex(ops, func, strlen(func), 0, enable);
Steven Rostedt2af15d62009-05-28 13:37:24 -04004342 }
4343}
4344
4345static void __init set_ftrace_early_filters(void)
4346{
4347 if (ftrace_filter_buf[0])
Steven Rostedt2a85a372011-12-19 21:57:44 -05004348 ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1);
Steven Rostedt2af15d62009-05-28 13:37:24 -04004349 if (ftrace_notrace_buf[0])
Steven Rostedt2a85a372011-12-19 21:57:44 -05004350 ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0);
Stefan Assmann369bc182009-10-12 22:17:21 +02004351#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4352 if (ftrace_graph_buf[0])
Namhyung Kim0d7d9a12014-06-13 01:23:50 +09004353 set_ftrace_early_graph(ftrace_graph_buf, 1);
4354 if (ftrace_graph_notrace_buf[0])
4355 set_ftrace_early_graph(ftrace_graph_notrace_buf, 0);
Stefan Assmann369bc182009-10-12 22:17:21 +02004356#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
Steven Rostedt2af15d62009-05-28 13:37:24 -04004357}
4358
Steven Rostedtfc13cb02011-12-19 14:41:25 -05004359int ftrace_regex_release(struct inode *inode, struct file *file)
Steven Rostedt5072c592008-05-12 21:20:43 +02004360{
4361 struct seq_file *m = (struct seq_file *)file->private_data;
Steven Rostedt (Red Hat)7485058e2015-01-13 14:03:38 -05004362 struct ftrace_ops_hash old_hash_ops;
Steven Rostedt5072c592008-05-12 21:20:43 +02004363 struct ftrace_iterator *iter;
Steven Rostedt33dc9b12011-05-02 17:34:47 -04004364 struct ftrace_hash **orig_hash;
Steven Rostedt (Red Hat)3296fc42014-07-24 15:33:41 -04004365 struct ftrace_hash *old_hash;
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02004366 struct trace_parser *parser;
Steven Rostedted926f92011-05-03 13:25:24 -04004367 int filter_hash;
Steven Rostedt33dc9b12011-05-02 17:34:47 -04004368 int ret;
Steven Rostedt5072c592008-05-12 21:20:43 +02004369
Steven Rostedt5072c592008-05-12 21:20:43 +02004370 if (file->f_mode & FMODE_READ) {
4371 iter = m->private;
Steven Rostedt5072c592008-05-12 21:20:43 +02004372 seq_release(inode, file);
4373 } else
4374 iter = file->private_data;
4375
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02004376 parser = &iter->parser;
4377 if (trace_parser_loaded(parser)) {
4378 parser->buffer[parser->idx] = 0;
Steven Rostedt1cf41dd72011-04-29 20:59:51 -04004379 ftrace_match_records(iter->hash, parser->buffer, parser->idx);
Steven Rostedt5072c592008-05-12 21:20:43 +02004380 }
4381
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02004382 trace_parser_put(parser);
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02004383
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04004384 mutex_lock(&iter->ops->func_hash->regex_lock);
Masami Hiramatsu3f2367b2013-05-09 14:44:21 +09004385
Steven Rostedt058e2972011-04-29 22:35:33 -04004386 if (file->f_mode & FMODE_WRITE) {
Steven Rostedted926f92011-05-03 13:25:24 -04004387 filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);
4388
4389 if (filter_hash)
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04004390 orig_hash = &iter->ops->func_hash->filter_hash;
Steven Rostedted926f92011-05-03 13:25:24 -04004391 else
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04004392 orig_hash = &iter->ops->func_hash->notrace_hash;
Steven Rostedt33dc9b12011-05-02 17:34:47 -04004393
Steven Rostedt058e2972011-04-29 22:35:33 -04004394 mutex_lock(&ftrace_lock);
Steven Rostedt (Red Hat)3296fc42014-07-24 15:33:41 -04004395 old_hash = *orig_hash;
Steven Rostedt (Red Hat)7485058e2015-01-13 14:03:38 -05004396 old_hash_ops.filter_hash = iter->ops->func_hash->filter_hash;
4397 old_hash_ops.notrace_hash = iter->ops->func_hash->notrace_hash;
Steven Rostedt41fb61c2011-07-13 15:03:44 -04004398 ret = ftrace_hash_move(iter->ops, filter_hash,
4399 orig_hash, iter->hash);
Steven Rostedt (Red Hat)3296fc42014-07-24 15:33:41 -04004400 if (!ret) {
Steven Rostedt (Red Hat)7485058e2015-01-13 14:03:38 -05004401 ftrace_ops_update_code(iter->ops, &old_hash_ops);
Steven Rostedt (Red Hat)3296fc42014-07-24 15:33:41 -04004402 free_ftrace_hash_rcu(old_hash);
4403 }
Steven Rostedt058e2972011-04-29 22:35:33 -04004404 mutex_unlock(&ftrace_lock);
4405 }
Masami Hiramatsu3f2367b2013-05-09 14:44:21 +09004406
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04004407 mutex_unlock(&iter->ops->func_hash->regex_lock);
Steven Rostedt33dc9b12011-05-02 17:34:47 -04004408 free_ftrace_hash(iter->hash);
4409 kfree(iter);
Steven Rostedt058e2972011-04-29 22:35:33 -04004410
Steven Rostedt5072c592008-05-12 21:20:43 +02004411 return 0;
4412}
4413
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004414static const struct file_operations ftrace_avail_fops = {
Steven Rostedt5072c592008-05-12 21:20:43 +02004415 .open = ftrace_avail_open,
4416 .read = seq_read,
4417 .llseek = seq_lseek,
Li Zefan3be04b42009-08-17 16:54:03 +08004418 .release = seq_release_private,
Steven Rostedt5072c592008-05-12 21:20:43 +02004419};
4420
Steven Rostedt647bcd02011-05-03 14:39:21 -04004421static const struct file_operations ftrace_enabled_fops = {
4422 .open = ftrace_enabled_open,
4423 .read = seq_read,
4424 .llseek = seq_lseek,
4425 .release = seq_release_private,
4426};
4427
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004428static const struct file_operations ftrace_filter_fops = {
Steven Rostedt5072c592008-05-12 21:20:43 +02004429 .open = ftrace_filter_open,
Lai Jiangshan850a80c2009-03-13 17:47:23 +08004430 .read = seq_read,
Steven Rostedt5072c592008-05-12 21:20:43 +02004431 .write = ftrace_filter_write,
Steven Rostedt (Red Hat)098c8792013-12-21 17:39:40 -05004432 .llseek = tracing_lseek,
Steven Rostedt1cf41dd72011-04-29 20:59:51 -04004433 .release = ftrace_regex_release,
Steven Rostedt5072c592008-05-12 21:20:43 +02004434};
4435
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004436static const struct file_operations ftrace_notrace_fops = {
Steven Rostedt41c52c02008-05-22 11:46:33 -04004437 .open = ftrace_notrace_open,
Lai Jiangshan850a80c2009-03-13 17:47:23 +08004438 .read = seq_read,
Steven Rostedt41c52c02008-05-22 11:46:33 -04004439 .write = ftrace_notrace_write,
Steven Rostedt (Red Hat)098c8792013-12-21 17:39:40 -05004440 .llseek = tracing_lseek,
Steven Rostedt1cf41dd72011-04-29 20:59:51 -04004441 .release = ftrace_regex_release,
Steven Rostedt41c52c02008-05-22 11:46:33 -04004442};
4443
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004444#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4445
4446static DEFINE_MUTEX(graph_lock);
4447
4448int ftrace_graph_count;
Namhyung Kim29ad23b2013-10-14 17:24:26 +09004449int ftrace_graph_notrace_count;
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004450unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
Namhyung Kim29ad23b2013-10-14 17:24:26 +09004451unsigned long ftrace_graph_notrace_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004452
Namhyung Kimfaf982a2013-10-14 17:24:24 +09004453struct ftrace_graph_data {
4454 unsigned long *table;
4455 size_t size;
4456 int *count;
4457 const struct seq_operations *seq_ops;
4458};
4459
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004460static void *
Li Zefan85951842009-06-24 09:54:00 +08004461__g_next(struct seq_file *m, loff_t *pos)
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004462{
Namhyung Kimfaf982a2013-10-14 17:24:24 +09004463 struct ftrace_graph_data *fgd = m->private;
4464
4465 if (*pos >= *fgd->count)
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004466 return NULL;
Namhyung Kimfaf982a2013-10-14 17:24:24 +09004467 return &fgd->table[*pos];
Li Zefan85951842009-06-24 09:54:00 +08004468}
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004469
Li Zefan85951842009-06-24 09:54:00 +08004470static void *
4471g_next(struct seq_file *m, void *v, loff_t *pos)
4472{
4473 (*pos)++;
4474 return __g_next(m, pos);
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004475}
4476
4477static void *g_start(struct seq_file *m, loff_t *pos)
4478{
Namhyung Kimfaf982a2013-10-14 17:24:24 +09004479 struct ftrace_graph_data *fgd = m->private;
4480
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004481 mutex_lock(&graph_lock);
4482
Frederic Weisbeckerf9349a82009-02-19 21:13:12 +01004483 /* Nothing, tell g_show to print all functions are enabled */
Namhyung Kimfaf982a2013-10-14 17:24:24 +09004484 if (!*fgd->count && !*pos)
Frederic Weisbeckerf9349a82009-02-19 21:13:12 +01004485 return (void *)1;
4486
Li Zefan85951842009-06-24 09:54:00 +08004487 return __g_next(m, pos);
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004488}
4489
4490static void g_stop(struct seq_file *m, void *p)
4491{
4492 mutex_unlock(&graph_lock);
4493}
4494
4495static int g_show(struct seq_file *m, void *v)
4496{
4497 unsigned long *ptr = v;
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004498
4499 if (!ptr)
4500 return 0;
4501
Frederic Weisbeckerf9349a82009-02-19 21:13:12 +01004502 if (ptr == (unsigned long *)1) {
Namhyung Kim280d1422014-06-13 01:23:51 +09004503 struct ftrace_graph_data *fgd = m->private;
4504
4505 if (fgd->table == ftrace_graph_funcs)
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01004506 seq_puts(m, "#### all functions enabled ####\n");
Namhyung Kim280d1422014-06-13 01:23:51 +09004507 else
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01004508 seq_puts(m, "#### no functions disabled ####\n");
Frederic Weisbeckerf9349a82009-02-19 21:13:12 +01004509 return 0;
4510 }
4511
Steven Rostedtb375a112009-09-17 00:05:58 -04004512 seq_printf(m, "%ps\n", (void *)*ptr);
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004513
4514 return 0;
4515}
4516
James Morris88e9d342009-09-22 16:43:43 -07004517static const struct seq_operations ftrace_graph_seq_ops = {
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004518 .start = g_start,
4519 .next = g_next,
4520 .stop = g_stop,
4521 .show = g_show,
4522};
4523
4524static int
Namhyung Kimfaf982a2013-10-14 17:24:24 +09004525__ftrace_graph_open(struct inode *inode, struct file *file,
4526 struct ftrace_graph_data *fgd)
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004527{
4528 int ret = 0;
4529
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004530 mutex_lock(&graph_lock);
4531 if ((file->f_mode & FMODE_WRITE) &&
Steven Rostedt8650ae32009-07-22 23:29:30 -04004532 (file->f_flags & O_TRUNC)) {
Namhyung Kimfaf982a2013-10-14 17:24:24 +09004533 *fgd->count = 0;
4534 memset(fgd->table, 0, fgd->size * sizeof(*fgd->table));
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004535 }
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004536 mutex_unlock(&graph_lock);
4537
Namhyung Kimfaf982a2013-10-14 17:24:24 +09004538 if (file->f_mode & FMODE_READ) {
4539 ret = seq_open(file, fgd->seq_ops);
4540 if (!ret) {
4541 struct seq_file *m = file->private_data;
4542 m->private = fgd;
4543 }
4544 } else
4545 file->private_data = fgd;
Li Zefana4ec5e02009-09-18 14:06:28 +08004546
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004547 return ret;
4548}
4549
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004550static int
Namhyung Kimfaf982a2013-10-14 17:24:24 +09004551ftrace_graph_open(struct inode *inode, struct file *file)
4552{
4553 struct ftrace_graph_data *fgd;
4554
4555 if (unlikely(ftrace_disabled))
4556 return -ENODEV;
4557
4558 fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
4559 if (fgd == NULL)
4560 return -ENOMEM;
4561
4562 fgd->table = ftrace_graph_funcs;
4563 fgd->size = FTRACE_GRAPH_MAX_FUNCS;
4564 fgd->count = &ftrace_graph_count;
4565 fgd->seq_ops = &ftrace_graph_seq_ops;
4566
4567 return __ftrace_graph_open(inode, file, fgd);
4568}
4569
4570static int
Namhyung Kim29ad23b2013-10-14 17:24:26 +09004571ftrace_graph_notrace_open(struct inode *inode, struct file *file)
4572{
4573 struct ftrace_graph_data *fgd;
4574
4575 if (unlikely(ftrace_disabled))
4576 return -ENODEV;
4577
4578 fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
4579 if (fgd == NULL)
4580 return -ENOMEM;
4581
4582 fgd->table = ftrace_graph_notrace_funcs;
4583 fgd->size = FTRACE_GRAPH_MAX_FUNCS;
4584 fgd->count = &ftrace_graph_notrace_count;
4585 fgd->seq_ops = &ftrace_graph_seq_ops;
4586
4587 return __ftrace_graph_open(inode, file, fgd);
4588}
4589
4590static int
Li Zefan87827112009-07-23 11:29:11 +08004591ftrace_graph_release(struct inode *inode, struct file *file)
4592{
Namhyung Kimfaf982a2013-10-14 17:24:24 +09004593 if (file->f_mode & FMODE_READ) {
4594 struct seq_file *m = file->private_data;
4595
4596 kfree(m->private);
Li Zefan87827112009-07-23 11:29:11 +08004597 seq_release(inode, file);
Namhyung Kimfaf982a2013-10-14 17:24:24 +09004598 } else {
4599 kfree(file->private_data);
4600 }
4601
Li Zefan87827112009-07-23 11:29:11 +08004602 return 0;
4603}
4604
4605static int
Namhyung Kimfaf982a2013-10-14 17:24:24 +09004606ftrace_set_func(unsigned long *array, int *idx, int size, char *buffer)
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004607{
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004608 struct dyn_ftrace *rec;
4609 struct ftrace_page *pg;
Frederic Weisbeckerf9349a82009-02-19 21:13:12 +01004610 int search_len;
Li Zefanc7c6b1f2010-02-10 15:43:04 +08004611 int fail = 1;
Frederic Weisbeckerf9349a82009-02-19 21:13:12 +01004612 int type, not;
4613 char *search;
4614 bool exists;
4615 int i;
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004616
Frederic Weisbeckerf9349a82009-02-19 21:13:12 +01004617 /* decode regex */
Frederic Weisbecker3f6fe062009-09-24 21:31:51 +02004618 type = filter_parse_regex(buffer, strlen(buffer), &search, &not);
Namhyung Kimfaf982a2013-10-14 17:24:24 +09004619 if (!not && *idx >= size)
Li Zefanc7c6b1f2010-02-10 15:43:04 +08004620 return -EBUSY;
Frederic Weisbeckerf9349a82009-02-19 21:13:12 +01004621
4622 search_len = strlen(search);
4623
Steven Rostedt52baf112009-02-14 01:15:39 -05004624 mutex_lock(&ftrace_lock);
Steven Rostedt45a4a232011-04-21 23:16:46 -04004625
4626 if (unlikely(ftrace_disabled)) {
4627 mutex_unlock(&ftrace_lock);
4628 return -ENODEV;
4629 }
4630
Steven Rostedt265c8312009-02-13 12:43:56 -05004631 do_for_each_ftrace_rec(pg, rec) {
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004632
Steven Rostedtb9df92d2011-04-28 20:32:08 -04004633 if (ftrace_match_record(rec, NULL, search, search_len, type)) {
Li Zefanc7c6b1f2010-02-10 15:43:04 +08004634 /* if it is in the array */
Frederic Weisbeckerf9349a82009-02-19 21:13:12 +01004635 exists = false;
Li Zefanc7c6b1f2010-02-10 15:43:04 +08004636 for (i = 0; i < *idx; i++) {
Frederic Weisbeckerf9349a82009-02-19 21:13:12 +01004637 if (array[i] == rec->ip) {
4638 exists = true;
Steven Rostedt265c8312009-02-13 12:43:56 -05004639 break;
4640 }
Li Zefanc7c6b1f2010-02-10 15:43:04 +08004641 }
4642
4643 if (!not) {
4644 fail = 0;
4645 if (!exists) {
4646 array[(*idx)++] = rec->ip;
Namhyung Kimfaf982a2013-10-14 17:24:24 +09004647 if (*idx >= size)
Li Zefanc7c6b1f2010-02-10 15:43:04 +08004648 goto out;
4649 }
4650 } else {
4651 if (exists) {
4652 array[i] = array[--(*idx)];
4653 array[*idx] = 0;
4654 fail = 0;
4655 }
4656 }
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004657 }
Steven Rostedt265c8312009-02-13 12:43:56 -05004658 } while_for_each_ftrace_rec();
Li Zefanc7c6b1f2010-02-10 15:43:04 +08004659out:
Steven Rostedt52baf112009-02-14 01:15:39 -05004660 mutex_unlock(&ftrace_lock);
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004661
Li Zefanc7c6b1f2010-02-10 15:43:04 +08004662 if (fail)
4663 return -EINVAL;
4664
Li Zefanc7c6b1f2010-02-10 15:43:04 +08004665 return 0;
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004666}
4667
4668static ssize_t
4669ftrace_graph_write(struct file *file, const char __user *ubuf,
4670 size_t cnt, loff_t *ppos)
4671{
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02004672 struct trace_parser parser;
Namhyung Kim6a101082013-10-14 17:24:25 +09004673 ssize_t read, ret = 0;
Namhyung Kimfaf982a2013-10-14 17:24:24 +09004674 struct ftrace_graph_data *fgd = file->private_data;
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004675
Li Zefanc7c6b1f2010-02-10 15:43:04 +08004676 if (!cnt)
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004677 return 0;
4678
Namhyung Kim6a101082013-10-14 17:24:25 +09004679 if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX))
4680 return -ENOMEM;
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004681
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02004682 read = trace_get_user(&parser, ubuf, cnt, ppos);
4683
Li Zefan4ba79782009-09-22 13:52:20 +08004684 if (read >= 0 && trace_parser_loaded((&parser))) {
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02004685 parser.buffer[parser.idx] = 0;
4686
Namhyung Kim6a101082013-10-14 17:24:25 +09004687 mutex_lock(&graph_lock);
4688
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02004689 /* we allow only one expression at a time */
Namhyung Kimfaf982a2013-10-14 17:24:24 +09004690 ret = ftrace_set_func(fgd->table, fgd->count, fgd->size,
4691 parser.buffer);
Namhyung Kim6a101082013-10-14 17:24:25 +09004692
4693 mutex_unlock(&graph_lock);
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004694 }
4695
Namhyung Kim6a101082013-10-14 17:24:25 +09004696 if (!ret)
4697 ret = read;
Li Zefan1eb90f12009-09-22 13:52:57 +08004698
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02004699 trace_parser_put(&parser);
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004700
4701 return ret;
4702}
4703
4704static const struct file_operations ftrace_graph_fops = {
Li Zefan87827112009-07-23 11:29:11 +08004705 .open = ftrace_graph_open,
4706 .read = seq_read,
4707 .write = ftrace_graph_write,
Steven Rostedt (Red Hat)098c8792013-12-21 17:39:40 -05004708 .llseek = tracing_lseek,
Li Zefan87827112009-07-23 11:29:11 +08004709 .release = ftrace_graph_release,
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004710};
Namhyung Kim29ad23b2013-10-14 17:24:26 +09004711
4712static const struct file_operations ftrace_graph_notrace_fops = {
4713 .open = ftrace_graph_notrace_open,
4714 .read = seq_read,
4715 .write = ftrace_graph_write,
Steven Rostedt (Red Hat)098c8792013-12-21 17:39:40 -05004716 .llseek = tracing_lseek,
Namhyung Kim29ad23b2013-10-14 17:24:26 +09004717 .release = ftrace_graph_release,
4718};
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004719#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
4720
Steven Rostedt (Red Hat)591dffd2014-01-10 16:17:45 -05004721void ftrace_create_filter_files(struct ftrace_ops *ops,
4722 struct dentry *parent)
4723{
4724
4725 trace_create_file("set_ftrace_filter", 0644, parent,
4726 ops, &ftrace_filter_fops);
4727
4728 trace_create_file("set_ftrace_notrace", 0644, parent,
4729 ops, &ftrace_notrace_fops);
4730}
4731
4732/*
4733 * The name "destroy_filter_files" is really a misnomer. Although
4734 * in the future, it may actualy delete the files, but this is
4735 * really intended to make sure the ops passed in are disabled
4736 * and that when this function returns, the caller is free to
4737 * free the ops.
4738 *
4739 * The "destroy" name is only to match the "create" name that this
4740 * should be paired with.
4741 */
4742void ftrace_destroy_filter_files(struct ftrace_ops *ops)
4743{
4744 mutex_lock(&ftrace_lock);
4745 if (ops->flags & FTRACE_OPS_FL_ENABLED)
4746 ftrace_shutdown(ops, 0);
4747 ops->flags |= FTRACE_OPS_FL_DELETED;
4748 mutex_unlock(&ftrace_lock);
4749}
4750
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05004751static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer)
Steven Rostedt5072c592008-05-12 21:20:43 +02004752{
Steven Rostedt5072c592008-05-12 21:20:43 +02004753
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004754 trace_create_file("available_filter_functions", 0444,
4755 d_tracer, NULL, &ftrace_avail_fops);
Steven Rostedt5072c592008-05-12 21:20:43 +02004756
Steven Rostedt647bcd02011-05-03 14:39:21 -04004757 trace_create_file("enabled_functions", 0444,
4758 d_tracer, NULL, &ftrace_enabled_fops);
4759
Steven Rostedt (Red Hat)591dffd2014-01-10 16:17:45 -05004760 ftrace_create_filter_files(&global_ops, d_tracer);
Steven Rostedtad90c0e2008-05-27 20:48:37 -04004761
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004762#ifdef CONFIG_FUNCTION_GRAPH_TRACER
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004763 trace_create_file("set_graph_function", 0444, d_tracer,
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004764 NULL,
4765 &ftrace_graph_fops);
Namhyung Kim29ad23b2013-10-14 17:24:26 +09004766 trace_create_file("set_graph_notrace", 0444, d_tracer,
4767 NULL,
4768 &ftrace_graph_notrace_fops);
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05004769#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
4770
Steven Rostedt5072c592008-05-12 21:20:43 +02004771 return 0;
4772}
4773
Steven Rostedt9fd49322012-04-24 22:32:06 -04004774static int ftrace_cmp_ips(const void *a, const void *b)
Steven Rostedt68950612011-12-16 17:06:45 -05004775{
Steven Rostedt9fd49322012-04-24 22:32:06 -04004776 const unsigned long *ipa = a;
4777 const unsigned long *ipb = b;
Steven Rostedt68950612011-12-16 17:06:45 -05004778
Steven Rostedt9fd49322012-04-24 22:32:06 -04004779 if (*ipa > *ipb)
4780 return 1;
4781 if (*ipa < *ipb)
4782 return -1;
4783 return 0;
4784}
4785
Jiri Olsa5cb084b2009-10-13 16:33:53 -04004786static int ftrace_process_locs(struct module *mod,
Steven Rostedt31e88902008-11-14 16:21:19 -08004787 unsigned long *start,
Steven Rostedt68bf21a2008-08-14 15:45:08 -04004788 unsigned long *end)
4789{
Steven Rostedt706c81f2012-04-24 23:45:26 -04004790 struct ftrace_page *start_pg;
Steven Rostedta7900872011-12-16 16:23:44 -05004791 struct ftrace_page *pg;
Steven Rostedt706c81f2012-04-24 23:45:26 -04004792 struct dyn_ftrace *rec;
Steven Rostedta7900872011-12-16 16:23:44 -05004793 unsigned long count;
Steven Rostedt68bf21a2008-08-14 15:45:08 -04004794 unsigned long *p;
4795 unsigned long addr;
Steven Rostedt4376cac2011-06-24 23:28:13 -04004796 unsigned long flags = 0; /* Shut up gcc */
Steven Rostedta7900872011-12-16 16:23:44 -05004797 int ret = -ENOMEM;
4798
4799 count = end - start;
4800
4801 if (!count)
4802 return 0;
4803
Steven Rostedt9fd49322012-04-24 22:32:06 -04004804 sort(start, count, sizeof(*start),
Rasmus Villemoes6db02902015-09-09 23:27:02 +02004805 ftrace_cmp_ips, NULL);
Steven Rostedt9fd49322012-04-24 22:32:06 -04004806
Steven Rostedt706c81f2012-04-24 23:45:26 -04004807 start_pg = ftrace_allocate_pages(count);
4808 if (!start_pg)
Steven Rostedta7900872011-12-16 16:23:44 -05004809 return -ENOMEM;
Steven Rostedt68bf21a2008-08-14 15:45:08 -04004810
Steven Rostedte6ea44e2009-02-14 01:42:44 -05004811 mutex_lock(&ftrace_lock);
Steven Rostedta7900872011-12-16 16:23:44 -05004812
Steven Rostedt320823092011-12-16 14:42:37 -05004813 /*
4814 * Core and each module needs their own pages, as
4815 * modules will free them when they are removed.
4816 * Force a new page to be allocated for modules.
4817 */
Steven Rostedta7900872011-12-16 16:23:44 -05004818 if (!mod) {
4819 WARN_ON(ftrace_pages || ftrace_pages_start);
4820 /* First initialization */
Steven Rostedt706c81f2012-04-24 23:45:26 -04004821 ftrace_pages = ftrace_pages_start = start_pg;
Steven Rostedta7900872011-12-16 16:23:44 -05004822 } else {
Steven Rostedt320823092011-12-16 14:42:37 -05004823 if (!ftrace_pages)
Steven Rostedta7900872011-12-16 16:23:44 -05004824 goto out;
Steven Rostedt320823092011-12-16 14:42:37 -05004825
Steven Rostedta7900872011-12-16 16:23:44 -05004826 if (WARN_ON(ftrace_pages->next)) {
4827 /* Hmm, we have free pages? */
4828 while (ftrace_pages->next)
4829 ftrace_pages = ftrace_pages->next;
Steven Rostedt320823092011-12-16 14:42:37 -05004830 }
Steven Rostedta7900872011-12-16 16:23:44 -05004831
Steven Rostedt706c81f2012-04-24 23:45:26 -04004832 ftrace_pages->next = start_pg;
Steven Rostedt320823092011-12-16 14:42:37 -05004833 }
4834
Steven Rostedt68bf21a2008-08-14 15:45:08 -04004835 p = start;
Steven Rostedt706c81f2012-04-24 23:45:26 -04004836 pg = start_pg;
Steven Rostedt68bf21a2008-08-14 15:45:08 -04004837 while (p < end) {
4838 addr = ftrace_call_adjust(*p++);
Steven Rostedt20e52272008-11-14 16:21:19 -08004839 /*
4840 * Some architecture linkers will pad between
4841 * the different mcount_loc sections of different
4842 * object files to satisfy alignments.
4843 * Skip any NULL pointers.
4844 */
4845 if (!addr)
4846 continue;
Steven Rostedt706c81f2012-04-24 23:45:26 -04004847
4848 if (pg->index == pg->size) {
4849 /* We should have allocated enough */
4850 if (WARN_ON(!pg->next))
4851 break;
4852 pg = pg->next;
4853 }
4854
4855 rec = &pg->records[pg->index++];
4856 rec->ip = addr;
Steven Rostedt68bf21a2008-08-14 15:45:08 -04004857 }
4858
Steven Rostedt706c81f2012-04-24 23:45:26 -04004859 /* We should have used all pages */
4860 WARN_ON(pg->next);
4861
4862 /* Assign the last page to ftrace_pages */
4863 ftrace_pages = pg;
4864
Steven Rostedta4f18ed2011-06-07 09:26:46 -04004865 /*
Steven Rostedt4376cac2011-06-24 23:28:13 -04004866 * We only need to disable interrupts on start up
4867 * because we are modifying code that an interrupt
4868 * may execute, and the modification is not atomic.
4869 * But for modules, nothing runs the code we modify
4870 * until we are finished with it, and there's no
4871 * reason to cause large interrupt latencies while we do it.
Steven Rostedta4f18ed2011-06-07 09:26:46 -04004872 */
Steven Rostedt4376cac2011-06-24 23:28:13 -04004873 if (!mod)
4874 local_irq_save(flags);
Jiri Slaby1dc43cf2014-02-24 19:59:56 +01004875 ftrace_update_code(mod, start_pg);
Steven Rostedt4376cac2011-06-24 23:28:13 -04004876 if (!mod)
4877 local_irq_restore(flags);
Steven Rostedta7900872011-12-16 16:23:44 -05004878 ret = 0;
4879 out:
Steven Rostedte6ea44e2009-02-14 01:42:44 -05004880 mutex_unlock(&ftrace_lock);
Steven Rostedt68bf21a2008-08-14 15:45:08 -04004881
Steven Rostedta7900872011-12-16 16:23:44 -05004882 return ret;
Steven Rostedt68bf21a2008-08-14 15:45:08 -04004883}
4884
Steven Rostedt93eb6772009-04-15 13:24:06 -04004885#ifdef CONFIG_MODULES
Steven Rostedt320823092011-12-16 14:42:37 -05004886
4887#define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next)
4888
jolsa@redhat.come7247a12009-10-07 19:00:35 +02004889void ftrace_release_mod(struct module *mod)
Steven Rostedt93eb6772009-04-15 13:24:06 -04004890{
4891 struct dyn_ftrace *rec;
Steven Rostedt320823092011-12-16 14:42:37 -05004892 struct ftrace_page **last_pg;
Steven Rostedt93eb6772009-04-15 13:24:06 -04004893 struct ftrace_page *pg;
Steven Rostedta7900872011-12-16 16:23:44 -05004894 int order;
Steven Rostedt93eb6772009-04-15 13:24:06 -04004895
Steven Rostedt93eb6772009-04-15 13:24:06 -04004896 mutex_lock(&ftrace_lock);
Steven Rostedt45a4a232011-04-21 23:16:46 -04004897
4898 if (ftrace_disabled)
4899 goto out_unlock;
4900
Steven Rostedt320823092011-12-16 14:42:37 -05004901 /*
4902 * Each module has its own ftrace_pages, remove
4903 * them from the list.
4904 */
4905 last_pg = &ftrace_pages_start;
4906 for (pg = ftrace_pages_start; pg; pg = *last_pg) {
4907 rec = &pg->records[0];
jolsa@redhat.come7247a12009-10-07 19:00:35 +02004908 if (within_module_core(rec->ip, mod)) {
Steven Rostedt93eb6772009-04-15 13:24:06 -04004909 /*
Steven Rostedt320823092011-12-16 14:42:37 -05004910 * As core pages are first, the first
4911 * page should never be a module page.
Steven Rostedt93eb6772009-04-15 13:24:06 -04004912 */
Steven Rostedt320823092011-12-16 14:42:37 -05004913 if (WARN_ON(pg == ftrace_pages_start))
4914 goto out_unlock;
4915
4916 /* Check if we are deleting the last page */
4917 if (pg == ftrace_pages)
4918 ftrace_pages = next_to_ftrace_page(last_pg);
4919
4920 *last_pg = pg->next;
Steven Rostedta7900872011-12-16 16:23:44 -05004921 order = get_count_order(pg->size / ENTRIES_PER_PAGE);
4922 free_pages((unsigned long)pg->records, order);
4923 kfree(pg);
Steven Rostedt320823092011-12-16 14:42:37 -05004924 } else
4925 last_pg = &pg->next;
4926 }
Steven Rostedt45a4a232011-04-21 23:16:46 -04004927 out_unlock:
Steven Rostedt93eb6772009-04-15 13:24:06 -04004928 mutex_unlock(&ftrace_lock);
4929}
4930
4931static void ftrace_init_module(struct module *mod,
4932 unsigned long *start, unsigned long *end)
Steven Rostedt90d595f2008-08-14 15:45:09 -04004933{
Steven Rostedt00fd61a2008-08-15 21:40:04 -04004934 if (ftrace_disabled || start == end)
Steven Rostedtfed19392008-08-14 22:47:19 -04004935 return;
Jiri Olsa5cb084b2009-10-13 16:33:53 -04004936 ftrace_process_locs(mod, start, end);
Steven Rostedt90d595f2008-08-14 15:45:09 -04004937}
4938
Steven Rostedt (Red Hat)a949ae52014-04-24 10:40:12 -04004939void ftrace_module_init(struct module *mod)
Steven Rostedt93eb6772009-04-15 13:24:06 -04004940{
Steven Rostedt (Red Hat)a949ae52014-04-24 10:40:12 -04004941 ftrace_init_module(mod, mod->ftrace_callsites,
4942 mod->ftrace_callsites +
4943 mod->num_ftrace_callsites);
Steven Rostedt (Red Hat)8c189ea2013-02-13 15:18:38 -05004944}
4945
4946static int ftrace_module_notify_exit(struct notifier_block *self,
4947 unsigned long val, void *data)
4948{
4949 struct module *mod = data;
4950
4951 if (val == MODULE_STATE_GOING)
jolsa@redhat.come7247a12009-10-07 19:00:35 +02004952 ftrace_release_mod(mod);
Steven Rostedt93eb6772009-04-15 13:24:06 -04004953
4954 return 0;
4955}
4956#else
Steven Rostedt (Red Hat)8c189ea2013-02-13 15:18:38 -05004957static int ftrace_module_notify_exit(struct notifier_block *self,
4958 unsigned long val, void *data)
Steven Rostedt93eb6772009-04-15 13:24:06 -04004959{
4960 return 0;
4961}
4962#endif /* CONFIG_MODULES */
4963
Steven Rostedt (Red Hat)8c189ea2013-02-13 15:18:38 -05004964struct notifier_block ftrace_module_exit_nb = {
4965 .notifier_call = ftrace_module_notify_exit,
4966 .priority = INT_MIN, /* Run after anything that can remove kprobes */
4967};
4968
Steven Rostedt68bf21a2008-08-14 15:45:08 -04004969void __init ftrace_init(void)
4970{
Jiri Slaby1dc43cf2014-02-24 19:59:56 +01004971 extern unsigned long __start_mcount_loc[];
4972 extern unsigned long __stop_mcount_loc[];
Jiri Slaby3a36cb12014-02-24 19:59:59 +01004973 unsigned long count, flags;
Steven Rostedt68bf21a2008-08-14 15:45:08 -04004974 int ret;
4975
Steven Rostedt68bf21a2008-08-14 15:45:08 -04004976 local_irq_save(flags);
Jiri Slaby3a36cb12014-02-24 19:59:59 +01004977 ret = ftrace_dyn_arch_init();
Steven Rostedt68bf21a2008-08-14 15:45:08 -04004978 local_irq_restore(flags);
Jiri Slabyaf64a7c2014-02-24 19:59:58 +01004979 if (ret)
Steven Rostedt68bf21a2008-08-14 15:45:08 -04004980 goto failed;
4981
4982 count = __stop_mcount_loc - __start_mcount_loc;
Jiri Slabyc867ccd2014-02-24 19:59:57 +01004983 if (!count) {
4984 pr_info("ftrace: No functions to be traced?\n");
Steven Rostedt68bf21a2008-08-14 15:45:08 -04004985 goto failed;
Jiri Slabyc867ccd2014-02-24 19:59:57 +01004986 }
4987
4988 pr_info("ftrace: allocating %ld entries in %ld pages\n",
4989 count, count / ENTRIES_PER_PAGE + 1);
Steven Rostedt68bf21a2008-08-14 15:45:08 -04004990
4991 last_ftrace_enabled = ftrace_enabled = 1;
4992
Jiri Olsa5cb084b2009-10-13 16:33:53 -04004993 ret = ftrace_process_locs(NULL,
Steven Rostedt31e88902008-11-14 16:21:19 -08004994 __start_mcount_loc,
Steven Rostedt68bf21a2008-08-14 15:45:08 -04004995 __stop_mcount_loc);
4996
Steven Rostedt (Red Hat)8c189ea2013-02-13 15:18:38 -05004997 ret = register_module_notifier(&ftrace_module_exit_nb);
4998 if (ret)
4999 pr_warning("Failed to register trace ftrace module exit notifier\n");
Steven Rostedt93eb6772009-04-15 13:24:06 -04005000
Steven Rostedt2af15d62009-05-28 13:37:24 -04005001 set_ftrace_early_filters();
5002
Steven Rostedt68bf21a2008-08-14 15:45:08 -04005003 return;
5004 failed:
5005 ftrace_disabled = 1;
5006}
Steven Rostedt68bf21a2008-08-14 15:45:08 -04005007
Steven Rostedt (Red Hat)f3bea492014-07-02 23:23:31 -04005008/* Do nothing if arch does not support this */
5009void __weak arch_ftrace_update_trampoline(struct ftrace_ops *ops)
5010{
5011}
5012
5013static void ftrace_update_trampoline(struct ftrace_ops *ops)
5014{
Steven Rostedt (Red Hat)12cce592014-07-03 15:48:16 -04005015
5016/*
5017 * Currently there's no safe way to free a trampoline when the kernel
5018 * is configured with PREEMPT. That is because a task could be preempted
5019 * when it jumped to the trampoline, it may be preempted for a long time
5020 * depending on the system load, and currently there's no way to know
5021 * when it will be off the trampoline. If the trampoline is freed
5022 * too early, when the task runs again, it will be executing on freed
5023 * memory and crash.
5024 */
5025#ifdef CONFIG_PREEMPT
Steven Rostedt (Red Hat)f3bea492014-07-02 23:23:31 -04005026 /* Currently, only non dynamic ops can have a trampoline */
5027 if (ops->flags & FTRACE_OPS_FL_DYNAMIC)
5028 return;
Steven Rostedt (Red Hat)12cce592014-07-03 15:48:16 -04005029#endif
Steven Rostedt (Red Hat)f3bea492014-07-02 23:23:31 -04005030
5031 arch_ftrace_update_trampoline(ops);
5032}
5033
Steven Rostedt3d083392008-05-12 21:20:42 +02005034#else
Frederic Weisbecker0b6e4d52008-10-28 20:17:38 +01005035
Steven Rostedt2b499382011-05-03 22:49:52 -04005036static struct ftrace_ops global_ops = {
Steven Rostedtbd69c302011-05-03 21:55:54 -04005037 .func = ftrace_stub,
Steven Rostedt (Red Hat)e3eea142015-07-24 10:38:12 -04005038 .flags = FTRACE_OPS_FL_RECURSION_SAFE |
5039 FTRACE_OPS_FL_INITIALIZED |
5040 FTRACE_OPS_FL_PID,
Steven Rostedtbd69c302011-05-03 21:55:54 -04005041};
5042
Frederic Weisbecker0b6e4d52008-10-28 20:17:38 +01005043static int __init ftrace_nodyn_init(void)
5044{
5045 ftrace_enabled = 1;
5046 return 0;
5047}
Steven Rostedt6f415672012-10-05 12:13:07 -04005048core_initcall(ftrace_nodyn_init);
Frederic Weisbecker0b6e4d52008-10-28 20:17:38 +01005049
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05005050static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; }
Steven Rostedtdf4fc312008-11-26 00:16:23 -05005051static inline void ftrace_startup_enable(int command) { }
Steven Rostedt (Red Hat)e1effa02014-08-05 17:19:38 -04005052static inline void ftrace_startup_all(int command) { }
Steven Rostedt5a45cfe2008-11-26 00:16:24 -05005053/* Keep as macros so we do not need to define the commands */
Steven Rostedt (Red Hat)8a56d772013-11-25 20:59:46 -05005054# define ftrace_startup(ops, command) \
5055 ({ \
5056 int ___ret = __register_ftrace_function(ops); \
5057 if (!___ret) \
5058 (ops)->flags |= FTRACE_OPS_FL_ENABLED; \
5059 ___ret; \
Steven Rostedt3b6cfdb2011-05-23 15:33:49 -04005060 })
Steven Rostedt (Red Hat)1fcc1552014-02-19 15:12:18 -05005061# define ftrace_shutdown(ops, command) \
5062 ({ \
5063 int ___ret = __unregister_ftrace_function(ops); \
5064 if (!___ret) \
5065 (ops)->flags &= ~FTRACE_OPS_FL_ENABLED; \
5066 ___ret; \
5067 })
Steven Rostedt (Red Hat)8a56d772013-11-25 20:59:46 -05005068
Ingo Molnarc7aafc52008-05-12 21:20:45 +02005069# define ftrace_startup_sysctl() do { } while (0)
5070# define ftrace_shutdown_sysctl() do { } while (0)
Steven Rostedtb8489142011-05-04 09:27:52 -04005071
5072static inline int
Steven Rostedt (Red Hat)195a8af2013-07-23 22:06:15 -04005073ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
Steven Rostedtb8489142011-05-04 09:27:52 -04005074{
5075 return 1;
5076}
5077
Steven Rostedt (Red Hat)f3bea492014-07-02 23:23:31 -04005078static void ftrace_update_trampoline(struct ftrace_ops *ops)
5079{
5080}
5081
Steven Rostedt3d083392008-05-12 21:20:42 +02005082#endif /* CONFIG_DYNAMIC_FTRACE */
5083
Steven Rostedt (Red Hat)4104d322014-01-10 17:01:58 -05005084__init void ftrace_init_global_array_ops(struct trace_array *tr)
5085{
5086 tr->ops = &global_ops;
5087 tr->ops->private = tr;
5088}
5089
5090void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func)
5091{
5092 /* If we filter on pids, update to use the pid function */
5093 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
5094 if (WARN_ON(tr->ops->func != ftrace_stub))
5095 printk("ftrace ops had %pS for function\n",
5096 tr->ops->func);
Steven Rostedt (Red Hat)4104d322014-01-10 17:01:58 -05005097 }
5098 tr->ops->func = func;
5099 tr->ops->private = tr;
5100}
5101
5102void ftrace_reset_array_ops(struct trace_array *tr)
5103{
5104 tr->ops->func = ftrace_stub;
5105}
5106
Steven Rostedtb8489142011-05-04 09:27:52 -04005107static void
Steven Rostedt2f5f6ad2011-08-08 16:57:47 -04005108ftrace_ops_control_func(unsigned long ip, unsigned long parent_ip,
Steven Rostedta1e2e312011-08-09 12:50:46 -04005109 struct ftrace_ops *op, struct pt_regs *regs)
Jiri Olsae2484912012-02-15 15:51:48 +01005110{
Jiri Olsae2484912012-02-15 15:51:48 +01005111 if (unlikely(trace_recursion_test(TRACE_CONTROL_BIT)))
5112 return;
5113
5114 /*
5115 * Some of the ops may be dynamically allocated,
5116 * they must be freed after a synchronize_sched().
5117 */
5118 preempt_disable_notrace();
5119 trace_recursion_set(TRACE_CONTROL_BIT);
Steven Rostedt (Red Hat)b5aa3a42013-11-04 18:34:44 -05005120
5121 /*
5122 * Control funcs (perf) uses RCU. Only trace if
5123 * RCU is currently active.
5124 */
5125 if (!rcu_is_watching())
5126 goto out;
5127
Steven Rostedt0a016402012-11-02 17:03:03 -04005128 do_for_each_ftrace_op(op, ftrace_control_list) {
Steven Rostedt (Red Hat)395b97a2013-03-27 09:31:28 -04005129 if (!(op->flags & FTRACE_OPS_FL_STUB) &&
5130 !ftrace_function_local_disabled(op) &&
Steven Rostedt (Red Hat)195a8af2013-07-23 22:06:15 -04005131 ftrace_ops_test(op, ip, regs))
Steven Rostedta1e2e312011-08-09 12:50:46 -04005132 op->func(ip, parent_ip, op, regs);
Steven Rostedt0a016402012-11-02 17:03:03 -04005133 } while_for_each_ftrace_op(op);
Steven Rostedt (Red Hat)b5aa3a42013-11-04 18:34:44 -05005134 out:
Jiri Olsae2484912012-02-15 15:51:48 +01005135 trace_recursion_clear(TRACE_CONTROL_BIT);
5136 preempt_enable_notrace();
5137}
5138
5139static struct ftrace_ops control_ops = {
Masami Hiramatsuf04f24fb2013-05-09 14:44:17 +09005140 .func = ftrace_ops_control_func,
5141 .flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
Steven Rostedt (Red Hat)33b7f992014-08-15 17:23:02 -04005142 INIT_OPS_HASH(control_ops)
Jiri Olsae2484912012-02-15 15:51:48 +01005143};
5144
Steven Rostedt2f5f6ad2011-08-08 16:57:47 -04005145static inline void
5146__ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
Steven Rostedta1e2e312011-08-09 12:50:46 -04005147 struct ftrace_ops *ignored, struct pt_regs *regs)
Steven Rostedtb8489142011-05-04 09:27:52 -04005148{
Steven Rostedtcdbe61b2011-05-05 21:14:55 -04005149 struct ftrace_ops *op;
Steven Rostedtedc15ca2012-11-02 17:47:21 -04005150 int bit;
Steven Rostedtb8489142011-05-04 09:27:52 -04005151
Steven Rostedtedc15ca2012-11-02 17:47:21 -04005152 bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
5153 if (bit < 0)
5154 return;
Steven Rostedtc29f1222012-11-02 17:17:59 -04005155
Steven Rostedtcdbe61b2011-05-05 21:14:55 -04005156 /*
5157 * Some of the ops may be dynamically allocated,
5158 * they must be freed after a synchronize_sched().
5159 */
5160 preempt_disable_notrace();
Steven Rostedt0a016402012-11-02 17:03:03 -04005161 do_for_each_ftrace_op(op, ftrace_ops_list) {
Steven Rostedt (Red Hat)4104d322014-01-10 17:01:58 -05005162 if (ftrace_ops_test(op, ip, regs)) {
Steven Rostedt (Red Hat)1d48d592014-06-25 11:54:03 -04005163 if (FTRACE_WARN_ON(!op->func)) {
5164 pr_warn("op=%p %pS\n", op, op);
Steven Rostedt (Red Hat)4104d322014-01-10 17:01:58 -05005165 goto out;
5166 }
Steven Rostedta1e2e312011-08-09 12:50:46 -04005167 op->func(ip, parent_ip, op, regs);
Steven Rostedt (Red Hat)4104d322014-01-10 17:01:58 -05005168 }
Steven Rostedt0a016402012-11-02 17:03:03 -04005169 } while_for_each_ftrace_op(op);
Steven Rostedt (Red Hat)4104d322014-01-10 17:01:58 -05005170out:
Steven Rostedtcdbe61b2011-05-05 21:14:55 -04005171 preempt_enable_notrace();
Steven Rostedtedc15ca2012-11-02 17:47:21 -04005172 trace_clear_recursion(bit);
Steven Rostedtb8489142011-05-04 09:27:52 -04005173}
5174
Steven Rostedt2f5f6ad2011-08-08 16:57:47 -04005175/*
5176 * Some archs only support passing ip and parent_ip. Even though
5177 * the list function ignores the op parameter, we do not want any
5178 * C side effects, where a function is called without the caller
5179 * sending a third parameter.
Steven Rostedta1e2e312011-08-09 12:50:46 -04005180 * Archs are to support both the regs and ftrace_ops at the same time.
5181 * If they support ftrace_ops, it is assumed they support regs.
5182 * If call backs want to use regs, they must either check for regs
Masami Hiramatsu06aeaae2012-09-28 17:15:17 +09005183 * being NULL, or CONFIG_DYNAMIC_FTRACE_WITH_REGS.
5184 * Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved.
Steven Rostedta1e2e312011-08-09 12:50:46 -04005185 * An architecture can pass partial regs with ftrace_ops and still
5186 * set the ARCH_SUPPORT_FTARCE_OPS.
Steven Rostedt2f5f6ad2011-08-08 16:57:47 -04005187 */
5188#if ARCH_SUPPORTS_FTRACE_OPS
5189static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
Steven Rostedta1e2e312011-08-09 12:50:46 -04005190 struct ftrace_ops *op, struct pt_regs *regs)
Steven Rostedt2f5f6ad2011-08-08 16:57:47 -04005191{
Steven Rostedta1e2e312011-08-09 12:50:46 -04005192 __ftrace_ops_list_func(ip, parent_ip, NULL, regs);
Steven Rostedt2f5f6ad2011-08-08 16:57:47 -04005193}
5194#else
5195static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip)
5196{
Steven Rostedta1e2e312011-08-09 12:50:46 -04005197 __ftrace_ops_list_func(ip, parent_ip, NULL, NULL);
Steven Rostedt2f5f6ad2011-08-08 16:57:47 -04005198}
5199#endif
5200
Steven Rostedt (Red Hat)f1ff6342014-07-22 20:16:57 -04005201/*
5202 * If there's only one function registered but it does not support
5203 * recursion, this function will be called by the mcount trampoline.
5204 * This function will handle recursion protection.
5205 */
5206static void ftrace_ops_recurs_func(unsigned long ip, unsigned long parent_ip,
5207 struct ftrace_ops *op, struct pt_regs *regs)
5208{
5209 int bit;
5210
5211 bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
5212 if (bit < 0)
5213 return;
5214
5215 op->func(ip, parent_ip, op, regs);
5216
5217 trace_clear_recursion(bit);
5218}
5219
Steven Rostedt (Red Hat)87354052014-07-22 20:41:42 -04005220/**
5221 * ftrace_ops_get_func - get the function a trampoline should call
5222 * @ops: the ops to get the function for
5223 *
5224 * Normally the mcount trampoline will call the ops->func, but there
5225 * are times that it should not. For example, if the ops does not
5226 * have its own recursion protection, then it should call the
5227 * ftrace_ops_recurs_func() instead.
5228 *
5229 * Returns the function that the trampoline should call for @ops.
5230 */
5231ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops)
5232{
5233 /*
Steven Rostedt (Red Hat)87354052014-07-22 20:41:42 -04005234 * If the func handles its own recursion, call it directly.
5235 * Otherwise call the recursion protected function that
5236 * will call the ftrace ops function.
5237 */
5238 if (!(ops->flags & FTRACE_OPS_FL_RECURSION_SAFE))
5239 return ftrace_ops_recurs_func;
5240
5241 return ops->func;
5242}
5243
Steven Rostedte32d8952008-12-04 00:26:41 -05005244static void clear_ftrace_swapper(void)
5245{
5246 struct task_struct *p;
5247 int cpu;
5248
5249 get_online_cpus();
5250 for_each_online_cpu(cpu) {
5251 p = idle_task(cpu);
5252 clear_tsk_trace_trace(p);
5253 }
5254 put_online_cpus();
5255}
5256
5257static void set_ftrace_swapper(void)
5258{
5259 struct task_struct *p;
5260 int cpu;
5261
5262 get_online_cpus();
5263 for_each_online_cpu(cpu) {
5264 p = idle_task(cpu);
5265 set_tsk_trace_trace(p);
5266 }
5267 put_online_cpus();
5268}
5269
5270static void clear_ftrace_pid(struct pid *pid)
Steven Rostedt978f3a42008-12-04 00:26:40 -05005271{
5272 struct task_struct *p;
5273
Oleg Nesterov229c4ef2009-02-03 20:39:04 +01005274 rcu_read_lock();
Steven Rostedte32d8952008-12-04 00:26:41 -05005275 do_each_pid_task(pid, PIDTYPE_PID, p) {
Steven Rostedt978f3a42008-12-04 00:26:40 -05005276 clear_tsk_trace_trace(p);
Steven Rostedte32d8952008-12-04 00:26:41 -05005277 } while_each_pid_task(pid, PIDTYPE_PID, p);
Oleg Nesterov229c4ef2009-02-03 20:39:04 +01005278 rcu_read_unlock();
5279
Steven Rostedte32d8952008-12-04 00:26:41 -05005280 put_pid(pid);
Steven Rostedt978f3a42008-12-04 00:26:40 -05005281}
5282
Steven Rostedte32d8952008-12-04 00:26:41 -05005283static void set_ftrace_pid(struct pid *pid)
Steven Rostedt978f3a42008-12-04 00:26:40 -05005284{
5285 struct task_struct *p;
5286
Oleg Nesterov229c4ef2009-02-03 20:39:04 +01005287 rcu_read_lock();
Steven Rostedt978f3a42008-12-04 00:26:40 -05005288 do_each_pid_task(pid, PIDTYPE_PID, p) {
5289 set_tsk_trace_trace(p);
5290 } while_each_pid_task(pid, PIDTYPE_PID, p);
Oleg Nesterov229c4ef2009-02-03 20:39:04 +01005291 rcu_read_unlock();
Steven Rostedt978f3a42008-12-04 00:26:40 -05005292}
5293
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04005294static void clear_ftrace_pid_task(struct pid *pid)
Steven Rostedte32d8952008-12-04 00:26:41 -05005295{
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04005296 if (pid == ftrace_swapper_pid)
Steven Rostedte32d8952008-12-04 00:26:41 -05005297 clear_ftrace_swapper();
5298 else
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04005299 clear_ftrace_pid(pid);
Steven Rostedte32d8952008-12-04 00:26:41 -05005300}
5301
5302static void set_ftrace_pid_task(struct pid *pid)
5303{
5304 if (pid == ftrace_swapper_pid)
5305 set_ftrace_swapper();
5306 else
5307 set_ftrace_pid(pid);
5308}
5309
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04005310static int ftrace_pid_add(int p)
5311{
5312 struct pid *pid;
5313 struct ftrace_pid *fpid;
5314 int ret = -EINVAL;
5315
5316 mutex_lock(&ftrace_lock);
5317
5318 if (!p)
5319 pid = ftrace_swapper_pid;
5320 else
5321 pid = find_get_pid(p);
5322
5323 if (!pid)
5324 goto out;
5325
5326 ret = 0;
5327
5328 list_for_each_entry(fpid, &ftrace_pids, list)
5329 if (fpid->pid == pid)
5330 goto out_put;
5331
5332 ret = -ENOMEM;
5333
5334 fpid = kmalloc(sizeof(*fpid), GFP_KERNEL);
5335 if (!fpid)
5336 goto out_put;
5337
5338 list_add(&fpid->list, &ftrace_pids);
5339 fpid->pid = pid;
5340
5341 set_ftrace_pid_task(pid);
5342
5343 ftrace_update_pid_func();
Steven Rostedt (Red Hat)e1effa02014-08-05 17:19:38 -04005344
5345 ftrace_startup_all(0);
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04005346
5347 mutex_unlock(&ftrace_lock);
5348 return 0;
5349
5350out_put:
5351 if (pid != ftrace_swapper_pid)
5352 put_pid(pid);
5353
5354out:
5355 mutex_unlock(&ftrace_lock);
5356 return ret;
5357}
5358
5359static void ftrace_pid_reset(void)
5360{
5361 struct ftrace_pid *fpid, *safe;
5362
5363 mutex_lock(&ftrace_lock);
5364 list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) {
5365 struct pid *pid = fpid->pid;
5366
5367 clear_ftrace_pid_task(pid);
5368
5369 list_del(&fpid->list);
5370 kfree(fpid);
5371 }
5372
5373 ftrace_update_pid_func();
Steven Rostedt (Red Hat)e1effa02014-08-05 17:19:38 -04005374 ftrace_startup_all(0);
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04005375
5376 mutex_unlock(&ftrace_lock);
5377}
5378
5379static void *fpid_start(struct seq_file *m, loff_t *pos)
5380{
5381 mutex_lock(&ftrace_lock);
5382
Steven Rostedt (Red Hat)e3eea142015-07-24 10:38:12 -04005383 if (!ftrace_pids_enabled() && (!*pos))
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04005384 return (void *) 1;
5385
5386 return seq_list_start(&ftrace_pids, *pos);
5387}
5388
5389static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
5390{
5391 if (v == (void *)1)
5392 return NULL;
5393
5394 return seq_list_next(v, &ftrace_pids, pos);
5395}
5396
5397static void fpid_stop(struct seq_file *m, void *p)
5398{
5399 mutex_unlock(&ftrace_lock);
5400}
5401
5402static int fpid_show(struct seq_file *m, void *v)
5403{
5404 const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list);
5405
5406 if (v == (void *)1) {
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01005407 seq_puts(m, "no pid\n");
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04005408 return 0;
5409 }
5410
5411 if (fpid->pid == ftrace_swapper_pid)
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01005412 seq_puts(m, "swapper tasks\n");
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04005413 else
5414 seq_printf(m, "%u\n", pid_vnr(fpid->pid));
5415
5416 return 0;
5417}
5418
5419static const struct seq_operations ftrace_pid_sops = {
5420 .start = fpid_start,
5421 .next = fpid_next,
5422 .stop = fpid_stop,
5423 .show = fpid_show,
5424};
5425
5426static int
5427ftrace_pid_open(struct inode *inode, struct file *file)
5428{
5429 int ret = 0;
5430
5431 if ((file->f_mode & FMODE_WRITE) &&
5432 (file->f_flags & O_TRUNC))
5433 ftrace_pid_reset();
5434
5435 if (file->f_mode & FMODE_READ)
5436 ret = seq_open(file, &ftrace_pid_sops);
5437
5438 return ret;
5439}
5440
Steven Rostedtdf4fc312008-11-26 00:16:23 -05005441static ssize_t
5442ftrace_pid_write(struct file *filp, const char __user *ubuf,
5443 size_t cnt, loff_t *ppos)
5444{
Ingo Molnar457dc922009-11-23 11:03:28 +01005445 char buf[64], *tmp;
Steven Rostedtdf4fc312008-11-26 00:16:23 -05005446 long val;
5447 int ret;
5448
5449 if (cnt >= sizeof(buf))
5450 return -EINVAL;
5451
5452 if (copy_from_user(&buf, ubuf, cnt))
5453 return -EFAULT;
5454
5455 buf[cnt] = 0;
5456
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04005457 /*
5458 * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid"
5459 * to clean the filter quietly.
5460 */
Ingo Molnar457dc922009-11-23 11:03:28 +01005461 tmp = strstrip(buf);
5462 if (strlen(tmp) == 0)
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04005463 return 1;
5464
Daniel Walterbcd83ea2012-09-26 22:08:38 +02005465 ret = kstrtol(tmp, 10, &val);
Steven Rostedtdf4fc312008-11-26 00:16:23 -05005466 if (ret < 0)
5467 return ret;
5468
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04005469 ret = ftrace_pid_add(val);
Steven Rostedt978f3a42008-12-04 00:26:40 -05005470
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04005471 return ret ? ret : cnt;
5472}
Steven Rostedtdf4fc312008-11-26 00:16:23 -05005473
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04005474static int
5475ftrace_pid_release(struct inode *inode, struct file *file)
5476{
5477 if (file->f_mode & FMODE_READ)
5478 seq_release(inode, file);
Steven Rostedtdf4fc312008-11-26 00:16:23 -05005479
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04005480 return 0;
Steven Rostedtdf4fc312008-11-26 00:16:23 -05005481}
5482
Steven Rostedt5e2336a2009-03-05 21:44:55 -05005483static const struct file_operations ftrace_pid_fops = {
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04005484 .open = ftrace_pid_open,
5485 .write = ftrace_pid_write,
5486 .read = seq_read,
Steven Rostedt (Red Hat)098c8792013-12-21 17:39:40 -05005487 .llseek = tracing_lseek,
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04005488 .release = ftrace_pid_release,
Steven Rostedtdf4fc312008-11-26 00:16:23 -05005489};
5490
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05005491static __init int ftrace_init_tracefs(void)
Steven Rostedtdf4fc312008-11-26 00:16:23 -05005492{
5493 struct dentry *d_tracer;
Steven Rostedtdf4fc312008-11-26 00:16:23 -05005494
5495 d_tracer = tracing_init_dentry();
Steven Rostedt (Red Hat)14a5ae42015-01-20 11:14:16 -05005496 if (IS_ERR(d_tracer))
Steven Rostedtdf4fc312008-11-26 00:16:23 -05005497 return 0;
5498
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05005499 ftrace_init_dyn_tracefs(d_tracer);
Steven Rostedtdf4fc312008-11-26 00:16:23 -05005500
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005501 trace_create_file("set_ftrace_pid", 0644, d_tracer,
5502 NULL, &ftrace_pid_fops);
Steven Rostedt493762f2009-03-23 17:12:36 -04005503
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05005504 ftrace_profile_tracefs(d_tracer);
Steven Rostedt493762f2009-03-23 17:12:36 -04005505
Steven Rostedtdf4fc312008-11-26 00:16:23 -05005506 return 0;
5507}
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05005508fs_initcall(ftrace_init_tracefs);
Steven Rostedtdf4fc312008-11-26 00:16:23 -05005509
Steven Rostedt3d083392008-05-12 21:20:42 +02005510/**
Steven Rostedt81adbdc2008-10-23 09:33:02 -04005511 * ftrace_kill - kill ftrace
Steven Rostedta2bb6a32008-07-10 20:58:15 -04005512 *
5513 * This function should be used by panic code. It stops ftrace
5514 * but in a not so nice way. If you need to simply kill ftrace
5515 * from a non-atomic section, use ftrace_kill.
5516 */
Steven Rostedt81adbdc2008-10-23 09:33:02 -04005517void ftrace_kill(void)
Steven Rostedta2bb6a32008-07-10 20:58:15 -04005518{
5519 ftrace_disabled = 1;
5520 ftrace_enabled = 0;
Steven Rostedta2bb6a32008-07-10 20:58:15 -04005521 clear_ftrace_function();
5522}
5523
5524/**
Steven Rostedte0a413f2011-09-29 21:26:16 -04005525 * Test if ftrace is dead or not.
5526 */
5527int ftrace_is_dead(void)
5528{
5529 return ftrace_disabled;
5530}
5531
5532/**
Steven Rostedt3d083392008-05-12 21:20:42 +02005533 * register_ftrace_function - register a function for profiling
5534 * @ops - ops structure that holds the function for profiling.
5535 *
5536 * Register a function to be called by all functions in the
5537 * kernel.
5538 *
5539 * Note: @ops->func and all the functions it calls must be labeled
5540 * with "notrace", otherwise it will go into a
5541 * recursive loop.
5542 */
5543int register_ftrace_function(struct ftrace_ops *ops)
5544{
Steven Rostedt45a4a232011-04-21 23:16:46 -04005545 int ret = -1;
Steven Rostedt4eebcc82008-05-12 21:20:48 +02005546
Masami Hiramatsuf04f24fb2013-05-09 14:44:17 +09005547 ftrace_ops_init(ops);
5548
Steven Rostedte6ea44e2009-02-14 01:42:44 -05005549 mutex_lock(&ftrace_lock);
Frederic Weisbeckere7d37372008-11-16 06:02:06 +01005550
Steven Rostedt (Red Hat)8a56d772013-11-25 20:59:46 -05005551 ret = ftrace_startup(ops, 0);
Steven Rostedtb8489142011-05-04 09:27:52 -04005552
Steven Rostedte6ea44e2009-02-14 01:42:44 -05005553 mutex_unlock(&ftrace_lock);
Borislav Petkov8d240dd2012-03-29 19:11:40 +02005554
Steven Rostedtb0fc4942008-05-12 21:20:43 +02005555 return ret;
Steven Rostedt3d083392008-05-12 21:20:42 +02005556}
Steven Rostedtcdbe61b2011-05-05 21:14:55 -04005557EXPORT_SYMBOL_GPL(register_ftrace_function);
Steven Rostedt3d083392008-05-12 21:20:42 +02005558
5559/**
Uwe Kleine-Koenig32632922009-01-12 23:35:50 +01005560 * unregister_ftrace_function - unregister a function for profiling.
Steven Rostedt3d083392008-05-12 21:20:42 +02005561 * @ops - ops structure that holds the function to unregister
5562 *
5563 * Unregister a function that was added to be called by ftrace profiling.
5564 */
5565int unregister_ftrace_function(struct ftrace_ops *ops)
5566{
5567 int ret;
5568
Steven Rostedte6ea44e2009-02-14 01:42:44 -05005569 mutex_lock(&ftrace_lock);
Steven Rostedt (Red Hat)8a56d772013-11-25 20:59:46 -05005570 ret = ftrace_shutdown(ops, 0);
Steven Rostedte6ea44e2009-02-14 01:42:44 -05005571 mutex_unlock(&ftrace_lock);
Steven Rostedtb0fc4942008-05-12 21:20:43 +02005572
5573 return ret;
5574}
Steven Rostedtcdbe61b2011-05-05 21:14:55 -04005575EXPORT_SYMBOL_GPL(unregister_ftrace_function);
Steven Rostedtb0fc4942008-05-12 21:20:43 +02005576
Ingo Molnare309b412008-05-12 21:20:51 +02005577int
Steven Rostedtb0fc4942008-05-12 21:20:43 +02005578ftrace_enable_sysctl(struct ctl_table *table, int write,
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07005579 void __user *buffer, size_t *lenp,
Steven Rostedtb0fc4942008-05-12 21:20:43 +02005580 loff_t *ppos)
5581{
Steven Rostedt45a4a232011-04-21 23:16:46 -04005582 int ret = -ENODEV;
Steven Rostedt4eebcc82008-05-12 21:20:48 +02005583
Steven Rostedte6ea44e2009-02-14 01:42:44 -05005584 mutex_lock(&ftrace_lock);
Steven Rostedtb0fc4942008-05-12 21:20:43 +02005585
Steven Rostedt45a4a232011-04-21 23:16:46 -04005586 if (unlikely(ftrace_disabled))
5587 goto out;
5588
5589 ret = proc_dointvec(table, write, buffer, lenp, ppos);
Steven Rostedtb0fc4942008-05-12 21:20:43 +02005590
Li Zefana32c7762009-06-26 16:55:51 +08005591 if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
Steven Rostedtb0fc4942008-05-12 21:20:43 +02005592 goto out;
5593
Li Zefana32c7762009-06-26 16:55:51 +08005594 last_ftrace_enabled = !!ftrace_enabled;
Steven Rostedtb0fc4942008-05-12 21:20:43 +02005595
5596 if (ftrace_enabled) {
5597
Steven Rostedtb0fc4942008-05-12 21:20:43 +02005598 /* we are starting ftrace again */
Jan Kiszka5000c412013-03-26 17:53:03 +01005599 if (ftrace_ops_list != &ftrace_list_end)
5600 update_ftrace_function();
Steven Rostedtb0fc4942008-05-12 21:20:43 +02005601
Steven Rostedt (Red Hat)524a3862015-03-06 19:55:13 -05005602 ftrace_startup_sysctl();
5603
Steven Rostedtb0fc4942008-05-12 21:20:43 +02005604 } else {
5605 /* stopping ftrace calls (just send to ftrace_stub) */
5606 ftrace_trace_function = ftrace_stub;
5607
5608 ftrace_shutdown_sysctl();
5609 }
5610
5611 out:
Steven Rostedte6ea44e2009-02-14 01:42:44 -05005612 mutex_unlock(&ftrace_lock);
Steven Rostedt3d083392008-05-12 21:20:42 +02005613 return ret;
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +02005614}
Ingo Molnarf17845e2008-10-24 12:47:10 +02005615
Frederic Weisbeckerfb526072008-11-25 21:07:04 +01005616#ifdef CONFIG_FUNCTION_GRAPH_TRACER
Frederic Weisbeckere7d37372008-11-16 06:02:06 +01005617
Steven Rostedt (Red Hat)5f151b22014-08-15 17:18:46 -04005618static struct ftrace_ops graph_ops = {
5619 .func = ftrace_stub,
5620 .flags = FTRACE_OPS_FL_RECURSION_SAFE |
5621 FTRACE_OPS_FL_INITIALIZED |
Steven Rostedt (Red Hat)e3eea142015-07-24 10:38:12 -04005622 FTRACE_OPS_FL_PID |
Steven Rostedt (Red Hat)5f151b22014-08-15 17:18:46 -04005623 FTRACE_OPS_FL_STUB,
5624#ifdef FTRACE_GRAPH_TRAMP_ADDR
5625 .trampoline = FTRACE_GRAPH_TRAMP_ADDR,
Steven Rostedt (Red Hat)aec0be22014-11-18 21:14:11 -05005626 /* trampoline_size is only needed for dynamically allocated tramps */
Steven Rostedt (Red Hat)5f151b22014-08-15 17:18:46 -04005627#endif
5628 ASSIGN_OPS_HASH(graph_ops, &global_ops.local_hash)
5629};
5630
Steven Rostedt (Red Hat)55577202015-09-29 19:06:50 -04005631void ftrace_graph_sleep_time_control(bool enable)
5632{
5633 fgraph_sleep_time = enable;
5634}
5635
5636void ftrace_graph_graph_time_control(bool enable)
5637{
5638 fgraph_graph_time = enable;
5639}
5640
Steven Rostedte49dc192008-12-02 23:50:05 -05005641int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
5642{
5643 return 0;
5644}
5645
Frederic Weisbecker287b6e62008-11-26 00:57:25 +01005646/* The callbacks that hook a function */
5647trace_func_graph_ret_t ftrace_graph_return =
5648 (trace_func_graph_ret_t)ftrace_stub;
Steven Rostedte49dc192008-12-02 23:50:05 -05005649trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
Steven Rostedt (Red Hat)23a8e842014-01-13 10:30:23 -05005650static trace_func_graph_ent_t __ftrace_graph_entry = ftrace_graph_entry_stub;
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01005651
5652/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
5653static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
5654{
5655 int i;
5656 int ret = 0;
5657 unsigned long flags;
5658 int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
5659 struct task_struct *g, *t;
5660
5661 for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
5662 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
5663 * sizeof(struct ftrace_ret_stack),
5664 GFP_KERNEL);
5665 if (!ret_stack_list[i]) {
5666 start = 0;
5667 end = i;
5668 ret = -ENOMEM;
5669 goto free;
5670 }
5671 }
5672
5673 read_lock_irqsave(&tasklist_lock, flags);
5674 do_each_thread(g, t) {
5675 if (start == end) {
5676 ret = -EAGAIN;
5677 goto unlock;
5678 }
5679
5680 if (t->ret_stack == NULL) {
Frederic Weisbecker380c4b12008-12-06 03:43:41 +01005681 atomic_set(&t->tracing_graph_pause, 0);
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01005682 atomic_set(&t->trace_overrun, 0);
Steven Rostedt26c01622009-06-02 14:01:19 -04005683 t->curr_ret_stack = -1;
5684 /* Make sure the tasks see the -1 first: */
5685 smp_wmb();
5686 t->ret_stack = ret_stack_list[start++];
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01005687 }
5688 } while_each_thread(g, t);
5689
5690unlock:
5691 read_unlock_irqrestore(&tasklist_lock, flags);
5692free:
5693 for (i = start; i < end; i++)
5694 kfree(ret_stack_list[i]);
5695 return ret;
5696}
5697
Steven Rostedt8aef2d22009-03-24 01:10:15 -04005698static void
Steven Rostedt38516ab2010-04-20 17:04:50 -04005699ftrace_graph_probe_sched_switch(void *ignore,
5700 struct task_struct *prev, struct task_struct *next)
Steven Rostedt8aef2d22009-03-24 01:10:15 -04005701{
5702 unsigned long long timestamp;
5703 int index;
5704
Steven Rostedtbe6f1642009-03-24 11:06:24 -04005705 /*
5706 * Does the user want to count the time a function was asleep.
5707 * If so, do not update the time stamps.
5708 */
Steven Rostedt (Red Hat)55577202015-09-29 19:06:50 -04005709 if (fgraph_sleep_time)
Steven Rostedtbe6f1642009-03-24 11:06:24 -04005710 return;
5711
Steven Rostedt8aef2d22009-03-24 01:10:15 -04005712 timestamp = trace_clock_local();
5713
5714 prev->ftrace_timestamp = timestamp;
5715
5716 /* only process tasks that we timestamped */
5717 if (!next->ftrace_timestamp)
5718 return;
5719
5720 /*
5721 * Update all the counters in next to make up for the
5722 * time next was sleeping.
5723 */
5724 timestamp -= next->ftrace_timestamp;
5725
5726 for (index = next->curr_ret_stack; index >= 0; index--)
5727 next->ret_stack[index].calltime += timestamp;
5728}
5729
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01005730/* Allocate a return stack for each task */
Frederic Weisbeckerfb526072008-11-25 21:07:04 +01005731static int start_graph_tracing(void)
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01005732{
5733 struct ftrace_ret_stack **ret_stack_list;
Frederic Weisbecker5b058bc2009-02-17 18:35:34 +01005734 int ret, cpu;
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01005735
5736 ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
5737 sizeof(struct ftrace_ret_stack *),
5738 GFP_KERNEL);
5739
5740 if (!ret_stack_list)
5741 return -ENOMEM;
5742
Frederic Weisbecker5b058bc2009-02-17 18:35:34 +01005743 /* The cpu_boot init_task->ret_stack will never be freed */
Steven Rostedt179c4982009-06-02 12:03:19 -04005744 for_each_online_cpu(cpu) {
5745 if (!idle_task(cpu)->ret_stack)
Steven Rostedt868baf02011-02-10 21:26:13 -05005746 ftrace_graph_init_idle_task(idle_task(cpu), cpu);
Steven Rostedt179c4982009-06-02 12:03:19 -04005747 }
Frederic Weisbecker5b058bc2009-02-17 18:35:34 +01005748
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01005749 do {
5750 ret = alloc_retstack_tasklist(ret_stack_list);
5751 } while (ret == -EAGAIN);
5752
Steven Rostedt8aef2d22009-03-24 01:10:15 -04005753 if (!ret) {
Steven Rostedt38516ab2010-04-20 17:04:50 -04005754 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
Steven Rostedt8aef2d22009-03-24 01:10:15 -04005755 if (ret)
5756 pr_info("ftrace_graph: Couldn't activate tracepoint"
5757 " probe to kernel_sched_switch\n");
5758 }
5759
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01005760 kfree(ret_stack_list);
5761 return ret;
5762}
5763
Frederic Weisbecker4a2b8dd2009-01-14 13:33:27 -08005764/*
5765 * Hibernation protection.
5766 * The state of the current task is too much unstable during
5767 * suspend/restore to disk. We want to protect against that.
5768 */
5769static int
5770ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
5771 void *unused)
5772{
5773 switch (state) {
5774 case PM_HIBERNATION_PREPARE:
5775 pause_graph_tracing();
5776 break;
5777
5778 case PM_POST_HIBERNATION:
5779 unpause_graph_tracing();
5780 break;
5781 }
5782 return NOTIFY_DONE;
5783}
5784
Steven Rostedt (Red Hat)23a8e842014-01-13 10:30:23 -05005785static int ftrace_graph_entry_test(struct ftrace_graph_ent *trace)
5786{
5787 if (!ftrace_ops_test(&global_ops, trace->func, NULL))
5788 return 0;
5789 return __ftrace_graph_entry(trace);
5790}
5791
5792/*
5793 * The function graph tracer should only trace the functions defined
5794 * by set_ftrace_filter and set_ftrace_notrace. If another function
5795 * tracer ops is registered, the graph tracer requires testing the
5796 * function against the global ops, and not just trace any function
5797 * that any ftrace_ops registered.
5798 */
5799static void update_function_graph_func(void)
5800{
Steven Rostedt (Red Hat)5f151b22014-08-15 17:18:46 -04005801 struct ftrace_ops *op;
5802 bool do_test = false;
5803
5804 /*
5805 * The graph and global ops share the same set of functions
5806 * to test. If any other ops is on the list, then
5807 * the graph tracing needs to test if its the function
5808 * it should call.
5809 */
5810 do_for_each_ftrace_op(op, ftrace_ops_list) {
5811 if (op != &global_ops && op != &graph_ops &&
5812 op != &ftrace_list_end) {
5813 do_test = true;
5814 /* in double loop, break out with goto */
5815 goto out;
5816 }
5817 } while_for_each_ftrace_op(op);
5818 out:
5819 if (do_test)
Steven Rostedt (Red Hat)23a8e842014-01-13 10:30:23 -05005820 ftrace_graph_entry = ftrace_graph_entry_test;
Steven Rostedt (Red Hat)5f151b22014-08-15 17:18:46 -04005821 else
5822 ftrace_graph_entry = __ftrace_graph_entry;
Steven Rostedt (Red Hat)23a8e842014-01-13 10:30:23 -05005823}
5824
Mathias Krause8275f692014-03-30 15:31:50 +02005825static struct notifier_block ftrace_suspend_notifier = {
5826 .notifier_call = ftrace_suspend_notifier_call,
5827};
5828
Frederic Weisbecker287b6e62008-11-26 00:57:25 +01005829int register_ftrace_graph(trace_func_graph_ret_t retfunc,
5830 trace_func_graph_ent_t entryfunc)
Frederic Weisbecker15e6cb32008-11-11 07:14:25 +01005831{
Frederic Weisbeckere7d37372008-11-16 06:02:06 +01005832 int ret = 0;
5833
Steven Rostedte6ea44e2009-02-14 01:42:44 -05005834 mutex_lock(&ftrace_lock);
Frederic Weisbeckere7d37372008-11-16 06:02:06 +01005835
Steven Rostedt05ce5812009-03-24 00:18:31 -04005836 /* we currently allow only one tracer registered at a time */
Steven Rostedt597af812009-04-03 15:24:12 -04005837 if (ftrace_graph_active) {
Steven Rostedt05ce5812009-03-24 00:18:31 -04005838 ret = -EBUSY;
5839 goto out;
5840 }
5841
Frederic Weisbecker4a2b8dd2009-01-14 13:33:27 -08005842 register_pm_notifier(&ftrace_suspend_notifier);
5843
Steven Rostedt597af812009-04-03 15:24:12 -04005844 ftrace_graph_active++;
Frederic Weisbeckerfb526072008-11-25 21:07:04 +01005845 ret = start_graph_tracing();
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01005846 if (ret) {
Steven Rostedt597af812009-04-03 15:24:12 -04005847 ftrace_graph_active--;
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01005848 goto out;
5849 }
Steven Rostedte53a6312008-11-26 00:16:25 -05005850
Frederic Weisbecker287b6e62008-11-26 00:57:25 +01005851 ftrace_graph_return = retfunc;
Steven Rostedt (Red Hat)23a8e842014-01-13 10:30:23 -05005852
5853 /*
5854 * Update the indirect function to the entryfunc, and the
5855 * function that gets called to the entry_test first. Then
5856 * call the update fgraph entry function to determine if
5857 * the entryfunc should be called directly or not.
5858 */
5859 __ftrace_graph_entry = entryfunc;
5860 ftrace_graph_entry = ftrace_graph_entry_test;
5861 update_function_graph_func();
Steven Rostedte53a6312008-11-26 00:16:25 -05005862
Steven Rostedt (Red Hat)5f151b22014-08-15 17:18:46 -04005863 ret = ftrace_startup(&graph_ops, FTRACE_START_FUNC_RET);
Frederic Weisbeckere7d37372008-11-16 06:02:06 +01005864out:
Steven Rostedte6ea44e2009-02-14 01:42:44 -05005865 mutex_unlock(&ftrace_lock);
Frederic Weisbeckere7d37372008-11-16 06:02:06 +01005866 return ret;
Frederic Weisbecker15e6cb32008-11-11 07:14:25 +01005867}
5868
Frederic Weisbeckerfb526072008-11-25 21:07:04 +01005869void unregister_ftrace_graph(void)
Frederic Weisbecker15e6cb32008-11-11 07:14:25 +01005870{
Steven Rostedte6ea44e2009-02-14 01:42:44 -05005871 mutex_lock(&ftrace_lock);
Frederic Weisbeckere7d37372008-11-16 06:02:06 +01005872
Steven Rostedt597af812009-04-03 15:24:12 -04005873 if (unlikely(!ftrace_graph_active))
Steven Rostedt2aad1b72009-03-30 11:11:28 -04005874 goto out;
5875
Steven Rostedt597af812009-04-03 15:24:12 -04005876 ftrace_graph_active--;
Frederic Weisbecker287b6e62008-11-26 00:57:25 +01005877 ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
Steven Rostedte49dc192008-12-02 23:50:05 -05005878 ftrace_graph_entry = ftrace_graph_entry_stub;
Steven Rostedt (Red Hat)23a8e842014-01-13 10:30:23 -05005879 __ftrace_graph_entry = ftrace_graph_entry_stub;
Steven Rostedt (Red Hat)5f151b22014-08-15 17:18:46 -04005880 ftrace_shutdown(&graph_ops, FTRACE_STOP_FUNC_RET);
Frederic Weisbecker4a2b8dd2009-01-14 13:33:27 -08005881 unregister_pm_notifier(&ftrace_suspend_notifier);
Steven Rostedt38516ab2010-04-20 17:04:50 -04005882 unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
Frederic Weisbeckere7d37372008-11-16 06:02:06 +01005883
Steven Rostedt (Red Hat)f3bea492014-07-02 23:23:31 -04005884#ifdef CONFIG_DYNAMIC_FTRACE
5885 /*
5886 * Function graph does not allocate the trampoline, but
5887 * other global_ops do. We need to reset the ALLOC_TRAMP flag
5888 * if one was used.
5889 */
5890 global_ops.trampoline = save_global_trampoline;
5891 if (save_global_flags & FTRACE_OPS_FL_ALLOC_TRAMP)
5892 global_ops.flags |= FTRACE_OPS_FL_ALLOC_TRAMP;
5893#endif
5894
Steven Rostedt2aad1b72009-03-30 11:11:28 -04005895 out:
Steven Rostedte6ea44e2009-02-14 01:42:44 -05005896 mutex_unlock(&ftrace_lock);
Frederic Weisbecker15e6cb32008-11-11 07:14:25 +01005897}
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01005898
Steven Rostedt868baf02011-02-10 21:26:13 -05005899static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);
5900
5901static void
5902graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
5903{
5904 atomic_set(&t->tracing_graph_pause, 0);
5905 atomic_set(&t->trace_overrun, 0);
5906 t->ftrace_timestamp = 0;
Lucas De Marchi25985ed2011-03-30 22:57:33 -03005907 /* make curr_ret_stack visible before we add the ret_stack */
Steven Rostedt868baf02011-02-10 21:26:13 -05005908 smp_wmb();
5909 t->ret_stack = ret_stack;
5910}
5911
5912/*
5913 * Allocate a return stack for the idle task. May be the first
5914 * time through, or it may be done by CPU hotplug online.
5915 */
5916void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
5917{
5918 t->curr_ret_stack = -1;
5919 /*
5920 * The idle task has no parent, it either has its own
5921 * stack or no stack at all.
5922 */
5923 if (t->ret_stack)
5924 WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));
5925
5926 if (ftrace_graph_active) {
5927 struct ftrace_ret_stack *ret_stack;
5928
5929 ret_stack = per_cpu(idle_ret_stack, cpu);
5930 if (!ret_stack) {
5931 ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
5932 * sizeof(struct ftrace_ret_stack),
5933 GFP_KERNEL);
5934 if (!ret_stack)
5935 return;
5936 per_cpu(idle_ret_stack, cpu) = ret_stack;
5937 }
5938 graph_init_task(t, ret_stack);
5939 }
5940}
5941
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01005942/* Allocate a return stack for newly created task */
Frederic Weisbeckerfb526072008-11-25 21:07:04 +01005943void ftrace_graph_init_task(struct task_struct *t)
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01005944{
Steven Rostedt84047e32009-06-02 16:51:55 -04005945 /* Make sure we do not use the parent ret_stack */
5946 t->ret_stack = NULL;
Steven Rostedtea14eb72010-03-12 19:41:23 -05005947 t->curr_ret_stack = -1;
Steven Rostedt84047e32009-06-02 16:51:55 -04005948
Steven Rostedt597af812009-04-03 15:24:12 -04005949 if (ftrace_graph_active) {
Steven Rostedt82310a32009-06-02 12:26:07 -04005950 struct ftrace_ret_stack *ret_stack;
5951
5952 ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01005953 * sizeof(struct ftrace_ret_stack),
5954 GFP_KERNEL);
Steven Rostedt82310a32009-06-02 12:26:07 -04005955 if (!ret_stack)
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01005956 return;
Steven Rostedt868baf02011-02-10 21:26:13 -05005957 graph_init_task(t, ret_stack);
Steven Rostedt84047e32009-06-02 16:51:55 -04005958 }
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01005959}
5960
Frederic Weisbeckerfb526072008-11-25 21:07:04 +01005961void ftrace_graph_exit_task(struct task_struct *t)
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01005962{
Frederic Weisbeckereae849c2008-11-23 17:33:12 +01005963 struct ftrace_ret_stack *ret_stack = t->ret_stack;
5964
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01005965 t->ret_stack = NULL;
Frederic Weisbeckereae849c2008-11-23 17:33:12 +01005966 /* NULL must become visible to IRQs before we free it: */
5967 barrier();
5968
5969 kfree(ret_stack);
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01005970}
Frederic Weisbecker15e6cb32008-11-11 07:14:25 +01005971#endif