blob: 7d79a10c3cde9962e63b498861babef6b57ff020 [file] [log] [blame]
Peter Zijlstraac199db2009-03-19 20:26:15 +01001/*
Frederic Weisbecker97d5a222010-03-05 05:35:37 +01002 * trace event based perf event profiling/tracing
Peter Zijlstraac199db2009-03-19 20:26:15 +01003 *
4 * Copyright (C) 2009 Red Hat Inc, Peter Zijlstra <pzijlstr@redhat.com>
Frederic Weisbeckerc5306652010-03-03 07:16:16 +01005 * Copyright (C) 2009-2010 Frederic Weisbecker <fweisbec@gmail.com>
Peter Zijlstraac199db2009-03-19 20:26:15 +01006 */
7
Li Zefan558e6542009-08-24 12:19:47 +08008#include <linux/module.h>
Xiao Guangrong430ad5a2010-01-28 09:32:29 +08009#include <linux/kprobes.h>
Peter Zijlstraac199db2009-03-19 20:26:15 +010010#include "trace.h"
11
Frederic Weisbeckerc5306652010-03-03 07:16:16 +010012DEFINE_PER_CPU(struct pt_regs, perf_trace_regs);
Xiao Guangrong639fe4b2010-03-11 15:30:35 +080013EXPORT_PER_CPU_SYMBOL_GPL(perf_trace_regs);
Frederic Weisbecker20ab44252009-09-18 06:10:28 +020014
Frederic Weisbeckerdcd5c162010-03-16 01:05:02 +010015EXPORT_SYMBOL_GPL(perf_arch_fetch_caller_regs);
16
Xiao Guangrong430ad5a2010-01-28 09:32:29 +080017static char *perf_trace_buf;
18static char *perf_trace_buf_nmi;
Frederic Weisbecker20ab44252009-09-18 06:10:28 +020019
Frederic Weisbecker97d5a222010-03-05 05:35:37 +010020typedef typeof(char [PERF_MAX_TRACE_SIZE]) perf_trace_t ;
Frederic Weisbeckerce71b9d2009-11-22 05:26:55 +010021
Frederic Weisbecker20ab44252009-09-18 06:10:28 +020022/* Count the events in use (per event id, not per instance) */
Frederic Weisbecker97d5a222010-03-05 05:35:37 +010023static int total_ref_count;
Frederic Weisbecker20ab44252009-09-18 06:10:28 +020024
Frederic Weisbecker97d5a222010-03-05 05:35:37 +010025static int perf_trace_event_enable(struct ftrace_event_call *event)
Frederic Weisbeckere5e25cf2009-09-18 00:54:43 +020026{
Frederic Weisbeckerce71b9d2009-11-22 05:26:55 +010027 char *buf;
Frederic Weisbecker20ab44252009-09-18 06:10:28 +020028 int ret = -ENOMEM;
29
Frederic Weisbecker97d5a222010-03-05 05:35:37 +010030 if (event->perf_refcount++ > 0)
Frederic Weisbeckere5e25cf2009-09-18 00:54:43 +020031 return 0;
32
Frederic Weisbecker97d5a222010-03-05 05:35:37 +010033 if (!total_ref_count) {
Frederic Weisbeckerce71b9d2009-11-22 05:26:55 +010034 buf = (char *)alloc_percpu(perf_trace_t);
Frederic Weisbecker20ab44252009-09-18 06:10:28 +020035 if (!buf)
36 goto fail_buf;
37
Frederic Weisbecker444a2a32009-11-06 04:13:05 +010038 rcu_assign_pointer(perf_trace_buf, buf);
Frederic Weisbecker20ab44252009-09-18 06:10:28 +020039
Frederic Weisbeckerce71b9d2009-11-22 05:26:55 +010040 buf = (char *)alloc_percpu(perf_trace_t);
Frederic Weisbecker20ab44252009-09-18 06:10:28 +020041 if (!buf)
42 goto fail_buf_nmi;
43
Frederic Weisbecker444a2a32009-11-06 04:13:05 +010044 rcu_assign_pointer(perf_trace_buf_nmi, buf);
Frederic Weisbecker20ab44252009-09-18 06:10:28 +020045 }
46
Frederic Weisbecker97d5a222010-03-05 05:35:37 +010047 ret = event->perf_event_enable(event);
Frederic Weisbeckerfe8e5b52009-10-03 14:55:18 +020048 if (!ret) {
Frederic Weisbecker97d5a222010-03-05 05:35:37 +010049 total_ref_count++;
Frederic Weisbecker20ab44252009-09-18 06:10:28 +020050 return 0;
Frederic Weisbeckerfe8e5b52009-10-03 14:55:18 +020051 }
Frederic Weisbecker20ab44252009-09-18 06:10:28 +020052
Frederic Weisbecker20ab44252009-09-18 06:10:28 +020053fail_buf_nmi:
Frederic Weisbecker97d5a222010-03-05 05:35:37 +010054 if (!total_ref_count) {
Frederic Weisbecker444a2a32009-11-06 04:13:05 +010055 free_percpu(perf_trace_buf_nmi);
56 free_percpu(perf_trace_buf);
57 perf_trace_buf_nmi = NULL;
58 perf_trace_buf = NULL;
Frederic Weisbeckerfe8e5b52009-10-03 14:55:18 +020059 }
Frederic Weisbecker20ab44252009-09-18 06:10:28 +020060fail_buf:
Frederic Weisbecker97d5a222010-03-05 05:35:37 +010061 event->perf_refcount--;
Frederic Weisbecker20ab44252009-09-18 06:10:28 +020062
63 return ret;
Frederic Weisbeckere5e25cf2009-09-18 00:54:43 +020064}
65
Frederic Weisbecker97d5a222010-03-05 05:35:37 +010066int perf_trace_enable(int event_id)
Peter Zijlstraac199db2009-03-19 20:26:15 +010067{
68 struct ftrace_event_call *event;
Li Zefan20c89282009-05-06 10:33:45 +080069 int ret = -EINVAL;
Peter Zijlstraac199db2009-03-19 20:26:15 +010070
Li Zefan20c89282009-05-06 10:33:45 +080071 mutex_lock(&event_mutex);
Steven Rostedta59fd602009-04-10 13:52:20 -040072 list_for_each_entry(event, &ftrace_events, list) {
Frederic Weisbecker97d5a222010-03-05 05:35:37 +010073 if (event->id == event_id && event->perf_event_enable &&
Li Zefan558e6542009-08-24 12:19:47 +080074 try_module_get(event->mod)) {
Frederic Weisbecker97d5a222010-03-05 05:35:37 +010075 ret = perf_trace_event_enable(event);
Li Zefan20c89282009-05-06 10:33:45 +080076 break;
77 }
Peter Zijlstraac199db2009-03-19 20:26:15 +010078 }
Li Zefan20c89282009-05-06 10:33:45 +080079 mutex_unlock(&event_mutex);
Peter Zijlstraac199db2009-03-19 20:26:15 +010080
Li Zefan20c89282009-05-06 10:33:45 +080081 return ret;
Peter Zijlstraac199db2009-03-19 20:26:15 +010082}
83
Frederic Weisbecker97d5a222010-03-05 05:35:37 +010084static void perf_trace_event_disable(struct ftrace_event_call *event)
Frederic Weisbeckere5e25cf2009-09-18 00:54:43 +020085{
Frederic Weisbeckerce71b9d2009-11-22 05:26:55 +010086 char *buf, *nmi_buf;
Frederic Weisbecker20ab44252009-09-18 06:10:28 +020087
Frederic Weisbecker97d5a222010-03-05 05:35:37 +010088 if (--event->perf_refcount > 0)
Frederic Weisbeckere5e25cf2009-09-18 00:54:43 +020089 return;
90
Frederic Weisbecker97d5a222010-03-05 05:35:37 +010091 event->perf_event_disable(event);
Frederic Weisbecker20ab44252009-09-18 06:10:28 +020092
Frederic Weisbecker97d5a222010-03-05 05:35:37 +010093 if (!--total_ref_count) {
Frederic Weisbecker444a2a32009-11-06 04:13:05 +010094 buf = perf_trace_buf;
95 rcu_assign_pointer(perf_trace_buf, NULL);
Frederic Weisbecker20ab44252009-09-18 06:10:28 +020096
Frederic Weisbecker444a2a32009-11-06 04:13:05 +010097 nmi_buf = perf_trace_buf_nmi;
98 rcu_assign_pointer(perf_trace_buf_nmi, NULL);
Frederic Weisbecker20ab44252009-09-18 06:10:28 +020099
100 /*
101 * Ensure every events in profiling have finished before
102 * releasing the buffers
103 */
104 synchronize_sched();
105
106 free_percpu(buf);
107 free_percpu(nmi_buf);
108 }
Frederic Weisbeckere5e25cf2009-09-18 00:54:43 +0200109}
110
Frederic Weisbecker97d5a222010-03-05 05:35:37 +0100111void perf_trace_disable(int event_id)
Peter Zijlstraac199db2009-03-19 20:26:15 +0100112{
113 struct ftrace_event_call *event;
114
Li Zefan20c89282009-05-06 10:33:45 +0800115 mutex_lock(&event_mutex);
Steven Rostedta59fd602009-04-10 13:52:20 -0400116 list_for_each_entry(event, &ftrace_events, list) {
Li Zefan20c89282009-05-06 10:33:45 +0800117 if (event->id == event_id) {
Frederic Weisbecker97d5a222010-03-05 05:35:37 +0100118 perf_trace_event_disable(event);
Li Zefan558e6542009-08-24 12:19:47 +0800119 module_put(event->mod);
Li Zefan20c89282009-05-06 10:33:45 +0800120 break;
121 }
Peter Zijlstraac199db2009-03-19 20:26:15 +0100122 }
Li Zefan20c89282009-05-06 10:33:45 +0800123 mutex_unlock(&event_mutex);
Peter Zijlstraac199db2009-03-19 20:26:15 +0100124}
Xiao Guangrong430ad5a2010-01-28 09:32:29 +0800125
Frederic Weisbecker97d5a222010-03-05 05:35:37 +0100126__kprobes void *perf_trace_buf_prepare(int size, unsigned short type,
127 int *rctxp, unsigned long *irq_flags)
Xiao Guangrong430ad5a2010-01-28 09:32:29 +0800128{
129 struct trace_entry *entry;
130 char *trace_buf, *raw_data;
131 int pc, cpu;
132
133 pc = preempt_count();
134
135 /* Protect the per cpu buffer, begin the rcu read side */
136 local_irq_save(*irq_flags);
137
138 *rctxp = perf_swevent_get_recursion_context();
139 if (*rctxp < 0)
140 goto err_recursion;
141
142 cpu = smp_processor_id();
143
144 if (in_nmi())
145 trace_buf = rcu_dereference(perf_trace_buf_nmi);
146 else
147 trace_buf = rcu_dereference(perf_trace_buf);
148
149 if (!trace_buf)
150 goto err;
151
152 raw_data = per_cpu_ptr(trace_buf, cpu);
153
154 /* zero the dead bytes from align to not leak stack to user */
155 *(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL;
156
157 entry = (struct trace_entry *)raw_data;
158 tracing_generic_entry_update(entry, *irq_flags, pc);
159 entry->type = type;
160
161 return raw_data;
162err:
163 perf_swevent_put_recursion_context(*rctxp);
164err_recursion:
165 local_irq_restore(*irq_flags);
166 return NULL;
167}
Frederic Weisbecker97d5a222010-03-05 05:35:37 +0100168EXPORT_SYMBOL_GPL(perf_trace_buf_prepare);