Steven Rostedt (VMware) | bcea3f9 | 2018-08-16 11:23:53 -0400 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0 |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 2 | /* |
Srivatsa S. Bhat (VMware) | 0c3c86b | 2019-10-10 11:51:17 -0700 | [diff] [blame] | 3 | * trace_hwlat.c - A simple Hardware Latency detector. |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 4 | * |
| 5 | * Use this tracer to detect large system latencies induced by the behavior of |
| 6 | * certain underlying system hardware or firmware, independent of Linux itself. |
| 7 | * The code was developed originally to detect the presence of SMIs on Intel |
| 8 | * and AMD systems, although there is no dependency upon x86 herein. |
| 9 | * |
| 10 | * The classical example usage of this tracer is in detecting the presence of |
| 11 | * SMIs or System Management Interrupts on Intel and AMD systems. An SMI is a |
| 12 | * somewhat special form of hardware interrupt spawned from earlier CPU debug |
| 13 | * modes in which the (BIOS/EFI/etc.) firmware arranges for the South Bridge |
| 14 | * LPC (or other device) to generate a special interrupt under certain |
| 15 | * circumstances, for example, upon expiration of a special SMI timer device, |
| 16 | * due to certain external thermal readings, on certain I/O address accesses, |
| 17 | * and other situations. An SMI hits a special CPU pin, triggers a special |
| 18 | * SMI mode (complete with special memory map), and the OS is unaware. |
| 19 | * |
| 20 | * Although certain hardware-inducing latencies are necessary (for example, |
| 21 | * a modern system often requires an SMI handler for correct thermal control |
| 22 | * and remote management) they can wreak havoc upon any OS-level performance |
| 23 | * guarantees toward low-latency, especially when the OS is not even made |
| 24 | * aware of the presence of these interrupts. For this reason, we need a |
| 25 | * somewhat brute force mechanism to detect these interrupts. In this case, |
| 26 | * we do it by hogging all of the CPU(s) for configurable timer intervals, |
| 27 | * sampling the built-in CPU timer, looking for discontiguous readings. |
| 28 | * |
| 29 | * WARNING: This implementation necessarily introduces latencies. Therefore, |
| 30 | * you should NEVER use this tracer while running in a production |
| 31 | * environment requiring any kind of low-latency performance |
| 32 | * guarantee(s). |
| 33 | * |
| 34 | * Copyright (C) 2008-2009 Jon Masters, Red Hat, Inc. <jcm@redhat.com> |
| 35 | * Copyright (C) 2013-2016 Steven Rostedt, Red Hat, Inc. <srostedt@redhat.com> |
| 36 | * |
Daniel Bristot de Oliveira | bb1b24c | 2021-06-22 16:42:19 +0200 | [diff] [blame] | 37 | * Includes useful feedback from Clark Williams <williams@redhat.com> |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 38 | * |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 39 | */ |
| 40 | #include <linux/kthread.h> |
| 41 | #include <linux/tracefs.h> |
| 42 | #include <linux/uaccess.h> |
Steven Rostedt (Red Hat) | 0330f7a | 2016-07-15 15:48:56 -0400 | [diff] [blame] | 43 | #include <linux/cpumask.h> |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 44 | #include <linux/delay.h> |
Ingo Molnar | e601757 | 2017-02-01 16:36:40 +0100 | [diff] [blame] | 45 | #include <linux/sched/clock.h> |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 46 | #include "trace.h" |
| 47 | |
| 48 | static struct trace_array *hwlat_trace; |
| 49 | |
| 50 | #define U64STR_SIZE 22 /* 20 digits max */ |
| 51 | |
| 52 | #define BANNER "hwlat_detector: " |
| 53 | #define DEFAULT_SAMPLE_WINDOW 1000000 /* 1s */ |
| 54 | #define DEFAULT_SAMPLE_WIDTH 500000 /* 0.5s */ |
| 55 | #define DEFAULT_LAT_THRESHOLD 10 /* 10us */ |
| 56 | |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 57 | static struct dentry *hwlat_sample_width; /* sample width us */ |
| 58 | static struct dentry *hwlat_sample_window; /* sample window us */ |
Daniel Bristot de Oliveira | 8fa826b | 2021-06-22 16:42:20 +0200 | [diff] [blame] | 59 | static struct dentry *hwlat_thread_mode; /* hwlat thread mode */ |
| 60 | |
| 61 | enum { |
| 62 | MODE_NONE = 0, |
| 63 | MODE_ROUND_ROBIN, |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 64 | MODE_PER_CPU, |
Daniel Bristot de Oliveira | 8fa826b | 2021-06-22 16:42:20 +0200 | [diff] [blame] | 65 | MODE_MAX |
| 66 | }; |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 67 | static char *thread_mode_str[] = { "none", "round-robin", "per-cpu" }; |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 68 | |
| 69 | /* Save the previous tracing_thresh value */ |
| 70 | static unsigned long save_tracing_thresh; |
| 71 | |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 72 | /* runtime kthread data */ |
| 73 | struct hwlat_kthread_data { |
| 74 | struct task_struct *kthread; |
| 75 | /* NMI timestamp counters */ |
| 76 | u64 nmi_ts_start; |
| 77 | u64 nmi_total_ts; |
| 78 | int nmi_count; |
| 79 | int nmi_cpu; |
| 80 | }; |
| 81 | |
Wang ShaoBo | 1d62889 | 2021-10-21 11:52:25 +0800 | [diff] [blame] | 82 | static struct hwlat_kthread_data hwlat_single_cpu_data; |
| 83 | static DEFINE_PER_CPU(struct hwlat_kthread_data, hwlat_per_cpu_data); |
Steven Rostedt (Red Hat) | 7b2c862 | 2016-08-04 12:49:53 -0400 | [diff] [blame] | 84 | |
| 85 | /* Tells NMIs to call back to the hwlat tracer to record timestamps */ |
| 86 | bool trace_hwlat_callback_enabled; |
| 87 | |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 88 | /* If the user changed threshold, remember it */ |
| 89 | static u64 last_tracing_thresh = DEFAULT_LAT_THRESHOLD * NSEC_PER_USEC; |
| 90 | |
| 91 | /* Individual latency samples are stored here when detected. */ |
| 92 | struct hwlat_sample { |
Deepa Dinamani | 51aad0a | 2017-05-08 15:59:13 -0700 | [diff] [blame] | 93 | u64 seqnum; /* unique sequence */ |
| 94 | u64 duration; /* delta */ |
| 95 | u64 outer_duration; /* delta (outer loop) */ |
| 96 | u64 nmi_total_ts; /* Total time spent in NMIs */ |
| 97 | struct timespec64 timestamp; /* wall time */ |
| 98 | int nmi_count; /* # NMIs during this sample */ |
Ingo Molnar | f2cc020 | 2021-03-23 18:49:35 +0100 | [diff] [blame] | 99 | int count; /* # of iterations over thresh */ |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 100 | }; |
| 101 | |
| 102 | /* keep the global state somewhere. */ |
| 103 | static struct hwlat_data { |
| 104 | |
| 105 | struct mutex lock; /* protect changes */ |
| 106 | |
| 107 | u64 count; /* total since reset */ |
| 108 | |
| 109 | u64 sample_window; /* total sampling window (on+off) */ |
| 110 | u64 sample_width; /* active sampling portion of window */ |
| 111 | |
Daniel Bristot de Oliveira | 8fa826b | 2021-06-22 16:42:20 +0200 | [diff] [blame] | 112 | int thread_mode; /* thread mode */ |
| 113 | |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 114 | } hwlat_data = { |
| 115 | .sample_window = DEFAULT_SAMPLE_WINDOW, |
| 116 | .sample_width = DEFAULT_SAMPLE_WIDTH, |
Daniel Bristot de Oliveira | 8fa826b | 2021-06-22 16:42:20 +0200 | [diff] [blame] | 117 | .thread_mode = MODE_ROUND_ROBIN |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 118 | }; |
| 119 | |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 120 | static struct hwlat_kthread_data *get_cpu_data(void) |
| 121 | { |
| 122 | if (hwlat_data.thread_mode == MODE_PER_CPU) |
| 123 | return this_cpu_ptr(&hwlat_per_cpu_data); |
| 124 | else |
| 125 | return &hwlat_single_cpu_data; |
| 126 | } |
| 127 | |
Daniel Bristot de Oliveira | 8fa826b | 2021-06-22 16:42:20 +0200 | [diff] [blame] | 128 | static bool hwlat_busy; |
| 129 | |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 130 | static void trace_hwlat_sample(struct hwlat_sample *sample) |
| 131 | { |
| 132 | struct trace_array *tr = hwlat_trace; |
| 133 | struct trace_event_call *call = &event_hwlat; |
Steven Rostedt (VMware) | 1329249 | 2019-12-13 13:58:57 -0500 | [diff] [blame] | 134 | struct trace_buffer *buffer = tr->array_buffer.buffer; |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 135 | struct ring_buffer_event *event; |
| 136 | struct hwlat_entry *entry; |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 137 | |
| 138 | event = trace_buffer_lock_reserve(buffer, TRACE_HWLAT, sizeof(*entry), |
Sebastian Andrzej Siewior | 36590c50 | 2021-01-25 20:45:08 +0100 | [diff] [blame] | 139 | tracing_gen_ctx()); |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 140 | if (!event) |
| 141 | return; |
| 142 | entry = ring_buffer_event_data(event); |
| 143 | entry->seqnum = sample->seqnum; |
| 144 | entry->duration = sample->duration; |
| 145 | entry->outer_duration = sample->outer_duration; |
| 146 | entry->timestamp = sample->timestamp; |
Steven Rostedt (Red Hat) | 7b2c862 | 2016-08-04 12:49:53 -0400 | [diff] [blame] | 147 | entry->nmi_total_ts = sample->nmi_total_ts; |
| 148 | entry->nmi_count = sample->nmi_count; |
Steven Rostedt (VMware) | b396bfd | 2020-02-12 12:21:03 -0500 | [diff] [blame] | 149 | entry->count = sample->count; |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 150 | |
| 151 | if (!call_filter_check_discard(call, entry, buffer, event)) |
Steven Rostedt (Red Hat) | 52ffabe3 | 2016-11-23 20:28:38 -0500 | [diff] [blame] | 152 | trace_buffer_unlock_commit_nostack(buffer, event); |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 153 | } |
| 154 | |
| 155 | /* Macros to encapsulate the time capturing infrastructure */ |
| 156 | #define time_type u64 |
| 157 | #define time_get() trace_clock_local() |
| 158 | #define time_to_us(x) div_u64(x, 1000) |
| 159 | #define time_sub(a, b) ((a) - (b)) |
| 160 | #define init_time(a, b) (a = b) |
| 161 | #define time_u64(a) a |
| 162 | |
Steven Rostedt (Red Hat) | 7b2c862 | 2016-08-04 12:49:53 -0400 | [diff] [blame] | 163 | void trace_hwlat_callback(bool enter) |
| 164 | { |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 165 | struct hwlat_kthread_data *kdata = get_cpu_data(); |
| 166 | |
| 167 | if (!kdata->kthread) |
Steven Rostedt (Red Hat) | 7b2c862 | 2016-08-04 12:49:53 -0400 | [diff] [blame] | 168 | return; |
| 169 | |
| 170 | /* |
| 171 | * Currently trace_clock_local() calls sched_clock() and the |
| 172 | * generic version is not NMI safe. |
| 173 | */ |
| 174 | if (!IS_ENABLED(CONFIG_GENERIC_SCHED_CLOCK)) { |
| 175 | if (enter) |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 176 | kdata->nmi_ts_start = time_get(); |
Steven Rostedt (Red Hat) | 7b2c862 | 2016-08-04 12:49:53 -0400 | [diff] [blame] | 177 | else |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 178 | kdata->nmi_total_ts += time_get() - kdata->nmi_ts_start; |
Steven Rostedt (Red Hat) | 7b2c862 | 2016-08-04 12:49:53 -0400 | [diff] [blame] | 179 | } |
| 180 | |
| 181 | if (enter) |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 182 | kdata->nmi_count++; |
Steven Rostedt (Red Hat) | 7b2c862 | 2016-08-04 12:49:53 -0400 | [diff] [blame] | 183 | } |
| 184 | |
Daniel Bristot de Oliveira | aa892f8 | 2021-06-22 16:42:25 +0200 | [diff] [blame] | 185 | /* |
| 186 | * hwlat_err - report a hwlat error. |
| 187 | */ |
| 188 | #define hwlat_err(msg) ({ \ |
| 189 | struct trace_array *tr = hwlat_trace; \ |
| 190 | \ |
| 191 | trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_, msg); \ |
| 192 | }) |
| 193 | |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 194 | /** |
| 195 | * get_sample - sample the CPU TSC and look for likely hardware latencies |
| 196 | * |
| 197 | * Used to repeatedly capture the CPU TSC (or similar), looking for potential |
| 198 | * hardware-induced latency. Called with interrupts disabled and with |
| 199 | * hwlat_data.lock held. |
| 200 | */ |
| 201 | static int get_sample(void) |
| 202 | { |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 203 | struct hwlat_kthread_data *kdata = get_cpu_data(); |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 204 | struct trace_array *tr = hwlat_trace; |
Steven Rostedt (VMware) | b396bfd | 2020-02-12 12:21:03 -0500 | [diff] [blame] | 205 | struct hwlat_sample s; |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 206 | time_type start, t1, t2, last_t2; |
Steven Rostedt (VMware) | b396bfd | 2020-02-12 12:21:03 -0500 | [diff] [blame] | 207 | s64 diff, outer_diff, total, last_total = 0; |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 208 | u64 sample = 0; |
| 209 | u64 thresh = tracing_thresh; |
| 210 | u64 outer_sample = 0; |
| 211 | int ret = -1; |
Steven Rostedt (VMware) | b396bfd | 2020-02-12 12:21:03 -0500 | [diff] [blame] | 212 | unsigned int count = 0; |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 213 | |
| 214 | do_div(thresh, NSEC_PER_USEC); /* modifies interval value */ |
| 215 | |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 216 | kdata->nmi_total_ts = 0; |
| 217 | kdata->nmi_count = 0; |
Steven Rostedt (Red Hat) | 7b2c862 | 2016-08-04 12:49:53 -0400 | [diff] [blame] | 218 | /* Make sure NMIs see this first */ |
| 219 | barrier(); |
| 220 | |
| 221 | trace_hwlat_callback_enabled = true; |
| 222 | |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 223 | init_time(last_t2, 0); |
| 224 | start = time_get(); /* start timestamp */ |
Steven Rostedt (VMware) | b396bfd | 2020-02-12 12:21:03 -0500 | [diff] [blame] | 225 | outer_diff = 0; |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 226 | |
| 227 | do { |
| 228 | |
| 229 | t1 = time_get(); /* we'll look for a discontinuity */ |
| 230 | t2 = time_get(); |
| 231 | |
| 232 | if (time_u64(last_t2)) { |
| 233 | /* Check the delta from outer loop (t2 to next t1) */ |
Steven Rostedt (VMware) | b396bfd | 2020-02-12 12:21:03 -0500 | [diff] [blame] | 234 | outer_diff = time_to_us(time_sub(t1, last_t2)); |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 235 | /* This shouldn't happen */ |
Steven Rostedt (VMware) | b396bfd | 2020-02-12 12:21:03 -0500 | [diff] [blame] | 236 | if (outer_diff < 0) { |
Daniel Bristot de Oliveira | aa892f8 | 2021-06-22 16:42:25 +0200 | [diff] [blame] | 237 | hwlat_err(BANNER "time running backwards\n"); |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 238 | goto out; |
| 239 | } |
Steven Rostedt (VMware) | b396bfd | 2020-02-12 12:21:03 -0500 | [diff] [blame] | 240 | if (outer_diff > outer_sample) |
| 241 | outer_sample = outer_diff; |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 242 | } |
| 243 | last_t2 = t2; |
| 244 | |
| 245 | total = time_to_us(time_sub(t2, start)); /* sample width */ |
| 246 | |
| 247 | /* Check for possible overflows */ |
| 248 | if (total < last_total) { |
Daniel Bristot de Oliveira | aa892f8 | 2021-06-22 16:42:25 +0200 | [diff] [blame] | 249 | hwlat_err("Time total overflowed\n"); |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 250 | break; |
| 251 | } |
| 252 | last_total = total; |
| 253 | |
| 254 | /* This checks the inner loop (t1 to t2) */ |
| 255 | diff = time_to_us(time_sub(t2, t1)); /* current diff */ |
| 256 | |
Steven Rostedt (VMware) | b396bfd | 2020-02-12 12:21:03 -0500 | [diff] [blame] | 257 | if (diff > thresh || outer_diff > thresh) { |
| 258 | if (!count) |
| 259 | ktime_get_real_ts64(&s.timestamp); |
| 260 | count++; |
| 261 | } |
| 262 | |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 263 | /* This shouldn't happen */ |
| 264 | if (diff < 0) { |
Daniel Bristot de Oliveira | aa892f8 | 2021-06-22 16:42:25 +0200 | [diff] [blame] | 265 | hwlat_err(BANNER "time running backwards\n"); |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 266 | goto out; |
| 267 | } |
| 268 | |
| 269 | if (diff > sample) |
| 270 | sample = diff; /* only want highest value */ |
| 271 | |
| 272 | } while (total <= hwlat_data.sample_width); |
| 273 | |
Steven Rostedt (Red Hat) | 7b2c862 | 2016-08-04 12:49:53 -0400 | [diff] [blame] | 274 | barrier(); /* finish the above in the view for NMIs */ |
| 275 | trace_hwlat_callback_enabled = false; |
| 276 | barrier(); /* Make sure nmi_total_ts is no longer updated */ |
| 277 | |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 278 | ret = 0; |
| 279 | |
| 280 | /* If we exceed the threshold value, we have found a hardware latency */ |
| 281 | if (sample > thresh || outer_sample > thresh) { |
Viktor Rosendahl (BMW) | 91edde2 | 2019-10-09 00:08:21 +0200 | [diff] [blame] | 282 | u64 latency; |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 283 | |
| 284 | ret = 1; |
| 285 | |
Steven Rostedt (Red Hat) | 7b2c862 | 2016-08-04 12:49:53 -0400 | [diff] [blame] | 286 | /* We read in microseconds */ |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 287 | if (kdata->nmi_total_ts) |
| 288 | do_div(kdata->nmi_total_ts, NSEC_PER_USEC); |
Steven Rostedt (Red Hat) | 7b2c862 | 2016-08-04 12:49:53 -0400 | [diff] [blame] | 289 | |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 290 | hwlat_data.count++; |
| 291 | s.seqnum = hwlat_data.count; |
| 292 | s.duration = sample; |
| 293 | s.outer_duration = outer_sample; |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 294 | s.nmi_total_ts = kdata->nmi_total_ts; |
| 295 | s.nmi_count = kdata->nmi_count; |
Steven Rostedt (VMware) | b396bfd | 2020-02-12 12:21:03 -0500 | [diff] [blame] | 296 | s.count = count; |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 297 | trace_hwlat_sample(&s); |
| 298 | |
Viktor Rosendahl (BMW) | 91edde2 | 2019-10-09 00:08:21 +0200 | [diff] [blame] | 299 | latency = max(sample, outer_sample); |
| 300 | |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 301 | /* Keep a running maximum ever recorded hardware latency */ |
Viktor Rosendahl (BMW) | 91edde2 | 2019-10-09 00:08:21 +0200 | [diff] [blame] | 302 | if (latency > tr->max_latency) { |
| 303 | tr->max_latency = latency; |
| 304 | latency_fsnotify(tr); |
| 305 | } |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 306 | } |
| 307 | |
| 308 | out: |
| 309 | return ret; |
| 310 | } |
| 311 | |
Steven Rostedt (Red Hat) | 0330f7a | 2016-07-15 15:48:56 -0400 | [diff] [blame] | 312 | static struct cpumask save_cpumask; |
Steven Rostedt (Red Hat) | 0330f7a | 2016-07-15 15:48:56 -0400 | [diff] [blame] | 313 | |
Steven Rostedt (VMware) | f447c19 | 2017-01-31 16:48:23 -0500 | [diff] [blame] | 314 | static void move_to_next_cpu(void) |
Steven Rostedt (Red Hat) | 0330f7a | 2016-07-15 15:48:56 -0400 | [diff] [blame] | 315 | { |
Steven Rostedt (VMware) | f447c19 | 2017-01-31 16:48:23 -0500 | [diff] [blame] | 316 | struct cpumask *current_mask = &save_cpumask; |
Kevin Hao | 96b4833 | 2020-07-30 16:23:18 +0800 | [diff] [blame] | 317 | struct trace_array *tr = hwlat_trace; |
Steven Rostedt (Red Hat) | 0330f7a | 2016-07-15 15:48:56 -0400 | [diff] [blame] | 318 | int next_cpu; |
| 319 | |
Steven Rostedt (Red Hat) | 0330f7a | 2016-07-15 15:48:56 -0400 | [diff] [blame] | 320 | /* |
| 321 | * If for some reason the user modifies the CPU affinity |
Srivatsa S. Bhat (VMware) | 0c3c86b | 2019-10-10 11:51:17 -0700 | [diff] [blame] | 322 | * of this thread, then stop migrating for the duration |
Steven Rostedt (Red Hat) | 0330f7a | 2016-07-15 15:48:56 -0400 | [diff] [blame] | 323 | * of the current test. |
| 324 | */ |
Sebastian Andrzej Siewior | 3bd3706 | 2019-04-23 16:26:36 +0200 | [diff] [blame] | 325 | if (!cpumask_equal(current_mask, current->cpus_ptr)) |
Daniel Bristot de Oliveira | 7bb7d80 | 2021-06-22 16:42:21 +0200 | [diff] [blame] | 326 | goto change_mode; |
Steven Rostedt (Red Hat) | 0330f7a | 2016-07-15 15:48:56 -0400 | [diff] [blame] | 327 | |
Sebastian Andrzej Siewior | 99c37d1 | 2021-08-03 16:16:19 +0200 | [diff] [blame] | 328 | cpus_read_lock(); |
Kevin Hao | 96b4833 | 2020-07-30 16:23:18 +0800 | [diff] [blame] | 329 | cpumask_and(current_mask, cpu_online_mask, tr->tracing_cpumask); |
Steven Rostedt (VMware) | 51397dc | 2021-08-04 14:18:48 -0400 | [diff] [blame] | 330 | next_cpu = cpumask_next(raw_smp_processor_id(), current_mask); |
Sebastian Andrzej Siewior | 99c37d1 | 2021-08-03 16:16:19 +0200 | [diff] [blame] | 331 | cpus_read_unlock(); |
Steven Rostedt (Red Hat) | 0330f7a | 2016-07-15 15:48:56 -0400 | [diff] [blame] | 332 | |
| 333 | if (next_cpu >= nr_cpu_ids) |
| 334 | next_cpu = cpumask_first(current_mask); |
| 335 | |
Steven Rostedt (Red Hat) | 0330f7a | 2016-07-15 15:48:56 -0400 | [diff] [blame] | 336 | if (next_cpu >= nr_cpu_ids) /* Shouldn't happen! */ |
Daniel Bristot de Oliveira | 7bb7d80 | 2021-06-22 16:42:21 +0200 | [diff] [blame] | 337 | goto change_mode; |
Steven Rostedt (Red Hat) | 0330f7a | 2016-07-15 15:48:56 -0400 | [diff] [blame] | 338 | |
| 339 | cpumask_clear(current_mask); |
| 340 | cpumask_set_cpu(next_cpu, current_mask); |
| 341 | |
| 342 | sched_setaffinity(0, current_mask); |
| 343 | return; |
| 344 | |
Daniel Bristot de Oliveira | 7bb7d80 | 2021-06-22 16:42:21 +0200 | [diff] [blame] | 345 | change_mode: |
| 346 | hwlat_data.thread_mode = MODE_NONE; |
| 347 | pr_info(BANNER "cpumask changed while in round-robin mode, switching to mode none\n"); |
Steven Rostedt (Red Hat) | 0330f7a | 2016-07-15 15:48:56 -0400 | [diff] [blame] | 348 | } |
| 349 | |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 350 | /* |
| 351 | * kthread_fn - The CPU time sampling/hardware latency detection kernel thread |
| 352 | * |
| 353 | * Used to periodically sample the CPU TSC via a call to get_sample. We |
| 354 | * disable interrupts, which does (intentionally) introduce latency since we |
| 355 | * need to ensure nothing else might be running (and thus preempting). |
| 356 | * Obviously this should never be used in production environments. |
| 357 | * |
Luiz Capitulino | 8e0f114 | 2017-02-13 12:25:17 -0500 | [diff] [blame] | 358 | * Executes one loop interaction on each CPU in tracing_cpumask sysfs file. |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 359 | */ |
| 360 | static int kthread_fn(void *data) |
| 361 | { |
| 362 | u64 interval; |
| 363 | |
| 364 | while (!kthread_should_stop()) { |
| 365 | |
Daniel Bristot de Oliveira | 8fa826b | 2021-06-22 16:42:20 +0200 | [diff] [blame] | 366 | if (hwlat_data.thread_mode == MODE_ROUND_ROBIN) |
| 367 | move_to_next_cpu(); |
Steven Rostedt (Red Hat) | 0330f7a | 2016-07-15 15:48:56 -0400 | [diff] [blame] | 368 | |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 369 | local_irq_disable(); |
| 370 | get_sample(); |
| 371 | local_irq_enable(); |
| 372 | |
| 373 | mutex_lock(&hwlat_data.lock); |
| 374 | interval = hwlat_data.sample_window - hwlat_data.sample_width; |
| 375 | mutex_unlock(&hwlat_data.lock); |
| 376 | |
| 377 | do_div(interval, USEC_PER_MSEC); /* modifies interval value */ |
| 378 | |
| 379 | /* Always sleep for at least 1ms */ |
| 380 | if (interval < 1) |
| 381 | interval = 1; |
| 382 | |
| 383 | if (msleep_interruptible(interval)) |
| 384 | break; |
| 385 | } |
| 386 | |
| 387 | return 0; |
| 388 | } |
| 389 | |
Daniel Bristot de Oliveira | 8fa826b | 2021-06-22 16:42:20 +0200 | [diff] [blame] | 390 | /* |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 391 | * stop_stop_kthread - Inform the hardware latency sampling/detector kthread to stop |
| 392 | * |
| 393 | * This kicks the running hardware latency sampling/detector kernel thread and |
| 394 | * tells it to stop sampling now. Use this on unload and at system shutdown. |
| 395 | */ |
| 396 | static void stop_single_kthread(void) |
| 397 | { |
| 398 | struct hwlat_kthread_data *kdata = get_cpu_data(); |
Daniel Bristot de Oliveira | 039a602 | 2021-06-22 16:42:29 +0200 | [diff] [blame] | 399 | struct task_struct *kthread; |
| 400 | |
Sebastian Andrzej Siewior | 99c37d1 | 2021-08-03 16:16:19 +0200 | [diff] [blame] | 401 | cpus_read_lock(); |
Daniel Bristot de Oliveira | 039a602 | 2021-06-22 16:42:29 +0200 | [diff] [blame] | 402 | kthread = kdata->kthread; |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 403 | |
| 404 | if (!kthread) |
Daniel Bristot de Oliveira | 039a602 | 2021-06-22 16:42:29 +0200 | [diff] [blame] | 405 | goto out_put_cpus; |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 406 | |
| 407 | kthread_stop(kthread); |
| 408 | kdata->kthread = NULL; |
Daniel Bristot de Oliveira | 039a602 | 2021-06-22 16:42:29 +0200 | [diff] [blame] | 409 | |
| 410 | out_put_cpus: |
Sebastian Andrzej Siewior | 99c37d1 | 2021-08-03 16:16:19 +0200 | [diff] [blame] | 411 | cpus_read_unlock(); |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 412 | } |
| 413 | |
| 414 | |
| 415 | /* |
| 416 | * start_single_kthread - Kick off the hardware latency sampling/detector kthread |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 417 | * |
| 418 | * This starts the kernel thread that will sit and sample the CPU timestamp |
| 419 | * counter (TSC or similar) and look for potential hardware latencies. |
| 420 | */ |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 421 | static int start_single_kthread(struct trace_array *tr) |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 422 | { |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 423 | struct hwlat_kthread_data *kdata = get_cpu_data(); |
Steven Rostedt (VMware) | f447c19 | 2017-01-31 16:48:23 -0500 | [diff] [blame] | 424 | struct cpumask *current_mask = &save_cpumask; |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 425 | struct task_struct *kthread; |
Steven Rostedt (VMware) | f447c19 | 2017-01-31 16:48:23 -0500 | [diff] [blame] | 426 | int next_cpu; |
| 427 | |
Sebastian Andrzej Siewior | 99c37d1 | 2021-08-03 16:16:19 +0200 | [diff] [blame] | 428 | cpus_read_lock(); |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 429 | if (kdata->kthread) |
Daniel Bristot de Oliveira | 039a602 | 2021-06-22 16:42:29 +0200 | [diff] [blame] | 430 | goto out_put_cpus; |
Erica Bugden | 82fbc8c | 2018-08-01 12:45:54 +0200 | [diff] [blame] | 431 | |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 432 | kthread = kthread_create(kthread_fn, NULL, "hwlatd"); |
| 433 | if (IS_ERR(kthread)) { |
| 434 | pr_err(BANNER "could not start sampling thread\n"); |
Sebastian Andrzej Siewior | 99c37d1 | 2021-08-03 16:16:19 +0200 | [diff] [blame] | 435 | cpus_read_unlock(); |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 436 | return -ENOMEM; |
| 437 | } |
Steven Rostedt (VMware) | f447c19 | 2017-01-31 16:48:23 -0500 | [diff] [blame] | 438 | |
Daniel Bristot de Oliveira | 8fa826b | 2021-06-22 16:42:20 +0200 | [diff] [blame] | 439 | /* Just pick the first CPU on first iteration */ |
Daniel Bristot de Oliveira | 8fa826b | 2021-06-22 16:42:20 +0200 | [diff] [blame] | 440 | cpumask_and(current_mask, cpu_online_mask, tr->tracing_cpumask); |
Daniel Bristot de Oliveira | 8fa826b | 2021-06-22 16:42:20 +0200 | [diff] [blame] | 441 | |
| 442 | if (hwlat_data.thread_mode == MODE_ROUND_ROBIN) { |
| 443 | next_cpu = cpumask_first(current_mask); |
| 444 | cpumask_clear(current_mask); |
| 445 | cpumask_set_cpu(next_cpu, current_mask); |
| 446 | |
| 447 | } |
| 448 | |
Steven Rostedt (VMware) | f447c19 | 2017-01-31 16:48:23 -0500 | [diff] [blame] | 449 | sched_setaffinity(kthread->pid, current_mask); |
| 450 | |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 451 | kdata->kthread = kthread; |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 452 | wake_up_process(kthread); |
| 453 | |
Daniel Bristot de Oliveira | 039a602 | 2021-06-22 16:42:29 +0200 | [diff] [blame] | 454 | out_put_cpus: |
Sebastian Andrzej Siewior | 99c37d1 | 2021-08-03 16:16:19 +0200 | [diff] [blame] | 455 | cpus_read_unlock(); |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 456 | return 0; |
| 457 | } |
| 458 | |
Daniel Bristot de Oliveira | 8fa826b | 2021-06-22 16:42:20 +0200 | [diff] [blame] | 459 | /* |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 460 | * stop_cpu_kthread - Stop a hwlat cpu kthread |
| 461 | */ |
| 462 | static void stop_cpu_kthread(unsigned int cpu) |
| 463 | { |
| 464 | struct task_struct *kthread; |
| 465 | |
| 466 | kthread = per_cpu(hwlat_per_cpu_data, cpu).kthread; |
| 467 | if (kthread) |
| 468 | kthread_stop(kthread); |
Daniel Bristot de Oliveira | ba998f7 | 2021-06-22 16:42:31 +0200 | [diff] [blame] | 469 | per_cpu(hwlat_per_cpu_data, cpu).kthread = NULL; |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 470 | } |
| 471 | |
| 472 | /* |
| 473 | * stop_per_cpu_kthreads - Inform the hardware latency sampling/detector kthread to stop |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 474 | * |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 475 | * This kicks the running hardware latency sampling/detector kernel threads and |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 476 | * tells it to stop sampling now. Use this on unload and at system shutdown. |
| 477 | */ |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 478 | static void stop_per_cpu_kthreads(void) |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 479 | { |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 480 | unsigned int cpu; |
| 481 | |
Sebastian Andrzej Siewior | 99c37d1 | 2021-08-03 16:16:19 +0200 | [diff] [blame] | 482 | cpus_read_lock(); |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 483 | for_each_online_cpu(cpu) |
| 484 | stop_cpu_kthread(cpu); |
Sebastian Andrzej Siewior | 99c37d1 | 2021-08-03 16:16:19 +0200 | [diff] [blame] | 485 | cpus_read_unlock(); |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 486 | } |
| 487 | |
| 488 | /* |
| 489 | * start_cpu_kthread - Start a hwlat cpu kthread |
| 490 | */ |
| 491 | static int start_cpu_kthread(unsigned int cpu) |
| 492 | { |
| 493 | struct task_struct *kthread; |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 494 | |
Cai Huoqing | ff78f66 | 2022-01-14 14:03:10 -0800 | [diff] [blame] | 495 | kthread = kthread_run_on_cpu(kthread_fn, NULL, cpu, "hwlatd/%u"); |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 496 | if (IS_ERR(kthread)) { |
| 497 | pr_err(BANNER "could not start sampling thread\n"); |
| 498 | return -ENOMEM; |
| 499 | } |
| 500 | |
| 501 | per_cpu(hwlat_per_cpu_data, cpu).kthread = kthread; |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 502 | |
| 503 | return 0; |
| 504 | } |
| 505 | |
Daniel Bristot de Oliveira | ba998f7 | 2021-06-22 16:42:31 +0200 | [diff] [blame] | 506 | #ifdef CONFIG_HOTPLUG_CPU |
| 507 | static void hwlat_hotplug_workfn(struct work_struct *dummy) |
| 508 | { |
| 509 | struct trace_array *tr = hwlat_trace; |
| 510 | unsigned int cpu = smp_processor_id(); |
| 511 | |
| 512 | mutex_lock(&trace_types_lock); |
| 513 | mutex_lock(&hwlat_data.lock); |
Sebastian Andrzej Siewior | 99c37d1 | 2021-08-03 16:16:19 +0200 | [diff] [blame] | 514 | cpus_read_lock(); |
Daniel Bristot de Oliveira | ba998f7 | 2021-06-22 16:42:31 +0200 | [diff] [blame] | 515 | |
| 516 | if (!hwlat_busy || hwlat_data.thread_mode != MODE_PER_CPU) |
| 517 | goto out_unlock; |
| 518 | |
| 519 | if (!cpumask_test_cpu(cpu, tr->tracing_cpumask)) |
| 520 | goto out_unlock; |
| 521 | |
| 522 | start_cpu_kthread(cpu); |
| 523 | |
| 524 | out_unlock: |
Sebastian Andrzej Siewior | 99c37d1 | 2021-08-03 16:16:19 +0200 | [diff] [blame] | 525 | cpus_read_unlock(); |
Daniel Bristot de Oliveira | ba998f7 | 2021-06-22 16:42:31 +0200 | [diff] [blame] | 526 | mutex_unlock(&hwlat_data.lock); |
| 527 | mutex_unlock(&trace_types_lock); |
| 528 | } |
| 529 | |
| 530 | static DECLARE_WORK(hwlat_hotplug_work, hwlat_hotplug_workfn); |
| 531 | |
| 532 | /* |
| 533 | * hwlat_cpu_init - CPU hotplug online callback function |
| 534 | */ |
| 535 | static int hwlat_cpu_init(unsigned int cpu) |
| 536 | { |
| 537 | schedule_work_on(cpu, &hwlat_hotplug_work); |
| 538 | return 0; |
| 539 | } |
| 540 | |
| 541 | /* |
| 542 | * hwlat_cpu_die - CPU hotplug offline callback function |
| 543 | */ |
| 544 | static int hwlat_cpu_die(unsigned int cpu) |
| 545 | { |
| 546 | stop_cpu_kthread(cpu); |
| 547 | return 0; |
| 548 | } |
| 549 | |
| 550 | static void hwlat_init_hotplug_support(void) |
| 551 | { |
| 552 | int ret; |
| 553 | |
| 554 | ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "trace/hwlat:online", |
| 555 | hwlat_cpu_init, hwlat_cpu_die); |
| 556 | if (ret < 0) |
| 557 | pr_warn(BANNER "Error to init cpu hotplug support\n"); |
| 558 | |
| 559 | return; |
| 560 | } |
| 561 | #else /* CONFIG_HOTPLUG_CPU */ |
| 562 | static void hwlat_init_hotplug_support(void) |
| 563 | { |
| 564 | return; |
| 565 | } |
| 566 | #endif /* CONFIG_HOTPLUG_CPU */ |
| 567 | |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 568 | /* |
| 569 | * start_per_cpu_kthreads - Kick off the hardware latency sampling/detector kthreads |
| 570 | * |
| 571 | * This starts the kernel threads that will sit on potentially all cpus and |
| 572 | * sample the CPU timestamp counter (TSC or similar) and look for potential |
| 573 | * hardware latencies. |
| 574 | */ |
| 575 | static int start_per_cpu_kthreads(struct trace_array *tr) |
| 576 | { |
| 577 | struct cpumask *current_mask = &save_cpumask; |
| 578 | unsigned int cpu; |
| 579 | int retval; |
| 580 | |
Sebastian Andrzej Siewior | 99c37d1 | 2021-08-03 16:16:19 +0200 | [diff] [blame] | 581 | cpus_read_lock(); |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 582 | /* |
| 583 | * Run only on CPUs in which hwlat is allowed to run. |
| 584 | */ |
| 585 | cpumask_and(current_mask, cpu_online_mask, tr->tracing_cpumask); |
| 586 | |
| 587 | for_each_online_cpu(cpu) |
| 588 | per_cpu(hwlat_per_cpu_data, cpu).kthread = NULL; |
| 589 | |
| 590 | for_each_cpu(cpu, current_mask) { |
| 591 | retval = start_cpu_kthread(cpu); |
| 592 | if (retval) |
| 593 | goto out_error; |
| 594 | } |
Sebastian Andrzej Siewior | 99c37d1 | 2021-08-03 16:16:19 +0200 | [diff] [blame] | 595 | cpus_read_unlock(); |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 596 | |
| 597 | return 0; |
| 598 | |
| 599 | out_error: |
Sebastian Andrzej Siewior | 99c37d1 | 2021-08-03 16:16:19 +0200 | [diff] [blame] | 600 | cpus_read_unlock(); |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 601 | stop_per_cpu_kthreads(); |
| 602 | return retval; |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 603 | } |
| 604 | |
Daniel Bristot de Oliveira | 8fa826b | 2021-06-22 16:42:20 +0200 | [diff] [blame] | 605 | static void *s_mode_start(struct seq_file *s, loff_t *pos) |
| 606 | { |
| 607 | int mode = *pos; |
| 608 | |
| 609 | mutex_lock(&hwlat_data.lock); |
| 610 | |
| 611 | if (mode >= MODE_MAX) |
| 612 | return NULL; |
| 613 | |
| 614 | return pos; |
| 615 | } |
| 616 | |
| 617 | static void *s_mode_next(struct seq_file *s, void *v, loff_t *pos) |
| 618 | { |
| 619 | int mode = ++(*pos); |
| 620 | |
| 621 | if (mode >= MODE_MAX) |
| 622 | return NULL; |
| 623 | |
| 624 | return pos; |
| 625 | } |
| 626 | |
| 627 | static int s_mode_show(struct seq_file *s, void *v) |
| 628 | { |
| 629 | loff_t *pos = v; |
| 630 | int mode = *pos; |
| 631 | |
| 632 | if (mode == hwlat_data.thread_mode) |
| 633 | seq_printf(s, "[%s]", thread_mode_str[mode]); |
| 634 | else |
| 635 | seq_printf(s, "%s", thread_mode_str[mode]); |
| 636 | |
| 637 | if (mode != MODE_MAX) |
| 638 | seq_puts(s, " "); |
| 639 | |
| 640 | return 0; |
| 641 | } |
| 642 | |
| 643 | static void s_mode_stop(struct seq_file *s, void *v) |
| 644 | { |
| 645 | seq_puts(s, "\n"); |
| 646 | mutex_unlock(&hwlat_data.lock); |
| 647 | } |
| 648 | |
| 649 | static const struct seq_operations thread_mode_seq_ops = { |
| 650 | .start = s_mode_start, |
| 651 | .next = s_mode_next, |
| 652 | .show = s_mode_show, |
| 653 | .stop = s_mode_stop |
| 654 | }; |
| 655 | |
| 656 | static int hwlat_mode_open(struct inode *inode, struct file *file) |
| 657 | { |
| 658 | return seq_open(file, &thread_mode_seq_ops); |
| 659 | }; |
| 660 | |
| 661 | static void hwlat_tracer_start(struct trace_array *tr); |
| 662 | static void hwlat_tracer_stop(struct trace_array *tr); |
| 663 | |
| 664 | /** |
| 665 | * hwlat_mode_write - Write function for "mode" entry |
| 666 | * @filp: The active open file structure |
| 667 | * @ubuf: The user buffer that contains the value to write |
| 668 | * @cnt: The maximum number of bytes to write to "file" |
| 669 | * @ppos: The current position in @file |
| 670 | * |
| 671 | * This function provides a write implementation for the "mode" interface |
| 672 | * to the hardware latency detector. hwlatd has different operation modes. |
| 673 | * The "none" sets the allowed cpumask for a single hwlatd thread at the |
| 674 | * startup and lets the scheduler handle the migration. The default mode is |
| 675 | * the "round-robin" one, in which a single hwlatd thread runs, migrating |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 676 | * among the allowed CPUs in a round-robin fashion. The "per-cpu" mode |
| 677 | * creates one hwlatd thread per allowed CPU. |
Daniel Bristot de Oliveira | 8fa826b | 2021-06-22 16:42:20 +0200 | [diff] [blame] | 678 | */ |
| 679 | static ssize_t hwlat_mode_write(struct file *filp, const char __user *ubuf, |
| 680 | size_t cnt, loff_t *ppos) |
| 681 | { |
| 682 | struct trace_array *tr = hwlat_trace; |
| 683 | const char *mode; |
| 684 | char buf[64]; |
| 685 | int ret, i; |
| 686 | |
| 687 | if (cnt >= sizeof(buf)) |
| 688 | return -EINVAL; |
| 689 | |
| 690 | if (copy_from_user(buf, ubuf, cnt)) |
| 691 | return -EFAULT; |
| 692 | |
| 693 | buf[cnt] = 0; |
| 694 | |
| 695 | mode = strstrip(buf); |
| 696 | |
| 697 | ret = -EINVAL; |
| 698 | |
| 699 | /* |
| 700 | * trace_types_lock is taken to avoid concurrency on start/stop |
| 701 | * and hwlat_busy. |
| 702 | */ |
| 703 | mutex_lock(&trace_types_lock); |
| 704 | if (hwlat_busy) |
| 705 | hwlat_tracer_stop(tr); |
| 706 | |
| 707 | mutex_lock(&hwlat_data.lock); |
| 708 | |
| 709 | for (i = 0; i < MODE_MAX; i++) { |
| 710 | if (strcmp(mode, thread_mode_str[i]) == 0) { |
| 711 | hwlat_data.thread_mode = i; |
| 712 | ret = cnt; |
| 713 | } |
| 714 | } |
| 715 | |
| 716 | mutex_unlock(&hwlat_data.lock); |
| 717 | |
| 718 | if (hwlat_busy) |
| 719 | hwlat_tracer_start(tr); |
| 720 | mutex_unlock(&trace_types_lock); |
| 721 | |
| 722 | *ppos += cnt; |
| 723 | |
| 724 | |
| 725 | |
| 726 | return ret; |
| 727 | } |
| 728 | |
Daniel Bristot de Oliveira | f27a1c9 | 2021-06-22 16:42:24 +0200 | [diff] [blame] | 729 | /* |
| 730 | * The width parameter is read/write using the generic trace_min_max_param |
| 731 | * method. The *val is protected by the hwlat_data lock and is upper |
| 732 | * bounded by the window parameter. |
| 733 | */ |
| 734 | static struct trace_min_max_param hwlat_width = { |
| 735 | .lock = &hwlat_data.lock, |
| 736 | .val = &hwlat_data.sample_width, |
| 737 | .max = &hwlat_data.sample_window, |
| 738 | .min = NULL, |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 739 | }; |
| 740 | |
Daniel Bristot de Oliveira | f27a1c9 | 2021-06-22 16:42:24 +0200 | [diff] [blame] | 741 | /* |
| 742 | * The window parameter is read/write using the generic trace_min_max_param |
| 743 | * method. The *val is protected by the hwlat_data lock and is lower |
| 744 | * bounded by the width parameter. |
| 745 | */ |
| 746 | static struct trace_min_max_param hwlat_window = { |
| 747 | .lock = &hwlat_data.lock, |
| 748 | .val = &hwlat_data.sample_window, |
| 749 | .max = NULL, |
| 750 | .min = &hwlat_data.sample_width, |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 751 | }; |
| 752 | |
Daniel Bristot de Oliveira | 8fa826b | 2021-06-22 16:42:20 +0200 | [diff] [blame] | 753 | static const struct file_operations thread_mode_fops = { |
| 754 | .open = hwlat_mode_open, |
| 755 | .read = seq_read, |
| 756 | .llseek = seq_lseek, |
| 757 | .release = seq_release, |
| 758 | .write = hwlat_mode_write |
| 759 | }; |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 760 | /** |
| 761 | * init_tracefs - A function to initialize the tracefs interface files |
| 762 | * |
| 763 | * This function creates entries in tracefs for "hwlat_detector". |
| 764 | * It creates the hwlat_detector directory in the tracing directory, |
| 765 | * and within that directory is the count, width and window files to |
| 766 | * change and view those values. |
| 767 | */ |
| 768 | static int init_tracefs(void) |
| 769 | { |
Wei Yang | 22c36b1 | 2020-07-12 09:10:36 +0800 | [diff] [blame] | 770 | int ret; |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 771 | struct dentry *top_dir; |
| 772 | |
Wei Yang | 22c36b1 | 2020-07-12 09:10:36 +0800 | [diff] [blame] | 773 | ret = tracing_init_dentry(); |
| 774 | if (ret) |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 775 | return -ENOMEM; |
| 776 | |
Wei Yang | 22c36b1 | 2020-07-12 09:10:36 +0800 | [diff] [blame] | 777 | top_dir = tracefs_create_dir("hwlat_detector", NULL); |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 778 | if (!top_dir) |
| 779 | return -ENOMEM; |
| 780 | |
Steven Rostedt (VMware) | 21ccc9c | 2021-08-18 11:24:51 -0400 | [diff] [blame] | 781 | hwlat_sample_window = tracefs_create_file("window", TRACE_MODE_WRITE, |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 782 | top_dir, |
Daniel Bristot de Oliveira | f27a1c9 | 2021-06-22 16:42:24 +0200 | [diff] [blame] | 783 | &hwlat_window, |
| 784 | &trace_min_max_fops); |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 785 | if (!hwlat_sample_window) |
| 786 | goto err; |
| 787 | |
Steven Rostedt (VMware) | 21ccc9c | 2021-08-18 11:24:51 -0400 | [diff] [blame] | 788 | hwlat_sample_width = tracefs_create_file("width", TRACE_MODE_WRITE, |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 789 | top_dir, |
Daniel Bristot de Oliveira | f27a1c9 | 2021-06-22 16:42:24 +0200 | [diff] [blame] | 790 | &hwlat_width, |
| 791 | &trace_min_max_fops); |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 792 | if (!hwlat_sample_width) |
| 793 | goto err; |
| 794 | |
Steven Rostedt (VMware) | 21ccc9c | 2021-08-18 11:24:51 -0400 | [diff] [blame] | 795 | hwlat_thread_mode = trace_create_file("mode", TRACE_MODE_WRITE, |
Daniel Bristot de Oliveira | 8fa826b | 2021-06-22 16:42:20 +0200 | [diff] [blame] | 796 | top_dir, |
| 797 | NULL, |
| 798 | &thread_mode_fops); |
| 799 | if (!hwlat_thread_mode) |
| 800 | goto err; |
| 801 | |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 802 | return 0; |
| 803 | |
| 804 | err: |
Al Viro | a3d1e7e | 2019-11-18 09:43:10 -0500 | [diff] [blame] | 805 | tracefs_remove(top_dir); |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 806 | return -ENOMEM; |
| 807 | } |
| 808 | |
| 809 | static void hwlat_tracer_start(struct trace_array *tr) |
| 810 | { |
| 811 | int err; |
| 812 | |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 813 | if (hwlat_data.thread_mode == MODE_PER_CPU) |
| 814 | err = start_per_cpu_kthreads(tr); |
| 815 | else |
| 816 | err = start_single_kthread(tr); |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 817 | if (err) |
| 818 | pr_err(BANNER "Cannot start hwlat kthread\n"); |
| 819 | } |
| 820 | |
| 821 | static void hwlat_tracer_stop(struct trace_array *tr) |
| 822 | { |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 823 | if (hwlat_data.thread_mode == MODE_PER_CPU) |
| 824 | stop_per_cpu_kthreads(); |
| 825 | else |
| 826 | stop_single_kthread(); |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 827 | } |
| 828 | |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 829 | static int hwlat_tracer_init(struct trace_array *tr) |
| 830 | { |
| 831 | /* Only allow one instance to enable this */ |
| 832 | if (hwlat_busy) |
| 833 | return -EBUSY; |
| 834 | |
| 835 | hwlat_trace = tr; |
| 836 | |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 837 | hwlat_data.count = 0; |
| 838 | tr->max_latency = 0; |
| 839 | save_tracing_thresh = tracing_thresh; |
| 840 | |
| 841 | /* tracing_thresh is in nsecs, we speak in usecs */ |
| 842 | if (!tracing_thresh) |
| 843 | tracing_thresh = last_tracing_thresh; |
| 844 | |
| 845 | if (tracer_tracing_is_on(tr)) |
| 846 | hwlat_tracer_start(tr); |
| 847 | |
| 848 | hwlat_busy = true; |
| 849 | |
| 850 | return 0; |
| 851 | } |
| 852 | |
| 853 | static void hwlat_tracer_reset(struct trace_array *tr) |
| 854 | { |
Daniel Bristot de Oliveira | f46b165 | 2021-06-22 16:42:22 +0200 | [diff] [blame] | 855 | hwlat_tracer_stop(tr); |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 856 | |
| 857 | /* the tracing threshold is static between runs */ |
| 858 | last_tracing_thresh = tracing_thresh; |
| 859 | |
| 860 | tracing_thresh = save_tracing_thresh; |
| 861 | hwlat_busy = false; |
| 862 | } |
| 863 | |
| 864 | static struct tracer hwlat_tracer __read_mostly = |
| 865 | { |
| 866 | .name = "hwlat", |
| 867 | .init = hwlat_tracer_init, |
| 868 | .reset = hwlat_tracer_reset, |
| 869 | .start = hwlat_tracer_start, |
| 870 | .stop = hwlat_tracer_stop, |
| 871 | .allow_instances = true, |
| 872 | }; |
| 873 | |
| 874 | __init static int init_hwlat_tracer(void) |
| 875 | { |
| 876 | int ret; |
| 877 | |
| 878 | mutex_init(&hwlat_data.lock); |
| 879 | |
| 880 | ret = register_tracer(&hwlat_tracer); |
| 881 | if (ret) |
| 882 | return ret; |
| 883 | |
Daniel Bristot de Oliveira | ba998f7 | 2021-06-22 16:42:31 +0200 | [diff] [blame] | 884 | hwlat_init_hotplug_support(); |
| 885 | |
Steven Rostedt (Red Hat) | e7c15cd | 2016-06-23 12:45:36 -0400 | [diff] [blame] | 886 | init_tracefs(); |
| 887 | |
| 888 | return 0; |
| 889 | } |
| 890 | late_initcall(init_hwlat_tracer); |