Joel Fernandes (Google) | c3bc8fd | 2018-07-30 15:24:23 -0700 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0 |
| 2 | /* |
| 3 | * preemptoff and irqoff tracepoints |
| 4 | * |
| 5 | * Copyright (C) Joel Fernandes (Google) <joel@joelfernandes.org> |
| 6 | */ |
| 7 | |
| 8 | #include <linux/kallsyms.h> |
| 9 | #include <linux/uaccess.h> |
| 10 | #include <linux/module.h> |
| 11 | #include <linux/ftrace.h> |
Masami Hiramatsu | eeeb080 | 2019-02-13 01:13:40 +0900 | [diff] [blame] | 12 | #include <linux/kprobes.h> |
Steven Rostedt (VMware) | 3f1756d | 2018-08-08 21:28:05 -0400 | [diff] [blame] | 13 | #include "trace.h" |
Joel Fernandes (Google) | c3bc8fd | 2018-07-30 15:24:23 -0700 | [diff] [blame] | 14 | |
| 15 | #define CREATE_TRACE_POINTS |
| 16 | #include <trace/events/preemptirq.h> |
| 17 | |
| 18 | #ifdef CONFIG_TRACE_IRQFLAGS |
| 19 | /* Per-cpu variable to prevent redundant calls when IRQs already off */ |
| 20 | static DEFINE_PER_CPU(int, tracing_irq_cpu); |
| 21 | |
Thomas Gleixner | 0995a5d | 2020-03-04 13:09:50 +0100 | [diff] [blame] | 22 | /* |
| 23 | * Like trace_hardirqs_on() but without the lockdep invocation. This is |
| 24 | * used in the low level entry code where the ordering vs. RCU is important |
| 25 | * and lockdep uses a staged approach which splits the lockdep hardirq |
| 26 | * tracking into a RCU on and a RCU off section. |
| 27 | */ |
| 28 | void trace_hardirqs_on_prepare(void) |
| 29 | { |
| 30 | if (this_cpu_read(tracing_irq_cpu)) { |
| 31 | if (!in_nmi()) |
| 32 | trace_irq_enable(CALLER_ADDR0, CALLER_ADDR1); |
| 33 | tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1); |
| 34 | this_cpu_write(tracing_irq_cpu, 0); |
| 35 | } |
| 36 | } |
| 37 | EXPORT_SYMBOL(trace_hardirqs_on_prepare); |
| 38 | NOKPROBE_SYMBOL(trace_hardirqs_on_prepare); |
| 39 | |
Joel Fernandes (Google) | c3bc8fd | 2018-07-30 15:24:23 -0700 | [diff] [blame] | 40 | void trace_hardirqs_on(void) |
| 41 | { |
Steven Rostedt (VMware) | bff1b20 | 2018-08-06 15:50:58 -0400 | [diff] [blame] | 42 | if (this_cpu_read(tracing_irq_cpu)) { |
Steven Rostedt (VMware) | 3f1756d | 2018-08-08 21:28:05 -0400 | [diff] [blame] | 43 | if (!in_nmi()) |
| 44 | trace_irq_enable_rcuidle(CALLER_ADDR0, CALLER_ADDR1); |
| 45 | tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1); |
Steven Rostedt (VMware) | bff1b20 | 2018-08-06 15:50:58 -0400 | [diff] [blame] | 46 | this_cpu_write(tracing_irq_cpu, 0); |
| 47 | } |
Joel Fernandes (Google) | c3bc8fd | 2018-07-30 15:24:23 -0700 | [diff] [blame] | 48 | |
Peter Zijlstra | c86e9b9 | 2020-03-18 14:22:03 +0100 | [diff] [blame] | 49 | lockdep_hardirqs_on_prepare(CALLER_ADDR0); |
Steven Rostedt (VMware) | bff1b20 | 2018-08-06 15:50:58 -0400 | [diff] [blame] | 50 | lockdep_hardirqs_on(CALLER_ADDR0); |
Joel Fernandes (Google) | c3bc8fd | 2018-07-30 15:24:23 -0700 | [diff] [blame] | 51 | } |
| 52 | EXPORT_SYMBOL(trace_hardirqs_on); |
Masami Hiramatsu | eeeb080 | 2019-02-13 01:13:40 +0900 | [diff] [blame] | 53 | NOKPROBE_SYMBOL(trace_hardirqs_on); |
Joel Fernandes (Google) | c3bc8fd | 2018-07-30 15:24:23 -0700 | [diff] [blame] | 54 | |
Thomas Gleixner | 0995a5d | 2020-03-04 13:09:50 +0100 | [diff] [blame] | 55 | /* |
| 56 | * Like trace_hardirqs_off() but without the lockdep invocation. This is |
| 57 | * used in the low level entry code where the ordering vs. RCU is important |
| 58 | * and lockdep uses a staged approach which splits the lockdep hardirq |
| 59 | * tracking into a RCU on and a RCU off section. |
| 60 | */ |
Peter Zijlstra | bf2b300 | 2020-05-29 23:27:40 +0200 | [diff] [blame] | 61 | void trace_hardirqs_off_finish(void) |
Thomas Gleixner | 0995a5d | 2020-03-04 13:09:50 +0100 | [diff] [blame] | 62 | { |
| 63 | if (!this_cpu_read(tracing_irq_cpu)) { |
| 64 | this_cpu_write(tracing_irq_cpu, 1); |
| 65 | tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1); |
| 66 | if (!in_nmi()) |
| 67 | trace_irq_disable(CALLER_ADDR0, CALLER_ADDR1); |
| 68 | } |
| 69 | |
| 70 | } |
Peter Zijlstra | bf2b300 | 2020-05-29 23:27:40 +0200 | [diff] [blame] | 71 | EXPORT_SYMBOL(trace_hardirqs_off_finish); |
| 72 | NOKPROBE_SYMBOL(trace_hardirqs_off_finish); |
Thomas Gleixner | 0995a5d | 2020-03-04 13:09:50 +0100 | [diff] [blame] | 73 | |
Joel Fernandes (Google) | c3bc8fd | 2018-07-30 15:24:23 -0700 | [diff] [blame] | 74 | void trace_hardirqs_off(void) |
| 75 | { |
Peter Zijlstra | bf2b300 | 2020-05-29 23:27:40 +0200 | [diff] [blame] | 76 | lockdep_hardirqs_off(CALLER_ADDR0); |
| 77 | |
Steven Rostedt (VMware) | bff1b20 | 2018-08-06 15:50:58 -0400 | [diff] [blame] | 78 | if (!this_cpu_read(tracing_irq_cpu)) { |
| 79 | this_cpu_write(tracing_irq_cpu, 1); |
Steven Rostedt (VMware) | 3f1756d | 2018-08-08 21:28:05 -0400 | [diff] [blame] | 80 | tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1); |
| 81 | if (!in_nmi()) |
| 82 | trace_irq_disable_rcuidle(CALLER_ADDR0, CALLER_ADDR1); |
Steven Rostedt (VMware) | bff1b20 | 2018-08-06 15:50:58 -0400 | [diff] [blame] | 83 | } |
Joel Fernandes (Google) | c3bc8fd | 2018-07-30 15:24:23 -0700 | [diff] [blame] | 84 | } |
| 85 | EXPORT_SYMBOL(trace_hardirqs_off); |
Masami Hiramatsu | eeeb080 | 2019-02-13 01:13:40 +0900 | [diff] [blame] | 86 | NOKPROBE_SYMBOL(trace_hardirqs_off); |
Joel Fernandes (Google) | c3bc8fd | 2018-07-30 15:24:23 -0700 | [diff] [blame] | 87 | |
| 88 | __visible void trace_hardirqs_on_caller(unsigned long caller_addr) |
| 89 | { |
Steven Rostedt (VMware) | bff1b20 | 2018-08-06 15:50:58 -0400 | [diff] [blame] | 90 | if (this_cpu_read(tracing_irq_cpu)) { |
Steven Rostedt (VMware) | 3f1756d | 2018-08-08 21:28:05 -0400 | [diff] [blame] | 91 | if (!in_nmi()) |
| 92 | trace_irq_enable_rcuidle(CALLER_ADDR0, caller_addr); |
| 93 | tracer_hardirqs_on(CALLER_ADDR0, caller_addr); |
Steven Rostedt (VMware) | bff1b20 | 2018-08-06 15:50:58 -0400 | [diff] [blame] | 94 | this_cpu_write(tracing_irq_cpu, 0); |
| 95 | } |
Joel Fernandes (Google) | c3bc8fd | 2018-07-30 15:24:23 -0700 | [diff] [blame] | 96 | |
Peter Zijlstra | c86e9b9 | 2020-03-18 14:22:03 +0100 | [diff] [blame] | 97 | lockdep_hardirqs_on_prepare(CALLER_ADDR0); |
Steven Rostedt (VMware) | bff1b20 | 2018-08-06 15:50:58 -0400 | [diff] [blame] | 98 | lockdep_hardirqs_on(CALLER_ADDR0); |
Joel Fernandes (Google) | c3bc8fd | 2018-07-30 15:24:23 -0700 | [diff] [blame] | 99 | } |
| 100 | EXPORT_SYMBOL(trace_hardirqs_on_caller); |
Masami Hiramatsu | eeeb080 | 2019-02-13 01:13:40 +0900 | [diff] [blame] | 101 | NOKPROBE_SYMBOL(trace_hardirqs_on_caller); |
Joel Fernandes (Google) | c3bc8fd | 2018-07-30 15:24:23 -0700 | [diff] [blame] | 102 | |
| 103 | __visible void trace_hardirqs_off_caller(unsigned long caller_addr) |
| 104 | { |
Sven Schnelle | 73ac74c | 2020-09-10 12:24:53 +0200 | [diff] [blame] | 105 | lockdep_hardirqs_off(CALLER_ADDR0); |
| 106 | |
Steven Rostedt (VMware) | bff1b20 | 2018-08-06 15:50:58 -0400 | [diff] [blame] | 107 | if (!this_cpu_read(tracing_irq_cpu)) { |
| 108 | this_cpu_write(tracing_irq_cpu, 1); |
Steven Rostedt (VMware) | 3f1756d | 2018-08-08 21:28:05 -0400 | [diff] [blame] | 109 | tracer_hardirqs_off(CALLER_ADDR0, caller_addr); |
| 110 | if (!in_nmi()) |
| 111 | trace_irq_disable_rcuidle(CALLER_ADDR0, caller_addr); |
Steven Rostedt (VMware) | bff1b20 | 2018-08-06 15:50:58 -0400 | [diff] [blame] | 112 | } |
Joel Fernandes (Google) | c3bc8fd | 2018-07-30 15:24:23 -0700 | [diff] [blame] | 113 | } |
| 114 | EXPORT_SYMBOL(trace_hardirqs_off_caller); |
Masami Hiramatsu | eeeb080 | 2019-02-13 01:13:40 +0900 | [diff] [blame] | 115 | NOKPROBE_SYMBOL(trace_hardirqs_off_caller); |
Joel Fernandes (Google) | c3bc8fd | 2018-07-30 15:24:23 -0700 | [diff] [blame] | 116 | #endif /* CONFIG_TRACE_IRQFLAGS */ |
| 117 | |
| 118 | #ifdef CONFIG_TRACE_PREEMPT_TOGGLE |
| 119 | |
| 120 | void trace_preempt_on(unsigned long a0, unsigned long a1) |
| 121 | { |
Steven Rostedt (VMware) | 3f1756d | 2018-08-08 21:28:05 -0400 | [diff] [blame] | 122 | if (!in_nmi()) |
| 123 | trace_preempt_enable_rcuidle(a0, a1); |
| 124 | tracer_preempt_on(a0, a1); |
Joel Fernandes (Google) | c3bc8fd | 2018-07-30 15:24:23 -0700 | [diff] [blame] | 125 | } |
| 126 | |
| 127 | void trace_preempt_off(unsigned long a0, unsigned long a1) |
| 128 | { |
Steven Rostedt (VMware) | 3f1756d | 2018-08-08 21:28:05 -0400 | [diff] [blame] | 129 | if (!in_nmi()) |
| 130 | trace_preempt_disable_rcuidle(a0, a1); |
| 131 | tracer_preempt_off(a0, a1); |
Joel Fernandes (Google) | c3bc8fd | 2018-07-30 15:24:23 -0700 | [diff] [blame] | 132 | } |
| 133 | #endif |