perf counters: protect them against CSTATE transitions
Impact: fix rare lost events problem
There are CPUs whose performance counters misbehave on CSTATE transitions,
so provide a way to just disable/enable them around deep idle methods.
(hw_perf_enable_all() is cheap on x86.)
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
diff --git a/arch/x86/kernel/cpu/perf_counter.c b/arch/x86/kernel/cpu/perf_counter.c
index 6a93d1f..0a7f3be 100644
--- a/arch/x86/kernel/cpu/perf_counter.c
+++ b/arch/x86/kernel/cpu/perf_counter.c
@@ -12,6 +12,7 @@
#include <linux/notifier.h>
#include <linux/hardirq.h>
#include <linux/kprobes.h>
+#include <linux/module.h>
#include <linux/kdebug.h>
#include <linux/sched.h>
@@ -119,10 +120,21 @@
wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, perf_counter_mask, 0);
}
-void hw_perf_disable_all(void)
+void hw_perf_restore_ctrl(u64 ctrl)
{
- wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0, 0);
+ wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, ctrl, 0);
}
+EXPORT_SYMBOL_GPL(hw_perf_restore_ctrl);
+
+u64 hw_perf_disable_all(void)
+{
+ u64 ctrl;
+
+ rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
+ wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0, 0);
+ return ctrl;
+}
+EXPORT_SYMBOL_GPL(hw_perf_disable_all);
static inline void
__hw_perf_counter_disable(struct hw_perf_counter *hwc, unsigned int idx)
diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c
index 5f8d746..cca804e 100644
--- a/drivers/acpi/processor_idle.c
+++ b/drivers/acpi/processor_idle.c
@@ -270,8 +270,11 @@
/* Common C-state entry for C2, C3, .. */
static void acpi_cstate_enter(struct acpi_processor_cx *cstate)
{
+ u64 pctrl;
+
/* Don't trace irqs off for idle */
stop_critical_timings();
+ pctrl = hw_perf_disable_all();
if (cstate->entry_method == ACPI_CSTATE_FFH) {
/* Call into architectural FFH based C-state */
acpi_processor_ffh_cstate_enter(cstate);
@@ -284,6 +287,7 @@
gets asserted in time to freeze execution properly. */
unused = inl(acpi_gbl_FADT.xpm_timer_block.address);
}
+ hw_perf_restore_ctrl(pctrl);
start_critical_timings();
}
#endif /* !CONFIG_CPU_IDLE */
@@ -1425,8 +1429,11 @@
*/
static inline void acpi_idle_do_entry(struct acpi_processor_cx *cx)
{
+ u64 pctrl;
+
/* Don't trace irqs off for idle */
stop_critical_timings();
+ pctrl = hw_perf_disable_all();
if (cx->entry_method == ACPI_CSTATE_FFH) {
/* Call into architectural FFH based C-state */
acpi_processor_ffh_cstate_enter(cx);
@@ -1441,6 +1448,7 @@
gets asserted in time to freeze execution properly. */
unused = inl(acpi_gbl_FADT.xpm_timer_block.address);
}
+ hw_perf_restore_ctrl(pctrl);
start_critical_timings();
}
diff --git a/include/linux/perf_counter.h b/include/linux/perf_counter.h
index 22c4469..5031b56 100644
--- a/include/linux/perf_counter.h
+++ b/include/linux/perf_counter.h
@@ -156,6 +156,8 @@
extern void perf_counter_init_task(struct task_struct *task);
extern void perf_counter_notify(struct pt_regs *regs);
extern void perf_counter_print_debug(void);
+extern void hw_perf_restore_ctrl(u64 ctrl);
+extern u64 hw_perf_disable_all(void);
#else
static inline void
perf_counter_task_sched_in(struct task_struct *task, int cpu) { }
@@ -166,6 +168,8 @@
static inline void perf_counter_init_task(struct task_struct *task) { }
static inline void perf_counter_notify(struct pt_regs *regs) { }
static inline void perf_counter_print_debug(void) { }
+static inline void hw_perf_restore_ctrl(u64 ctrl) { }
+static inline u64 hw_perf_disable_all(void) { return 0; }
#endif
#endif /* _LINUX_PERF_COUNTER_H */