Thomas Gleixner | 2025cf9 | 2019-05-29 07:18:02 -0700 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-only |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 2 | /* |
| 3 | * Intel(R) Processor Trace PMU driver for perf |
| 4 | * Copyright (c) 2013-2014, Intel Corporation. |
| 5 | * |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 6 | * Intel PT is specified in the Intel Architecture Instruction Set Extensions |
| 7 | * Programming Reference: |
| 8 | * http://software.intel.com/en-us/intel-isa-extensions |
| 9 | */ |
| 10 | |
| 11 | #undef DEBUG |
| 12 | |
| 13 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
| 14 | |
| 15 | #include <linux/types.h> |
| 16 | #include <linux/slab.h> |
| 17 | #include <linux/device.h> |
| 18 | |
| 19 | #include <asm/perf_event.h> |
| 20 | #include <asm/insn.h> |
| 21 | #include <asm/io.h> |
Takao Indoh | 24cc12b | 2015-11-04 14:22:32 +0900 | [diff] [blame] | 22 | #include <asm/intel_pt.h> |
Alexander Shishkin | d35869b | 2017-02-06 16:41:40 +0200 | [diff] [blame] | 23 | #include <asm/intel-family.h> |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 24 | |
Borislav Petkov | 27f6d22 | 2016-02-10 10:55:23 +0100 | [diff] [blame] | 25 | #include "../perf_event.h" |
Borislav Petkov | fd1c601 | 2016-02-10 10:55:13 +0100 | [diff] [blame] | 26 | #include "pt.h" |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 27 | |
| 28 | static DEFINE_PER_CPU(struct pt, pt_ctx); |
| 29 | |
| 30 | static struct pt_pmu pt_pmu; |
| 31 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 32 | /* |
| 33 | * Capabilities of Intel PT hardware, such as number of address bits or |
| 34 | * supported output schemes, are cached and exported to userspace as "caps" |
| 35 | * attribute group of pt pmu device |
| 36 | * (/sys/bus/event_source/devices/intel_pt/caps/) so that userspace can store |
| 37 | * relevant bits together with intel_pt traces. |
| 38 | * |
| 39 | * These are necessary for both trace decoding (payloads_lip, contains address |
| 40 | * width encoded in IP-related packets), and event configuration (bitmasks with |
| 41 | * permitted values for certain bit fields). |
| 42 | */ |
| 43 | #define PT_CAP(_n, _l, _r, _m) \ |
| 44 | [PT_CAP_ ## _n] = { .name = __stringify(_n), .leaf = _l, \ |
| 45 | .reg = _r, .mask = _m } |
| 46 | |
| 47 | static struct pt_cap_desc { |
| 48 | const char *name; |
| 49 | u32 leaf; |
| 50 | u8 reg; |
| 51 | u32 mask; |
| 52 | } pt_caps[] = { |
He Chen | 47f10a3 | 2016-11-11 17:25:34 +0800 | [diff] [blame] | 53 | PT_CAP(max_subleaf, 0, CPUID_EAX, 0xffffffff), |
| 54 | PT_CAP(cr3_filtering, 0, CPUID_EBX, BIT(0)), |
| 55 | PT_CAP(psb_cyc, 0, CPUID_EBX, BIT(1)), |
| 56 | PT_CAP(ip_filtering, 0, CPUID_EBX, BIT(2)), |
| 57 | PT_CAP(mtc, 0, CPUID_EBX, BIT(3)), |
| 58 | PT_CAP(ptwrite, 0, CPUID_EBX, BIT(4)), |
| 59 | PT_CAP(power_event_trace, 0, CPUID_EBX, BIT(5)), |
| 60 | PT_CAP(topa_output, 0, CPUID_ECX, BIT(0)), |
| 61 | PT_CAP(topa_multiple_entries, 0, CPUID_ECX, BIT(1)), |
| 62 | PT_CAP(single_range_output, 0, CPUID_ECX, BIT(2)), |
Luwei Kang | e0018af | 2018-10-24 16:05:09 +0800 | [diff] [blame] | 63 | PT_CAP(output_subsys, 0, CPUID_ECX, BIT(3)), |
He Chen | 47f10a3 | 2016-11-11 17:25:34 +0800 | [diff] [blame] | 64 | PT_CAP(payloads_lip, 0, CPUID_ECX, BIT(31)), |
Xiaoyao Li | c53c6b7 | 2021-08-24 12:06:22 +0800 | [diff] [blame] | 65 | PT_CAP(num_address_ranges, 1, CPUID_EAX, 0x7), |
He Chen | 47f10a3 | 2016-11-11 17:25:34 +0800 | [diff] [blame] | 66 | PT_CAP(mtc_periods, 1, CPUID_EAX, 0xffff0000), |
| 67 | PT_CAP(cycle_thresholds, 1, CPUID_EBX, 0xffff), |
| 68 | PT_CAP(psb_periods, 1, CPUID_EBX, 0xffff0000), |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 69 | }; |
| 70 | |
Luwei Kang | 61be299 | 2018-10-24 16:05:07 +0800 | [diff] [blame] | 71 | u32 intel_pt_validate_cap(u32 *caps, enum pt_capabilities capability) |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 72 | { |
Luwei Kang | 61be299 | 2018-10-24 16:05:07 +0800 | [diff] [blame] | 73 | struct pt_cap_desc *cd = &pt_caps[capability]; |
| 74 | u32 c = caps[cd->leaf * PT_CPUID_REGS_NUM + cd->reg]; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 75 | unsigned int shift = __ffs(cd->mask); |
| 76 | |
| 77 | return (c & cd->mask) >> shift; |
| 78 | } |
Luwei Kang | 61be299 | 2018-10-24 16:05:07 +0800 | [diff] [blame] | 79 | EXPORT_SYMBOL_GPL(intel_pt_validate_cap); |
| 80 | |
| 81 | u32 intel_pt_validate_hw_cap(enum pt_capabilities cap) |
| 82 | { |
| 83 | return intel_pt_validate_cap(pt_pmu.caps, cap); |
| 84 | } |
Chao Peng | f6d079c | 2018-10-24 16:05:06 +0800 | [diff] [blame] | 85 | EXPORT_SYMBOL_GPL(intel_pt_validate_hw_cap); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 86 | |
| 87 | static ssize_t pt_cap_show(struct device *cdev, |
| 88 | struct device_attribute *attr, |
| 89 | char *buf) |
| 90 | { |
| 91 | struct dev_ext_attribute *ea = |
| 92 | container_of(attr, struct dev_ext_attribute, attr); |
| 93 | enum pt_capabilities cap = (long)ea->var; |
| 94 | |
Chao Peng | f6d079c | 2018-10-24 16:05:06 +0800 | [diff] [blame] | 95 | return snprintf(buf, PAGE_SIZE, "%x\n", intel_pt_validate_hw_cap(cap)); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 96 | } |
| 97 | |
Zubin Mithra | 49e7324 | 2018-09-12 09:45:10 -0700 | [diff] [blame] | 98 | static struct attribute_group pt_cap_group __ro_after_init = { |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 99 | .name = "caps", |
| 100 | }; |
| 101 | |
Alexander Shishkin | d35869b | 2017-02-06 16:41:40 +0200 | [diff] [blame] | 102 | PMU_FORMAT_ATTR(pt, "config:0" ); |
Alexander Shishkin | b1bf72d | 2015-07-30 16:15:31 +0300 | [diff] [blame] | 103 | PMU_FORMAT_ATTR(cyc, "config:1" ); |
Alexander Shishkin | 5443624 | 2017-01-27 17:16:43 +0200 | [diff] [blame] | 104 | PMU_FORMAT_ATTR(pwr_evt, "config:4" ); |
| 105 | PMU_FORMAT_ATTR(fup_on_ptw, "config:5" ); |
Alexander Shishkin | b1bf72d | 2015-07-30 16:15:31 +0300 | [diff] [blame] | 106 | PMU_FORMAT_ATTR(mtc, "config:9" ); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 107 | PMU_FORMAT_ATTR(tsc, "config:10" ); |
| 108 | PMU_FORMAT_ATTR(noretcomp, "config:11" ); |
Alexander Shishkin | 5443624 | 2017-01-27 17:16:43 +0200 | [diff] [blame] | 109 | PMU_FORMAT_ATTR(ptw, "config:12" ); |
Alexander Shishkin | d35869b | 2017-02-06 16:41:40 +0200 | [diff] [blame] | 110 | PMU_FORMAT_ATTR(branch, "config:13" ); |
Alexander Shishkin | b1bf72d | 2015-07-30 16:15:31 +0300 | [diff] [blame] | 111 | PMU_FORMAT_ATTR(mtc_period, "config:14-17" ); |
| 112 | PMU_FORMAT_ATTR(cyc_thresh, "config:19-22" ); |
| 113 | PMU_FORMAT_ATTR(psb_period, "config:24-27" ); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 114 | |
| 115 | static struct attribute *pt_formats_attr[] = { |
Alexander Shishkin | d35869b | 2017-02-06 16:41:40 +0200 | [diff] [blame] | 116 | &format_attr_pt.attr, |
Alexander Shishkin | b1bf72d | 2015-07-30 16:15:31 +0300 | [diff] [blame] | 117 | &format_attr_cyc.attr, |
Alexander Shishkin | 5443624 | 2017-01-27 17:16:43 +0200 | [diff] [blame] | 118 | &format_attr_pwr_evt.attr, |
| 119 | &format_attr_fup_on_ptw.attr, |
Alexander Shishkin | b1bf72d | 2015-07-30 16:15:31 +0300 | [diff] [blame] | 120 | &format_attr_mtc.attr, |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 121 | &format_attr_tsc.attr, |
| 122 | &format_attr_noretcomp.attr, |
Alexander Shishkin | 5443624 | 2017-01-27 17:16:43 +0200 | [diff] [blame] | 123 | &format_attr_ptw.attr, |
Alexander Shishkin | d35869b | 2017-02-06 16:41:40 +0200 | [diff] [blame] | 124 | &format_attr_branch.attr, |
Alexander Shishkin | b1bf72d | 2015-07-30 16:15:31 +0300 | [diff] [blame] | 125 | &format_attr_mtc_period.attr, |
| 126 | &format_attr_cyc_thresh.attr, |
| 127 | &format_attr_psb_period.attr, |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 128 | NULL, |
| 129 | }; |
| 130 | |
| 131 | static struct attribute_group pt_format_group = { |
| 132 | .name = "format", |
| 133 | .attrs = pt_formats_attr, |
| 134 | }; |
| 135 | |
Alexander Shishkin | 65c7e6f | 2015-08-19 17:02:10 +0300 | [diff] [blame] | 136 | static ssize_t |
| 137 | pt_timing_attr_show(struct device *dev, struct device_attribute *attr, |
| 138 | char *page) |
| 139 | { |
| 140 | struct perf_pmu_events_attr *pmu_attr = |
| 141 | container_of(attr, struct perf_pmu_events_attr, attr); |
| 142 | |
| 143 | switch (pmu_attr->id) { |
| 144 | case 0: |
| 145 | return sprintf(page, "%lu\n", pt_pmu.max_nonturbo_ratio); |
| 146 | case 1: |
| 147 | return sprintf(page, "%u:%u\n", |
| 148 | pt_pmu.tsc_art_num, |
| 149 | pt_pmu.tsc_art_den); |
| 150 | default: |
| 151 | break; |
| 152 | } |
| 153 | |
| 154 | return -EINVAL; |
| 155 | } |
| 156 | |
| 157 | PMU_EVENT_ATTR(max_nonturbo_ratio, timing_attr_max_nonturbo_ratio, 0, |
| 158 | pt_timing_attr_show); |
| 159 | PMU_EVENT_ATTR(tsc_art_ratio, timing_attr_tsc_art_ratio, 1, |
| 160 | pt_timing_attr_show); |
| 161 | |
| 162 | static struct attribute *pt_timing_attr[] = { |
| 163 | &timing_attr_max_nonturbo_ratio.attr.attr, |
| 164 | &timing_attr_tsc_art_ratio.attr.attr, |
| 165 | NULL, |
| 166 | }; |
| 167 | |
| 168 | static struct attribute_group pt_timing_group = { |
| 169 | .attrs = pt_timing_attr, |
| 170 | }; |
| 171 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 172 | static const struct attribute_group *pt_attr_groups[] = { |
| 173 | &pt_cap_group, |
| 174 | &pt_format_group, |
Alexander Shishkin | 65c7e6f | 2015-08-19 17:02:10 +0300 | [diff] [blame] | 175 | &pt_timing_group, |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 176 | NULL, |
| 177 | }; |
| 178 | |
| 179 | static int __init pt_pmu_hw_init(void) |
| 180 | { |
| 181 | struct dev_ext_attribute *de_attrs; |
| 182 | struct attribute **attrs; |
| 183 | size_t size; |
Alexander Shishkin | 1c5ac21 | 2016-03-29 17:43:10 +0300 | [diff] [blame] | 184 | u64 reg; |
Ingo Molnar | 066450b | 2015-04-12 11:11:21 +0200 | [diff] [blame] | 185 | int ret; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 186 | long i; |
| 187 | |
Alexander Shishkin | 65c7e6f | 2015-08-19 17:02:10 +0300 | [diff] [blame] | 188 | rdmsrl(MSR_PLATFORM_INFO, reg); |
| 189 | pt_pmu.max_nonturbo_ratio = (reg & 0xff00) >> 8; |
| 190 | |
| 191 | /* |
| 192 | * if available, read in TSC to core crystal clock ratio, |
| 193 | * otherwise, zero for numerator stands for "not enumerated" |
| 194 | * as per SDM |
| 195 | */ |
| 196 | if (boot_cpu_data.cpuid_level >= CPUID_TSC_LEAF) { |
| 197 | u32 eax, ebx, ecx, edx; |
| 198 | |
| 199 | cpuid(CPUID_TSC_LEAF, &eax, &ebx, &ecx, &edx); |
| 200 | |
| 201 | pt_pmu.tsc_art_num = ebx; |
| 202 | pt_pmu.tsc_art_den = eax; |
| 203 | } |
| 204 | |
Alexander Shishkin | d35869b | 2017-02-06 16:41:40 +0200 | [diff] [blame] | 205 | /* model-specific quirks */ |
| 206 | switch (boot_cpu_data.x86_model) { |
Peter Zijlstra | c66f78a | 2019-08-27 21:48:21 +0200 | [diff] [blame] | 207 | case INTEL_FAM6_BROADWELL: |
Peter Zijlstra | 5ebb34e | 2019-08-27 21:48:24 +0200 | [diff] [blame] | 208 | case INTEL_FAM6_BROADWELL_D: |
Peter Zijlstra | 5e74140 | 2019-08-27 21:48:23 +0200 | [diff] [blame] | 209 | case INTEL_FAM6_BROADWELL_G: |
Alexander Shishkin | d35869b | 2017-02-06 16:41:40 +0200 | [diff] [blame] | 210 | case INTEL_FAM6_BROADWELL_X: |
| 211 | /* not setting BRANCH_EN will #GP, erratum BDM106 */ |
| 212 | pt_pmu.branch_en_always_on = true; |
| 213 | break; |
| 214 | default: |
| 215 | break; |
| 216 | } |
| 217 | |
Alexander Shishkin | 1c5ac21 | 2016-03-29 17:43:10 +0300 | [diff] [blame] | 218 | if (boot_cpu_has(X86_FEATURE_VMX)) { |
| 219 | /* |
| 220 | * Intel SDM, 36.5 "Tracing post-VMXON" says that |
| 221 | * "IA32_VMX_MISC[bit 14]" being 1 means PT can trace |
| 222 | * post-VMXON. |
| 223 | */ |
| 224 | rdmsrl(MSR_IA32_VMX_MISC, reg); |
| 225 | if (reg & BIT(14)) |
| 226 | pt_pmu.vmx = true; |
| 227 | } |
| 228 | |
Ingo Molnar | 066450b | 2015-04-12 11:11:21 +0200 | [diff] [blame] | 229 | for (i = 0; i < PT_CPUID_LEAVES; i++) { |
| 230 | cpuid_count(20, i, |
He Chen | 47f10a3 | 2016-11-11 17:25:34 +0800 | [diff] [blame] | 231 | &pt_pmu.caps[CPUID_EAX + i*PT_CPUID_REGS_NUM], |
| 232 | &pt_pmu.caps[CPUID_EBX + i*PT_CPUID_REGS_NUM], |
| 233 | &pt_pmu.caps[CPUID_ECX + i*PT_CPUID_REGS_NUM], |
| 234 | &pt_pmu.caps[CPUID_EDX + i*PT_CPUID_REGS_NUM]); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 235 | } |
| 236 | |
Ingo Molnar | 066450b | 2015-04-12 11:11:21 +0200 | [diff] [blame] | 237 | ret = -ENOMEM; |
| 238 | size = sizeof(struct attribute *) * (ARRAY_SIZE(pt_caps)+1); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 239 | attrs = kzalloc(size, GFP_KERNEL); |
| 240 | if (!attrs) |
Ingo Molnar | 066450b | 2015-04-12 11:11:21 +0200 | [diff] [blame] | 241 | goto fail; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 242 | |
Ingo Molnar | 066450b | 2015-04-12 11:11:21 +0200 | [diff] [blame] | 243 | size = sizeof(struct dev_ext_attribute) * (ARRAY_SIZE(pt_caps)+1); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 244 | de_attrs = kzalloc(size, GFP_KERNEL); |
| 245 | if (!de_attrs) |
Ingo Molnar | 066450b | 2015-04-12 11:11:21 +0200 | [diff] [blame] | 246 | goto fail; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 247 | |
| 248 | for (i = 0; i < ARRAY_SIZE(pt_caps); i++) { |
Ingo Molnar | 066450b | 2015-04-12 11:11:21 +0200 | [diff] [blame] | 249 | struct dev_ext_attribute *de_attr = de_attrs + i; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 250 | |
Ingo Molnar | 066450b | 2015-04-12 11:11:21 +0200 | [diff] [blame] | 251 | de_attr->attr.attr.name = pt_caps[i].name; |
| 252 | |
Alexander Shishkin | b44a2b5 | 2015-06-04 16:31:47 +0300 | [diff] [blame] | 253 | sysfs_attr_init(&de_attr->attr.attr); |
Ingo Molnar | 066450b | 2015-04-12 11:11:21 +0200 | [diff] [blame] | 254 | |
| 255 | de_attr->attr.attr.mode = S_IRUGO; |
| 256 | de_attr->attr.show = pt_cap_show; |
| 257 | de_attr->var = (void *)i; |
| 258 | |
| 259 | attrs[i] = &de_attr->attr.attr; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 260 | } |
| 261 | |
| 262 | pt_cap_group.attrs = attrs; |
Ingo Molnar | 066450b | 2015-04-12 11:11:21 +0200 | [diff] [blame] | 263 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 264 | return 0; |
| 265 | |
Ingo Molnar | 066450b | 2015-04-12 11:11:21 +0200 | [diff] [blame] | 266 | fail: |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 267 | kfree(attrs); |
| 268 | |
Ingo Molnar | 066450b | 2015-04-12 11:11:21 +0200 | [diff] [blame] | 269 | return ret; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 270 | } |
| 271 | |
Alexander Shishkin | b1bf72d | 2015-07-30 16:15:31 +0300 | [diff] [blame] | 272 | #define RTIT_CTL_CYC_PSB (RTIT_CTL_CYCLEACC | \ |
| 273 | RTIT_CTL_CYC_THRESH | \ |
| 274 | RTIT_CTL_PSB_FREQ) |
| 275 | |
| 276 | #define RTIT_CTL_MTC (RTIT_CTL_MTC_EN | \ |
| 277 | RTIT_CTL_MTC_RANGE) |
| 278 | |
Alexander Shishkin | 8ee83b2 | 2016-09-16 16:48:19 +0300 | [diff] [blame] | 279 | #define RTIT_CTL_PTW (RTIT_CTL_PTW_EN | \ |
| 280 | RTIT_CTL_FUP_ON_PTW) |
| 281 | |
Alexander Shishkin | d35869b | 2017-02-06 16:41:40 +0200 | [diff] [blame] | 282 | /* |
| 283 | * Bit 0 (TraceEn) in the attr.config is meaningless as the |
| 284 | * corresponding bit in the RTIT_CTL can only be controlled |
| 285 | * by the driver; therefore, repurpose it to mean: pass |
| 286 | * through the bit that was previously assumed to be always |
| 287 | * on for PT, thereby allowing the user to *not* set it if |
| 288 | * they so wish. See also pt_event_valid() and pt_config(). |
| 289 | */ |
| 290 | #define RTIT_CTL_PASSTHROUGH RTIT_CTL_TRACEEN |
| 291 | |
| 292 | #define PT_CONFIG_MASK (RTIT_CTL_TRACEEN | \ |
| 293 | RTIT_CTL_TSC_EN | \ |
Alexander Shishkin | b1bf72d | 2015-07-30 16:15:31 +0300 | [diff] [blame] | 294 | RTIT_CTL_DISRETC | \ |
Alexander Shishkin | d35869b | 2017-02-06 16:41:40 +0200 | [diff] [blame] | 295 | RTIT_CTL_BRANCH_EN | \ |
Alexander Shishkin | b1bf72d | 2015-07-30 16:15:31 +0300 | [diff] [blame] | 296 | RTIT_CTL_CYC_PSB | \ |
Alexander Shishkin | 8ee83b2 | 2016-09-16 16:48:19 +0300 | [diff] [blame] | 297 | RTIT_CTL_MTC | \ |
| 298 | RTIT_CTL_PWR_EVT_EN | \ |
| 299 | RTIT_CTL_FUP_ON_PTW | \ |
| 300 | RTIT_CTL_PTW_EN) |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 301 | |
| 302 | static bool pt_event_valid(struct perf_event *event) |
| 303 | { |
| 304 | u64 config = event->attr.config; |
Alexander Shishkin | b1bf72d | 2015-07-30 16:15:31 +0300 | [diff] [blame] | 305 | u64 allowed, requested; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 306 | |
| 307 | if ((config & PT_CONFIG_MASK) != config) |
| 308 | return false; |
| 309 | |
Alexander Shishkin | b1bf72d | 2015-07-30 16:15:31 +0300 | [diff] [blame] | 310 | if (config & RTIT_CTL_CYC_PSB) { |
Chao Peng | f6d079c | 2018-10-24 16:05:06 +0800 | [diff] [blame] | 311 | if (!intel_pt_validate_hw_cap(PT_CAP_psb_cyc)) |
Alexander Shishkin | b1bf72d | 2015-07-30 16:15:31 +0300 | [diff] [blame] | 312 | return false; |
| 313 | |
Chao Peng | f6d079c | 2018-10-24 16:05:06 +0800 | [diff] [blame] | 314 | allowed = intel_pt_validate_hw_cap(PT_CAP_psb_periods); |
Alexander Shishkin | b1bf72d | 2015-07-30 16:15:31 +0300 | [diff] [blame] | 315 | requested = (config & RTIT_CTL_PSB_FREQ) >> |
| 316 | RTIT_CTL_PSB_FREQ_OFFSET; |
| 317 | if (requested && (!(allowed & BIT(requested)))) |
| 318 | return false; |
| 319 | |
Chao Peng | f6d079c | 2018-10-24 16:05:06 +0800 | [diff] [blame] | 320 | allowed = intel_pt_validate_hw_cap(PT_CAP_cycle_thresholds); |
Alexander Shishkin | b1bf72d | 2015-07-30 16:15:31 +0300 | [diff] [blame] | 321 | requested = (config & RTIT_CTL_CYC_THRESH) >> |
| 322 | RTIT_CTL_CYC_THRESH_OFFSET; |
| 323 | if (requested && (!(allowed & BIT(requested)))) |
| 324 | return false; |
| 325 | } |
| 326 | |
| 327 | if (config & RTIT_CTL_MTC) { |
| 328 | /* |
| 329 | * In the unlikely case that CPUID lists valid mtc periods, |
| 330 | * but not the mtc capability, drop out here. |
| 331 | * |
| 332 | * Spec says that setting mtc period bits while mtc bit in |
| 333 | * CPUID is 0 will #GP, so better safe than sorry. |
| 334 | */ |
Chao Peng | f6d079c | 2018-10-24 16:05:06 +0800 | [diff] [blame] | 335 | if (!intel_pt_validate_hw_cap(PT_CAP_mtc)) |
Alexander Shishkin | b1bf72d | 2015-07-30 16:15:31 +0300 | [diff] [blame] | 336 | return false; |
| 337 | |
Chao Peng | f6d079c | 2018-10-24 16:05:06 +0800 | [diff] [blame] | 338 | allowed = intel_pt_validate_hw_cap(PT_CAP_mtc_periods); |
Alexander Shishkin | b1bf72d | 2015-07-30 16:15:31 +0300 | [diff] [blame] | 339 | if (!allowed) |
| 340 | return false; |
| 341 | |
| 342 | requested = (config & RTIT_CTL_MTC_RANGE) >> |
| 343 | RTIT_CTL_MTC_RANGE_OFFSET; |
| 344 | |
| 345 | if (!(allowed & BIT(requested))) |
| 346 | return false; |
| 347 | } |
| 348 | |
Alexander Shishkin | 8ee83b2 | 2016-09-16 16:48:19 +0300 | [diff] [blame] | 349 | if (config & RTIT_CTL_PWR_EVT_EN && |
Chao Peng | f6d079c | 2018-10-24 16:05:06 +0800 | [diff] [blame] | 350 | !intel_pt_validate_hw_cap(PT_CAP_power_event_trace)) |
Alexander Shishkin | 8ee83b2 | 2016-09-16 16:48:19 +0300 | [diff] [blame] | 351 | return false; |
| 352 | |
| 353 | if (config & RTIT_CTL_PTW) { |
Chao Peng | f6d079c | 2018-10-24 16:05:06 +0800 | [diff] [blame] | 354 | if (!intel_pt_validate_hw_cap(PT_CAP_ptwrite)) |
Alexander Shishkin | 8ee83b2 | 2016-09-16 16:48:19 +0300 | [diff] [blame] | 355 | return false; |
| 356 | |
| 357 | /* FUPonPTW without PTW doesn't make sense */ |
| 358 | if ((config & RTIT_CTL_FUP_ON_PTW) && |
| 359 | !(config & RTIT_CTL_PTW_EN)) |
| 360 | return false; |
| 361 | } |
| 362 | |
Alexander Shishkin | d35869b | 2017-02-06 16:41:40 +0200 | [diff] [blame] | 363 | /* |
| 364 | * Setting bit 0 (TraceEn in RTIT_CTL MSR) in the attr.config |
Ingo Molnar | d9f6e12 | 2021-03-18 15:28:01 +0100 | [diff] [blame] | 365 | * clears the assumption that BranchEn must always be enabled, |
Alexander Shishkin | d35869b | 2017-02-06 16:41:40 +0200 | [diff] [blame] | 366 | * as was the case with the first implementation of PT. |
| 367 | * If this bit is not set, the legacy behavior is preserved |
| 368 | * for compatibility with the older userspace. |
| 369 | * |
| 370 | * Re-using bit 0 for this purpose is fine because it is never |
| 371 | * directly set by the user; previous attempts at setting it in |
| 372 | * the attr.config resulted in -EINVAL. |
| 373 | */ |
| 374 | if (config & RTIT_CTL_PASSTHROUGH) { |
| 375 | /* |
| 376 | * Disallow not setting BRANCH_EN where BRANCH_EN is |
| 377 | * always required. |
| 378 | */ |
| 379 | if (pt_pmu.branch_en_always_on && |
| 380 | !(config & RTIT_CTL_BRANCH_EN)) |
| 381 | return false; |
| 382 | } else { |
| 383 | /* |
| 384 | * Disallow BRANCH_EN without the PASSTHROUGH. |
| 385 | */ |
| 386 | if (config & RTIT_CTL_BRANCH_EN) |
| 387 | return false; |
| 388 | } |
| 389 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 390 | return true; |
| 391 | } |
| 392 | |
| 393 | /* |
| 394 | * PT configuration helpers |
| 395 | * These all are cpu affine and operate on a local PT |
| 396 | */ |
| 397 | |
Alexander Shishkin | 8e105a1 | 2019-10-25 17:08:34 +0300 | [diff] [blame] | 398 | static void pt_config_start(struct perf_event *event) |
| 399 | { |
| 400 | struct pt *pt = this_cpu_ptr(&pt_ctx); |
| 401 | u64 ctl = event->hw.config; |
| 402 | |
| 403 | ctl |= RTIT_CTL_TRACEEN; |
| 404 | if (READ_ONCE(pt->vmx_on)) |
| 405 | perf_aux_output_flag(&pt->handle, PERF_AUX_FLAG_PARTIAL); |
| 406 | else |
| 407 | wrmsrl(MSR_IA32_RTIT_CTL, ctl); |
| 408 | |
| 409 | WRITE_ONCE(event->hw.config, ctl); |
| 410 | } |
| 411 | |
Alexander Shishkin | eadf48c | 2016-04-27 18:44:47 +0300 | [diff] [blame] | 412 | /* Address ranges and their corresponding msr configuration registers */ |
| 413 | static const struct pt_address_range { |
| 414 | unsigned long msr_a; |
| 415 | unsigned long msr_b; |
| 416 | unsigned int reg_off; |
| 417 | } pt_address_ranges[] = { |
| 418 | { |
| 419 | .msr_a = MSR_IA32_RTIT_ADDR0_A, |
| 420 | .msr_b = MSR_IA32_RTIT_ADDR0_B, |
| 421 | .reg_off = RTIT_CTL_ADDR0_OFFSET, |
| 422 | }, |
| 423 | { |
| 424 | .msr_a = MSR_IA32_RTIT_ADDR1_A, |
| 425 | .msr_b = MSR_IA32_RTIT_ADDR1_B, |
| 426 | .reg_off = RTIT_CTL_ADDR1_OFFSET, |
| 427 | }, |
| 428 | { |
| 429 | .msr_a = MSR_IA32_RTIT_ADDR2_A, |
| 430 | .msr_b = MSR_IA32_RTIT_ADDR2_B, |
| 431 | .reg_off = RTIT_CTL_ADDR2_OFFSET, |
| 432 | }, |
| 433 | { |
| 434 | .msr_a = MSR_IA32_RTIT_ADDR3_A, |
| 435 | .msr_b = MSR_IA32_RTIT_ADDR3_B, |
| 436 | .reg_off = RTIT_CTL_ADDR3_OFFSET, |
| 437 | } |
| 438 | }; |
| 439 | |
| 440 | static u64 pt_config_filters(struct perf_event *event) |
| 441 | { |
| 442 | struct pt_filters *filters = event->hw.addr_filters; |
| 443 | struct pt *pt = this_cpu_ptr(&pt_ctx); |
| 444 | unsigned int range = 0; |
| 445 | u64 rtit_ctl = 0; |
| 446 | |
| 447 | if (!filters) |
| 448 | return 0; |
| 449 | |
| 450 | perf_event_addr_filters_sync(event); |
| 451 | |
| 452 | for (range = 0; range < filters->nr_filters; range++) { |
| 453 | struct pt_filter *filter = &filters->filter[range]; |
| 454 | |
| 455 | /* |
| 456 | * Note, if the range has zero start/end addresses due |
| 457 | * to its dynamic object not being loaded yet, we just |
| 458 | * go ahead and program zeroed range, which will simply |
| 459 | * produce no data. Note^2: if executable code at 0x0 |
| 460 | * is a concern, we can set up an "invalid" configuration |
| 461 | * such as msr_b < msr_a. |
| 462 | */ |
| 463 | |
| 464 | /* avoid redundant msr writes */ |
| 465 | if (pt->filters.filter[range].msr_a != filter->msr_a) { |
| 466 | wrmsrl(pt_address_ranges[range].msr_a, filter->msr_a); |
| 467 | pt->filters.filter[range].msr_a = filter->msr_a; |
| 468 | } |
| 469 | |
| 470 | if (pt->filters.filter[range].msr_b != filter->msr_b) { |
| 471 | wrmsrl(pt_address_ranges[range].msr_b, filter->msr_b); |
| 472 | pt->filters.filter[range].msr_b = filter->msr_b; |
| 473 | } |
| 474 | |
| 475 | rtit_ctl |= filter->config << pt_address_ranges[range].reg_off; |
| 476 | } |
| 477 | |
| 478 | return rtit_ctl; |
| 479 | } |
| 480 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 481 | static void pt_config(struct perf_event *event) |
| 482 | { |
Alexander Shishkin | 6706384 | 2019-11-05 10:27:00 +0200 | [diff] [blame] | 483 | struct pt *pt = this_cpu_ptr(&pt_ctx); |
| 484 | struct pt_buffer *buf = perf_get_aux(&pt->handle); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 485 | u64 reg; |
| 486 | |
Alexander Shishkin | 8d4e6c4 | 2017-03-30 18:39:56 +0300 | [diff] [blame] | 487 | /* First round: clear STATUS, in particular the PSB byte counter. */ |
| 488 | if (!event->hw.config) { |
| 489 | perf_event_itrace_started(event); |
Alexander Shishkin | 9a6694c | 2015-07-30 16:48:24 +0300 | [diff] [blame] | 490 | wrmsrl(MSR_IA32_RTIT_STATUS, 0); |
| 491 | } |
| 492 | |
Alexander Shishkin | eadf48c | 2016-04-27 18:44:47 +0300 | [diff] [blame] | 493 | reg = pt_config_filters(event); |
Alexander Shishkin | 6706384 | 2019-11-05 10:27:00 +0200 | [diff] [blame] | 494 | reg |= RTIT_CTL_TRACEEN; |
| 495 | if (!buf->single) |
| 496 | reg |= RTIT_CTL_TOPA; |
Alexander Shishkin | d35869b | 2017-02-06 16:41:40 +0200 | [diff] [blame] | 497 | |
| 498 | /* |
| 499 | * Previously, we had BRANCH_EN on by default, but now that PT has |
| 500 | * grown features outside of branch tracing, it is useful to allow |
| 501 | * the user to disable it. Setting bit 0 in the event's attr.config |
| 502 | * allows BRANCH_EN to pass through instead of being always on. See |
| 503 | * also the comment in pt_event_valid(). |
| 504 | */ |
| 505 | if (event->attr.config & BIT(0)) { |
| 506 | reg |= event->attr.config & RTIT_CTL_BRANCH_EN; |
| 507 | } else { |
| 508 | reg |= RTIT_CTL_BRANCH_EN; |
| 509 | } |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 510 | |
| 511 | if (!event->attr.exclude_kernel) |
| 512 | reg |= RTIT_CTL_OS; |
| 513 | if (!event->attr.exclude_user) |
| 514 | reg |= RTIT_CTL_USR; |
| 515 | |
| 516 | reg |= (event->attr.config & PT_CONFIG_MASK); |
| 517 | |
Alexander Shishkin | 1c5ac21 | 2016-03-29 17:43:10 +0300 | [diff] [blame] | 518 | event->hw.config = reg; |
Alexander Shishkin | 8e105a1 | 2019-10-25 17:08:34 +0300 | [diff] [blame] | 519 | pt_config_start(event); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 520 | } |
| 521 | |
Alexander Shishkin | 1c5ac21 | 2016-03-29 17:43:10 +0300 | [diff] [blame] | 522 | static void pt_config_stop(struct perf_event *event) |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 523 | { |
Alexander Shishkin | ee36842 | 2017-02-20 15:33:52 +0200 | [diff] [blame] | 524 | struct pt *pt = this_cpu_ptr(&pt_ctx); |
Alexander Shishkin | 1c5ac21 | 2016-03-29 17:43:10 +0300 | [diff] [blame] | 525 | u64 ctl = READ_ONCE(event->hw.config); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 526 | |
Alexander Shishkin | 1c5ac21 | 2016-03-29 17:43:10 +0300 | [diff] [blame] | 527 | /* may be already stopped by a PMI */ |
| 528 | if (!(ctl & RTIT_CTL_TRACEEN)) |
| 529 | return; |
| 530 | |
| 531 | ctl &= ~RTIT_CTL_TRACEEN; |
Alexander Shishkin | ee36842 | 2017-02-20 15:33:52 +0200 | [diff] [blame] | 532 | if (!READ_ONCE(pt->vmx_on)) |
| 533 | wrmsrl(MSR_IA32_RTIT_CTL, ctl); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 534 | |
Alexander Shishkin | 1c5ac21 | 2016-03-29 17:43:10 +0300 | [diff] [blame] | 535 | WRITE_ONCE(event->hw.config, ctl); |
| 536 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 537 | /* |
| 538 | * A wrmsr that disables trace generation serializes other PT |
| 539 | * registers and causes all data packets to be written to memory, |
| 540 | * but a fence is required for the data to become globally visible. |
| 541 | * |
| 542 | * The below WMB, separating data store and aux_head store matches |
| 543 | * the consumer's RMB that separates aux_head load and data load. |
| 544 | */ |
Alexander Shishkin | 1c5ac21 | 2016-03-29 17:43:10 +0300 | [diff] [blame] | 545 | wmb(); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 546 | } |
| 547 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 548 | /** |
Alexander Shishkin | 38bb8d7 | 2019-08-21 15:47:25 +0300 | [diff] [blame] | 549 | * struct topa - ToPA metadata |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 550 | * @list: linkage to struct pt_buffer's list of tables |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 551 | * @offset: offset of the first entry in this table in the buffer |
| 552 | * @size: total size of all entries in this table |
| 553 | * @last: index of the last initialized entry in this table |
Alexander Shishkin | 39152ee | 2019-08-21 15:47:27 +0300 | [diff] [blame] | 554 | * @z_count: how many times the first entry repeats |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 555 | */ |
| 556 | struct topa { |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 557 | struct list_head list; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 558 | u64 offset; |
| 559 | size_t size; |
| 560 | int last; |
Alexander Shishkin | 39152ee | 2019-08-21 15:47:27 +0300 | [diff] [blame] | 561 | unsigned int z_count; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 562 | }; |
| 563 | |
Alexander Shishkin | 38bb8d7 | 2019-08-21 15:47:25 +0300 | [diff] [blame] | 564 | /* |
| 565 | * Keep ToPA table-related metadata on the same page as the actual table, |
| 566 | * taking up a few words from the top |
| 567 | */ |
| 568 | |
| 569 | #define TENTS_PER_PAGE \ |
| 570 | ((PAGE_SIZE - sizeof(struct topa)) / sizeof(struct topa_entry)) |
| 571 | |
| 572 | /** |
| 573 | * struct topa_page - page-sized ToPA table with metadata at the top |
| 574 | * @table: actual ToPA table entries, as understood by PT hardware |
| 575 | * @topa: metadata |
| 576 | */ |
| 577 | struct topa_page { |
| 578 | struct topa_entry table[TENTS_PER_PAGE]; |
| 579 | struct topa topa; |
| 580 | }; |
| 581 | |
| 582 | static inline struct topa_page *topa_to_page(struct topa *topa) |
| 583 | { |
| 584 | return container_of(topa, struct topa_page, topa); |
| 585 | } |
| 586 | |
| 587 | static inline struct topa_page *topa_entry_to_page(struct topa_entry *te) |
| 588 | { |
| 589 | return (struct topa_page *)((unsigned long)te & PAGE_MASK); |
| 590 | } |
| 591 | |
Alexander Shishkin | 91feca5 | 2019-08-21 15:47:26 +0300 | [diff] [blame] | 592 | static inline phys_addr_t topa_pfn(struct topa *topa) |
| 593 | { |
| 594 | return PFN_DOWN(virt_to_phys(topa_to_page(topa))); |
| 595 | } |
| 596 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 597 | /* make -1 stand for the last table entry */ |
Alexander Shishkin | 38bb8d7 | 2019-08-21 15:47:25 +0300 | [diff] [blame] | 598 | #define TOPA_ENTRY(t, i) \ |
| 599 | ((i) == -1 \ |
| 600 | ? &topa_to_page(t)->table[(t)->last] \ |
| 601 | : &topa_to_page(t)->table[(i)]) |
Alexander Shishkin | fffec50 | 2019-08-21 15:47:23 +0300 | [diff] [blame] | 602 | #define TOPA_ENTRY_SIZE(t, i) (sizes(TOPA_ENTRY((t), (i))->size)) |
Alexander Shishkin | 39152ee | 2019-08-21 15:47:27 +0300 | [diff] [blame] | 603 | #define TOPA_ENTRY_PAGES(t, i) (1 << TOPA_ENTRY((t), (i))->size) |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 604 | |
Alexander Shishkin | 6706384 | 2019-11-05 10:27:00 +0200 | [diff] [blame] | 605 | static void pt_config_buffer(struct pt_buffer *buf) |
| 606 | { |
Alexander Shishkin | 295c52e | 2019-11-05 10:27:01 +0200 | [diff] [blame] | 607 | struct pt *pt = this_cpu_ptr(&pt_ctx); |
Alexander Shishkin | 6706384 | 2019-11-05 10:27:00 +0200 | [diff] [blame] | 608 | u64 reg, mask; |
| 609 | void *base; |
| 610 | |
| 611 | if (buf->single) { |
| 612 | base = buf->data_pages[0]; |
| 613 | mask = (buf->nr_pages * PAGE_SIZE - 1) >> 7; |
| 614 | } else { |
| 615 | base = topa_to_page(buf->cur)->table; |
| 616 | mask = (u64)buf->cur_idx; |
| 617 | } |
| 618 | |
Alexander Shishkin | 295c52e | 2019-11-05 10:27:01 +0200 | [diff] [blame] | 619 | reg = virt_to_phys(base); |
| 620 | if (pt->output_base != reg) { |
| 621 | pt->output_base = reg; |
| 622 | wrmsrl(MSR_IA32_RTIT_OUTPUT_BASE, reg); |
| 623 | } |
Alexander Shishkin | 6706384 | 2019-11-05 10:27:00 +0200 | [diff] [blame] | 624 | |
| 625 | reg = 0x7f | (mask << 7) | ((u64)buf->output_off << 32); |
Alexander Shishkin | 295c52e | 2019-11-05 10:27:01 +0200 | [diff] [blame] | 626 | if (pt->output_mask != reg) { |
| 627 | pt->output_mask = reg; |
| 628 | wrmsrl(MSR_IA32_RTIT_OUTPUT_MASK, reg); |
| 629 | } |
Alexander Shishkin | 6706384 | 2019-11-05 10:27:00 +0200 | [diff] [blame] | 630 | } |
| 631 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 632 | /** |
| 633 | * topa_alloc() - allocate page-sized ToPA table |
| 634 | * @cpu: CPU on which to allocate. |
| 635 | * @gfp: Allocation flags. |
| 636 | * |
| 637 | * Return: On success, return the pointer to ToPA table page. |
| 638 | */ |
| 639 | static struct topa *topa_alloc(int cpu, gfp_t gfp) |
| 640 | { |
| 641 | int node = cpu_to_node(cpu); |
Alexander Shishkin | 38bb8d7 | 2019-08-21 15:47:25 +0300 | [diff] [blame] | 642 | struct topa_page *tp; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 643 | struct page *p; |
| 644 | |
| 645 | p = alloc_pages_node(node, gfp | __GFP_ZERO, 0); |
| 646 | if (!p) |
| 647 | return NULL; |
| 648 | |
Alexander Shishkin | 38bb8d7 | 2019-08-21 15:47:25 +0300 | [diff] [blame] | 649 | tp = page_address(p); |
| 650 | tp->topa.last = 0; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 651 | |
| 652 | /* |
| 653 | * In case of singe-entry ToPA, always put the self-referencing END |
| 654 | * link as the 2nd entry in the table |
| 655 | */ |
Chao Peng | f6d079c | 2018-10-24 16:05:06 +0800 | [diff] [blame] | 656 | if (!intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries)) { |
Jiri Olsa | 13301c6 | 2019-10-20 00:07:26 +0200 | [diff] [blame] | 657 | TOPA_ENTRY(&tp->topa, 1)->base = page_to_phys(p) >> TOPA_SHIFT; |
Alexander Shishkin | 38bb8d7 | 2019-08-21 15:47:25 +0300 | [diff] [blame] | 658 | TOPA_ENTRY(&tp->topa, 1)->end = 1; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 659 | } |
| 660 | |
Alexander Shishkin | 38bb8d7 | 2019-08-21 15:47:25 +0300 | [diff] [blame] | 661 | return &tp->topa; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 662 | } |
| 663 | |
| 664 | /** |
| 665 | * topa_free() - free a page-sized ToPA table |
| 666 | * @topa: Table to deallocate. |
| 667 | */ |
| 668 | static void topa_free(struct topa *topa) |
| 669 | { |
| 670 | free_page((unsigned long)topa); |
| 671 | } |
| 672 | |
| 673 | /** |
| 674 | * topa_insert_table() - insert a ToPA table into a buffer |
| 675 | * @buf: PT buffer that's being extended. |
| 676 | * @topa: New topa table to be inserted. |
| 677 | * |
| 678 | * If it's the first table in this buffer, set up buffer's pointers |
| 679 | * accordingly; otherwise, add a END=1 link entry to @topa to the current |
| 680 | * "last" table and adjust the last table pointer to @topa. |
| 681 | */ |
| 682 | static void topa_insert_table(struct pt_buffer *buf, struct topa *topa) |
| 683 | { |
| 684 | struct topa *last = buf->last; |
| 685 | |
| 686 | list_add_tail(&topa->list, &buf->tables); |
| 687 | |
| 688 | if (!buf->first) { |
| 689 | buf->first = buf->last = buf->cur = topa; |
| 690 | return; |
| 691 | } |
| 692 | |
| 693 | topa->offset = last->offset + last->size; |
| 694 | buf->last = topa; |
| 695 | |
Chao Peng | f6d079c | 2018-10-24 16:05:06 +0800 | [diff] [blame] | 696 | if (!intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries)) |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 697 | return; |
| 698 | |
| 699 | BUG_ON(last->last != TENTS_PER_PAGE - 1); |
| 700 | |
Alexander Shishkin | 91feca5 | 2019-08-21 15:47:26 +0300 | [diff] [blame] | 701 | TOPA_ENTRY(last, -1)->base = topa_pfn(topa); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 702 | TOPA_ENTRY(last, -1)->end = 1; |
| 703 | } |
| 704 | |
| 705 | /** |
| 706 | * topa_table_full() - check if a ToPA table is filled up |
| 707 | * @topa: ToPA table. |
| 708 | */ |
| 709 | static bool topa_table_full(struct topa *topa) |
| 710 | { |
| 711 | /* single-entry ToPA is a special case */ |
Chao Peng | f6d079c | 2018-10-24 16:05:06 +0800 | [diff] [blame] | 712 | if (!intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries)) |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 713 | return !!topa->last; |
| 714 | |
| 715 | return topa->last == TENTS_PER_PAGE - 1; |
| 716 | } |
| 717 | |
| 718 | /** |
| 719 | * topa_insert_pages() - create a list of ToPA tables |
| 720 | * @buf: PT buffer being initialized. |
| 721 | * @gfp: Allocation flags. |
| 722 | * |
| 723 | * This initializes a list of ToPA tables with entries from |
| 724 | * the data_pages provided by rb_alloc_aux(). |
| 725 | * |
| 726 | * Return: 0 on success or error code. |
| 727 | */ |
Alexander Shishkin | 90583af | 2019-08-21 15:47:22 +0300 | [diff] [blame] | 728 | static int topa_insert_pages(struct pt_buffer *buf, int cpu, gfp_t gfp) |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 729 | { |
| 730 | struct topa *topa = buf->last; |
| 731 | int order = 0; |
| 732 | struct page *p; |
| 733 | |
| 734 | p = virt_to_page(buf->data_pages[buf->nr_pages]); |
| 735 | if (PagePrivate(p)) |
| 736 | order = page_private(p); |
| 737 | |
| 738 | if (topa_table_full(topa)) { |
Alexander Shishkin | 90583af | 2019-08-21 15:47:22 +0300 | [diff] [blame] | 739 | topa = topa_alloc(cpu, gfp); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 740 | if (!topa) |
| 741 | return -ENOMEM; |
| 742 | |
| 743 | topa_insert_table(buf, topa); |
| 744 | } |
| 745 | |
Alexander Shishkin | 39152ee | 2019-08-21 15:47:27 +0300 | [diff] [blame] | 746 | if (topa->z_count == topa->last - 1) { |
| 747 | if (order == TOPA_ENTRY(topa, topa->last - 1)->size) |
| 748 | topa->z_count++; |
| 749 | } |
| 750 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 751 | TOPA_ENTRY(topa, -1)->base = page_to_phys(p) >> TOPA_SHIFT; |
| 752 | TOPA_ENTRY(topa, -1)->size = order; |
Chao Peng | f6d079c | 2018-10-24 16:05:06 +0800 | [diff] [blame] | 753 | if (!buf->snapshot && |
| 754 | !intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries)) { |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 755 | TOPA_ENTRY(topa, -1)->intr = 1; |
| 756 | TOPA_ENTRY(topa, -1)->stop = 1; |
| 757 | } |
| 758 | |
| 759 | topa->last++; |
| 760 | topa->size += sizes(order); |
| 761 | |
| 762 | buf->nr_pages += 1ul << order; |
| 763 | |
| 764 | return 0; |
| 765 | } |
| 766 | |
| 767 | /** |
| 768 | * pt_topa_dump() - print ToPA tables and their entries |
| 769 | * @buf: PT buffer. |
| 770 | */ |
| 771 | static void pt_topa_dump(struct pt_buffer *buf) |
| 772 | { |
| 773 | struct topa *topa; |
| 774 | |
| 775 | list_for_each_entry(topa, &buf->tables, list) { |
Alexander Shishkin | 38bb8d7 | 2019-08-21 15:47:25 +0300 | [diff] [blame] | 776 | struct topa_page *tp = topa_to_page(topa); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 777 | int i; |
| 778 | |
Alexander Shishkin | 91feca5 | 2019-08-21 15:47:26 +0300 | [diff] [blame] | 779 | pr_debug("# table @%p, off %llx size %zx\n", tp->table, |
| 780 | topa->offset, topa->size); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 781 | for (i = 0; i < TENTS_PER_PAGE; i++) { |
| 782 | pr_debug("# entry @%p (%lx sz %u %c%c%c) raw=%16llx\n", |
Alexander Shishkin | 38bb8d7 | 2019-08-21 15:47:25 +0300 | [diff] [blame] | 783 | &tp->table[i], |
| 784 | (unsigned long)tp->table[i].base << TOPA_SHIFT, |
| 785 | sizes(tp->table[i].size), |
| 786 | tp->table[i].end ? 'E' : ' ', |
| 787 | tp->table[i].intr ? 'I' : ' ', |
| 788 | tp->table[i].stop ? 'S' : ' ', |
| 789 | *(u64 *)&tp->table[i]); |
Chao Peng | f6d079c | 2018-10-24 16:05:06 +0800 | [diff] [blame] | 790 | if ((intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries) && |
Alexander Shishkin | 38bb8d7 | 2019-08-21 15:47:25 +0300 | [diff] [blame] | 791 | tp->table[i].stop) || |
| 792 | tp->table[i].end) |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 793 | break; |
Alexander Shishkin | 39152ee | 2019-08-21 15:47:27 +0300 | [diff] [blame] | 794 | if (!i && topa->z_count) |
| 795 | i += topa->z_count; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 796 | } |
| 797 | } |
| 798 | } |
| 799 | |
| 800 | /** |
| 801 | * pt_buffer_advance() - advance to the next output region |
| 802 | * @buf: PT buffer. |
| 803 | * |
| 804 | * Advance the current pointers in the buffer to the next ToPA entry. |
| 805 | */ |
| 806 | static void pt_buffer_advance(struct pt_buffer *buf) |
| 807 | { |
| 808 | buf->output_off = 0; |
| 809 | buf->cur_idx++; |
| 810 | |
| 811 | if (buf->cur_idx == buf->cur->last) { |
| 812 | if (buf->cur == buf->last) |
| 813 | buf->cur = buf->first; |
| 814 | else |
| 815 | buf->cur = list_entry(buf->cur->list.next, struct topa, |
| 816 | list); |
| 817 | buf->cur_idx = 0; |
| 818 | } |
| 819 | } |
| 820 | |
| 821 | /** |
| 822 | * pt_update_head() - calculate current offsets and sizes |
| 823 | * @pt: Per-cpu pt context. |
| 824 | * |
| 825 | * Update buffer's current write pointer position and data size. |
| 826 | */ |
| 827 | static void pt_update_head(struct pt *pt) |
| 828 | { |
| 829 | struct pt_buffer *buf = perf_get_aux(&pt->handle); |
| 830 | u64 topa_idx, base, old; |
| 831 | |
Alexander Shishkin | 6706384 | 2019-11-05 10:27:00 +0200 | [diff] [blame] | 832 | if (buf->single) { |
| 833 | local_set(&buf->data_size, buf->output_off); |
| 834 | return; |
| 835 | } |
| 836 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 837 | /* offset of the first region in this table from the beginning of buf */ |
| 838 | base = buf->cur->offset + buf->output_off; |
| 839 | |
| 840 | /* offset of the current output region within this table */ |
| 841 | for (topa_idx = 0; topa_idx < buf->cur_idx; topa_idx++) |
Alexander Shishkin | fffec50 | 2019-08-21 15:47:23 +0300 | [diff] [blame] | 842 | base += TOPA_ENTRY_SIZE(buf->cur, topa_idx); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 843 | |
| 844 | if (buf->snapshot) { |
| 845 | local_set(&buf->data_size, base); |
| 846 | } else { |
| 847 | old = (local64_xchg(&buf->head, base) & |
| 848 | ((buf->nr_pages << PAGE_SHIFT) - 1)); |
| 849 | if (base < old) |
| 850 | base += buf->nr_pages << PAGE_SHIFT; |
| 851 | |
| 852 | local_add(base - old, &buf->data_size); |
| 853 | } |
| 854 | } |
| 855 | |
| 856 | /** |
| 857 | * pt_buffer_region() - obtain current output region's address |
| 858 | * @buf: PT buffer. |
| 859 | */ |
| 860 | static void *pt_buffer_region(struct pt_buffer *buf) |
| 861 | { |
Alexander Shishkin | 38bb8d7 | 2019-08-21 15:47:25 +0300 | [diff] [blame] | 862 | return phys_to_virt(TOPA_ENTRY(buf->cur, buf->cur_idx)->base << TOPA_SHIFT); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 863 | } |
| 864 | |
| 865 | /** |
| 866 | * pt_buffer_region_size() - obtain current output region's size |
| 867 | * @buf: PT buffer. |
| 868 | */ |
| 869 | static size_t pt_buffer_region_size(struct pt_buffer *buf) |
| 870 | { |
Alexander Shishkin | fffec50 | 2019-08-21 15:47:23 +0300 | [diff] [blame] | 871 | return TOPA_ENTRY_SIZE(buf->cur, buf->cur_idx); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 872 | } |
| 873 | |
| 874 | /** |
| 875 | * pt_handle_status() - take care of possible status conditions |
| 876 | * @pt: Per-cpu pt context. |
| 877 | */ |
| 878 | static void pt_handle_status(struct pt *pt) |
| 879 | { |
| 880 | struct pt_buffer *buf = perf_get_aux(&pt->handle); |
| 881 | int advance = 0; |
| 882 | u64 status; |
| 883 | |
| 884 | rdmsrl(MSR_IA32_RTIT_STATUS, status); |
| 885 | |
| 886 | if (status & RTIT_STATUS_ERROR) { |
| 887 | pr_err_ratelimited("ToPA ERROR encountered, trying to recover\n"); |
| 888 | pt_topa_dump(buf); |
| 889 | status &= ~RTIT_STATUS_ERROR; |
| 890 | } |
| 891 | |
| 892 | if (status & RTIT_STATUS_STOPPED) { |
| 893 | status &= ~RTIT_STATUS_STOPPED; |
| 894 | |
| 895 | /* |
| 896 | * On systems that only do single-entry ToPA, hitting STOP |
| 897 | * means we are already losing data; need to let the decoder |
| 898 | * know. |
| 899 | */ |
Chao Peng | f6d079c | 2018-10-24 16:05:06 +0800 | [diff] [blame] | 900 | if (!intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries) || |
Alexander Shishkin | fffec50 | 2019-08-21 15:47:23 +0300 | [diff] [blame] | 901 | buf->output_off == pt_buffer_region_size(buf)) { |
Will Deacon | f4c0b0a | 2017-02-20 15:33:50 +0200 | [diff] [blame] | 902 | perf_aux_output_flag(&pt->handle, |
| 903 | PERF_AUX_FLAG_TRUNCATED); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 904 | advance++; |
| 905 | } |
| 906 | } |
| 907 | |
| 908 | /* |
| 909 | * Also on single-entry ToPA implementations, interrupt will come |
| 910 | * before the output reaches its output region's boundary. |
| 911 | */ |
Chao Peng | f6d079c | 2018-10-24 16:05:06 +0800 | [diff] [blame] | 912 | if (!intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries) && |
| 913 | !buf->snapshot && |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 914 | pt_buffer_region_size(buf) - buf->output_off <= TOPA_PMI_MARGIN) { |
| 915 | void *head = pt_buffer_region(buf); |
| 916 | |
| 917 | /* everything within this margin needs to be zeroed out */ |
| 918 | memset(head + buf->output_off, 0, |
| 919 | pt_buffer_region_size(buf) - |
| 920 | buf->output_off); |
| 921 | advance++; |
| 922 | } |
| 923 | |
| 924 | if (advance) |
| 925 | pt_buffer_advance(buf); |
| 926 | |
| 927 | wrmsrl(MSR_IA32_RTIT_STATUS, status); |
| 928 | } |
| 929 | |
| 930 | /** |
| 931 | * pt_read_offset() - translate registers into buffer pointers |
| 932 | * @buf: PT buffer. |
| 933 | * |
| 934 | * Set buffer's output pointers from MSR values. |
| 935 | */ |
| 936 | static void pt_read_offset(struct pt_buffer *buf) |
| 937 | { |
Alexander Shishkin | 295c52e | 2019-11-05 10:27:01 +0200 | [diff] [blame] | 938 | struct pt *pt = this_cpu_ptr(&pt_ctx); |
Alexander Shishkin | 38bb8d7 | 2019-08-21 15:47:25 +0300 | [diff] [blame] | 939 | struct topa_page *tp; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 940 | |
Alexander Shishkin | 6706384 | 2019-11-05 10:27:00 +0200 | [diff] [blame] | 941 | if (!buf->single) { |
Alexander Shishkin | 295c52e | 2019-11-05 10:27:01 +0200 | [diff] [blame] | 942 | rdmsrl(MSR_IA32_RTIT_OUTPUT_BASE, pt->output_base); |
| 943 | tp = phys_to_virt(pt->output_base); |
Alexander Shishkin | 6706384 | 2019-11-05 10:27:00 +0200 | [diff] [blame] | 944 | buf->cur = &tp->topa; |
| 945 | } |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 946 | |
Alexander Shishkin | 295c52e | 2019-11-05 10:27:01 +0200 | [diff] [blame] | 947 | rdmsrl(MSR_IA32_RTIT_OUTPUT_MASK, pt->output_mask); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 948 | /* offset within current output region */ |
Alexander Shishkin | 295c52e | 2019-11-05 10:27:01 +0200 | [diff] [blame] | 949 | buf->output_off = pt->output_mask >> 32; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 950 | /* index of current output region within this table */ |
Alexander Shishkin | 6706384 | 2019-11-05 10:27:00 +0200 | [diff] [blame] | 951 | if (!buf->single) |
Alexander Shishkin | 295c52e | 2019-11-05 10:27:01 +0200 | [diff] [blame] | 952 | buf->cur_idx = (pt->output_mask & 0xffffff80) >> 7; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 953 | } |
| 954 | |
Alexander Shishkin | 39152ee | 2019-08-21 15:47:27 +0300 | [diff] [blame] | 955 | static struct topa_entry * |
| 956 | pt_topa_entry_for_page(struct pt_buffer *buf, unsigned int pg) |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 957 | { |
Alexander Shishkin | 39152ee | 2019-08-21 15:47:27 +0300 | [diff] [blame] | 958 | struct topa_page *tp; |
| 959 | struct topa *topa; |
| 960 | unsigned int idx, cur_pg = 0, z_pg = 0, start_idx = 0; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 961 | |
Alexander Shishkin | 39152ee | 2019-08-21 15:47:27 +0300 | [diff] [blame] | 962 | /* |
| 963 | * Indicates a bug in the caller. |
| 964 | */ |
| 965 | if (WARN_ON_ONCE(pg >= buf->nr_pages)) |
| 966 | return NULL; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 967 | |
Alexander Shishkin | 39152ee | 2019-08-21 15:47:27 +0300 | [diff] [blame] | 968 | /* |
| 969 | * First, find the ToPA table where @pg fits. With high |
| 970 | * order allocations, there shouldn't be many of these. |
| 971 | */ |
| 972 | list_for_each_entry(topa, &buf->tables, list) { |
| 973 | if (topa->offset + topa->size > pg << PAGE_SHIFT) |
| 974 | goto found; |
| 975 | } |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 976 | |
Alexander Shishkin | 39152ee | 2019-08-21 15:47:27 +0300 | [diff] [blame] | 977 | /* |
| 978 | * Hitting this means we have a problem in the ToPA |
| 979 | * allocation code. |
| 980 | */ |
| 981 | WARN_ON_ONCE(1); |
| 982 | |
| 983 | return NULL; |
| 984 | |
| 985 | found: |
| 986 | /* |
| 987 | * Indicates a problem in the ToPA allocation code. |
| 988 | */ |
| 989 | if (WARN_ON_ONCE(topa->last == -1)) |
| 990 | return NULL; |
| 991 | |
| 992 | tp = topa_to_page(topa); |
| 993 | cur_pg = PFN_DOWN(topa->offset); |
| 994 | if (topa->z_count) { |
| 995 | z_pg = TOPA_ENTRY_PAGES(topa, 0) * (topa->z_count + 1); |
| 996 | start_idx = topa->z_count + 1; |
| 997 | } |
| 998 | |
| 999 | /* |
| 1000 | * Multiple entries at the beginning of the table have the same size, |
| 1001 | * ideally all of them; if @pg falls there, the search is done. |
| 1002 | */ |
| 1003 | if (pg >= cur_pg && pg < cur_pg + z_pg) { |
| 1004 | idx = (pg - cur_pg) / TOPA_ENTRY_PAGES(topa, 0); |
| 1005 | return &tp->table[idx]; |
| 1006 | } |
| 1007 | |
| 1008 | /* |
| 1009 | * Otherwise, slow path: iterate through the remaining entries. |
| 1010 | */ |
| 1011 | for (idx = start_idx, cur_pg += z_pg; idx < topa->last; idx++) { |
| 1012 | if (cur_pg + TOPA_ENTRY_PAGES(topa, idx) > pg) |
| 1013 | return &tp->table[idx]; |
| 1014 | |
| 1015 | cur_pg += TOPA_ENTRY_PAGES(topa, idx); |
| 1016 | } |
| 1017 | |
| 1018 | /* |
| 1019 | * Means we couldn't find a ToPA entry in the table that does match. |
| 1020 | */ |
| 1021 | WARN_ON_ONCE(1); |
| 1022 | |
| 1023 | return NULL; |
| 1024 | } |
| 1025 | |
| 1026 | static struct topa_entry * |
| 1027 | pt_topa_prev_entry(struct pt_buffer *buf, struct topa_entry *te) |
| 1028 | { |
| 1029 | unsigned long table = (unsigned long)te & ~(PAGE_SIZE - 1); |
| 1030 | struct topa_page *tp; |
| 1031 | struct topa *topa; |
| 1032 | |
| 1033 | tp = (struct topa_page *)table; |
| 1034 | if (tp->table != te) |
| 1035 | return --te; |
| 1036 | |
| 1037 | topa = &tp->topa; |
| 1038 | if (topa == buf->first) |
| 1039 | topa = buf->last; |
| 1040 | else |
| 1041 | topa = list_prev_entry(topa, list); |
| 1042 | |
| 1043 | tp = topa_to_page(topa); |
| 1044 | |
| 1045 | return &tp->table[topa->last - 1]; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1046 | } |
| 1047 | |
| 1048 | /** |
| 1049 | * pt_buffer_reset_markers() - place interrupt and stop bits in the buffer |
| 1050 | * @buf: PT buffer. |
| 1051 | * @handle: Current output handle. |
| 1052 | * |
| 1053 | * Place INT and STOP marks to prevent overwriting old data that the consumer |
Alexander Shishkin | cf302bf | 2015-04-21 16:16:15 +0300 | [diff] [blame] | 1054 | * hasn't yet collected and waking up the consumer after a certain fraction of |
| 1055 | * the buffer has filled up. Only needed and sensible for non-snapshot counters. |
| 1056 | * |
| 1057 | * This obviously relies on buf::head to figure out buffer markers, so it has |
| 1058 | * to be called after pt_buffer_reset_offsets() and before the hardware tracing |
| 1059 | * is enabled. |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1060 | */ |
| 1061 | static int pt_buffer_reset_markers(struct pt_buffer *buf, |
| 1062 | struct perf_output_handle *handle) |
| 1063 | |
| 1064 | { |
Alexander Shishkin | f73ec48 | 2015-05-22 18:30:22 +0300 | [diff] [blame] | 1065 | unsigned long head = local64_read(&buf->head); |
| 1066 | unsigned long idx, npages, wakeup; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1067 | |
Alexander Shishkin | 6706384 | 2019-11-05 10:27:00 +0200 | [diff] [blame] | 1068 | if (buf->single) |
| 1069 | return 0; |
| 1070 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1071 | /* can't stop in the middle of an output region */ |
Alexander Shishkin | fffec50 | 2019-08-21 15:47:23 +0300 | [diff] [blame] | 1072 | if (buf->output_off + handle->size + 1 < pt_buffer_region_size(buf)) { |
Will Deacon | f4c0b0a | 2017-02-20 15:33:50 +0200 | [diff] [blame] | 1073 | perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1074 | return -EINVAL; |
Will Deacon | f4c0b0a | 2017-02-20 15:33:50 +0200 | [diff] [blame] | 1075 | } |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1076 | |
| 1077 | |
| 1078 | /* single entry ToPA is handled by marking all regions STOP=1 INT=1 */ |
Chao Peng | f6d079c | 2018-10-24 16:05:06 +0800 | [diff] [blame] | 1079 | if (!intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries)) |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1080 | return 0; |
| 1081 | |
| 1082 | /* clear STOP and INT from current entry */ |
Alexander Shishkin | 39152ee | 2019-08-21 15:47:27 +0300 | [diff] [blame] | 1083 | if (buf->stop_te) { |
| 1084 | buf->stop_te->stop = 0; |
| 1085 | buf->stop_te->intr = 0; |
| 1086 | } |
| 1087 | |
| 1088 | if (buf->intr_te) |
| 1089 | buf->intr_te->intr = 0; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1090 | |
Alexander Shishkin | f73ec48 | 2015-05-22 18:30:22 +0300 | [diff] [blame] | 1091 | /* how many pages till the STOP marker */ |
| 1092 | npages = handle->size >> PAGE_SHIFT; |
| 1093 | |
| 1094 | /* if it's on a page boundary, fill up one more page */ |
| 1095 | if (!offset_in_page(head + handle->size + 1)) |
| 1096 | npages++; |
| 1097 | |
| 1098 | idx = (head >> PAGE_SHIFT) + npages; |
| 1099 | idx &= buf->nr_pages - 1; |
Alexander Shishkin | 39152ee | 2019-08-21 15:47:27 +0300 | [diff] [blame] | 1100 | |
| 1101 | if (idx != buf->stop_pos) { |
| 1102 | buf->stop_pos = idx; |
| 1103 | buf->stop_te = pt_topa_entry_for_page(buf, idx); |
| 1104 | buf->stop_te = pt_topa_prev_entry(buf, buf->stop_te); |
| 1105 | } |
Alexander Shishkin | f73ec48 | 2015-05-22 18:30:22 +0300 | [diff] [blame] | 1106 | |
| 1107 | wakeup = handle->wakeup >> PAGE_SHIFT; |
| 1108 | |
| 1109 | /* in the worst case, wake up the consumer one page before hard stop */ |
| 1110 | idx = (head >> PAGE_SHIFT) + npages - 1; |
| 1111 | if (idx > wakeup) |
| 1112 | idx = wakeup; |
| 1113 | |
| 1114 | idx &= buf->nr_pages - 1; |
Alexander Shishkin | 39152ee | 2019-08-21 15:47:27 +0300 | [diff] [blame] | 1115 | if (idx != buf->intr_pos) { |
| 1116 | buf->intr_pos = idx; |
| 1117 | buf->intr_te = pt_topa_entry_for_page(buf, idx); |
| 1118 | buf->intr_te = pt_topa_prev_entry(buf, buf->intr_te); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1119 | } |
| 1120 | |
Alexander Shishkin | 39152ee | 2019-08-21 15:47:27 +0300 | [diff] [blame] | 1121 | buf->stop_te->stop = 1; |
| 1122 | buf->stop_te->intr = 1; |
| 1123 | buf->intr_te->intr = 1; |
| 1124 | |
| 1125 | return 0; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1126 | } |
| 1127 | |
| 1128 | /** |
| 1129 | * pt_buffer_reset_offsets() - adjust buffer's write pointers from aux_head |
| 1130 | * @buf: PT buffer. |
| 1131 | * @head: Write pointer (aux_head) from AUX buffer. |
| 1132 | * |
| 1133 | * Find the ToPA table and entry corresponding to given @head and set buffer's |
Alexander Shishkin | 5b1dbd1 | 2015-04-21 16:16:16 +0300 | [diff] [blame] | 1134 | * "current" pointers accordingly. This is done after we have obtained the |
| 1135 | * current aux_head position from a successful call to perf_aux_output_begin() |
| 1136 | * to make sure the hardware is writing to the right place. |
| 1137 | * |
| 1138 | * This function modifies buf::{cur,cur_idx,output_off} that will be programmed |
| 1139 | * into PT msrs when the tracing is enabled and buf::head and buf::data_size, |
| 1140 | * which are used to determine INT and STOP markers' locations by a subsequent |
| 1141 | * call to pt_buffer_reset_markers(). |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1142 | */ |
| 1143 | static void pt_buffer_reset_offsets(struct pt_buffer *buf, unsigned long head) |
| 1144 | { |
Alexander Shishkin | 38bb8d7 | 2019-08-21 15:47:25 +0300 | [diff] [blame] | 1145 | struct topa_page *cur_tp; |
Alexander Shishkin | 39152ee | 2019-08-21 15:47:27 +0300 | [diff] [blame] | 1146 | struct topa_entry *te; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1147 | int pg; |
| 1148 | |
| 1149 | if (buf->snapshot) |
| 1150 | head &= (buf->nr_pages << PAGE_SHIFT) - 1; |
| 1151 | |
Alexander Shishkin | 6706384 | 2019-11-05 10:27:00 +0200 | [diff] [blame] | 1152 | if (!buf->single) { |
| 1153 | pg = (head >> PAGE_SHIFT) & (buf->nr_pages - 1); |
| 1154 | te = pt_topa_entry_for_page(buf, pg); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1155 | |
Alexander Shishkin | 6706384 | 2019-11-05 10:27:00 +0200 | [diff] [blame] | 1156 | cur_tp = topa_entry_to_page(te); |
| 1157 | buf->cur = &cur_tp->topa; |
| 1158 | buf->cur_idx = te - TOPA_ENTRY(buf->cur, 0); |
| 1159 | buf->output_off = head & (pt_buffer_region_size(buf) - 1); |
| 1160 | } else { |
| 1161 | buf->output_off = head; |
| 1162 | } |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1163 | |
| 1164 | local64_set(&buf->head, head); |
| 1165 | local_set(&buf->data_size, 0); |
| 1166 | } |
| 1167 | |
| 1168 | /** |
| 1169 | * pt_buffer_fini_topa() - deallocate ToPA structure of a buffer |
| 1170 | * @buf: PT buffer. |
| 1171 | */ |
| 1172 | static void pt_buffer_fini_topa(struct pt_buffer *buf) |
| 1173 | { |
| 1174 | struct topa *topa, *iter; |
| 1175 | |
Alexander Shishkin | 6706384 | 2019-11-05 10:27:00 +0200 | [diff] [blame] | 1176 | if (buf->single) |
| 1177 | return; |
| 1178 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1179 | list_for_each_entry_safe(topa, iter, &buf->tables, list) { |
| 1180 | /* |
| 1181 | * right now, this is in free_aux() path only, so |
| 1182 | * no need to unlink this table from the list |
| 1183 | */ |
| 1184 | topa_free(topa); |
| 1185 | } |
| 1186 | } |
| 1187 | |
| 1188 | /** |
| 1189 | * pt_buffer_init_topa() - initialize ToPA table for pt buffer |
| 1190 | * @buf: PT buffer. |
| 1191 | * @size: Total size of all regions within this ToPA. |
| 1192 | * @gfp: Allocation flags. |
| 1193 | */ |
Alexander Shishkin | 90583af | 2019-08-21 15:47:22 +0300 | [diff] [blame] | 1194 | static int pt_buffer_init_topa(struct pt_buffer *buf, int cpu, |
| 1195 | unsigned long nr_pages, gfp_t gfp) |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1196 | { |
| 1197 | struct topa *topa; |
| 1198 | int err; |
| 1199 | |
Alexander Shishkin | 90583af | 2019-08-21 15:47:22 +0300 | [diff] [blame] | 1200 | topa = topa_alloc(cpu, gfp); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1201 | if (!topa) |
| 1202 | return -ENOMEM; |
| 1203 | |
| 1204 | topa_insert_table(buf, topa); |
| 1205 | |
| 1206 | while (buf->nr_pages < nr_pages) { |
Alexander Shishkin | 90583af | 2019-08-21 15:47:22 +0300 | [diff] [blame] | 1207 | err = topa_insert_pages(buf, cpu, gfp); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1208 | if (err) { |
| 1209 | pt_buffer_fini_topa(buf); |
| 1210 | return -ENOMEM; |
| 1211 | } |
| 1212 | } |
| 1213 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1214 | /* link last table to the first one, unless we're double buffering */ |
Chao Peng | f6d079c | 2018-10-24 16:05:06 +0800 | [diff] [blame] | 1215 | if (intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries)) { |
Alexander Shishkin | 91feca5 | 2019-08-21 15:47:26 +0300 | [diff] [blame] | 1216 | TOPA_ENTRY(buf->last, -1)->base = topa_pfn(buf->first); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1217 | TOPA_ENTRY(buf->last, -1)->end = 1; |
| 1218 | } |
| 1219 | |
| 1220 | pt_topa_dump(buf); |
| 1221 | return 0; |
| 1222 | } |
| 1223 | |
Alexander Shishkin | 6706384 | 2019-11-05 10:27:00 +0200 | [diff] [blame] | 1224 | static int pt_buffer_try_single(struct pt_buffer *buf, int nr_pages) |
| 1225 | { |
| 1226 | struct page *p = virt_to_page(buf->data_pages[0]); |
| 1227 | int ret = -ENOTSUPP, order = 0; |
| 1228 | |
| 1229 | /* |
| 1230 | * We can use single range output mode |
| 1231 | * + in snapshot mode, where we don't need interrupts; |
| 1232 | * + if the hardware supports it; |
| 1233 | * + if the entire buffer is one contiguous allocation. |
| 1234 | */ |
| 1235 | if (!buf->snapshot) |
| 1236 | goto out; |
| 1237 | |
| 1238 | if (!intel_pt_validate_hw_cap(PT_CAP_single_range_output)) |
| 1239 | goto out; |
| 1240 | |
| 1241 | if (PagePrivate(p)) |
| 1242 | order = page_private(p); |
| 1243 | |
| 1244 | if (1 << order != nr_pages) |
| 1245 | goto out; |
| 1246 | |
| 1247 | buf->single = true; |
| 1248 | buf->nr_pages = nr_pages; |
| 1249 | ret = 0; |
| 1250 | out: |
| 1251 | return ret; |
| 1252 | } |
| 1253 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1254 | /** |
| 1255 | * pt_buffer_setup_aux() - set up topa tables for a PT buffer |
| 1256 | * @cpu: Cpu on which to allocate, -1 means current. |
| 1257 | * @pages: Array of pointers to buffer pages passed from perf core. |
| 1258 | * @nr_pages: Number of pages in the buffer. |
| 1259 | * @snapshot: If this is a snapshot/overwrite counter. |
| 1260 | * |
| 1261 | * This is a pmu::setup_aux callback that sets up ToPA tables and all the |
| 1262 | * bookkeeping for an AUX buffer. |
| 1263 | * |
| 1264 | * Return: Our private PT buffer structure. |
| 1265 | */ |
| 1266 | static void * |
Mathieu Poirier | 8400186 | 2019-01-31 11:47:08 -0700 | [diff] [blame] | 1267 | pt_buffer_setup_aux(struct perf_event *event, void **pages, |
| 1268 | int nr_pages, bool snapshot) |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1269 | { |
| 1270 | struct pt_buffer *buf; |
Mathieu Poirier | 8400186 | 2019-01-31 11:47:08 -0700 | [diff] [blame] | 1271 | int node, ret, cpu = event->cpu; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1272 | |
| 1273 | if (!nr_pages) |
| 1274 | return NULL; |
| 1275 | |
Alexander Shishkin | 25e8920 | 2019-10-25 17:08:35 +0300 | [diff] [blame] | 1276 | /* |
| 1277 | * Only support AUX sampling in snapshot mode, where we don't |
| 1278 | * generate NMIs. |
| 1279 | */ |
| 1280 | if (event->attr.aux_sample_size && !snapshot) |
| 1281 | return NULL; |
| 1282 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1283 | if (cpu == -1) |
| 1284 | cpu = raw_smp_processor_id(); |
| 1285 | node = cpu_to_node(cpu); |
| 1286 | |
Alexander Shishkin | 39152ee | 2019-08-21 15:47:27 +0300 | [diff] [blame] | 1287 | buf = kzalloc_node(sizeof(struct pt_buffer), GFP_KERNEL, node); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1288 | if (!buf) |
| 1289 | return NULL; |
| 1290 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1291 | buf->snapshot = snapshot; |
| 1292 | buf->data_pages = pages; |
Alexander Shishkin | 39152ee | 2019-08-21 15:47:27 +0300 | [diff] [blame] | 1293 | buf->stop_pos = -1; |
| 1294 | buf->intr_pos = -1; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1295 | |
| 1296 | INIT_LIST_HEAD(&buf->tables); |
| 1297 | |
Alexander Shishkin | 6706384 | 2019-11-05 10:27:00 +0200 | [diff] [blame] | 1298 | ret = pt_buffer_try_single(buf, nr_pages); |
| 1299 | if (!ret) |
| 1300 | return buf; |
| 1301 | |
Alexander Shishkin | 90583af | 2019-08-21 15:47:22 +0300 | [diff] [blame] | 1302 | ret = pt_buffer_init_topa(buf, cpu, nr_pages, GFP_KERNEL); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1303 | if (ret) { |
| 1304 | kfree(buf); |
| 1305 | return NULL; |
| 1306 | } |
| 1307 | |
| 1308 | return buf; |
| 1309 | } |
| 1310 | |
| 1311 | /** |
| 1312 | * pt_buffer_free_aux() - perf AUX deallocation path callback |
| 1313 | * @data: PT buffer. |
| 1314 | */ |
| 1315 | static void pt_buffer_free_aux(void *data) |
| 1316 | { |
| 1317 | struct pt_buffer *buf = data; |
| 1318 | |
| 1319 | pt_buffer_fini_topa(buf); |
| 1320 | kfree(buf); |
| 1321 | } |
| 1322 | |
Alexander Shishkin | eadf48c | 2016-04-27 18:44:47 +0300 | [diff] [blame] | 1323 | static int pt_addr_filters_init(struct perf_event *event) |
| 1324 | { |
| 1325 | struct pt_filters *filters; |
| 1326 | int node = event->cpu == -1 ? -1 : cpu_to_node(event->cpu); |
| 1327 | |
Chao Peng | f6d079c | 2018-10-24 16:05:06 +0800 | [diff] [blame] | 1328 | if (!intel_pt_validate_hw_cap(PT_CAP_num_address_ranges)) |
Alexander Shishkin | eadf48c | 2016-04-27 18:44:47 +0300 | [diff] [blame] | 1329 | return 0; |
| 1330 | |
| 1331 | filters = kzalloc_node(sizeof(struct pt_filters), GFP_KERNEL, node); |
| 1332 | if (!filters) |
| 1333 | return -ENOMEM; |
| 1334 | |
| 1335 | if (event->parent) |
| 1336 | memcpy(filters, event->parent->hw.addr_filters, |
| 1337 | sizeof(*filters)); |
| 1338 | |
| 1339 | event->hw.addr_filters = filters; |
| 1340 | |
| 1341 | return 0; |
| 1342 | } |
| 1343 | |
| 1344 | static void pt_addr_filters_fini(struct perf_event *event) |
| 1345 | { |
| 1346 | kfree(event->hw.addr_filters); |
| 1347 | event->hw.addr_filters = NULL; |
| 1348 | } |
| 1349 | |
Alexander Shishkin | ddfdad9 | 2016-09-15 18:13:51 +0300 | [diff] [blame] | 1350 | static inline bool valid_kernel_ip(unsigned long ip) |
| 1351 | { |
| 1352 | return virt_addr_valid(ip) && kernel_ip(ip); |
| 1353 | } |
| 1354 | |
Alexander Shishkin | eadf48c | 2016-04-27 18:44:47 +0300 | [diff] [blame] | 1355 | static int pt_event_addr_filters_validate(struct list_head *filters) |
| 1356 | { |
| 1357 | struct perf_addr_filter *filter; |
| 1358 | int range = 0; |
| 1359 | |
| 1360 | list_for_each_entry(filter, filters, entry) { |
Alexander Shishkin | 6ed70cf | 2018-03-29 15:06:48 +0300 | [diff] [blame] | 1361 | /* |
| 1362 | * PT doesn't support single address triggers and |
| 1363 | * 'start' filters. |
| 1364 | */ |
| 1365 | if (!filter->size || |
| 1366 | filter->action == PERF_ADDR_FILTER_ACTION_START) |
Alexander Shishkin | eadf48c | 2016-04-27 18:44:47 +0300 | [diff] [blame] | 1367 | return -EOPNOTSUPP; |
| 1368 | |
Song Liu | 9511bce | 2018-04-17 23:29:07 -0700 | [diff] [blame] | 1369 | if (!filter->path.dentry) { |
Alexander Shishkin | 1155baf | 2016-09-15 18:13:52 +0300 | [diff] [blame] | 1370 | if (!valid_kernel_ip(filter->offset)) |
| 1371 | return -EINVAL; |
| 1372 | |
| 1373 | if (!valid_kernel_ip(filter->offset + filter->size)) |
| 1374 | return -EINVAL; |
| 1375 | } |
Alexander Shishkin | eadf48c | 2016-04-27 18:44:47 +0300 | [diff] [blame] | 1376 | |
Chao Peng | f6d079c | 2018-10-24 16:05:06 +0800 | [diff] [blame] | 1377 | if (++range > intel_pt_validate_hw_cap(PT_CAP_num_address_ranges)) |
Alexander Shishkin | eadf48c | 2016-04-27 18:44:47 +0300 | [diff] [blame] | 1378 | return -EOPNOTSUPP; |
| 1379 | } |
| 1380 | |
| 1381 | return 0; |
| 1382 | } |
| 1383 | |
| 1384 | static void pt_event_addr_filters_sync(struct perf_event *event) |
| 1385 | { |
| 1386 | struct perf_addr_filters_head *head = perf_event_addr_filters(event); |
Alexander Shishkin | c60f83b | 2019-02-15 13:56:55 +0200 | [diff] [blame] | 1387 | unsigned long msr_a, msr_b; |
| 1388 | struct perf_addr_filter_range *fr = event->addr_filter_ranges; |
Alexander Shishkin | eadf48c | 2016-04-27 18:44:47 +0300 | [diff] [blame] | 1389 | struct pt_filters *filters = event->hw.addr_filters; |
| 1390 | struct perf_addr_filter *filter; |
| 1391 | int range = 0; |
| 1392 | |
| 1393 | if (!filters) |
| 1394 | return; |
| 1395 | |
| 1396 | list_for_each_entry(filter, &head->list, entry) { |
Alexander Shishkin | c60f83b | 2019-02-15 13:56:55 +0200 | [diff] [blame] | 1397 | if (filter->path.dentry && !fr[range].start) { |
Alexander Shishkin | eadf48c | 2016-04-27 18:44:47 +0300 | [diff] [blame] | 1398 | msr_a = msr_b = 0; |
| 1399 | } else { |
| 1400 | /* apply the offset */ |
Alexander Shishkin | c60f83b | 2019-02-15 13:56:55 +0200 | [diff] [blame] | 1401 | msr_a = fr[range].start; |
| 1402 | msr_b = msr_a + fr[range].size - 1; |
Alexander Shishkin | eadf48c | 2016-04-27 18:44:47 +0300 | [diff] [blame] | 1403 | } |
| 1404 | |
| 1405 | filters->filter[range].msr_a = msr_a; |
| 1406 | filters->filter[range].msr_b = msr_b; |
Alexander Shishkin | 6ed70cf | 2018-03-29 15:06:48 +0300 | [diff] [blame] | 1407 | if (filter->action == PERF_ADDR_FILTER_ACTION_FILTER) |
| 1408 | filters->filter[range].config = 1; |
| 1409 | else |
| 1410 | filters->filter[range].config = 2; |
Alexander Shishkin | eadf48c | 2016-04-27 18:44:47 +0300 | [diff] [blame] | 1411 | range++; |
| 1412 | } |
| 1413 | |
| 1414 | filters->nr_filters = range; |
| 1415 | } |
| 1416 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1417 | /** |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1418 | * intel_pt_interrupt() - PT PMI handler |
| 1419 | */ |
| 1420 | void intel_pt_interrupt(void) |
| 1421 | { |
| 1422 | struct pt *pt = this_cpu_ptr(&pt_ctx); |
| 1423 | struct pt_buffer *buf; |
| 1424 | struct perf_event *event = pt->handle.event; |
| 1425 | |
| 1426 | /* |
| 1427 | * There may be a dangling PT bit in the interrupt status register |
| 1428 | * after PT has been disabled by pt_event_stop(). Make sure we don't |
| 1429 | * do anything (particularly, re-enable) for this event here. |
| 1430 | */ |
Alexander Shishkin | 1b6de59 | 2016-04-28 18:35:44 +0300 | [diff] [blame] | 1431 | if (!READ_ONCE(pt->handle_nmi)) |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1432 | return; |
| 1433 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1434 | if (!event) |
| 1435 | return; |
| 1436 | |
Alexander Shishkin | 1c5ac21 | 2016-03-29 17:43:10 +0300 | [diff] [blame] | 1437 | pt_config_stop(event); |
| 1438 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1439 | buf = perf_get_aux(&pt->handle); |
| 1440 | if (!buf) |
| 1441 | return; |
| 1442 | |
| 1443 | pt_read_offset(buf); |
| 1444 | |
| 1445 | pt_handle_status(pt); |
| 1446 | |
| 1447 | pt_update_head(pt); |
| 1448 | |
Will Deacon | f4c0b0a | 2017-02-20 15:33:50 +0200 | [diff] [blame] | 1449 | perf_aux_output_end(&pt->handle, local_xchg(&buf->data_size, 0)); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1450 | |
| 1451 | if (!event->hw.state) { |
| 1452 | int ret; |
| 1453 | |
| 1454 | buf = perf_aux_output_begin(&pt->handle, event); |
| 1455 | if (!buf) { |
| 1456 | event->hw.state = PERF_HES_STOPPED; |
| 1457 | return; |
| 1458 | } |
| 1459 | |
| 1460 | pt_buffer_reset_offsets(buf, pt->handle.head); |
Alexander Shishkin | cf302bf | 2015-04-21 16:16:15 +0300 | [diff] [blame] | 1461 | /* snapshot counters don't use PMI, so it's safe */ |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1462 | ret = pt_buffer_reset_markers(buf, &pt->handle); |
| 1463 | if (ret) { |
Will Deacon | f4c0b0a | 2017-02-20 15:33:50 +0200 | [diff] [blame] | 1464 | perf_aux_output_end(&pt->handle, 0); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1465 | return; |
| 1466 | } |
| 1467 | |
Alexander Shishkin | 6706384 | 2019-11-05 10:27:00 +0200 | [diff] [blame] | 1468 | pt_config_buffer(buf); |
Alexander Shishkin | 8e105a1 | 2019-10-25 17:08:34 +0300 | [diff] [blame] | 1469 | pt_config_start(event); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1470 | } |
| 1471 | } |
| 1472 | |
Alexander Shishkin | 1c5ac21 | 2016-03-29 17:43:10 +0300 | [diff] [blame] | 1473 | void intel_pt_handle_vmx(int on) |
| 1474 | { |
| 1475 | struct pt *pt = this_cpu_ptr(&pt_ctx); |
| 1476 | struct perf_event *event; |
| 1477 | unsigned long flags; |
| 1478 | |
| 1479 | /* PT plays nice with VMX, do nothing */ |
| 1480 | if (pt_pmu.vmx) |
| 1481 | return; |
| 1482 | |
| 1483 | /* |
| 1484 | * VMXON will clear RTIT_CTL.TraceEn; we need to make |
| 1485 | * sure to not try to set it while VMX is on. Disable |
| 1486 | * interrupts to avoid racing with pmu callbacks; |
| 1487 | * concurrent PMI should be handled fine. |
| 1488 | */ |
| 1489 | local_irq_save(flags); |
| 1490 | WRITE_ONCE(pt->vmx_on, on); |
| 1491 | |
Alexander Shishkin | ee36842 | 2017-02-20 15:33:52 +0200 | [diff] [blame] | 1492 | /* |
| 1493 | * If an AUX transaction is in progress, it will contain |
| 1494 | * gap(s), so flag it PARTIAL to inform the user. |
| 1495 | */ |
| 1496 | event = pt->handle.event; |
| 1497 | if (event) |
| 1498 | perf_aux_output_flag(&pt->handle, |
| 1499 | PERF_AUX_FLAG_PARTIAL); |
| 1500 | |
| 1501 | /* Turn PTs back on */ |
| 1502 | if (!on && event) |
| 1503 | wrmsrl(MSR_IA32_RTIT_CTL, event->hw.config); |
| 1504 | |
Alexander Shishkin | 1c5ac21 | 2016-03-29 17:43:10 +0300 | [diff] [blame] | 1505 | local_irq_restore(flags); |
| 1506 | } |
| 1507 | EXPORT_SYMBOL_GPL(intel_pt_handle_vmx); |
| 1508 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1509 | /* |
| 1510 | * PMU callbacks |
| 1511 | */ |
| 1512 | |
| 1513 | static void pt_event_start(struct perf_event *event, int mode) |
| 1514 | { |
Alexander Shishkin | 66d2190 | 2016-03-04 15:42:48 +0200 | [diff] [blame] | 1515 | struct hw_perf_event *hwc = &event->hw; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1516 | struct pt *pt = this_cpu_ptr(&pt_ctx); |
Alexander Shishkin | 66d2190 | 2016-03-04 15:42:48 +0200 | [diff] [blame] | 1517 | struct pt_buffer *buf; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1518 | |
Alexander Shishkin | 66d2190 | 2016-03-04 15:42:48 +0200 | [diff] [blame] | 1519 | buf = perf_aux_output_begin(&pt->handle, event); |
| 1520 | if (!buf) |
| 1521 | goto fail_stop; |
| 1522 | |
| 1523 | pt_buffer_reset_offsets(buf, pt->handle.head); |
| 1524 | if (!buf->snapshot) { |
| 1525 | if (pt_buffer_reset_markers(buf, &pt->handle)) |
| 1526 | goto fail_end_stop; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1527 | } |
| 1528 | |
Alexander Shishkin | 1b6de59 | 2016-04-28 18:35:44 +0300 | [diff] [blame] | 1529 | WRITE_ONCE(pt->handle_nmi, 1); |
Alexander Shishkin | 66d2190 | 2016-03-04 15:42:48 +0200 | [diff] [blame] | 1530 | hwc->state = 0; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1531 | |
Alexander Shishkin | 6706384 | 2019-11-05 10:27:00 +0200 | [diff] [blame] | 1532 | pt_config_buffer(buf); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1533 | pt_config(event); |
Alexander Shishkin | 66d2190 | 2016-03-04 15:42:48 +0200 | [diff] [blame] | 1534 | |
| 1535 | return; |
| 1536 | |
| 1537 | fail_end_stop: |
Will Deacon | f4c0b0a | 2017-02-20 15:33:50 +0200 | [diff] [blame] | 1538 | perf_aux_output_end(&pt->handle, 0); |
Alexander Shishkin | 66d2190 | 2016-03-04 15:42:48 +0200 | [diff] [blame] | 1539 | fail_stop: |
| 1540 | hwc->state = PERF_HES_STOPPED; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1541 | } |
| 1542 | |
| 1543 | static void pt_event_stop(struct perf_event *event, int mode) |
| 1544 | { |
| 1545 | struct pt *pt = this_cpu_ptr(&pt_ctx); |
| 1546 | |
| 1547 | /* |
| 1548 | * Protect against the PMI racing with disabling wrmsr, |
| 1549 | * see comment in intel_pt_interrupt(). |
| 1550 | */ |
Alexander Shishkin | 1b6de59 | 2016-04-28 18:35:44 +0300 | [diff] [blame] | 1551 | WRITE_ONCE(pt->handle_nmi, 0); |
Alexander Shishkin | 1c5ac21 | 2016-03-29 17:43:10 +0300 | [diff] [blame] | 1552 | |
| 1553 | pt_config_stop(event); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1554 | |
| 1555 | if (event->hw.state == PERF_HES_STOPPED) |
| 1556 | return; |
| 1557 | |
| 1558 | event->hw.state = PERF_HES_STOPPED; |
| 1559 | |
| 1560 | if (mode & PERF_EF_UPDATE) { |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1561 | struct pt_buffer *buf = perf_get_aux(&pt->handle); |
| 1562 | |
| 1563 | if (!buf) |
| 1564 | return; |
| 1565 | |
| 1566 | if (WARN_ON_ONCE(pt->handle.event != event)) |
| 1567 | return; |
| 1568 | |
| 1569 | pt_read_offset(buf); |
| 1570 | |
| 1571 | pt_handle_status(pt); |
| 1572 | |
| 1573 | pt_update_head(pt); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1574 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1575 | if (buf->snapshot) |
| 1576 | pt->handle.head = |
| 1577 | local_xchg(&buf->data_size, |
| 1578 | buf->nr_pages << PAGE_SHIFT); |
Will Deacon | f4c0b0a | 2017-02-20 15:33:50 +0200 | [diff] [blame] | 1579 | perf_aux_output_end(&pt->handle, local_xchg(&buf->data_size, 0)); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1580 | } |
| 1581 | } |
| 1582 | |
Alexander Shishkin | 25e8920 | 2019-10-25 17:08:35 +0300 | [diff] [blame] | 1583 | static long pt_event_snapshot_aux(struct perf_event *event, |
| 1584 | struct perf_output_handle *handle, |
| 1585 | unsigned long size) |
| 1586 | { |
| 1587 | struct pt *pt = this_cpu_ptr(&pt_ctx); |
| 1588 | struct pt_buffer *buf = perf_get_aux(&pt->handle); |
| 1589 | unsigned long from = 0, to; |
| 1590 | long ret; |
| 1591 | |
| 1592 | if (WARN_ON_ONCE(!buf)) |
| 1593 | return 0; |
| 1594 | |
| 1595 | /* |
| 1596 | * Sampling is only allowed on snapshot events; |
| 1597 | * see pt_buffer_setup_aux(). |
| 1598 | */ |
| 1599 | if (WARN_ON_ONCE(!buf->snapshot)) |
| 1600 | return 0; |
| 1601 | |
| 1602 | /* |
| 1603 | * Here, handle_nmi tells us if the tracing is on |
| 1604 | */ |
| 1605 | if (READ_ONCE(pt->handle_nmi)) |
| 1606 | pt_config_stop(event); |
| 1607 | |
| 1608 | pt_read_offset(buf); |
| 1609 | pt_update_head(pt); |
| 1610 | |
| 1611 | to = local_read(&buf->data_size); |
| 1612 | if (to < size) |
| 1613 | from = buf->nr_pages << PAGE_SHIFT; |
| 1614 | from += to - size; |
| 1615 | |
| 1616 | ret = perf_output_copy_aux(&pt->handle, handle, from, to); |
| 1617 | |
| 1618 | /* |
| 1619 | * If the tracing was on when we turned up, restart it. |
| 1620 | * Compiler barrier not needed as we couldn't have been |
| 1621 | * preempted by anything that touches pt->handle_nmi. |
| 1622 | */ |
| 1623 | if (pt->handle_nmi) |
| 1624 | pt_config_start(event); |
| 1625 | |
| 1626 | return ret; |
| 1627 | } |
| 1628 | |
Alexander Shishkin | 66d2190 | 2016-03-04 15:42:48 +0200 | [diff] [blame] | 1629 | static void pt_event_del(struct perf_event *event, int mode) |
| 1630 | { |
| 1631 | pt_event_stop(event, PERF_EF_UPDATE); |
| 1632 | } |
| 1633 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1634 | static int pt_event_add(struct perf_event *event, int mode) |
| 1635 | { |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1636 | struct pt *pt = this_cpu_ptr(&pt_ctx); |
| 1637 | struct hw_perf_event *hwc = &event->hw; |
| 1638 | int ret = -EBUSY; |
| 1639 | |
| 1640 | if (pt->handle.event) |
Ingo Molnar | 0c99241 | 2015-04-16 12:38:30 +0200 | [diff] [blame] | 1641 | goto fail; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1642 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1643 | if (mode & PERF_EF_START) { |
| 1644 | pt_event_start(event, 0); |
Alexander Shishkin | 66d2190 | 2016-03-04 15:42:48 +0200 | [diff] [blame] | 1645 | ret = -EINVAL; |
Ingo Molnar | 0c99241 | 2015-04-16 12:38:30 +0200 | [diff] [blame] | 1646 | if (hwc->state == PERF_HES_STOPPED) |
Alexander Shishkin | 66d2190 | 2016-03-04 15:42:48 +0200 | [diff] [blame] | 1647 | goto fail; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1648 | } else { |
| 1649 | hwc->state = PERF_HES_STOPPED; |
| 1650 | } |
| 1651 | |
Alexander Shishkin | 66d2190 | 2016-03-04 15:42:48 +0200 | [diff] [blame] | 1652 | ret = 0; |
Ingo Molnar | 0c99241 | 2015-04-16 12:38:30 +0200 | [diff] [blame] | 1653 | fail: |
Alexander Shishkin | 66d2190 | 2016-03-04 15:42:48 +0200 | [diff] [blame] | 1654 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1655 | return ret; |
| 1656 | } |
| 1657 | |
| 1658 | static void pt_event_read(struct perf_event *event) |
| 1659 | { |
| 1660 | } |
| 1661 | |
| 1662 | static void pt_event_destroy(struct perf_event *event) |
| 1663 | { |
Alexander Shishkin | eadf48c | 2016-04-27 18:44:47 +0300 | [diff] [blame] | 1664 | pt_addr_filters_fini(event); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1665 | x86_del_exclusive(x86_lbr_exclusive_pt); |
| 1666 | } |
| 1667 | |
| 1668 | static int pt_event_init(struct perf_event *event) |
| 1669 | { |
| 1670 | if (event->attr.type != pt_pmu.pmu.type) |
| 1671 | return -ENOENT; |
| 1672 | |
| 1673 | if (!pt_event_valid(event)) |
| 1674 | return -EINVAL; |
| 1675 | |
| 1676 | if (x86_add_exclusive(x86_lbr_exclusive_pt)) |
| 1677 | return -EBUSY; |
| 1678 | |
Alexander Shishkin | eadf48c | 2016-04-27 18:44:47 +0300 | [diff] [blame] | 1679 | if (pt_addr_filters_init(event)) { |
| 1680 | x86_del_exclusive(x86_lbr_exclusive_pt); |
| 1681 | return -ENOMEM; |
| 1682 | } |
| 1683 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1684 | event->destroy = pt_event_destroy; |
| 1685 | |
| 1686 | return 0; |
| 1687 | } |
| 1688 | |
Takao Indoh | 24cc12b | 2015-11-04 14:22:32 +0900 | [diff] [blame] | 1689 | void cpu_emergency_stop_pt(void) |
| 1690 | { |
| 1691 | struct pt *pt = this_cpu_ptr(&pt_ctx); |
| 1692 | |
| 1693 | if (pt->handle.event) |
| 1694 | pt_event_stop(pt->handle.event, PERF_EF_UPDATE); |
| 1695 | } |
| 1696 | |
Alexander Shishkin | 42880f7 | 2019-08-06 11:46:01 +0300 | [diff] [blame] | 1697 | int is_intel_pt_event(struct perf_event *event) |
| 1698 | { |
| 1699 | return event->pmu == &pt_pmu.pmu; |
| 1700 | } |
| 1701 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1702 | static __init int pt_init(void) |
| 1703 | { |
| 1704 | int ret, cpu, prior_warn = 0; |
| 1705 | |
| 1706 | BUILD_BUG_ON(sizeof(struct topa) > PAGE_SIZE); |
Huaitong Han | 73fdeb6 | 2015-08-31 16:21:02 +0800 | [diff] [blame] | 1707 | |
Alexander Shishkin | e465de1 | 2016-04-06 17:35:07 +0300 | [diff] [blame] | 1708 | if (!boot_cpu_has(X86_FEATURE_INTEL_PT)) |
Huaitong Han | 73fdeb6 | 2015-08-31 16:21:02 +0800 | [diff] [blame] | 1709 | return -ENODEV; |
| 1710 | |
Sebastian Andrzej Siewior | eda8a2c | 2021-08-03 16:15:53 +0200 | [diff] [blame^] | 1711 | cpus_read_lock(); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1712 | for_each_online_cpu(cpu) { |
| 1713 | u64 ctl; |
| 1714 | |
| 1715 | ret = rdmsrl_safe_on_cpu(cpu, MSR_IA32_RTIT_CTL, &ctl); |
| 1716 | if (!ret && (ctl & RTIT_CTL_TRACEEN)) |
| 1717 | prior_warn++; |
| 1718 | } |
Sebastian Andrzej Siewior | eda8a2c | 2021-08-03 16:15:53 +0200 | [diff] [blame^] | 1719 | cpus_read_unlock(); |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1720 | |
| 1721 | if (prior_warn) { |
| 1722 | x86_add_exclusive(x86_lbr_exclusive_pt); |
| 1723 | pr_warn("PT is enabled at boot time, doing nothing\n"); |
| 1724 | |
| 1725 | return -EBUSY; |
| 1726 | } |
| 1727 | |
| 1728 | ret = pt_pmu_hw_init(); |
| 1729 | if (ret) |
| 1730 | return ret; |
| 1731 | |
Chao Peng | f6d079c | 2018-10-24 16:05:06 +0800 | [diff] [blame] | 1732 | if (!intel_pt_validate_hw_cap(PT_CAP_topa_output)) { |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1733 | pr_warn("ToPA output is not supported on this CPU\n"); |
| 1734 | return -ENODEV; |
| 1735 | } |
| 1736 | |
Chao Peng | f6d079c | 2018-10-24 16:05:06 +0800 | [diff] [blame] | 1737 | if (!intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries)) |
Alexander Shishkin | 72e830f | 2019-05-03 11:55:36 +0300 | [diff] [blame] | 1738 | pt_pmu.pmu.capabilities = PERF_PMU_CAP_AUX_NO_SG; |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1739 | |
| 1740 | pt_pmu.pmu.capabilities |= PERF_PMU_CAP_EXCLUSIVE | PERF_PMU_CAP_ITRACE; |
Alexander Shishkin | eadf48c | 2016-04-27 18:44:47 +0300 | [diff] [blame] | 1741 | pt_pmu.pmu.attr_groups = pt_attr_groups; |
| 1742 | pt_pmu.pmu.task_ctx_nr = perf_sw_context; |
| 1743 | pt_pmu.pmu.event_init = pt_event_init; |
| 1744 | pt_pmu.pmu.add = pt_event_add; |
| 1745 | pt_pmu.pmu.del = pt_event_del; |
| 1746 | pt_pmu.pmu.start = pt_event_start; |
| 1747 | pt_pmu.pmu.stop = pt_event_stop; |
Alexander Shishkin | 25e8920 | 2019-10-25 17:08:35 +0300 | [diff] [blame] | 1748 | pt_pmu.pmu.snapshot_aux = pt_event_snapshot_aux; |
Alexander Shishkin | eadf48c | 2016-04-27 18:44:47 +0300 | [diff] [blame] | 1749 | pt_pmu.pmu.read = pt_event_read; |
| 1750 | pt_pmu.pmu.setup_aux = pt_buffer_setup_aux; |
| 1751 | pt_pmu.pmu.free_aux = pt_buffer_free_aux; |
| 1752 | pt_pmu.pmu.addr_filters_sync = pt_event_addr_filters_sync; |
| 1753 | pt_pmu.pmu.addr_filters_validate = pt_event_addr_filters_validate; |
| 1754 | pt_pmu.pmu.nr_addr_filters = |
Chao Peng | f6d079c | 2018-10-24 16:05:06 +0800 | [diff] [blame] | 1755 | intel_pt_validate_hw_cap(PT_CAP_num_address_ranges); |
Alexander Shishkin | eadf48c | 2016-04-27 18:44:47 +0300 | [diff] [blame] | 1756 | |
Alexander Shishkin | 52ca9ce | 2015-01-30 12:39:52 +0200 | [diff] [blame] | 1757 | ret = perf_pmu_register(&pt_pmu.pmu, "intel_pt", -1); |
| 1758 | |
| 1759 | return ret; |
| 1760 | } |
Paul Gortmaker | 5b00c1e | 2015-05-01 21:57:34 -0400 | [diff] [blame] | 1761 | arch_initcall(pt_init); |