blob: e05d92d67525002822f9afb888125f6ca8cd416f [file] [log] [blame]
Thomas Gleixnerf6cc69f2019-05-29 16:57:24 -07001// SPDX-License-Identifier: GPL-2.0-only
Jacob Pan2d281d82013-10-17 10:28:35 -07002/*
3 * Intel Running Average Power Limit (RAPL) Driver
4 * Copyright (c) 2013, Intel Corporation.
Jacob Pan2d281d82013-10-17 10:28:35 -07005 */
6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7
8#include <linux/kernel.h>
9#include <linux/module.h>
10#include <linux/list.h>
11#include <linux/types.h>
12#include <linux/device.h>
13#include <linux/slab.h>
14#include <linux/log2.h>
15#include <linux/bitmap.h>
16#include <linux/delay.h>
17#include <linux/sysfs.h>
18#include <linux/cpu.h>
19#include <linux/powercap.h>
Zhen Han52b36722018-01-10 08:38:23 +080020#include <linux/suspend.h>
Zhang Ruiff956822019-07-10 21:44:24 +080021#include <linux/intel_rapl.h>
Jacob Pan2d281d82013-10-17 10:28:35 -070022
Zhang Ruiff956822019-07-10 21:44:24 +080023#include <asm/iosf_mbi.h>
Jacob Pan2d281d82013-10-17 10:28:35 -070024#include <asm/processor.h>
25#include <asm/cpu_device_id.h>
Dave Hansen62d16732016-06-02 17:19:36 -070026#include <asm/intel-family.h>
Jacob Pan2d281d82013-10-17 10:28:35 -070027
Srinivas Pandruvada3521ba12016-04-17 15:03:01 -070028/* Local defines */
29#define MSR_PLATFORM_POWER_LIMIT 0x0000065C
30
Jacob Pan2d281d82013-10-17 10:28:35 -070031/* bitmasks for RAPL MSRs, used by primitive access functions */
32#define ENERGY_STATUS_MASK 0xffffffff
33
34#define POWER_LIMIT1_MASK 0x7FFF
35#define POWER_LIMIT1_ENABLE BIT(15)
36#define POWER_LIMIT1_CLAMP BIT(16)
37
38#define POWER_LIMIT2_MASK (0x7FFFULL<<32)
39#define POWER_LIMIT2_ENABLE BIT_ULL(47)
40#define POWER_LIMIT2_CLAMP BIT_ULL(48)
41#define POWER_PACKAGE_LOCK BIT_ULL(63)
42#define POWER_PP_LOCK BIT(31)
43
44#define TIME_WINDOW1_MASK (0x7FULL<<17)
45#define TIME_WINDOW2_MASK (0x7FULL<<49)
46
47#define POWER_UNIT_OFFSET 0
48#define POWER_UNIT_MASK 0x0F
49
50#define ENERGY_UNIT_OFFSET 0x08
51#define ENERGY_UNIT_MASK 0x1F00
52
53#define TIME_UNIT_OFFSET 0x10
54#define TIME_UNIT_MASK 0xF0000
55
56#define POWER_INFO_MAX_MASK (0x7fffULL<<32)
57#define POWER_INFO_MIN_MASK (0x7fffULL<<16)
58#define POWER_INFO_MAX_TIME_WIN_MASK (0x3fULL<<48)
59#define POWER_INFO_THERMAL_SPEC_MASK 0x7fff
60
61#define PERF_STATUS_THROTTLE_TIME_MASK 0xffffffff
62#define PP_POLICY_MASK 0x1F
63
64/* Non HW constants */
65#define RAPL_PRIMITIVE_DERIVED BIT(1) /* not from raw data */
66#define RAPL_PRIMITIVE_DUMMY BIT(2)
67
Jacob Pan2d281d82013-10-17 10:28:35 -070068#define TIME_WINDOW_MAX_MSEC 40000
69#define TIME_WINDOW_MIN_MSEC 250
Jacob Pand474a4d2015-03-13 03:48:56 -070070#define ENERGY_UNIT_SCALE 1000 /* scale from driver unit to powercap unit */
Jacob Pan2d281d82013-10-17 10:28:35 -070071enum unit_type {
72 ARBITRARY_UNIT, /* no translation */
73 POWER_UNIT,
74 ENERGY_UNIT,
75 TIME_UNIT,
76};
77
Zhang Rui7ebf8ef2019-07-10 21:44:25 +080078/* private data for RAPL MSR Interface */
79static struct rapl_if_priv rapl_msr_priv;
80
Jacob Pan2d281d82013-10-17 10:28:35 -070081/* per domain data, some are optional */
Jacob Pan2d281d82013-10-17 10:28:35 -070082#define NR_RAW_PRIMITIVES (NR_RAPL_PRIMITIVES - 2)
83
Jacob Panf14a1392016-02-24 13:31:36 -080084struct msrl_action {
85 u32 msr_no;
86 u64 clear_mask;
87 u64 set_mask;
88 int err;
89};
Jacob Pan2d281d82013-10-17 10:28:35 -070090
91#define DOMAIN_STATE_INACTIVE BIT(0)
92#define DOMAIN_STATE_POWER_LIMIT_SET BIT(1)
93#define DOMAIN_STATE_BIOS_LOCKED BIT(2)
94
Jacob Pan2d281d82013-10-17 10:28:35 -070095static const char pl1_name[] = "long_term";
96static const char pl2_name[] = "short_term";
97
Jacob Pan2d281d82013-10-17 10:28:35 -070098#define power_zone_to_rapl_domain(_zone) \
99 container_of(_zone, struct rapl_domain, power_zone)
100
Jacob Pan087e9cb2014-11-07 09:29:25 -0800101struct rapl_defaults {
Ajay Thomas51b63402015-04-30 01:43:23 +0530102 u8 floor_freq_reg_addr;
Jacob Pan087e9cb2014-11-07 09:29:25 -0800103 int (*check_unit)(struct rapl_package *rp, int cpu);
104 void (*set_floor_freq)(struct rapl_domain *rd, bool mode);
105 u64 (*compute_time_window)(struct rapl_package *rp, u64 val,
106 bool to_raw);
Jacob Pand474a4d2015-03-13 03:48:56 -0700107 unsigned int dram_domain_energy_unit;
Jacob Pan087e9cb2014-11-07 09:29:25 -0800108};
109static struct rapl_defaults *rapl_defaults;
110
Jacob Pan3c2c0842014-11-07 09:29:26 -0800111/* Sideband MBI registers */
Ajay Thomas51b63402015-04-30 01:43:23 +0530112#define IOSF_CPU_POWER_BUDGET_CTL_BYT (0x2)
113#define IOSF_CPU_POWER_BUDGET_CTL_TNG (0xdf)
Jacob Pan3c2c0842014-11-07 09:29:26 -0800114
Jacob Pan2d281d82013-10-17 10:28:35 -0700115#define PACKAGE_PLN_INT_SAVED BIT(0)
116#define MAX_PRIM_NAME (32)
117
118/* per domain data. used to describe individual knobs such that access function
119 * can be consolidated into one instead of many inline functions.
120 */
121struct rapl_primitive_info {
122 const char *name;
123 u64 mask;
124 int shift;
Zhang Ruif7c4e0c2019-07-10 21:44:22 +0800125 enum rapl_domain_reg_id id;
Jacob Pan2d281d82013-10-17 10:28:35 -0700126 enum unit_type unit;
127 u32 flag;
128};
129
130#define PRIMITIVE_INFO_INIT(p, m, s, i, u, f) { \
131 .name = #p, \
132 .mask = m, \
133 .shift = s, \
134 .id = i, \
135 .unit = u, \
136 .flag = f \
137 }
138
139static void rapl_init_domains(struct rapl_package *rp);
140static int rapl_read_data_raw(struct rapl_domain *rd,
141 enum rapl_primitives prim,
142 bool xlate, u64 *data);
143static int rapl_write_data_raw(struct rapl_domain *rd,
144 enum rapl_primitives prim,
145 unsigned long long value);
Jacob Pan309557f2016-02-24 13:31:37 -0800146static u64 rapl_unit_xlate(struct rapl_domain *rd,
Jacob Pand474a4d2015-03-13 03:48:56 -0700147 enum unit_type type, u64 value,
Jacob Pan2d281d82013-10-17 10:28:35 -0700148 int to_raw);
Jacob Pan309557f2016-02-24 13:31:37 -0800149static void package_power_limit_irq_save(struct rapl_package *rp);
Jacob Pan2d281d82013-10-17 10:28:35 -0700150
151static LIST_HEAD(rapl_packages); /* guarded by CPU hotplug lock */
152
153static const char * const rapl_domain_names[] = {
154 "package",
155 "core",
156 "uncore",
157 "dram",
Srinivas Pandruvada3521ba12016-04-17 15:03:01 -0700158 "psys",
Jacob Pan2d281d82013-10-17 10:28:35 -0700159};
160
Jacob Pan2d281d82013-10-17 10:28:35 -0700161/* caller to ensure CPU hotplug lock is held */
Zhang Rui7ebf8ef2019-07-10 21:44:25 +0800162static struct rapl_package *rapl_find_package_domain(int cpu, struct rapl_if_priv *priv)
Jacob Pan2d281d82013-10-17 10:28:35 -0700163{
Zhang Rui32fb4802019-05-13 13:58:51 -0400164 int id = topology_logical_die_id(cpu);
Jacob Pan2d281d82013-10-17 10:28:35 -0700165 struct rapl_package *rp;
166
167 list_for_each_entry(rp, &rapl_packages, plist) {
Zhang Rui7ebf8ef2019-07-10 21:44:25 +0800168 if (rp->id == id && rp->priv->control_type == priv->control_type)
Jacob Pan2d281d82013-10-17 10:28:35 -0700169 return rp;
170 }
171
172 return NULL;
173}
174
Jacob Pan2d281d82013-10-17 10:28:35 -0700175static int get_energy_counter(struct powercap_zone *power_zone, u64 *energy_raw)
176{
177 struct rapl_domain *rd;
178 u64 energy_now;
179
180 /* prevent CPU hotplug, make sure the RAPL domain does not go
181 * away while reading the counter.
182 */
183 get_online_cpus();
184 rd = power_zone_to_rapl_domain(power_zone);
185
186 if (!rapl_read_data_raw(rd, ENERGY_COUNTER, true, &energy_now)) {
187 *energy_raw = energy_now;
188 put_online_cpus();
189
190 return 0;
191 }
192 put_online_cpus();
193
194 return -EIO;
195}
196
197static int get_max_energy_counter(struct powercap_zone *pcd_dev, u64 *energy)
198{
Jacob Pand474a4d2015-03-13 03:48:56 -0700199 struct rapl_domain *rd = power_zone_to_rapl_domain(pcd_dev);
200
Jacob Pan309557f2016-02-24 13:31:37 -0800201 *energy = rapl_unit_xlate(rd, ENERGY_UNIT, ENERGY_STATUS_MASK, 0);
Jacob Pan2d281d82013-10-17 10:28:35 -0700202 return 0;
203}
204
205static int release_zone(struct powercap_zone *power_zone)
206{
207 struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone);
Jacob Pan309557f2016-02-24 13:31:37 -0800208 struct rapl_package *rp = rd->rp;
Jacob Pan2d281d82013-10-17 10:28:35 -0700209
210 /* package zone is the last zone of a package, we can free
211 * memory here since all children has been unregistered.
212 */
213 if (rd->id == RAPL_DOMAIN_PACKAGE) {
Jacob Pan2d281d82013-10-17 10:28:35 -0700214 kfree(rd);
215 rp->domains = NULL;
216 }
217
218 return 0;
219
220}
221
222static int find_nr_power_limit(struct rapl_domain *rd)
223{
Jacob Pane1399ba2016-05-31 13:41:29 -0700224 int i, nr_pl = 0;
Jacob Pan2d281d82013-10-17 10:28:35 -0700225
226 for (i = 0; i < NR_POWER_LIMITS; i++) {
Jacob Pane1399ba2016-05-31 13:41:29 -0700227 if (rd->rpl[i].name)
228 nr_pl++;
Jacob Pan2d281d82013-10-17 10:28:35 -0700229 }
230
Jacob Pane1399ba2016-05-31 13:41:29 -0700231 return nr_pl;
Jacob Pan2d281d82013-10-17 10:28:35 -0700232}
233
234static int set_domain_enable(struct powercap_zone *power_zone, bool mode)
235{
236 struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone);
Jacob Pan2d281d82013-10-17 10:28:35 -0700237
238 if (rd->state & DOMAIN_STATE_BIOS_LOCKED)
239 return -EACCES;
Jacob Pan3c2c0842014-11-07 09:29:26 -0800240
Jacob Pan2d281d82013-10-17 10:28:35 -0700241 get_online_cpus();
Jacob Pan2d281d82013-10-17 10:28:35 -0700242 rapl_write_data_raw(rd, PL1_ENABLE, mode);
Ajay Thomas51b63402015-04-30 01:43:23 +0530243 if (rapl_defaults->set_floor_freq)
244 rapl_defaults->set_floor_freq(rd, mode);
Jacob Pan2d281d82013-10-17 10:28:35 -0700245 put_online_cpus();
246
247 return 0;
248}
249
250static int get_domain_enable(struct powercap_zone *power_zone, bool *mode)
251{
252 struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone);
253 u64 val;
254
255 if (rd->state & DOMAIN_STATE_BIOS_LOCKED) {
256 *mode = false;
257 return 0;
258 }
259 get_online_cpus();
260 if (rapl_read_data_raw(rd, PL1_ENABLE, true, &val)) {
261 put_online_cpus();
262 return -EIO;
263 }
264 *mode = val;
265 put_online_cpus();
266
267 return 0;
268}
269
270/* per RAPL domain ops, in the order of rapl_domain_type */
Julia Lawall600c3952015-12-23 22:59:55 +0100271static const struct powercap_zone_ops zone_ops[] = {
Jacob Pan2d281d82013-10-17 10:28:35 -0700272 /* RAPL_DOMAIN_PACKAGE */
273 {
274 .get_energy_uj = get_energy_counter,
275 .get_max_energy_range_uj = get_max_energy_counter,
276 .release = release_zone,
277 .set_enable = set_domain_enable,
278 .get_enable = get_domain_enable,
279 },
280 /* RAPL_DOMAIN_PP0 */
281 {
282 .get_energy_uj = get_energy_counter,
283 .get_max_energy_range_uj = get_max_energy_counter,
284 .release = release_zone,
285 .set_enable = set_domain_enable,
286 .get_enable = get_domain_enable,
287 },
288 /* RAPL_DOMAIN_PP1 */
289 {
290 .get_energy_uj = get_energy_counter,
291 .get_max_energy_range_uj = get_max_energy_counter,
292 .release = release_zone,
293 .set_enable = set_domain_enable,
294 .get_enable = get_domain_enable,
295 },
296 /* RAPL_DOMAIN_DRAM */
297 {
298 .get_energy_uj = get_energy_counter,
299 .get_max_energy_range_uj = get_max_energy_counter,
300 .release = release_zone,
301 .set_enable = set_domain_enable,
302 .get_enable = get_domain_enable,
303 },
Srinivas Pandruvada3521ba12016-04-17 15:03:01 -0700304 /* RAPL_DOMAIN_PLATFORM */
305 {
306 .get_energy_uj = get_energy_counter,
307 .get_max_energy_range_uj = get_max_energy_counter,
308 .release = release_zone,
309 .set_enable = set_domain_enable,
310 .get_enable = get_domain_enable,
311 },
Jacob Pan2d281d82013-10-17 10:28:35 -0700312};
313
Jacob Pane1399ba2016-05-31 13:41:29 -0700314
315/*
316 * Constraint index used by powercap can be different than power limit (PL)
317 * index in that some PLs maybe missing due to non-existant MSRs. So we
318 * need to convert here by finding the valid PLs only (name populated).
319 */
320static int contraint_to_pl(struct rapl_domain *rd, int cid)
321{
322 int i, j;
323
324 for (i = 0, j = 0; i < NR_POWER_LIMITS; i++) {
325 if ((rd->rpl[i].name) && j++ == cid) {
326 pr_debug("%s: index %d\n", __func__, i);
327 return i;
328 }
329 }
Jacob Pancb43f812016-11-28 13:53:11 -0800330 pr_err("Cannot find matching power limit for constraint %d\n", cid);
Jacob Pane1399ba2016-05-31 13:41:29 -0700331
332 return -EINVAL;
333}
334
335static int set_power_limit(struct powercap_zone *power_zone, int cid,
Jacob Pan2d281d82013-10-17 10:28:35 -0700336 u64 power_limit)
337{
338 struct rapl_domain *rd;
339 struct rapl_package *rp;
340 int ret = 0;
Jacob Pane1399ba2016-05-31 13:41:29 -0700341 int id;
Jacob Pan2d281d82013-10-17 10:28:35 -0700342
343 get_online_cpus();
344 rd = power_zone_to_rapl_domain(power_zone);
Jacob Pane1399ba2016-05-31 13:41:29 -0700345 id = contraint_to_pl(rd, cid);
Jacob Pancb43f812016-11-28 13:53:11 -0800346 if (id < 0) {
347 ret = id;
348 goto set_exit;
349 }
Jacob Pane1399ba2016-05-31 13:41:29 -0700350
Jacob Pan309557f2016-02-24 13:31:37 -0800351 rp = rd->rp;
Jacob Pan2d281d82013-10-17 10:28:35 -0700352
353 if (rd->state & DOMAIN_STATE_BIOS_LOCKED) {
354 dev_warn(&power_zone->dev, "%s locked by BIOS, monitoring only\n",
355 rd->name);
356 ret = -EACCES;
357 goto set_exit;
358 }
359
360 switch (rd->rpl[id].prim_id) {
361 case PL1_ENABLE:
362 rapl_write_data_raw(rd, POWER_LIMIT1, power_limit);
363 break;
364 case PL2_ENABLE:
365 rapl_write_data_raw(rd, POWER_LIMIT2, power_limit);
366 break;
367 default:
368 ret = -EINVAL;
369 }
370 if (!ret)
Jacob Pan309557f2016-02-24 13:31:37 -0800371 package_power_limit_irq_save(rp);
Jacob Pan2d281d82013-10-17 10:28:35 -0700372set_exit:
373 put_online_cpus();
374 return ret;
375}
376
Jacob Pane1399ba2016-05-31 13:41:29 -0700377static int get_current_power_limit(struct powercap_zone *power_zone, int cid,
Jacob Pan2d281d82013-10-17 10:28:35 -0700378 u64 *data)
379{
380 struct rapl_domain *rd;
381 u64 val;
382 int prim;
383 int ret = 0;
Jacob Pane1399ba2016-05-31 13:41:29 -0700384 int id;
Jacob Pan2d281d82013-10-17 10:28:35 -0700385
386 get_online_cpus();
387 rd = power_zone_to_rapl_domain(power_zone);
Jacob Pane1399ba2016-05-31 13:41:29 -0700388 id = contraint_to_pl(rd, cid);
Jacob Pancb43f812016-11-28 13:53:11 -0800389 if (id < 0) {
390 ret = id;
391 goto get_exit;
392 }
393
Jacob Pan2d281d82013-10-17 10:28:35 -0700394 switch (rd->rpl[id].prim_id) {
395 case PL1_ENABLE:
396 prim = POWER_LIMIT1;
397 break;
398 case PL2_ENABLE:
399 prim = POWER_LIMIT2;
400 break;
401 default:
402 put_online_cpus();
403 return -EINVAL;
404 }
405 if (rapl_read_data_raw(rd, prim, true, &val))
406 ret = -EIO;
407 else
408 *data = val;
409
Jacob Pancb43f812016-11-28 13:53:11 -0800410get_exit:
Jacob Pan2d281d82013-10-17 10:28:35 -0700411 put_online_cpus();
412
413 return ret;
414}
415
Jacob Pane1399ba2016-05-31 13:41:29 -0700416static int set_time_window(struct powercap_zone *power_zone, int cid,
Jacob Pan2d281d82013-10-17 10:28:35 -0700417 u64 window)
418{
419 struct rapl_domain *rd;
420 int ret = 0;
Jacob Pane1399ba2016-05-31 13:41:29 -0700421 int id;
Jacob Pan2d281d82013-10-17 10:28:35 -0700422
423 get_online_cpus();
424 rd = power_zone_to_rapl_domain(power_zone);
Jacob Pane1399ba2016-05-31 13:41:29 -0700425 id = contraint_to_pl(rd, cid);
Jacob Pancb43f812016-11-28 13:53:11 -0800426 if (id < 0) {
427 ret = id;
428 goto set_time_exit;
429 }
Jacob Pane1399ba2016-05-31 13:41:29 -0700430
Jacob Pan2d281d82013-10-17 10:28:35 -0700431 switch (rd->rpl[id].prim_id) {
432 case PL1_ENABLE:
433 rapl_write_data_raw(rd, TIME_WINDOW1, window);
434 break;
435 case PL2_ENABLE:
436 rapl_write_data_raw(rd, TIME_WINDOW2, window);
437 break;
438 default:
439 ret = -EINVAL;
440 }
Jacob Pancb43f812016-11-28 13:53:11 -0800441
442set_time_exit:
Jacob Pan2d281d82013-10-17 10:28:35 -0700443 put_online_cpus();
444 return ret;
445}
446
Jacob Pane1399ba2016-05-31 13:41:29 -0700447static int get_time_window(struct powercap_zone *power_zone, int cid, u64 *data)
Jacob Pan2d281d82013-10-17 10:28:35 -0700448{
449 struct rapl_domain *rd;
450 u64 val;
451 int ret = 0;
Jacob Pane1399ba2016-05-31 13:41:29 -0700452 int id;
Jacob Pan2d281d82013-10-17 10:28:35 -0700453
454 get_online_cpus();
455 rd = power_zone_to_rapl_domain(power_zone);
Jacob Pane1399ba2016-05-31 13:41:29 -0700456 id = contraint_to_pl(rd, cid);
Jacob Pancb43f812016-11-28 13:53:11 -0800457 if (id < 0) {
458 ret = id;
459 goto get_time_exit;
460 }
Jacob Pane1399ba2016-05-31 13:41:29 -0700461
Jacob Pan2d281d82013-10-17 10:28:35 -0700462 switch (rd->rpl[id].prim_id) {
463 case PL1_ENABLE:
464 ret = rapl_read_data_raw(rd, TIME_WINDOW1, true, &val);
465 break;
466 case PL2_ENABLE:
467 ret = rapl_read_data_raw(rd, TIME_WINDOW2, true, &val);
468 break;
469 default:
470 put_online_cpus();
471 return -EINVAL;
472 }
473 if (!ret)
474 *data = val;
Jacob Pancb43f812016-11-28 13:53:11 -0800475
476get_time_exit:
Jacob Pan2d281d82013-10-17 10:28:35 -0700477 put_online_cpus();
478
479 return ret;
480}
481
Jacob Pane1399ba2016-05-31 13:41:29 -0700482static const char *get_constraint_name(struct powercap_zone *power_zone, int cid)
Jacob Pan2d281d82013-10-17 10:28:35 -0700483{
Jacob Pan2d281d82013-10-17 10:28:35 -0700484 struct rapl_domain *rd;
Jacob Pane1399ba2016-05-31 13:41:29 -0700485 int id;
Jacob Pan2d281d82013-10-17 10:28:35 -0700486
487 rd = power_zone_to_rapl_domain(power_zone);
Jacob Pane1399ba2016-05-31 13:41:29 -0700488 id = contraint_to_pl(rd, cid);
489 if (id >= 0)
490 return rd->rpl[id].name;
Jacob Pan2d281d82013-10-17 10:28:35 -0700491
Jacob Pane1399ba2016-05-31 13:41:29 -0700492 return NULL;
Jacob Pan2d281d82013-10-17 10:28:35 -0700493}
494
495
496static int get_max_power(struct powercap_zone *power_zone, int id,
497 u64 *data)
498{
499 struct rapl_domain *rd;
500 u64 val;
501 int prim;
502 int ret = 0;
503
504 get_online_cpus();
505 rd = power_zone_to_rapl_domain(power_zone);
506 switch (rd->rpl[id].prim_id) {
507 case PL1_ENABLE:
508 prim = THERMAL_SPEC_POWER;
509 break;
510 case PL2_ENABLE:
511 prim = MAX_POWER;
512 break;
513 default:
514 put_online_cpus();
515 return -EINVAL;
516 }
517 if (rapl_read_data_raw(rd, prim, true, &val))
518 ret = -EIO;
519 else
520 *data = val;
521
522 put_online_cpus();
523
524 return ret;
525}
526
Julia Lawall600c3952015-12-23 22:59:55 +0100527static const struct powercap_zone_constraint_ops constraint_ops = {
Jacob Pan2d281d82013-10-17 10:28:35 -0700528 .set_power_limit_uw = set_power_limit,
529 .get_power_limit_uw = get_current_power_limit,
530 .set_time_window_us = set_time_window,
531 .get_time_window_us = get_time_window,
532 .get_max_power_uw = get_max_power,
533 .get_name = get_constraint_name,
534};
535
536/* called after domain detection and package level data are set */
537static void rapl_init_domains(struct rapl_package *rp)
538{
539 int i;
540 struct rapl_domain *rd = rp->domains;
541
542 for (i = 0; i < RAPL_DOMAIN_MAX; i++) {
543 unsigned int mask = rp->domain_map & (1 << i);
544 switch (mask) {
545 case BIT(RAPL_DOMAIN_PACKAGE):
546 rd->name = rapl_domain_names[RAPL_DOMAIN_PACKAGE];
547 rd->id = RAPL_DOMAIN_PACKAGE;
Zhang Rui8310e822019-07-10 21:44:23 +0800548 rd->regs[RAPL_DOMAIN_REG_LIMIT] = MSR_PKG_POWER_LIMIT;
549 rd->regs[RAPL_DOMAIN_REG_STATUS] = MSR_PKG_ENERGY_STATUS;
550 rd->regs[RAPL_DOMAIN_REG_PERF] = MSR_PKG_PERF_STATUS;
551 rd->regs[RAPL_DOMAIN_REG_POLICY] = 0;
552 rd->regs[RAPL_DOMAIN_REG_INFO] = MSR_PKG_POWER_INFO;
Jacob Pan2d281d82013-10-17 10:28:35 -0700553 rd->rpl[0].prim_id = PL1_ENABLE;
554 rd->rpl[0].name = pl1_name;
555 rd->rpl[1].prim_id = PL2_ENABLE;
556 rd->rpl[1].name = pl2_name;
557 break;
558 case BIT(RAPL_DOMAIN_PP0):
559 rd->name = rapl_domain_names[RAPL_DOMAIN_PP0];
560 rd->id = RAPL_DOMAIN_PP0;
Zhang Rui8310e822019-07-10 21:44:23 +0800561 rd->regs[RAPL_DOMAIN_REG_LIMIT] = MSR_PP0_POWER_LIMIT;
562 rd->regs[RAPL_DOMAIN_REG_STATUS] = MSR_PP0_ENERGY_STATUS;
563 rd->regs[RAPL_DOMAIN_REG_PERF] = 0;
564 rd->regs[RAPL_DOMAIN_REG_POLICY] = MSR_PP0_POLICY;
565 rd->regs[RAPL_DOMAIN_REG_INFO] = 0;
Jacob Pan2d281d82013-10-17 10:28:35 -0700566 rd->rpl[0].prim_id = PL1_ENABLE;
567 rd->rpl[0].name = pl1_name;
568 break;
569 case BIT(RAPL_DOMAIN_PP1):
570 rd->name = rapl_domain_names[RAPL_DOMAIN_PP1];
571 rd->id = RAPL_DOMAIN_PP1;
Zhang Rui8310e822019-07-10 21:44:23 +0800572 rd->regs[RAPL_DOMAIN_REG_LIMIT] = MSR_PP1_POWER_LIMIT;
573 rd->regs[RAPL_DOMAIN_REG_STATUS] = MSR_PP1_ENERGY_STATUS;
574 rd->regs[RAPL_DOMAIN_REG_PERF] = 0;
575 rd->regs[RAPL_DOMAIN_REG_POLICY] = MSR_PP1_POLICY;
576 rd->regs[RAPL_DOMAIN_REG_INFO] = 0;
Jacob Pan2d281d82013-10-17 10:28:35 -0700577 rd->rpl[0].prim_id = PL1_ENABLE;
578 rd->rpl[0].name = pl1_name;
579 break;
580 case BIT(RAPL_DOMAIN_DRAM):
581 rd->name = rapl_domain_names[RAPL_DOMAIN_DRAM];
582 rd->id = RAPL_DOMAIN_DRAM;
Zhang Rui8310e822019-07-10 21:44:23 +0800583 rd->regs[RAPL_DOMAIN_REG_LIMIT] = MSR_DRAM_POWER_LIMIT;
584 rd->regs[RAPL_DOMAIN_REG_STATUS] = MSR_DRAM_ENERGY_STATUS;
585 rd->regs[RAPL_DOMAIN_REG_PERF] = MSR_DRAM_PERF_STATUS;
586 rd->regs[RAPL_DOMAIN_REG_POLICY] = 0;
587 rd->regs[RAPL_DOMAIN_REG_INFO] = MSR_DRAM_POWER_INFO;
Jacob Pan2d281d82013-10-17 10:28:35 -0700588 rd->rpl[0].prim_id = PL1_ENABLE;
589 rd->rpl[0].name = pl1_name;
Jacob Pand474a4d2015-03-13 03:48:56 -0700590 rd->domain_energy_unit =
591 rapl_defaults->dram_domain_energy_unit;
592 if (rd->domain_energy_unit)
593 pr_info("DRAM domain energy unit %dpj\n",
594 rd->domain_energy_unit);
Jacob Pan2d281d82013-10-17 10:28:35 -0700595 break;
596 }
597 if (mask) {
Jacob Pan309557f2016-02-24 13:31:37 -0800598 rd->rp = rp;
Jacob Pan2d281d82013-10-17 10:28:35 -0700599 rd++;
600 }
601 }
602}
603
Jacob Pan309557f2016-02-24 13:31:37 -0800604static u64 rapl_unit_xlate(struct rapl_domain *rd, enum unit_type type,
605 u64 value, int to_raw)
Jacob Pan2d281d82013-10-17 10:28:35 -0700606{
Jacob Pan3c2c0842014-11-07 09:29:26 -0800607 u64 units = 1;
Jacob Pan309557f2016-02-24 13:31:37 -0800608 struct rapl_package *rp = rd->rp;
Jacob Pand474a4d2015-03-13 03:48:56 -0700609 u64 scale = 1;
Jacob Pan2d281d82013-10-17 10:28:35 -0700610
Jacob Pan2d281d82013-10-17 10:28:35 -0700611 switch (type) {
612 case POWER_UNIT:
Jacob Pan3c2c0842014-11-07 09:29:26 -0800613 units = rp->power_unit;
Jacob Pan2d281d82013-10-17 10:28:35 -0700614 break;
615 case ENERGY_UNIT:
Jacob Pand474a4d2015-03-13 03:48:56 -0700616 scale = ENERGY_UNIT_SCALE;
617 /* per domain unit takes precedence */
Jacob Pancb43f812016-11-28 13:53:11 -0800618 if (rd->domain_energy_unit)
Jacob Pand474a4d2015-03-13 03:48:56 -0700619 units = rd->domain_energy_unit;
620 else
621 units = rp->energy_unit;
Jacob Pan2d281d82013-10-17 10:28:35 -0700622 break;
623 case TIME_UNIT:
Jacob Pan3c2c0842014-11-07 09:29:26 -0800624 return rapl_defaults->compute_time_window(rp, value, to_raw);
Jacob Pan2d281d82013-10-17 10:28:35 -0700625 case ARBITRARY_UNIT:
626 default:
627 return value;
628 };
629
630 if (to_raw)
Jacob Pand474a4d2015-03-13 03:48:56 -0700631 return div64_u64(value, units) * scale;
Jacob Pan3c2c0842014-11-07 09:29:26 -0800632
633 value *= units;
634
Jacob Pand474a4d2015-03-13 03:48:56 -0700635 return div64_u64(value, scale);
Jacob Pan2d281d82013-10-17 10:28:35 -0700636}
637
638/* in the order of enum rapl_primitives */
639static struct rapl_primitive_info rpi[] = {
640 /* name, mask, shift, msr index, unit divisor */
641 PRIMITIVE_INFO_INIT(ENERGY_COUNTER, ENERGY_STATUS_MASK, 0,
Zhang Ruif7c4e0c2019-07-10 21:44:22 +0800642 RAPL_DOMAIN_REG_STATUS, ENERGY_UNIT, 0),
Jacob Pan2d281d82013-10-17 10:28:35 -0700643 PRIMITIVE_INFO_INIT(POWER_LIMIT1, POWER_LIMIT1_MASK, 0,
Zhang Ruif7c4e0c2019-07-10 21:44:22 +0800644 RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),
Jacob Pan2d281d82013-10-17 10:28:35 -0700645 PRIMITIVE_INFO_INIT(POWER_LIMIT2, POWER_LIMIT2_MASK, 32,
Zhang Ruif7c4e0c2019-07-10 21:44:22 +0800646 RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),
Jacob Pan2d281d82013-10-17 10:28:35 -0700647 PRIMITIVE_INFO_INIT(FW_LOCK, POWER_PP_LOCK, 31,
Zhang Ruif7c4e0c2019-07-10 21:44:22 +0800648 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
Jacob Pan2d281d82013-10-17 10:28:35 -0700649 PRIMITIVE_INFO_INIT(PL1_ENABLE, POWER_LIMIT1_ENABLE, 15,
Zhang Ruif7c4e0c2019-07-10 21:44:22 +0800650 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
Jacob Pan2d281d82013-10-17 10:28:35 -0700651 PRIMITIVE_INFO_INIT(PL1_CLAMP, POWER_LIMIT1_CLAMP, 16,
Zhang Ruif7c4e0c2019-07-10 21:44:22 +0800652 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
Jacob Pan2d281d82013-10-17 10:28:35 -0700653 PRIMITIVE_INFO_INIT(PL2_ENABLE, POWER_LIMIT2_ENABLE, 47,
Zhang Ruif7c4e0c2019-07-10 21:44:22 +0800654 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
Jacob Pan2d281d82013-10-17 10:28:35 -0700655 PRIMITIVE_INFO_INIT(PL2_CLAMP, POWER_LIMIT2_CLAMP, 48,
Zhang Ruif7c4e0c2019-07-10 21:44:22 +0800656 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
Jacob Pan2d281d82013-10-17 10:28:35 -0700657 PRIMITIVE_INFO_INIT(TIME_WINDOW1, TIME_WINDOW1_MASK, 17,
Zhang Ruif7c4e0c2019-07-10 21:44:22 +0800658 RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),
Jacob Pan2d281d82013-10-17 10:28:35 -0700659 PRIMITIVE_INFO_INIT(TIME_WINDOW2, TIME_WINDOW2_MASK, 49,
Zhang Ruif7c4e0c2019-07-10 21:44:22 +0800660 RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),
Jacob Pan2d281d82013-10-17 10:28:35 -0700661 PRIMITIVE_INFO_INIT(THERMAL_SPEC_POWER, POWER_INFO_THERMAL_SPEC_MASK,
Zhang Ruif7c4e0c2019-07-10 21:44:22 +0800662 0, RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
Jacob Pan2d281d82013-10-17 10:28:35 -0700663 PRIMITIVE_INFO_INIT(MAX_POWER, POWER_INFO_MAX_MASK, 32,
Zhang Ruif7c4e0c2019-07-10 21:44:22 +0800664 RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
Jacob Pan2d281d82013-10-17 10:28:35 -0700665 PRIMITIVE_INFO_INIT(MIN_POWER, POWER_INFO_MIN_MASK, 16,
Zhang Ruif7c4e0c2019-07-10 21:44:22 +0800666 RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
Jacob Pan2d281d82013-10-17 10:28:35 -0700667 PRIMITIVE_INFO_INIT(MAX_TIME_WINDOW, POWER_INFO_MAX_TIME_WIN_MASK, 48,
Zhang Ruif7c4e0c2019-07-10 21:44:22 +0800668 RAPL_DOMAIN_REG_INFO, TIME_UNIT, 0),
Jacob Pan2d281d82013-10-17 10:28:35 -0700669 PRIMITIVE_INFO_INIT(THROTTLED_TIME, PERF_STATUS_THROTTLE_TIME_MASK, 0,
Zhang Ruif7c4e0c2019-07-10 21:44:22 +0800670 RAPL_DOMAIN_REG_PERF, TIME_UNIT, 0),
Jacob Pan2d281d82013-10-17 10:28:35 -0700671 PRIMITIVE_INFO_INIT(PRIORITY_LEVEL, PP_POLICY_MASK, 0,
Zhang Ruif7c4e0c2019-07-10 21:44:22 +0800672 RAPL_DOMAIN_REG_POLICY, ARBITRARY_UNIT, 0),
Jacob Pan2d281d82013-10-17 10:28:35 -0700673 /* non-hardware */
674 PRIMITIVE_INFO_INIT(AVERAGE_POWER, 0, 0, 0, POWER_UNIT,
675 RAPL_PRIMITIVE_DERIVED),
676 {NULL, 0, 0, 0},
677};
678
679/* Read primitive data based on its related struct rapl_primitive_info.
680 * if xlate flag is set, return translated data based on data units, i.e.
681 * time, energy, and power.
682 * RAPL MSRs are non-architectual and are laid out not consistently across
683 * domains. Here we use primitive info to allow writing consolidated access
684 * functions.
685 * For a given primitive, it is processed by MSR mask and shift. Unit conversion
686 * is pre-assigned based on RAPL unit MSRs read at init time.
687 * 63-------------------------- 31--------------------------- 0
688 * | xxxxx (mask) |
689 * | |<- shift ----------------|
690 * 63-------------------------- 31--------------------------- 0
691 */
692static int rapl_read_data_raw(struct rapl_domain *rd,
693 enum rapl_primitives prim,
694 bool xlate, u64 *data)
695{
696 u64 value, final;
697 u32 msr;
698 struct rapl_primitive_info *rp = &rpi[prim];
699 int cpu;
700
701 if (!rp->name || rp->flag & RAPL_PRIMITIVE_DUMMY)
702 return -EINVAL;
703
Zhang Ruif7c4e0c2019-07-10 21:44:22 +0800704 msr = rd->regs[rp->id];
Jacob Pan2d281d82013-10-17 10:28:35 -0700705 if (!msr)
706 return -EINVAL;
Jacob Pan323ee642016-02-24 13:31:38 -0800707
708 cpu = rd->rp->lead_cpu;
Jacob Pan2d281d82013-10-17 10:28:35 -0700709
710 /* special-case package domain, which uses a different bit*/
711 if (prim == FW_LOCK && rd->id == RAPL_DOMAIN_PACKAGE) {
712 rp->mask = POWER_PACKAGE_LOCK;
713 rp->shift = 63;
714 }
715 /* non-hardware data are collected by the polling thread */
716 if (rp->flag & RAPL_PRIMITIVE_DERIVED) {
717 *data = rd->rdd.primitives[prim];
718 return 0;
719 }
720
721 if (rdmsrl_safe_on_cpu(cpu, msr, &value)) {
722 pr_debug("failed to read msr 0x%x on cpu %d\n", msr, cpu);
723 return -EIO;
724 }
725
726 final = value & rp->mask;
727 final = final >> rp->shift;
728 if (xlate)
Jacob Pan309557f2016-02-24 13:31:37 -0800729 *data = rapl_unit_xlate(rd, rp->unit, final, 0);
Jacob Pan2d281d82013-10-17 10:28:35 -0700730 else
731 *data = final;
732
733 return 0;
734}
735
Jacob Panf14a1392016-02-24 13:31:36 -0800736
737static int msrl_update_safe(u32 msr_no, u64 clear_mask, u64 set_mask)
738{
739 int err;
740 u64 val;
741
742 err = rdmsrl_safe(msr_no, &val);
743 if (err)
744 goto out;
745
746 val &= ~clear_mask;
747 val |= set_mask;
748
749 err = wrmsrl_safe(msr_no, val);
750
751out:
752 return err;
753}
754
755static void msrl_update_func(void *info)
756{
757 struct msrl_action *ma = info;
758
759 ma->err = msrl_update_safe(ma->msr_no, ma->clear_mask, ma->set_mask);
760}
761
Jacob Pan2d281d82013-10-17 10:28:35 -0700762/* Similar use of primitive info in the read counterpart */
763static int rapl_write_data_raw(struct rapl_domain *rd,
764 enum rapl_primitives prim,
765 unsigned long long value)
766{
Jacob Pan2d281d82013-10-17 10:28:35 -0700767 struct rapl_primitive_info *rp = &rpi[prim];
768 int cpu;
Jacob Panf14a1392016-02-24 13:31:36 -0800769 u64 bits;
770 struct msrl_action ma;
771 int ret;
Jacob Pan2d281d82013-10-17 10:28:35 -0700772
Jacob Pan323ee642016-02-24 13:31:38 -0800773 cpu = rd->rp->lead_cpu;
Jacob Pan309557f2016-02-24 13:31:37 -0800774 bits = rapl_unit_xlate(rd, rp->unit, value, 1);
Adam Lessnauedbdabc2017-06-01 11:21:50 +0200775 bits <<= rp->shift;
776 bits &= rp->mask;
777
Jacob Panf14a1392016-02-24 13:31:36 -0800778 memset(&ma, 0, sizeof(ma));
779
Zhang Ruif7c4e0c2019-07-10 21:44:22 +0800780 ma.msr_no = rd->regs[rp->id];
Jacob Panf14a1392016-02-24 13:31:36 -0800781 ma.clear_mask = rp->mask;
782 ma.set_mask = bits;
783
784 ret = smp_call_function_single(cpu, msrl_update_func, &ma, 1);
785 if (ret)
786 WARN_ON_ONCE(ret);
787 else
788 ret = ma.err;
789
790 return ret;
Jacob Pan2d281d82013-10-17 10:28:35 -0700791}
792
Jacob Pan3c2c0842014-11-07 09:29:26 -0800793/*
794 * Raw RAPL data stored in MSRs are in certain scales. We need to
795 * convert them into standard units based on the units reported in
796 * the RAPL unit MSRs. This is specific to CPUs as the method to
797 * calculate units differ on different CPUs.
798 * We convert the units to below format based on CPUs.
799 * i.e.
Jacob Pand474a4d2015-03-13 03:48:56 -0700800 * energy unit: picoJoules : Represented in picoJoules by default
Jacob Pan3c2c0842014-11-07 09:29:26 -0800801 * power unit : microWatts : Represented in milliWatts by default
802 * time unit : microseconds: Represented in seconds by default
803 */
804static int rapl_check_unit_core(struct rapl_package *rp, int cpu)
Jacob Pan2d281d82013-10-17 10:28:35 -0700805{
806 u64 msr_val;
807 u32 value;
808
809 if (rdmsrl_safe_on_cpu(cpu, MSR_RAPL_POWER_UNIT, &msr_val)) {
810 pr_err("Failed to read power unit MSR 0x%x on CPU %d, exit.\n",
811 MSR_RAPL_POWER_UNIT, cpu);
812 return -ENODEV;
813 }
814
Jacob Pan2d281d82013-10-17 10:28:35 -0700815 value = (msr_val & ENERGY_UNIT_MASK) >> ENERGY_UNIT_OFFSET;
Jacob Pand474a4d2015-03-13 03:48:56 -0700816 rp->energy_unit = ENERGY_UNIT_SCALE * 1000000 / (1 << value);
Jacob Pan2d281d82013-10-17 10:28:35 -0700817
818 value = (msr_val & POWER_UNIT_MASK) >> POWER_UNIT_OFFSET;
Jacob Pan3c2c0842014-11-07 09:29:26 -0800819 rp->power_unit = 1000000 / (1 << value);
Jacob Pan2d281d82013-10-17 10:28:35 -0700820
821 value = (msr_val & TIME_UNIT_MASK) >> TIME_UNIT_OFFSET;
Jacob Pan3c2c0842014-11-07 09:29:26 -0800822 rp->time_unit = 1000000 / (1 << value);
Jacob Pan2d281d82013-10-17 10:28:35 -0700823
Zhang Rui9ea76122019-05-13 13:58:53 -0400824 pr_debug("Core CPU %s energy=%dpJ, time=%dus, power=%duW\n",
825 rp->name, rp->energy_unit, rp->time_unit, rp->power_unit);
Jacob Pan2d281d82013-10-17 10:28:35 -0700826
827 return 0;
828}
829
Jacob Pan3c2c0842014-11-07 09:29:26 -0800830static int rapl_check_unit_atom(struct rapl_package *rp, int cpu)
831{
832 u64 msr_val;
833 u32 value;
834
835 if (rdmsrl_safe_on_cpu(cpu, MSR_RAPL_POWER_UNIT, &msr_val)) {
836 pr_err("Failed to read power unit MSR 0x%x on CPU %d, exit.\n",
837 MSR_RAPL_POWER_UNIT, cpu);
838 return -ENODEV;
839 }
840 value = (msr_val & ENERGY_UNIT_MASK) >> ENERGY_UNIT_OFFSET;
Jacob Pand474a4d2015-03-13 03:48:56 -0700841 rp->energy_unit = ENERGY_UNIT_SCALE * 1 << value;
Jacob Pan3c2c0842014-11-07 09:29:26 -0800842
843 value = (msr_val & POWER_UNIT_MASK) >> POWER_UNIT_OFFSET;
844 rp->power_unit = (1 << value) * 1000;
845
846 value = (msr_val & TIME_UNIT_MASK) >> TIME_UNIT_OFFSET;
847 rp->time_unit = 1000000 / (1 << value);
848
Zhang Rui9ea76122019-05-13 13:58:53 -0400849 pr_debug("Atom %s energy=%dpJ, time=%dus, power=%duW\n",
850 rp->name, rp->energy_unit, rp->time_unit, rp->power_unit);
Jacob Pan3c2c0842014-11-07 09:29:26 -0800851
852 return 0;
853}
854
Jacob Panf14a1392016-02-24 13:31:36 -0800855static void power_limit_irq_save_cpu(void *info)
856{
857 u32 l, h = 0;
858 struct rapl_package *rp = (struct rapl_package *)info;
859
860 /* save the state of PLN irq mask bit before disabling it */
861 rdmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, &l, &h);
862 if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED)) {
863 rp->power_limit_irq = l & PACKAGE_THERM_INT_PLN_ENABLE;
864 rp->power_limit_irq |= PACKAGE_PLN_INT_SAVED;
865 }
866 l &= ~PACKAGE_THERM_INT_PLN_ENABLE;
867 wrmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
868}
869
Jacob Pan3c2c0842014-11-07 09:29:26 -0800870
Jacob Pan2d281d82013-10-17 10:28:35 -0700871/* REVISIT:
872 * When package power limit is set artificially low by RAPL, LVT
873 * thermal interrupt for package power limit should be ignored
874 * since we are not really exceeding the real limit. The intention
875 * is to avoid excessive interrupts while we are trying to save power.
876 * A useful feature might be routing the package_power_limit interrupt
877 * to userspace via eventfd. once we have a usecase, this is simple
878 * to do by adding an atomic notifier.
879 */
880
Jacob Pan309557f2016-02-24 13:31:37 -0800881static void package_power_limit_irq_save(struct rapl_package *rp)
Jacob Pan2d281d82013-10-17 10:28:35 -0700882{
Jacob Pan2d281d82013-10-17 10:28:35 -0700883 if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN))
884 return;
885
Jacob Pan323ee642016-02-24 13:31:38 -0800886 smp_call_function_single(rp->lead_cpu, power_limit_irq_save_cpu, rp, 1);
Jacob Panf14a1392016-02-24 13:31:36 -0800887}
888
Thomas Gleixner58705062016-11-22 21:16:02 +0000889/*
890 * Restore per package power limit interrupt enable state. Called from cpu
891 * hotplug code on package removal.
892 */
893static void package_power_limit_irq_restore(struct rapl_package *rp)
Jacob Panf14a1392016-02-24 13:31:36 -0800894{
Thomas Gleixner58705062016-11-22 21:16:02 +0000895 u32 l, h;
896
897 if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN))
898 return;
899
900 /* irq enable state not saved, nothing to restore */
901 if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED))
902 return;
Jacob Panf14a1392016-02-24 13:31:36 -0800903
904 rdmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, &l, &h);
905
906 if (rp->power_limit_irq & PACKAGE_THERM_INT_PLN_ENABLE)
907 l |= PACKAGE_THERM_INT_PLN_ENABLE;
908 else
909 l &= ~PACKAGE_THERM_INT_PLN_ENABLE;
910
911 wrmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
Jacob Pan2d281d82013-10-17 10:28:35 -0700912}
913
Jacob Pan3c2c0842014-11-07 09:29:26 -0800914static void set_floor_freq_default(struct rapl_domain *rd, bool mode)
915{
916 int nr_powerlimit = find_nr_power_limit(rd);
917
918 /* always enable clamp such that p-state can go below OS requested
919 * range. power capping priority over guranteed frequency.
920 */
921 rapl_write_data_raw(rd, PL1_CLAMP, mode);
922
923 /* some domains have pl2 */
924 if (nr_powerlimit > 1) {
925 rapl_write_data_raw(rd, PL2_ENABLE, mode);
926 rapl_write_data_raw(rd, PL2_CLAMP, mode);
927 }
928}
929
930static void set_floor_freq_atom(struct rapl_domain *rd, bool enable)
931{
932 static u32 power_ctrl_orig_val;
933 u32 mdata;
934
Ajay Thomas51b63402015-04-30 01:43:23 +0530935 if (!rapl_defaults->floor_freq_reg_addr) {
936 pr_err("Invalid floor frequency config register\n");
937 return;
938 }
939
Jacob Pan3c2c0842014-11-07 09:29:26 -0800940 if (!power_ctrl_orig_val)
Andy Shevchenko4077a382015-11-11 19:59:29 +0200941 iosf_mbi_read(BT_MBI_UNIT_PMC, MBI_CR_READ,
942 rapl_defaults->floor_freq_reg_addr,
943 &power_ctrl_orig_val);
Jacob Pan3c2c0842014-11-07 09:29:26 -0800944 mdata = power_ctrl_orig_val;
945 if (enable) {
946 mdata &= ~(0x7f << 8);
947 mdata |= 1 << 8;
948 }
Andy Shevchenko4077a382015-11-11 19:59:29 +0200949 iosf_mbi_write(BT_MBI_UNIT_PMC, MBI_CR_WRITE,
950 rapl_defaults->floor_freq_reg_addr, mdata);
Jacob Pan3c2c0842014-11-07 09:29:26 -0800951}
952
953static u64 rapl_compute_time_window_core(struct rapl_package *rp, u64 value,
954 bool to_raw)
955{
956 u64 f, y; /* fraction and exp. used for time unit */
957
958 /*
959 * Special processing based on 2^Y*(1+F/4), refer
960 * to Intel Software Developer's manual Vol.3B: CH 14.9.3.
961 */
962 if (!to_raw) {
963 f = (value & 0x60) >> 5;
964 y = value & 0x1f;
965 value = (1 << y) * (4 + f) * rp->time_unit / 4;
966 } else {
967 do_div(value, rp->time_unit);
968 y = ilog2(value);
969 f = div64_u64(4 * (value - (1 << y)), 1 << y);
970 value = (y & 0x1f) | ((f & 0x3) << 5);
971 }
972 return value;
973}
974
975static u64 rapl_compute_time_window_atom(struct rapl_package *rp, u64 value,
976 bool to_raw)
977{
978 /*
979 * Atom time unit encoding is straight forward val * time_unit,
980 * where time_unit is default to 1 sec. Never 0.
981 */
982 if (!to_raw)
983 return (value) ? value *= rp->time_unit : rp->time_unit;
984 else
985 value = div64_u64(value, rp->time_unit);
986
987 return value;
988}
989
Jacob Pan087e9cb2014-11-07 09:29:25 -0800990static const struct rapl_defaults rapl_defaults_core = {
Ajay Thomas51b63402015-04-30 01:43:23 +0530991 .floor_freq_reg_addr = 0,
Jacob Pan3c2c0842014-11-07 09:29:26 -0800992 .check_unit = rapl_check_unit_core,
993 .set_floor_freq = set_floor_freq_default,
994 .compute_time_window = rapl_compute_time_window_core,
Jacob Pan087e9cb2014-11-07 09:29:25 -0800995};
996
Jacob Pand474a4d2015-03-13 03:48:56 -0700997static const struct rapl_defaults rapl_defaults_hsw_server = {
998 .check_unit = rapl_check_unit_core,
999 .set_floor_freq = set_floor_freq_default,
1000 .compute_time_window = rapl_compute_time_window_core,
1001 .dram_domain_energy_unit = 15300,
1002};
1003
Ajay Thomas51b63402015-04-30 01:43:23 +05301004static const struct rapl_defaults rapl_defaults_byt = {
1005 .floor_freq_reg_addr = IOSF_CPU_POWER_BUDGET_CTL_BYT,
Jacob Pan3c2c0842014-11-07 09:29:26 -08001006 .check_unit = rapl_check_unit_atom,
1007 .set_floor_freq = set_floor_freq_atom,
1008 .compute_time_window = rapl_compute_time_window_atom,
Jacob Pan087e9cb2014-11-07 09:29:25 -08001009};
1010
Ajay Thomas51b63402015-04-30 01:43:23 +05301011static const struct rapl_defaults rapl_defaults_tng = {
1012 .floor_freq_reg_addr = IOSF_CPU_POWER_BUDGET_CTL_TNG,
1013 .check_unit = rapl_check_unit_atom,
1014 .set_floor_freq = set_floor_freq_atom,
1015 .compute_time_window = rapl_compute_time_window_atom,
1016};
1017
1018static const struct rapl_defaults rapl_defaults_ann = {
1019 .floor_freq_reg_addr = 0,
1020 .check_unit = rapl_check_unit_atom,
1021 .set_floor_freq = NULL,
1022 .compute_time_window = rapl_compute_time_window_atom,
1023};
1024
1025static const struct rapl_defaults rapl_defaults_cht = {
1026 .floor_freq_reg_addr = 0,
1027 .check_unit = rapl_check_unit_atom,
1028 .set_floor_freq = NULL,
1029 .compute_time_window = rapl_compute_time_window_atom,
1030};
1031
Mathias Krauseea85dbc2015-03-25 22:15:52 +01001032static const struct x86_cpu_id rapl_ids[] __initconst = {
Andy Shevchenko17ed1512018-08-31 11:25:13 +03001033 INTEL_CPU_FAM6(SANDYBRIDGE, rapl_defaults_core),
1034 INTEL_CPU_FAM6(SANDYBRIDGE_X, rapl_defaults_core),
Dave Hansen0bb04b52016-06-02 17:19:37 -07001035
Andy Shevchenko17ed1512018-08-31 11:25:13 +03001036 INTEL_CPU_FAM6(IVYBRIDGE, rapl_defaults_core),
1037 INTEL_CPU_FAM6(IVYBRIDGE_X, rapl_defaults_core),
Dave Hansen0bb04b52016-06-02 17:19:37 -07001038
Andy Shevchenko17ed1512018-08-31 11:25:13 +03001039 INTEL_CPU_FAM6(HASWELL_CORE, rapl_defaults_core),
1040 INTEL_CPU_FAM6(HASWELL_ULT, rapl_defaults_core),
1041 INTEL_CPU_FAM6(HASWELL_GT3E, rapl_defaults_core),
1042 INTEL_CPU_FAM6(HASWELL_X, rapl_defaults_hsw_server),
Dave Hansen0bb04b52016-06-02 17:19:37 -07001043
Andy Shevchenko17ed1512018-08-31 11:25:13 +03001044 INTEL_CPU_FAM6(BROADWELL_CORE, rapl_defaults_core),
1045 INTEL_CPU_FAM6(BROADWELL_GT3E, rapl_defaults_core),
1046 INTEL_CPU_FAM6(BROADWELL_XEON_D, rapl_defaults_core),
1047 INTEL_CPU_FAM6(BROADWELL_X, rapl_defaults_hsw_server),
Dave Hansen0bb04b52016-06-02 17:19:37 -07001048
Andy Shevchenko17ed1512018-08-31 11:25:13 +03001049 INTEL_CPU_FAM6(SKYLAKE_DESKTOP, rapl_defaults_core),
1050 INTEL_CPU_FAM6(SKYLAKE_MOBILE, rapl_defaults_core),
1051 INTEL_CPU_FAM6(SKYLAKE_X, rapl_defaults_hsw_server),
1052 INTEL_CPU_FAM6(KABYLAKE_MOBILE, rapl_defaults_core),
1053 INTEL_CPU_FAM6(KABYLAKE_DESKTOP, rapl_defaults_core),
1054 INTEL_CPU_FAM6(CANNONLAKE_MOBILE, rapl_defaults_core),
Gayatri Kammelaba6f3ec2019-02-18 15:01:02 +08001055 INTEL_CPU_FAM6(ICELAKE_MOBILE, rapl_defaults_core),
Dave Hansen0bb04b52016-06-02 17:19:37 -07001056
Linus Torvaldsc05f3642018-10-23 13:32:18 +01001057 INTEL_CPU_FAM6(ATOM_SILVERMONT, rapl_defaults_byt),
Andy Shevchenko17ed1512018-08-31 11:25:13 +03001058 INTEL_CPU_FAM6(ATOM_AIRMONT, rapl_defaults_cht),
Linus Torvaldsc05f3642018-10-23 13:32:18 +01001059 INTEL_CPU_FAM6(ATOM_SILVERMONT_MID, rapl_defaults_tng),
1060 INTEL_CPU_FAM6(ATOM_AIRMONT_MID, rapl_defaults_ann),
Andy Shevchenko17ed1512018-08-31 11:25:13 +03001061 INTEL_CPU_FAM6(ATOM_GOLDMONT, rapl_defaults_core),
Linus Torvaldsc05f3642018-10-23 13:32:18 +01001062 INTEL_CPU_FAM6(ATOM_GOLDMONT_PLUS, rapl_defaults_core),
1063 INTEL_CPU_FAM6(ATOM_GOLDMONT_X, rapl_defaults_core),
Zhang Ruidf7f8e02019-02-12 10:47:10 +08001064 INTEL_CPU_FAM6(ATOM_TREMONT_X, rapl_defaults_core),
Dave Hansen0bb04b52016-06-02 17:19:37 -07001065
Andy Shevchenko17ed1512018-08-31 11:25:13 +03001066 INTEL_CPU_FAM6(XEON_PHI_KNL, rapl_defaults_hsw_server),
1067 INTEL_CPU_FAM6(XEON_PHI_KNM, rapl_defaults_hsw_server),
Jacob Pan2d281d82013-10-17 10:28:35 -07001068 {}
1069};
1070MODULE_DEVICE_TABLE(x86cpu, rapl_ids);
1071
Thomas Gleixnerbed5ab62016-11-22 21:15:58 +00001072/* Read once for all raw primitive data for domains */
1073static void rapl_update_domain_data(struct rapl_package *rp)
Jacob Pan2d281d82013-10-17 10:28:35 -07001074{
1075 int dmn, prim;
1076 u64 val;
Jacob Pan2d281d82013-10-17 10:28:35 -07001077
Thomas Gleixnerbed5ab62016-11-22 21:15:58 +00001078 for (dmn = 0; dmn < rp->nr_domains; dmn++) {
Zhang Rui9ea76122019-05-13 13:58:53 -04001079 pr_debug("update %s domain %s data\n", rp->name,
Thomas Gleixnerbed5ab62016-11-22 21:15:58 +00001080 rp->domains[dmn].name);
1081 /* exclude non-raw primitives */
1082 for (prim = 0; prim < NR_RAW_PRIMITIVES; prim++) {
1083 if (!rapl_read_data_raw(&rp->domains[dmn], prim,
1084 rpi[prim].unit, &val))
1085 rp->domains[dmn].rdd.primitives[prim] = val;
Jacob Pan2d281d82013-10-17 10:28:35 -07001086 }
1087 }
1088
1089}
1090
Thomas Gleixner58705062016-11-22 21:16:02 +00001091static void rapl_unregister_powercap(void)
Jacob Pan2d281d82013-10-17 10:28:35 -07001092{
Zhang Rui7ebf8ef2019-07-10 21:44:25 +08001093 if (&rapl_msr_priv.platform_rapl_domain) {
1094 powercap_unregister_zone(rapl_msr_priv.control_type,
1095 &rapl_msr_priv.platform_rapl_domain->power_zone);
1096 kfree(rapl_msr_priv.platform_rapl_domain);
Srinivas Pandruvada3521ba12016-04-17 15:03:01 -07001097 }
Zhang Rui7ebf8ef2019-07-10 21:44:25 +08001098 powercap_unregister_control_type(rapl_msr_priv.control_type);
Jacob Pan2d281d82013-10-17 10:28:35 -07001099}
1100
1101static int rapl_package_register_powercap(struct rapl_package *rp)
1102{
1103 struct rapl_domain *rd;
Jacob Pan2d281d82013-10-17 10:28:35 -07001104 struct powercap_zone *power_zone = NULL;
Luis de Bethencourt01857cf2018-01-17 10:30:34 +00001105 int nr_pl, ret;
Thomas Gleixnerbed5ab62016-11-22 21:15:58 +00001106
1107 /* Update the domain data of the new package */
1108 rapl_update_domain_data(rp);
Jacob Pan2d281d82013-10-17 10:28:35 -07001109
1110 /* first we register package domain as the parent zone*/
1111 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
1112 if (rd->id == RAPL_DOMAIN_PACKAGE) {
1113 nr_pl = find_nr_power_limit(rd);
Zhang Rui9ea76122019-05-13 13:58:53 -04001114 pr_debug("register package domain %s\n", rp->name);
Jacob Pan2d281d82013-10-17 10:28:35 -07001115 power_zone = powercap_register_zone(&rd->power_zone,
Zhang Rui7ebf8ef2019-07-10 21:44:25 +08001116 rp->priv->control_type,
Zhang Rui9ea76122019-05-13 13:58:53 -04001117 rp->name, NULL,
Jacob Pan2d281d82013-10-17 10:28:35 -07001118 &zone_ops[rd->id],
1119 nr_pl,
1120 &constraint_ops);
1121 if (IS_ERR(power_zone)) {
Zhang Rui9ea76122019-05-13 13:58:53 -04001122 pr_debug("failed to register power zone %s\n",
1123 rp->name);
Thomas Gleixnerbed5ab62016-11-22 21:15:58 +00001124 return PTR_ERR(power_zone);
Jacob Pan2d281d82013-10-17 10:28:35 -07001125 }
1126 /* track parent zone in per package/socket data */
1127 rp->power_zone = power_zone;
1128 /* done, only one package domain per socket */
1129 break;
1130 }
1131 }
1132 if (!power_zone) {
1133 pr_err("no package domain found, unknown topology!\n");
Thomas Gleixnerbed5ab62016-11-22 21:15:58 +00001134 return -ENODEV;
Jacob Pan2d281d82013-10-17 10:28:35 -07001135 }
1136 /* now register domains as children of the socket/package*/
1137 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
1138 if (rd->id == RAPL_DOMAIN_PACKAGE)
1139 continue;
1140 /* number of power limits per domain varies */
1141 nr_pl = find_nr_power_limit(rd);
1142 power_zone = powercap_register_zone(&rd->power_zone,
Zhang Rui7ebf8ef2019-07-10 21:44:25 +08001143 rp->priv->control_type, rd->name,
Jacob Pan2d281d82013-10-17 10:28:35 -07001144 rp->power_zone,
1145 &zone_ops[rd->id], nr_pl,
1146 &constraint_ops);
1147
1148 if (IS_ERR(power_zone)) {
Zhang Rui9ea76122019-05-13 13:58:53 -04001149 pr_debug("failed to register power_zone, %s:%s\n",
1150 rp->name, rd->name);
Jacob Pan2d281d82013-10-17 10:28:35 -07001151 ret = PTR_ERR(power_zone);
1152 goto err_cleanup;
1153 }
1154 }
Thomas Gleixnerbed5ab62016-11-22 21:15:58 +00001155 return 0;
Jacob Pan2d281d82013-10-17 10:28:35 -07001156
Jacob Pan2d281d82013-10-17 10:28:35 -07001157err_cleanup:
Thomas Gleixner58705062016-11-22 21:16:02 +00001158 /*
1159 * Clean up previously initialized domains within the package if we
Jacob Pan2d281d82013-10-17 10:28:35 -07001160 * failed after the first domain setup.
1161 */
1162 while (--rd >= rp->domains) {
Zhang Rui9ea76122019-05-13 13:58:53 -04001163 pr_debug("unregister %s domain %s\n", rp->name, rd->name);
Zhang Rui7ebf8ef2019-07-10 21:44:25 +08001164 powercap_unregister_zone(rp->priv->control_type, &rd->power_zone);
Jacob Pan2d281d82013-10-17 10:28:35 -07001165 }
1166
1167 return ret;
1168}
1169
Thomas Gleixner58705062016-11-22 21:16:02 +00001170static int __init rapl_register_psys(void)
Srinivas Pandruvada3521ba12016-04-17 15:03:01 -07001171{
1172 struct rapl_domain *rd;
1173 struct powercap_zone *power_zone;
1174 u64 val;
1175
1176 if (rdmsrl_safe_on_cpu(0, MSR_PLATFORM_ENERGY_STATUS, &val) || !val)
1177 return -ENODEV;
1178
1179 if (rdmsrl_safe_on_cpu(0, MSR_PLATFORM_POWER_LIMIT, &val) || !val)
1180 return -ENODEV;
1181
1182 rd = kzalloc(sizeof(*rd), GFP_KERNEL);
1183 if (!rd)
1184 return -ENOMEM;
1185
1186 rd->name = rapl_domain_names[RAPL_DOMAIN_PLATFORM];
1187 rd->id = RAPL_DOMAIN_PLATFORM;
Zhang Rui8310e822019-07-10 21:44:23 +08001188 rd->regs[RAPL_DOMAIN_REG_LIMIT] = MSR_PLATFORM_POWER_LIMIT;
1189 rd->regs[RAPL_DOMAIN_REG_STATUS] = MSR_PLATFORM_ENERGY_STATUS;
Srinivas Pandruvada3521ba12016-04-17 15:03:01 -07001190 rd->rpl[0].prim_id = PL1_ENABLE;
1191 rd->rpl[0].name = pl1_name;
1192 rd->rpl[1].prim_id = PL2_ENABLE;
1193 rd->rpl[1].name = pl2_name;
Zhang Rui7ebf8ef2019-07-10 21:44:25 +08001194 rd->rp = rapl_find_package_domain(0, &rapl_msr_priv);
Srinivas Pandruvada3521ba12016-04-17 15:03:01 -07001195
Zhang Rui7ebf8ef2019-07-10 21:44:25 +08001196 power_zone = powercap_register_zone(&rd->power_zone, rapl_msr_priv.control_type,
Srinivas Pandruvada3521ba12016-04-17 15:03:01 -07001197 "psys", NULL,
1198 &zone_ops[RAPL_DOMAIN_PLATFORM],
1199 2, &constraint_ops);
1200
1201 if (IS_ERR(power_zone)) {
1202 kfree(rd);
1203 return PTR_ERR(power_zone);
1204 }
1205
Zhang Rui7ebf8ef2019-07-10 21:44:25 +08001206 rapl_msr_priv.platform_rapl_domain = rd;
Srinivas Pandruvada3521ba12016-04-17 15:03:01 -07001207
1208 return 0;
1209}
1210
Thomas Gleixner58705062016-11-22 21:16:02 +00001211static int __init rapl_register_powercap(void)
Jacob Pan2d281d82013-10-17 10:28:35 -07001212{
Zhang Rui7ebf8ef2019-07-10 21:44:25 +08001213 rapl_msr_priv.control_type = powercap_register_control_type(NULL, "intel-rapl", NULL);
1214 if (IS_ERR(rapl_msr_priv.control_type)) {
Jacob Pan2d281d82013-10-17 10:28:35 -07001215 pr_debug("failed to register powercap control_type.\n");
Zhang Rui7ebf8ef2019-07-10 21:44:25 +08001216 return PTR_ERR(rapl_msr_priv.control_type);
Jacob Pan2d281d82013-10-17 10:28:35 -07001217 }
Thomas Gleixner58705062016-11-22 21:16:02 +00001218 return 0;
Jacob Pan2d281d82013-10-17 10:28:35 -07001219}
1220
1221static int rapl_check_domain(int cpu, int domain)
1222{
1223 unsigned msr;
Jacob Pan9d31c672014-04-29 15:33:06 -07001224 u64 val = 0;
Jacob Pan2d281d82013-10-17 10:28:35 -07001225
1226 switch (domain) {
1227 case RAPL_DOMAIN_PACKAGE:
1228 msr = MSR_PKG_ENERGY_STATUS;
1229 break;
1230 case RAPL_DOMAIN_PP0:
1231 msr = MSR_PP0_ENERGY_STATUS;
1232 break;
1233 case RAPL_DOMAIN_PP1:
1234 msr = MSR_PP1_ENERGY_STATUS;
1235 break;
1236 case RAPL_DOMAIN_DRAM:
1237 msr = MSR_DRAM_ENERGY_STATUS;
1238 break;
Srinivas Pandruvada3521ba12016-04-17 15:03:01 -07001239 case RAPL_DOMAIN_PLATFORM:
1240 /* PSYS(PLATFORM) is not a CPU domain, so avoid printng error */
1241 return -EINVAL;
Jacob Pan2d281d82013-10-17 10:28:35 -07001242 default:
1243 pr_err("invalid domain id %d\n", domain);
1244 return -EINVAL;
1245 }
Jacob Pan9d31c672014-04-29 15:33:06 -07001246 /* make sure domain counters are available and contains non-zero
1247 * values, otherwise skip it.
1248 */
1249 if (rdmsrl_safe_on_cpu(cpu, msr, &val) || !val)
Jacob Pan2d281d82013-10-17 10:28:35 -07001250 return -ENODEV;
1251
Jacob Pan9d31c672014-04-29 15:33:06 -07001252 return 0;
Jacob Pan2d281d82013-10-17 10:28:35 -07001253}
1254
Jacob Pane1399ba2016-05-31 13:41:29 -07001255
1256/*
1257 * Check if power limits are available. Two cases when they are not available:
1258 * 1. Locked by BIOS, in this case we still provide read-only access so that
1259 * users can see what limit is set by the BIOS.
1260 * 2. Some CPUs make some domains monitoring only which means PLx MSRs may not
1261 * exist at all. In this case, we do not show the contraints in powercap.
1262 *
1263 * Called after domains are detected and initialized.
1264 */
1265static void rapl_detect_powerlimit(struct rapl_domain *rd)
1266{
1267 u64 val64;
1268 int i;
1269
1270 /* check if the domain is locked by BIOS, ignore if MSR doesn't exist */
1271 if (!rapl_read_data_raw(rd, FW_LOCK, false, &val64)) {
1272 if (val64) {
Zhang Rui9ea76122019-05-13 13:58:53 -04001273 pr_info("RAPL %s domain %s locked by BIOS\n",
1274 rd->rp->name, rd->name);
Jacob Pane1399ba2016-05-31 13:41:29 -07001275 rd->state |= DOMAIN_STATE_BIOS_LOCKED;
1276 }
1277 }
1278 /* check if power limit MSRs exists, otherwise domain is monitoring only */
1279 for (i = 0; i < NR_POWER_LIMITS; i++) {
1280 int prim = rd->rpl[i].prim_id;
1281 if (rapl_read_data_raw(rd, prim, false, &val64))
1282 rd->rpl[i].name = NULL;
1283 }
1284}
1285
Jacob Pan2d281d82013-10-17 10:28:35 -07001286/* Detect active and valid domains for the given CPU, caller must
1287 * ensure the CPU belongs to the targeted package and CPU hotlug is disabled.
1288 */
1289static int rapl_detect_domains(struct rapl_package *rp, int cpu)
1290{
Jacob Pan2d281d82013-10-17 10:28:35 -07001291 struct rapl_domain *rd;
Thomas Gleixner58705062016-11-22 21:16:02 +00001292 int i;
Jacob Pan2d281d82013-10-17 10:28:35 -07001293
1294 for (i = 0; i < RAPL_DOMAIN_MAX; i++) {
1295 /* use physical package id to read counters */
Jacob Panfcdf1792014-09-02 02:55:21 -07001296 if (!rapl_check_domain(cpu, i)) {
Jacob Pan2d281d82013-10-17 10:28:35 -07001297 rp->domain_map |= 1 << i;
Jacob Panfcdf1792014-09-02 02:55:21 -07001298 pr_info("Found RAPL domain %s\n", rapl_domain_names[i]);
1299 }
Jacob Pan2d281d82013-10-17 10:28:35 -07001300 }
1301 rp->nr_domains = bitmap_weight(&rp->domain_map, RAPL_DOMAIN_MAX);
1302 if (!rp->nr_domains) {
Zhang Rui9ea76122019-05-13 13:58:53 -04001303 pr_debug("no valid rapl domains found in %s\n", rp->name);
Thomas Gleixner58705062016-11-22 21:16:02 +00001304 return -ENODEV;
Jacob Pan2d281d82013-10-17 10:28:35 -07001305 }
Zhang Rui9ea76122019-05-13 13:58:53 -04001306 pr_debug("found %d domains on %s\n", rp->nr_domains, rp->name);
Jacob Pan2d281d82013-10-17 10:28:35 -07001307
1308 rp->domains = kcalloc(rp->nr_domains + 1, sizeof(struct rapl_domain),
1309 GFP_KERNEL);
Thomas Gleixner58705062016-11-22 21:16:02 +00001310 if (!rp->domains)
1311 return -ENOMEM;
1312
Jacob Pan2d281d82013-10-17 10:28:35 -07001313 rapl_init_domains(rp);
1314
Jacob Pane1399ba2016-05-31 13:41:29 -07001315 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++)
1316 rapl_detect_powerlimit(rd);
1317
Jacob Pan2d281d82013-10-17 10:28:35 -07001318 return 0;
1319}
1320
1321/* called from CPU hotplug notifier, hotplug lock held */
1322static void rapl_remove_package(struct rapl_package *rp)
1323{
1324 struct rapl_domain *rd, *rd_package = NULL;
1325
Thomas Gleixner58705062016-11-22 21:16:02 +00001326 package_power_limit_irq_restore(rp);
1327
Jacob Pan2d281d82013-10-17 10:28:35 -07001328 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
Thomas Gleixner58705062016-11-22 21:16:02 +00001329 rapl_write_data_raw(rd, PL1_ENABLE, 0);
1330 rapl_write_data_raw(rd, PL1_CLAMP, 0);
1331 if (find_nr_power_limit(rd) > 1) {
1332 rapl_write_data_raw(rd, PL2_ENABLE, 0);
1333 rapl_write_data_raw(rd, PL2_CLAMP, 0);
1334 }
Jacob Pan2d281d82013-10-17 10:28:35 -07001335 if (rd->id == RAPL_DOMAIN_PACKAGE) {
1336 rd_package = rd;
1337 continue;
1338 }
Zhang Rui9ea76122019-05-13 13:58:53 -04001339 pr_debug("remove package, undo power limit on %s: %s\n",
1340 rp->name, rd->name);
Zhang Rui7ebf8ef2019-07-10 21:44:25 +08001341 powercap_unregister_zone(rp->priv->control_type, &rd->power_zone);
Jacob Pan2d281d82013-10-17 10:28:35 -07001342 }
1343 /* do parent zone last */
Zhang Rui7ebf8ef2019-07-10 21:44:25 +08001344 powercap_unregister_zone(rp->priv->control_type, &rd_package->power_zone);
Jacob Pan2d281d82013-10-17 10:28:35 -07001345 list_del(&rp->plist);
1346 kfree(rp);
1347}
1348
1349/* called from CPU hotplug notifier, hotplug lock held */
Zhang Rui7ebf8ef2019-07-10 21:44:25 +08001350static struct rapl_package *rapl_add_package(int cpu, struct rapl_if_priv *priv)
Jacob Pan2d281d82013-10-17 10:28:35 -07001351{
Zhang Rui32fb4802019-05-13 13:58:51 -04001352 int id = topology_logical_die_id(cpu);
Jacob Pan2d281d82013-10-17 10:28:35 -07001353 struct rapl_package *rp;
Zhang Rui9ea76122019-05-13 13:58:53 -04001354 struct cpuinfo_x86 *c = &cpu_data(cpu);
Thomas Gleixnerb4005e92016-11-22 21:16:05 +00001355 int ret;
Jacob Pan2d281d82013-10-17 10:28:35 -07001356
Jacob Pan2d281d82013-10-17 10:28:35 -07001357 rp = kzalloc(sizeof(struct rapl_package), GFP_KERNEL);
1358 if (!rp)
Thomas Gleixnerb4005e92016-11-22 21:16:05 +00001359 return ERR_PTR(-ENOMEM);
Jacob Pan2d281d82013-10-17 10:28:35 -07001360
1361 /* add the new package to the list */
Zhang Ruiaadf7b32019-05-13 13:58:50 -04001362 rp->id = id;
Jacob Pan323ee642016-02-24 13:31:38 -08001363 rp->lead_cpu = cpu;
Zhang Rui7ebf8ef2019-07-10 21:44:25 +08001364 rp->priv = priv;
Jacob Pan323ee642016-02-24 13:31:38 -08001365
Zhang Rui9ea76122019-05-13 13:58:53 -04001366 if (topology_max_die_per_package() > 1)
1367 snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH,
1368 "package-%d-die-%d", c->phys_proc_id, c->cpu_die_id);
1369 else
1370 snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d",
1371 c->phys_proc_id);
1372
Jacob Pan2d281d82013-10-17 10:28:35 -07001373 /* check if the package contains valid domains */
1374 if (rapl_detect_domains(rp, cpu) ||
Jacob Pan3c2c0842014-11-07 09:29:26 -08001375 rapl_defaults->check_unit(rp, cpu)) {
Jacob Pan2d281d82013-10-17 10:28:35 -07001376 ret = -ENODEV;
1377 goto err_free_package;
1378 }
Thomas Gleixnera74f4362016-11-22 21:15:59 +00001379 ret = rapl_package_register_powercap(rp);
1380 if (!ret) {
Jacob Pan2d281d82013-10-17 10:28:35 -07001381 INIT_LIST_HEAD(&rp->plist);
1382 list_add(&rp->plist, &rapl_packages);
Thomas Gleixnerb4005e92016-11-22 21:16:05 +00001383 return rp;
Jacob Pan2d281d82013-10-17 10:28:35 -07001384 }
1385
1386err_free_package:
1387 kfree(rp->domains);
1388 kfree(rp);
Thomas Gleixnerb4005e92016-11-22 21:16:05 +00001389 return ERR_PTR(ret);
Jacob Pan2d281d82013-10-17 10:28:35 -07001390}
1391
1392/* Handles CPU hotplug on multi-socket systems.
1393 * If a CPU goes online as the first CPU of the physical package
1394 * we add the RAPL package to the system. Similarly, when the last
1395 * CPU of the package is removed, we remove the RAPL package and its
1396 * associated domains. Cooling devices are handled accordingly at
1397 * per-domain level.
1398 */
Sebastian Andrzej Siewior5e4dc792016-11-22 21:16:00 +00001399static int rapl_cpu_online(unsigned int cpu)
Jacob Pan2d281d82013-10-17 10:28:35 -07001400{
Sebastian Andrzej Siewior5e4dc792016-11-22 21:16:00 +00001401 struct rapl_package *rp;
Sebastian Andrzej Siewior5e4dc792016-11-22 21:16:00 +00001402
Zhang Rui7ebf8ef2019-07-10 21:44:25 +08001403 rp = rapl_find_package_domain(cpu, &rapl_msr_priv);
Thomas Gleixnerb4005e92016-11-22 21:16:05 +00001404 if (!rp) {
Zhang Rui7ebf8ef2019-07-10 21:44:25 +08001405 rp = rapl_add_package(cpu, &rapl_msr_priv);
Thomas Gleixnerb4005e92016-11-22 21:16:05 +00001406 if (IS_ERR(rp))
1407 return PTR_ERR(rp);
Thomas Gleixner58705062016-11-22 21:16:02 +00001408 }
Thomas Gleixnerb4005e92016-11-22 21:16:05 +00001409 cpumask_set_cpu(cpu, &rp->cpumask);
1410 return 0;
Sebastian Andrzej Siewior5e4dc792016-11-22 21:16:00 +00001411}
1412
1413static int rapl_cpu_down_prep(unsigned int cpu)
1414{
Jacob Pan2d281d82013-10-17 10:28:35 -07001415 struct rapl_package *rp;
Jacob Pan323ee642016-02-24 13:31:38 -08001416 int lead_cpu;
Jacob Pan2d281d82013-10-17 10:28:35 -07001417
Zhang Rui7ebf8ef2019-07-10 21:44:25 +08001418 rp = rapl_find_package_domain(cpu, &rapl_msr_priv);
Sebastian Andrzej Siewior5e4dc792016-11-22 21:16:00 +00001419 if (!rp)
1420 return 0;
Thomas Gleixnerb4005e92016-11-22 21:16:05 +00001421
1422 cpumask_clear_cpu(cpu, &rp->cpumask);
1423 lead_cpu = cpumask_first(&rp->cpumask);
1424 if (lead_cpu >= nr_cpu_ids)
Sebastian Andrzej Siewior5e4dc792016-11-22 21:16:00 +00001425 rapl_remove_package(rp);
Thomas Gleixnerb4005e92016-11-22 21:16:05 +00001426 else if (rp->lead_cpu == cpu)
1427 rp->lead_cpu = lead_cpu;
Sebastian Andrzej Siewior5e4dc792016-11-22 21:16:00 +00001428 return 0;
Jacob Pan2d281d82013-10-17 10:28:35 -07001429}
1430
Zhen Han52b36722018-01-10 08:38:23 +08001431static void power_limit_state_save(void)
1432{
1433 struct rapl_package *rp;
1434 struct rapl_domain *rd;
1435 int nr_pl, ret, i;
1436
1437 get_online_cpus();
1438 list_for_each_entry(rp, &rapl_packages, plist) {
1439 if (!rp->power_zone)
1440 continue;
1441 rd = power_zone_to_rapl_domain(rp->power_zone);
1442 nr_pl = find_nr_power_limit(rd);
1443 for (i = 0; i < nr_pl; i++) {
1444 switch (rd->rpl[i].prim_id) {
1445 case PL1_ENABLE:
1446 ret = rapl_read_data_raw(rd,
1447 POWER_LIMIT1,
1448 true,
1449 &rd->rpl[i].last_power_limit);
1450 if (ret)
1451 rd->rpl[i].last_power_limit = 0;
1452 break;
1453 case PL2_ENABLE:
1454 ret = rapl_read_data_raw(rd,
1455 POWER_LIMIT2,
1456 true,
1457 &rd->rpl[i].last_power_limit);
1458 if (ret)
1459 rd->rpl[i].last_power_limit = 0;
1460 break;
1461 }
1462 }
1463 }
1464 put_online_cpus();
1465}
1466
1467static void power_limit_state_restore(void)
1468{
1469 struct rapl_package *rp;
1470 struct rapl_domain *rd;
1471 int nr_pl, i;
1472
1473 get_online_cpus();
1474 list_for_each_entry(rp, &rapl_packages, plist) {
1475 if (!rp->power_zone)
1476 continue;
1477 rd = power_zone_to_rapl_domain(rp->power_zone);
1478 nr_pl = find_nr_power_limit(rd);
1479 for (i = 0; i < nr_pl; i++) {
1480 switch (rd->rpl[i].prim_id) {
1481 case PL1_ENABLE:
1482 if (rd->rpl[i].last_power_limit)
1483 rapl_write_data_raw(rd,
1484 POWER_LIMIT1,
1485 rd->rpl[i].last_power_limit);
1486 break;
1487 case PL2_ENABLE:
1488 if (rd->rpl[i].last_power_limit)
1489 rapl_write_data_raw(rd,
1490 POWER_LIMIT2,
1491 rd->rpl[i].last_power_limit);
1492 break;
1493 }
1494 }
1495 }
1496 put_online_cpus();
1497}
1498
1499static int rapl_pm_callback(struct notifier_block *nb,
1500 unsigned long mode, void *_unused)
1501{
1502 switch (mode) {
1503 case PM_SUSPEND_PREPARE:
1504 power_limit_state_save();
1505 break;
1506 case PM_POST_SUSPEND:
1507 power_limit_state_restore();
1508 break;
1509 }
1510 return NOTIFY_OK;
1511}
1512
1513static struct notifier_block rapl_pm_notifier = {
1514 .notifier_call = rapl_pm_callback,
1515};
1516
Jacob Pan2d281d82013-10-17 10:28:35 -07001517static int __init rapl_init(void)
1518{
Jacob Pan087e9cb2014-11-07 09:29:25 -08001519 const struct x86_cpu_id *id;
Thomas Gleixner58705062016-11-22 21:16:02 +00001520 int ret;
Jacob Pan2d281d82013-10-17 10:28:35 -07001521
Jacob Pan087e9cb2014-11-07 09:29:25 -08001522 id = x86_match_cpu(rapl_ids);
1523 if (!id) {
Jacob Pan2d281d82013-10-17 10:28:35 -07001524 pr_err("driver does not support CPU family %d model %d\n",
1525 boot_cpu_data.x86, boot_cpu_data.x86_model);
1526
1527 return -ENODEV;
1528 }
Srivatsa S. Bhat009f2252014-03-11 02:09:26 +05301529
Jacob Pan087e9cb2014-11-07 09:29:25 -08001530 rapl_defaults = (struct rapl_defaults *)id->driver_data;
1531
Thomas Gleixner58705062016-11-22 21:16:02 +00001532 ret = rapl_register_powercap();
Jacob Pan2d281d82013-10-17 10:28:35 -07001533 if (ret)
Thomas Gleixner58705062016-11-22 21:16:02 +00001534 return ret;
Jacob Pan2d281d82013-10-17 10:28:35 -07001535
Thomas Gleixner58705062016-11-22 21:16:02 +00001536 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "powercap/rapl:online",
1537 rapl_cpu_online, rapl_cpu_down_prep);
Sebastian Andrzej Siewior5e4dc792016-11-22 21:16:00 +00001538 if (ret < 0)
1539 goto err_unreg;
Zhang Rui7ebf8ef2019-07-10 21:44:25 +08001540 rapl_msr_priv.pcap_rapl_online = ret;
Thomas Gleixner58705062016-11-22 21:16:02 +00001541
1542 /* Don't bail out if PSys is not supported */
1543 rapl_register_psys();
Zhen Han52b36722018-01-10 08:38:23 +08001544
1545 ret = register_pm_notifier(&rapl_pm_notifier);
1546 if (ret)
1547 goto err_unreg_all;
1548
Sebastian Andrzej Siewior5e4dc792016-11-22 21:16:00 +00001549 return 0;
Jacob Pan2d281d82013-10-17 10:28:35 -07001550
Zhen Han52b36722018-01-10 08:38:23 +08001551err_unreg_all:
Zhang Rui7ebf8ef2019-07-10 21:44:25 +08001552 cpuhp_remove_state(rapl_msr_priv.pcap_rapl_online);
Zhen Han52b36722018-01-10 08:38:23 +08001553
Sebastian Andrzej Siewior5e4dc792016-11-22 21:16:00 +00001554err_unreg:
1555 rapl_unregister_powercap();
Jacob Pan2d281d82013-10-17 10:28:35 -07001556 return ret;
1557}
1558
1559static void __exit rapl_exit(void)
1560{
Zhen Han52b36722018-01-10 08:38:23 +08001561 unregister_pm_notifier(&rapl_pm_notifier);
Zhang Rui7ebf8ef2019-07-10 21:44:25 +08001562 cpuhp_remove_state(rapl_msr_priv.pcap_rapl_online);
Jacob Pan2d281d82013-10-17 10:28:35 -07001563 rapl_unregister_powercap();
Jacob Pan2d281d82013-10-17 10:28:35 -07001564}
1565
1566module_init(rapl_init);
1567module_exit(rapl_exit);
1568
1569MODULE_DESCRIPTION("Driver for Intel RAPL (Running Average Power Limit)");
1570MODULE_AUTHOR("Jacob Pan <jacob.jun.pan@intel.com>");
1571MODULE_LICENSE("GPL v2");