2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2018, Matthew Macy
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 #include <sys/types.h>
32 #include <sys/errno.h>
34 #include <sys/sysctl.h>
43 #include <libpmcstat.h>
44 #include "pmu-events/pmu-events.h"
46 #if defined(__amd64__) || defined(__i386__)
58 static struct pmu_alias pmu_intel_alias_table[] = {
59 {"UNHALTED_CORE_CYCLES", "CPU_CLK_UNHALTED.THREAD_P_ANY"},
60 {"UNHALTED-CORE-CYCLES", "CPU_CLK_UNHALTED.THREAD_P_ANY"},
61 {"LLC_MISSES", "LONGEST_LAT_CACHE.MISS"},
62 {"LLC-MISSES", "LONGEST_LAT_CACHE.MISS"},
63 {"LLC_REFERENCE", "LONGEST_LAT_CACHE.REFERENCE"},
64 {"LLC-REFERENCE", "LONGEST_LAT_CACHE.REFERENCE"},
65 {"LLC_MISS_RHITM", "mem_load_l3_miss_retired.remote_hitm"},
66 {"LLC-MISS-RHITM", "mem_load_l3_miss_retired.remote_hitm"},
67 {"RESOURCE_STALL", "RESOURCE_STALLS.ANY"},
68 {"RESOURCE_STALLS_ANY", "RESOURCE_STALLS.ANY"},
69 {"BRANCH_INSTRUCTION_RETIRED", "BR_INST_RETIRED.ALL_BRANCHES"},
70 {"BRANCH-INSTRUCTION-RETIRED", "BR_INST_RETIRED.ALL_BRANCHES"},
71 {"BRANCH_MISSES_RETIRED", "BR_MISP_RETIRED.ALL_BRANCHES"},
72 {"BRANCH-MISSES-RETIRED", "BR_MISP_RETIRED.ALL_BRANCHES"},
73 {"cycles", "tsc-tsc"},
74 {"unhalted-cycles", "CPU_CLK_UNHALTED.THREAD_P_ANY"},
75 {"instructions", "inst-retired.any_p"},
76 {"branch-mispredicts", "br_misp_retired.all_branches"},
77 {"branches", "br_inst_retired.all_branches"},
78 {"interrupts", "hw_interrupts.received"},
79 {"ic-misses", "frontend_retired.l1i_miss"},
83 static struct pmu_alias pmu_amd_alias_table[] = {
84 {"UNHALTED_CORE_CYCLES", "ls_not_halted_cyc"},
85 {"UNHALTED-CORE-CYCLES", "ls_not_halted_cyc"},
93 char buf[PMC_CPUID_LEN];
94 size_t s = sizeof(buf);
97 if (sysctlbyname("kern.hwpmc.cpuid", buf, &s,
98 (void *)NULL, 0) == -1)
100 if (strcasestr(buf, "AuthenticAMD") != NULL ||
101 strcasestr(buf, "HygonGenuine") != NULL)
103 else if (strcasestr(buf, "GenuineIntel") != NULL)
111 * The Intel fixed mode counters are:
112 * "inst_retired.any",
113 * "cpu_clk_unhalted.thread",
114 * "cpu_clk_unhalted.thread_any",
115 * "cpu_clk_unhalted.ref_tsc",
120 pmu_alias_get(const char *name)
123 struct pmu_alias *pa;
124 struct pmu_alias *pmu_alias_table;
126 if ((mfr = pmu_events_mfr()) == PMU_INVALID)
129 pmu_alias_table = pmu_amd_alias_table;
130 else if (mfr == PMU_INTEL)
131 pmu_alias_table = pmu_intel_alias_table;
135 for (pa = pmu_alias_table; pa->pa_alias != NULL; pa++)
136 if (strcasecmp(name, pa->pa_alias) == 0)
137 return (pa->pa_name);
142 struct pmu_event_desc {
144 uint64_t ped_offcore_rsp;
145 uint64_t ped_l3_thread;
146 uint64_t ped_l3_slice;
148 uint32_t ped_frontend;
150 uint32_t ped_config1;
160 static const struct pmu_events_map *
161 pmu_events_map_get(const char *cpuid)
164 regmatch_t pmatch[1];
165 char buf[PMC_CPUID_LEN];
166 size_t s = sizeof(buf);
168 const struct pmu_events_map *pme;
171 strlcpy(buf, cpuid, s);
173 if (sysctlbyname("kern.hwpmc.cpuid", buf, &s,
174 (void *)NULL, 0) == -1)
177 for (pme = pmu_events_map; pme->cpuid != NULL; pme++) {
178 if (regcomp(&re, pme->cpuid, REG_EXTENDED) != 0) {
179 printf("regex '%s' failed to compile, ignoring\n",
183 match = regexec(&re, buf, 1, pmatch, 0);
186 if (pmatch[0].rm_so == 0 && (buf[pmatch[0].rm_eo] == 0
187 || buf[pmatch[0].rm_eo] == '-'))
194 static const struct pmu_event *
195 pmu_event_get(const char *cpuid, const char *event_name, int *idx)
197 const struct pmu_events_map *pme;
198 const struct pmu_event *pe;
201 if ((pme = pmu_events_map_get(cpuid)) == NULL)
203 for (i = 0, pe = pme->table; pe->name || pe->desc || pe->event; pe++, i++) {
204 if (pe->name == NULL)
206 if (strcasecmp(pe->name, event_name) == 0) {
216 pmc_pmu_idx_get_by_event(const char *cpuid, const char *event)
219 const char *realname;
221 realname = pmu_alias_get(event);
222 if (pmu_event_get(cpuid, realname, &idx) == NULL)
228 pmc_pmu_event_get_by_idx(const char *cpuid, int idx)
230 const struct pmu_events_map *pme;
232 if ((pme = pmu_events_map_get(cpuid)) == NULL)
234 assert(pme->table[idx].name);
235 return (pme->table[idx].name);
239 pmu_parse_event(struct pmu_event_desc *ped, const char *eventin)
242 char *kvp, *key, *value, *r;
245 if ((event = strdup(eventin)) == NULL)
248 bzero(ped, sizeof(*ped));
249 ped->ped_period = DEFAULT_SAMPLE_COUNT;
251 while ((kvp = strsep(&event, ",")) != NULL) {
252 key = strsep(&kvp, "=");
256 if (strcmp(key, "umask") == 0)
257 ped->ped_umask = strtol(value, NULL, 16);
258 else if (strcmp(key, "event") == 0)
259 ped->ped_event = strtol(value, NULL, 16);
260 else if (strcmp(key, "period") == 0)
261 ped->ped_period = strtol(value, NULL, 10);
262 else if (strcmp(key, "offcore_rsp") == 0)
263 ped->ped_offcore_rsp = strtol(value, NULL, 16);
264 else if (strcmp(key, "any") == 0)
265 ped->ped_any = strtol(value, NULL, 10);
266 else if (strcmp(key, "cmask") == 0)
267 ped->ped_cmask = strtol(value, NULL, 10);
268 else if (strcmp(key, "inv") == 0)
269 ped->ped_inv = strtol(value, NULL, 10);
270 else if (strcmp(key, "edge") == 0)
271 ped->ped_edge = strtol(value, NULL, 10);
272 else if (strcmp(key, "frontend") == 0)
273 ped->ped_frontend = strtol(value, NULL, 16);
274 else if (strcmp(key, "ldlat") == 0)
275 ped->ped_ldlat = strtol(value, NULL, 16);
276 else if (strcmp(key, "fc_mask") == 0)
277 ped->ped_fc_mask = strtol(value, NULL, 16);
278 else if (strcmp(key, "ch_mask") == 0)
279 ped->ped_ch_mask = strtol(value, NULL, 16);
280 else if (strcmp(key, "config1") == 0)
281 ped->ped_config1 = strtol(value, NULL, 16);
282 else if (strcmp(key, "l3_thread_mask") == 0)
283 ped->ped_l3_thread = strtol(value, NULL, 16);
284 else if (strcmp(key, "l3_slice_mask") == 0)
285 ped->ped_l3_slice = strtol(value, NULL, 16);
287 debug = getenv("PMUDEBUG");
288 if (debug != NULL && strcmp(debug, "true") == 0 && value != NULL)
289 printf("unrecognized kvpair: %s:%s\n", key, value);
297 pmc_pmu_sample_rate_get(const char *event_name)
299 const struct pmu_event *pe;
300 struct pmu_event_desc ped;
302 event_name = pmu_alias_get(event_name);
303 if ((pe = pmu_event_get(NULL, event_name, NULL)) == NULL)
304 return (DEFAULT_SAMPLE_COUNT);
305 if (pe->alias && (pe = pmu_event_get(NULL, pe->alias, NULL)) == NULL)
306 return (DEFAULT_SAMPLE_COUNT);
307 if (pe->event == NULL)
308 return (DEFAULT_SAMPLE_COUNT);
309 if (pmu_parse_event(&ped, pe->event))
310 return (DEFAULT_SAMPLE_COUNT);
311 return (ped.ped_period);
315 pmc_pmu_enabled(void)
318 return (pmu_events_map_get(NULL) != NULL);
322 pmc_pmu_print_counters(const char *event_name)
324 const struct pmu_events_map *pme;
325 const struct pmu_event *pe;
326 struct pmu_event_desc ped;
330 debug = getenv("PMUDEBUG");
333 if (debug != NULL && strcmp(debug, "true") == 0)
335 if ((pme = pmu_events_map_get(NULL)) == NULL)
337 for (pe = pme->table; pe->name || pe->desc || pe->event; pe++) {
338 if (pe->name == NULL)
340 if (event_name != NULL && strcasestr(pe->name, event_name) == NULL)
342 printf("\t%s\n", pe->name);
344 pmu_parse_event(&ped, pe->event);
349 pmc_pmu_print_counter_desc(const char *ev)
351 const struct pmu_events_map *pme;
352 const struct pmu_event *pe;
354 if ((pme = pmu_events_map_get(NULL)) == NULL)
356 for (pe = pme->table; pe->name || pe->desc || pe->event; pe++) {
357 if (pe->name == NULL)
359 if (strcasestr(pe->name, ev) != NULL &&
361 printf("%s:\t%s\n", pe->name, pe->desc);
366 pmc_pmu_print_counter_desc_long(const char *ev)
368 const struct pmu_events_map *pme;
369 const struct pmu_event *pe;
371 if ((pme = pmu_events_map_get(NULL)) == NULL)
373 for (pe = pme->table; pe->name || pe->desc || pe->event; pe++) {
374 if (pe->name == NULL)
376 if (strcasestr(pe->name, ev) != NULL) {
377 if (pe->long_desc != NULL)
378 printf("%s:\n%s\n", pe->name, pe->long_desc);
379 else if (pe->desc != NULL)
380 printf("%s:\t%s\n", pe->name, pe->desc);
386 pmc_pmu_print_counter_full(const char *ev)
388 const struct pmu_events_map *pme;
389 const struct pmu_event *pe;
391 if ((pme = pmu_events_map_get(NULL)) == NULL)
393 for (pe = pme->table; pe->name || pe->desc || pe->event; pe++) {
394 if (pe->name == NULL)
396 if (strcasestr(pe->name, ev) == NULL)
398 printf("name: %s\n", pe->name);
399 if (pe->long_desc != NULL)
400 printf("desc: %s\n", pe->long_desc);
401 else if (pe->desc != NULL)
402 printf("desc: %s\n", pe->desc);
403 if (pe->event != NULL)
404 printf("event: %s\n", pe->event);
405 if (pe->topic != NULL)
406 printf("topic: %s\n", pe->topic);
408 printf("pmu: %s\n", pe->pmu);
409 if (pe->unit != NULL)
410 printf("unit: %s\n", pe->unit);
411 if (pe->perpkg != NULL)
412 printf("perpkg: %s\n", pe->perpkg);
413 if (pe->metric_expr != NULL)
414 printf("metric_expr: %s\n", pe->metric_expr);
415 if (pe->metric_name != NULL)
416 printf("metric_name: %s\n", pe->metric_name);
417 if (pe->metric_group != NULL)
418 printf("metric_group: %s\n", pe->metric_group);
423 pmc_pmu_amd_pmcallocate(const char *event_name, struct pmc_op_pmcallocate *pm,
424 struct pmu_event_desc *ped)
426 struct pmc_md_amd_op_pmcallocate *amd;
427 const struct pmu_event *pe;
430 amd = &pm->pm_md.pm_amd;
431 if (ped->ped_umask > 0) {
432 pm->pm_caps |= PMC_CAP_QUALIFIER;
433 amd->pm_amd_config |= AMD_PMC_TO_UNITMASK(ped->ped_umask);
435 pm->pm_class = PMC_CLASS_K8;
436 pe = pmu_event_get(NULL, event_name, &idx);
438 if (strcmp("l3cache", pe->topic) == 0){
439 amd->pm_amd_config |= AMD_PMC_TO_EVENTMASK(ped->ped_event);
440 amd->pm_amd_sub_class = PMC_AMD_SUB_CLASS_L3_CACHE;
441 amd->pm_amd_config |= AMD_PMC_TO_L3SLICE(ped->ped_l3_slice);
442 amd->pm_amd_config |= AMD_PMC_TO_L3CORE(ped->ped_l3_thread);
444 else if (strcmp("data fabric", pe->topic) == 0){
446 amd->pm_amd_config |= AMD_PMC_TO_EVENTMASK_DF(ped->ped_event);
447 amd->pm_amd_sub_class = PMC_AMD_SUB_CLASS_DATA_FABRIC;
450 amd->pm_amd_config |= AMD_PMC_TO_EVENTMASK(ped->ped_event);
451 amd->pm_amd_sub_class = PMC_AMD_SUB_CLASS_CORE;
452 if ((pm->pm_caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0 ||
453 (pm->pm_caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) ==
454 (PMC_CAP_USER|PMC_CAP_SYSTEM))
455 amd->pm_amd_config |= (AMD_PMC_USR | AMD_PMC_OS);
456 else if (pm->pm_caps & PMC_CAP_USER)
457 amd->pm_amd_config |= AMD_PMC_USR;
458 else if (pm->pm_caps & PMC_CAP_SYSTEM)
459 amd->pm_amd_config |= AMD_PMC_OS;
461 amd->pm_amd_config |= AMD_PMC_EDGE;
463 amd->pm_amd_config |= AMD_PMC_EDGE;
464 if (pm->pm_caps & PMC_CAP_INTERRUPT)
465 amd->pm_amd_config |= AMD_PMC_INT;
471 pmc_pmu_intel_pmcallocate(const char *event_name, struct pmc_op_pmcallocate *pm,
472 struct pmu_event_desc *ped)
474 struct pmc_md_iap_op_pmcallocate *iap;
478 iap = &pm->pm_md.pm_iap;
479 if (strcasestr(event_name, "UNC_") == event_name ||
480 strcasestr(event_name, "uncore") != NULL) {
481 pm->pm_class = PMC_CLASS_UCP;
482 pm->pm_caps |= PMC_CAP_QUALIFIER;
483 } else if ((ped->ped_umask == -1) ||
484 (ped->ped_event == 0x0 && ped->ped_umask == 0x3)) {
485 pm->pm_class = PMC_CLASS_IAF;
487 pm->pm_class = PMC_CLASS_IAP;
488 pm->pm_caps |= PMC_CAP_QUALIFIER;
490 iap->pm_iap_config |= IAP_EVSEL(ped->ped_event);
491 if (ped->ped_umask > 0)
492 iap->pm_iap_config |= IAP_UMASK(ped->ped_umask);
493 iap->pm_iap_config |= IAP_CMASK(ped->ped_cmask);
494 iap->pm_iap_rsp = ped->ped_offcore_rsp;
496 if ((pm->pm_caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0 ||
497 (pm->pm_caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) ==
498 (PMC_CAP_USER|PMC_CAP_SYSTEM))
499 iap->pm_iap_config |= (IAP_USR | IAP_OS);
500 else if (pm->pm_caps & PMC_CAP_USER)
501 iap->pm_iap_config |= IAP_USR;
502 else if (pm->pm_caps & PMC_CAP_SYSTEM)
503 iap->pm_iap_config |= IAP_OS;
505 iap->pm_iap_config |= IAP_EDGE;
507 iap->pm_iap_config |= IAP_ANY;
509 iap->pm_iap_config |= IAP_EDGE;
510 if (pm->pm_caps & PMC_CAP_INTERRUPT)
511 iap->pm_iap_config |= IAP_INT;
516 pmc_pmu_pmcallocate(const char *event_name, struct pmc_op_pmcallocate *pm)
518 const struct pmu_event *pe;
519 struct pmu_event_desc ped;
523 if ((mfr = pmu_events_mfr()) == PMU_INVALID)
526 bzero(&pm->pm_md, sizeof(pm->pm_md));
527 pm->pm_caps |= (PMC_CAP_READ | PMC_CAP_WRITE);
528 event_name = pmu_alias_get(event_name);
529 if ((pe = pmu_event_get(NULL, event_name, &idx)) == NULL)
531 if (pe->alias && (pe = pmu_event_get(NULL, pe->alias, &idx)) == NULL)
536 if (pe->event == NULL)
538 if (pmu_parse_event(&ped, pe->event))
541 if (mfr == PMU_INTEL)
542 return (pmc_pmu_intel_pmcallocate(event_name, pm, &ped));
544 return (pmc_pmu_amd_pmcallocate(event_name, pm, &ped));
548 * Ultimately rely on AMD calling theirs the same
550 static const char *stat_mode_cntrs[] = {
551 "cpu_clk_unhalted.thread",
553 "br_inst_retired.all_branches",
554 "br_misp_retired.all_branches",
555 "longest_lat_cache.reference",
556 "longest_lat_cache.miss",
560 pmc_pmu_stat_mode(const char ***cntrs)
562 if (pmc_pmu_enabled()) {
563 *cntrs = stat_mode_cntrs;
572 pmc_pmu_sample_rate_get(const char *event_name __unused)
574 return (DEFAULT_SAMPLE_COUNT);
578 pmc_pmu_print_counters(const char *event_name __unused)
583 pmc_pmu_print_counter_desc(const char *e __unused)
588 pmc_pmu_print_counter_desc_long(const char *e __unused)
593 pmc_pmu_print_counter_full(const char *e __unused)
599 pmc_pmu_enabled(void)
605 pmc_pmu_pmcallocate(const char *e __unused, struct pmc_op_pmcallocate *p __unused)
611 pmc_pmu_event_get_by_idx(const char *c __unused, int idx __unused)
617 pmc_pmu_stat_mode(const char ***a __unused)
623 pmc_pmu_idx_get_by_event(const char *c __unused, const char *e __unused)