2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2008 Joseph Koshy
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33 #include <sys/cdefs.h>
34 __FBSDID("$FreeBSD$");
36 #include <sys/param.h>
39 #include <sys/pmckern.h>
40 #include <sys/systm.h>
42 #include <machine/intr_machdep.h>
43 #if (__FreeBSD_version >= 1100000)
44 #include <x86/apicvar.h>
46 #include <machine/apicvar.h>
48 #include <machine/cpu.h>
49 #include <machine/cpufunc.h>
50 #include <machine/md_var.h>
51 #include <machine/specialreg.h>
53 #define CORE_CPUID_REQUEST 0xA
54 #define CORE_CPUID_REQUEST_SIZE 0x4
55 #define CORE_CPUID_EAX 0x0
56 #define CORE_CPUID_EBX 0x1
57 #define CORE_CPUID_ECX 0x2
58 #define CORE_CPUID_EDX 0x3
60 #define IAF_PMC_CAPS \
61 (PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INTERRUPT | \
62 PMC_CAP_USER | PMC_CAP_SYSTEM)
63 #define IAF_RI_TO_MSR(RI) ((RI) + (1 << 30))
65 #define IAP_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | PMC_CAP_SYSTEM | \
66 PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \
67 PMC_CAP_INVERT | PMC_CAP_QUALIFIER | PMC_CAP_PRECISE)
69 #define EV_IS_NOTARCH 0
70 #define EV_IS_ARCH_SUPP 1
71 #define EV_IS_ARCH_NOTSUPP -1
74 * "Architectural" events defined by Intel. The values of these
75 * symbols correspond to positions in the bitmask returned by
76 * the CPUID.0AH instruction.
78 enum core_arch_events {
79 CORE_AE_BRANCH_INSTRUCTION_RETIRED = 5,
80 CORE_AE_BRANCH_MISSES_RETIRED = 6,
81 CORE_AE_INSTRUCTION_RETIRED = 1,
82 CORE_AE_LLC_MISSES = 4,
83 CORE_AE_LLC_REFERENCE = 3,
84 CORE_AE_UNHALTED_REFERENCE_CYCLES = 2,
85 CORE_AE_UNHALTED_CORE_CYCLES = 0
88 static enum pmc_cputype core_cputype;
91 volatile uint32_t pc_resync;
92 volatile uint32_t pc_iafctrl; /* Fixed function control. */
93 volatile uint64_t pc_globalctrl; /* Global control register. */
94 struct pmc_hw pc_corepmcs[];
97 static struct core_cpu **core_pcpu;
99 static uint32_t core_architectural_events;
100 static uint64_t core_pmcmask;
102 static int core_iaf_ri; /* relative index of fixed counters */
103 static int core_iaf_width;
104 static int core_iaf_npmc;
106 static int core_iap_width;
107 static int core_iap_npmc;
108 static int core_iap_wroffset;
111 core_pcpu_noop(struct pmc_mdep *md, int cpu)
119 core_pcpu_init(struct pmc_mdep *md, int cpu)
124 int core_ri, n, npmc;
126 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
127 ("[iaf,%d] insane cpu number %d", __LINE__, cpu));
129 PMCDBG1(MDP,INI,1,"core-init cpu=%d", cpu);
131 core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri;
132 npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num;
134 if (core_cputype != PMC_CPU_INTEL_CORE)
135 npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num;
137 cc = malloc(sizeof(struct core_cpu) + npmc * sizeof(struct pmc_hw),
138 M_PMC, M_WAITOK | M_ZERO);
143 KASSERT(pc != NULL && cc != NULL,
144 ("[core,%d] NULL per-cpu structures cpu=%d", __LINE__, cpu));
146 for (n = 0, phw = cc->pc_corepmcs; n < npmc; n++, phw++) {
147 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
148 PMC_PHW_CPU_TO_STATE(cpu) |
149 PMC_PHW_INDEX_TO_STATE(n + core_ri);
151 pc->pc_hwpmcs[n + core_ri] = phw;
158 core_pcpu_fini(struct pmc_mdep *md, int cpu)
160 int core_ri, n, npmc;
165 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
166 ("[core,%d] insane cpu number (%d)", __LINE__, cpu));
168 PMCDBG1(MDP,INI,1,"core-pcpu-fini cpu=%d", cpu);
170 if ((cc = core_pcpu[cpu]) == NULL)
173 core_pcpu[cpu] = NULL;
177 KASSERT(pc != NULL, ("[core,%d] NULL per-cpu %d state", __LINE__,
180 npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num;
181 core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri;
183 for (n = 0; n < npmc; n++) {
184 msr = rdmsr(IAP_EVSEL0 + n) & ~IAP_EVSEL_MASK;
185 wrmsr(IAP_EVSEL0 + n, msr);
188 if (core_cputype != PMC_CPU_INTEL_CORE) {
189 msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK;
190 wrmsr(IAF_CTRL, msr);
191 npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num;
194 for (n = 0; n < npmc; n++)
195 pc->pc_hwpmcs[n + core_ri] = NULL;
203 * Fixed function counters.
207 iaf_perfctr_value_to_reload_count(pmc_value_t v)
210 /* If the PMC has overflowed, return a reload count of zero. */
211 if ((v & (1ULL << (core_iaf_width - 1))) == 0)
213 v &= (1ULL << core_iaf_width) - 1;
214 return (1ULL << core_iaf_width) - v;
218 iaf_reload_count_to_perfctr_value(pmc_value_t rlc)
220 return (1ULL << core_iaf_width) - rlc;
224 iaf_allocate_pmc(int cpu, int ri, struct pmc *pm,
225 const struct pmc_op_pmcallocate *a)
228 uint32_t caps, flags, config;
229 const struct pmc_md_iap_op_pmcallocate *iap;
231 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
232 ("[core,%d] illegal CPU %d", __LINE__, cpu));
234 PMCDBG2(MDP,ALL,1, "iaf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps);
236 if (ri < 0 || ri > core_iaf_npmc)
241 if (a->pm_class != PMC_CLASS_IAF ||
242 (caps & IAF_PMC_CAPS) != caps)
245 iap = &a->pm_md.pm_iap;
246 config = iap->pm_iap_config;
247 ev = IAP_EVSEL_GET(config);
248 umask = IAP_UMASK_GET(config);
250 /* INST_RETIRED.ANY */
251 if (ev == 0xC0 && ri != 0)
253 /* CPU_CLK_UNHALTED.THREAD */
254 else if (ev == 0x3C && ri != 1)
256 /* CPU_CLK_UNHALTED.REF */
257 else if (ev == 0x0 && umask == 0x3 && ri != 2)
265 if (config & IAP_USR)
267 if (config & IAP_ANY)
269 if (config & IAP_INT)
272 if (caps & PMC_CAP_INTERRUPT)
274 if (caps & PMC_CAP_SYSTEM)
276 if (caps & PMC_CAP_USER)
278 if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0)
279 flags |= (IAF_OS | IAF_USR);
281 pm->pm_md.pm_iaf.pm_iaf_ctrl = (flags << (ri * 4));
283 PMCDBG1(MDP,ALL,2, "iaf-allocate config=0x%jx",
284 (uintmax_t) pm->pm_md.pm_iaf.pm_iaf_ctrl);
290 iaf_config_pmc(int cpu, int ri, struct pmc *pm)
292 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
293 ("[core,%d] illegal CPU %d", __LINE__, cpu));
295 KASSERT(ri >= 0 && ri < core_iaf_npmc,
296 ("[core,%d] illegal row-index %d", __LINE__, ri));
298 PMCDBG3(MDP,CFG,1, "iaf-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
300 KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__,
303 core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc = pm;
309 iaf_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
313 char iaf_name[PMC_NAME_MAX];
315 phw = &core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri];
317 (void) snprintf(iaf_name, sizeof(iaf_name), "IAF-%d", ri);
318 if ((error = copystr(iaf_name, pi->pm_name, PMC_NAME_MAX,
322 pi->pm_class = PMC_CLASS_IAF;
324 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
325 pi->pm_enabled = TRUE;
326 *ppmc = phw->phw_pmc;
328 pi->pm_enabled = FALSE;
336 iaf_get_config(int cpu, int ri, struct pmc **ppm)
338 *ppm = core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc;
344 iaf_get_msr(int ri, uint32_t *msr)
346 KASSERT(ri >= 0 && ri < core_iaf_npmc,
347 ("[iaf,%d] ri %d out of range", __LINE__, ri));
349 *msr = IAF_RI_TO_MSR(ri);
355 iaf_read_pmc(int cpu, int ri, pmc_value_t *v)
360 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
361 ("[core,%d] illegal cpu value %d", __LINE__, cpu));
362 KASSERT(ri >= 0 && ri < core_iaf_npmc,
363 ("[core,%d] illegal row-index %d", __LINE__, ri));
365 pm = core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc;
368 ("[core,%d] cpu %d ri %d(%d) pmc not configured", __LINE__, cpu,
369 ri, ri + core_iaf_ri));
371 tmp = rdpmc(IAF_RI_TO_MSR(ri));
373 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
374 *v = iaf_perfctr_value_to_reload_count(tmp);
376 *v = tmp & ((1ULL << core_iaf_width) - 1);
378 PMCDBG4(MDP,REA,1, "iaf-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
379 IAF_RI_TO_MSR(ri), *v);
385 iaf_release_pmc(int cpu, int ri, struct pmc *pmc)
387 PMCDBG3(MDP,REL,1, "iaf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc);
389 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
390 ("[core,%d] illegal CPU value %d", __LINE__, cpu));
391 KASSERT(ri >= 0 && ri < core_iaf_npmc,
392 ("[core,%d] illegal row-index %d", __LINE__, ri));
394 KASSERT(core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc == NULL,
395 ("[core,%d] PHW pmc non-NULL", __LINE__));
401 iaf_start_pmc(int cpu, int ri)
404 struct core_cpu *iafc;
407 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
408 ("[core,%d] illegal CPU value %d", __LINE__, cpu));
409 KASSERT(ri >= 0 && ri < core_iaf_npmc,
410 ("[core,%d] illegal row-index %d", __LINE__, ri));
412 PMCDBG2(MDP,STA,1,"iaf-start cpu=%d ri=%d", cpu, ri);
414 iafc = core_pcpu[cpu];
415 pm = iafc->pc_corepmcs[ri + core_iaf_ri].phw_pmc;
417 iafc->pc_iafctrl |= pm->pm_md.pm_iaf.pm_iaf_ctrl;
419 msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK;
420 wrmsr(IAF_CTRL, msr | (iafc->pc_iafctrl & IAF_CTRL_MASK));
424 iafc->pc_globalctrl |= (1ULL << (ri + IAF_OFFSET));
425 msr = rdmsr(IA_GLOBAL_CTRL) & ~IAF_GLOBAL_CTRL_MASK;
426 wrmsr(IA_GLOBAL_CTRL, msr | (iafc->pc_globalctrl &
427 IAF_GLOBAL_CTRL_MASK));
428 } while (iafc->pc_resync != 0);
430 PMCDBG4(MDP,STA,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)",
431 iafc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL),
432 iafc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL));
438 iaf_stop_pmc(int cpu, int ri)
441 struct core_cpu *iafc;
444 PMCDBG2(MDP,STO,1,"iaf-stop cpu=%d ri=%d", cpu, ri);
446 iafc = core_pcpu[cpu];
448 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
449 ("[core,%d] illegal CPU value %d", __LINE__, cpu));
450 KASSERT(ri >= 0 && ri < core_iaf_npmc,
451 ("[core,%d] illegal row-index %d", __LINE__, ri));
453 fc = (IAF_MASK << (ri * 4));
455 iafc->pc_iafctrl &= ~fc;
457 PMCDBG1(MDP,STO,1,"iaf-stop iafctrl=%x", iafc->pc_iafctrl);
458 msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK;
459 wrmsr(IAF_CTRL, msr | (iafc->pc_iafctrl & IAF_CTRL_MASK));
463 iafc->pc_globalctrl &= ~(1ULL << (ri + IAF_OFFSET));
464 msr = rdmsr(IA_GLOBAL_CTRL) & ~IAF_GLOBAL_CTRL_MASK;
465 wrmsr(IA_GLOBAL_CTRL, msr | (iafc->pc_globalctrl &
466 IAF_GLOBAL_CTRL_MASK));
467 } while (iafc->pc_resync != 0);
469 PMCDBG4(MDP,STO,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)",
470 iafc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL),
471 iafc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL));
477 iaf_write_pmc(int cpu, int ri, pmc_value_t v)
483 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
484 ("[core,%d] illegal cpu value %d", __LINE__, cpu));
485 KASSERT(ri >= 0 && ri < core_iaf_npmc,
486 ("[core,%d] illegal row-index %d", __LINE__, ri));
489 pm = cc->pc_corepmcs[ri + core_iaf_ri].phw_pmc;
492 ("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
494 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
495 v = iaf_reload_count_to_perfctr_value(v);
497 /* Turn off fixed counters */
498 msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK;
499 wrmsr(IAF_CTRL, msr);
501 wrmsr(IAF_CTR0 + ri, v & ((1ULL << core_iaf_width) - 1));
503 /* Turn on fixed counters */
504 msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK;
505 wrmsr(IAF_CTRL, msr | (cc->pc_iafctrl & IAF_CTRL_MASK));
507 PMCDBG6(MDP,WRI,1, "iaf-write cpu=%d ri=%d msr=0x%x v=%jx iafctrl=%jx "
508 "pmc=%jx", cpu, ri, IAF_RI_TO_MSR(ri), v,
509 (uintmax_t) rdmsr(IAF_CTRL),
510 (uintmax_t) rdpmc(IAF_RI_TO_MSR(ri)));
517 iaf_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth)
519 struct pmc_classdep *pcd;
521 KASSERT(md != NULL, ("[iaf,%d] md is NULL", __LINE__));
523 PMCDBG0(MDP,INI,1, "iaf-initialize");
525 pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF];
527 pcd->pcd_caps = IAF_PMC_CAPS;
528 pcd->pcd_class = PMC_CLASS_IAF;
530 pcd->pcd_ri = md->pmd_npmc;
531 pcd->pcd_width = pmcwidth;
533 pcd->pcd_allocate_pmc = iaf_allocate_pmc;
534 pcd->pcd_config_pmc = iaf_config_pmc;
535 pcd->pcd_describe = iaf_describe;
536 pcd->pcd_get_config = iaf_get_config;
537 pcd->pcd_get_msr = iaf_get_msr;
538 pcd->pcd_pcpu_fini = core_pcpu_noop;
539 pcd->pcd_pcpu_init = core_pcpu_noop;
540 pcd->pcd_read_pmc = iaf_read_pmc;
541 pcd->pcd_release_pmc = iaf_release_pmc;
542 pcd->pcd_start_pmc = iaf_start_pmc;
543 pcd->pcd_stop_pmc = iaf_stop_pmc;
544 pcd->pcd_write_pmc = iaf_write_pmc;
546 md->pmd_npmc += npmc;
550 * Intel programmable PMCs.
553 /* Sub fields of UMASK that this event supports. */
554 #define IAP_M_CORE (1 << 0) /* Core specificity */
555 #define IAP_M_AGENT (1 << 1) /* Agent specificity */
556 #define IAP_M_PREFETCH (1 << 2) /* Prefetch */
557 #define IAP_M_MESI (1 << 3) /* MESI */
558 #define IAP_M_SNOOPRESPONSE (1 << 4) /* Snoop response */
559 #define IAP_M_SNOOPTYPE (1 << 5) /* Snoop type */
560 #define IAP_M_TRANSITION (1 << 6) /* Transition */
562 #define IAP_F_CORE (0x3 << 14) /* Core specificity */
563 #define IAP_F_AGENT (0x1 << 13) /* Agent specificity */
564 #define IAP_F_PREFETCH (0x3 << 12) /* Prefetch */
565 #define IAP_F_MESI (0xF << 8) /* MESI */
566 #define IAP_F_SNOOPRESPONSE (0xB << 8) /* Snoop response */
567 #define IAP_F_SNOOPTYPE (0x3 << 8) /* Snoop type */
568 #define IAP_F_TRANSITION (0x1 << 12) /* Transition */
570 #define IAP_PREFETCH_RESERVED (0x2 << 12)
571 #define IAP_CORE_THIS (0x1 << 14)
572 #define IAP_CORE_ALL (0x3 << 14)
573 #define IAP_F_CMASK 0xFF000000
576 iap_perfctr_value_to_reload_count(pmc_value_t v)
579 /* If the PMC has overflowed, return a reload count of zero. */
580 if ((v & (1ULL << (core_iap_width - 1))) == 0)
582 v &= (1ULL << core_iap_width) - 1;
583 return (1ULL << core_iap_width) - v;
587 iap_reload_count_to_perfctr_value(pmc_value_t rlc)
589 return (1ULL << core_iap_width) - rlc;
593 iap_pmc_has_overflowed(int ri)
598 * We treat a Core (i.e., Intel architecture v1) PMC as has
599 * having overflowed if its MSB is zero.
602 return ((v & (1ULL << (core_iap_width - 1))) == 0);
606 iap_event_corei7_ok_on_counter(uint8_t evsel, int ri)
612 * Events valid only on counter 0, 1.
624 mask = ~0; /* Any row index is ok. */
627 return (mask & (1 << ri));
631 iap_event_westmere_ok_on_counter(uint8_t evsel, int ri)
637 * Events valid only on counter 0.
645 * Events valid only on counter 0, 1.
655 mask = ~0; /* Any row index is ok. */
658 return (mask & (1 << ri));
662 iap_event_sb_sbx_ib_ibx_ok_on_counter(uint8_t evsel, int ri)
667 /* Events valid only on counter 0. */
671 /* Events valid only on counter 1. */
675 /* Events valid only on counter 2. */
681 /* Events valid only on counter 3. */
687 mask = ~0; /* Any row index is ok. */
690 return (mask & (1 << ri));
694 iap_event_ok_on_counter(uint8_t evsel, int ri)
700 * Events valid only on counter 0.
712 * Events valid only on counter 1.
721 mask = ~0; /* Any row index is ok. */
724 return (mask & (1 << ri));
728 iap_allocate_pmc(int cpu, int ri, struct pmc *pm,
729 const struct pmc_op_pmcallocate *a)
734 const struct pmc_md_iap_op_pmcallocate *iap;
736 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
737 ("[core,%d] illegal CPU %d", __LINE__, cpu));
738 KASSERT(ri >= 0 && ri < core_iap_npmc,
739 ("[core,%d] illegal row-index value %d", __LINE__, ri));
741 /* check requested capabilities */
743 if ((IAP_PMC_CAPS & caps) != caps)
745 map = 0; /* XXX: silent GCC warning */
746 iap = &a->pm_md.pm_iap;
747 ev = IAP_EVSEL_GET(iap->pm_iap_config);
749 switch (core_cputype) {
750 case PMC_CPU_INTEL_COREI7:
751 case PMC_CPU_INTEL_NEHALEM_EX:
752 if (iap_event_corei7_ok_on_counter(ev, ri) == 0)
755 case PMC_CPU_INTEL_SKYLAKE:
756 case PMC_CPU_INTEL_SKYLAKE_XEON:
757 case PMC_CPU_INTEL_BROADWELL:
758 case PMC_CPU_INTEL_BROADWELL_XEON:
759 case PMC_CPU_INTEL_SANDYBRIDGE:
760 case PMC_CPU_INTEL_SANDYBRIDGE_XEON:
761 case PMC_CPU_INTEL_IVYBRIDGE:
762 case PMC_CPU_INTEL_IVYBRIDGE_XEON:
763 case PMC_CPU_INTEL_HASWELL:
764 case PMC_CPU_INTEL_HASWELL_XEON:
765 if (iap_event_sb_sbx_ib_ibx_ok_on_counter(ev, ri) == 0)
768 case PMC_CPU_INTEL_WESTMERE:
769 case PMC_CPU_INTEL_WESTMERE_EX:
770 if (iap_event_westmere_ok_on_counter(ev, ri) == 0)
774 if (iap_event_ok_on_counter(ev, ri) == 0)
778 pm->pm_md.pm_iap.pm_iap_evsel = iap->pm_iap_config;
783 iap_config_pmc(int cpu, int ri, struct pmc *pm)
785 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
786 ("[core,%d] illegal CPU %d", __LINE__, cpu));
788 KASSERT(ri >= 0 && ri < core_iap_npmc,
789 ("[core,%d] illegal row-index %d", __LINE__, ri));
791 PMCDBG3(MDP,CFG,1, "iap-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
793 KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__,
796 core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc = pm;
802 iap_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
806 char iap_name[PMC_NAME_MAX];
808 phw = &core_pcpu[cpu]->pc_corepmcs[ri];
810 (void) snprintf(iap_name, sizeof(iap_name), "IAP-%d", ri);
811 if ((error = copystr(iap_name, pi->pm_name, PMC_NAME_MAX,
815 pi->pm_class = PMC_CLASS_IAP;
817 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
818 pi->pm_enabled = TRUE;
819 *ppmc = phw->phw_pmc;
821 pi->pm_enabled = FALSE;
829 iap_get_config(int cpu, int ri, struct pmc **ppm)
831 *ppm = core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc;
837 iap_get_msr(int ri, uint32_t *msr)
839 KASSERT(ri >= 0 && ri < core_iap_npmc,
840 ("[iap,%d] ri %d out of range", __LINE__, ri));
848 iap_read_pmc(int cpu, int ri, pmc_value_t *v)
853 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
854 ("[core,%d] illegal cpu value %d", __LINE__, cpu));
855 KASSERT(ri >= 0 && ri < core_iap_npmc,
856 ("[core,%d] illegal row-index %d", __LINE__, ri));
858 pm = core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc;
861 ("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu,
865 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
866 *v = iap_perfctr_value_to_reload_count(tmp);
868 *v = tmp & ((1ULL << core_iap_width) - 1);
870 PMCDBG4(MDP,REA,1, "iap-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
877 iap_release_pmc(int cpu, int ri, struct pmc *pm)
881 PMCDBG3(MDP,REL,1, "iap-release cpu=%d ri=%d pm=%p", cpu, ri,
884 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
885 ("[core,%d] illegal CPU value %d", __LINE__, cpu));
886 KASSERT(ri >= 0 && ri < core_iap_npmc,
887 ("[core,%d] illegal row-index %d", __LINE__, ri));
889 KASSERT(core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc
890 == NULL, ("[core,%d] PHW pmc non-NULL", __LINE__));
896 iap_start_pmc(int cpu, int ri)
902 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
903 ("[core,%d] illegal CPU value %d", __LINE__, cpu));
904 KASSERT(ri >= 0 && ri < core_iap_npmc,
905 ("[core,%d] illegal row-index %d", __LINE__, ri));
908 pm = cc->pc_corepmcs[ri].phw_pmc;
911 ("[core,%d] starting cpu%d,ri%d with no pmc configured",
914 PMCDBG2(MDP,STA,1, "iap-start cpu=%d ri=%d", cpu, ri);
916 evsel = pm->pm_md.pm_iap.pm_iap_evsel;
918 PMCDBG4(MDP,STA,2, "iap-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x",
919 cpu, ri, IAP_EVSEL0 + ri, evsel);
921 /* Event specific configuration. */
923 switch (IAP_EVSEL_GET(evsel)) {
925 wrmsr(IA_OFFCORE_RSP0, pm->pm_md.pm_iap.pm_iap_rsp);
928 wrmsr(IA_OFFCORE_RSP1, pm->pm_md.pm_iap.pm_iap_rsp);
934 wrmsr(IAP_EVSEL0 + ri, evsel | IAP_EN);
936 if (core_cputype == PMC_CPU_INTEL_CORE)
941 cc->pc_globalctrl |= (1ULL << ri);
942 wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);
943 } while (cc->pc_resync != 0);
949 iap_stop_pmc(int cpu, int ri)
955 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
956 ("[core,%d] illegal cpu value %d", __LINE__, cpu));
957 KASSERT(ri >= 0 && ri < core_iap_npmc,
958 ("[core,%d] illegal row index %d", __LINE__, ri));
961 pm = cc->pc_corepmcs[ri].phw_pmc;
964 ("[core,%d] cpu%d ri%d no configured PMC to stop", __LINE__,
967 PMCDBG2(MDP,STO,1, "iap-stop cpu=%d ri=%d", cpu, ri);
969 msr = rdmsr(IAP_EVSEL0 + ri) & ~IAP_EVSEL_MASK;
970 wrmsr(IAP_EVSEL0 + ri, msr); /* stop hw */
972 if (core_cputype == PMC_CPU_INTEL_CORE)
978 cc->pc_globalctrl &= ~(1ULL << ri);
979 msr = rdmsr(IA_GLOBAL_CTRL) & ~IA_GLOBAL_CTRL_MASK;
980 wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);
981 } while (cc->pc_resync != 0);
987 iap_write_pmc(int cpu, int ri, pmc_value_t v)
992 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
993 ("[core,%d] illegal cpu value %d", __LINE__, cpu));
994 KASSERT(ri >= 0 && ri < core_iap_npmc,
995 ("[core,%d] illegal row index %d", __LINE__, ri));
998 pm = cc->pc_corepmcs[ri].phw_pmc;
1001 ("[core,%d] cpu%d ri%d no configured PMC to stop", __LINE__,
1004 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1005 v = iap_reload_count_to_perfctr_value(v);
1007 v &= (1ULL << core_iap_width) - 1;
1009 PMCDBG4(MDP,WRI,1, "iap-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri,
1013 * Write the new value to the counter (or it's alias). The
1014 * counter will be in a stopped state when the pcd_write()
1015 * entry point is called.
1017 wrmsr(core_iap_wroffset + IAP_PMC0 + ri, v);
1023 iap_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth,
1026 struct pmc_classdep *pcd;
1028 KASSERT(md != NULL, ("[iap,%d] md is NULL", __LINE__));
1030 PMCDBG0(MDP,INI,1, "iap-initialize");
1032 /* Remember the set of architectural events supported. */
1033 core_architectural_events = ~flags;
1035 pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP];
1037 pcd->pcd_caps = IAP_PMC_CAPS;
1038 pcd->pcd_class = PMC_CLASS_IAP;
1039 pcd->pcd_num = npmc;
1040 pcd->pcd_ri = md->pmd_npmc;
1041 pcd->pcd_width = pmcwidth;
1043 pcd->pcd_allocate_pmc = iap_allocate_pmc;
1044 pcd->pcd_config_pmc = iap_config_pmc;
1045 pcd->pcd_describe = iap_describe;
1046 pcd->pcd_get_config = iap_get_config;
1047 pcd->pcd_get_msr = iap_get_msr;
1048 pcd->pcd_pcpu_fini = core_pcpu_fini;
1049 pcd->pcd_pcpu_init = core_pcpu_init;
1050 pcd->pcd_read_pmc = iap_read_pmc;
1051 pcd->pcd_release_pmc = iap_release_pmc;
1052 pcd->pcd_start_pmc = iap_start_pmc;
1053 pcd->pcd_stop_pmc = iap_stop_pmc;
1054 pcd->pcd_write_pmc = iap_write_pmc;
1056 md->pmd_npmc += npmc;
1060 core_intr(int cpu, struct trapframe *tf)
1064 struct core_cpu *cc;
1065 int error, found_interrupt, ri;
1068 PMCDBG3(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf,
1069 TRAPF_USERMODE(tf));
1071 found_interrupt = 0;
1072 cc = core_pcpu[cpu];
1074 for (ri = 0; ri < core_iap_npmc; ri++) {
1076 if ((pm = cc->pc_corepmcs[ri].phw_pmc) == NULL ||
1077 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1080 if (!iap_pmc_has_overflowed(ri))
1083 found_interrupt = 1;
1085 if (pm->pm_state != PMC_STATE_RUNNING)
1088 error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
1089 TRAPF_USERMODE(tf));
1091 v = pm->pm_sc.pm_reloadcount;
1092 v = iap_reload_count_to_perfctr_value(v);
1095 * Stop the counter, reload it but only restart it if
1096 * the PMC is not stalled.
1098 msr = rdmsr(IAP_EVSEL0 + ri) & ~IAP_EVSEL_MASK;
1099 wrmsr(IAP_EVSEL0 + ri, msr);
1100 wrmsr(core_iap_wroffset + IAP_PMC0 + ri, v);
1105 wrmsr(IAP_EVSEL0 + ri, msr | (pm->pm_md.pm_iap.pm_iap_evsel |
1109 if (found_interrupt)
1110 lapic_reenable_pmc();
1112 if (found_interrupt)
1113 counter_u64_add(pmc_stats.pm_intr_processed, 1);
1115 counter_u64_add(pmc_stats.pm_intr_ignored, 1);
1117 return (found_interrupt);
1121 core2_intr(int cpu, struct trapframe *tf)
1123 int error, found_interrupt, n;
1124 uint64_t flag, intrstatus, intrenable, msr;
1126 struct core_cpu *cc;
1129 PMCDBG3(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf,
1130 TRAPF_USERMODE(tf));
1133 * The IA_GLOBAL_STATUS (MSR 0x38E) register indicates which
1134 * PMCs have a pending PMI interrupt. We take a 'snapshot' of
1135 * the current set of interrupting PMCs and process these
1136 * after stopping them.
1138 intrstatus = rdmsr(IA_GLOBAL_STATUS);
1139 intrenable = intrstatus & core_pmcmask;
1141 PMCDBG2(MDP,INT, 1, "cpu=%d intrstatus=%jx", cpu,
1142 (uintmax_t) intrstatus);
1144 found_interrupt = 0;
1145 cc = core_pcpu[cpu];
1147 KASSERT(cc != NULL, ("[core,%d] null pcpu", __LINE__));
1149 cc->pc_globalctrl &= ~intrenable;
1150 cc->pc_resync = 1; /* MSRs now potentially out of sync. */
1153 * Stop PMCs and clear overflow status bits.
1155 msr = rdmsr(IA_GLOBAL_CTRL) & ~IA_GLOBAL_CTRL_MASK;
1156 wrmsr(IA_GLOBAL_CTRL, msr);
1157 wrmsr(IA_GLOBAL_OVF_CTRL, intrenable |
1158 IA_GLOBAL_STATUS_FLAG_OVFBUF |
1159 IA_GLOBAL_STATUS_FLAG_CONDCHG);
1162 * Look for interrupts from fixed function PMCs.
1164 for (n = 0, flag = (1ULL << IAF_OFFSET); n < core_iaf_npmc;
1167 if ((intrstatus & flag) == 0)
1170 found_interrupt = 1;
1172 pm = cc->pc_corepmcs[n + core_iaf_ri].phw_pmc;
1173 if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING ||
1174 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1177 error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
1178 TRAPF_USERMODE(tf));
1181 intrenable &= ~flag;
1183 v = iaf_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount);
1185 /* Reload sampling count. */
1186 wrmsr(IAF_CTR0 + n, v);
1188 PMCDBG4(MDP,INT, 1, "iaf-intr cpu=%d error=%d v=%jx(%jx)", cpu,
1189 error, (uintmax_t) v, (uintmax_t) rdpmc(IAF_RI_TO_MSR(n)));
1193 * Process interrupts from the programmable counters.
1195 for (n = 0, flag = 1; n < core_iap_npmc; n++, flag <<= 1) {
1196 if ((intrstatus & flag) == 0)
1199 found_interrupt = 1;
1201 pm = cc->pc_corepmcs[n].phw_pmc;
1202 if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING ||
1203 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1206 error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
1207 TRAPF_USERMODE(tf));
1209 intrenable &= ~flag;
1211 v = iap_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount);
1213 PMCDBG3(MDP,INT, 1, "iap-intr cpu=%d error=%d v=%jx", cpu, error,
1216 /* Reload sampling count. */
1217 wrmsr(core_iap_wroffset + IAP_PMC0 + n, v);
1221 * Reenable all non-stalled PMCs.
1223 PMCDBG2(MDP,INT, 1, "cpu=%d intrenable=%jx", cpu,
1224 (uintmax_t) intrenable);
1226 cc->pc_globalctrl |= intrenable;
1228 wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl & IA_GLOBAL_CTRL_MASK);
1230 PMCDBG5(MDP,INT, 1, "cpu=%d fixedctrl=%jx globalctrl=%jx status=%jx "
1231 "ovf=%jx", cpu, (uintmax_t) rdmsr(IAF_CTRL),
1232 (uintmax_t) rdmsr(IA_GLOBAL_CTRL),
1233 (uintmax_t) rdmsr(IA_GLOBAL_STATUS),
1234 (uintmax_t) rdmsr(IA_GLOBAL_OVF_CTRL));
1236 if (found_interrupt)
1237 lapic_reenable_pmc();
1239 if (found_interrupt)
1240 counter_u64_add(pmc_stats.pm_intr_processed, 1);
1242 counter_u64_add(pmc_stats.pm_intr_ignored, 1);
1244 return (found_interrupt);
1248 pmc_core_initialize(struct pmc_mdep *md, int maxcpu, int version_override)
1250 int cpuid[CORE_CPUID_REQUEST_SIZE];
1251 int ipa_version, flags, nflags;
1253 do_cpuid(CORE_CPUID_REQUEST, cpuid);
1255 ipa_version = (version_override > 0) ? version_override :
1256 cpuid[CORE_CPUID_EAX] & 0xFF;
1257 core_cputype = md->pmd_cputype;
1259 PMCDBG3(MDP,INI,1,"core-init cputype=%d ncpu=%d ipa-version=%d",
1260 core_cputype, maxcpu, ipa_version);
1262 if (ipa_version < 1 || ipa_version > 4 ||
1263 (core_cputype != PMC_CPU_INTEL_CORE && ipa_version == 1)) {
1264 /* Unknown PMC architecture. */
1265 printf("hwpc_core: unknown PMC architecture: %d\n",
1267 return (EPROGMISMATCH);
1270 core_iap_wroffset = 0;
1271 if (cpu_feature2 & CPUID2_PDCM) {
1272 if (rdmsr(IA32_PERF_CAPABILITIES) & PERFCAP_FW_WRITE) {
1273 PMCDBG0(MDP, INI, 1,
1274 "core-init full-width write supported");
1275 core_iap_wroffset = IAP_A_PMC0 - IAP_PMC0;
1277 PMCDBG0(MDP, INI, 1,
1278 "core-init full-width write NOT supported");
1280 PMCDBG0(MDP, INI, 1, "core-init pdcm not supported");
1285 * Initialize programmable counters.
1287 core_iap_npmc = (cpuid[CORE_CPUID_EAX] >> 8) & 0xFF;
1288 core_iap_width = (cpuid[CORE_CPUID_EAX] >> 16) & 0xFF;
1290 core_pmcmask |= ((1ULL << core_iap_npmc) - 1);
1292 nflags = (cpuid[CORE_CPUID_EAX] >> 24) & 0xFF;
1293 flags = cpuid[CORE_CPUID_EBX] & ((1 << nflags) - 1);
1295 iap_initialize(md, maxcpu, core_iap_npmc, core_iap_width, flags);
1298 * Initialize fixed function counters, if present.
1300 if (core_cputype != PMC_CPU_INTEL_CORE) {
1301 core_iaf_ri = core_iap_npmc;
1302 core_iaf_npmc = cpuid[CORE_CPUID_EDX] & 0x1F;
1303 core_iaf_width = (cpuid[CORE_CPUID_EDX] >> 5) & 0xFF;
1305 iaf_initialize(md, maxcpu, core_iaf_npmc, core_iaf_width);
1306 core_pmcmask |= ((1ULL << core_iaf_npmc) - 1) << IAF_OFFSET;
1309 PMCDBG2(MDP,INI,1,"core-init pmcmask=0x%jx iafri=%d", core_pmcmask,
1312 core_pcpu = malloc(sizeof(*core_pcpu) * maxcpu, M_PMC,
1316 * Choose the appropriate interrupt handler.
1318 if (ipa_version == 1)
1319 md->pmd_intr = core_intr;
1321 md->pmd_intr = core2_intr;
1323 md->pmd_pcpu_fini = NULL;
1324 md->pmd_pcpu_init = NULL;
1330 pmc_core_finalize(struct pmc_mdep *md)
1332 PMCDBG0(MDP,INI,1, "core-finalize");
1334 free(core_pcpu, M_PMC);