2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2008 Joseph Koshy
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33 #include <sys/cdefs.h>
34 __FBSDID("$FreeBSD$");
36 #include <sys/param.h>
39 #include <sys/pmckern.h>
40 #include <sys/systm.h>
42 #include <machine/intr_machdep.h>
43 #if (__FreeBSD_version >= 1100000)
44 #include <x86/apicvar.h>
46 #include <machine/apicvar.h>
48 #include <machine/cpu.h>
49 #include <machine/cpufunc.h>
50 #include <machine/md_var.h>
51 #include <machine/specialreg.h>
53 #define CORE_CPUID_REQUEST 0xA
54 #define CORE_CPUID_REQUEST_SIZE 0x4
55 #define CORE_CPUID_EAX 0x0
56 #define CORE_CPUID_EBX 0x1
57 #define CORE_CPUID_ECX 0x2
58 #define CORE_CPUID_EDX 0x3
60 #define IAF_PMC_CAPS \
61 (PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INTERRUPT | \
62 PMC_CAP_USER | PMC_CAP_SYSTEM)
63 #define IAF_RI_TO_MSR(RI) ((RI) + (1 << 30))
65 #define IAP_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | PMC_CAP_SYSTEM | \
66 PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \
67 PMC_CAP_INVERT | PMC_CAP_QUALIFIER | PMC_CAP_PRECISE)
69 #define EV_IS_NOTARCH 0
70 #define EV_IS_ARCH_SUPP 1
71 #define EV_IS_ARCH_NOTSUPP -1
74 * "Architectural" events defined by Intel. The values of these
75 * symbols correspond to positions in the bitmask returned by
76 * the CPUID.0AH instruction.
78 enum core_arch_events {
79 CORE_AE_BRANCH_INSTRUCTION_RETIRED = 5,
80 CORE_AE_BRANCH_MISSES_RETIRED = 6,
81 CORE_AE_INSTRUCTION_RETIRED = 1,
82 CORE_AE_LLC_MISSES = 4,
83 CORE_AE_LLC_REFERENCE = 3,
84 CORE_AE_UNHALTED_REFERENCE_CYCLES = 2,
85 CORE_AE_UNHALTED_CORE_CYCLES = 0
88 static enum pmc_cputype core_cputype;
91 volatile uint32_t pc_resync;
92 volatile uint32_t pc_iafctrl; /* Fixed function control. */
93 volatile uint64_t pc_globalctrl; /* Global control register. */
94 struct pmc_hw pc_corepmcs[];
97 static struct core_cpu **core_pcpu;
99 static uint32_t core_architectural_events;
100 static uint64_t core_pmcmask;
102 static int core_iaf_ri; /* relative index of fixed counters */
103 static int core_iaf_width;
104 static int core_iaf_npmc;
106 static int core_iap_width;
107 static int core_iap_npmc;
108 static int core_iap_wroffset;
111 core_pcpu_noop(struct pmc_mdep *md, int cpu)
119 core_pcpu_init(struct pmc_mdep *md, int cpu)
124 int core_ri, n, npmc;
126 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
127 ("[iaf,%d] insane cpu number %d", __LINE__, cpu));
129 PMCDBG1(MDP,INI,1,"core-init cpu=%d", cpu);
131 core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri;
132 npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num;
134 if (core_cputype != PMC_CPU_INTEL_CORE)
135 npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num;
137 cc = malloc(sizeof(struct core_cpu) + npmc * sizeof(struct pmc_hw),
138 M_PMC, M_WAITOK | M_ZERO);
143 KASSERT(pc != NULL && cc != NULL,
144 ("[core,%d] NULL per-cpu structures cpu=%d", __LINE__, cpu));
146 for (n = 0, phw = cc->pc_corepmcs; n < npmc; n++, phw++) {
147 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
148 PMC_PHW_CPU_TO_STATE(cpu) |
149 PMC_PHW_INDEX_TO_STATE(n + core_ri);
151 pc->pc_hwpmcs[n + core_ri] = phw;
158 core_pcpu_fini(struct pmc_mdep *md, int cpu)
160 int core_ri, n, npmc;
165 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
166 ("[core,%d] insane cpu number (%d)", __LINE__, cpu));
168 PMCDBG1(MDP,INI,1,"core-pcpu-fini cpu=%d", cpu);
170 if ((cc = core_pcpu[cpu]) == NULL)
173 core_pcpu[cpu] = NULL;
177 KASSERT(pc != NULL, ("[core,%d] NULL per-cpu %d state", __LINE__,
180 npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num;
181 core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri;
183 for (n = 0; n < npmc; n++) {
184 msr = rdmsr(IAP_EVSEL0 + n) & ~IAP_EVSEL_MASK;
185 wrmsr(IAP_EVSEL0 + n, msr);
188 if (core_cputype != PMC_CPU_INTEL_CORE) {
189 msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK;
190 wrmsr(IAF_CTRL, msr);
191 npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num;
194 for (n = 0; n < npmc; n++)
195 pc->pc_hwpmcs[n + core_ri] = NULL;
203 * Fixed function counters.
207 iaf_perfctr_value_to_reload_count(pmc_value_t v)
210 /* If the PMC has overflowed, return a reload count of zero. */
211 if ((v & (1ULL << (core_iaf_width - 1))) == 0)
213 v &= (1ULL << core_iaf_width) - 1;
214 return (1ULL << core_iaf_width) - v;
218 iaf_reload_count_to_perfctr_value(pmc_value_t rlc)
220 return (1ULL << core_iaf_width) - rlc;
224 iaf_allocate_pmc(int cpu, int ri, struct pmc *pm,
225 const struct pmc_op_pmcallocate *a)
228 uint32_t caps, flags, validflags;
230 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
231 ("[core,%d] illegal CPU %d", __LINE__, cpu));
233 PMCDBG2(MDP,ALL,1, "iaf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps);
235 if (ri < 0 || ri > core_iaf_npmc)
240 if (a->pm_class != PMC_CLASS_IAF ||
241 (caps & IAF_PMC_CAPS) != caps)
245 if (ev < PMC_EV_IAF_FIRST || ev > PMC_EV_IAF_LAST)
248 if (ev == PMC_EV_IAF_INSTR_RETIRED_ANY && ri != 0)
250 if (ev == PMC_EV_IAF_CPU_CLK_UNHALTED_CORE && ri != 1)
252 if (ev == PMC_EV_IAF_CPU_CLK_UNHALTED_REF && ri != 2)
255 flags = a->pm_md.pm_iaf.pm_iaf_flags;
257 validflags = IAF_MASK;
259 if (caps & PMC_CAP_INTERRUPT)
261 if (caps & PMC_CAP_SYSTEM)
263 if (caps & PMC_CAP_USER)
265 if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0)
266 flags |= (IAF_OS | IAF_USR);
268 pm->pm_md.pm_iaf.pm_iaf_ctrl = (flags << (ri * 4));
270 PMCDBG1(MDP,ALL,2, "iaf-allocate config=0x%jx",
271 (uintmax_t) pm->pm_md.pm_iaf.pm_iaf_ctrl);
277 iaf_config_pmc(int cpu, int ri, struct pmc *pm)
279 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
280 ("[core,%d] illegal CPU %d", __LINE__, cpu));
282 KASSERT(ri >= 0 && ri < core_iaf_npmc,
283 ("[core,%d] illegal row-index %d", __LINE__, ri));
285 PMCDBG3(MDP,CFG,1, "iaf-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
287 KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__,
290 core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc = pm;
296 iaf_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
300 char iaf_name[PMC_NAME_MAX];
302 phw = &core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri];
304 (void) snprintf(iaf_name, sizeof(iaf_name), "IAF-%d", ri);
305 if ((error = copystr(iaf_name, pi->pm_name, PMC_NAME_MAX,
309 pi->pm_class = PMC_CLASS_IAF;
311 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
312 pi->pm_enabled = TRUE;
313 *ppmc = phw->phw_pmc;
315 pi->pm_enabled = FALSE;
323 iaf_get_config(int cpu, int ri, struct pmc **ppm)
325 *ppm = core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc;
331 iaf_get_msr(int ri, uint32_t *msr)
333 KASSERT(ri >= 0 && ri < core_iaf_npmc,
334 ("[iaf,%d] ri %d out of range", __LINE__, ri));
336 *msr = IAF_RI_TO_MSR(ri);
342 iaf_read_pmc(int cpu, int ri, pmc_value_t *v)
347 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
348 ("[core,%d] illegal cpu value %d", __LINE__, cpu));
349 KASSERT(ri >= 0 && ri < core_iaf_npmc,
350 ("[core,%d] illegal row-index %d", __LINE__, ri));
352 pm = core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc;
355 ("[core,%d] cpu %d ri %d(%d) pmc not configured", __LINE__, cpu,
356 ri, ri + core_iaf_ri));
358 tmp = rdpmc(IAF_RI_TO_MSR(ri));
360 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
361 *v = iaf_perfctr_value_to_reload_count(tmp);
363 *v = tmp & ((1ULL << core_iaf_width) - 1);
365 PMCDBG4(MDP,REA,1, "iaf-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
366 IAF_RI_TO_MSR(ri), *v);
372 iaf_release_pmc(int cpu, int ri, struct pmc *pmc)
374 PMCDBG3(MDP,REL,1, "iaf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc);
376 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
377 ("[core,%d] illegal CPU value %d", __LINE__, cpu));
378 KASSERT(ri >= 0 && ri < core_iaf_npmc,
379 ("[core,%d] illegal row-index %d", __LINE__, ri));
381 KASSERT(core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc == NULL,
382 ("[core,%d] PHW pmc non-NULL", __LINE__));
388 iaf_start_pmc(int cpu, int ri)
391 struct core_cpu *iafc;
394 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
395 ("[core,%d] illegal CPU value %d", __LINE__, cpu));
396 KASSERT(ri >= 0 && ri < core_iaf_npmc,
397 ("[core,%d] illegal row-index %d", __LINE__, ri));
399 PMCDBG2(MDP,STA,1,"iaf-start cpu=%d ri=%d", cpu, ri);
401 iafc = core_pcpu[cpu];
402 pm = iafc->pc_corepmcs[ri + core_iaf_ri].phw_pmc;
404 iafc->pc_iafctrl |= pm->pm_md.pm_iaf.pm_iaf_ctrl;
406 msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK;
407 wrmsr(IAF_CTRL, msr | (iafc->pc_iafctrl & IAF_CTRL_MASK));
411 iafc->pc_globalctrl |= (1ULL << (ri + IAF_OFFSET));
412 msr = rdmsr(IA_GLOBAL_CTRL) & ~IAF_GLOBAL_CTRL_MASK;
413 wrmsr(IA_GLOBAL_CTRL, msr | (iafc->pc_globalctrl &
414 IAF_GLOBAL_CTRL_MASK));
415 } while (iafc->pc_resync != 0);
417 PMCDBG4(MDP,STA,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)",
418 iafc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL),
419 iafc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL));
425 iaf_stop_pmc(int cpu, int ri)
428 struct core_cpu *iafc;
431 PMCDBG2(MDP,STO,1,"iaf-stop cpu=%d ri=%d", cpu, ri);
433 iafc = core_pcpu[cpu];
435 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
436 ("[core,%d] illegal CPU value %d", __LINE__, cpu));
437 KASSERT(ri >= 0 && ri < core_iaf_npmc,
438 ("[core,%d] illegal row-index %d", __LINE__, ri));
440 fc = (IAF_MASK << (ri * 4));
442 iafc->pc_iafctrl &= ~fc;
444 PMCDBG1(MDP,STO,1,"iaf-stop iafctrl=%x", iafc->pc_iafctrl);
445 msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK;
446 wrmsr(IAF_CTRL, msr | (iafc->pc_iafctrl & IAF_CTRL_MASK));
450 iafc->pc_globalctrl &= ~(1ULL << (ri + IAF_OFFSET));
451 msr = rdmsr(IA_GLOBAL_CTRL) & ~IAF_GLOBAL_CTRL_MASK;
452 wrmsr(IA_GLOBAL_CTRL, msr | (iafc->pc_globalctrl &
453 IAF_GLOBAL_CTRL_MASK));
454 } while (iafc->pc_resync != 0);
456 PMCDBG4(MDP,STO,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)",
457 iafc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL),
458 iafc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL));
464 iaf_write_pmc(int cpu, int ri, pmc_value_t v)
470 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
471 ("[core,%d] illegal cpu value %d", __LINE__, cpu));
472 KASSERT(ri >= 0 && ri < core_iaf_npmc,
473 ("[core,%d] illegal row-index %d", __LINE__, ri));
476 pm = cc->pc_corepmcs[ri + core_iaf_ri].phw_pmc;
479 ("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
481 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
482 v = iaf_reload_count_to_perfctr_value(v);
484 /* Turn off fixed counters */
485 msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK;
486 wrmsr(IAF_CTRL, msr);
488 wrmsr(IAF_CTR0 + ri, v & ((1ULL << core_iaf_width) - 1));
490 /* Turn on fixed counters */
491 msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK;
492 wrmsr(IAF_CTRL, msr | (cc->pc_iafctrl & IAF_CTRL_MASK));
494 PMCDBG6(MDP,WRI,1, "iaf-write cpu=%d ri=%d msr=0x%x v=%jx iafctrl=%jx "
495 "pmc=%jx", cpu, ri, IAF_RI_TO_MSR(ri), v,
496 (uintmax_t) rdmsr(IAF_CTRL),
497 (uintmax_t) rdpmc(IAF_RI_TO_MSR(ri)));
504 iaf_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth)
506 struct pmc_classdep *pcd;
508 KASSERT(md != NULL, ("[iaf,%d] md is NULL", __LINE__));
510 PMCDBG0(MDP,INI,1, "iaf-initialize");
512 pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF];
514 pcd->pcd_caps = IAF_PMC_CAPS;
515 pcd->pcd_class = PMC_CLASS_IAF;
517 pcd->pcd_ri = md->pmd_npmc;
518 pcd->pcd_width = pmcwidth;
520 pcd->pcd_allocate_pmc = iaf_allocate_pmc;
521 pcd->pcd_config_pmc = iaf_config_pmc;
522 pcd->pcd_describe = iaf_describe;
523 pcd->pcd_get_config = iaf_get_config;
524 pcd->pcd_get_msr = iaf_get_msr;
525 pcd->pcd_pcpu_fini = core_pcpu_noop;
526 pcd->pcd_pcpu_init = core_pcpu_noop;
527 pcd->pcd_read_pmc = iaf_read_pmc;
528 pcd->pcd_release_pmc = iaf_release_pmc;
529 pcd->pcd_start_pmc = iaf_start_pmc;
530 pcd->pcd_stop_pmc = iaf_stop_pmc;
531 pcd->pcd_write_pmc = iaf_write_pmc;
533 md->pmd_npmc += npmc;
537 * Intel programmable PMCs.
540 /* Sub fields of UMASK that this event supports. */
541 #define IAP_M_CORE (1 << 0) /* Core specificity */
542 #define IAP_M_AGENT (1 << 1) /* Agent specificity */
543 #define IAP_M_PREFETCH (1 << 2) /* Prefetch */
544 #define IAP_M_MESI (1 << 3) /* MESI */
545 #define IAP_M_SNOOPRESPONSE (1 << 4) /* Snoop response */
546 #define IAP_M_SNOOPTYPE (1 << 5) /* Snoop type */
547 #define IAP_M_TRANSITION (1 << 6) /* Transition */
549 #define IAP_F_CORE (0x3 << 14) /* Core specificity */
550 #define IAP_F_AGENT (0x1 << 13) /* Agent specificity */
551 #define IAP_F_PREFETCH (0x3 << 12) /* Prefetch */
552 #define IAP_F_MESI (0xF << 8) /* MESI */
553 #define IAP_F_SNOOPRESPONSE (0xB << 8) /* Snoop response */
554 #define IAP_F_SNOOPTYPE (0x3 << 8) /* Snoop type */
555 #define IAP_F_TRANSITION (0x1 << 12) /* Transition */
557 #define IAP_PREFETCH_RESERVED (0x2 << 12)
558 #define IAP_CORE_THIS (0x1 << 14)
559 #define IAP_CORE_ALL (0x3 << 14)
560 #define IAP_F_CMASK 0xFF000000
563 iap_perfctr_value_to_reload_count(pmc_value_t v)
566 /* If the PMC has overflowed, return a reload count of zero. */
567 if ((v & (1ULL << (core_iap_width - 1))) == 0)
569 v &= (1ULL << core_iap_width) - 1;
570 return (1ULL << core_iap_width) - v;
574 iap_reload_count_to_perfctr_value(pmc_value_t rlc)
576 return (1ULL << core_iap_width) - rlc;
580 iap_pmc_has_overflowed(int ri)
585 * We treat a Core (i.e., Intel architecture v1) PMC as has
586 * having overflowed if its MSB is zero.
589 return ((v & (1ULL << (core_iap_width - 1))) == 0);
593 iap_event_corei7_ok_on_counter(uint8_t evsel, int ri)
599 * Events valid only on counter 0, 1.
611 mask = ~0; /* Any row index is ok. */
614 return (mask & (1 << ri));
618 iap_event_westmere_ok_on_counter(uint8_t evsel, int ri)
624 * Events valid only on counter 0.
632 * Events valid only on counter 0, 1.
642 mask = ~0; /* Any row index is ok. */
645 return (mask & (1 << ri));
649 iap_event_sb_sbx_ib_ibx_ok_on_counter(uint8_t evsel, int ri)
654 /* Events valid only on counter 0. */
658 /* Events valid only on counter 1. */
662 /* Events valid only on counter 2. */
668 /* Events valid only on counter 3. */
674 mask = ~0; /* Any row index is ok. */
677 return (mask & (1 << ri));
681 iap_event_ok_on_counter(uint8_t evsel, int ri)
687 * Events valid only on counter 0.
699 * Events valid only on counter 1.
708 mask = ~0; /* Any row index is ok. */
711 return (mask & (1 << ri));
715 iap_allocate_pmc(int cpu, int ri, struct pmc *pm,
716 const struct pmc_op_pmcallocate *a)
721 const struct pmc_md_iap_op_pmcallocate *iap;
723 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
724 ("[core,%d] illegal CPU %d", __LINE__, cpu));
725 KASSERT(ri >= 0 && ri < core_iap_npmc,
726 ("[core,%d] illegal row-index value %d", __LINE__, ri));
728 /* check requested capabilities */
730 if ((IAP_PMC_CAPS & caps) != caps)
732 map = 0; /* XXX: silent GCC warning */
733 iap = &a->pm_md.pm_iap;
734 ev = IAP_EVSEL_GET(iap->pm_iap_config);
736 switch (core_cputype) {
737 case PMC_CPU_INTEL_COREI7:
738 case PMC_CPU_INTEL_NEHALEM_EX:
739 if (iap_event_corei7_ok_on_counter(ev, ri) == 0)
742 case PMC_CPU_INTEL_SKYLAKE:
743 case PMC_CPU_INTEL_SKYLAKE_XEON:
744 case PMC_CPU_INTEL_BROADWELL:
745 case PMC_CPU_INTEL_BROADWELL_XEON:
746 case PMC_CPU_INTEL_SANDYBRIDGE:
747 case PMC_CPU_INTEL_SANDYBRIDGE_XEON:
748 case PMC_CPU_INTEL_IVYBRIDGE:
749 case PMC_CPU_INTEL_IVYBRIDGE_XEON:
750 case PMC_CPU_INTEL_HASWELL:
751 case PMC_CPU_INTEL_HASWELL_XEON:
752 if (iap_event_sb_sbx_ib_ibx_ok_on_counter(ev, ri) == 0)
755 case PMC_CPU_INTEL_WESTMERE:
756 case PMC_CPU_INTEL_WESTMERE_EX:
757 if (iap_event_westmere_ok_on_counter(ev, ri) == 0)
761 if (iap_event_ok_on_counter(ev, ri) == 0)
765 pm->pm_md.pm_iap.pm_iap_evsel = iap->pm_iap_config;
770 iap_config_pmc(int cpu, int ri, struct pmc *pm)
772 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
773 ("[core,%d] illegal CPU %d", __LINE__, cpu));
775 KASSERT(ri >= 0 && ri < core_iap_npmc,
776 ("[core,%d] illegal row-index %d", __LINE__, ri));
778 PMCDBG3(MDP,CFG,1, "iap-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
780 KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__,
783 core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc = pm;
789 iap_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
793 char iap_name[PMC_NAME_MAX];
795 phw = &core_pcpu[cpu]->pc_corepmcs[ri];
797 (void) snprintf(iap_name, sizeof(iap_name), "IAP-%d", ri);
798 if ((error = copystr(iap_name, pi->pm_name, PMC_NAME_MAX,
802 pi->pm_class = PMC_CLASS_IAP;
804 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
805 pi->pm_enabled = TRUE;
806 *ppmc = phw->phw_pmc;
808 pi->pm_enabled = FALSE;
816 iap_get_config(int cpu, int ri, struct pmc **ppm)
818 *ppm = core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc;
824 iap_get_msr(int ri, uint32_t *msr)
826 KASSERT(ri >= 0 && ri < core_iap_npmc,
827 ("[iap,%d] ri %d out of range", __LINE__, ri));
835 iap_read_pmc(int cpu, int ri, pmc_value_t *v)
840 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
841 ("[core,%d] illegal cpu value %d", __LINE__, cpu));
842 KASSERT(ri >= 0 && ri < core_iap_npmc,
843 ("[core,%d] illegal row-index %d", __LINE__, ri));
845 pm = core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc;
848 ("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu,
852 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
853 *v = iap_perfctr_value_to_reload_count(tmp);
855 *v = tmp & ((1ULL << core_iap_width) - 1);
857 PMCDBG4(MDP,REA,1, "iap-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
864 iap_release_pmc(int cpu, int ri, struct pmc *pm)
868 PMCDBG3(MDP,REL,1, "iap-release cpu=%d ri=%d pm=%p", cpu, ri,
871 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
872 ("[core,%d] illegal CPU value %d", __LINE__, cpu));
873 KASSERT(ri >= 0 && ri < core_iap_npmc,
874 ("[core,%d] illegal row-index %d", __LINE__, ri));
876 KASSERT(core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc
877 == NULL, ("[core,%d] PHW pmc non-NULL", __LINE__));
883 iap_start_pmc(int cpu, int ri)
889 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
890 ("[core,%d] illegal CPU value %d", __LINE__, cpu));
891 KASSERT(ri >= 0 && ri < core_iap_npmc,
892 ("[core,%d] illegal row-index %d", __LINE__, ri));
895 pm = cc->pc_corepmcs[ri].phw_pmc;
898 ("[core,%d] starting cpu%d,ri%d with no pmc configured",
901 PMCDBG2(MDP,STA,1, "iap-start cpu=%d ri=%d", cpu, ri);
903 evsel = pm->pm_md.pm_iap.pm_iap_evsel;
905 PMCDBG4(MDP,STA,2, "iap-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x",
906 cpu, ri, IAP_EVSEL0 + ri, evsel);
908 /* Event specific configuration. */
910 switch (IAP_EVSEL_GET(evsel)) {
912 wrmsr(IA_OFFCORE_RSP0, pm->pm_md.pm_iap.pm_iap_rsp);
915 wrmsr(IA_OFFCORE_RSP1, pm->pm_md.pm_iap.pm_iap_rsp);
921 wrmsr(IAP_EVSEL0 + ri, evsel | IAP_EN);
923 if (core_cputype == PMC_CPU_INTEL_CORE)
928 cc->pc_globalctrl |= (1ULL << ri);
929 wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);
930 } while (cc->pc_resync != 0);
936 iap_stop_pmc(int cpu, int ri)
942 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
943 ("[core,%d] illegal cpu value %d", __LINE__, cpu));
944 KASSERT(ri >= 0 && ri < core_iap_npmc,
945 ("[core,%d] illegal row index %d", __LINE__, ri));
948 pm = cc->pc_corepmcs[ri].phw_pmc;
951 ("[core,%d] cpu%d ri%d no configured PMC to stop", __LINE__,
954 PMCDBG2(MDP,STO,1, "iap-stop cpu=%d ri=%d", cpu, ri);
956 msr = rdmsr(IAP_EVSEL0 + ri) & ~IAP_EVSEL_MASK;
957 wrmsr(IAP_EVSEL0 + ri, msr); /* stop hw */
959 if (core_cputype == PMC_CPU_INTEL_CORE)
965 cc->pc_globalctrl &= ~(1ULL << ri);
966 msr = rdmsr(IA_GLOBAL_CTRL) & ~IA_GLOBAL_CTRL_MASK;
967 wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);
968 } while (cc->pc_resync != 0);
974 iap_write_pmc(int cpu, int ri, pmc_value_t v)
979 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
980 ("[core,%d] illegal cpu value %d", __LINE__, cpu));
981 KASSERT(ri >= 0 && ri < core_iap_npmc,
982 ("[core,%d] illegal row index %d", __LINE__, ri));
985 pm = cc->pc_corepmcs[ri].phw_pmc;
988 ("[core,%d] cpu%d ri%d no configured PMC to stop", __LINE__,
991 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
992 v = iap_reload_count_to_perfctr_value(v);
994 v &= (1ULL << core_iap_width) - 1;
996 PMCDBG4(MDP,WRI,1, "iap-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri,
1000 * Write the new value to the counter (or it's alias). The
1001 * counter will be in a stopped state when the pcd_write()
1002 * entry point is called.
1004 wrmsr(core_iap_wroffset + IAP_PMC0 + ri, v);
1010 iap_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth,
1013 struct pmc_classdep *pcd;
1015 KASSERT(md != NULL, ("[iap,%d] md is NULL", __LINE__));
1017 PMCDBG0(MDP,INI,1, "iap-initialize");
1019 /* Remember the set of architectural events supported. */
1020 core_architectural_events = ~flags;
1022 pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP];
1024 pcd->pcd_caps = IAP_PMC_CAPS;
1025 pcd->pcd_class = PMC_CLASS_IAP;
1026 pcd->pcd_num = npmc;
1027 pcd->pcd_ri = md->pmd_npmc;
1028 pcd->pcd_width = pmcwidth;
1030 pcd->pcd_allocate_pmc = iap_allocate_pmc;
1031 pcd->pcd_config_pmc = iap_config_pmc;
1032 pcd->pcd_describe = iap_describe;
1033 pcd->pcd_get_config = iap_get_config;
1034 pcd->pcd_get_msr = iap_get_msr;
1035 pcd->pcd_pcpu_fini = core_pcpu_fini;
1036 pcd->pcd_pcpu_init = core_pcpu_init;
1037 pcd->pcd_read_pmc = iap_read_pmc;
1038 pcd->pcd_release_pmc = iap_release_pmc;
1039 pcd->pcd_start_pmc = iap_start_pmc;
1040 pcd->pcd_stop_pmc = iap_stop_pmc;
1041 pcd->pcd_write_pmc = iap_write_pmc;
1043 md->pmd_npmc += npmc;
1047 core_intr(int cpu, struct trapframe *tf)
1051 struct core_cpu *cc;
1052 int error, found_interrupt, ri;
1055 PMCDBG3(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf,
1056 TRAPF_USERMODE(tf));
1058 found_interrupt = 0;
1059 cc = core_pcpu[cpu];
1061 for (ri = 0; ri < core_iap_npmc; ri++) {
1063 if ((pm = cc->pc_corepmcs[ri].phw_pmc) == NULL ||
1064 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1067 if (!iap_pmc_has_overflowed(ri))
1070 found_interrupt = 1;
1072 if (pm->pm_state != PMC_STATE_RUNNING)
1075 error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
1076 TRAPF_USERMODE(tf));
1078 v = pm->pm_sc.pm_reloadcount;
1079 v = iap_reload_count_to_perfctr_value(v);
1082 * Stop the counter, reload it but only restart it if
1083 * the PMC is not stalled.
1085 msr = rdmsr(IAP_EVSEL0 + ri) & ~IAP_EVSEL_MASK;
1086 wrmsr(IAP_EVSEL0 + ri, msr);
1087 wrmsr(core_iap_wroffset + IAP_PMC0 + ri, v);
1092 wrmsr(IAP_EVSEL0 + ri, msr | (pm->pm_md.pm_iap.pm_iap_evsel |
1096 if (found_interrupt)
1097 lapic_reenable_pmc();
1099 if (found_interrupt)
1100 counter_u64_add(pmc_stats.pm_intr_processed, 1);
1102 counter_u64_add(pmc_stats.pm_intr_ignored, 1);
1104 return (found_interrupt);
1108 core2_intr(int cpu, struct trapframe *tf)
1110 int error, found_interrupt, n;
1111 uint64_t flag, intrstatus, intrenable, msr;
1113 struct core_cpu *cc;
1116 PMCDBG3(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf,
1117 TRAPF_USERMODE(tf));
1120 * The IA_GLOBAL_STATUS (MSR 0x38E) register indicates which
1121 * PMCs have a pending PMI interrupt. We take a 'snapshot' of
1122 * the current set of interrupting PMCs and process these
1123 * after stopping them.
1125 intrstatus = rdmsr(IA_GLOBAL_STATUS);
1126 intrenable = intrstatus & core_pmcmask;
1128 PMCDBG2(MDP,INT, 1, "cpu=%d intrstatus=%jx", cpu,
1129 (uintmax_t) intrstatus);
1131 found_interrupt = 0;
1132 cc = core_pcpu[cpu];
1134 KASSERT(cc != NULL, ("[core,%d] null pcpu", __LINE__));
1136 cc->pc_globalctrl &= ~intrenable;
1137 cc->pc_resync = 1; /* MSRs now potentially out of sync. */
1140 * Stop PMCs and clear overflow status bits.
1142 msr = rdmsr(IA_GLOBAL_CTRL) & ~IA_GLOBAL_CTRL_MASK;
1143 wrmsr(IA_GLOBAL_CTRL, msr);
1144 wrmsr(IA_GLOBAL_OVF_CTRL, intrenable |
1145 IA_GLOBAL_STATUS_FLAG_OVFBUF |
1146 IA_GLOBAL_STATUS_FLAG_CONDCHG);
1149 * Look for interrupts from fixed function PMCs.
1151 for (n = 0, flag = (1ULL << IAF_OFFSET); n < core_iaf_npmc;
1154 if ((intrstatus & flag) == 0)
1157 found_interrupt = 1;
1159 pm = cc->pc_corepmcs[n + core_iaf_ri].phw_pmc;
1160 if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING ||
1161 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1164 error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
1165 TRAPF_USERMODE(tf));
1168 intrenable &= ~flag;
1170 v = iaf_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount);
1172 /* Reload sampling count. */
1173 wrmsr(IAF_CTR0 + n, v);
1175 PMCDBG4(MDP,INT, 1, "iaf-intr cpu=%d error=%d v=%jx(%jx)", cpu,
1176 error, (uintmax_t) v, (uintmax_t) rdpmc(IAF_RI_TO_MSR(n)));
1180 * Process interrupts from the programmable counters.
1182 for (n = 0, flag = 1; n < core_iap_npmc; n++, flag <<= 1) {
1183 if ((intrstatus & flag) == 0)
1186 found_interrupt = 1;
1188 pm = cc->pc_corepmcs[n].phw_pmc;
1189 if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING ||
1190 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1193 error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
1194 TRAPF_USERMODE(tf));
1196 intrenable &= ~flag;
1198 v = iap_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount);
1200 PMCDBG3(MDP,INT, 1, "iap-intr cpu=%d error=%d v=%jx", cpu, error,
1203 /* Reload sampling count. */
1204 wrmsr(core_iap_wroffset + IAP_PMC0 + n, v);
1208 * Reenable all non-stalled PMCs.
1210 PMCDBG2(MDP,INT, 1, "cpu=%d intrenable=%jx", cpu,
1211 (uintmax_t) intrenable);
1213 cc->pc_globalctrl |= intrenable;
1215 wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl & IA_GLOBAL_CTRL_MASK);
1217 PMCDBG5(MDP,INT, 1, "cpu=%d fixedctrl=%jx globalctrl=%jx status=%jx "
1218 "ovf=%jx", cpu, (uintmax_t) rdmsr(IAF_CTRL),
1219 (uintmax_t) rdmsr(IA_GLOBAL_CTRL),
1220 (uintmax_t) rdmsr(IA_GLOBAL_STATUS),
1221 (uintmax_t) rdmsr(IA_GLOBAL_OVF_CTRL));
1223 if (found_interrupt)
1224 lapic_reenable_pmc();
1226 if (found_interrupt)
1227 counter_u64_add(pmc_stats.pm_intr_processed, 1);
1229 counter_u64_add(pmc_stats.pm_intr_ignored, 1);
1231 return (found_interrupt);
1235 pmc_core_initialize(struct pmc_mdep *md, int maxcpu, int version_override)
1237 int cpuid[CORE_CPUID_REQUEST_SIZE];
1238 int ipa_version, flags, nflags;
1240 do_cpuid(CORE_CPUID_REQUEST, cpuid);
1242 ipa_version = (version_override > 0) ? version_override :
1243 cpuid[CORE_CPUID_EAX] & 0xFF;
1244 core_cputype = md->pmd_cputype;
1246 PMCDBG3(MDP,INI,1,"core-init cputype=%d ncpu=%d ipa-version=%d",
1247 core_cputype, maxcpu, ipa_version);
1249 if (ipa_version < 1 || ipa_version > 4 ||
1250 (core_cputype != PMC_CPU_INTEL_CORE && ipa_version == 1)) {
1251 /* Unknown PMC architecture. */
1252 printf("hwpc_core: unknown PMC architecture: %d\n",
1254 return (EPROGMISMATCH);
1257 core_iap_wroffset = 0;
1258 if (cpu_feature2 & CPUID2_PDCM) {
1259 if (rdmsr(IA32_PERF_CAPABILITIES) & PERFCAP_FW_WRITE) {
1260 PMCDBG0(MDP, INI, 1,
1261 "core-init full-width write supported");
1262 core_iap_wroffset = IAP_A_PMC0 - IAP_PMC0;
1264 PMCDBG0(MDP, INI, 1,
1265 "core-init full-width write NOT supported");
1267 PMCDBG0(MDP, INI, 1, "core-init pdcm not supported");
1272 * Initialize programmable counters.
1274 core_iap_npmc = (cpuid[CORE_CPUID_EAX] >> 8) & 0xFF;
1275 core_iap_width = (cpuid[CORE_CPUID_EAX] >> 16) & 0xFF;
1277 core_pmcmask |= ((1ULL << core_iap_npmc) - 1);
1279 nflags = (cpuid[CORE_CPUID_EAX] >> 24) & 0xFF;
1280 flags = cpuid[CORE_CPUID_EBX] & ((1 << nflags) - 1);
1282 iap_initialize(md, maxcpu, core_iap_npmc, core_iap_width, flags);
1285 * Initialize fixed function counters, if present.
1287 if (core_cputype != PMC_CPU_INTEL_CORE) {
1288 core_iaf_ri = core_iap_npmc;
1289 core_iaf_npmc = cpuid[CORE_CPUID_EDX] & 0x1F;
1290 core_iaf_width = (cpuid[CORE_CPUID_EDX] >> 5) & 0xFF;
1292 iaf_initialize(md, maxcpu, core_iaf_npmc, core_iaf_width);
1293 core_pmcmask |= ((1ULL << core_iaf_npmc) - 1) << IAF_OFFSET;
1296 PMCDBG2(MDP,INI,1,"core-init pmcmask=0x%jx iafri=%d", core_pmcmask,
1299 core_pcpu = malloc(sizeof(*core_pcpu) * maxcpu, M_PMC,
1303 * Choose the appropriate interrupt handler.
1305 if (ipa_version == 1)
1306 md->pmd_intr = core_intr;
1308 md->pmd_intr = core2_intr;
1310 md->pmd_pcpu_fini = NULL;
1311 md->pmd_pcpu_init = NULL;
1317 pmc_core_finalize(struct pmc_mdep *md)
1319 PMCDBG0(MDP,INI,1, "core-finalize");
1321 free(core_pcpu, M_PMC);