sys/dev/hwpmc/hwpmc_ppro.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2003-2005,2008 Joseph Koshy
   5  * Copyright (c) 2007 The FreeBSD Foundation
   6  * All rights reserved.
   7  *
   8  * Portions of this software were developed by A. Joseph Koshy under
   9  * sponsorship from the FreeBSD Foundation and Google, Inc.
  10  *
  11  * Redistribution and use in source and binary forms, with or without
  12  * modification, are permitted provided that the following conditions
  13  * are met:
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in the
  18  *    documentation and/or other materials provided with the distribution.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  30  * SUCH DAMAGE.
  31  */
  32
  33 #include <sys/cdefs.h>
  34 __FBSDID("$FreeBSD$");
  35
  36 #include <sys/param.h>
  37 #include <sys/bus.h>
  38 #include <sys/lock.h>
  39 #include <sys/mutex.h>
  40 #include <sys/pmc.h>
  41 #include <sys/pmckern.h>
  42 #include <sys/smp.h>
  43 #include <sys/systm.h>
  44
  45 #include <machine/intr_machdep.h>
  46 #include <x86/apicvar.h>
  47 #include <machine/cpu.h>
  48 #include <machine/cpufunc.h>
  49 #include <machine/cputypes.h>
  50 #include <machine/md_var.h>
  51 #include <machine/pmc_mdep.h>
  52 #include <machine/specialreg.h>
  53
  54 /*
  55  * PENTIUM PRO SUPPORT
  56  *
  57  * Quirks:
  58  *
  59  * - Both PMCs are enabled by a single bit P6_EVSEL_EN in performance
  60  *   counter '0'.  This bit needs to be '1' if any of the two
  61  *   performance counters are in use.  Perf counters can also be
  62  *   switched off by writing zeros to their EVSEL register.
  63  *
  64  * - While the width of these counters is 40 bits, we do not appear to
  65  *   have a way of writing 40 bits to the counter MSRs.  A WRMSR
  66  *   instruction will sign extend bit 31 of the value being written to
  67  *   the perf counter -- a value of 0x80000000 written to an perf
  68  *   counter register will be sign extended to 0xFF80000000.
  69  *
  70  *   This quirk primarily affects thread-mode PMCs in counting mode, as
  71  *   these PMCs read and write PMC registers at every context switch.
  72  */
  73
  74 struct p6pmc_descr {
  75         struct pmc_descr pm_descr; /* common information */
  76         uint32_t        pm_pmc_msr;
  77         uint32_t        pm_evsel_msr;
  78 };
  79
  80 static struct p6pmc_descr p6_pmcdesc[P6_NPMCS] = {
  81
  82 #define P6_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | PMC_CAP_SYSTEM | \
  83     PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE |    \
  84     PMC_CAP_INVERT | PMC_CAP_QUALIFIER)
  85
  86         /* PMC 0 */
  87         {
  88                 .pm_descr =
  89                 {
  90                         .pd_name  ="P6-0",
  91                         .pd_class = PMC_CLASS_P6,
  92                         .pd_caps  = P6_PMC_CAPS,
  93                         .pd_width = 40
  94                 },
  95                 .pm_pmc_msr   = P6_MSR_PERFCTR0,
  96                 .pm_evsel_msr = P6_MSR_EVSEL0
  97         },
  98
  99         /* PMC 1 */
 100         {
 101                 .pm_descr =
 102                 {
 103                         .pd_name  ="P6-1",
 104                         .pd_class = PMC_CLASS_P6,
 105                         .pd_caps  = P6_PMC_CAPS,
 106                         .pd_width = 40
 107                 },
 108                 .pm_pmc_msr   = P6_MSR_PERFCTR1,
 109                 .pm_evsel_msr = P6_MSR_EVSEL1
 110         }
 111 };
 112
 113 static enum pmc_cputype p6_cputype;
 114
 115 /*
 116  * P6 Event descriptor
 117  *
 118  * The 'pm_flags' field has the following structure:
 119  * - The upper 4 bits are used to track which counter an event is valid on.
 120  * - The lower bits form a bitmask of flags indicating support for the event
 121  *   on a given CPU.
 122  */
 123
 124 struct p6_event_descr {
 125         const enum pmc_event pm_event;
 126         uint32_t             pm_evsel;
 127         uint32_t             pm_flags;
 128         uint32_t             pm_unitmask;
 129 };
 130
 131 #define P6F_CTR(C)      (1 << (28 + (C)))
 132 #define P6F_CTR0        P6F_CTR(0)
 133 #define P6F_CTR1        P6F_CTR(1)
 134 #define P6F(CPU)        (1 << ((CPU) - PMC_CPU_INTEL_P6))
 135 #define _P6F(C)         P6F(PMC_CPU_INTEL_##C)
 136 #define P6F_P6          _P6F(P6)
 137 #define P6F_CL          _P6F(CL)
 138 #define P6F_PII         _P6F(PII)
 139 #define P6F_PIII        _P6F(PIII)
 140 #define P6F_PM          _P6F(PM)
 141 #define P6F_ALL_CPUS    (P6F_P6 | P6F_PII | P6F_CL | P6F_PIII | P6F_PM)
 142 #define P6F_ALL_CTRS    (P6F_CTR0 | P6F_CTR1)
 143 #define P6F_ALL         (P6F_ALL_CPUS | P6F_ALL_CTRS)
 144
 145 #define P6_EVENT_VALID_FOR_CPU(P,CPU)   ((P)->pm_flags & P6F(CPU))
 146 #define P6_EVENT_VALID_FOR_CTR(P,CTR)   ((P)->pm_flags & P6F_CTR(CTR))
 147
 148 static const struct p6_event_descr p6_events[] = {
 149
 150 #define P6_EVDESCR(NAME, EVSEL, FLAGS, UMASK)   \
 151         {                                       \
 152                 .pm_event = PMC_EV_P6_##NAME,   \
 153                 .pm_evsel = (EVSEL),            \
 154                 .pm_flags = (FLAGS),            \
 155                 .pm_unitmask = (UMASK)          \
 156         }
 157
 158 P6_EVDESCR(DATA_MEM_REFS,               0x43, P6F_ALL, 0x00),
 159 P6_EVDESCR(DCU_LINES_IN,                0x45, P6F_ALL, 0x00),
 160 P6_EVDESCR(DCU_M_LINES_IN,              0x46, P6F_ALL, 0x00),
 161 P6_EVDESCR(DCU_M_LINES_OUT,             0x47, P6F_ALL, 0x00),
 162 P6_EVDESCR(DCU_MISS_OUTSTANDING,        0x47, P6F_ALL, 0x00),
 163 P6_EVDESCR(IFU_FETCH,                   0x80, P6F_ALL, 0x00),
 164 P6_EVDESCR(IFU_FETCH_MISS,              0x81, P6F_ALL, 0x00),
 165 P6_EVDESCR(ITLB_MISS,                   0x85, P6F_ALL, 0x00),
 166 P6_EVDESCR(IFU_MEM_STALL,               0x86, P6F_ALL, 0x00),
 167 P6_EVDESCR(ILD_STALL,                   0x87, P6F_ALL, 0x00),
 168 P6_EVDESCR(L2_IFETCH,                   0x28, P6F_ALL, 0x0F),
 169 P6_EVDESCR(L2_LD,                       0x29, P6F_ALL, 0x0F),
 170 P6_EVDESCR(L2_ST,                       0x2A, P6F_ALL, 0x0F),
 171 P6_EVDESCR(L2_LINES_IN,                 0x24, P6F_ALL, 0x0F),
 172 P6_EVDESCR(L2_LINES_OUT,                0x26, P6F_ALL, 0x0F),
 173 P6_EVDESCR(L2_M_LINES_INM,              0x25, P6F_ALL, 0x00),
 174 P6_EVDESCR(L2_M_LINES_OUTM,             0x27, P6F_ALL, 0x0F),
 175 P6_EVDESCR(L2_RQSTS,                    0x2E, P6F_ALL, 0x0F),
 176 P6_EVDESCR(L2_ADS,                      0x21, P6F_ALL, 0x00),
 177 P6_EVDESCR(L2_DBUS_BUSY,                0x22, P6F_ALL, 0x00),
 178 P6_EVDESCR(L2_DBUS_BUSY_RD,             0x23, P6F_ALL, 0x00),
 179 P6_EVDESCR(BUS_DRDY_CLOCKS,             0x62, P6F_ALL, 0x20),
 180 P6_EVDESCR(BUS_LOCK_CLOCKS,             0x63, P6F_ALL, 0x20),
 181 P6_EVDESCR(BUS_REQ_OUTSTANDING,         0x60, P6F_ALL, 0x00),
 182 P6_EVDESCR(BUS_TRAN_BRD,                0x65, P6F_ALL, 0x20),
 183 P6_EVDESCR(BUS_TRAN_RFO,                0x66, P6F_ALL, 0x20),
 184 P6_EVDESCR(BUS_TRANS_WB,                0x67, P6F_ALL, 0x20),
 185 P6_EVDESCR(BUS_TRAN_IFETCH,             0x68, P6F_ALL, 0x20),
 186 P6_EVDESCR(BUS_TRAN_INVAL,              0x69, P6F_ALL, 0x20),
 187 P6_EVDESCR(BUS_TRAN_PWR,                0x6A, P6F_ALL, 0x20),
 188 P6_EVDESCR(BUS_TRANS_P,                 0x6B, P6F_ALL, 0x20),
 189 P6_EVDESCR(BUS_TRANS_IO,                0x6C, P6F_ALL, 0x20),
 190 P6_EVDESCR(BUS_TRAN_DEF,                0x6D, P6F_ALL, 0x20),
 191 P6_EVDESCR(BUS_TRAN_BURST,              0x6E, P6F_ALL, 0x20),
 192 P6_EVDESCR(BUS_TRAN_ANY,                0x70, P6F_ALL, 0x20),
 193 P6_EVDESCR(BUS_TRAN_MEM,                0x6F, P6F_ALL, 0x20),
 194 P6_EVDESCR(BUS_DATA_RCV,                0x64, P6F_ALL, 0x00),
 195 P6_EVDESCR(BUS_BNR_DRV,                 0x61, P6F_ALL, 0x00),
 196 P6_EVDESCR(BUS_HIT_DRV,                 0x7A, P6F_ALL, 0x00),
 197 P6_EVDESCR(BUS_HITM_DRV,                0x7B, P6F_ALL, 0x00),
 198 P6_EVDESCR(BUS_SNOOP_STALL,             0x7E, P6F_ALL, 0x00),
 199 P6_EVDESCR(FLOPS,                       0xC1, P6F_ALL_CPUS | P6F_CTR0, 0x00),
 200 P6_EVDESCR(FP_COMPS_OPS_EXE,            0x10, P6F_ALL_CPUS | P6F_CTR0, 0x00),
 201 P6_EVDESCR(FP_ASSIST,                   0x11, P6F_ALL_CPUS | P6F_CTR1, 0x00),
 202 P6_EVDESCR(MUL,                         0x12, P6F_ALL_CPUS | P6F_CTR1, 0x00),
 203 P6_EVDESCR(DIV,                         0x13, P6F_ALL_CPUS | P6F_CTR1, 0x00),
 204 P6_EVDESCR(CYCLES_DIV_BUSY,             0x14, P6F_ALL_CPUS | P6F_CTR0, 0x00),
 205 P6_EVDESCR(LD_BLOCKS,                   0x03, P6F_ALL, 0x00),
 206 P6_EVDESCR(SB_DRAINS,                   0x04, P6F_ALL, 0x00),
 207 P6_EVDESCR(MISALIGN_MEM_REF,            0x05, P6F_ALL, 0x00),
 208 P6_EVDESCR(EMON_KNI_PREF_DISPATCHED,    0x07, P6F_PIII | P6F_ALL_CTRS, 0x03),
 209 P6_EVDESCR(EMON_KNI_PREF_MISS,          0x4B, P6F_PIII | P6F_ALL_CTRS, 0x03),
 210 P6_EVDESCR(INST_RETIRED,                0xC0, P6F_ALL, 0x00),
 211 P6_EVDESCR(UOPS_RETIRED,                0xC2, P6F_ALL, 0x00),
 212 P6_EVDESCR(INST_DECODED,                0xD0, P6F_ALL, 0x00),
 213 P6_EVDESCR(EMON_KNI_INST_RETIRED,       0xD8, P6F_PIII | P6F_ALL_CTRS, 0x01),
 214 P6_EVDESCR(EMON_KNI_COMP_INST_RET,      0xD9, P6F_PIII | P6F_ALL_CTRS, 0x01),
 215 P6_EVDESCR(HW_INT_RX,                   0xC8, P6F_ALL, 0x00),
 216 P6_EVDESCR(CYCLES_INT_MASKED,           0xC6, P6F_ALL, 0x00),
 217 P6_EVDESCR(CYCLES_INT_PENDING_AND_MASKED, 0xC7, P6F_ALL, 0x00),
 218 P6_EVDESCR(BR_INST_RETIRED,             0xC4, P6F_ALL, 0x00),
 219 P6_EVDESCR(BR_MISS_PRED_RETIRED,        0xC5, P6F_ALL, 0x00),
 220 P6_EVDESCR(BR_TAKEN_RETIRED,            0xC9, P6F_ALL, 0x00),
 221 P6_EVDESCR(BR_MISS_PRED_TAKEN_RET,      0xCA, P6F_ALL, 0x00),
 222 P6_EVDESCR(BR_INST_DECODED,             0xE0, P6F_ALL, 0x00),
 223 P6_EVDESCR(BTB_MISSES,                  0xE2, P6F_ALL, 0x00),
 224 P6_EVDESCR(BR_BOGUS,                    0xE4, P6F_ALL, 0x00),
 225 P6_EVDESCR(BACLEARS,                    0xE6, P6F_ALL, 0x00),
 226 P6_EVDESCR(RESOURCE_STALLS,             0xA2, P6F_ALL, 0x00),
 227 P6_EVDESCR(PARTIAL_RAT_STALLS,          0xD2, P6F_ALL, 0x00),
 228 P6_EVDESCR(SEGMENT_REG_LOADS,           0x06, P6F_ALL, 0x00),
 229 P6_EVDESCR(CPU_CLK_UNHALTED,            0x79, P6F_ALL, 0x00),
 230 P6_EVDESCR(MMX_INSTR_EXEC,              0xB0,
 231                         P6F_ALL_CTRS | P6F_CL | P6F_PII, 0x00),
 232 P6_EVDESCR(MMX_SAT_INSTR_EXEC,          0xB1,
 233                         P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x00),
 234 P6_EVDESCR(MMX_UOPS_EXEC,               0xB2,
 235                         P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x0F),
 236 P6_EVDESCR(MMX_INSTR_TYPE_EXEC,         0xB3,
 237                         P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x3F),
 238 P6_EVDESCR(FP_MMX_TRANS,                0xCC,
 239                         P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x01),
 240 P6_EVDESCR(MMX_ASSIST,                  0xCD,
 241                         P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x00),
 242 P6_EVDESCR(MMX_INSTR_RET,               0xCE, P6F_ALL_CTRS | P6F_PII, 0x00),
 243 P6_EVDESCR(SEG_RENAME_STALLS,           0xD4,
 244                         P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x0F),
 245 P6_EVDESCR(SEG_REG_RENAMES,             0xD5,
 246                         P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x0F),
 247 P6_EVDESCR(RET_SEG_RENAMES,             0xD6,
 248                         P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x00),
 249 P6_EVDESCR(EMON_EST_TRANS,              0x58, P6F_ALL_CTRS | P6F_PM, 0x02),
 250 P6_EVDESCR(EMON_THERMAL_TRIP,           0x59, P6F_ALL_CTRS | P6F_PM, 0x00),
 251 P6_EVDESCR(BR_INST_EXEC,                0x88, P6F_ALL_CTRS | P6F_PM, 0x00),
 252 P6_EVDESCR(BR_MISSP_EXEC,               0x89, P6F_ALL_CTRS | P6F_PM, 0x00),
 253 P6_EVDESCR(BR_BAC_MISSP_EXEC,           0x8A, P6F_ALL_CTRS | P6F_PM, 0x00),
 254 P6_EVDESCR(BR_CND_EXEC,                 0x8B, P6F_ALL_CTRS | P6F_PM, 0x00),
 255 P6_EVDESCR(BR_CND_MISSP_EXEC,           0x8C, P6F_ALL_CTRS | P6F_PM, 0x00),
 256 P6_EVDESCR(BR_IND_EXEC,                 0x8D, P6F_ALL_CTRS | P6F_PM, 0x00),
 257 P6_EVDESCR(BR_IND_MISSP_EXEC,           0x8E, P6F_ALL_CTRS | P6F_PM, 0x00),
 258 P6_EVDESCR(BR_RET_EXEC,                 0x8F, P6F_ALL_CTRS | P6F_PM, 0x00),
 259 P6_EVDESCR(BR_RET_MISSP_EXEC,           0x90, P6F_ALL_CTRS | P6F_PM, 0x00),
 260 P6_EVDESCR(BR_RET_BAC_MISSP_EXEC,       0x91, P6F_ALL_CTRS | P6F_PM, 0x00),
 261 P6_EVDESCR(BR_CALL_EXEC,                0x92, P6F_ALL_CTRS | P6F_PM, 0x00),
 262 P6_EVDESCR(BR_CALL_MISSP_EXEC,          0x93, P6F_ALL_CTRS | P6F_PM, 0x00),
 263 P6_EVDESCR(BR_IND_CALL_EXEC,            0x94, P6F_ALL_CTRS | P6F_PM, 0x00),
 264 P6_EVDESCR(EMON_SIMD_INSTR_RETIRED,     0xCE, P6F_ALL_CTRS | P6F_PM, 0x00),
 265 P6_EVDESCR(EMON_SYNCH_UOPS,             0xD3, P6F_ALL_CTRS | P6F_PM, 0x00),
 266 P6_EVDESCR(EMON_ESP_UOPS,               0xD7, P6F_ALL_CTRS | P6F_PM, 0x00),
 267 P6_EVDESCR(EMON_FUSED_UOPS_RET,         0xDA, P6F_ALL_CTRS | P6F_PM, 0x03),
 268 P6_EVDESCR(EMON_UNFUSION,               0xDB, P6F_ALL_CTRS | P6F_PM, 0x00),
 269 P6_EVDESCR(EMON_PREF_RQSTS_UP,          0xF0, P6F_ALL_CTRS | P6F_PM, 0x00),
 270 P6_EVDESCR(EMON_PREF_RQSTS_DN,          0xD8, P6F_ALL_CTRS | P6F_PM, 0x00),
 271 P6_EVDESCR(EMON_SSE_SSE2_INST_RETIRED,  0xD8, P6F_ALL_CTRS | P6F_PM, 0x03),
 272 P6_EVDESCR(EMON_SSE_SSE2_COMP_INST_RETIRED, 0xD9, P6F_ALL_CTRS | P6F_PM, 0x03)
 273
 274 #undef  P6_EVDESCR
 275 };
 276
 277 #define P6_NEVENTS      (PMC_EV_P6_LAST - PMC_EV_P6_FIRST + 1)
 278
 279 static const struct p6_event_descr *
 280 p6_find_event(enum pmc_event ev)
 281 {
 282         int n;
 283
 284         for (n = 0; n < P6_NEVENTS; n++)
 285                 if (p6_events[n].pm_event == ev)
 286                         break;
 287         if (n == P6_NEVENTS)
 288                 return NULL;
 289         return &p6_events[n];
 290 }
 291
 292 /*
 293  * Per-CPU data structure for P6 class CPUs
 294  *
 295  * [common stuff]
 296  * [flags for maintaining PMC start/stop state]
 297  * [3 struct pmc_hw pointers]
 298  * [3 struct pmc_hw structures]
 299  */
 300
 301 struct p6_cpu {
 302         struct pmc_hw   pc_p6pmcs[P6_NPMCS];
 303         uint32_t        pc_state;
 304 };
 305
 306 static struct p6_cpu **p6_pcpu;
 307
 308 /*
 309  * If CTR1 is active, we need to keep the 'EN' bit if CTR0 set,
 310  * with the rest of CTR0 being zero'ed out.
 311  */
 312 #define P6_SYNC_CTR_STATE(PC) do {                              \
 313                 uint32_t _config, _enable;                      \
 314                 _enable = 0;                                    \
 315                 if ((PC)->pc_state & 0x02)                      \
 316                         _enable |= P6_EVSEL_EN;                 \
 317                 if ((PC)->pc_state & 0x01)                      \
 318                         _config = rdmsr(P6_MSR_EVSEL0) |        \
 319                             P6_EVSEL_EN;                        \
 320                 else                                            \
 321                         _config = 0;                            \
 322                 wrmsr(P6_MSR_EVSEL0, _config | _enable);        \
 323         } while (0)
 324
 325 #define P6_MARK_STARTED(PC,RI) do {                             \
 326                 (PC)->pc_state |= (1 << ((RI)-1));              \
 327         } while (0)
 328
 329 #define P6_MARK_STOPPED(PC,RI) do {                             \
 330                 (PC)->pc_state &= ~(1<< ((RI)-1));              \
 331         } while (0)
 332
 333 static int
 334 p6_pcpu_init(struct pmc_mdep *md, int cpu)
 335 {
 336         int first_ri, n;
 337         struct p6_cpu *p6c;
 338         struct pmc_cpu *pc;
 339         struct pmc_hw *phw;
 340
 341         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 342             ("[p6,%d] bad cpu %d", __LINE__, cpu));
 343
 344         PMCDBG1(MDP,INI,0,"p6-init cpu=%d", cpu);
 345
 346         p6c = malloc(sizeof (struct p6_cpu), M_PMC, M_WAITOK|M_ZERO);
 347         pc = pmc_pcpu[cpu];
 348
 349         KASSERT(pc != NULL, ("[p6,%d] cpu %d null per-cpu", __LINE__, cpu));
 350
 351         phw = p6c->pc_p6pmcs;
 352         p6_pcpu[cpu] = p6c;
 353
 354         first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P6].pcd_ri;
 355
 356         for (n = 0; n < P6_NPMCS; n++, phw++) {
 357                 phw->phw_state   = PMC_PHW_FLAG_IS_ENABLED |
 358                     PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n);
 359                 phw->phw_pmc     = NULL;
 360                 pc->pc_hwpmcs[n + first_ri] = phw;
 361         }
 362
 363         return (0);
 364 }
 365
 366 static int
 367 p6_pcpu_fini(struct pmc_mdep *md, int cpu)
 368 {
 369         int first_ri, n;
 370         struct p6_cpu *p6c;
 371         struct pmc_cpu *pc;
 372
 373         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 374             ("[p6,%d] bad cpu %d", __LINE__, cpu));
 375
 376         PMCDBG1(MDP,INI,0,"p6-cleanup cpu=%d", cpu);
 377
 378         p6c = p6_pcpu[cpu];
 379         p6_pcpu[cpu] = NULL;
 380
 381         KASSERT(p6c != NULL, ("[p6,%d] null pcpu", __LINE__));
 382
 383         free(p6c, M_PMC);
 384
 385         first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P6].pcd_ri;
 386         pc = pmc_pcpu[cpu];
 387         for (n = 0; n < P6_NPMCS; n++)
 388                 pc->pc_hwpmcs[n + first_ri] = NULL;
 389
 390         return (0);
 391 }
 392
 393 static int
 394 p6_read_pmc(int cpu, int ri, pmc_value_t *v)
 395 {
 396         struct pmc *pm;
 397         struct p6pmc_descr *pd;
 398         pmc_value_t tmp;
 399
 400         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 401             ("[p6,%d] illegal cpu value %d", __LINE__, cpu));
 402         KASSERT(ri >= 0 && ri < P6_NPMCS,
 403             ("[p6,%d] illegal row-index %d", __LINE__, ri));
 404
 405         pm = p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc;
 406         pd = &p6_pmcdesc[ri];
 407
 408         KASSERT(pm,
 409             ("[p6,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
 410
 411         tmp = rdmsr(pd->pm_pmc_msr) & P6_PERFCTR_READ_MASK;
 412         if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
 413                 *v = P6_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp);
 414         else
 415                 *v = tmp;
 416
 417         PMCDBG4(MDP,REA,1, "p6-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
 418             pd->pm_pmc_msr, *v);
 419
 420         return (0);
 421 }
 422
 423 static int
 424 p6_write_pmc(int cpu, int ri, pmc_value_t v)
 425 {
 426         struct pmc *pm;
 427         struct p6pmc_descr *pd;
 428
 429         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 430             ("[p6,%d] illegal cpu value %d", __LINE__, cpu));
 431         KASSERT(ri >= 0 && ri < P6_NPMCS,
 432             ("[p6,%d] illegal row-index %d", __LINE__, ri));
 433
 434         pm = p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc;
 435         pd = &p6_pmcdesc[ri];
 436
 437         KASSERT(pm,
 438             ("[p6,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
 439
 440         PMCDBG4(MDP,WRI,1, "p6-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri,
 441             pd->pm_pmc_msr, v);
 442
 443         if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
 444                 v = P6_RELOAD_COUNT_TO_PERFCTR_VALUE(v);
 445
 446         wrmsr(pd->pm_pmc_msr, v & P6_PERFCTR_WRITE_MASK);
 447
 448         return (0);
 449 }
 450
 451 static int
 452 p6_config_pmc(int cpu, int ri, struct pmc *pm)
 453 {
 454         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 455             ("[p6,%d] illegal CPU %d", __LINE__, cpu));
 456
 457         KASSERT(ri >= 0 && ri < P6_NPMCS,
 458             ("[p6,%d] illegal row-index %d", __LINE__, ri));
 459
 460         PMCDBG3(MDP,CFG,1, "p6-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
 461
 462         KASSERT(p6_pcpu[cpu] != NULL, ("[p6,%d] null per-cpu %d", __LINE__,
 463             cpu));
 464
 465         p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc = pm;
 466
 467         return (0);
 468 }
 469
 470 /*
 471  * Retrieve a configured PMC pointer from hardware state.
 472  */
 473
 474 static int
 475 p6_get_config(int cpu, int ri, struct pmc **ppm)
 476 {
 477
 478         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 479             ("[p6,%d] illegal CPU %d", __LINE__, cpu));
 480         KASSERT(ri >= 0 && ri < P6_NPMCS,
 481             ("[p6,%d] illegal row-index %d", __LINE__, ri));
 482
 483         *ppm = p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc;
 484
 485         return (0);
 486 }
 487
 488
 489 /*
 490  * A pmc may be allocated to a given row index if:
 491  * - the event is valid for this CPU
 492  * - the event is valid for this counter index
 493  */
 494
 495 static int
 496 p6_allocate_pmc(int cpu, int ri, struct pmc *pm,
 497     const struct pmc_op_pmcallocate *a)
 498 {
 499         uint32_t allowed_unitmask, caps, config, unitmask;
 500         const struct p6pmc_descr *pd;
 501         const struct p6_event_descr *pevent;
 502         enum pmc_event ev;
 503
 504         (void) cpu;
 505
 506         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 507             ("[p6,%d] illegal CPU %d", __LINE__, cpu));
 508         KASSERT(ri >= 0 && ri < P6_NPMCS,
 509             ("[p6,%d] illegal row-index value %d", __LINE__, ri));
 510
 511         pd = &p6_pmcdesc[ri];
 512
 513         PMCDBG4(MDP,ALL,1, "p6-allocate ri=%d class=%d pmccaps=0x%x "
 514             "reqcaps=0x%x", ri, pd->pm_descr.pd_class, pd->pm_descr.pd_caps,
 515             pm->pm_caps);
 516
 517         /* check class */
 518         if (pd->pm_descr.pd_class != a->pm_class)
 519                 return (EINVAL);
 520
 521         /* check requested capabilities */
 522         caps = a->pm_caps;
 523         if ((pd->pm_descr.pd_caps & caps) != caps)
 524                 return (EPERM);
 525
 526         ev = pm->pm_event;
 527
 528         if (ev < PMC_EV_P6_FIRST || ev > PMC_EV_P6_LAST)
 529                 return (EINVAL);
 530
 531         if ((pevent = p6_find_event(ev)) == NULL)
 532                 return (ESRCH);
 533
 534         if (!P6_EVENT_VALID_FOR_CPU(pevent, p6_cputype) ||
 535             !P6_EVENT_VALID_FOR_CTR(pevent, (ri-1)))
 536                 return (EINVAL);
 537
 538         /* For certain events, Pentium M differs from the stock P6 */
 539         allowed_unitmask = 0;
 540         if (p6_cputype == PMC_CPU_INTEL_PM) {
 541                 if (ev == PMC_EV_P6_L2_LD || ev == PMC_EV_P6_L2_LINES_IN ||
 542                     ev == PMC_EV_P6_L2_LINES_OUT)
 543                         allowed_unitmask = P6_EVSEL_TO_UMASK(0x3F);
 544                 else if (ev == PMC_EV_P6_L2_M_LINES_OUTM)
 545                         allowed_unitmask = P6_EVSEL_TO_UMASK(0x30);
 546         } else
 547                 allowed_unitmask = P6_EVSEL_TO_UMASK(pevent->pm_unitmask);
 548
 549         unitmask = a->pm_md.pm_ppro.pm_ppro_config & P6_EVSEL_UMASK_MASK;
 550         if (unitmask & ~allowed_unitmask) /* disallow reserved bits */
 551                 return (EINVAL);
 552
 553         if (ev == PMC_EV_P6_MMX_UOPS_EXEC) /* hardcoded mask */
 554                 unitmask = P6_EVSEL_TO_UMASK(0x0F);
 555
 556         config = 0;
 557
 558         config |= P6_EVSEL_EVENT_SELECT(pevent->pm_evsel);
 559
 560         if (unitmask & (caps & PMC_CAP_QUALIFIER))
 561                 config |= unitmask;
 562
 563         if (caps & PMC_CAP_THRESHOLD)
 564                 config |= a->pm_md.pm_ppro.pm_ppro_config &
 565                     P6_EVSEL_CMASK_MASK;
 566
 567         /* set at least one of the 'usr' or 'os' caps */
 568         if (caps & PMC_CAP_USER)
 569                 config |= P6_EVSEL_USR;
 570         if (caps & PMC_CAP_SYSTEM)
 571                 config |= P6_EVSEL_OS;
 572         if ((caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0)
 573                 config |= (P6_EVSEL_USR|P6_EVSEL_OS);
 574
 575         if (caps & PMC_CAP_EDGE)
 576                 config |= P6_EVSEL_E;
 577         if (caps & PMC_CAP_INVERT)
 578                 config |= P6_EVSEL_INV;
 579         if (caps & PMC_CAP_INTERRUPT)
 580                 config |= P6_EVSEL_INT;
 581
 582         pm->pm_md.pm_ppro.pm_ppro_evsel = config;
 583
 584         PMCDBG1(MDP,ALL,2, "p6-allocate config=0x%x", config);
 585
 586         return (0);
 587 }
 588
 589 static int
 590 p6_release_pmc(int cpu, int ri, struct pmc *pm)
 591 {
 592         (void) pm;
 593
 594         PMCDBG3(MDP,REL,1, "p6-release cpu=%d ri=%d pm=%p", cpu, ri, pm);
 595
 596         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 597             ("[p6,%d] illegal CPU value %d", __LINE__, cpu));
 598         KASSERT(ri >= 0 && ri < P6_NPMCS,
 599             ("[p6,%d] illegal row-index %d", __LINE__, ri));
 600
 601         KASSERT(p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc == NULL,
 602             ("[p6,%d] PHW pmc non-NULL", __LINE__));
 603
 604         return (0);
 605 }
 606
 607 static int
 608 p6_start_pmc(int cpu, int ri)
 609 {
 610         uint32_t config;
 611         struct pmc *pm;
 612         struct p6_cpu *pc;
 613         const struct p6pmc_descr *pd;
 614
 615         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 616             ("[p6,%d] illegal CPU value %d", __LINE__, cpu));
 617         KASSERT(ri >= 0 && ri < P6_NPMCS,
 618             ("[p6,%d] illegal row-index %d", __LINE__, ri));
 619
 620         pc = p6_pcpu[cpu];
 621         pm = pc->pc_p6pmcs[ri].phw_pmc;
 622         pd = &p6_pmcdesc[ri];
 623
 624         KASSERT(pm,
 625             ("[p6,%d] starting cpu%d,ri%d with no pmc configured",
 626                 __LINE__, cpu, ri));
 627
 628         PMCDBG2(MDP,STA,1, "p6-start cpu=%d ri=%d", cpu, ri);
 629
 630         config = pm->pm_md.pm_ppro.pm_ppro_evsel;
 631
 632         PMCDBG4(MDP,STA,2, "p6-start/2 cpu=%d ri=%d evselmsr=0x%x config=0x%x",
 633             cpu, ri, pd->pm_evsel_msr, config);
 634
 635         P6_MARK_STARTED(pc, ri);
 636         wrmsr(pd->pm_evsel_msr, config);
 637
 638         P6_SYNC_CTR_STATE(pc);
 639
 640         return (0);
 641 }
 642
 643 static int
 644 p6_stop_pmc(int cpu, int ri)
 645 {
 646         struct pmc *pm;
 647         struct p6_cpu *pc;
 648         struct p6pmc_descr *pd;
 649
 650         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 651             ("[p6,%d] illegal cpu value %d", __LINE__, cpu));
 652         KASSERT(ri >= 0 && ri < P6_NPMCS,
 653             ("[p6,%d] illegal row index %d", __LINE__, ri));
 654
 655         pc = p6_pcpu[cpu];
 656         pm = pc->pc_p6pmcs[ri].phw_pmc;
 657         pd = &p6_pmcdesc[ri];
 658
 659         KASSERT(pm,
 660             ("[p6,%d] cpu%d ri%d no configured PMC to stop", __LINE__,
 661                 cpu, ri));
 662
 663         PMCDBG2(MDP,STO,1, "p6-stop cpu=%d ri=%d", cpu, ri);
 664
 665         wrmsr(pd->pm_evsel_msr, 0);     /* stop hw */
 666         P6_MARK_STOPPED(pc, ri);        /* update software state */
 667
 668         P6_SYNC_CTR_STATE(pc);          /* restart CTR1 if need be */
 669
 670         PMCDBG2(MDP,STO,2, "p6-stop/2 cpu=%d ri=%d", cpu, ri);
 671
 672         return (0);
 673 }
 674
 675 static int
 676 p6_intr(int cpu, struct trapframe *tf)
 677 {
 678         int error, retval, ri;
 679         uint32_t perf0cfg;
 680         struct pmc *pm;
 681         struct p6_cpu *pc;
 682         pmc_value_t v;
 683
 684         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 685             ("[p6,%d] CPU %d out of range", __LINE__, cpu));
 686
 687         retval = 0;
 688         pc = p6_pcpu[cpu];
 689
 690         /* stop both PMCs */
 691         perf0cfg = rdmsr(P6_MSR_EVSEL0);
 692         wrmsr(P6_MSR_EVSEL0, perf0cfg & ~P6_EVSEL_EN);
 693
 694         for (ri = 0; ri < P6_NPMCS; ri++) {
 695
 696                 if ((pm = pc->pc_p6pmcs[ri].phw_pmc) == NULL ||
 697                     !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
 698                         continue;
 699                 }
 700
 701                 if (!P6_PMC_HAS_OVERFLOWED(ri))
 702                         continue;
 703
 704                 retval = 1;
 705
 706                 if (pm->pm_state != PMC_STATE_RUNNING)
 707                         continue;
 708
 709                 error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
 710                     TRAPF_USERMODE(tf));
 711                 if (error)
 712                         P6_MARK_STOPPED(pc,ri);
 713
 714                 /* reload sampling count */
 715                 v = pm->pm_sc.pm_reloadcount;
 716                 wrmsr(P6_MSR_PERFCTR0 + ri,
 717                     P6_RELOAD_COUNT_TO_PERFCTR_VALUE(v));
 718
 719         }
 720
 721         /*
 722          * On P6 processors, the LAPIC needs to have its PMC interrupt
 723          * unmasked after a PMC interrupt.
 724          */
 725         if (retval)
 726                 lapic_reenable_pmc();
 727
 728         atomic_add_int(retval ? &pmc_stats.pm_intr_processed :
 729             &pmc_stats.pm_intr_ignored, 1);
 730
 731         /* restart counters that can be restarted */
 732         P6_SYNC_CTR_STATE(pc);
 733
 734         return (retval);
 735 }
 736
 737 static int
 738 p6_describe(int cpu, int ri, struct pmc_info *pi,
 739     struct pmc **ppmc)
 740 {
 741         int error;
 742         size_t copied;
 743         struct pmc_hw *phw;
 744         struct p6pmc_descr *pd;
 745
 746         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 747             ("[p6,%d] illegal CPU %d", __LINE__, cpu));
 748         KASSERT(ri >= 0 && ri < P6_NPMCS,
 749             ("[p6,%d] row-index %d out of range", __LINE__, ri));
 750
 751         phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
 752         pd  = &p6_pmcdesc[ri];
 753
 754         KASSERT(phw == &p6_pcpu[cpu]->pc_p6pmcs[ri],
 755             ("[p6,%d] phw mismatch", __LINE__));
 756
 757         if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name,
 758                  PMC_NAME_MAX, &copied)) != 0)
 759                 return (error);
 760
 761         pi->pm_class = pd->pm_descr.pd_class;
 762
 763         if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
 764                 pi->pm_enabled = TRUE;
 765                 *ppmc          = phw->phw_pmc;
 766         } else {
 767                 pi->pm_enabled = FALSE;
 768                 *ppmc          = NULL;
 769         }
 770
 771         return (0);
 772 }
 773
 774 static int
 775 p6_get_msr(int ri, uint32_t *msr)
 776 {
 777         KASSERT(ri >= 0 && ri < P6_NPMCS,
 778             ("[p6,%d ri %d out of range", __LINE__, ri));
 779
 780         *msr = p6_pmcdesc[ri].pm_pmc_msr - P6_MSR_PERFCTR0;
 781
 782         return (0);
 783 }
 784
 785 int
 786 pmc_p6_initialize(struct pmc_mdep *md, int ncpus)
 787 {
 788         struct pmc_classdep *pcd;
 789
 790         KASSERT(cpu_vendor_id == CPU_VENDOR_INTEL,
 791             ("[p6,%d] Initializing non-intel processor", __LINE__));
 792
 793         PMCDBG0(MDP,INI,1, "p6-initialize");
 794
 795         /* Allocate space for pointers to per-cpu descriptors. */
 796         p6_pcpu = malloc(sizeof(struct p6_cpu **) * ncpus, M_PMC,
 797             M_ZERO|M_WAITOK);
 798
 799         /* Fill in the class dependent descriptor. */
 800         pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P6];
 801
 802         switch (md->pmd_cputype) {
 803
 804                 /*
 805                  * P6 Family Processors
 806                  */
 807         case PMC_CPU_INTEL_P6:
 808         case PMC_CPU_INTEL_CL:
 809         case PMC_CPU_INTEL_PII:
 810         case PMC_CPU_INTEL_PIII:
 811         case PMC_CPU_INTEL_PM:
 812
 813                 p6_cputype = md->pmd_cputype;
 814
 815                 pcd->pcd_caps           = P6_PMC_CAPS;
 816                 pcd->pcd_class          = PMC_CLASS_P6;
 817                 pcd->pcd_num            = P6_NPMCS;
 818                 pcd->pcd_ri             = md->pmd_npmc;
 819                 pcd->pcd_width          = 40;
 820
 821                 pcd->pcd_allocate_pmc   = p6_allocate_pmc;
 822                 pcd->pcd_config_pmc     = p6_config_pmc;
 823                 pcd->pcd_describe       = p6_describe;
 824                 pcd->pcd_get_config     = p6_get_config;
 825                 pcd->pcd_get_msr        = p6_get_msr;
 826                 pcd->pcd_pcpu_fini      = p6_pcpu_fini;
 827                 pcd->pcd_pcpu_init      = p6_pcpu_init;
 828                 pcd->pcd_read_pmc       = p6_read_pmc;
 829                 pcd->pcd_release_pmc    = p6_release_pmc;
 830                 pcd->pcd_start_pmc      = p6_start_pmc;
 831                 pcd->pcd_stop_pmc       = p6_stop_pmc;
 832                 pcd->pcd_write_pmc      = p6_write_pmc;
 833
 834                 md->pmd_pcpu_fini       = NULL;
 835                 md->pmd_pcpu_init       = NULL;
 836                 md->pmd_intr            = p6_intr;
 837
 838                 md->pmd_npmc           += P6_NPMCS;
 839
 840                 break;
 841
 842         default:
 843                 KASSERT(0,("[p6,%d] Unknown CPU type", __LINE__));
 844                 return ENOSYS;
 845         }
 846
 847         return (0);
 848 }
 849
 850 void
 851 pmc_p6_finalize(struct pmc_mdep *md)
 852 {
 853 #if     defined(INVARIANTS)
 854         int i, ncpus;
 855 #endif
 856
 857         KASSERT(p6_pcpu != NULL, ("[p6,%d] NULL p6_pcpu", __LINE__));
 858
 859 #if     defined(INVARIANTS)
 860         ncpus = pmc_cpu_max();
 861         for (i = 0; i < ncpus; i++)
 862                 KASSERT(p6_pcpu[i] == NULL, ("[p6,%d] non-null pcpu %d",
 863                     __LINE__, i));
 864 #endif
 865
 866         free(p6_pcpu, M_PMC);
 867         p6_pcpu = NULL;
 868 }