sys/dev/hwpmc/hwpmc_ppro.c

   1 /*-
   2  * Copyright (c) 2003-2005,2008 Joseph Koshy
   3  * Copyright (c) 2007 The FreeBSD Foundation
   4  * All rights reserved.
   5  *
   6  * Portions of this software were developed by A. Joseph Koshy under
   7  * sponsorship from the FreeBSD Foundation and Google, Inc.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33
  34 #include <sys/param.h>
  35 #include <sys/bus.h>
  36 #include <sys/lock.h>
  37 #include <sys/mutex.h>
  38 #include <sys/pmc.h>
  39 #include <sys/pmckern.h>
  40 #include <sys/smp.h>
  41 #include <sys/systm.h>
  42
  43 #include <machine/intr_machdep.h>
  44 #include <machine/apicvar.h>
  45 #include <machine/cpu.h>
  46 #include <machine/cpufunc.h>
  47 #include <machine/cputypes.h>
  48 #include <machine/md_var.h>
  49 #include <machine/pmc_mdep.h>
  50 #include <machine/specialreg.h>
  51
  52 /*
  53  * PENTIUM PRO SUPPORT
  54  *
  55  * Quirks:
  56  *
  57  * - Both PMCs are enabled by a single bit P6_EVSEL_EN in performance
  58  *   counter '0'.  This bit needs to be '1' if any of the two
  59  *   performance counters are in use.  Perf counters can also be
  60  *   switched off by writing zeros to their EVSEL register.
  61  *
  62  * - While the width of these counters is 40 bits, we do not appear to
  63  *   have a way of writing 40 bits to the counter MSRs.  A WRMSR
  64  *   instruction will sign extend bit 31 of the value being written to
  65  *   the perf counter -- a value of 0x80000000 written to an perf
  66  *   counter register will be sign extended to 0xFF80000000.
  67  *
  68  *   This quirk primarily affects thread-mode PMCs in counting mode, as
  69  *   these PMCs read and write PMC registers at every context switch.
  70  */
  71
  72 struct p6pmc_descr {
  73         struct pmc_descr pm_descr; /* common information */
  74         uint32_t        pm_pmc_msr;
  75         uint32_t        pm_evsel_msr;
  76 };
  77
  78 static struct p6pmc_descr p6_pmcdesc[P6_NPMCS] = {
  79
  80 #define P6_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | PMC_CAP_SYSTEM | \
  81     PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE |    \
  82     PMC_CAP_INVERT | PMC_CAP_QUALIFIER)
  83
  84         /* PMC 0 */
  85         {
  86                 .pm_descr =
  87                 {
  88                         .pd_name  ="P6-0",
  89                         .pd_class = PMC_CLASS_P6,
  90                         .pd_caps  = P6_PMC_CAPS,
  91                         .pd_width = 40
  92                 },
  93                 .pm_pmc_msr   = P6_MSR_PERFCTR0,
  94                 .pm_evsel_msr = P6_MSR_EVSEL0
  95         },
  96
  97         /* PMC 1 */
  98         {
  99                 .pm_descr =
 100                 {
 101                         .pd_name  ="P6-1",
 102                         .pd_class = PMC_CLASS_P6,
 103                         .pd_caps  = P6_PMC_CAPS,
 104                         .pd_width = 40
 105                 },
 106                 .pm_pmc_msr   = P6_MSR_PERFCTR1,
 107                 .pm_evsel_msr = P6_MSR_EVSEL1
 108         }
 109 };
 110
 111 static enum pmc_cputype p6_cputype;
 112
 113 /*
 114  * P6 Event descriptor
 115  *
 116  * The 'pm_flags' field has the following structure:
 117  * - The upper 4 bits are used to track which counter an event is valid on.
 118  * - The lower bits form a bitmask of flags indicating support for the event
 119  *   on a given CPU.
 120  */
 121
 122 struct p6_event_descr {
 123         const enum pmc_event pm_event;
 124         uint32_t             pm_evsel;
 125         uint32_t             pm_flags;
 126         uint32_t             pm_unitmask;
 127 };
 128
 129 #define P6F_CTR(C)      (1 << (28 + (C)))
 130 #define P6F_CTR0        P6F_CTR(0)
 131 #define P6F_CTR1        P6F_CTR(1)
 132 #define P6F(CPU)        (1 << ((CPU) - PMC_CPU_INTEL_P6))
 133 #define _P6F(C)         P6F(PMC_CPU_INTEL_##C)
 134 #define P6F_P6          _P6F(P6)
 135 #define P6F_CL          _P6F(CL)
 136 #define P6F_PII         _P6F(PII)
 137 #define P6F_PIII        _P6F(PIII)
 138 #define P6F_PM          _P6F(PM)
 139 #define P6F_ALL_CPUS    (P6F_P6 | P6F_PII | P6F_CL | P6F_PIII | P6F_PM)
 140 #define P6F_ALL_CTRS    (P6F_CTR0 | P6F_CTR1)
 141 #define P6F_ALL         (P6F_ALL_CPUS | P6F_ALL_CTRS)
 142
 143 #define P6_EVENT_VALID_FOR_CPU(P,CPU)   ((P)->pm_flags & P6F(CPU))
 144 #define P6_EVENT_VALID_FOR_CTR(P,CTR)   ((P)->pm_flags & P6F_CTR(CTR))
 145
 146 static const struct p6_event_descr p6_events[] = {
 147
 148 #define P6_EVDESCR(NAME, EVSEL, FLAGS, UMASK)   \
 149         {                                       \
 150                 .pm_event = PMC_EV_P6_##NAME,   \
 151                 .pm_evsel = (EVSEL),            \
 152                 .pm_flags = (FLAGS),            \
 153                 .pm_unitmask = (UMASK)          \
 154         }
 155
 156 P6_EVDESCR(DATA_MEM_REFS,               0x43, P6F_ALL, 0x00),
 157 P6_EVDESCR(DCU_LINES_IN,                0x45, P6F_ALL, 0x00),
 158 P6_EVDESCR(DCU_M_LINES_IN,              0x46, P6F_ALL, 0x00),
 159 P6_EVDESCR(DCU_M_LINES_OUT,             0x47, P6F_ALL, 0x00),
 160 P6_EVDESCR(DCU_MISS_OUTSTANDING,        0x47, P6F_ALL, 0x00),
 161 P6_EVDESCR(IFU_FETCH,                   0x80, P6F_ALL, 0x00),
 162 P6_EVDESCR(IFU_FETCH_MISS,              0x81, P6F_ALL, 0x00),
 163 P6_EVDESCR(ITLB_MISS,                   0x85, P6F_ALL, 0x00),
 164 P6_EVDESCR(IFU_MEM_STALL,               0x86, P6F_ALL, 0x00),
 165 P6_EVDESCR(ILD_STALL,                   0x87, P6F_ALL, 0x00),
 166 P6_EVDESCR(L2_IFETCH,                   0x28, P6F_ALL, 0x0F),
 167 P6_EVDESCR(L2_LD,                       0x29, P6F_ALL, 0x0F),
 168 P6_EVDESCR(L2_ST,                       0x2A, P6F_ALL, 0x0F),
 169 P6_EVDESCR(L2_LINES_IN,                 0x24, P6F_ALL, 0x0F),
 170 P6_EVDESCR(L2_LINES_OUT,                0x26, P6F_ALL, 0x0F),
 171 P6_EVDESCR(L2_M_LINES_INM,              0x25, P6F_ALL, 0x00),
 172 P6_EVDESCR(L2_M_LINES_OUTM,             0x27, P6F_ALL, 0x0F),
 173 P6_EVDESCR(L2_RQSTS,                    0x2E, P6F_ALL, 0x0F),
 174 P6_EVDESCR(L2_ADS,                      0x21, P6F_ALL, 0x00),
 175 P6_EVDESCR(L2_DBUS_BUSY,                0x22, P6F_ALL, 0x00),
 176 P6_EVDESCR(L2_DBUS_BUSY_RD,             0x23, P6F_ALL, 0x00),
 177 P6_EVDESCR(BUS_DRDY_CLOCKS,             0x62, P6F_ALL, 0x20),
 178 P6_EVDESCR(BUS_LOCK_CLOCKS,             0x63, P6F_ALL, 0x20),
 179 P6_EVDESCR(BUS_REQ_OUTSTANDING,         0x60, P6F_ALL, 0x00),
 180 P6_EVDESCR(BUS_TRAN_BRD,                0x65, P6F_ALL, 0x20),
 181 P6_EVDESCR(BUS_TRAN_RFO,                0x66, P6F_ALL, 0x20),
 182 P6_EVDESCR(BUS_TRANS_WB,                0x67, P6F_ALL, 0x20),
 183 P6_EVDESCR(BUS_TRAN_IFETCH,             0x68, P6F_ALL, 0x20),
 184 P6_EVDESCR(BUS_TRAN_INVAL,              0x69, P6F_ALL, 0x20),
 185 P6_EVDESCR(BUS_TRAN_PWR,                0x6A, P6F_ALL, 0x20),
 186 P6_EVDESCR(BUS_TRANS_P,                 0x6B, P6F_ALL, 0x20),
 187 P6_EVDESCR(BUS_TRANS_IO,                0x6C, P6F_ALL, 0x20),
 188 P6_EVDESCR(BUS_TRAN_DEF,                0x6D, P6F_ALL, 0x20),
 189 P6_EVDESCR(BUS_TRAN_BURST,              0x6E, P6F_ALL, 0x20),
 190 P6_EVDESCR(BUS_TRAN_ANY,                0x70, P6F_ALL, 0x20),
 191 P6_EVDESCR(BUS_TRAN_MEM,                0x6F, P6F_ALL, 0x20),
 192 P6_EVDESCR(BUS_DATA_RCV,                0x64, P6F_ALL, 0x00),
 193 P6_EVDESCR(BUS_BNR_DRV,                 0x61, P6F_ALL, 0x00),
 194 P6_EVDESCR(BUS_HIT_DRV,                 0x7A, P6F_ALL, 0x00),
 195 P6_EVDESCR(BUS_HITM_DRV,                0x7B, P6F_ALL, 0x00),
 196 P6_EVDESCR(BUS_SNOOP_STALL,             0x7E, P6F_ALL, 0x00),
 197 P6_EVDESCR(FLOPS,                       0xC1, P6F_ALL_CPUS | P6F_CTR0, 0x00),
 198 P6_EVDESCR(FP_COMPS_OPS_EXE,            0x10, P6F_ALL_CPUS | P6F_CTR0, 0x00),
 199 P6_EVDESCR(FP_ASSIST,                   0x11, P6F_ALL_CPUS | P6F_CTR1, 0x00),
 200 P6_EVDESCR(MUL,                         0x12, P6F_ALL_CPUS | P6F_CTR1, 0x00),
 201 P6_EVDESCR(DIV,                         0x13, P6F_ALL_CPUS | P6F_CTR1, 0x00),
 202 P6_EVDESCR(CYCLES_DIV_BUSY,             0x14, P6F_ALL_CPUS | P6F_CTR0, 0x00),
 203 P6_EVDESCR(LD_BLOCKS,                   0x03, P6F_ALL, 0x00),
 204 P6_EVDESCR(SB_DRAINS,                   0x04, P6F_ALL, 0x00),
 205 P6_EVDESCR(MISALIGN_MEM_REF,            0x05, P6F_ALL, 0x00),
 206 P6_EVDESCR(EMON_KNI_PREF_DISPATCHED,    0x07, P6F_PIII | P6F_ALL_CTRS, 0x03),
 207 P6_EVDESCR(EMON_KNI_PREF_MISS,          0x4B, P6F_PIII | P6F_ALL_CTRS, 0x03),
 208 P6_EVDESCR(INST_RETIRED,                0xC0, P6F_ALL, 0x00),
 209 P6_EVDESCR(UOPS_RETIRED,                0xC2, P6F_ALL, 0x00),
 210 P6_EVDESCR(INST_DECODED,                0xD0, P6F_ALL, 0x00),
 211 P6_EVDESCR(EMON_KNI_INST_RETIRED,       0xD8, P6F_PIII | P6F_ALL_CTRS, 0x01),
 212 P6_EVDESCR(EMON_KNI_COMP_INST_RET,      0xD9, P6F_PIII | P6F_ALL_CTRS, 0x01),
 213 P6_EVDESCR(HW_INT_RX,                   0xC8, P6F_ALL, 0x00),
 214 P6_EVDESCR(CYCLES_INT_MASKED,           0xC6, P6F_ALL, 0x00),
 215 P6_EVDESCR(CYCLES_INT_PENDING_AND_MASKED, 0xC7, P6F_ALL, 0x00),
 216 P6_EVDESCR(BR_INST_RETIRED,             0xC4, P6F_ALL, 0x00),
 217 P6_EVDESCR(BR_MISS_PRED_RETIRED,        0xC5, P6F_ALL, 0x00),
 218 P6_EVDESCR(BR_TAKEN_RETIRED,            0xC9, P6F_ALL, 0x00),
 219 P6_EVDESCR(BR_MISS_PRED_TAKEN_RET,      0xCA, P6F_ALL, 0x00),
 220 P6_EVDESCR(BR_INST_DECODED,             0xE0, P6F_ALL, 0x00),
 221 P6_EVDESCR(BTB_MISSES,                  0xE2, P6F_ALL, 0x00),
 222 P6_EVDESCR(BR_BOGUS,                    0xE4, P6F_ALL, 0x00),
 223 P6_EVDESCR(BACLEARS,                    0xE6, P6F_ALL, 0x00),
 224 P6_EVDESCR(RESOURCE_STALLS,             0xA2, P6F_ALL, 0x00),
 225 P6_EVDESCR(PARTIAL_RAT_STALLS,          0xD2, P6F_ALL, 0x00),
 226 P6_EVDESCR(SEGMENT_REG_LOADS,           0x06, P6F_ALL, 0x00),
 227 P6_EVDESCR(CPU_CLK_UNHALTED,            0x79, P6F_ALL, 0x00),
 228 P6_EVDESCR(MMX_INSTR_EXEC,              0xB0,
 229                         P6F_ALL_CTRS | P6F_CL | P6F_PII, 0x00),
 230 P6_EVDESCR(MMX_SAT_INSTR_EXEC,          0xB1,
 231                         P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x00),
 232 P6_EVDESCR(MMX_UOPS_EXEC,               0xB2,
 233                         P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x0F),
 234 P6_EVDESCR(MMX_INSTR_TYPE_EXEC,         0xB3,
 235                         P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x3F),
 236 P6_EVDESCR(FP_MMX_TRANS,                0xCC,
 237                         P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x01),
 238 P6_EVDESCR(MMX_ASSIST,                  0xCD,
 239                         P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x00),
 240 P6_EVDESCR(MMX_INSTR_RET,               0xCE, P6F_ALL_CTRS | P6F_PII, 0x00),
 241 P6_EVDESCR(SEG_RENAME_STALLS,           0xD4,
 242                         P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x0F),
 243 P6_EVDESCR(SEG_REG_RENAMES,             0xD5,
 244                         P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x0F),
 245 P6_EVDESCR(RET_SEG_RENAMES,             0xD6,
 246                         P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x00),
 247 P6_EVDESCR(EMON_EST_TRANS,              0x58, P6F_ALL_CTRS | P6F_PM, 0x02),
 248 P6_EVDESCR(EMON_THERMAL_TRIP,           0x59, P6F_ALL_CTRS | P6F_PM, 0x00),
 249 P6_EVDESCR(BR_INST_EXEC,                0x88, P6F_ALL_CTRS | P6F_PM, 0x00),
 250 P6_EVDESCR(BR_MISSP_EXEC,               0x89, P6F_ALL_CTRS | P6F_PM, 0x00),
 251 P6_EVDESCR(BR_BAC_MISSP_EXEC,           0x8A, P6F_ALL_CTRS | P6F_PM, 0x00),
 252 P6_EVDESCR(BR_CND_EXEC,                 0x8B, P6F_ALL_CTRS | P6F_PM, 0x00),
 253 P6_EVDESCR(BR_CND_MISSP_EXEC,           0x8C, P6F_ALL_CTRS | P6F_PM, 0x00),
 254 P6_EVDESCR(BR_IND_EXEC,                 0x8D, P6F_ALL_CTRS | P6F_PM, 0x00),
 255 P6_EVDESCR(BR_IND_MISSP_EXEC,           0x8E, P6F_ALL_CTRS | P6F_PM, 0x00),
 256 P6_EVDESCR(BR_RET_EXEC,                 0x8F, P6F_ALL_CTRS | P6F_PM, 0x00),
 257 P6_EVDESCR(BR_RET_MISSP_EXEC,           0x90, P6F_ALL_CTRS | P6F_PM, 0x00),
 258 P6_EVDESCR(BR_RET_BAC_MISSP_EXEC,       0x91, P6F_ALL_CTRS | P6F_PM, 0x00),
 259 P6_EVDESCR(BR_CALL_EXEC,                0x92, P6F_ALL_CTRS | P6F_PM, 0x00),
 260 P6_EVDESCR(BR_CALL_MISSP_EXEC,          0x93, P6F_ALL_CTRS | P6F_PM, 0x00),
 261 P6_EVDESCR(BR_IND_CALL_EXEC,            0x94, P6F_ALL_CTRS | P6F_PM, 0x00),
 262 P6_EVDESCR(EMON_SIMD_INSTR_RETIRED,     0xCE, P6F_ALL_CTRS | P6F_PM, 0x00),
 263 P6_EVDESCR(EMON_SYNCH_UOPS,             0xD3, P6F_ALL_CTRS | P6F_PM, 0x00),
 264 P6_EVDESCR(EMON_ESP_UOPS,               0xD7, P6F_ALL_CTRS | P6F_PM, 0x00),
 265 P6_EVDESCR(EMON_FUSED_UOPS_RET,         0xDA, P6F_ALL_CTRS | P6F_PM, 0x03),
 266 P6_EVDESCR(EMON_UNFUSION,               0xDB, P6F_ALL_CTRS | P6F_PM, 0x00),
 267 P6_EVDESCR(EMON_PREF_RQSTS_UP,          0xF0, P6F_ALL_CTRS | P6F_PM, 0x00),
 268 P6_EVDESCR(EMON_PREF_RQSTS_DN,          0xD8, P6F_ALL_CTRS | P6F_PM, 0x00),
 269 P6_EVDESCR(EMON_SSE_SSE2_INST_RETIRED,  0xD8, P6F_ALL_CTRS | P6F_PM, 0x03),
 270 P6_EVDESCR(EMON_SSE_SSE2_COMP_INST_RETIRED, 0xD9, P6F_ALL_CTRS | P6F_PM, 0x03)
 271
 272 #undef  P6_EVDESCR
 273 };
 274
 275 #define P6_NEVENTS      (PMC_EV_P6_LAST - PMC_EV_P6_FIRST + 1)
 276
 277 static const struct p6_event_descr *
 278 p6_find_event(enum pmc_event ev)
 279 {
 280         int n;
 281
 282         for (n = 0; n < P6_NEVENTS; n++)
 283                 if (p6_events[n].pm_event == ev)
 284                         break;
 285         if (n == P6_NEVENTS)
 286                 return NULL;
 287         return &p6_events[n];
 288 }
 289
 290 /*
 291  * Per-CPU data structure for P6 class CPUs
 292  *
 293  * [common stuff]
 294  * [flags for maintaining PMC start/stop state]
 295  * [3 struct pmc_hw pointers]
 296  * [3 struct pmc_hw structures]
 297  */
 298
 299 struct p6_cpu {
 300         struct pmc_hw   pc_p6pmcs[P6_NPMCS];
 301         uint32_t        pc_state;
 302 };
 303
 304 static struct p6_cpu **p6_pcpu;
 305
 306 /*
 307  * If CTR1 is active, we need to keep the 'EN' bit if CTR0 set,
 308  * with the rest of CTR0 being zero'ed out.
 309  */
 310 #define P6_SYNC_CTR_STATE(PC) do {                              \
 311                 uint32_t _config, _enable;                      \
 312                 _enable = 0;                                    \
 313                 if ((PC)->pc_state & 0x02)                      \
 314                         _enable |= P6_EVSEL_EN;                 \
 315                 if ((PC)->pc_state & 0x01)                      \
 316                         _config = rdmsr(P6_MSR_EVSEL0) |        \
 317                             P6_EVSEL_EN;                        \
 318                 else                                            \
 319                         _config = 0;                            \
 320                 wrmsr(P6_MSR_EVSEL0, _config | _enable);        \
 321         } while (0)
 322
 323 #define P6_MARK_STARTED(PC,RI) do {                             \
 324                 (PC)->pc_state |= (1 << ((RI)-1));              \
 325         } while (0)
 326
 327 #define P6_MARK_STOPPED(PC,RI) do {                             \
 328                 (PC)->pc_state &= ~(1<< ((RI)-1));              \
 329         } while (0)
 330
 331 static int
 332 p6_pcpu_init(struct pmc_mdep *md, int cpu)
 333 {
 334         int first_ri, n;
 335         struct p6_cpu *p6c;
 336         struct pmc_cpu *pc;
 337         struct pmc_hw *phw;
 338
 339         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 340             ("[p6,%d] bad cpu %d", __LINE__, cpu));
 341
 342         PMCDBG(MDP,INI,0,"p6-init cpu=%d", cpu);
 343
 344         p6c = malloc(sizeof (struct p6_cpu), M_PMC, M_WAITOK|M_ZERO);
 345         pc = pmc_pcpu[cpu];
 346
 347         KASSERT(pc != NULL, ("[p6,%d] cpu %d null per-cpu", __LINE__, cpu));
 348
 349         phw = p6c->pc_p6pmcs;
 350         p6_pcpu[cpu] = p6c;
 351
 352         first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P6].pcd_ri;
 353
 354         for (n = 0; n < P6_NPMCS; n++, phw++) {
 355                 phw->phw_state   = PMC_PHW_FLAG_IS_ENABLED |
 356                     PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n);
 357                 phw->phw_pmc     = NULL;
 358                 pc->pc_hwpmcs[n + first_ri] = phw;
 359         }
 360
 361         return (0);
 362 }
 363
 364 static int
 365 p6_pcpu_fini(struct pmc_mdep *md, int cpu)
 366 {
 367         int first_ri, n;
 368         struct p6_cpu *p6c;
 369         struct pmc_cpu *pc;
 370
 371         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 372             ("[p6,%d] bad cpu %d", __LINE__, cpu));
 373
 374         PMCDBG(MDP,INI,0,"p6-cleanup cpu=%d", cpu);
 375
 376         p6c = p6_pcpu[cpu];
 377         p6_pcpu[cpu] = NULL;
 378
 379         KASSERT(p6c != NULL, ("[p6,%d] null pcpu", __LINE__));
 380
 381         free(p6c, M_PMC);
 382
 383         first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P6].pcd_ri;
 384         pc = pmc_pcpu[cpu];
 385         for (n = 0; n < P6_NPMCS; n++)
 386                 pc->pc_hwpmcs[n + first_ri] = NULL;
 387
 388         return (0);
 389 }
 390
 391 static int
 392 p6_read_pmc(int cpu, int ri, pmc_value_t *v)
 393 {
 394         struct pmc *pm;
 395         struct p6pmc_descr *pd;
 396         pmc_value_t tmp;
 397
 398         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 399             ("[p6,%d] illegal cpu value %d", __LINE__, cpu));
 400         KASSERT(ri >= 0 && ri < P6_NPMCS,
 401             ("[p6,%d] illegal row-index %d", __LINE__, ri));
 402
 403         pm = p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc;
 404         pd = &p6_pmcdesc[ri];
 405
 406         KASSERT(pm,
 407             ("[p6,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
 408
 409         tmp = rdmsr(pd->pm_pmc_msr) & P6_PERFCTR_READ_MASK;
 410         if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
 411                 *v = P6_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp);
 412         else
 413                 *v = tmp;
 414
 415         PMCDBG(MDP,REA,1, "p6-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
 416             pd->pm_pmc_msr, *v);
 417
 418         return (0);
 419 }
 420
 421 static int
 422 p6_write_pmc(int cpu, int ri, pmc_value_t v)
 423 {
 424         struct pmc *pm;
 425         struct p6pmc_descr *pd;
 426
 427         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 428             ("[p6,%d] illegal cpu value %d", __LINE__, cpu));
 429         KASSERT(ri >= 0 && ri < P6_NPMCS,
 430             ("[p6,%d] illegal row-index %d", __LINE__, ri));
 431
 432         pm = p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc;
 433         pd = &p6_pmcdesc[ri];
 434
 435         KASSERT(pm,
 436             ("[p6,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
 437
 438         PMCDBG(MDP,WRI,1, "p6-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri,
 439             pd->pm_pmc_msr, v);
 440
 441         if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
 442                 v = P6_RELOAD_COUNT_TO_PERFCTR_VALUE(v);
 443
 444         wrmsr(pd->pm_pmc_msr, v & P6_PERFCTR_WRITE_MASK);
 445
 446         return (0);
 447 }
 448
 449 static int
 450 p6_config_pmc(int cpu, int ri, struct pmc *pm)
 451 {
 452         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 453             ("[p6,%d] illegal CPU %d", __LINE__, cpu));
 454
 455         KASSERT(ri >= 0 && ri < P6_NPMCS,
 456             ("[p6,%d] illegal row-index %d", __LINE__, ri));
 457
 458         PMCDBG(MDP,CFG,1, "p6-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
 459
 460         KASSERT(p6_pcpu[cpu] != NULL, ("[p6,%d] null per-cpu %d", __LINE__,
 461             cpu));
 462
 463         p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc = pm;
 464
 465         return (0);
 466 }
 467
 468 /*
 469  * Retrieve a configured PMC pointer from hardware state.
 470  */
 471
 472 static int
 473 p6_get_config(int cpu, int ri, struct pmc **ppm)
 474 {
 475
 476         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 477             ("[p6,%d] illegal CPU %d", __LINE__, cpu));
 478         KASSERT(ri >= 0 && ri < P6_NPMCS,
 479             ("[p6,%d] illegal row-index %d", __LINE__, ri));
 480
 481         *ppm = p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc;
 482
 483         return (0);
 484 }
 485
 486
 487 /*
 488  * A pmc may be allocated to a given row index if:
 489  * - the event is valid for this CPU
 490  * - the event is valid for this counter index
 491  */
 492
 493 static int
 494 p6_allocate_pmc(int cpu, int ri, struct pmc *pm,
 495     const struct pmc_op_pmcallocate *a)
 496 {
 497         uint32_t allowed_unitmask, caps, config, unitmask;
 498         const struct p6pmc_descr *pd;
 499         const struct p6_event_descr *pevent;
 500         enum pmc_event ev;
 501
 502         (void) cpu;
 503
 504         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 505             ("[p6,%d] illegal CPU %d", __LINE__, cpu));
 506         KASSERT(ri >= 0 && ri < P6_NPMCS,
 507             ("[p6,%d] illegal row-index value %d", __LINE__, ri));
 508
 509         pd = &p6_pmcdesc[ri];
 510
 511         PMCDBG(MDP,ALL,1, "p6-allocate ri=%d class=%d pmccaps=0x%x "
 512             "reqcaps=0x%x", ri, pd->pm_descr.pd_class, pd->pm_descr.pd_caps,
 513             pm->pm_caps);
 514
 515         /* check class */
 516         if (pd->pm_descr.pd_class != a->pm_class)
 517                 return (EINVAL);
 518
 519         /* check requested capabilities */
 520         caps = a->pm_caps;
 521         if ((pd->pm_descr.pd_caps & caps) != caps)
 522                 return (EPERM);
 523
 524         ev = pm->pm_event;
 525
 526         if (ev < PMC_EV_P6_FIRST || ev > PMC_EV_P6_LAST)
 527                 return (EINVAL);
 528
 529         if ((pevent = p6_find_event(ev)) == NULL)
 530                 return (ESRCH);
 531
 532         if (!P6_EVENT_VALID_FOR_CPU(pevent, p6_cputype) ||
 533             !P6_EVENT_VALID_FOR_CTR(pevent, (ri-1)))
 534                 return (EINVAL);
 535
 536         /* For certain events, Pentium M differs from the stock P6 */
 537         allowed_unitmask = 0;
 538         if (p6_cputype == PMC_CPU_INTEL_PM) {
 539                 if (ev == PMC_EV_P6_L2_LD || ev == PMC_EV_P6_L2_LINES_IN ||
 540                     ev == PMC_EV_P6_L2_LINES_OUT)
 541                         allowed_unitmask = P6_EVSEL_TO_UMASK(0x3F);
 542                 else if (ev == PMC_EV_P6_L2_M_LINES_OUTM)
 543                         allowed_unitmask = P6_EVSEL_TO_UMASK(0x30);
 544         } else
 545                 allowed_unitmask = P6_EVSEL_TO_UMASK(pevent->pm_unitmask);
 546
 547         unitmask = a->pm_md.pm_ppro.pm_ppro_config & P6_EVSEL_UMASK_MASK;
 548         if (unitmask & ~allowed_unitmask) /* disallow reserved bits */
 549                 return (EINVAL);
 550
 551         if (ev == PMC_EV_P6_MMX_UOPS_EXEC) /* hardcoded mask */
 552                 unitmask = P6_EVSEL_TO_UMASK(0x0F);
 553
 554         config = 0;
 555
 556         config |= P6_EVSEL_EVENT_SELECT(pevent->pm_evsel);
 557
 558         if (unitmask & (caps & PMC_CAP_QUALIFIER))
 559                 config |= unitmask;
 560
 561         if (caps & PMC_CAP_THRESHOLD)
 562                 config |= a->pm_md.pm_ppro.pm_ppro_config &
 563                     P6_EVSEL_CMASK_MASK;
 564
 565         /* set at least one of the 'usr' or 'os' caps */
 566         if (caps & PMC_CAP_USER)
 567                 config |= P6_EVSEL_USR;
 568         if (caps & PMC_CAP_SYSTEM)
 569                 config |= P6_EVSEL_OS;
 570         if ((caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0)
 571                 config |= (P6_EVSEL_USR|P6_EVSEL_OS);
 572
 573         if (caps & PMC_CAP_EDGE)
 574                 config |= P6_EVSEL_E;
 575         if (caps & PMC_CAP_INVERT)
 576                 config |= P6_EVSEL_INV;
 577         if (caps & PMC_CAP_INTERRUPT)
 578                 config |= P6_EVSEL_INT;
 579
 580         pm->pm_md.pm_ppro.pm_ppro_evsel = config;
 581
 582         PMCDBG(MDP,ALL,2, "p6-allocate config=0x%x", config);
 583
 584         return (0);
 585 }
 586
 587 static int
 588 p6_release_pmc(int cpu, int ri, struct pmc *pm)
 589 {
 590         (void) pm;
 591
 592         PMCDBG(MDP,REL,1, "p6-release cpu=%d ri=%d pm=%p", cpu, ri, pm);
 593
 594         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 595             ("[p6,%d] illegal CPU value %d", __LINE__, cpu));
 596         KASSERT(ri >= 0 && ri < P6_NPMCS,
 597             ("[p6,%d] illegal row-index %d", __LINE__, ri));
 598
 599         KASSERT(p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc == NULL,
 600             ("[p6,%d] PHW pmc non-NULL", __LINE__));
 601
 602         return (0);
 603 }
 604
 605 static int
 606 p6_start_pmc(int cpu, int ri)
 607 {
 608         uint32_t config;
 609         struct pmc *pm;
 610         struct p6_cpu *pc;
 611         const struct p6pmc_descr *pd;
 612
 613         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 614             ("[p6,%d] illegal CPU value %d", __LINE__, cpu));
 615         KASSERT(ri >= 0 && ri < P6_NPMCS,
 616             ("[p6,%d] illegal row-index %d", __LINE__, ri));
 617
 618         pc = p6_pcpu[cpu];
 619         pm = pc->pc_p6pmcs[ri].phw_pmc;
 620         pd = &p6_pmcdesc[ri];
 621
 622         KASSERT(pm,
 623             ("[p6,%d] starting cpu%d,ri%d with no pmc configured",
 624                 __LINE__, cpu, ri));
 625
 626         PMCDBG(MDP,STA,1, "p6-start cpu=%d ri=%d", cpu, ri);
 627
 628         config = pm->pm_md.pm_ppro.pm_ppro_evsel;
 629
 630         PMCDBG(MDP,STA,2, "p6-start/2 cpu=%d ri=%d evselmsr=0x%x config=0x%x",
 631             cpu, ri, pd->pm_evsel_msr, config);
 632
 633         P6_MARK_STARTED(pc, ri);
 634         wrmsr(pd->pm_evsel_msr, config);
 635
 636         P6_SYNC_CTR_STATE(pc);
 637
 638         return (0);
 639 }
 640
 641 static int
 642 p6_stop_pmc(int cpu, int ri)
 643 {
 644         struct pmc *pm;
 645         struct p6_cpu *pc;
 646         struct p6pmc_descr *pd;
 647
 648         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 649             ("[p6,%d] illegal cpu value %d", __LINE__, cpu));
 650         KASSERT(ri >= 0 && ri < P6_NPMCS,
 651             ("[p6,%d] illegal row index %d", __LINE__, ri));
 652
 653         pc = p6_pcpu[cpu];
 654         pm = pc->pc_p6pmcs[ri].phw_pmc;
 655         pd = &p6_pmcdesc[ri];
 656
 657         KASSERT(pm,
 658             ("[p6,%d] cpu%d ri%d no configured PMC to stop", __LINE__,
 659                 cpu, ri));
 660
 661         PMCDBG(MDP,STO,1, "p6-stop cpu=%d ri=%d", cpu, ri);
 662
 663         wrmsr(pd->pm_evsel_msr, 0);     /* stop hw */
 664         P6_MARK_STOPPED(pc, ri);        /* update software state */
 665
 666         P6_SYNC_CTR_STATE(pc);          /* restart CTR1 if need be */
 667
 668         PMCDBG(MDP,STO,2, "p6-stop/2 cpu=%d ri=%d", cpu, ri);
 669
 670         return (0);
 671 }
 672
 673 static int
 674 p6_intr(int cpu, struct trapframe *tf)
 675 {
 676         int error, retval, ri;
 677         uint32_t perf0cfg;
 678         struct pmc *pm;
 679         struct p6_cpu *pc;
 680         pmc_value_t v;
 681
 682         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 683             ("[p6,%d] CPU %d out of range", __LINE__, cpu));
 684
 685         retval = 0;
 686         pc = p6_pcpu[cpu];
 687
 688         /* stop both PMCs */
 689         perf0cfg = rdmsr(P6_MSR_EVSEL0);
 690         wrmsr(P6_MSR_EVSEL0, perf0cfg & ~P6_EVSEL_EN);
 691
 692         for (ri = 0; ri < P6_NPMCS; ri++) {
 693
 694                 if ((pm = pc->pc_p6pmcs[ri].phw_pmc) == NULL ||
 695                     !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
 696                         continue;
 697                 }
 698
 699                 if (!P6_PMC_HAS_OVERFLOWED(ri))
 700                         continue;
 701
 702                 retval = 1;
 703
 704                 if (pm->pm_state != PMC_STATE_RUNNING)
 705                         continue;
 706
 707                 error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
 708                     TRAPF_USERMODE(tf));
 709                 if (error)
 710                         P6_MARK_STOPPED(pc,ri);
 711
 712                 /* reload sampling count */
 713                 v = pm->pm_sc.pm_reloadcount;
 714                 wrmsr(P6_MSR_PERFCTR0 + ri,
 715                     P6_RELOAD_COUNT_TO_PERFCTR_VALUE(v));
 716
 717         }
 718
 719         /*
 720          * On P6 processors, the LAPIC needs to have its PMC interrupt
 721          * unmasked after a PMC interrupt.
 722          */
 723         if (retval)
 724                 lapic_reenable_pmc();
 725
 726         atomic_add_int(retval ? &pmc_stats.pm_intr_processed :
 727             &pmc_stats.pm_intr_ignored, 1);
 728
 729         /* restart counters that can be restarted */
 730         P6_SYNC_CTR_STATE(pc);
 731
 732         return (retval);
 733 }
 734
 735 static int
 736 p6_describe(int cpu, int ri, struct pmc_info *pi,
 737     struct pmc **ppmc)
 738 {
 739         int error;
 740         size_t copied;
 741         struct pmc_hw *phw;
 742         struct p6pmc_descr *pd;
 743
 744         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 745             ("[p6,%d] illegal CPU %d", __LINE__, cpu));
 746         KASSERT(ri >= 0 && ri < P6_NPMCS,
 747             ("[p6,%d] row-index %d out of range", __LINE__, ri));
 748
 749         phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
 750         pd  = &p6_pmcdesc[ri];
 751
 752         KASSERT(phw == &p6_pcpu[cpu]->pc_p6pmcs[ri],
 753             ("[p6,%d] phw mismatch", __LINE__));
 754
 755         if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name,
 756                  PMC_NAME_MAX, &copied)) != 0)
 757                 return (error);
 758
 759         pi->pm_class = pd->pm_descr.pd_class;
 760
 761         if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
 762                 pi->pm_enabled = TRUE;
 763                 *ppmc          = phw->phw_pmc;
 764         } else {
 765                 pi->pm_enabled = FALSE;
 766                 *ppmc          = NULL;
 767         }
 768
 769         return (0);
 770 }
 771
 772 static int
 773 p6_get_msr(int ri, uint32_t *msr)
 774 {
 775         KASSERT(ri >= 0 && ri < P6_NPMCS,
 776             ("[p6,%d ri %d out of range", __LINE__, ri));
 777
 778         *msr = p6_pmcdesc[ri].pm_pmc_msr - P6_MSR_PERFCTR0;
 779
 780         return (0);
 781 }
 782
 783 int
 784 pmc_p6_initialize(struct pmc_mdep *md, int ncpus)
 785 {
 786         struct pmc_classdep *pcd;
 787
 788         KASSERT(cpu_vendor_id == CPU_VENDOR_INTEL,
 789             ("[p6,%d] Initializing non-intel processor", __LINE__));
 790
 791         PMCDBG(MDP,INI,1, "%s", "p6-initialize");
 792
 793         /* Allocate space for pointers to per-cpu descriptors. */
 794         p6_pcpu = malloc(sizeof(struct p6_cpu **) * ncpus, M_PMC,
 795             M_ZERO|M_WAITOK);
 796
 797         /* Fill in the class dependent descriptor. */
 798         pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P6];
 799
 800         switch (md->pmd_cputype) {
 801
 802                 /*
 803                  * P6 Family Processors
 804                  */
 805         case PMC_CPU_INTEL_P6:
 806         case PMC_CPU_INTEL_CL:
 807         case PMC_CPU_INTEL_PII:
 808         case PMC_CPU_INTEL_PIII:
 809         case PMC_CPU_INTEL_PM:
 810
 811                 p6_cputype = md->pmd_cputype;
 812
 813                 pcd->pcd_caps           = P6_PMC_CAPS;
 814                 pcd->pcd_class          = PMC_CLASS_P6;
 815                 pcd->pcd_num            = P6_NPMCS;
 816                 pcd->pcd_ri             = md->pmd_npmc;
 817                 pcd->pcd_width          = 40;
 818
 819                 pcd->pcd_allocate_pmc   = p6_allocate_pmc;
 820                 pcd->pcd_config_pmc     = p6_config_pmc;
 821                 pcd->pcd_describe       = p6_describe;
 822                 pcd->pcd_get_config     = p6_get_config;
 823                 pcd->pcd_get_msr        = p6_get_msr;
 824                 pcd->pcd_pcpu_fini      = p6_pcpu_fini;
 825                 pcd->pcd_pcpu_init      = p6_pcpu_init;
 826                 pcd->pcd_read_pmc       = p6_read_pmc;
 827                 pcd->pcd_release_pmc    = p6_release_pmc;
 828                 pcd->pcd_start_pmc      = p6_start_pmc;
 829                 pcd->pcd_stop_pmc       = p6_stop_pmc;
 830                 pcd->pcd_write_pmc      = p6_write_pmc;
 831
 832                 md->pmd_pcpu_fini       = NULL;
 833                 md->pmd_pcpu_init       = NULL;
 834                 md->pmd_intr            = p6_intr;
 835
 836                 md->pmd_npmc           += P6_NPMCS;
 837
 838                 break;
 839
 840         default:
 841                 KASSERT(0,("[p6,%d] Unknown CPU type", __LINE__));
 842                 return ENOSYS;
 843         }
 844
 845         return (0);
 846 }
 847
 848 void
 849 pmc_p6_finalize(struct pmc_mdep *md)
 850 {
 851 #if     defined(INVARIANTS)
 852         int i, ncpus;
 853 #endif
 854
 855         KASSERT(p6_pcpu != NULL, ("[p6,%d] NULL p6_pcpu", __LINE__));
 856
 857 #if     defined(INVARIANTS)
 858         ncpus = pmc_cpu_max();
 859         for (i = 0; i < ncpus; i++)
 860                 KASSERT(p6_pcpu[i] == NULL, ("[p6,%d] non-null pcpu %d",
 861                     __LINE__, i));
 862 #endif
 863
 864         free(p6_pcpu, M_PMC);
 865         p6_pcpu = NULL;
 866 }