sys/dev/hwpmc/hwpmc_ppro.c

   1 /*-
   2  * Copyright (c) 2003-2005 Joseph Koshy
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #include <sys/cdefs.h>
  28 __FBSDID("$FreeBSD$");
  29
  30 #include <sys/param.h>
  31 #include <sys/lock.h>
  32 #include <sys/mutex.h>
  33 #include <sys/pmc.h>
  34 #include <sys/pmckern.h>
  35 #include <sys/smp.h>
  36 #include <sys/systm.h>
  37
  38 #include <machine/cpufunc.h>
  39 #include <machine/md_var.h>
  40 #include <machine/pmc_mdep.h>
  41 #include <machine/specialreg.h>
  42
  43 /*
  44  * PENTIUM PRO SUPPORT
  45  *
  46  * Quirks:
  47  *
  48  * - Both PMCs are enabled by a single bit P6_EVSEL_EN in performance
  49  *   counter '0'.  This bit needs to be '1' if any of the two
  50  *   performance counters are in use.  Perf counters can also be
  51  *   switched off by writing zeros to their EVSEL register.
  52  *
  53  * - While the width of these counters is 40 bits, we do not appear to
  54  *   have a way of writing 40 bits to the counter MSRs.  A WRMSR
  55  *   instruction will sign extend bit 31 of the value being written to
  56  *   the perf counter -- a value of 0x80000000 written to an perf
  57  *   counter register will be sign extended to 0xFF80000000.
  58  *
  59  *   This quirk primarily affects thread-mode PMCs in counting mode, as
  60  *   these PMCs read and write PMC registers at every context switch.
  61  */
  62
  63 struct p6pmc_descr {
  64         struct pmc_descr pm_descr; /* common information */
  65         uint32_t        pm_pmc_msr;
  66         uint32_t        pm_evsel_msr;
  67 };
  68
  69 static struct p6pmc_descr p6_pmcdesc[P6_NPMCS] = {
  70
  71         /* TSC */
  72         {
  73                 .pm_descr =
  74                 {
  75                         .pd_name  = "TSC",
  76                         .pd_class = PMC_CLASS_TSC,
  77                         .pd_caps  = PMC_CAP_READ,
  78                         .pd_width = 64
  79                 },
  80                 .pm_pmc_msr   = 0x10,
  81                 .pm_evsel_msr = ~0
  82         },
  83
  84 #define P6_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | PMC_CAP_SYSTEM | \
  85     PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE |    \
  86     PMC_CAP_INVERT | PMC_CAP_QUALIFIER)
  87
  88         /* PMC 0 */
  89         {
  90                 .pm_descr =
  91                 {
  92                         .pd_name  ="P6-0",
  93                         .pd_class = PMC_CLASS_P6,
  94                         .pd_caps  = P6_PMC_CAPS,
  95                         .pd_width = 40
  96                 },
  97                 .pm_pmc_msr   = P6_MSR_PERFCTR0,
  98                 .pm_evsel_msr = P6_MSR_EVSEL0
  99         },
 100
 101         /* PMC 1 */
 102         {
 103                 .pm_descr =
 104                 {
 105                         .pd_name  ="P6-1",
 106                         .pd_class = PMC_CLASS_P6,
 107                         .pd_caps  = P6_PMC_CAPS,
 108                         .pd_width = 40
 109                 },
 110                 .pm_pmc_msr   = P6_MSR_PERFCTR1,
 111                 .pm_evsel_msr = P6_MSR_EVSEL1
 112         }
 113 };
 114
 115 static enum pmc_cputype p6_cputype;
 116
 117 /*
 118  * P6 Event descriptor
 119  */
 120
 121 struct p6_event_descr {
 122         const enum pmc_event pm_event;
 123         uint32_t             pm_evsel;
 124         uint32_t             pm_flags;
 125         uint32_t             pm_unitmask;
 126 };
 127
 128 static const struct p6_event_descr p6_events[] = {
 129
 130 #define P6_EVDESCR(NAME, EVSEL, FLAGS, UMASK)   \
 131         {                                       \
 132                 .pm_event = PMC_EV_P6_##NAME,   \
 133                 .pm_evsel = (EVSEL),            \
 134                 .pm_flags = (FLAGS),            \
 135                 .pm_unitmask = (UMASK)          \
 136         }
 137
 138 #define P6F_P6          (1 << PMC_CPU_INTEL_P6)
 139 #define P6F_CL          (1 << PMC_CPU_INTEL_CL)
 140 #define P6F_PII         (1 << PMC_CPU_INTEL_PII)
 141 #define P6F_PIII        (1 << PMC_CPU_INTEL_PIII)
 142 #define P6F_PM          (1 << PMC_CPU_INTEL_PM)
 143 #define P6F_CTR0        0x0001
 144 #define P6F_CTR1        0x0002
 145 #define P6F_ALL_CPUS    (P6F_P6 | P6F_PII | P6F_CL | P6F_PIII | P6F_PM)
 146 #define P6F_ALL_CTRS    (P6F_CTR0 | P6F_CTR1)
 147 #define P6F_ALL         (P6F_ALL_CPUS | P6F_ALL_CTRS)
 148
 149 #define P6_EVENT_VALID_FOR_CPU(P,CPU)   ((P)->pm_flags & (1 << (CPU)))
 150 #define P6_EVENT_VALID_FOR_CTR(P,CTR)   ((P)->pm_flags & (1 << (CTR)))
 151
 152 P6_EVDESCR(DATA_MEM_REFS,               0x43, P6F_ALL, 0x00),
 153 P6_EVDESCR(DCU_LINES_IN,                0x45, P6F_ALL, 0x00),
 154 P6_EVDESCR(DCU_M_LINES_IN,              0x46, P6F_ALL, 0x00),
 155 P6_EVDESCR(DCU_M_LINES_OUT,             0x47, P6F_ALL, 0x00),
 156 P6_EVDESCR(DCU_MISS_OUTSTANDING,        0x47, P6F_ALL, 0x00),
 157 P6_EVDESCR(IFU_FETCH,                   0x80, P6F_ALL, 0x00),
 158 P6_EVDESCR(IFU_FETCH_MISS,              0x81, P6F_ALL, 0x00),
 159 P6_EVDESCR(ITLB_MISS,                   0x85, P6F_ALL, 0x00),
 160 P6_EVDESCR(IFU_MEM_STALL,               0x86, P6F_ALL, 0x00),
 161 P6_EVDESCR(ILD_STALL,                   0x87, P6F_ALL, 0x00),
 162 P6_EVDESCR(L2_IFETCH,                   0x28, P6F_ALL, 0x0F),
 163 P6_EVDESCR(L2_LD,                       0x29, P6F_ALL, 0x0F),
 164 P6_EVDESCR(L2_ST,                       0x2A, P6F_ALL, 0x0F),
 165 P6_EVDESCR(L2_LINES_IN,                 0x24, P6F_ALL, 0x0F),
 166 P6_EVDESCR(L2_LINES_OUT,                0x26, P6F_ALL, 0x0F),
 167 P6_EVDESCR(L2_M_LINES_INM,              0x25, P6F_ALL, 0x00),
 168 P6_EVDESCR(L2_M_LINES_OUTM,             0x27, P6F_ALL, 0x0F),
 169 P6_EVDESCR(L2_RQSTS,                    0x2E, P6F_ALL, 0x0F),
 170 P6_EVDESCR(L2_ADS,                      0x21, P6F_ALL, 0x00),
 171 P6_EVDESCR(L2_DBUS_BUSY,                0x22, P6F_ALL, 0x00),
 172 P6_EVDESCR(L2_DBUS_BUSY_RD,             0x23, P6F_ALL, 0x00),
 173 P6_EVDESCR(BUS_DRDY_CLOCKS,             0x62, P6F_ALL, 0x20),
 174 P6_EVDESCR(BUS_LOCK_CLOCKS,             0x63, P6F_ALL, 0x20),
 175 P6_EVDESCR(BUS_REQ_OUTSTANDING,         0x60, P6F_ALL, 0x00),
 176 P6_EVDESCR(BUS_TRAN_BRD,                0x65, P6F_ALL, 0x20),
 177 P6_EVDESCR(BUS_TRAN_RFO,                0x66, P6F_ALL, 0x20),
 178 P6_EVDESCR(BUS_TRANS_WB,                0x67, P6F_ALL, 0x20),
 179 P6_EVDESCR(BUS_TRAN_IFETCH,             0x68, P6F_ALL, 0x20),
 180 P6_EVDESCR(BUS_TRAN_INVAL,              0x69, P6F_ALL, 0x20),
 181 P6_EVDESCR(BUS_TRAN_PWR,                0x6A, P6F_ALL, 0x20),
 182 P6_EVDESCR(BUS_TRANS_P,                 0x6B, P6F_ALL, 0x20),
 183 P6_EVDESCR(BUS_TRANS_IO,                0x6C, P6F_ALL, 0x20),
 184 P6_EVDESCR(BUS_TRAN_DEF,                0x6D, P6F_ALL, 0x20),
 185 P6_EVDESCR(BUS_TRAN_BURST,              0x6E, P6F_ALL, 0x20),
 186 P6_EVDESCR(BUS_TRAN_ANY,                0x70, P6F_ALL, 0x20),
 187 P6_EVDESCR(BUS_TRAN_MEM,                0x6F, P6F_ALL, 0x20),
 188 P6_EVDESCR(BUS_DATA_RCV,                0x64, P6F_ALL, 0x00),
 189 P6_EVDESCR(BUS_BNR_DRV,                 0x61, P6F_ALL, 0x00),
 190 P6_EVDESCR(BUS_HIT_DRV,                 0x7A, P6F_ALL, 0x00),
 191 P6_EVDESCR(BUS_HITM_DRV,                0x7B, P6F_ALL, 0x00),
 192 P6_EVDESCR(BUS_SNOOP_STALL,             0x7E, P6F_ALL, 0x00),
 193 P6_EVDESCR(FLOPS,                       0xC1, P6F_ALL_CPUS | P6F_CTR0, 0x00),
 194 P6_EVDESCR(FP_COMPS_OPS_EXE,            0x10, P6F_ALL_CPUS | P6F_CTR0, 0x00),
 195 P6_EVDESCR(FP_ASSIST,                   0x11, P6F_ALL_CPUS | P6F_CTR1, 0x00),
 196 P6_EVDESCR(MUL,                         0x12, P6F_ALL_CPUS | P6F_CTR1, 0x00),
 197 P6_EVDESCR(DIV,                         0x13, P6F_ALL_CPUS | P6F_CTR1, 0x00),
 198 P6_EVDESCR(CYCLES_DIV_BUSY,             0x14, P6F_ALL_CPUS | P6F_CTR0, 0x00),
 199 P6_EVDESCR(LD_BLOCKS,                   0x03, P6F_ALL, 0x00),
 200 P6_EVDESCR(SB_DRAINS,                   0x04, P6F_ALL, 0x00),
 201 P6_EVDESCR(MISALIGN_MEM_REF,            0x05, P6F_ALL, 0x00),
 202 P6_EVDESCR(EMON_KNI_PREF_DISPATCHED,    0x07, P6F_PIII | P6F_ALL_CTRS, 0x03),
 203 P6_EVDESCR(EMON_KNI_PREF_MISS,          0x4B, P6F_PIII | P6F_ALL_CTRS, 0x03),
 204 P6_EVDESCR(INST_RETIRED,                0xC0, P6F_ALL, 0x00),
 205 P6_EVDESCR(UOPS_RETIRED,                0xC2, P6F_ALL, 0x00),
 206 P6_EVDESCR(INST_DECODED,                0xD0, P6F_ALL, 0x00),
 207 P6_EVDESCR(EMON_KNI_INST_RETIRED,       0xD8, P6F_PIII | P6F_ALL_CTRS, 0x01),
 208 P6_EVDESCR(EMON_KNI_COMP_INST_RET,      0xD9, P6F_PIII | P6F_ALL_CTRS, 0x01),
 209 P6_EVDESCR(HW_INT_RX,                   0xC8, P6F_ALL, 0x00),
 210 P6_EVDESCR(CYCLES_INT_MASKED,           0xC6, P6F_ALL, 0x00),
 211 P6_EVDESCR(CYCLES_INT_PENDING_AND_MASKED, 0xC7, P6F_ALL, 0x00),
 212 P6_EVDESCR(BR_INST_RETIRED,             0xC4, P6F_ALL, 0x00),
 213 P6_EVDESCR(BR_MISS_PRED_RETIRED,        0xC5, P6F_ALL, 0x00),
 214 P6_EVDESCR(BR_TAKEN_RETIRED,            0xC9, P6F_ALL, 0x00),
 215 P6_EVDESCR(BR_MISS_PRED_TAKEN_RET,      0xCA, P6F_ALL, 0x00),
 216 P6_EVDESCR(BR_INST_DECODED,             0xE0, P6F_ALL, 0x00),
 217 P6_EVDESCR(BTB_MISSES,                  0xE2, P6F_ALL, 0x00),
 218 P6_EVDESCR(BR_BOGUS,                    0xE4, P6F_ALL, 0x00),
 219 P6_EVDESCR(BACLEARS,                    0xE6, P6F_ALL, 0x00),
 220 P6_EVDESCR(RESOURCE_STALLS,             0xA2, P6F_ALL, 0x00),
 221 P6_EVDESCR(PARTIAL_RAT_STALLS,          0xD2, P6F_ALL, 0x00),
 222 P6_EVDESCR(SEGMENT_REG_LOADS,           0x06, P6F_ALL, 0x00),
 223 P6_EVDESCR(CPU_CLK_UNHALTED,            0x79, P6F_ALL, 0x00),
 224 P6_EVDESCR(MMX_INSTR_EXEC,              0xB0,
 225                         P6F_ALL_CTRS | P6F_CL | P6F_PII, 0x00),
 226 P6_EVDESCR(MMX_SAT_INSTR_EXEC,          0xB1,
 227                         P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x00),
 228 P6_EVDESCR(MMX_UOPS_EXEC,               0xB2,
 229                         P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x0F),
 230 P6_EVDESCR(MMX_INSTR_TYPE_EXEC,         0xB3,
 231                         P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x3F),
 232 P6_EVDESCR(FP_MMX_TRANS,                0xCC,
 233                         P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x01),
 234 P6_EVDESCR(MMX_ASSIST,                  0xCD,
 235                         P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x00),
 236 P6_EVDESCR(MMX_INSTR_RET,               0xCE, P6F_ALL_CTRS | P6F_PII, 0x00),
 237 P6_EVDESCR(SEG_RENAME_STALLS,           0xD4,
 238                         P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x0F),
 239 P6_EVDESCR(SEG_REG_RENAMES,             0xD5,
 240                         P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x0F),
 241 P6_EVDESCR(RET_SEG_RENAMES,             0xD6,
 242                         P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x00),
 243 P6_EVDESCR(EMON_EST_TRANS,              0x58, P6F_ALL_CTRS | P6F_PM, 0x02),
 244 P6_EVDESCR(EMON_THERMAL_TRIP,           0x59, P6F_ALL_CTRS | P6F_PM, 0x00),
 245 P6_EVDESCR(BR_INST_EXEC,                0x88, P6F_ALL_CTRS | P6F_PM, 0x00),
 246 P6_EVDESCR(BR_MISSP_EXEC,               0x89, P6F_ALL_CTRS | P6F_PM, 0x00),
 247 P6_EVDESCR(BR_BAC_MISSP_EXEC,           0x8A, P6F_ALL_CTRS | P6F_PM, 0x00),
 248 P6_EVDESCR(BR_CND_EXEC,                 0x8B, P6F_ALL_CTRS | P6F_PM, 0x00),
 249 P6_EVDESCR(BR_CND_MISSP_EXEC,           0x8C, P6F_ALL_CTRS | P6F_PM, 0x00),
 250 P6_EVDESCR(BR_IND_EXEC,                 0x8D, P6F_ALL_CTRS | P6F_PM, 0x00),
 251 P6_EVDESCR(BR_IND_MISSP_EXEC,           0x8E, P6F_ALL_CTRS | P6F_PM, 0x00),
 252 P6_EVDESCR(BR_RET_EXEC,                 0x8F, P6F_ALL_CTRS | P6F_PM, 0x00),
 253 P6_EVDESCR(BR_RET_MISSP_EXEC,           0x90, P6F_ALL_CTRS | P6F_PM, 0x00),
 254 P6_EVDESCR(BR_RET_BAC_MISSP_EXEC,       0x91, P6F_ALL_CTRS | P6F_PM, 0x00),
 255 P6_EVDESCR(BR_CALL_EXEC,                0x92, P6F_ALL_CTRS | P6F_PM, 0x00),
 256 P6_EVDESCR(BR_CALL_MISSP_EXEC,          0x93, P6F_ALL_CTRS | P6F_PM, 0x00),
 257 P6_EVDESCR(BR_IND_CALL_EXEC,            0x94, P6F_ALL_CTRS | P6F_PM, 0x00),
 258 P6_EVDESCR(EMON_SIMD_INSTR_RETIRED,     0xCE, P6F_ALL_CTRS | P6F_PM, 0x00),
 259 P6_EVDESCR(EMON_SYNCH_UOPS,             0xD3, P6F_ALL_CTRS | P6F_PM, 0x00),
 260 P6_EVDESCR(EMON_ESP_UOPS,               0xD7, P6F_ALL_CTRS | P6F_PM, 0x00),
 261 P6_EVDESCR(EMON_FUSED_UOPS_RET,         0xDA, P6F_ALL_CTRS | P6F_PM, 0x03),
 262 P6_EVDESCR(EMON_UNFUSION,               0xDB, P6F_ALL_CTRS | P6F_PM, 0x00),
 263 P6_EVDESCR(EMON_PREF_RQSTS_UP,          0xF0, P6F_ALL_CTRS | P6F_PM, 0x00),
 264 P6_EVDESCR(EMON_PREF_RQSTS_DN,          0xD8, P6F_ALL_CTRS | P6F_PM, 0x00),
 265 P6_EVDESCR(EMON_SSE_SSE2_INST_RETIRED,  0xD8, P6F_ALL_CTRS | P6F_PM, 0x03),
 266 P6_EVDESCR(EMON_SSE_SSE2_COMP_INST_RETIRED, 0xD9, P6F_ALL_CTRS | P6F_PM, 0x03)
 267
 268 #undef  P6_EVDESCR
 269 };
 270
 271 #define P6_NEVENTS      (PMC_EV_P6_LAST - PMC_EV_P6_FIRST + 1)
 272
 273 static const struct p6_event_descr *
 274 p6_find_event(enum pmc_event ev)
 275 {
 276         int n;
 277
 278         for (n = 0; n < P6_NEVENTS; n++)
 279                 if (p6_events[n].pm_event == ev)
 280                         break;
 281         if (n == P6_NEVENTS)
 282                 return NULL;
 283         return &p6_events[n];
 284 }
 285
 286 /*
 287  * Per-CPU data structure for P6 class CPUs
 288  *
 289  * [common stuff]
 290  * [flags for maintaining PMC start/stop state]
 291  * [3 struct pmc_hw pointers]
 292  * [3 struct pmc_hw structures]
 293  */
 294
 295 struct p6_cpu {
 296         struct pmc_cpu  pc_common;
 297         struct pmc_hw   *pc_hwpmcs[P6_NPMCS];
 298         struct pmc_hw   pc_p6pmcs[P6_NPMCS];
 299         uint32_t        pc_state;
 300 };
 301
 302 /*
 303  * If CTR1 is active, we need to keep the 'EN' bit if CTR0 set,
 304  * with the rest of CTR0 being zero'ed out.
 305  */
 306 #define P6_SYNC_CTR_STATE(PC) do {                              \
 307                 uint32_t _config, _enable;                      \
 308                 _enable = 0;                                    \
 309                 if ((PC)->pc_state & 0x02)                      \
 310                         _enable |= P6_EVSEL_EN;                 \
 311                 if ((PC)->pc_state & 0x01)                      \
 312                         _config = rdmsr(P6_MSR_EVSEL0) |        \
 313                             P6_EVSEL_EN;                        \
 314                 else                                            \
 315                         _config = 0;                            \
 316                 wrmsr(P6_MSR_EVSEL0, _config | _enable);        \
 317         } while (0)
 318
 319 #define P6_MARK_STARTED(PC,RI) do {                             \
 320                 (PC)->pc_state |= (1 << ((RI)-1));              \
 321         } while (0)
 322
 323 #define P6_MARK_STOPPED(PC,RI) do {                             \
 324                 (PC)->pc_state &= ~(1<< ((RI)-1));              \
 325         } while (0)
 326
 327 static int
 328 p6_init(int cpu)
 329 {
 330         int n;
 331         struct p6_cpu *pcs;
 332         struct pmc_hw *phw;
 333
 334         KASSERT(cpu >= 0 && cpu < mp_ncpus,
 335             ("[p6,%d] bad cpu %d", __LINE__, cpu));
 336
 337         PMCDBG(MDP,INI,0,"p6-init cpu=%d", cpu);
 338
 339         MALLOC(pcs, struct p6_cpu *, sizeof(struct p6_cpu), M_PMC,
 340             M_WAITOK|M_ZERO);
 341
 342         phw = pcs->pc_p6pmcs;
 343
 344         for (n = 0; n < P6_NPMCS; n++, phw++) {
 345                 phw->phw_state   = PMC_PHW_FLAG_IS_ENABLED |
 346                     PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n);
 347                 phw->phw_pmc     = NULL;
 348                 pcs->pc_hwpmcs[n] = phw;
 349         }
 350
 351         /* Mark the TSC as shareable */
 352         pcs->pc_hwpmcs[0]->phw_state |= PMC_PHW_FLAG_IS_SHAREABLE;
 353
 354         pmc_pcpu[cpu] = (struct pmc_cpu *) pcs;
 355
 356         return 0;
 357 }
 358
 359 static int
 360 p6_cleanup(int cpu)
 361 {
 362         struct pmc_cpu *pcs;
 363
 364         KASSERT(cpu >= 0 && cpu < mp_ncpus,
 365             ("[p6,%d] bad cpu %d", __LINE__, cpu));
 366
 367         PMCDBG(MDP,INI,0,"p6-cleanup cpu=%d", cpu);
 368
 369         if ((pcs = pmc_pcpu[cpu]) != NULL)
 370                 FREE(pcs, M_PMC);
 371         pmc_pcpu[cpu] = NULL;
 372
 373         return 0;
 374 }
 375
 376 static int
 377 p6_switch_in(struct pmc_cpu *pc, struct pmc_process *pp)
 378 {
 379         (void) pc;
 380
 381         PMCDBG(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp,
 382             pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS);
 383
 384         /* allow the RDPMC instruction if needed */
 385         if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS)
 386                 load_cr4(rcr4() | CR4_PCE);
 387
 388         PMCDBG(MDP,SWI,1, "cr4=0x%x", rcr4());
 389
 390         return 0;
 391 }
 392
 393 static int
 394 p6_switch_out(struct pmc_cpu *pc, struct pmc_process *pp)
 395 {
 396         (void) pc;
 397         (void) pp;              /* can be NULL */
 398
 399         PMCDBG(MDP,SWO,1, "pc=%p pp=%p cr4=0x%x", pc, pp, rcr4());
 400
 401         /* always turn off the RDPMC instruction */
 402         load_cr4(rcr4() & ~CR4_PCE);
 403
 404         return 0;
 405 }
 406
 407 static int
 408 p6_read_pmc(int cpu, int ri, pmc_value_t *v)
 409 {
 410         struct pmc_hw *phw;
 411         struct pmc *pm;
 412         struct p6pmc_descr *pd;
 413         pmc_value_t tmp;
 414
 415         phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
 416         pm  = phw->phw_pmc;
 417         pd  = &p6_pmcdesc[ri];
 418
 419         KASSERT(pm,
 420             ("[p6,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
 421
 422         if (pd->pm_descr.pd_class == PMC_CLASS_TSC) {
 423                 *v = rdtsc();
 424                 return 0;
 425         }
 426
 427         tmp = rdmsr(pd->pm_pmc_msr) & P6_PERFCTR_READ_MASK;
 428         if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
 429                 *v = P6_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp);
 430         else
 431                 *v = tmp;
 432
 433         PMCDBG(MDP,REA,1, "p6-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
 434             pd->pm_pmc_msr, *v);
 435
 436         return 0;
 437 }
 438
 439 static int
 440 p6_write_pmc(int cpu, int ri, pmc_value_t v)
 441 {
 442         struct pmc_hw *phw;
 443         struct pmc *pm;
 444         struct p6pmc_descr *pd;
 445
 446         phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
 447         pm  = phw->phw_pmc;
 448         pd  = &p6_pmcdesc[ri];
 449
 450         KASSERT(pm,
 451             ("[p6,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
 452
 453         if (pd->pm_descr.pd_class == PMC_CLASS_TSC)
 454                 return 0;
 455
 456         PMCDBG(MDP,WRI,1, "p6-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri,
 457             pd->pm_pmc_msr, v);
 458
 459         if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
 460                 v = P6_RELOAD_COUNT_TO_PERFCTR_VALUE(v);
 461
 462         wrmsr(pd->pm_pmc_msr, v & P6_PERFCTR_WRITE_MASK);
 463
 464         return 0;
 465 }
 466
 467 static int
 468 p6_config_pmc(int cpu, int ri, struct pmc *pm)
 469 {
 470         struct pmc_hw *phw;
 471
 472         PMCDBG(MDP,CFG,1, "p6-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
 473
 474         phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
 475         phw->phw_pmc = pm;
 476
 477         return 0;
 478 }
 479
 480 /*
 481  * Retrieve a configured PMC pointer from hardware state.
 482  */
 483
 484 static int
 485 p6_get_config(int cpu, int ri, struct pmc **ppm)
 486 {
 487         *ppm = pmc_pcpu[cpu]->pc_hwpmcs[ri]->phw_pmc;
 488
 489         return 0;
 490 }
 491
 492
 493 /*
 494  * A pmc may be allocated to a given row index if:
 495  * - the event is valid for this CPU
 496  * - the event is valid for this counter index
 497  */
 498
 499 static int
 500 p6_allocate_pmc(int cpu, int ri, struct pmc *pm,
 501     const struct pmc_op_pmcallocate *a)
 502 {
 503         uint32_t allowed_unitmask, caps, config, unitmask;
 504         const struct p6pmc_descr *pd;
 505         const struct p6_event_descr *pevent;
 506         enum pmc_event ev;
 507
 508         (void) cpu;
 509
 510         KASSERT(cpu >= 0 && cpu < mp_ncpus,
 511             ("[p4,%d] illegal CPU %d", __LINE__, cpu));
 512         KASSERT(ri >= 0 && ri < P6_NPMCS,
 513             ("[p4,%d] illegal row-index value %d", __LINE__, ri));
 514
 515         pd = &p6_pmcdesc[ri];
 516
 517         PMCDBG(MDP,ALL,1, "p6-allocate ri=%d class=%d pmccaps=0x%x "
 518             "reqcaps=0x%x", ri, pd->pm_descr.pd_class, pd->pm_descr.pd_caps,
 519             pm->pm_caps);
 520
 521         /* check class */
 522         if (pd->pm_descr.pd_class != a->pm_class)
 523                 return EINVAL;
 524
 525         /* check requested capabilities */
 526         caps = a->pm_caps;
 527         if ((pd->pm_descr.pd_caps & caps) != caps)
 528                 return EPERM;
 529
 530         if (pd->pm_descr.pd_class == PMC_CLASS_TSC) {
 531                 /* TSC's are always allocated in system-wide counting mode */
 532                 if (a->pm_ev != PMC_EV_TSC_TSC ||
 533                     a->pm_mode != PMC_MODE_SC)
 534                         return EINVAL;
 535                 return 0;
 536         }
 537
 538         /*
 539          * P6 class events
 540          */
 541
 542         ev = pm->pm_event;
 543
 544         if (ev < PMC_EV_P6_FIRST || ev > PMC_EV_P6_LAST)
 545                 return EINVAL;
 546
 547         if ((pevent = p6_find_event(ev)) == NULL)
 548                 return ESRCH;
 549
 550         if (!P6_EVENT_VALID_FOR_CPU(pevent, p6_cputype) ||
 551             !P6_EVENT_VALID_FOR_CTR(pevent, (ri-1)))
 552                 return EINVAL;
 553
 554         /* For certain events, Pentium M differs from the stock P6 */
 555         allowed_unitmask = 0;
 556         if (p6_cputype == PMC_CPU_INTEL_PM) {
 557                 if (ev == PMC_EV_P6_L2_LD || ev == PMC_EV_P6_L2_LINES_IN ||
 558                     ev == PMC_EV_P6_L2_LINES_OUT)
 559                         allowed_unitmask = P6_EVSEL_TO_UMASK(0x3F);
 560                 else if (ev == PMC_EV_P6_L2_M_LINES_OUTM)
 561                         allowed_unitmask = P6_EVSEL_TO_UMASK(0x30);
 562         } else
 563                 allowed_unitmask = P6_EVSEL_TO_UMASK(pevent->pm_unitmask);
 564
 565         unitmask = a->pm_md.pm_ppro.pm_ppro_config & P6_EVSEL_UMASK_MASK;
 566         if (unitmask & ~allowed_unitmask) /* disallow reserved bits */
 567                 return EINVAL;
 568
 569         if (ev == PMC_EV_P6_MMX_UOPS_EXEC) /* hardcoded mask */
 570                 unitmask = P6_EVSEL_TO_UMASK(0x0F);
 571
 572         config = 0;
 573
 574         config |= P6_EVSEL_EVENT_SELECT(pevent->pm_evsel);
 575
 576         if (unitmask & (caps & PMC_CAP_QUALIFIER))
 577                 config |= unitmask;
 578
 579         if (caps & PMC_CAP_THRESHOLD)
 580                 config |= a->pm_md.pm_ppro.pm_ppro_config &
 581                     P6_EVSEL_CMASK_MASK;
 582
 583         /* set at least one of the 'usr' or 'os' caps */
 584         if (caps & PMC_CAP_USER)
 585                 config |= P6_EVSEL_USR;
 586         if (caps & PMC_CAP_SYSTEM)
 587                 config |= P6_EVSEL_OS;
 588         if ((caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0)
 589                 config |= (P6_EVSEL_USR|P6_EVSEL_OS);
 590
 591         if (caps & PMC_CAP_EDGE)
 592                 config |= P6_EVSEL_E;
 593         if (caps & PMC_CAP_INVERT)
 594                 config |= P6_EVSEL_INV;
 595         if (caps & PMC_CAP_INTERRUPT)
 596                 config |= P6_EVSEL_INT;
 597
 598         pm->pm_md.pm_ppro.pm_ppro_evsel = config;
 599
 600         PMCDBG(MDP,ALL,2, "p6-allocate config=0x%x", config);
 601
 602         return 0;
 603 }
 604
 605 static int
 606 p6_release_pmc(int cpu, int ri, struct pmc *pm)
 607 {
 608         struct pmc_hw *phw;
 609
 610         (void) pm;
 611
 612         PMCDBG(MDP,REL,1, "p6-release cpu=%d ri=%d pm=%p", cpu, ri, pm);
 613
 614         KASSERT(cpu >= 0 && cpu < mp_ncpus,
 615             ("[p6,%d] illegal CPU value %d", __LINE__, cpu));
 616         KASSERT(ri >= 0 && ri < P6_NPMCS,
 617             ("[p6,%d] illegal row-index %d", __LINE__, ri));
 618
 619         phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
 620
 621         KASSERT(phw->phw_pmc == NULL,
 622             ("[p6,%d] PHW pmc %p != pmc %p", __LINE__, phw->phw_pmc, pm));
 623
 624         return 0;
 625 }
 626
 627 static int
 628 p6_start_pmc(int cpu, int ri)
 629 {
 630         uint32_t config;
 631         struct pmc *pm;
 632         struct p6_cpu *pc;
 633         struct pmc_hw *phw;
 634         const struct p6pmc_descr *pd;
 635
 636         KASSERT(cpu >= 0 && cpu < mp_ncpus,
 637             ("[p6,%d] illegal CPU value %d", __LINE__, cpu));
 638         KASSERT(ri >= 0 && ri < P6_NPMCS,
 639             ("[p6,%d] illegal row-index %d", __LINE__, ri));
 640
 641         pc  = (struct p6_cpu *) pmc_pcpu[cpu];
 642         phw = pc->pc_common.pc_hwpmcs[ri];
 643         pm  = phw->phw_pmc;
 644         pd  = &p6_pmcdesc[ri];
 645
 646         KASSERT(pm,
 647             ("[p6,%d] starting cpu%d,ri%d with no pmc configured",
 648                 __LINE__, cpu, ri));
 649
 650         PMCDBG(MDP,STA,1, "p6-start cpu=%d ri=%d", cpu, ri);
 651
 652         if (pd->pm_descr.pd_class == PMC_CLASS_TSC)
 653                 return 0;       /* TSC are always running */
 654
 655         KASSERT(pd->pm_descr.pd_class == PMC_CLASS_P6,
 656             ("[p6,%d] unknown PMC class %d", __LINE__,
 657                 pd->pm_descr.pd_class));
 658
 659         config = pm->pm_md.pm_ppro.pm_ppro_evsel;
 660
 661         PMCDBG(MDP,STA,2, "p6-start/2 cpu=%d ri=%d evselmsr=0x%x config=0x%x",
 662             cpu, ri, pd->pm_evsel_msr, config);
 663
 664         P6_MARK_STARTED(pc, ri);
 665         wrmsr(pd->pm_evsel_msr, config);
 666
 667         P6_SYNC_CTR_STATE(pc);
 668
 669         return 0;
 670 }
 671
 672 static int
 673 p6_stop_pmc(int cpu, int ri)
 674 {
 675         struct pmc *pm;
 676         struct p6_cpu *pc;
 677         struct pmc_hw *phw;
 678         struct p6pmc_descr *pd;
 679
 680         KASSERT(cpu >= 0 && cpu < mp_ncpus,
 681             ("[p6,%d] illegal cpu value %d", __LINE__, cpu));
 682         KASSERT(ri >= 0 && ri < P6_NPMCS,
 683             ("[p6,%d] illegal row index %d", __LINE__, ri));
 684
 685         pc  = (struct p6_cpu *) pmc_pcpu[cpu];
 686         phw = pc->pc_common.pc_hwpmcs[ri];
 687         pm  = phw->phw_pmc;
 688         pd  = &p6_pmcdesc[ri];
 689
 690         KASSERT(pm,
 691             ("[p6,%d] cpu%d ri%d no configured PMC to stop", __LINE__,
 692                 cpu, ri));
 693
 694         if (pd->pm_descr.pd_class == PMC_CLASS_TSC)
 695                 return 0;
 696
 697         KASSERT(pd->pm_descr.pd_class == PMC_CLASS_P6,
 698             ("[p6,%d] unknown PMC class %d", __LINE__,
 699                 pd->pm_descr.pd_class));
 700
 701         PMCDBG(MDP,STO,1, "p6-stop cpu=%d ri=%d", cpu, ri);
 702
 703         wrmsr(pd->pm_evsel_msr, 0);     /* stop hw */
 704         P6_MARK_STOPPED(pc, ri);        /* update software state */
 705
 706         P6_SYNC_CTR_STATE(pc);          /* restart CTR1 if need be */
 707
 708         PMCDBG(MDP,STO,2, "p6-stop/2 cpu=%d ri=%d", cpu, ri);
 709         return 0;
 710 }
 711
 712 static int
 713 p6_intr(int cpu, uintptr_t eip, int usermode)
 714 {
 715         int i, error, retval, ri;
 716         uint32_t perf0cfg;
 717         struct pmc *pm;
 718         struct p6_cpu *pc;
 719         struct pmc_hw *phw;
 720         pmc_value_t v;
 721
 722         KASSERT(cpu >= 0 && cpu < mp_ncpus,
 723             ("[p6,%d] CPU %d out of range", __LINE__, cpu));
 724
 725         retval = 0;
 726         pc = (struct p6_cpu *) pmc_pcpu[cpu];
 727
 728         /* stop both PMCs */
 729         perf0cfg = rdmsr(P6_MSR_EVSEL0);
 730         wrmsr(P6_MSR_EVSEL0, perf0cfg & ~P6_EVSEL_EN);
 731
 732         for (i = 0; i < P6_NPMCS-1; i++) {
 733                 ri = i + 1;
 734
 735                 if (!P6_PMC_HAS_OVERFLOWED(i))
 736                         continue;
 737
 738                 phw = pc->pc_common.pc_hwpmcs[ri];
 739
 740                 if ((pm = phw->phw_pmc) == NULL ||
 741                     pm->pm_state != PMC_STATE_RUNNING ||
 742                     !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
 743                         continue;
 744                 }
 745
 746                 retval = 1;
 747
 748                 error = pmc_process_interrupt(cpu, pm, eip, usermode);
 749                 if (error)
 750                         P6_MARK_STOPPED(pc,ri);
 751
 752                 /* reload sampling count */
 753                 v = pm->pm_sc.pm_reloadcount;
 754                 wrmsr(P6_MSR_PERFCTR0 + i,
 755                     P6_RELOAD_COUNT_TO_PERFCTR_VALUE(v));
 756
 757         }
 758
 759         /*
 760          * On P6 processors, the LAPIC needs to have its PMC interrupt
 761          * unmasked after a PMC interrupt.
 762          */
 763         if (retval)
 764                 pmc_x86_lapic_enable_pmc_interrupt();
 765
 766         atomic_add_int(retval ? &pmc_stats.pm_intr_processed :
 767             &pmc_stats.pm_intr_ignored, 1);
 768
 769         /* restart counters that can be restarted */
 770         P6_SYNC_CTR_STATE(pc);
 771
 772         return retval;
 773 }
 774
 775 static int
 776 p6_describe(int cpu, int ri, struct pmc_info *pi,
 777     struct pmc **ppmc)
 778 {
 779         int error;
 780         size_t copied;
 781         struct pmc_hw *phw;
 782         struct p6pmc_descr *pd;
 783
 784         phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
 785         pd  = &p6_pmcdesc[ri];
 786
 787         if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name,
 788                  PMC_NAME_MAX, &copied)) != 0)
 789                 return error;
 790
 791         pi->pm_class = pd->pm_descr.pd_class;
 792
 793         if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
 794                 pi->pm_enabled = TRUE;
 795                 *ppmc          = phw->phw_pmc;
 796         } else {
 797                 pi->pm_enabled = FALSE;
 798                 *ppmc          = NULL;
 799         }
 800
 801         return 0;
 802 }
 803
 804 static int
 805 p6_get_msr(int ri, uint32_t *msr)
 806 {
 807         KASSERT(ri >= 0 && ri < P6_NPMCS,
 808             ("[p6,%d ri %d out of range", __LINE__, ri));
 809
 810         *msr = p6_pmcdesc[ri].pm_pmc_msr - P6_MSR_PERFCTR0;
 811         return 0;
 812 }
 813
 814 int
 815 pmc_initialize_p6(struct pmc_mdep *pmc_mdep)
 816 {
 817         KASSERT(strcmp(cpu_vendor, "GenuineIntel") == 0,
 818             ("[p6,%d] Initializing non-intel processor", __LINE__));
 819
 820         PMCDBG(MDP,INI,1, "%s", "p6-initialize");
 821
 822         switch (pmc_mdep->pmd_cputype) {
 823
 824                 /*
 825                  * P6 Family Processors
 826                  */
 827
 828         case PMC_CPU_INTEL_P6:
 829         case PMC_CPU_INTEL_CL:
 830         case PMC_CPU_INTEL_PII:
 831         case PMC_CPU_INTEL_PIII:
 832         case PMC_CPU_INTEL_PM:
 833
 834                 p6_cputype = pmc_mdep->pmd_cputype;
 835
 836                 pmc_mdep->pmd_npmc          = P6_NPMCS;
 837                 pmc_mdep->pmd_classes[1].pm_class = PMC_CLASS_P6;
 838                 pmc_mdep->pmd_classes[1].pm_caps  = P6_PMC_CAPS;
 839                 pmc_mdep->pmd_classes[1].pm_width = 40;
 840                 pmc_mdep->pmd_nclasspmcs[1] = 2;
 841
 842                 pmc_mdep->pmd_init          = p6_init;
 843                 pmc_mdep->pmd_cleanup       = p6_cleanup;
 844                 pmc_mdep->pmd_switch_in     = p6_switch_in;
 845                 pmc_mdep->pmd_switch_out    = p6_switch_out;
 846                 pmc_mdep->pmd_read_pmc      = p6_read_pmc;
 847                 pmc_mdep->pmd_write_pmc     = p6_write_pmc;
 848                 pmc_mdep->pmd_config_pmc    = p6_config_pmc;
 849                 pmc_mdep->pmd_get_config    = p6_get_config;
 850                 pmc_mdep->pmd_allocate_pmc  = p6_allocate_pmc;
 851                 pmc_mdep->pmd_release_pmc   = p6_release_pmc;
 852                 pmc_mdep->pmd_start_pmc     = p6_start_pmc;
 853                 pmc_mdep->pmd_stop_pmc      = p6_stop_pmc;
 854                 pmc_mdep->pmd_intr          = p6_intr;
 855                 pmc_mdep->pmd_describe      = p6_describe;
 856                 pmc_mdep->pmd_get_msr       = p6_get_msr; /* i386 */
 857
 858                 break;
 859         default:
 860                 KASSERT(0,("[p6,%d] Unknown CPU type", __LINE__));
 861                 return ENOSYS;
 862         }
 863
 864         return 0;
 865 }