lib/libpmc/libpmc.c

   1 /*-
   2  * Copyright (c) 2003-2008 Joseph Koshy
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #include <sys/cdefs.h>
  28 __FBSDID("$FreeBSD$");
  29
  30 #include <sys/types.h>
  31 #include <sys/module.h>
  32 #include <sys/pmc.h>
  33 #include <sys/syscall.h>
  34
  35 #include <ctype.h>
  36 #include <errno.h>
  37 #include <fcntl.h>
  38 #include <pmc.h>
  39 #include <stdio.h>
  40 #include <stdlib.h>
  41 #include <string.h>
  42 #include <strings.h>
  43 #include <unistd.h>
  44
  45 #include "libpmcinternal.h"
  46
  47 /* Function prototypes */
  48 #if defined(__i386__)
  49 static int k7_allocate_pmc(enum pmc_event _pe, char *_ctrspec,
  50     struct pmc_op_pmcallocate *_pmc_config);
  51 #endif
  52 #if defined(__amd64__) || defined(__i386__)
  53 static int iaf_allocate_pmc(enum pmc_event _pe, char *_ctrspec,
  54     struct pmc_op_pmcallocate *_pmc_config);
  55 static int iap_allocate_pmc(enum pmc_event _pe, char *_ctrspec,
  56     struct pmc_op_pmcallocate *_pmc_config);
  57 static int k8_allocate_pmc(enum pmc_event _pe, char *_ctrspec,
  58     struct pmc_op_pmcallocate *_pmc_config);
  59 static int p4_allocate_pmc(enum pmc_event _pe, char *_ctrspec,
  60     struct pmc_op_pmcallocate *_pmc_config);
  61 #endif
  62 #if defined(__i386__)
  63 static int p5_allocate_pmc(enum pmc_event _pe, char *_ctrspec,
  64     struct pmc_op_pmcallocate *_pmc_config);
  65 static int p6_allocate_pmc(enum pmc_event _pe, char *_ctrspec,
  66     struct pmc_op_pmcallocate *_pmc_config);
  67 #endif
  68 #if defined(__amd64__) || defined(__i386__)
  69 static int tsc_allocate_pmc(enum pmc_event _pe, char *_ctrspec,
  70     struct pmc_op_pmcallocate *_pmc_config);
  71 #endif
  72
  73 #define PMC_CALL(cmd, params)                           \
  74         syscall(pmc_syscall, PMC_OP_##cmd, (params))
  75
  76 /*
  77  * Event aliases provide a way for the user to ask for generic events
  78  * like "cache-misses", or "instructions-retired".  These aliases are
  79  * mapped to the appropriate canonical event descriptions using a
  80  * lookup table.
  81  */
  82 struct pmc_event_alias {
  83         const char      *pm_alias;
  84         const char      *pm_spec;
  85 };
  86
  87 static const struct pmc_event_alias *pmc_mdep_event_aliases;
  88
  89 /*
  90  * The pmc_event_descr structure maps symbolic names known to the user
  91  * to integer codes used by the PMC KLD.
  92  */
  93 struct pmc_event_descr {
  94         const char      *pm_ev_name;
  95         enum pmc_event  pm_ev_code;
  96 };
  97
  98 /*
  99  * The pmc_class_descr structure maps class name prefixes for
 100  * event names to event tables and other PMC class data.
 101  */
 102 struct pmc_class_descr {
 103         const char      *pm_evc_name;
 104         size_t          pm_evc_name_size;
 105         enum pmc_class  pm_evc_class;
 106         const struct pmc_event_descr *pm_evc_event_table;
 107         size_t          pm_evc_event_table_size;
 108         int             (*pm_evc_allocate_pmc)(enum pmc_event _pe,
 109                             char *_ctrspec, struct pmc_op_pmcallocate *_pa);
 110 };
 111
 112 #define PMC_TABLE_SIZE(N)       (sizeof(N)/sizeof(N[0]))
 113 #define PMC_EVENT_TABLE_SIZE(N) PMC_TABLE_SIZE(N##_event_table)
 114
 115 #undef  __PMC_EV
 116 #define __PMC_EV(C,N) { #N, PMC_EV_ ## C ## _ ## N },
 117
 118 /*
 119  * PMC_CLASSDEP_TABLE(NAME, CLASS)
 120  *
 121  * Define a table mapping event names and aliases to HWPMC event IDs.
 122  */
 123 #define PMC_CLASSDEP_TABLE(N, C)                                \
 124         static const struct pmc_event_descr N##_event_table[] = \
 125         {                                                       \
 126                 __PMC_EV_##C()                                  \
 127         }
 128
 129 PMC_CLASSDEP_TABLE(iaf, IAF);
 130 PMC_CLASSDEP_TABLE(k7, K7);
 131 PMC_CLASSDEP_TABLE(k8, K8);
 132 PMC_CLASSDEP_TABLE(p4, P4);
 133 PMC_CLASSDEP_TABLE(p5, P5);
 134 PMC_CLASSDEP_TABLE(p6, P6);
 135
 136 #undef  __PMC_EV_ALIAS
 137 #define __PMC_EV_ALIAS(N,CODE)  { N, PMC_EV_##CODE },
 138
 139 static const struct pmc_event_descr atom_event_table[] =
 140 {
 141         __PMC_EV_ALIAS_ATOM()
 142 };
 143
 144 static const struct pmc_event_descr core_event_table[] =
 145 {
 146         __PMC_EV_ALIAS_CORE()
 147 };
 148
 149
 150 static const struct pmc_event_descr core2_event_table[] =
 151 {
 152         __PMC_EV_ALIAS_CORE2()
 153 };
 154
 155 static const struct pmc_event_descr corei7_event_table[] =
 156 {
 157         __PMC_EV_ALIAS_COREI7()
 158 };
 159
 160 /*
 161  * PMC_MDEP_TABLE(NAME, PRIMARYCLASS, ADDITIONAL_CLASSES...)
 162  *
 163  * Map a CPU to the PMC classes it supports.
 164  */
 165 #define PMC_MDEP_TABLE(N,C,...)                         \
 166         static const enum pmc_class N##_pmc_classes[] = {       \
 167                 PMC_CLASS_##C, __VA_ARGS__                      \
 168         }
 169
 170 PMC_MDEP_TABLE(atom, IAP, PMC_CLASS_IAF, PMC_CLASS_TSC);
 171 PMC_MDEP_TABLE(core, IAP, PMC_CLASS_TSC);
 172 PMC_MDEP_TABLE(core2, IAP, PMC_CLASS_IAF, PMC_CLASS_TSC);
 173 PMC_MDEP_TABLE(corei7, IAP, PMC_CLASS_IAF, PMC_CLASS_TSC);
 174 PMC_MDEP_TABLE(k7, K7, PMC_CLASS_TSC);
 175 PMC_MDEP_TABLE(k8, K8, PMC_CLASS_TSC);
 176 PMC_MDEP_TABLE(p4, P4, PMC_CLASS_TSC);
 177 PMC_MDEP_TABLE(p5, P5, PMC_CLASS_TSC);
 178 PMC_MDEP_TABLE(p6, P6, PMC_CLASS_TSC);
 179
 180 static const struct pmc_event_descr tsc_event_table[] =
 181 {
 182         __PMC_EV_TSC()
 183 };
 184
 185 #undef  PMC_CLASS_TABLE_DESC
 186 #define PMC_CLASS_TABLE_DESC(NAME, CLASS, EVENTS, ALLOCATOR)    \
 187 static const struct pmc_class_descr NAME##_class_table_descr =  \
 188         {                                                       \
 189                 .pm_evc_name  = #CLASS "-",                     \
 190                 .pm_evc_name_size = sizeof(#CLASS "-") - 1,     \
 191                 .pm_evc_class = PMC_CLASS_##CLASS ,             \
 192                 .pm_evc_event_table = EVENTS##_event_table ,    \
 193                 .pm_evc_event_table_size =                      \
 194                         PMC_EVENT_TABLE_SIZE(EVENTS),           \
 195                 .pm_evc_allocate_pmc = ALLOCATOR##_allocate_pmc \
 196         }
 197
 198 #if     defined(__i386__) || defined(__amd64__)
 199 PMC_CLASS_TABLE_DESC(iaf, IAF, iaf, iaf);
 200 PMC_CLASS_TABLE_DESC(atom, IAP, atom, iap);
 201 PMC_CLASS_TABLE_DESC(core, IAP, core, iap);
 202 PMC_CLASS_TABLE_DESC(core2, IAP, core2, iap);
 203 PMC_CLASS_TABLE_DESC(corei7, IAP, corei7, iap);
 204 #endif
 205 #if     defined(__i386__)
 206 PMC_CLASS_TABLE_DESC(k7, K7, k7, k7);
 207 #endif
 208 #if     defined(__i386__) || defined(__amd64__)
 209 PMC_CLASS_TABLE_DESC(k8, K8, k8, k8);
 210 PMC_CLASS_TABLE_DESC(p4, P4, p4, p4);
 211 #endif
 212 #if     defined(__i386__)
 213 PMC_CLASS_TABLE_DESC(p5, P5, p5, p5);
 214 PMC_CLASS_TABLE_DESC(p6, P6, p6, p6);
 215 #endif
 216 #if     defined(__i386__) || defined(__amd64__)
 217 PMC_CLASS_TABLE_DESC(tsc, TSC, tsc, tsc);
 218 #endif
 219
 220 #undef  PMC_CLASS_TABLE_DESC
 221
 222 static const struct pmc_class_descr **pmc_class_table;
 223 #define PMC_CLASS_TABLE_SIZE    cpu_info.pm_nclass
 224
 225 static const enum pmc_class *pmc_mdep_class_list;
 226 static size_t pmc_mdep_class_list_size;
 227
 228 /*
 229  * Mapping tables, mapping enumeration values to human readable
 230  * strings.
 231  */
 232
 233 static const char * pmc_capability_names[] = {
 234 #undef  __PMC_CAP
 235 #define __PMC_CAP(N,V,D)        #N ,
 236         __PMC_CAPS()
 237 };
 238
 239 static const char * pmc_class_names[] = {
 240 #undef  __PMC_CLASS
 241 #define __PMC_CLASS(C)  #C ,
 242         __PMC_CLASSES()
 243 };
 244
 245 struct pmc_cputype_map {
 246         enum pmc_class  pm_cputype;
 247         const char      *pm_name;
 248 };
 249
 250 static const struct pmc_cputype_map pmc_cputype_names[] = {
 251 #undef  __PMC_CPU
 252 #define __PMC_CPU(S, V, D) { .pm_cputype = PMC_CPU_##S, .pm_name = #S } ,
 253         __PMC_CPUS()
 254 };
 255
 256 static const char * pmc_disposition_names[] = {
 257 #undef  __PMC_DISP
 258 #define __PMC_DISP(D)   #D ,
 259         __PMC_DISPOSITIONS()
 260 };
 261
 262 static const char * pmc_mode_names[] = {
 263 #undef  __PMC_MODE
 264 #define __PMC_MODE(M,N) #M ,
 265         __PMC_MODES()
 266 };
 267
 268 static const char * pmc_state_names[] = {
 269 #undef  __PMC_STATE
 270 #define __PMC_STATE(S) #S ,
 271         __PMC_STATES()
 272 };
 273
 274 static int pmc_syscall = -1;            /* filled in by pmc_init() */
 275
 276 static struct pmc_cpuinfo cpu_info;     /* filled in by pmc_init() */
 277
 278 /* Event masks for events */
 279 struct pmc_masks {
 280         const char      *pm_name;
 281         const uint32_t  pm_value;
 282 };
 283 #define PMCMASK(N,V)    { .pm_name = #N, .pm_value = (V) }
 284 #define NULLMASK        PMCMASK(NULL,0)
 285
 286 #if defined(__amd64__) || defined(__i386__)
 287 static int
 288 pmc_parse_mask(const struct pmc_masks *pmask, char *p, uint32_t *evmask)
 289 {
 290         const struct pmc_masks *pm;
 291         char *q, *r;
 292         int c;
 293
 294         if (pmask == NULL)      /* no mask keywords */
 295                 return (-1);
 296         q = strchr(p, '=');     /* skip '=' */
 297         if (*++q == '\0')       /* no more data */
 298                 return (-1);
 299         c = 0;                  /* count of mask keywords seen */
 300         while ((r = strsep(&q, "+")) != NULL) {
 301                 for (pm = pmask; pm->pm_name && strcasecmp(r, pm->pm_name);
 302                     pm++)
 303                         ;
 304                 if (pm->pm_name == NULL) /* not found */
 305                         return (-1);
 306                 *evmask |= pm->pm_value;
 307                 c++;
 308         }
 309         return (c);
 310 }
 311 #endif
 312
 313 #define KWMATCH(p,kw)           (strcasecmp((p), (kw)) == 0)
 314 #define KWPREFIXMATCH(p,kw)     (strncasecmp((p), (kw), sizeof((kw)) - 1) == 0)
 315 #define EV_ALIAS(N,S)           { .pm_alias = N, .pm_spec = S }
 316
 317 #if defined(__i386__)
 318
 319 /*
 320  * AMD K7 (Athlon) CPUs.
 321  */
 322
 323 static struct pmc_event_alias k7_aliases[] = {
 324         EV_ALIAS("branches",            "k7-retired-branches"),
 325         EV_ALIAS("branch-mispredicts",  "k7-retired-branches-mispredicted"),
 326         EV_ALIAS("cycles",              "tsc"),
 327         EV_ALIAS("dc-misses",           "k7-dc-misses"),
 328         EV_ALIAS("ic-misses",           "k7-ic-misses"),
 329         EV_ALIAS("instructions",        "k7-retired-instructions"),
 330         EV_ALIAS("interrupts",          "k7-hardware-interrupts"),
 331         EV_ALIAS(NULL, NULL)
 332 };
 333
 334 #define K7_KW_COUNT     "count"
 335 #define K7_KW_EDGE      "edge"
 336 #define K7_KW_INV       "inv"
 337 #define K7_KW_OS        "os"
 338 #define K7_KW_UNITMASK  "unitmask"
 339 #define K7_KW_USR       "usr"
 340
 341 static int
 342 k7_allocate_pmc(enum pmc_event pe, char *ctrspec,
 343     struct pmc_op_pmcallocate *pmc_config)
 344 {
 345         char            *e, *p, *q;
 346         int             c, has_unitmask;
 347         uint32_t        count, unitmask;
 348
 349         pmc_config->pm_md.pm_amd.pm_amd_config = 0;
 350         pmc_config->pm_caps |= (PMC_CAP_READ | PMC_CAP_WRITE);
 351
 352         if (pe == PMC_EV_K7_DC_REFILLS_FROM_L2 ||
 353             pe == PMC_EV_K7_DC_REFILLS_FROM_SYSTEM ||
 354             pe == PMC_EV_K7_DC_WRITEBACKS) {
 355                 has_unitmask = 1;
 356                 unitmask = AMD_PMC_UNITMASK_MOESI;
 357         } else
 358                 unitmask = has_unitmask = 0;
 359
 360         while ((p = strsep(&ctrspec, ",")) != NULL) {
 361                 if (KWPREFIXMATCH(p, K7_KW_COUNT "=")) {
 362                         q = strchr(p, '=');
 363                         if (*++q == '\0') /* skip '=' */
 364                                 return (-1);
 365
 366                         count = strtol(q, &e, 0);
 367                         if (e == q || *e != '\0')
 368                                 return (-1);
 369
 370                         pmc_config->pm_caps |= PMC_CAP_THRESHOLD;
 371                         pmc_config->pm_md.pm_amd.pm_amd_config |=
 372                             AMD_PMC_TO_COUNTER(count);
 373
 374                 } else if (KWMATCH(p, K7_KW_EDGE)) {
 375                         pmc_config->pm_caps |= PMC_CAP_EDGE;
 376                 } else if (KWMATCH(p, K7_KW_INV)) {
 377                         pmc_config->pm_caps |= PMC_CAP_INVERT;
 378                 } else if (KWMATCH(p, K7_KW_OS)) {
 379                         pmc_config->pm_caps |= PMC_CAP_SYSTEM;
 380                 } else if (KWPREFIXMATCH(p, K7_KW_UNITMASK "=")) {
 381                         if (has_unitmask == 0)
 382                                 return (-1);
 383                         unitmask = 0;
 384                         q = strchr(p, '=');
 385                         if (*++q == '\0') /* skip '=' */
 386                                 return (-1);
 387
 388                         while ((c = tolower(*q++)) != 0)
 389                                 if (c == 'm')
 390                                         unitmask |= AMD_PMC_UNITMASK_M;
 391                                 else if (c == 'o')
 392                                         unitmask |= AMD_PMC_UNITMASK_O;
 393                                 else if (c == 'e')
 394                                         unitmask |= AMD_PMC_UNITMASK_E;
 395                                 else if (c == 's')
 396                                         unitmask |= AMD_PMC_UNITMASK_S;
 397                                 else if (c == 'i')
 398                                         unitmask |= AMD_PMC_UNITMASK_I;
 399                                 else if (c == '+')
 400                                         continue;
 401                                 else
 402                                         return (-1);
 403
 404                         if (unitmask == 0)
 405                                 return (-1);
 406
 407                 } else if (KWMATCH(p, K7_KW_USR)) {
 408                         pmc_config->pm_caps |= PMC_CAP_USER;
 409                 } else
 410                         return (-1);
 411         }
 412
 413         if (has_unitmask) {
 414                 pmc_config->pm_caps |= PMC_CAP_QUALIFIER;
 415                 pmc_config->pm_md.pm_amd.pm_amd_config |=
 416                     AMD_PMC_TO_UNITMASK(unitmask);
 417         }
 418
 419         return (0);
 420
 421 }
 422
 423 #endif
 424
 425 #if defined(__amd64__) || defined(__i386__)
 426
 427 /*
 428  * Intel Core (Family 6, Model E) PMCs.
 429  */
 430
 431 static struct pmc_event_alias core_aliases[] = {
 432         EV_ALIAS("branches",            "iap-br-instr-ret"),
 433         EV_ALIAS("branch-mispredicts",  "iap-br-mispred-ret"),
 434         EV_ALIAS("cycles",              "tsc-tsc"),
 435         EV_ALIAS("ic-misses",           "iap-icache-misses"),
 436         EV_ALIAS("instructions",        "iap-instr-ret"),
 437         EV_ALIAS("interrupts",          "iap-core-hw-int-rx"),
 438         EV_ALIAS("unhalted-cycles",     "iap-unhalted-core-cycles"),
 439         EV_ALIAS(NULL, NULL)
 440 };
 441
 442 /*
 443  * Intel Core2 (Family 6, Model F), Core2Extreme (Family 6, Model 17H)
 444  * and Atom (Family 6, model 1CH) PMCs.
 445  *
 446  * We map aliases to events on the fixed-function counters if these
 447  * are present.  Note that not all CPUs in this family contain fixed-function
 448  * counters.
 449  */
 450
 451 static struct pmc_event_alias core2_aliases[] = {
 452         EV_ALIAS("branches",            "iap-br-inst-retired.any"),
 453         EV_ALIAS("branch-mispredicts",  "iap-br-inst-retired.mispred"),
 454         EV_ALIAS("cycles",              "tsc-tsc"),
 455         EV_ALIAS("ic-misses",           "iap-l1i-misses"),
 456         EV_ALIAS("instructions",        "iaf-instr-retired.any"),
 457         EV_ALIAS("interrupts",          "iap-hw-int-rcv"),
 458         EV_ALIAS("unhalted-cycles",     "iaf-cpu-clk-unhalted.core"),
 459         EV_ALIAS(NULL, NULL)
 460 };
 461
 462 static struct pmc_event_alias core2_aliases_without_iaf[] = {
 463         EV_ALIAS("branches",            "iap-br-inst-retired.any"),
 464         EV_ALIAS("branch-mispredicts",  "iap-br-inst-retired.mispred"),
 465         EV_ALIAS("cycles",              "tsc-tsc"),
 466         EV_ALIAS("ic-misses",           "iap-l1i-misses"),
 467         EV_ALIAS("instructions",        "iap-inst-retired.any_p"),
 468         EV_ALIAS("interrupts",          "iap-hw-int-rcv"),
 469         EV_ALIAS("unhalted-cycles",     "iap-cpu-clk-unhalted.core_p"),
 470         EV_ALIAS(NULL, NULL)
 471 };
 472
 473 #define atom_aliases                    core2_aliases
 474 #define atom_aliases_without_iaf        core2_aliases_without_iaf
 475 #define corei7_aliases                  core2_aliases
 476 #define corei7_aliases_without_iaf      core2_aliases_without_iaf
 477
 478 #define IAF_KW_OS               "os"
 479 #define IAF_KW_USR              "usr"
 480 #define IAF_KW_ANYTHREAD        "anythread"
 481
 482 /*
 483  * Parse an event specifier for Intel fixed function counters.
 484  */
 485 static int
 486 iaf_allocate_pmc(enum pmc_event pe, char *ctrspec,
 487     struct pmc_op_pmcallocate *pmc_config)
 488 {
 489         char *p;
 490
 491         (void) pe;
 492
 493         pmc_config->pm_caps |= (PMC_CAP_READ | PMC_CAP_WRITE);
 494         pmc_config->pm_md.pm_iaf.pm_iaf_flags = 0;
 495
 496         while ((p = strsep(&ctrspec, ",")) != NULL) {
 497                 if (KWMATCH(p, IAF_KW_OS))
 498                         pmc_config->pm_caps |= PMC_CAP_SYSTEM;
 499                 else if (KWMATCH(p, IAF_KW_USR))
 500                         pmc_config->pm_caps |= PMC_CAP_USER;
 501                 else if (KWMATCH(p, IAF_KW_ANYTHREAD))
 502                         pmc_config->pm_md.pm_iaf.pm_iaf_flags |= IAF_ANY;
 503                 else
 504                         return (-1);
 505         }
 506
 507         return (0);
 508 }
 509
 510 /*
 511  * Core/Core2 support.
 512  */
 513
 514 #define IAP_KW_AGENT            "agent"
 515 #define IAP_KW_ANYTHREAD        "anythread"
 516 #define IAP_KW_CACHESTATE       "cachestate"
 517 #define IAP_KW_CMASK            "cmask"
 518 #define IAP_KW_CORE             "core"
 519 #define IAP_KW_EDGE             "edge"
 520 #define IAP_KW_INV              "inv"
 521 #define IAP_KW_OS               "os"
 522 #define IAP_KW_PREFETCH         "prefetch"
 523 #define IAP_KW_SNOOPRESPONSE    "snoopresponse"
 524 #define IAP_KW_SNOOPTYPE        "snooptype"
 525 #define IAP_KW_TRANSITION       "trans"
 526 #define IAP_KW_USR              "usr"
 527
 528 static struct pmc_masks iap_core_mask[] = {
 529         PMCMASK(all,    (0x3 << 14)),
 530         PMCMASK(this,   (0x1 << 14)),
 531         NULLMASK
 532 };
 533
 534 static struct pmc_masks iap_agent_mask[] = {
 535         PMCMASK(this,   0),
 536         PMCMASK(any,    (0x1 << 13)),
 537         NULLMASK
 538 };
 539
 540 static struct pmc_masks iap_prefetch_mask[] = {
 541         PMCMASK(both,           (0x3 << 12)),
 542         PMCMASK(only,           (0x1 << 12)),
 543         PMCMASK(exclude,        0),
 544         NULLMASK
 545 };
 546
 547 static struct pmc_masks iap_cachestate_mask[] = {
 548         PMCMASK(i,              (1 <<  8)),
 549         PMCMASK(s,              (1 <<  9)),
 550         PMCMASK(e,              (1 << 10)),
 551         PMCMASK(m,              (1 << 11)),
 552         NULLMASK
 553 };
 554
 555 static struct pmc_masks iap_snoopresponse_mask[] = {
 556         PMCMASK(clean,          (1 << 8)),
 557         PMCMASK(hit,            (1 << 9)),
 558         PMCMASK(hitm,           (1 << 11)),
 559         NULLMASK
 560 };
 561
 562 static struct pmc_masks iap_snooptype_mask[] = {
 563         PMCMASK(cmp2s,          (1 << 8)),
 564         PMCMASK(cmp2i,          (1 << 9)),
 565         NULLMASK
 566 };
 567
 568 static struct pmc_masks iap_transition_mask[] = {
 569         PMCMASK(any,            0x00),
 570         PMCMASK(frequency,      0x10),
 571         NULLMASK
 572 };
 573
 574 static int
 575 iap_allocate_pmc(enum pmc_event pe, char *ctrspec,
 576     struct pmc_op_pmcallocate *pmc_config)
 577 {
 578         char *e, *p, *q;
 579         uint32_t cachestate, evmask;
 580         int count, n;
 581
 582         pmc_config->pm_caps |= (PMC_CAP_READ | PMC_CAP_WRITE |
 583             PMC_CAP_QUALIFIER);
 584         pmc_config->pm_md.pm_iap.pm_iap_config = 0;
 585
 586         cachestate = evmask = 0;
 587
 588         /* Parse additional modifiers if present */
 589         while ((p = strsep(&ctrspec, ",")) != NULL) {
 590
 591                 n = 0;
 592                 if (KWPREFIXMATCH(p, IAP_KW_CMASK "=")) {
 593                         q = strchr(p, '=');
 594                         if (*++q == '\0') /* skip '=' */
 595                                 return (-1);
 596                         count = strtol(q, &e, 0);
 597                         if (e == q || *e != '\0')
 598                                 return (-1);
 599                         pmc_config->pm_caps |= PMC_CAP_THRESHOLD;
 600                         pmc_config->pm_md.pm_iap.pm_iap_config |=
 601                             IAP_CMASK(count);
 602                 } else if (KWMATCH(p, IAP_KW_EDGE)) {
 603                         pmc_config->pm_caps |= PMC_CAP_EDGE;
 604                 } else if (KWMATCH(p, IAP_KW_INV)) {
 605                         pmc_config->pm_caps |= PMC_CAP_INVERT;
 606                 } else if (KWMATCH(p, IAP_KW_OS)) {
 607                         pmc_config->pm_caps |= PMC_CAP_SYSTEM;
 608                 } else if (KWMATCH(p, IAP_KW_USR)) {
 609                         pmc_config->pm_caps |= PMC_CAP_USER;
 610                 } else if (KWMATCH(p, IAP_KW_ANYTHREAD)) {
 611                         pmc_config->pm_md.pm_iap.pm_iap_config |= IAP_ANY;
 612                 } else if (KWPREFIXMATCH(p, IAP_KW_CORE "=")) {
 613                         n = pmc_parse_mask(iap_core_mask, p, &evmask);
 614                         if (n != 1)
 615                                 return (-1);
 616                 } else if (KWPREFIXMATCH(p, IAP_KW_AGENT "=")) {
 617                         n = pmc_parse_mask(iap_agent_mask, p, &evmask);
 618                         if (n != 1)
 619                                 return (-1);
 620                 } else if (KWPREFIXMATCH(p, IAP_KW_PREFETCH "=")) {
 621                         n = pmc_parse_mask(iap_prefetch_mask, p, &evmask);
 622                         if (n != 1)
 623                                 return (-1);
 624                 } else if (KWPREFIXMATCH(p, IAP_KW_CACHESTATE "=")) {
 625                         n = pmc_parse_mask(iap_cachestate_mask, p, &cachestate);
 626                 } else if (cpu_info.pm_cputype == PMC_CPU_INTEL_CORE &&
 627                     KWPREFIXMATCH(p, IAP_KW_TRANSITION "=")) {
 628                         n = pmc_parse_mask(iap_transition_mask, p, &evmask);
 629                         if (n != 1)
 630                                 return (-1);
 631                 } else if (cpu_info.pm_cputype == PMC_CPU_INTEL_ATOM ||
 632                     cpu_info.pm_cputype == PMC_CPU_INTEL_CORE2 ||
 633                     cpu_info.pm_cputype == PMC_CPU_INTEL_CORE2EXTREME ||
 634                     cpu_info.pm_cputype == PMC_CPU_INTEL_COREI7) {
 635                         if (KWPREFIXMATCH(p, IAP_KW_SNOOPRESPONSE "=")) {
 636                                 n = pmc_parse_mask(iap_snoopresponse_mask, p,
 637                                     &evmask);
 638                         } else if (KWPREFIXMATCH(p, IAP_KW_SNOOPTYPE "=")) {
 639                                 n = pmc_parse_mask(iap_snooptype_mask, p,
 640                                     &evmask);
 641                         } else
 642                                 return (-1);
 643                 } else
 644                         return (-1);
 645
 646                 if (n < 0)      /* Parsing failed. */
 647                         return (-1);
 648         }
 649
 650         pmc_config->pm_md.pm_iap.pm_iap_config |= evmask;
 651
 652         /*
 653          * If the event requires a 'cachestate' qualifier but was not
 654          * specified by the user, use a sensible default.
 655          */
 656         switch (pe) {
 657         case PMC_EV_IAP_EVENT_28H: /* Core, Core2, Atom */
 658         case PMC_EV_IAP_EVENT_29H: /* Core, Core2, Atom */
 659         case PMC_EV_IAP_EVENT_2AH: /* Core, Core2, Atom */
 660         case PMC_EV_IAP_EVENT_2BH: /* Atom, Core2 */
 661         case PMC_EV_IAP_EVENT_2EH: /* Core, Core2, Atom */
 662         case PMC_EV_IAP_EVENT_30H: /* Core, Core2, Atom */
 663         case PMC_EV_IAP_EVENT_32H: /* Core */
 664         case PMC_EV_IAP_EVENT_40H: /* Core */
 665         case PMC_EV_IAP_EVENT_41H: /* Core */
 666         case PMC_EV_IAP_EVENT_42H: /* Core, Core2, Atom */
 667         case PMC_EV_IAP_EVENT_77H: /* Core */
 668                 if (cachestate == 0)
 669                         cachestate = (0xF << 8);
 670         default:
 671                 break;
 672         }
 673
 674         pmc_config->pm_md.pm_iap.pm_iap_config |= cachestate;
 675
 676         return (0);
 677 }
 678
 679 /*
 680  * AMD K8 PMCs.
 681  *
 682  * These are very similar to AMD K7 PMCs, but support more kinds of
 683  * events.
 684  */
 685
 686 static struct pmc_event_alias k8_aliases[] = {
 687         EV_ALIAS("branches",            "k8-fr-retired-taken-branches"),
 688         EV_ALIAS("branch-mispredicts",
 689             "k8-fr-retired-taken-branches-mispredicted"),
 690         EV_ALIAS("cycles",              "tsc"),
 691         EV_ALIAS("dc-misses",           "k8-dc-miss"),
 692         EV_ALIAS("ic-misses",           "k8-ic-miss"),
 693         EV_ALIAS("instructions",        "k8-fr-retired-x86-instructions"),
 694         EV_ALIAS("interrupts",          "k8-fr-taken-hardware-interrupts"),
 695         EV_ALIAS("unhalted-cycles",     "k8-bu-cpu-clk-unhalted"),
 696         EV_ALIAS(NULL, NULL)
 697 };
 698
 699 #define __K8MASK(N,V) PMCMASK(N,(1 << (V)))
 700
 701 /*
 702  * Parsing tables
 703  */
 704
 705 /* fp dispatched fpu ops */
 706 static const struct pmc_masks k8_mask_fdfo[] = {
 707         __K8MASK(add-pipe-excluding-junk-ops,   0),
 708         __K8MASK(multiply-pipe-excluding-junk-ops,      1),
 709         __K8MASK(store-pipe-excluding-junk-ops, 2),
 710         __K8MASK(add-pipe-junk-ops,             3),
 711         __K8MASK(multiply-pipe-junk-ops,        4),
 712         __K8MASK(store-pipe-junk-ops,           5),
 713         NULLMASK
 714 };
 715
 716 /* ls segment register loads */
 717 static const struct pmc_masks k8_mask_lsrl[] = {
 718         __K8MASK(es,    0),
 719         __K8MASK(cs,    1),
 720         __K8MASK(ss,    2),
 721         __K8MASK(ds,    3),
 722         __K8MASK(fs,    4),
 723         __K8MASK(gs,    5),
 724         __K8MASK(hs,    6),
 725         NULLMASK
 726 };
 727
 728 /* ls locked operation */
 729 static const struct pmc_masks k8_mask_llo[] = {
 730         __K8MASK(locked-instructions,   0),
 731         __K8MASK(cycles-in-request,     1),
 732         __K8MASK(cycles-to-complete,    2),
 733         NULLMASK
 734 };
 735
 736 /* dc refill from {l2,system} and dc copyback */
 737 static const struct pmc_masks k8_mask_dc[] = {
 738         __K8MASK(invalid,       0),
 739         __K8MASK(shared,        1),
 740         __K8MASK(exclusive,     2),
 741         __K8MASK(owner,         3),
 742         __K8MASK(modified,      4),
 743         NULLMASK
 744 };
 745
 746 /* dc one bit ecc error */
 747 static const struct pmc_masks k8_mask_dobee[] = {
 748         __K8MASK(scrubber,      0),
 749         __K8MASK(piggyback,     1),
 750         NULLMASK
 751 };
 752
 753 /* dc dispatched prefetch instructions */
 754 static const struct pmc_masks k8_mask_ddpi[] = {
 755         __K8MASK(load,  0),
 756         __K8MASK(store, 1),
 757         __K8MASK(nta,   2),
 758         NULLMASK
 759 };
 760
 761 /* dc dcache accesses by locks */
 762 static const struct pmc_masks k8_mask_dabl[] = {
 763         __K8MASK(accesses,      0),
 764         __K8MASK(misses,        1),
 765         NULLMASK
 766 };
 767
 768 /* bu internal l2 request */
 769 static const struct pmc_masks k8_mask_bilr[] = {
 770         __K8MASK(ic-fill,       0),
 771         __K8MASK(dc-fill,       1),
 772         __K8MASK(tlb-reload,    2),
 773         __K8MASK(tag-snoop,     3),
 774         __K8MASK(cancelled,     4),
 775         NULLMASK
 776 };
 777
 778 /* bu fill request l2 miss */
 779 static const struct pmc_masks k8_mask_bfrlm[] = {
 780         __K8MASK(ic-fill,       0),
 781         __K8MASK(dc-fill,       1),
 782         __K8MASK(tlb-reload,    2),
 783         NULLMASK
 784 };
 785
 786 /* bu fill into l2 */
 787 static const struct pmc_masks k8_mask_bfil[] = {
 788         __K8MASK(dirty-l2-victim,       0),
 789         __K8MASK(victim-from-l2,        1),
 790         NULLMASK
 791 };
 792
 793 /* fr retired fpu instructions */
 794 static const struct pmc_masks k8_mask_frfi[] = {
 795         __K8MASK(x87,                   0),
 796         __K8MASK(mmx-3dnow,             1),
 797         __K8MASK(packed-sse-sse2,       2),
 798         __K8MASK(scalar-sse-sse2,       3),
 799         NULLMASK
 800 };
 801
 802 /* fr retired fastpath double op instructions */
 803 static const struct pmc_masks k8_mask_frfdoi[] = {
 804         __K8MASK(low-op-pos-0,          0),
 805         __K8MASK(low-op-pos-1,          1),
 806         __K8MASK(low-op-pos-2,          2),
 807         NULLMASK
 808 };
 809
 810 /* fr fpu exceptions */
 811 static const struct pmc_masks k8_mask_ffe[] = {
 812         __K8MASK(x87-reclass-microfaults,       0),
 813         __K8MASK(sse-retype-microfaults,        1),
 814         __K8MASK(sse-reclass-microfaults,       2),
 815         __K8MASK(sse-and-x87-microtraps,        3),
 816         NULLMASK
 817 };
 818
 819 /* nb memory controller page access event */
 820 static const struct pmc_masks k8_mask_nmcpae[] = {
 821         __K8MASK(page-hit,      0),
 822         __K8MASK(page-miss,     1),
 823         __K8MASK(page-conflict, 2),
 824         NULLMASK
 825 };
 826
 827 /* nb memory controller turnaround */
 828 static const struct pmc_masks k8_mask_nmct[] = {
 829         __K8MASK(dimm-turnaround,               0),
 830         __K8MASK(read-to-write-turnaround,      1),
 831         __K8MASK(write-to-read-turnaround,      2),
 832         NULLMASK
 833 };
 834
 835 /* nb memory controller bypass saturation */
 836 static const struct pmc_masks k8_mask_nmcbs[] = {
 837         __K8MASK(memory-controller-hi-pri-bypass,       0),
 838         __K8MASK(memory-controller-lo-pri-bypass,       1),
 839         __K8MASK(dram-controller-interface-bypass,      2),
 840         __K8MASK(dram-controller-queue-bypass,          3),
 841         NULLMASK
 842 };
 843
 844 /* nb sized commands */
 845 static const struct pmc_masks k8_mask_nsc[] = {
 846         __K8MASK(nonpostwrszbyte,       0),
 847         __K8MASK(nonpostwrszdword,      1),
 848         __K8MASK(postwrszbyte,          2),
 849         __K8MASK(postwrszdword,         3),
 850         __K8MASK(rdszbyte,              4),
 851         __K8MASK(rdszdword,             5),
 852         __K8MASK(rdmodwr,               6),
 853         NULLMASK
 854 };
 855
 856 /* nb probe result */
 857 static const struct pmc_masks k8_mask_npr[] = {
 858         __K8MASK(probe-miss,            0),
 859         __K8MASK(probe-hit,             1),
 860         __K8MASK(probe-hit-dirty-no-memory-cancel, 2),
 861         __K8MASK(probe-hit-dirty-with-memory-cancel, 3),
 862         NULLMASK
 863 };
 864
 865 /* nb hypertransport bus bandwidth */
 866 static const struct pmc_masks k8_mask_nhbb[] = { /* HT bus bandwidth */
 867         __K8MASK(command,       0),
 868         __K8MASK(data,  1),
 869         __K8MASK(buffer-release, 2),
 870         __K8MASK(nop,   3),
 871         NULLMASK
 872 };
 873
 874 #undef  __K8MASK
 875
 876 #define K8_KW_COUNT     "count"
 877 #define K8_KW_EDGE      "edge"
 878 #define K8_KW_INV       "inv"
 879 #define K8_KW_MASK      "mask"
 880 #define K8_KW_OS        "os"
 881 #define K8_KW_USR       "usr"
 882
 883 static int
 884 k8_allocate_pmc(enum pmc_event pe, char *ctrspec,
 885     struct pmc_op_pmcallocate *pmc_config)
 886 {
 887         char            *e, *p, *q;
 888         int             n;
 889         uint32_t        count, evmask;
 890         const struct pmc_masks  *pm, *pmask;
 891
 892         pmc_config->pm_caps |= (PMC_CAP_READ | PMC_CAP_WRITE);
 893         pmc_config->pm_md.pm_amd.pm_amd_config = 0;
 894
 895         pmask = NULL;
 896         evmask = 0;
 897
 898 #define __K8SETMASK(M) pmask = k8_mask_##M
 899
 900         /* setup parsing tables */
 901         switch (pe) {
 902         case PMC_EV_K8_FP_DISPATCHED_FPU_OPS:
 903                 __K8SETMASK(fdfo);
 904                 break;
 905         case PMC_EV_K8_LS_SEGMENT_REGISTER_LOAD:
 906                 __K8SETMASK(lsrl);
 907                 break;
 908         case PMC_EV_K8_LS_LOCKED_OPERATION:
 909                 __K8SETMASK(llo);
 910                 break;
 911         case PMC_EV_K8_DC_REFILL_FROM_L2:
 912         case PMC_EV_K8_DC_REFILL_FROM_SYSTEM:
 913         case PMC_EV_K8_DC_COPYBACK:
 914                 __K8SETMASK(dc);
 915                 break;
 916         case PMC_EV_K8_DC_ONE_BIT_ECC_ERROR:
 917                 __K8SETMASK(dobee);
 918                 break;
 919         case PMC_EV_K8_DC_DISPATCHED_PREFETCH_INSTRUCTIONS:
 920                 __K8SETMASK(ddpi);
 921                 break;
 922         case PMC_EV_K8_DC_DCACHE_ACCESSES_BY_LOCKS:
 923                 __K8SETMASK(dabl);
 924                 break;
 925         case PMC_EV_K8_BU_INTERNAL_L2_REQUEST:
 926                 __K8SETMASK(bilr);
 927                 break;
 928         case PMC_EV_K8_BU_FILL_REQUEST_L2_MISS:
 929                 __K8SETMASK(bfrlm);
 930                 break;
 931         case PMC_EV_K8_BU_FILL_INTO_L2:
 932                 __K8SETMASK(bfil);
 933                 break;
 934         case PMC_EV_K8_FR_RETIRED_FPU_INSTRUCTIONS:
 935                 __K8SETMASK(frfi);
 936                 break;
 937         case PMC_EV_K8_FR_RETIRED_FASTPATH_DOUBLE_OP_INSTRUCTIONS:
 938                 __K8SETMASK(frfdoi);
 939                 break;
 940         case PMC_EV_K8_FR_FPU_EXCEPTIONS:
 941                 __K8SETMASK(ffe);
 942                 break;
 943         case PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_ACCESS_EVENT:
 944                 __K8SETMASK(nmcpae);
 945                 break;
 946         case PMC_EV_K8_NB_MEMORY_CONTROLLER_TURNAROUND:
 947                 __K8SETMASK(nmct);
 948                 break;
 949         case PMC_EV_K8_NB_MEMORY_CONTROLLER_BYPASS_SATURATION:
 950                 __K8SETMASK(nmcbs);
 951                 break;
 952         case PMC_EV_K8_NB_SIZED_COMMANDS:
 953                 __K8SETMASK(nsc);
 954                 break;
 955         case PMC_EV_K8_NB_PROBE_RESULT:
 956                 __K8SETMASK(npr);
 957                 break;
 958         case PMC_EV_K8_NB_HT_BUS0_BANDWIDTH:
 959         case PMC_EV_K8_NB_HT_BUS1_BANDWIDTH:
 960         case PMC_EV_K8_NB_HT_BUS2_BANDWIDTH:
 961                 __K8SETMASK(nhbb);
 962                 break;
 963
 964         default:
 965                 break;          /* no options defined */
 966         }
 967
 968         while ((p = strsep(&ctrspec, ",")) != NULL) {
 969                 if (KWPREFIXMATCH(p, K8_KW_COUNT "=")) {
 970                         q = strchr(p, '=');
 971                         if (*++q == '\0') /* skip '=' */
 972                                 return (-1);
 973
 974                         count = strtol(q, &e, 0);
 975                         if (e == q || *e != '\0')
 976                                 return (-1);
 977
 978                         pmc_config->pm_caps |= PMC_CAP_THRESHOLD;
 979                         pmc_config->pm_md.pm_amd.pm_amd_config |=
 980                             AMD_PMC_TO_COUNTER(count);
 981
 982                 } else if (KWMATCH(p, K8_KW_EDGE)) {
 983                         pmc_config->pm_caps |= PMC_CAP_EDGE;
 984                 } else if (KWMATCH(p, K8_KW_INV)) {
 985                         pmc_config->pm_caps |= PMC_CAP_INVERT;
 986                 } else if (KWPREFIXMATCH(p, K8_KW_MASK "=")) {
 987                         if ((n = pmc_parse_mask(pmask, p, &evmask)) < 0)
 988                                 return (-1);
 989                         pmc_config->pm_caps |= PMC_CAP_QUALIFIER;
 990                 } else if (KWMATCH(p, K8_KW_OS)) {
 991                         pmc_config->pm_caps |= PMC_CAP_SYSTEM;
 992                 } else if (KWMATCH(p, K8_KW_USR)) {
 993                         pmc_config->pm_caps |= PMC_CAP_USER;
 994                 } else
 995                         return (-1);
 996         }
 997
 998         /* other post processing */
 999         switch (pe) {
1000         case PMC_EV_K8_FP_DISPATCHED_FPU_OPS:
1001         case PMC_EV_K8_FP_CYCLES_WITH_NO_FPU_OPS_RETIRED:
1002         case PMC_EV_K8_FP_DISPATCHED_FPU_FAST_FLAG_OPS:
1003         case PMC_EV_K8_FR_RETIRED_FASTPATH_DOUBLE_OP_INSTRUCTIONS:
1004         case PMC_EV_K8_FR_RETIRED_FPU_INSTRUCTIONS:
1005         case PMC_EV_K8_FR_FPU_EXCEPTIONS:
1006                 /* XXX only available in rev B and later */
1007                 break;
1008         case PMC_EV_K8_DC_DCACHE_ACCESSES_BY_LOCKS:
1009                 /* XXX only available in rev C and later */
1010                 break;
1011         case PMC_EV_K8_LS_LOCKED_OPERATION:
1012                 /* XXX CPU Rev A,B evmask is to be zero */
1013                 if (evmask & (evmask - 1)) /* > 1 bit set */
1014                         return (-1);
1015                 if (evmask == 0) {
1016                         evmask = 0x01; /* Rev C and later: #instrs */
1017                         pmc_config->pm_caps |= PMC_CAP_QUALIFIER;
1018                 }
1019                 break;
1020         default:
1021                 if (evmask == 0 && pmask != NULL) {
1022                         for (pm = pmask; pm->pm_name; pm++)
1023                                 evmask |= pm->pm_value;
1024                         pmc_config->pm_caps |= PMC_CAP_QUALIFIER;
1025                 }
1026         }
1027
1028         if (pmc_config->pm_caps & PMC_CAP_QUALIFIER)
1029                 pmc_config->pm_md.pm_amd.pm_amd_config =
1030                     AMD_PMC_TO_UNITMASK(evmask);
1031
1032         return (0);
1033 }
1034
1035 #endif
1036
1037 #if defined(__amd64__) || defined(__i386__)
1038
1039 /*
1040  * Intel P4 PMCs
1041  */
1042
1043 static struct pmc_event_alias p4_aliases[] = {
1044         EV_ALIAS("branches",            "p4-branch-retired,mask=mmtp+mmtm"),
1045         EV_ALIAS("branch-mispredicts",  "p4-mispred-branch-retired"),
1046         EV_ALIAS("cycles",              "tsc"),
1047         EV_ALIAS("instructions",
1048             "p4-instr-retired,mask=nbogusntag+nbogustag"),
1049         EV_ALIAS("unhalted-cycles",     "p4-global-power-events"),
1050         EV_ALIAS(NULL, NULL)
1051 };
1052
1053 #define P4_KW_ACTIVE    "active"
1054 #define P4_KW_ACTIVE_ANY "any"
1055 #define P4_KW_ACTIVE_BOTH "both"
1056 #define P4_KW_ACTIVE_NONE "none"
1057 #define P4_KW_ACTIVE_SINGLE "single"
1058 #define P4_KW_BUSREQTYPE "busreqtype"
1059 #define P4_KW_CASCADE   "cascade"
1060 #define P4_KW_EDGE      "edge"
1061 #define P4_KW_INV       "complement"
1062 #define P4_KW_OS        "os"
1063 #define P4_KW_MASK      "mask"
1064 #define P4_KW_PRECISE   "precise"
1065 #define P4_KW_TAG       "tag"
1066 #define P4_KW_THRESHOLD "threshold"
1067 #define P4_KW_USR       "usr"
1068
1069 #define __P4MASK(N,V) PMCMASK(N, (1 << (V)))
1070
1071 static const struct pmc_masks p4_mask_tcdm[] = { /* tc deliver mode */
1072         __P4MASK(dd, 0),
1073         __P4MASK(db, 1),
1074         __P4MASK(di, 2),
1075         __P4MASK(bd, 3),
1076         __P4MASK(bb, 4),
1077         __P4MASK(bi, 5),
1078         __P4MASK(id, 6),
1079         __P4MASK(ib, 7),
1080         NULLMASK
1081 };
1082
1083 static const struct pmc_masks p4_mask_bfr[] = { /* bpu fetch request */
1084         __P4MASK(tcmiss, 0),
1085         NULLMASK,
1086 };
1087
1088 static const struct pmc_masks p4_mask_ir[] = { /* itlb reference */
1089         __P4MASK(hit, 0),
1090         __P4MASK(miss, 1),
1091         __P4MASK(hit-uc, 2),
1092         NULLMASK
1093 };
1094
1095 static const struct pmc_masks p4_mask_memcan[] = { /* memory cancel */
1096         __P4MASK(st-rb-full, 2),
1097         __P4MASK(64k-conf, 3),
1098         NULLMASK
1099 };
1100
1101 static const struct pmc_masks p4_mask_memcomp[] = { /* memory complete */
1102         __P4MASK(lsc, 0),
1103         __P4MASK(ssc, 1),
1104         NULLMASK
1105 };
1106
1107 static const struct pmc_masks p4_mask_lpr[] = { /* load port replay */
1108         __P4MASK(split-ld, 1),
1109         NULLMASK
1110 };
1111
1112 static const struct pmc_masks p4_mask_spr[] = { /* store port replay */
1113         __P4MASK(split-st, 1),
1114         NULLMASK
1115 };
1116
1117 static const struct pmc_masks p4_mask_mlr[] = { /* mob load replay */
1118         __P4MASK(no-sta, 1),
1119         __P4MASK(no-std, 3),
1120         __P4MASK(partial-data, 4),
1121         __P4MASK(unalgn-addr, 5),
1122         NULLMASK
1123 };
1124
1125 static const struct pmc_masks p4_mask_pwt[] = { /* page walk type */
1126         __P4MASK(dtmiss, 0),
1127         __P4MASK(itmiss, 1),
1128         NULLMASK
1129 };
1130
1131 static const struct pmc_masks p4_mask_bcr[] = { /* bsq cache reference */
1132         __P4MASK(rd-2ndl-hits, 0),
1133         __P4MASK(rd-2ndl-hite, 1),
1134         __P4MASK(rd-2ndl-hitm, 2),
1135         __P4MASK(rd-3rdl-hits, 3),
1136         __P4MASK(rd-3rdl-hite, 4),
1137         __P4MASK(rd-3rdl-hitm, 5),
1138         __P4MASK(rd-2ndl-miss, 8),
1139         __P4MASK(rd-3rdl-miss, 9),
1140         __P4MASK(wr-2ndl-miss, 10),
1141         NULLMASK
1142 };
1143
1144 static const struct pmc_masks p4_mask_ia[] = { /* ioq allocation */
1145         __P4MASK(all-read, 5),
1146         __P4MASK(all-write, 6),
1147         __P4MASK(mem-uc, 7),
1148         __P4MASK(mem-wc, 8),
1149         __P4MASK(mem-wt, 9),
1150         __P4MASK(mem-wp, 10),
1151         __P4MASK(mem-wb, 11),
1152         __P4MASK(own, 13),
1153         __P4MASK(other, 14),
1154         __P4MASK(prefetch, 15),
1155         NULLMASK
1156 };
1157
1158 static const struct pmc_masks p4_mask_iae[] = { /* ioq active entries */
1159         __P4MASK(all-read, 5),
1160         __P4MASK(all-write, 6),
1161         __P4MASK(mem-uc, 7),
1162         __P4MASK(mem-wc, 8),
1163         __P4MASK(mem-wt, 9),
1164         __P4MASK(mem-wp, 10),
1165         __P4MASK(mem-wb, 11),
1166         __P4MASK(own, 13),
1167         __P4MASK(other, 14),
1168         __P4MASK(prefetch, 15),
1169         NULLMASK
1170 };
1171
1172 static const struct pmc_masks p4_mask_fda[] = { /* fsb data activity */
1173         __P4MASK(drdy-drv, 0),
1174         __P4MASK(drdy-own, 1),
1175         __P4MASK(drdy-other, 2),
1176         __P4MASK(dbsy-drv, 3),
1177         __P4MASK(dbsy-own, 4),
1178         __P4MASK(dbsy-other, 5),
1179         NULLMASK
1180 };
1181
1182 static const struct pmc_masks p4_mask_ba[] = { /* bsq allocation */
1183         __P4MASK(req-type0, 0),
1184         __P4MASK(req-type1, 1),
1185         __P4MASK(req-len0, 2),
1186         __P4MASK(req-len1, 3),
1187         __P4MASK(req-io-type, 5),
1188         __P4MASK(req-lock-type, 6),
1189         __P4MASK(req-cache-type, 7),
1190         __P4MASK(req-split-type, 8),
1191         __P4MASK(req-dem-type, 9),
1192         __P4MASK(req-ord-type, 10),
1193         __P4MASK(mem-type0, 11),
1194         __P4MASK(mem-type1, 12),
1195         __P4MASK(mem-type2, 13),
1196         NULLMASK
1197 };
1198
1199 static const struct pmc_masks p4_mask_sia[] = { /* sse input assist */
1200         __P4MASK(all, 15),
1201         NULLMASK
1202 };
1203
1204 static const struct pmc_masks p4_mask_psu[] = { /* packed sp uop */
1205         __P4MASK(all, 15),
1206         NULLMASK
1207 };
1208
1209 static const struct pmc_masks p4_mask_pdu[] = { /* packed dp uop */
1210         __P4MASK(all, 15),
1211         NULLMASK
1212 };
1213
1214 static const struct pmc_masks p4_mask_ssu[] = { /* scalar sp uop */
1215         __P4MASK(all, 15),
1216         NULLMASK
1217 };
1218
1219 static const struct pmc_masks p4_mask_sdu[] = { /* scalar dp uop */
1220         __P4MASK(all, 15),
1221         NULLMASK
1222 };
1223
1224 static const struct pmc_masks p4_mask_64bmu[] = { /* 64 bit mmx uop */
1225         __P4MASK(all, 15),
1226         NULLMASK
1227 };
1228
1229 static const struct pmc_masks p4_mask_128bmu[] = { /* 128 bit mmx uop */
1230         __P4MASK(all, 15),
1231         NULLMASK
1232 };
1233
1234 static const struct pmc_masks p4_mask_xfu[] = { /* X87 fp uop */
1235         __P4MASK(all, 15),
1236         NULLMASK
1237 };
1238
1239 static const struct pmc_masks p4_mask_xsmu[] = { /* x87 simd moves uop */
1240         __P4MASK(allp0, 3),
1241         __P4MASK(allp2, 4),
1242         NULLMASK
1243 };
1244
1245 static const struct pmc_masks p4_mask_gpe[] = { /* global power events */
1246         __P4MASK(running, 0),
1247         NULLMASK
1248 };
1249
1250 static const struct pmc_masks p4_mask_tmx[] = { /* TC ms xfer */
1251         __P4MASK(cisc, 0),
1252         NULLMASK
1253 };
1254
1255 static const struct pmc_masks p4_mask_uqw[] = { /* uop queue writes */
1256         __P4MASK(from-tc-build, 0),
1257         __P4MASK(from-tc-deliver, 1),
1258         __P4MASK(from-rom, 2),
1259         NULLMASK
1260 };
1261
1262 static const struct pmc_masks p4_mask_rmbt[] = {
1263         /* retired mispred branch type */
1264         __P4MASK(conditional, 1),
1265         __P4MASK(call, 2),
1266         __P4MASK(return, 3),
1267         __P4MASK(indirect, 4),
1268         NULLMASK
1269 };
1270
1271 static const struct pmc_masks p4_mask_rbt[] = { /* retired branch type */
1272         __P4MASK(conditional, 1),
1273         __P4MASK(call, 2),
1274         __P4MASK(retired, 3),
1275         __P4MASK(indirect, 4),
1276         NULLMASK
1277 };
1278
1279 static const struct pmc_masks p4_mask_rs[] = { /* resource stall */
1280         __P4MASK(sbfull, 5),
1281         NULLMASK
1282 };
1283
1284 static const struct pmc_masks p4_mask_wb[] = { /* WC buffer */
1285         __P4MASK(wcb-evicts, 0),
1286         __P4MASK(wcb-full-evict, 1),
1287         NULLMASK
1288 };
1289
1290 static const struct pmc_masks p4_mask_fee[] = { /* front end event */
1291         __P4MASK(nbogus, 0),
1292         __P4MASK(bogus, 1),
1293         NULLMASK
1294 };
1295
1296 static const struct pmc_masks p4_mask_ee[] = { /* execution event */
1297         __P4MASK(nbogus0, 0),
1298         __P4MASK(nbogus1, 1),
1299         __P4MASK(nbogus2, 2),
1300         __P4MASK(nbogus3, 3),
1301         __P4MASK(bogus0, 4),
1302         __P4MASK(bogus1, 5),
1303         __P4MASK(bogus2, 6),
1304         __P4MASK(bogus3, 7),
1305         NULLMASK
1306 };
1307
1308 static const struct pmc_masks p4_mask_re[] = { /* replay event */
1309         __P4MASK(nbogus, 0),
1310         __P4MASK(bogus, 1),
1311         NULLMASK
1312 };
1313
1314 static const struct pmc_masks p4_mask_insret[] = { /* instr retired */
1315         __P4MASK(nbogusntag, 0),
1316         __P4MASK(nbogustag, 1),
1317         __P4MASK(bogusntag, 2),
1318         __P4MASK(bogustag, 3),
1319         NULLMASK
1320 };
1321
1322 static const struct pmc_masks p4_mask_ur[] = { /* uops retired */
1323         __P4MASK(nbogus, 0),
1324         __P4MASK(bogus, 1),
1325         NULLMASK
1326 };
1327
1328 static const struct pmc_masks p4_mask_ut[] = { /* uop type */
1329         __P4MASK(tagloads, 1),
1330         __P4MASK(tagstores, 2),
1331         NULLMASK
1332 };
1333
1334 static const struct pmc_masks p4_mask_br[] = { /* branch retired */
1335         __P4MASK(mmnp, 0),
1336         __P4MASK(mmnm, 1),
1337         __P4MASK(mmtp, 2),
1338         __P4MASK(mmtm, 3),
1339         NULLMASK
1340 };
1341
1342 static const struct pmc_masks p4_mask_mbr[] = { /* mispred branch retired */
1343         __P4MASK(nbogus, 0),
1344         NULLMASK
1345 };
1346
1347 static const struct pmc_masks p4_mask_xa[] = { /* x87 assist */
1348         __P4MASK(fpsu, 0),
1349         __P4MASK(fpso, 1),
1350         __P4MASK(poao, 2),
1351         __P4MASK(poau, 3),
1352         __P4MASK(prea, 4),
1353         NULLMASK
1354 };
1355
1356 static const struct pmc_masks p4_mask_machclr[] = { /* machine clear */
1357         __P4MASK(clear, 0),
1358         __P4MASK(moclear, 2),
1359         __P4MASK(smclear, 3),
1360         NULLMASK
1361 };
1362
1363 /* P4 event parser */
1364 static int
1365 p4_allocate_pmc(enum pmc_event pe, char *ctrspec,
1366     struct pmc_op_pmcallocate *pmc_config)
1367 {
1368
1369         char    *e, *p, *q;
1370         int     count, has_tag, has_busreqtype, n;
1371         uint32_t evmask, cccractivemask;
1372         const struct pmc_masks *pm, *pmask;
1373
1374         pmc_config->pm_caps |= (PMC_CAP_READ | PMC_CAP_WRITE);
1375         pmc_config->pm_md.pm_p4.pm_p4_cccrconfig =
1376             pmc_config->pm_md.pm_p4.pm_p4_escrconfig = 0;
1377
1378         pmask   = NULL;
1379         evmask  = 0;
1380         cccractivemask = 0x3;
1381         has_tag = has_busreqtype = 0;
1382
1383 #define __P4SETMASK(M) do {                             \
1384         pmask = p4_mask_##M;                            \
1385 } while (0)
1386
1387         switch (pe) {
1388         case PMC_EV_P4_TC_DELIVER_MODE:
1389                 __P4SETMASK(tcdm);
1390                 break;
1391         case PMC_EV_P4_BPU_FETCH_REQUEST:
1392                 __P4SETMASK(bfr);
1393                 break;
1394         case PMC_EV_P4_ITLB_REFERENCE:
1395                 __P4SETMASK(ir);
1396                 break;
1397         case PMC_EV_P4_MEMORY_CANCEL:
1398                 __P4SETMASK(memcan);
1399                 break;
1400         case PMC_EV_P4_MEMORY_COMPLETE:
1401                 __P4SETMASK(memcomp);
1402                 break;
1403         case PMC_EV_P4_LOAD_PORT_REPLAY:
1404                 __P4SETMASK(lpr);
1405                 break;
1406         case PMC_EV_P4_STORE_PORT_REPLAY:
1407                 __P4SETMASK(spr);
1408                 break;
1409         case PMC_EV_P4_MOB_LOAD_REPLAY:
1410                 __P4SETMASK(mlr);
1411                 break;
1412         case PMC_EV_P4_PAGE_WALK_TYPE:
1413                 __P4SETMASK(pwt);
1414                 break;
1415         case PMC_EV_P4_BSQ_CACHE_REFERENCE:
1416                 __P4SETMASK(bcr);
1417                 break;
1418         case PMC_EV_P4_IOQ_ALLOCATION:
1419                 __P4SETMASK(ia);
1420                 has_busreqtype = 1;
1421                 break;
1422         case PMC_EV_P4_IOQ_ACTIVE_ENTRIES:
1423                 __P4SETMASK(iae);
1424                 has_busreqtype = 1;
1425                 break;
1426         case PMC_EV_P4_FSB_DATA_ACTIVITY:
1427                 __P4SETMASK(fda);
1428                 break;
1429         case PMC_EV_P4_BSQ_ALLOCATION:
1430                 __P4SETMASK(ba);
1431                 break;
1432         case PMC_EV_P4_SSE_INPUT_ASSIST:
1433                 __P4SETMASK(sia);
1434                 break;
1435         case PMC_EV_P4_PACKED_SP_UOP:
1436                 __P4SETMASK(psu);
1437                 break;
1438         case PMC_EV_P4_PACKED_DP_UOP:
1439                 __P4SETMASK(pdu);
1440                 break;
1441         case PMC_EV_P4_SCALAR_SP_UOP:
1442                 __P4SETMASK(ssu);
1443                 break;
1444         case PMC_EV_P4_SCALAR_DP_UOP:
1445                 __P4SETMASK(sdu);
1446                 break;
1447         case PMC_EV_P4_64BIT_MMX_UOP:
1448                 __P4SETMASK(64bmu);
1449                 break;
1450         case PMC_EV_P4_128BIT_MMX_UOP:
1451                 __P4SETMASK(128bmu);
1452                 break;
1453         case PMC_EV_P4_X87_FP_UOP:
1454                 __P4SETMASK(xfu);
1455                 break;
1456         case PMC_EV_P4_X87_SIMD_MOVES_UOP:
1457                 __P4SETMASK(xsmu);
1458                 break;
1459         case PMC_EV_P4_GLOBAL_POWER_EVENTS:
1460                 __P4SETMASK(gpe);
1461                 break;
1462         case PMC_EV_P4_TC_MS_XFER:
1463                 __P4SETMASK(tmx);
1464                 break;
1465         case PMC_EV_P4_UOP_QUEUE_WRITES:
1466                 __P4SETMASK(uqw);
1467                 break;
1468         case PMC_EV_P4_RETIRED_MISPRED_BRANCH_TYPE:
1469                 __P4SETMASK(rmbt);
1470                 break;
1471         case PMC_EV_P4_RETIRED_BRANCH_TYPE:
1472                 __P4SETMASK(rbt);
1473                 break;
1474         case PMC_EV_P4_RESOURCE_STALL:
1475                 __P4SETMASK(rs);
1476                 break;
1477         case PMC_EV_P4_WC_BUFFER:
1478                 __P4SETMASK(wb);
1479                 break;
1480         case PMC_EV_P4_BSQ_ACTIVE_ENTRIES:
1481         case PMC_EV_P4_B2B_CYCLES:
1482         case PMC_EV_P4_BNR:
1483         case PMC_EV_P4_SNOOP:
1484         case PMC_EV_P4_RESPONSE:
1485                 break;
1486         case PMC_EV_P4_FRONT_END_EVENT:
1487                 __P4SETMASK(fee);
1488                 break;
1489         case PMC_EV_P4_EXECUTION_EVENT:
1490                 __P4SETMASK(ee);
1491                 break;
1492         case PMC_EV_P4_REPLAY_EVENT:
1493                 __P4SETMASK(re);
1494                 break;
1495         case PMC_EV_P4_INSTR_RETIRED:
1496                 __P4SETMASK(insret);
1497                 break;
1498         case PMC_EV_P4_UOPS_RETIRED:
1499                 __P4SETMASK(ur);
1500                 break;
1501         case PMC_EV_P4_UOP_TYPE:
1502                 __P4SETMASK(ut);
1503                 break;
1504         case PMC_EV_P4_BRANCH_RETIRED:
1505                 __P4SETMASK(br);
1506                 break;
1507         case PMC_EV_P4_MISPRED_BRANCH_RETIRED:
1508                 __P4SETMASK(mbr);
1509                 break;
1510         case PMC_EV_P4_X87_ASSIST:
1511                 __P4SETMASK(xa);
1512                 break;
1513         case PMC_EV_P4_MACHINE_CLEAR:
1514                 __P4SETMASK(machclr);
1515                 break;
1516         default:
1517                 return (-1);
1518         }
1519
1520         /* process additional flags */
1521         while ((p = strsep(&ctrspec, ",")) != NULL) {
1522                 if (KWPREFIXMATCH(p, P4_KW_ACTIVE)) {
1523                         q = strchr(p, '=');
1524                         if (*++q == '\0') /* skip '=' */
1525                                 return (-1);
1526
1527                         if (strcasecmp(q, P4_KW_ACTIVE_NONE) == 0)
1528                                 cccractivemask = 0x0;
1529                         else if (strcasecmp(q, P4_KW_ACTIVE_SINGLE) == 0)
1530                                 cccractivemask = 0x1;
1531                         else if (strcasecmp(q, P4_KW_ACTIVE_BOTH) == 0)
1532                                 cccractivemask = 0x2;
1533                         else if (strcasecmp(q, P4_KW_ACTIVE_ANY) == 0)
1534                                 cccractivemask = 0x3;
1535                         else
1536                                 return (-1);
1537
1538                 } else if (KWPREFIXMATCH(p, P4_KW_BUSREQTYPE)) {
1539                         if (has_busreqtype == 0)
1540                                 return (-1);
1541
1542                         q = strchr(p, '=');
1543                         if (*++q == '\0') /* skip '=' */
1544                                 return (-1);
1545
1546                         count = strtol(q, &e, 0);
1547                         if (e == q || *e != '\0')
1548                                 return (-1);
1549                         evmask = (evmask & ~0x1F) | (count & 0x1F);
1550                 } else if (KWMATCH(p, P4_KW_CASCADE))
1551                         pmc_config->pm_caps |= PMC_CAP_CASCADE;
1552                 else if (KWMATCH(p, P4_KW_EDGE))
1553                         pmc_config->pm_caps |= PMC_CAP_EDGE;
1554                 else if (KWMATCH(p, P4_KW_INV))
1555                         pmc_config->pm_caps |= PMC_CAP_INVERT;
1556                 else if (KWPREFIXMATCH(p, P4_KW_MASK "=")) {
1557                         if ((n = pmc_parse_mask(pmask, p, &evmask)) < 0)
1558                                 return (-1);
1559                         pmc_config->pm_caps |= PMC_CAP_QUALIFIER;
1560                 } else if (KWMATCH(p, P4_KW_OS))
1561                         pmc_config->pm_caps |= PMC_CAP_SYSTEM;
1562                 else if (KWMATCH(p, P4_KW_PRECISE))
1563                         pmc_config->pm_caps |= PMC_CAP_PRECISE;
1564                 else if (KWPREFIXMATCH(p, P4_KW_TAG "=")) {
1565                         if (has_tag == 0)
1566                                 return (-1);
1567
1568                         q = strchr(p, '=');
1569                         if (*++q == '\0') /* skip '=' */
1570                                 return (-1);
1571
1572                         count = strtol(q, &e, 0);
1573                         if (e == q || *e != '\0')
1574                                 return (-1);
1575
1576                         pmc_config->pm_caps |= PMC_CAP_TAGGING;
1577                         pmc_config->pm_md.pm_p4.pm_p4_escrconfig |=
1578                             P4_ESCR_TO_TAG_VALUE(count);
1579                 } else if (KWPREFIXMATCH(p, P4_KW_THRESHOLD "=")) {
1580                         q = strchr(p, '=');
1581                         if (*++q == '\0') /* skip '=' */
1582                                 return (-1);
1583
1584                         count = strtol(q, &e, 0);
1585                         if (e == q || *e != '\0')
1586                                 return (-1);
1587
1588                         pmc_config->pm_caps |= PMC_CAP_THRESHOLD;
1589                         pmc_config->pm_md.pm_p4.pm_p4_cccrconfig &=
1590                             ~P4_CCCR_THRESHOLD_MASK;
1591                         pmc_config->pm_md.pm_p4.pm_p4_cccrconfig |=
1592                             P4_CCCR_TO_THRESHOLD(count);
1593                 } else if (KWMATCH(p, P4_KW_USR))
1594                         pmc_config->pm_caps |= PMC_CAP_USER;
1595                 else
1596                         return (-1);
1597         }
1598
1599         /* other post processing */
1600         if (pe == PMC_EV_P4_IOQ_ALLOCATION ||
1601             pe == PMC_EV_P4_FSB_DATA_ACTIVITY ||
1602             pe == PMC_EV_P4_BSQ_ALLOCATION)
1603                 pmc_config->pm_caps |= PMC_CAP_EDGE;
1604
1605         /* fill in thread activity mask */
1606         pmc_config->pm_md.pm_p4.pm_p4_cccrconfig |=
1607             P4_CCCR_TO_ACTIVE_THREAD(cccractivemask);
1608
1609         if (evmask)
1610                 pmc_config->pm_caps |= PMC_CAP_QUALIFIER;
1611
1612         switch (pe) {
1613         case PMC_EV_P4_FSB_DATA_ACTIVITY:
1614                 if ((evmask & 0x06) == 0x06 ||
1615                     (evmask & 0x18) == 0x18)
1616                         return (-1); /* can't have own+other bits together */
1617                 if (evmask == 0) /* default:drdy-{drv,own}+dbsy{drv,own} */
1618                         evmask = 0x1D;
1619                 break;
1620         case PMC_EV_P4_MACHINE_CLEAR:
1621                 /* only one bit is allowed to be set */
1622                 if ((evmask & (evmask - 1)) != 0)
1623                         return (-1);
1624                 if (evmask == 0) {
1625                         evmask = 0x1;   /* 'CLEAR' */
1626                         pmc_config->pm_caps |= PMC_CAP_QUALIFIER;
1627                 }
1628                 break;
1629         default:
1630                 if (evmask == 0 && pmask) {
1631                         for (pm = pmask; pm->pm_name; pm++)
1632                                 evmask |= pm->pm_value;
1633                         pmc_config->pm_caps |= PMC_CAP_QUALIFIER;
1634                 }
1635         }
1636
1637         pmc_config->pm_md.pm_p4.pm_p4_escrconfig =
1638             P4_ESCR_TO_EVENT_MASK(evmask);
1639
1640         return (0);
1641 }
1642
1643 #endif
1644
1645 #if defined(__i386__)
1646
1647 /*
1648  * Pentium style PMCs
1649  */
1650
1651 static struct pmc_event_alias p5_aliases[] = {
1652         EV_ALIAS("branches",            "p5-taken-branches"),
1653         EV_ALIAS("cycles",              "tsc"),
1654         EV_ALIAS("dc-misses",           "p5-data-read-miss-or-write-miss"),
1655         EV_ALIAS("ic-misses",           "p5-code-cache-miss"),
1656         EV_ALIAS("instructions",        "p5-instructions-executed"),
1657         EV_ALIAS("interrupts",          "p5-hardware-interrupts"),
1658         EV_ALIAS("unhalted-cycles",
1659             "p5-number-of-cycles-not-in-halt-state"),
1660         EV_ALIAS(NULL, NULL)
1661 };
1662
1663 static int
1664 p5_allocate_pmc(enum pmc_event pe, char *ctrspec,
1665     struct pmc_op_pmcallocate *pmc_config)
1666 {
1667         return (-1 || pe || ctrspec || pmc_config); /* shut up gcc */
1668 }
1669
1670 /*
1671  * Pentium Pro style PMCs.  These PMCs are found in Pentium II, Pentium III,
1672  * and Pentium M CPUs.
1673  */
1674
1675 static struct pmc_event_alias p6_aliases[] = {
1676         EV_ALIAS("branches",            "p6-br-inst-retired"),
1677         EV_ALIAS("branch-mispredicts",  "p6-br-miss-pred-retired"),
1678         EV_ALIAS("cycles",              "tsc"),
1679         EV_ALIAS("dc-misses",           "p6-dcu-lines-in"),
1680         EV_ALIAS("ic-misses",           "p6-ifu-fetch-miss"),
1681         EV_ALIAS("instructions",        "p6-inst-retired"),
1682         EV_ALIAS("interrupts",          "p6-hw-int-rx"),
1683         EV_ALIAS("unhalted-cycles",     "p6-cpu-clk-unhalted"),
1684         EV_ALIAS(NULL, NULL)
1685 };
1686
1687 #define P6_KW_CMASK     "cmask"
1688 #define P6_KW_EDGE      "edge"
1689 #define P6_KW_INV       "inv"
1690 #define P6_KW_OS        "os"
1691 #define P6_KW_UMASK     "umask"
1692 #define P6_KW_USR       "usr"
1693
1694 static struct pmc_masks p6_mask_mesi[] = {
1695         PMCMASK(m,      0x01),
1696         PMCMASK(e,      0x02),
1697         PMCMASK(s,      0x04),
1698         PMCMASK(i,      0x08),
1699         NULLMASK
1700 };
1701
1702 static struct pmc_masks p6_mask_mesihw[] = {
1703         PMCMASK(m,      0x01),
1704         PMCMASK(e,      0x02),
1705         PMCMASK(s,      0x04),
1706         PMCMASK(i,      0x08),
1707         PMCMASK(nonhw,  0x00),
1708         PMCMASK(hw,     0x10),
1709         PMCMASK(both,   0x30),
1710         NULLMASK
1711 };
1712
1713 static struct pmc_masks p6_mask_hw[] = {
1714         PMCMASK(nonhw,  0x00),
1715         PMCMASK(hw,     0x10),
1716         PMCMASK(both,   0x30),
1717         NULLMASK
1718 };
1719
1720 static struct pmc_masks p6_mask_any[] = {
1721         PMCMASK(self,   0x00),
1722         PMCMASK(any,    0x20),
1723         NULLMASK
1724 };
1725
1726 static struct pmc_masks p6_mask_ekp[] = {
1727         PMCMASK(nta,    0x00),
1728         PMCMASK(t1,     0x01),
1729         PMCMASK(t2,     0x02),
1730         PMCMASK(wos,    0x03),
1731         NULLMASK
1732 };
1733
1734 static struct pmc_masks p6_mask_pps[] = {
1735         PMCMASK(packed-and-scalar, 0x00),
1736         PMCMASK(scalar, 0x01),
1737         NULLMASK
1738 };
1739
1740 static struct pmc_masks p6_mask_mite[] = {
1741         PMCMASK(packed-multiply,         0x01),
1742         PMCMASK(packed-shift,           0x02),
1743         PMCMASK(pack,                   0x04),
1744         PMCMASK(unpack,                 0x08),
1745         PMCMASK(packed-logical,         0x10),
1746         PMCMASK(packed-arithmetic,      0x20),
1747         NULLMASK
1748 };
1749
1750 static struct pmc_masks p6_mask_fmt[] = {
1751         PMCMASK(mmxtofp,        0x00),
1752         PMCMASK(fptommx,        0x01),
1753         NULLMASK
1754 };
1755
1756 static struct pmc_masks p6_mask_sr[] = {
1757         PMCMASK(es,     0x01),
1758         PMCMASK(ds,     0x02),
1759         PMCMASK(fs,     0x04),
1760         PMCMASK(gs,     0x08),
1761         NULLMASK
1762 };
1763
1764 static struct pmc_masks p6_mask_eet[] = {
1765         PMCMASK(all,    0x00),
1766         PMCMASK(freq,   0x02),
1767         NULLMASK
1768 };
1769
1770 static struct pmc_masks p6_mask_efur[] = {
1771         PMCMASK(all,    0x00),
1772         PMCMASK(loadop, 0x01),
1773         PMCMASK(stdsta, 0x02),
1774         NULLMASK
1775 };
1776
1777 static struct pmc_masks p6_mask_essir[] = {
1778         PMCMASK(sse-packed-single,      0x00),
1779         PMCMASK(sse-packed-single-scalar-single, 0x01),
1780         PMCMASK(sse2-packed-double,     0x02),
1781         PMCMASK(sse2-scalar-double,     0x03),
1782         NULLMASK
1783 };
1784
1785 static struct pmc_masks p6_mask_esscir[] = {
1786         PMCMASK(sse-packed-single,      0x00),
1787         PMCMASK(sse-scalar-single,      0x01),
1788         PMCMASK(sse2-packed-double,     0x02),
1789         PMCMASK(sse2-scalar-double,     0x03),
1790         NULLMASK
1791 };
1792
1793 /* P6 event parser */
1794 static int
1795 p6_allocate_pmc(enum pmc_event pe, char *ctrspec,
1796     struct pmc_op_pmcallocate *pmc_config)
1797 {
1798         char *e, *p, *q;
1799         uint32_t evmask;
1800         int count, n;
1801         const struct pmc_masks *pm, *pmask;
1802
1803         pmc_config->pm_caps |= (PMC_CAP_READ | PMC_CAP_WRITE);
1804         pmc_config->pm_md.pm_ppro.pm_ppro_config = 0;
1805
1806         evmask = 0;
1807
1808 #define P6MASKSET(M)    pmask = p6_mask_ ## M
1809
1810         switch(pe) {
1811         case PMC_EV_P6_L2_IFETCH:       P6MASKSET(mesi); break;
1812         case PMC_EV_P6_L2_LD:           P6MASKSET(mesi); break;
1813         case PMC_EV_P6_L2_ST:           P6MASKSET(mesi); break;
1814         case PMC_EV_P6_L2_RQSTS:        P6MASKSET(mesi); break;
1815         case PMC_EV_P6_BUS_DRDY_CLOCKS:
1816         case PMC_EV_P6_BUS_LOCK_CLOCKS:
1817         case PMC_EV_P6_BUS_TRAN_BRD:
1818         case PMC_EV_P6_BUS_TRAN_RFO:
1819         case PMC_EV_P6_BUS_TRANS_WB:
1820         case PMC_EV_P6_BUS_TRAN_IFETCH:
1821         case PMC_EV_P6_BUS_TRAN_INVAL:
1822         case PMC_EV_P6_BUS_TRAN_PWR:
1823         case PMC_EV_P6_BUS_TRANS_P:
1824         case PMC_EV_P6_BUS_TRANS_IO:
1825         case PMC_EV_P6_BUS_TRAN_DEF:
1826         case PMC_EV_P6_BUS_TRAN_BURST:
1827         case PMC_EV_P6_BUS_TRAN_ANY:
1828         case PMC_EV_P6_BUS_TRAN_MEM:
1829                 P6MASKSET(any); break;
1830         case PMC_EV_P6_EMON_KNI_PREF_DISPATCHED:
1831         case PMC_EV_P6_EMON_KNI_PREF_MISS:
1832                 P6MASKSET(ekp); break;
1833         case PMC_EV_P6_EMON_KNI_INST_RETIRED:
1834         case PMC_EV_P6_EMON_KNI_COMP_INST_RET:
1835                 P6MASKSET(pps); break;
1836         case PMC_EV_P6_MMX_INSTR_TYPE_EXEC:
1837                 P6MASKSET(mite); break;
1838         case PMC_EV_P6_FP_MMX_TRANS:
1839                 P6MASKSET(fmt); break;
1840         case PMC_EV_P6_SEG_RENAME_STALLS:
1841         case PMC_EV_P6_SEG_REG_RENAMES:
1842                 P6MASKSET(sr);  break;
1843         case PMC_EV_P6_EMON_EST_TRANS:
1844                 P6MASKSET(eet); break;
1845         case PMC_EV_P6_EMON_FUSED_UOPS_RET:
1846                 P6MASKSET(efur); break;
1847         case PMC_EV_P6_EMON_SSE_SSE2_INST_RETIRED:
1848                 P6MASKSET(essir); break;
1849         case PMC_EV_P6_EMON_SSE_SSE2_COMP_INST_RETIRED:
1850                 P6MASKSET(esscir); break;
1851         default:
1852                 pmask = NULL;
1853                 break;
1854         }
1855
1856         /* Pentium M PMCs have a few events with different semantics */
1857         if (cpu_info.pm_cputype == PMC_CPU_INTEL_PM) {
1858                 if (pe == PMC_EV_P6_L2_LD ||
1859                     pe == PMC_EV_P6_L2_LINES_IN ||
1860                     pe == PMC_EV_P6_L2_LINES_OUT)
1861                         P6MASKSET(mesihw);
1862                 else if (pe == PMC_EV_P6_L2_M_LINES_OUTM)
1863                         P6MASKSET(hw);
1864         }
1865
1866         /* Parse additional modifiers if present */
1867         while ((p = strsep(&ctrspec, ",")) != NULL) {
1868                 if (KWPREFIXMATCH(p, P6_KW_CMASK "=")) {
1869                         q = strchr(p, '=');
1870                         if (*++q == '\0') /* skip '=' */
1871                                 return (-1);
1872                         count = strtol(q, &e, 0);
1873                         if (e == q || *e != '\0')
1874                                 return (-1);
1875                         pmc_config->pm_caps |= PMC_CAP_THRESHOLD;
1876                         pmc_config->pm_md.pm_ppro.pm_ppro_config |=
1877                             P6_EVSEL_TO_CMASK(count);
1878                 } else if (KWMATCH(p, P6_KW_EDGE)) {
1879                         pmc_config->pm_caps |= PMC_CAP_EDGE;
1880                 } else if (KWMATCH(p, P6_KW_INV)) {
1881                         pmc_config->pm_caps |= PMC_CAP_INVERT;
1882                 } else if (KWMATCH(p, P6_KW_OS)) {
1883                         pmc_config->pm_caps |= PMC_CAP_SYSTEM;
1884                 } else if (KWPREFIXMATCH(p, P6_KW_UMASK "=")) {
1885                         evmask = 0;
1886                         if ((n = pmc_parse_mask(pmask, p, &evmask)) < 0)
1887                                 return (-1);
1888                         if ((pe == PMC_EV_P6_BUS_DRDY_CLOCKS ||
1889                              pe == PMC_EV_P6_BUS_LOCK_CLOCKS ||
1890                              pe == PMC_EV_P6_BUS_TRAN_BRD ||
1891                              pe == PMC_EV_P6_BUS_TRAN_RFO ||
1892                              pe == PMC_EV_P6_BUS_TRAN_IFETCH ||
1893                              pe == PMC_EV_P6_BUS_TRAN_INVAL ||
1894                              pe == PMC_EV_P6_BUS_TRAN_PWR ||
1895                              pe == PMC_EV_P6_BUS_TRAN_DEF ||
1896                              pe == PMC_EV_P6_BUS_TRAN_BURST ||
1897                              pe == PMC_EV_P6_BUS_TRAN_ANY ||
1898                              pe == PMC_EV_P6_BUS_TRAN_MEM ||
1899                              pe == PMC_EV_P6_BUS_TRANS_IO ||
1900                              pe == PMC_EV_P6_BUS_TRANS_P ||
1901                              pe == PMC_EV_P6_BUS_TRANS_WB ||
1902                              pe == PMC_EV_P6_EMON_EST_TRANS ||
1903                              pe == PMC_EV_P6_EMON_FUSED_UOPS_RET ||
1904                              pe == PMC_EV_P6_EMON_KNI_COMP_INST_RET ||
1905                              pe == PMC_EV_P6_EMON_KNI_INST_RETIRED ||
1906                              pe == PMC_EV_P6_EMON_KNI_PREF_DISPATCHED ||
1907                              pe == PMC_EV_P6_EMON_KNI_PREF_MISS ||
1908                              pe == PMC_EV_P6_EMON_SSE_SSE2_COMP_INST_RETIRED ||
1909                              pe == PMC_EV_P6_EMON_SSE_SSE2_INST_RETIRED ||
1910                              pe == PMC_EV_P6_FP_MMX_TRANS)
1911                             && (n > 1)) /* Only one mask keyword is allowed. */
1912                                 return (-1);
1913                         pmc_config->pm_caps |= PMC_CAP_QUALIFIER;
1914                 } else if (KWMATCH(p, P6_KW_USR)) {
1915                         pmc_config->pm_caps |= PMC_CAP_USER;
1916                 } else
1917                         return (-1);
1918         }
1919
1920         /* post processing */
1921         switch (pe) {
1922
1923                 /*
1924                  * The following events default to an evmask of 0
1925                  */
1926
1927                 /* default => 'self' */
1928         case PMC_EV_P6_BUS_DRDY_CLOCKS:
1929         case PMC_EV_P6_BUS_LOCK_CLOCKS:
1930         case PMC_EV_P6_BUS_TRAN_BRD:
1931         case PMC_EV_P6_BUS_TRAN_RFO:
1932         case PMC_EV_P6_BUS_TRANS_WB:
1933         case PMC_EV_P6_BUS_TRAN_IFETCH:
1934         case PMC_EV_P6_BUS_TRAN_INVAL:
1935         case PMC_EV_P6_BUS_TRAN_PWR:
1936         case PMC_EV_P6_BUS_TRANS_P:
1937         case PMC_EV_P6_BUS_TRANS_IO:
1938         case PMC_EV_P6_BUS_TRAN_DEF:
1939         case PMC_EV_P6_BUS_TRAN_BURST:
1940         case PMC_EV_P6_BUS_TRAN_ANY:
1941         case PMC_EV_P6_BUS_TRAN_MEM:
1942
1943                 /* default => 'nta' */
1944         case PMC_EV_P6_EMON_KNI_PREF_DISPATCHED:
1945         case PMC_EV_P6_EMON_KNI_PREF_MISS:
1946
1947                 /* default => 'packed and scalar' */
1948         case PMC_EV_P6_EMON_KNI_INST_RETIRED:
1949         case PMC_EV_P6_EMON_KNI_COMP_INST_RET:
1950
1951                 /* default => 'mmx to fp transitions' */
1952         case PMC_EV_P6_FP_MMX_TRANS:
1953
1954                 /* default => 'SSE Packed Single' */
1955         case PMC_EV_P6_EMON_SSE_SSE2_INST_RETIRED:
1956         case PMC_EV_P6_EMON_SSE_SSE2_COMP_INST_RETIRED:
1957
1958                 /* default => 'all fused micro-ops' */
1959         case PMC_EV_P6_EMON_FUSED_UOPS_RET:
1960
1961                 /* default => 'all transitions' */
1962         case PMC_EV_P6_EMON_EST_TRANS:
1963                 break;
1964
1965         case PMC_EV_P6_MMX_UOPS_EXEC:
1966                 evmask = 0x0F;          /* only value allowed */
1967                 break;
1968
1969         default:
1970                 /*
1971                  * For all other events, set the default event mask
1972                  * to a logical OR of all the allowed event mask bits.
1973                  */
1974                 if (evmask == 0 && pmask) {
1975                         for (pm = pmask; pm->pm_name; pm++)
1976                                 evmask |= pm->pm_value;
1977                         pmc_config->pm_caps |= PMC_CAP_QUALIFIER;
1978                 }
1979
1980                 break;
1981         }
1982
1983         if (pmc_config->pm_caps & PMC_CAP_QUALIFIER)
1984                 pmc_config->pm_md.pm_ppro.pm_ppro_config |=
1985                     P6_EVSEL_TO_UMASK(evmask);
1986
1987         return (0);
1988 }
1989
1990 #endif
1991
1992 #if     defined(__i386__) || defined(__amd64__)
1993 static int
1994 tsc_allocate_pmc(enum pmc_event pe, char *ctrspec,
1995     struct pmc_op_pmcallocate *pmc_config)
1996 {
1997         if (pe != PMC_EV_TSC_TSC)
1998                 return (-1);
1999
2000         /* TSC events must be unqualified. */
2001         if (ctrspec && *ctrspec != '\0')
2002                 return (-1);
2003
2004         pmc_config->pm_md.pm_amd.pm_amd_config = 0;
2005         pmc_config->pm_caps |= PMC_CAP_READ;
2006
2007         return (0);
2008 }
2009 #endif
2010
2011 /*
2012  * Match an event name `name' with its canonical form.
2013  *
2014  * Matches are case insensitive and spaces, periods, underscores and
2015  * hyphen characters are considered to match each other.
2016  *
2017  * Returns 1 for a match, 0 otherwise.
2018  */
2019
2020 static int
2021 pmc_match_event_name(const char *name, const char *canonicalname)
2022 {
2023         int cc, nc;
2024         const unsigned char *c, *n;
2025
2026         c = (const unsigned char *) canonicalname;
2027         n = (const unsigned char *) name;
2028
2029         for (; (nc = *n) && (cc = *c); n++, c++) {
2030
2031                 if ((nc == ' ' || nc == '_' || nc == '-' || nc == '.') &&
2032                     (cc == ' ' || cc == '_' || cc == '-' || cc == '.'))
2033                         continue;
2034
2035                 if (toupper(nc) == toupper(cc))
2036                         continue;
2037
2038
2039                 return (0);
2040         }
2041
2042         if (*n == '\0' && *c == '\0')
2043                 return (1);
2044
2045         return (0);
2046 }
2047
2048 /*
2049  * Match an event name against all the event named supported by a
2050  * PMC class.
2051  *
2052  * Returns an event descriptor pointer on match or NULL otherwise.
2053  */
2054 static const struct pmc_event_descr *
2055 pmc_match_event_class(const char *name,
2056     const struct pmc_class_descr *pcd)
2057 {
2058         size_t n;
2059         const struct pmc_event_descr *ev;
2060
2061         ev = pcd->pm_evc_event_table;
2062         for (n = 0; n < pcd->pm_evc_event_table_size; n++, ev++)
2063                 if (pmc_match_event_name(name, ev->pm_ev_name))
2064                         return (ev);
2065
2066         return (NULL);
2067 }
2068
2069 static int
2070 pmc_mdep_is_compatible_class(enum pmc_class pc)
2071 {
2072         size_t n;
2073
2074         for (n = 0; n < pmc_mdep_class_list_size; n++)
2075                 if (pmc_mdep_class_list[n] == pc)
2076                         return (1);
2077         return (0);
2078 }
2079
2080 /*
2081  * API entry points
2082  */
2083
2084 int
2085 pmc_allocate(const char *ctrspec, enum pmc_mode mode,
2086     uint32_t flags, int cpu, pmc_id_t *pmcid)
2087 {
2088         size_t n;
2089         int retval;
2090         char *r, *spec_copy;
2091         const char *ctrname;
2092         const struct pmc_event_descr *ev;
2093         const struct pmc_event_alias *alias;
2094         struct pmc_op_pmcallocate pmc_config;
2095         const struct pmc_class_descr *pcd;
2096
2097         spec_copy = NULL;
2098         retval    = -1;
2099
2100         if (mode != PMC_MODE_SS && mode != PMC_MODE_TS &&
2101             mode != PMC_MODE_SC && mode != PMC_MODE_TC) {
2102                 errno = EINVAL;
2103                 goto out;
2104         }
2105
2106         /* replace an event alias with the canonical event specifier */
2107         if (pmc_mdep_event_aliases)
2108                 for (alias = pmc_mdep_event_aliases; alias->pm_alias; alias++)
2109                         if (!strcasecmp(ctrspec, alias->pm_alias)) {
2110                                 spec_copy = strdup(alias->pm_spec);
2111                                 break;
2112                         }
2113
2114         if (spec_copy == NULL)
2115                 spec_copy = strdup(ctrspec);
2116
2117         r = spec_copy;
2118         ctrname = strsep(&r, ",");
2119
2120         /*
2121          * If a explicit class prefix was given by the user, restrict the
2122          * search for the event to the specified PMC class.
2123          */
2124         ev = NULL;
2125         for (n = 0; n < PMC_CLASS_TABLE_SIZE; n++) {
2126                 pcd = pmc_class_table[n];
2127                 if (pmc_mdep_is_compatible_class(pcd->pm_evc_class) &&
2128                     strncasecmp(ctrname, pcd->pm_evc_name,
2129                                 pcd->pm_evc_name_size) == 0) {
2130                         if ((ev = pmc_match_event_class(ctrname +
2131                             pcd->pm_evc_name_size, pcd)) == NULL) {
2132                                 errno = EINVAL;
2133                                 goto out;
2134                         }
2135                         break;
2136                 }
2137         }
2138
2139         /*
2140          * Otherwise, search for this event in all compatible PMC
2141          * classes.
2142          */
2143         for (n = 0; ev == NULL && n < PMC_CLASS_TABLE_SIZE; n++) {
2144                 pcd = pmc_class_table[n];
2145                 if (pmc_mdep_is_compatible_class(pcd->pm_evc_class))
2146                         ev = pmc_match_event_class(ctrname, pcd);
2147         }
2148
2149         if (ev == NULL) {
2150                 errno = EINVAL;
2151                 goto out;
2152         }
2153
2154         bzero(&pmc_config, sizeof(pmc_config));
2155         pmc_config.pm_ev    = ev->pm_ev_code;
2156         pmc_config.pm_class = pcd->pm_evc_class;
2157         pmc_config.pm_cpu   = cpu;
2158         pmc_config.pm_mode  = mode;
2159         pmc_config.pm_flags = flags;
2160
2161         if (PMC_IS_SAMPLING_MODE(mode))
2162                 pmc_config.pm_caps |= PMC_CAP_INTERRUPT;
2163
2164         if (pcd->pm_evc_allocate_pmc(ev->pm_ev_code, r, &pmc_config) < 0) {
2165                 errno = EINVAL;
2166                 goto out;
2167         }
2168
2169         if (PMC_CALL(PMCALLOCATE, &pmc_config) < 0)
2170                 goto out;
2171
2172         *pmcid = pmc_config.pm_pmcid;
2173
2174         retval = 0;
2175
2176  out:
2177         if (spec_copy)
2178                 free(spec_copy);
2179
2180         return (retval);
2181 }
2182
2183 int
2184 pmc_attach(pmc_id_t pmc, pid_t pid)
2185 {
2186         struct pmc_op_pmcattach pmc_attach_args;
2187
2188         pmc_attach_args.pm_pmc = pmc;
2189         pmc_attach_args.pm_pid = pid;
2190
2191         return (PMC_CALL(PMCATTACH, &pmc_attach_args));
2192 }
2193
2194 int
2195 pmc_capabilities(pmc_id_t pmcid, uint32_t *caps)
2196 {
2197         unsigned int i;
2198         enum pmc_class cl;
2199
2200         cl = PMC_ID_TO_CLASS(pmcid);
2201         for (i = 0; i < cpu_info.pm_nclass; i++)
2202                 if (cpu_info.pm_classes[i].pm_class == cl) {
2203                         *caps = cpu_info.pm_classes[i].pm_caps;
2204                         return (0);
2205                 }
2206         errno = EINVAL;
2207         return (-1);
2208 }
2209
2210 int
2211 pmc_configure_logfile(int fd)
2212 {
2213         struct pmc_op_configurelog cla;
2214
2215         cla.pm_logfd = fd;
2216         if (PMC_CALL(CONFIGURELOG, &cla) < 0)
2217                 return (-1);
2218         return (0);
2219 }
2220
2221 int
2222 pmc_cpuinfo(const struct pmc_cpuinfo **pci)
2223 {
2224         if (pmc_syscall == -1) {
2225                 errno = ENXIO;
2226                 return (-1);
2227         }
2228
2229         *pci = &cpu_info;
2230         return (0);
2231 }
2232
2233 int
2234 pmc_detach(pmc_id_t pmc, pid_t pid)
2235 {
2236         struct pmc_op_pmcattach pmc_detach_args;
2237
2238         pmc_detach_args.pm_pmc = pmc;
2239         pmc_detach_args.pm_pid = pid;
2240         return (PMC_CALL(PMCDETACH, &pmc_detach_args));
2241 }
2242
2243 int
2244 pmc_disable(int cpu, int pmc)
2245 {
2246         struct pmc_op_pmcadmin ssa;
2247
2248         ssa.pm_cpu = cpu;
2249         ssa.pm_pmc = pmc;
2250         ssa.pm_state = PMC_STATE_DISABLED;
2251         return (PMC_CALL(PMCADMIN, &ssa));
2252 }
2253
2254 int
2255 pmc_enable(int cpu, int pmc)
2256 {
2257         struct pmc_op_pmcadmin ssa;
2258
2259         ssa.pm_cpu = cpu;
2260         ssa.pm_pmc = pmc;
2261         ssa.pm_state = PMC_STATE_FREE;
2262         return (PMC_CALL(PMCADMIN, &ssa));
2263 }
2264
2265 /*
2266  * Return a list of events known to a given PMC class.  'cl' is the
2267  * PMC class identifier, 'eventnames' is the returned list of 'const
2268  * char *' pointers pointing to the names of the events. 'nevents' is
2269  * the number of event name pointers returned.
2270  *
2271  * The space for 'eventnames' is allocated using malloc(3).  The caller
2272  * is responsible for freeing this space when done.
2273  */
2274 int
2275 pmc_event_names_of_class(enum pmc_class cl, const char ***eventnames,
2276     int *nevents)
2277 {
2278         int count;
2279         const char **names;
2280         const struct pmc_event_descr *ev;
2281
2282         switch (cl)
2283         {
2284         case PMC_CLASS_IAF:
2285                 ev = iaf_event_table;
2286                 count = PMC_EVENT_TABLE_SIZE(iaf);
2287                 break;
2288         case PMC_CLASS_IAP:
2289                 /*
2290                  * Return the most appropriate set of event name
2291                  * spellings for the current CPU.
2292                  */
2293                 switch (cpu_info.pm_cputype) {
2294                 default:
2295                 case PMC_CPU_INTEL_ATOM:
2296                         ev = atom_event_table;
2297                         count = PMC_EVENT_TABLE_SIZE(atom);
2298                         break;
2299                 case PMC_CPU_INTEL_CORE:
2300                         ev = core_event_table;
2301                         count = PMC_EVENT_TABLE_SIZE(core);
2302                         break;
2303                 case PMC_CPU_INTEL_CORE2:
2304                 case PMC_CPU_INTEL_CORE2EXTREME:
2305                         ev = core2_event_table;
2306                         count = PMC_EVENT_TABLE_SIZE(core2);
2307                         break;
2308                 case PMC_CPU_INTEL_COREI7:
2309                         ev = corei7_event_table;
2310                         count = PMC_EVENT_TABLE_SIZE(corei7);
2311                         break;
2312                 }
2313                 break;
2314         case PMC_CLASS_TSC:
2315                 ev = tsc_event_table;
2316                 count = PMC_EVENT_TABLE_SIZE(tsc);
2317                 break;
2318         case PMC_CLASS_K7:
2319                 ev = k7_event_table;
2320                 count = PMC_EVENT_TABLE_SIZE(k7);
2321                 break;
2322         case PMC_CLASS_K8:
2323                 ev = k8_event_table;
2324                 count = PMC_EVENT_TABLE_SIZE(k8);
2325                 break;
2326         case PMC_CLASS_P4:
2327                 ev = p4_event_table;
2328                 count = PMC_EVENT_TABLE_SIZE(p4);
2329                 break;
2330         case PMC_CLASS_P5:
2331                 ev = p5_event_table;
2332                 count = PMC_EVENT_TABLE_SIZE(p5);
2333                 break;
2334         case PMC_CLASS_P6:
2335                 ev = p6_event_table;
2336                 count = PMC_EVENT_TABLE_SIZE(p6);
2337                 break;
2338         default:
2339                 errno = EINVAL;
2340                 return (-1);
2341         }
2342
2343         if ((names = malloc(count * sizeof(const char *))) == NULL)
2344                 return (-1);
2345
2346         *eventnames = names;
2347         *nevents = count;
2348
2349         for (;count--; ev++, names++)
2350                 *names = ev->pm_ev_name;
2351         return (0);
2352 }
2353
2354 int
2355 pmc_flush_logfile(void)
2356 {
2357         return (PMC_CALL(FLUSHLOG,0));
2358 }
2359
2360 int
2361 pmc_get_driver_stats(struct pmc_driverstats *ds)
2362 {
2363         struct pmc_op_getdriverstats gms;
2364
2365         if (PMC_CALL(GETDRIVERSTATS, &gms) < 0)
2366                 return (-1);
2367
2368         /* copy out fields in the current userland<->library interface */
2369         ds->pm_intr_ignored    = gms.pm_intr_ignored;
2370         ds->pm_intr_processed  = gms.pm_intr_processed;
2371         ds->pm_intr_bufferfull = gms.pm_intr_bufferfull;
2372         ds->pm_syscalls        = gms.pm_syscalls;
2373         ds->pm_syscall_errors  = gms.pm_syscall_errors;
2374         ds->pm_buffer_requests = gms.pm_buffer_requests;
2375         ds->pm_buffer_requests_failed = gms.pm_buffer_requests_failed;
2376         ds->pm_log_sweeps      = gms.pm_log_sweeps;
2377         return (0);
2378 }
2379
2380 int
2381 pmc_get_msr(pmc_id_t pmc, uint32_t *msr)
2382 {
2383         struct pmc_op_getmsr gm;
2384
2385         gm.pm_pmcid = pmc;
2386         if (PMC_CALL(PMCGETMSR, &gm) < 0)
2387                 return (-1);
2388         *msr = gm.pm_msr;
2389         return (0);
2390 }
2391
2392 int
2393 pmc_init(void)
2394 {
2395         int error, pmc_mod_id;
2396         unsigned int n;
2397         uint32_t abi_version;
2398         struct module_stat pmc_modstat;
2399         struct pmc_op_getcpuinfo op_cpu_info;
2400 #if defined(__amd64__) || defined(__i386__)
2401         int cpu_has_iaf_counters;
2402         unsigned int t;
2403 #endif
2404
2405         if (pmc_syscall != -1) /* already inited */
2406                 return (0);
2407
2408         /* retrieve the system call number from the KLD */
2409         if ((pmc_mod_id = modfind(PMC_MODULE_NAME)) < 0)
2410                 return (-1);
2411
2412         pmc_modstat.version = sizeof(struct module_stat);
2413         if ((error = modstat(pmc_mod_id, &pmc_modstat)) < 0)
2414                 return (-1);
2415
2416         pmc_syscall = pmc_modstat.data.intval;
2417
2418         /* check the kernel module's ABI against our compiled-in version */
2419         abi_version = PMC_VERSION;
2420         if (PMC_CALL(GETMODULEVERSION, &abi_version) < 0)
2421                 return (pmc_syscall = -1);
2422
2423         /* ignore patch & minor numbers for the comparision */
2424         if ((abi_version & 0xFF000000) != (PMC_VERSION & 0xFF000000)) {
2425                 errno  = EPROGMISMATCH;
2426                 return (pmc_syscall = -1);
2427         }
2428
2429         if (PMC_CALL(GETCPUINFO, &op_cpu_info) < 0)
2430                 return (pmc_syscall = -1);
2431
2432         cpu_info.pm_cputype = op_cpu_info.pm_cputype;
2433         cpu_info.pm_ncpu    = op_cpu_info.pm_ncpu;
2434         cpu_info.pm_npmc    = op_cpu_info.pm_npmc;
2435         cpu_info.pm_nclass  = op_cpu_info.pm_nclass;
2436         for (n = 0; n < cpu_info.pm_nclass; n++)
2437                 cpu_info.pm_classes[n] = op_cpu_info.pm_classes[n];
2438
2439         pmc_class_table = malloc(PMC_CLASS_TABLE_SIZE *
2440             sizeof(struct pmc_class_descr *));
2441
2442         if (pmc_class_table == NULL)
2443                 return (-1);
2444
2445         for (n = 0; n < PMC_CLASS_TABLE_SIZE; n++)
2446                 pmc_class_table[n] = NULL;
2447
2448         /*
2449          * Fill in the class table.
2450          */
2451         n = 0;
2452 #if defined(__amd64__) || defined(__i386__)
2453         pmc_class_table[n++] = &tsc_class_table_descr;
2454
2455         /*
2456          * Check if this CPU has fixed function counters.
2457          */
2458         cpu_has_iaf_counters = 0;
2459         for (t = 0; t < cpu_info.pm_nclass; t++)
2460                 if (cpu_info.pm_classes[t].pm_class == PMC_CLASS_IAF)
2461                         cpu_has_iaf_counters = 1;
2462 #endif
2463
2464 #define PMC_MDEP_INIT(C) do {                                   \
2465                 pmc_mdep_event_aliases    = C##_aliases;        \
2466                 pmc_mdep_class_list  = C##_pmc_classes;         \
2467                 pmc_mdep_class_list_size =                      \
2468                     PMC_TABLE_SIZE(C##_pmc_classes);            \
2469         } while (0)
2470
2471 #define PMC_MDEP_INIT_INTEL_V2(C) do {                                  \
2472                 PMC_MDEP_INIT(C);                                       \
2473                 if (cpu_has_iaf_counters)                               \
2474                         pmc_class_table[n++] = &iaf_class_table_descr;  \
2475                 else                                                    \
2476                         pmc_mdep_event_aliases =                        \
2477                                 C##_aliases_without_iaf;                \
2478                 pmc_class_table[n] = &C##_class_table_descr;            \
2479         } while (0)
2480
2481         /* Configure the event name parser. */
2482         switch (cpu_info.pm_cputype) {
2483 #if defined(__i386__)
2484         case PMC_CPU_AMD_K7:
2485                 PMC_MDEP_INIT(k7);
2486                 pmc_class_table[n] = &k7_class_table_descr;
2487                 break;
2488         case PMC_CPU_INTEL_P5:
2489                 PMC_MDEP_INIT(p5);
2490                 pmc_class_table[n]  = &p5_class_table_descr;
2491                 break;
2492         case PMC_CPU_INTEL_P6:          /* P6 ... Pentium M CPUs have */
2493         case PMC_CPU_INTEL_PII:         /* similar PMCs. */
2494         case PMC_CPU_INTEL_PIII:
2495         case PMC_CPU_INTEL_PM:
2496                 PMC_MDEP_INIT(p6);
2497                 pmc_class_table[n] = &p6_class_table_descr;
2498                 break;
2499 #endif
2500 #if defined(__amd64__) || defined(__i386__)
2501         case PMC_CPU_AMD_K8:
2502                 PMC_MDEP_INIT(k8);
2503                 pmc_class_table[n] = &k8_class_table_descr;
2504                 break;
2505         case PMC_CPU_INTEL_ATOM:
2506                 PMC_MDEP_INIT_INTEL_V2(atom);
2507                 break;
2508         case PMC_CPU_INTEL_CORE:
2509                 PMC_MDEP_INIT(core);
2510                 break;
2511         case PMC_CPU_INTEL_CORE2:
2512         case PMC_CPU_INTEL_CORE2EXTREME:
2513                 PMC_MDEP_INIT_INTEL_V2(core2);
2514                 break;
2515         case PMC_CPU_INTEL_COREI7:
2516                 PMC_MDEP_INIT_INTEL_V2(corei7);
2517                 break;
2518         case PMC_CPU_INTEL_PIV:
2519                 PMC_MDEP_INIT(p4);
2520                 pmc_class_table[n] = &p4_class_table_descr;
2521                 break;
2522 #endif
2523
2524
2525         default:
2526                 /*
2527                  * Some kind of CPU this version of the library knows nothing
2528                  * about.  This shouldn't happen since the abi version check
2529                  * should have caught this.
2530                  */
2531                 errno = ENXIO;
2532                 return (pmc_syscall = -1);
2533         }
2534
2535         return (0);
2536 }
2537
2538 const char *
2539 pmc_name_of_capability(enum pmc_caps cap)
2540 {
2541         int i;
2542
2543         /*
2544          * 'cap' should have a single bit set and should be in
2545          * range.
2546          */
2547         if ((cap & (cap - 1)) || cap < PMC_CAP_FIRST ||
2548             cap > PMC_CAP_LAST) {
2549                 errno = EINVAL;
2550                 return (NULL);
2551         }
2552
2553         i = ffs(cap);
2554         return (pmc_capability_names[i - 1]);
2555 }
2556
2557 const char *
2558 pmc_name_of_class(enum pmc_class pc)
2559 {
2560         if ((int) pc >= PMC_CLASS_FIRST &&
2561             pc <= PMC_CLASS_LAST)
2562                 return (pmc_class_names[pc]);
2563
2564         errno = EINVAL;
2565         return (NULL);
2566 }
2567
2568 const char *
2569 pmc_name_of_cputype(enum pmc_cputype cp)
2570 {
2571         size_t n;
2572
2573         for (n = 0; n < PMC_TABLE_SIZE(pmc_cputype_names); n++)
2574                 if (cp == pmc_cputype_names[n].pm_cputype)
2575                         return (pmc_cputype_names[n].pm_name);
2576
2577         errno = EINVAL;
2578         return (NULL);
2579 }
2580
2581 const char *
2582 pmc_name_of_disposition(enum pmc_disp pd)
2583 {
2584         if ((int) pd >= PMC_DISP_FIRST &&
2585             pd <= PMC_DISP_LAST)
2586                 return (pmc_disposition_names[pd]);
2587
2588         errno = EINVAL;
2589         return (NULL);
2590 }
2591
2592 const char *
2593 _pmc_name_of_event(enum pmc_event pe, enum pmc_cputype cpu)
2594 {
2595         const struct pmc_event_descr *ev, *evfence;
2596
2597         ev = evfence = NULL;
2598         if (pe >= PMC_EV_IAF_FIRST && pe <= PMC_EV_IAF_LAST) {
2599                 ev = iaf_event_table;
2600                 evfence = iaf_event_table + PMC_EVENT_TABLE_SIZE(iaf);
2601         } else if (pe >= PMC_EV_IAP_FIRST && pe <= PMC_EV_IAP_LAST) {
2602                 switch (cpu) {
2603                 case PMC_CPU_INTEL_ATOM:
2604                         ev = atom_event_table;
2605                         evfence = atom_event_table + PMC_EVENT_TABLE_SIZE(atom);
2606                         break;
2607                 case PMC_CPU_INTEL_CORE:
2608                         ev = core_event_table;
2609                         evfence = core_event_table + PMC_EVENT_TABLE_SIZE(core);
2610                         break;
2611                 case PMC_CPU_INTEL_CORE2:
2612                 case PMC_CPU_INTEL_CORE2EXTREME:
2613                         ev = core2_event_table;
2614                         evfence = core2_event_table + PMC_EVENT_TABLE_SIZE(core2);
2615                         break;
2616                 case PMC_CPU_INTEL_COREI7:
2617                         ev = corei7_event_table;
2618                         evfence = corei7_event_table + PMC_EVENT_TABLE_SIZE(corei7);
2619                         break;
2620                 default:        /* Unknown CPU type. */
2621                         break;
2622                 }
2623         } if (pe >= PMC_EV_K7_FIRST && pe <= PMC_EV_K7_LAST) {
2624                 ev = k7_event_table;
2625                 evfence = k7_event_table + PMC_EVENT_TABLE_SIZE(k7);
2626         } else if (pe >= PMC_EV_K8_FIRST && pe <= PMC_EV_K8_LAST) {
2627                 ev = k8_event_table;
2628                 evfence = k8_event_table + PMC_EVENT_TABLE_SIZE(k8);
2629         } else if (pe >= PMC_EV_P4_FIRST && pe <= PMC_EV_P4_LAST) {
2630                 ev = p4_event_table;
2631                 evfence = p4_event_table + PMC_EVENT_TABLE_SIZE(p4);
2632         } else if (pe >= PMC_EV_P5_FIRST && pe <= PMC_EV_P5_LAST) {
2633                 ev = p5_event_table;
2634                 evfence = p5_event_table + PMC_EVENT_TABLE_SIZE(p5);
2635         } else if (pe >= PMC_EV_P6_FIRST && pe <= PMC_EV_P6_LAST) {
2636                 ev = p6_event_table;
2637                 evfence = p6_event_table + PMC_EVENT_TABLE_SIZE(p6);
2638         } else if (pe == PMC_EV_TSC_TSC) {
2639                 ev = tsc_event_table;
2640                 evfence = tsc_event_table + PMC_EVENT_TABLE_SIZE(tsc);
2641         }
2642
2643         for (; ev != evfence; ev++)
2644                 if (pe == ev->pm_ev_code)
2645                         return (ev->pm_ev_name);
2646
2647         return (NULL);
2648 }
2649
2650 const char *
2651 pmc_name_of_event(enum pmc_event pe)
2652 {
2653         const char *n;
2654
2655         if ((n = _pmc_name_of_event(pe, cpu_info.pm_cputype)) != NULL)
2656                 return (n);
2657
2658         errno = EINVAL;
2659         return (NULL);
2660 }
2661
2662 const char *
2663 pmc_name_of_mode(enum pmc_mode pm)
2664 {
2665         if ((int) pm >= PMC_MODE_FIRST &&
2666             pm <= PMC_MODE_LAST)
2667                 return (pmc_mode_names[pm]);
2668
2669         errno = EINVAL;
2670         return (NULL);
2671 }
2672
2673 const char *
2674 pmc_name_of_state(enum pmc_state ps)
2675 {
2676         if ((int) ps >= PMC_STATE_FIRST &&
2677             ps <= PMC_STATE_LAST)
2678                 return (pmc_state_names[ps]);
2679
2680         errno = EINVAL;
2681         return (NULL);
2682 }
2683
2684 int
2685 pmc_ncpu(void)
2686 {
2687         if (pmc_syscall == -1) {
2688                 errno = ENXIO;
2689                 return (-1);
2690         }
2691
2692         return (cpu_info.pm_ncpu);
2693 }
2694
2695 int
2696 pmc_npmc(int cpu)
2697 {
2698         if (pmc_syscall == -1) {
2699                 errno = ENXIO;
2700                 return (-1);
2701         }
2702
2703         if (cpu < 0 || cpu >= (int) cpu_info.pm_ncpu) {
2704                 errno = EINVAL;
2705                 return (-1);
2706         }
2707
2708         return (cpu_info.pm_npmc);
2709 }
2710
2711 int
2712 pmc_pmcinfo(int cpu, struct pmc_pmcinfo **ppmci)
2713 {
2714         int nbytes, npmc;
2715         struct pmc_op_getpmcinfo *pmci;
2716
2717         if ((npmc = pmc_npmc(cpu)) < 0)
2718                 return (-1);
2719
2720         nbytes = sizeof(struct pmc_op_getpmcinfo) +
2721             npmc * sizeof(struct pmc_info);
2722
2723         if ((pmci = calloc(1, nbytes)) == NULL)
2724                 return (-1);
2725
2726         pmci->pm_cpu  = cpu;
2727
2728         if (PMC_CALL(GETPMCINFO, pmci) < 0) {
2729                 free(pmci);
2730                 return (-1);
2731         }
2732
2733         /* kernel<->library, library<->userland interfaces are identical */
2734         *ppmci = (struct pmc_pmcinfo *) pmci;
2735         return (0);
2736 }
2737
2738 int
2739 pmc_read(pmc_id_t pmc, pmc_value_t *value)
2740 {
2741         struct pmc_op_pmcrw pmc_read_op;
2742
2743         pmc_read_op.pm_pmcid = pmc;
2744         pmc_read_op.pm_flags = PMC_F_OLDVALUE;
2745         pmc_read_op.pm_value = -1;
2746
2747         if (PMC_CALL(PMCRW, &pmc_read_op) < 0)
2748                 return (-1);
2749
2750         *value = pmc_read_op.pm_value;
2751         return (0);
2752 }
2753
2754 int
2755 pmc_release(pmc_id_t pmc)
2756 {
2757         struct pmc_op_simple    pmc_release_args;
2758
2759         pmc_release_args.pm_pmcid = pmc;
2760         return (PMC_CALL(PMCRELEASE, &pmc_release_args));
2761 }
2762
2763 int
2764 pmc_rw(pmc_id_t pmc, pmc_value_t newvalue, pmc_value_t *oldvaluep)
2765 {
2766         struct pmc_op_pmcrw pmc_rw_op;
2767
2768         pmc_rw_op.pm_pmcid = pmc;
2769         pmc_rw_op.pm_flags = PMC_F_NEWVALUE | PMC_F_OLDVALUE;
2770         pmc_rw_op.pm_value = newvalue;
2771
2772         if (PMC_CALL(PMCRW, &pmc_rw_op) < 0)
2773                 return (-1);
2774
2775         *oldvaluep = pmc_rw_op.pm_value;
2776         return (0);
2777 }
2778
2779 int
2780 pmc_set(pmc_id_t pmc, pmc_value_t value)
2781 {
2782         struct pmc_op_pmcsetcount sc;
2783
2784         sc.pm_pmcid = pmc;
2785         sc.pm_count = value;
2786
2787         if (PMC_CALL(PMCSETCOUNT, &sc) < 0)
2788                 return (-1);
2789         return (0);
2790 }
2791
2792 int
2793 pmc_start(pmc_id_t pmc)
2794 {
2795         struct pmc_op_simple    pmc_start_args;
2796
2797         pmc_start_args.pm_pmcid = pmc;
2798         return (PMC_CALL(PMCSTART, &pmc_start_args));
2799 }
2800
2801 int
2802 pmc_stop(pmc_id_t pmc)
2803 {
2804         struct pmc_op_simple    pmc_stop_args;
2805
2806         pmc_stop_args.pm_pmcid = pmc;
2807         return (PMC_CALL(PMCSTOP, &pmc_stop_args));
2808 }
2809
2810 int
2811 pmc_width(pmc_id_t pmcid, uint32_t *width)
2812 {
2813         unsigned int i;
2814         enum pmc_class cl;
2815
2816         cl = PMC_ID_TO_CLASS(pmcid);
2817         for (i = 0; i < cpu_info.pm_nclass; i++)
2818                 if (cpu_info.pm_classes[i].pm_class == cl) {
2819                         *width = cpu_info.pm_classes[i].pm_width;
2820                         return (0);
2821                 }
2822         errno = EINVAL;
2823         return (-1);
2824 }
2825
2826 int
2827 pmc_write(pmc_id_t pmc, pmc_value_t value)
2828 {
2829         struct pmc_op_pmcrw pmc_write_op;
2830
2831         pmc_write_op.pm_pmcid = pmc;
2832         pmc_write_op.pm_flags = PMC_F_NEWVALUE;
2833         pmc_write_op.pm_value = value;
2834         return (PMC_CALL(PMCRW, &pmc_write_op));
2835 }
2836
2837 int
2838 pmc_writelog(uint32_t userdata)
2839 {
2840         struct pmc_op_writelog wl;
2841
2842         wl.pm_userdata = userdata;
2843         return (PMC_CALL(WRITELOG, &wl));
2844 }