sys/dev/hwpmc/hwpmc_mod.c

   1 /*-
   2  * Copyright (c) 2003-2008 Joseph Koshy
   3  * Copyright (c) 2007 The FreeBSD Foundation
   4  * All rights reserved.
   5  *
   6  * Portions of this software were developed by A. Joseph Koshy under
   7  * sponsorship from the FreeBSD Foundation and Google, Inc.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  *
  30  */
  31
  32 #include <sys/cdefs.h>
  33 __FBSDID("$FreeBSD$");
  34
  35 #include <sys/param.h>
  36 #include <sys/eventhandler.h>
  37 #include <sys/jail.h>
  38 #include <sys/kernel.h>
  39 #include <sys/kthread.h>
  40 #include <sys/limits.h>
  41 #include <sys/lock.h>
  42 #include <sys/malloc.h>
  43 #include <sys/module.h>
  44 #include <sys/mount.h>
  45 #include <sys/mutex.h>
  46 #include <sys/pmc.h>
  47 #include <sys/pmckern.h>
  48 #include <sys/pmclog.h>
  49 #include <sys/priv.h>
  50 #include <sys/proc.h>
  51 #include <sys/queue.h>
  52 #include <sys/resourcevar.h>
  53 #include <sys/sched.h>
  54 #include <sys/signalvar.h>
  55 #include <sys/smp.h>
  56 #include <sys/sx.h>
  57 #include <sys/sysctl.h>
  58 #include <sys/sysent.h>
  59 #include <sys/systm.h>
  60 #include <sys/vnode.h>
  61
  62 #include <sys/linker.h>         /* needs to be after <sys/malloc.h> */
  63
  64 #include <machine/atomic.h>
  65 #include <machine/md_var.h>
  66
  67 #include <vm/vm.h>
  68 #include <vm/vm_extern.h>
  69 #include <vm/pmap.h>
  70 #include <vm/vm_map.h>
  71 #include <vm/vm_object.h>
  72
  73 /*
  74  * Types
  75  */
  76
  77 enum pmc_flags {
  78         PMC_FLAG_NONE     = 0x00, /* do nothing */
  79         PMC_FLAG_REMOVE   = 0x01, /* atomically remove entry from hash */
  80         PMC_FLAG_ALLOCATE = 0x02, /* add entry to hash if not found */
  81 };
  82
  83 /*
  84  * The offset in sysent where the syscall is allocated.
  85  */
  86
  87 static int pmc_syscall_num = NO_SYSCALL;
  88 struct pmc_cpu          **pmc_pcpu;      /* per-cpu state */
  89 pmc_value_t             *pmc_pcpu_saved; /* saved PMC values: CSW handling */
  90
  91 #define PMC_PCPU_SAVED(C,R)     pmc_pcpu_saved[(R) + md->pmd_npmc*(C)]
  92
  93 struct mtx_pool         *pmc_mtxpool;
  94 static int              *pmc_pmcdisp;    /* PMC row dispositions */
  95
  96 #define PMC_ROW_DISP_IS_FREE(R)         (pmc_pmcdisp[(R)] == 0)
  97 #define PMC_ROW_DISP_IS_THREAD(R)       (pmc_pmcdisp[(R)] > 0)
  98 #define PMC_ROW_DISP_IS_STANDALONE(R)   (pmc_pmcdisp[(R)] < 0)
  99
 100 #define PMC_MARK_ROW_FREE(R) do {                                         \
 101         pmc_pmcdisp[(R)] = 0;                                             \
 102 } while (0)
 103
 104 #define PMC_MARK_ROW_STANDALONE(R) do {                                   \
 105         KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \
 106                     __LINE__));                                           \
 107         atomic_add_int(&pmc_pmcdisp[(R)], -1);                            \
 108         KASSERT(pmc_pmcdisp[(R)] >= (-pmc_cpu_max_active()),              \
 109                 ("[pmc,%d] row disposition error", __LINE__));            \
 110 } while (0)
 111
 112 #define PMC_UNMARK_ROW_STANDALONE(R) do {                                 \
 113         atomic_add_int(&pmc_pmcdisp[(R)], 1);                             \
 114         KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \
 115                     __LINE__));                                           \
 116 } while (0)
 117
 118 #define PMC_MARK_ROW_THREAD(R) do {                                       \
 119         KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \
 120                     __LINE__));                                           \
 121         atomic_add_int(&pmc_pmcdisp[(R)], 1);                             \
 122 } while (0)
 123
 124 #define PMC_UNMARK_ROW_THREAD(R) do {                                     \
 125         atomic_add_int(&pmc_pmcdisp[(R)], -1);                            \
 126         KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \
 127                     __LINE__));                                           \
 128 } while (0)
 129
 130
 131 /* various event handlers */
 132 static eventhandler_tag pmc_exit_tag, pmc_fork_tag;
 133
 134 /* Module statistics */
 135 struct pmc_op_getdriverstats pmc_stats;
 136
 137 /* Machine/processor dependent operations */
 138 static struct pmc_mdep  *md;
 139
 140 /*
 141  * Hash tables mapping owner processes and target threads to PMCs.
 142  */
 143
 144 struct mtx pmc_processhash_mtx;         /* spin mutex */
 145 static u_long pmc_processhashmask;
 146 static LIST_HEAD(pmc_processhash, pmc_process)  *pmc_processhash;
 147
 148 /*
 149  * Hash table of PMC owner descriptors.  This table is protected by
 150  * the shared PMC "sx" lock.
 151  */
 152
 153 static u_long pmc_ownerhashmask;
 154 static LIST_HEAD(pmc_ownerhash, pmc_owner)      *pmc_ownerhash;
 155
 156 /*
 157  * List of PMC owners with system-wide sampling PMCs.
 158  */
 159
 160 static LIST_HEAD(, pmc_owner)                   pmc_ss_owners;
 161
 162
 163 /*
 164  * A map of row indices to classdep structures.
 165  */
 166 static struct pmc_classdep **pmc_rowindex_to_classdep;
 167
 168 /*
 169  * Prototypes
 170  */
 171
 172 #ifdef  DEBUG
 173 static int      pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS);
 174 static int      pmc_debugflags_parse(char *newstr, char *fence);
 175 #endif
 176
 177 static int      load(struct module *module, int cmd, void *arg);
 178 static int      pmc_attach_process(struct proc *p, struct pmc *pm);
 179 static struct pmc *pmc_allocate_pmc_descriptor(void);
 180 static struct pmc_owner *pmc_allocate_owner_descriptor(struct proc *p);
 181 static int      pmc_attach_one_process(struct proc *p, struct pmc *pm);
 182 static int      pmc_can_allocate_rowindex(struct proc *p, unsigned int ri,
 183     int cpu);
 184 static int      pmc_can_attach(struct pmc *pm, struct proc *p);
 185 static void     pmc_capture_user_callchain(int cpu, struct trapframe *tf);
 186 static void     pmc_cleanup(void);
 187 static int      pmc_detach_process(struct proc *p, struct pmc *pm);
 188 static int      pmc_detach_one_process(struct proc *p, struct pmc *pm,
 189     int flags);
 190 static void     pmc_destroy_owner_descriptor(struct pmc_owner *po);
 191 static struct pmc_owner *pmc_find_owner_descriptor(struct proc *p);
 192 static int      pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm);
 193 static struct pmc *pmc_find_pmc_descriptor_in_process(struct pmc_owner *po,
 194     pmc_id_t pmc);
 195 static struct pmc_process *pmc_find_process_descriptor(struct proc *p,
 196     uint32_t mode);
 197 static void     pmc_force_context_switch(void);
 198 static void     pmc_link_target_process(struct pmc *pm,
 199     struct pmc_process *pp);
 200 static void     pmc_log_all_process_mappings(struct pmc_owner *po);
 201 static void     pmc_log_kernel_mappings(struct pmc *pm);
 202 static void     pmc_log_process_mappings(struct pmc_owner *po, struct proc *p);
 203 static void     pmc_maybe_remove_owner(struct pmc_owner *po);
 204 static void     pmc_process_csw_in(struct thread *td);
 205 static void     pmc_process_csw_out(struct thread *td);
 206 static void     pmc_process_exit(void *arg, struct proc *p);
 207 static void     pmc_process_fork(void *arg, struct proc *p1,
 208     struct proc *p2, int n);
 209 static void     pmc_process_samples(int cpu);
 210 static void     pmc_release_pmc_descriptor(struct pmc *pmc);
 211 static void     pmc_remove_owner(struct pmc_owner *po);
 212 static void     pmc_remove_process_descriptor(struct pmc_process *pp);
 213 static void     pmc_restore_cpu_binding(struct pmc_binding *pb);
 214 static void     pmc_save_cpu_binding(struct pmc_binding *pb);
 215 static void     pmc_select_cpu(int cpu);
 216 static int      pmc_start(struct pmc *pm);
 217 static int      pmc_stop(struct pmc *pm);
 218 static int      pmc_syscall_handler(struct thread *td, void *syscall_args);
 219 static void     pmc_unlink_target_process(struct pmc *pmc,
 220     struct pmc_process *pp);
 221
 222 /*
 223  * Kernel tunables and sysctl(8) interface.
 224  */
 225
 226 SYSCTL_NODE(_kern, OID_AUTO, hwpmc, CTLFLAG_RW, 0, "HWPMC parameters");
 227
 228 static int pmc_callchaindepth = PMC_CALLCHAIN_DEPTH;
 229 TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "callchaindepth", &pmc_callchaindepth);
 230 SYSCTL_INT(_kern_hwpmc, OID_AUTO, callchaindepth, CTLFLAG_TUN|CTLFLAG_RD,
 231     &pmc_callchaindepth, 0, "depth of call chain records");
 232
 233 #ifdef  DEBUG
 234 struct pmc_debugflags pmc_debugflags = PMC_DEBUG_DEFAULT_FLAGS;
 235 char    pmc_debugstr[PMC_DEBUG_STRSIZE];
 236 TUNABLE_STR(PMC_SYSCTL_NAME_PREFIX "debugflags", pmc_debugstr,
 237     sizeof(pmc_debugstr));
 238 SYSCTL_PROC(_kern_hwpmc, OID_AUTO, debugflags,
 239     CTLTYPE_STRING|CTLFLAG_RW|CTLFLAG_TUN,
 240     0, 0, pmc_debugflags_sysctl_handler, "A", "debug flags");
 241 #endif
 242
 243 /*
 244  * kern.hwpmc.hashrows -- determines the number of rows in the
 245  * of the hash table used to look up threads
 246  */
 247
 248 static int pmc_hashsize = PMC_HASH_SIZE;
 249 TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "hashsize", &pmc_hashsize);
 250 SYSCTL_INT(_kern_hwpmc, OID_AUTO, hashsize, CTLFLAG_TUN|CTLFLAG_RD,
 251     &pmc_hashsize, 0, "rows in hash tables");
 252
 253 /*
 254  * kern.hwpmc.nsamples --- number of PC samples/callchain stacks per CPU
 255  */
 256
 257 static int pmc_nsamples = PMC_NSAMPLES;
 258 TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "nsamples", &pmc_nsamples);
 259 SYSCTL_INT(_kern_hwpmc, OID_AUTO, nsamples, CTLFLAG_TUN|CTLFLAG_RD,
 260     &pmc_nsamples, 0, "number of PC samples per CPU");
 261
 262
 263 /*
 264  * kern.hwpmc.mtxpoolsize -- number of mutexes in the mutex pool.
 265  */
 266
 267 static int pmc_mtxpool_size = PMC_MTXPOOL_SIZE;
 268 TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "mtxpoolsize", &pmc_mtxpool_size);
 269 SYSCTL_INT(_kern_hwpmc, OID_AUTO, mtxpoolsize, CTLFLAG_TUN|CTLFLAG_RD,
 270     &pmc_mtxpool_size, 0, "size of spin mutex pool");
 271
 272
 273 /*
 274  * security.bsd.unprivileged_syspmcs -- allow non-root processes to
 275  * allocate system-wide PMCs.
 276  *
 277  * Allowing unprivileged processes to allocate system PMCs is convenient
 278  * if system-wide measurements need to be taken concurrently with other
 279  * per-process measurements.  This feature is turned off by default.
 280  */
 281
 282 static int pmc_unprivileged_syspmcs = 0;
 283 TUNABLE_INT("security.bsd.unprivileged_syspmcs", &pmc_unprivileged_syspmcs);
 284 SYSCTL_INT(_security_bsd, OID_AUTO, unprivileged_syspmcs, CTLFLAG_RW,
 285     &pmc_unprivileged_syspmcs, 0,
 286     "allow unprivileged process to allocate system PMCs");
 287
 288 /*
 289  * Hash function.  Discard the lower 2 bits of the pointer since
 290  * these are always zero for our uses.  The hash multiplier is
 291  * round((2^LONG_BIT) * ((sqrt(5)-1)/2)).
 292  */
 293
 294 #if     LONG_BIT == 64
 295 #define _PMC_HM         11400714819323198486u
 296 #elif   LONG_BIT == 32
 297 #define _PMC_HM         2654435769u
 298 #else
 299 #error  Must know the size of 'long' to compile
 300 #endif
 301
 302 #define PMC_HASH_PTR(P,M)       ((((unsigned long) (P) >> 2) * _PMC_HM) & (M))
 303
 304 /*
 305  * Syscall structures
 306  */
 307
 308 /* The `sysent' for the new syscall */
 309 static struct sysent pmc_sysent = {
 310         2,                      /* sy_narg */
 311         pmc_syscall_handler     /* sy_call */
 312 };
 313
 314 static struct syscall_module_data pmc_syscall_mod = {
 315         load,
 316         NULL,
 317         &pmc_syscall_num,
 318         &pmc_sysent,
 319         { 0, NULL }
 320 };
 321
 322 static moduledata_t pmc_mod = {
 323         PMC_MODULE_NAME,
 324         syscall_module_handler,
 325         &pmc_syscall_mod
 326 };
 327
 328 DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SMP, SI_ORDER_ANY);
 329 MODULE_VERSION(pmc, PMC_VERSION);
 330
 331 #ifdef  DEBUG
 332 enum pmc_dbgparse_state {
 333         PMCDS_WS,               /* in whitespace */
 334         PMCDS_MAJOR,            /* seen a major keyword */
 335         PMCDS_MINOR
 336 };
 337
 338 static int
 339 pmc_debugflags_parse(char *newstr, char *fence)
 340 {
 341         char c, *p, *q;
 342         struct pmc_debugflags *tmpflags;
 343         int error, found, *newbits, tmp;
 344         size_t kwlen;
 345
 346         tmpflags = malloc(sizeof(*tmpflags), M_PMC, M_WAITOK|M_ZERO);
 347
 348         p = newstr;
 349         error = 0;
 350
 351         for (; p < fence && (c = *p); p++) {
 352
 353                 /* skip white space */
 354                 if (c == ' ' || c == '\t')
 355                         continue;
 356
 357                 /* look for a keyword followed by "=" */
 358                 for (q = p; p < fence && (c = *p) && c != '='; p++)
 359                         ;
 360                 if (c != '=') {
 361                         error = EINVAL;
 362                         goto done;
 363                 }
 364
 365                 kwlen = p - q;
 366                 newbits = NULL;
 367
 368                 /* lookup flag group name */
 369 #define DBG_SET_FLAG_MAJ(S,F)                                           \
 370                 if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0)  \
 371                         newbits = &tmpflags->pdb_ ## F;
 372
 373                 DBG_SET_FLAG_MAJ("cpu",         CPU);
 374                 DBG_SET_FLAG_MAJ("csw",         CSW);
 375                 DBG_SET_FLAG_MAJ("logging",     LOG);
 376                 DBG_SET_FLAG_MAJ("module",      MOD);
 377                 DBG_SET_FLAG_MAJ("md",          MDP);
 378                 DBG_SET_FLAG_MAJ("owner",       OWN);
 379                 DBG_SET_FLAG_MAJ("pmc",         PMC);
 380                 DBG_SET_FLAG_MAJ("process",     PRC);
 381                 DBG_SET_FLAG_MAJ("sampling",    SAM);
 382
 383                 if (newbits == NULL) {
 384                         error = EINVAL;
 385                         goto done;
 386                 }
 387
 388                 p++;            /* skip the '=' */
 389
 390                 /* Now parse the individual flags */
 391                 tmp = 0;
 392         newflag:
 393                 for (q = p; p < fence && (c = *p); p++)
 394                         if (c == ' ' || c == '\t' || c == ',')
 395                                 break;
 396
 397                 /* p == fence or c == ws or c == "," or c == 0 */
 398
 399                 if ((kwlen = p - q) == 0) {
 400                         *newbits = tmp;
 401                         continue;
 402                 }
 403
 404                 found = 0;
 405 #define DBG_SET_FLAG_MIN(S,F)                                           \
 406                 if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0)  \
 407                         tmp |= found = (1 << PMC_DEBUG_MIN_ ## F)
 408
 409                 /* a '*' denotes all possible flags in the group */
 410                 if (kwlen == 1 && *q == '*')
 411                         tmp = found = ~0;
 412                 /* look for individual flag names */
 413                 DBG_SET_FLAG_MIN("allocaterow", ALR);
 414                 DBG_SET_FLAG_MIN("allocate",    ALL);
 415                 DBG_SET_FLAG_MIN("attach",      ATT);
 416                 DBG_SET_FLAG_MIN("bind",        BND);
 417                 DBG_SET_FLAG_MIN("config",      CFG);
 418                 DBG_SET_FLAG_MIN("exec",        EXC);
 419                 DBG_SET_FLAG_MIN("exit",        EXT);
 420                 DBG_SET_FLAG_MIN("find",        FND);
 421                 DBG_SET_FLAG_MIN("flush",       FLS);
 422                 DBG_SET_FLAG_MIN("fork",        FRK);
 423                 DBG_SET_FLAG_MIN("getbuf",      GTB);
 424                 DBG_SET_FLAG_MIN("hook",        PMH);
 425                 DBG_SET_FLAG_MIN("init",        INI);
 426                 DBG_SET_FLAG_MIN("intr",        INT);
 427                 DBG_SET_FLAG_MIN("linktarget",  TLK);
 428                 DBG_SET_FLAG_MIN("mayberemove", OMR);
 429                 DBG_SET_FLAG_MIN("ops",         OPS);
 430                 DBG_SET_FLAG_MIN("read",        REA);
 431                 DBG_SET_FLAG_MIN("register",    REG);
 432                 DBG_SET_FLAG_MIN("release",     REL);
 433                 DBG_SET_FLAG_MIN("remove",      ORM);
 434                 DBG_SET_FLAG_MIN("sample",      SAM);
 435                 DBG_SET_FLAG_MIN("scheduleio",  SIO);
 436                 DBG_SET_FLAG_MIN("select",      SEL);
 437                 DBG_SET_FLAG_MIN("signal",      SIG);
 438                 DBG_SET_FLAG_MIN("swi",         SWI);
 439                 DBG_SET_FLAG_MIN("swo",         SWO);
 440                 DBG_SET_FLAG_MIN("start",       STA);
 441                 DBG_SET_FLAG_MIN("stop",        STO);
 442                 DBG_SET_FLAG_MIN("syscall",     PMS);
 443                 DBG_SET_FLAG_MIN("unlinktarget", TUL);
 444                 DBG_SET_FLAG_MIN("write",       WRI);
 445                 if (found == 0) {
 446                         /* unrecognized flag name */
 447                         error = EINVAL;
 448                         goto done;
 449                 }
 450
 451                 if (c == 0 || c == ' ' || c == '\t') {  /* end of flag group */
 452                         *newbits = tmp;
 453                         continue;
 454                 }
 455
 456                 p++;
 457                 goto newflag;
 458         }
 459
 460         /* save the new flag set */
 461         bcopy(tmpflags, &pmc_debugflags, sizeof(pmc_debugflags));
 462
 463  done:
 464         free(tmpflags, M_PMC);
 465         return error;
 466 }
 467
 468 static int
 469 pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS)
 470 {
 471         char *fence, *newstr;
 472         int error;
 473         unsigned int n;
 474
 475         (void) arg1; (void) arg2; /* unused parameters */
 476
 477         n = sizeof(pmc_debugstr);
 478         newstr = malloc(n, M_PMC, M_WAITOK|M_ZERO);
 479         (void) strlcpy(newstr, pmc_debugstr, n);
 480
 481         error = sysctl_handle_string(oidp, newstr, n, req);
 482
 483         /* if there is a new string, parse and copy it */
 484         if (error == 0 && req->newptr != NULL) {
 485                 fence = newstr + (n < req->newlen ? n : req->newlen + 1);
 486                 if ((error = pmc_debugflags_parse(newstr, fence)) == 0)
 487                         (void) strlcpy(pmc_debugstr, newstr,
 488                             sizeof(pmc_debugstr));
 489         }
 490
 491         free(newstr, M_PMC);
 492
 493         return error;
 494 }
 495 #endif
 496
 497 /*
 498  * Map a row index to a classdep structure and return the adjusted row
 499  * index for the PMC class index.
 500  */
 501 static struct pmc_classdep *
 502 pmc_ri_to_classdep(struct pmc_mdep *md, int ri, int *adjri)
 503 {
 504         struct pmc_classdep *pcd;
 505
 506         (void) md;
 507
 508         KASSERT(ri >= 0 && ri < md->pmd_npmc,
 509             ("[pmc,%d] illegal row-index %d", __LINE__, ri));
 510
 511         pcd = pmc_rowindex_to_classdep[ri];
 512
 513         KASSERT(pcd != NULL,
 514             ("[amd,%d] ri %d null pcd", __LINE__, ri));
 515
 516         *adjri = ri - pcd->pcd_ri;
 517
 518         KASSERT(*adjri >= 0 && *adjri < pcd->pcd_num,
 519             ("[pmc,%d] adjusted row-index %d", __LINE__, *adjri));
 520
 521         return (pcd);
 522 }
 523
 524 /*
 525  * Concurrency Control
 526  *
 527  * The driver manages the following data structures:
 528  *
 529  *   - target process descriptors, one per target process
 530  *   - owner process descriptors (and attached lists), one per owner process
 531  *   - lookup hash tables for owner and target processes
 532  *   - PMC descriptors (and attached lists)
 533  *   - per-cpu hardware state
 534  *   - the 'hook' variable through which the kernel calls into
 535  *     this module
 536  *   - the machine hardware state (managed by the MD layer)
 537  *
 538  * These data structures are accessed from:
 539  *
 540  * - thread context-switch code
 541  * - interrupt handlers (possibly on multiple cpus)
 542  * - kernel threads on multiple cpus running on behalf of user
 543  *   processes doing system calls
 544  * - this driver's private kernel threads
 545  *
 546  * = Locks and Locking strategy =
 547  *
 548  * The driver uses four locking strategies for its operation:
 549  *
 550  * - The global SX lock "pmc_sx" is used to protect internal
 551  *   data structures.
 552  *
 553  *   Calls into the module by syscall() start with this lock being
 554  *   held in exclusive mode.  Depending on the requested operation,
 555  *   the lock may be downgraded to 'shared' mode to allow more
 556  *   concurrent readers into the module.  Calls into the module from
 557  *   other parts of the kernel acquire the lock in shared mode.
 558  *
 559  *   This SX lock is held in exclusive mode for any operations that
 560  *   modify the linkages between the driver's internal data structures.
 561  *
 562  *   The 'pmc_hook' function pointer is also protected by this lock.
 563  *   It is only examined with the sx lock held in exclusive mode.  The
 564  *   kernel module is allowed to be unloaded only with the sx lock held
 565  *   in exclusive mode.  In normal syscall handling, after acquiring the
 566  *   pmc_sx lock we first check that 'pmc_hook' is non-null before
 567  *   proceeding.  This prevents races between the thread unloading the module
 568  *   and other threads seeking to use the module.
 569  *
 570  * - Lookups of target process structures and owner process structures
 571  *   cannot use the global "pmc_sx" SX lock because these lookups need
 572  *   to happen during context switches and in other critical sections
 573  *   where sleeping is not allowed.  We protect these lookup tables
 574  *   with their own private spin-mutexes, "pmc_processhash_mtx" and
 575  *   "pmc_ownerhash_mtx".
 576  *
 577  * - Interrupt handlers work in a lock free manner.  At interrupt
 578  *   time, handlers look at the PMC pointer (phw->phw_pmc) configured
 579  *   when the PMC was started.  If this pointer is NULL, the interrupt
 580  *   is ignored after updating driver statistics.  We ensure that this
 581  *   pointer is set (using an atomic operation if necessary) before the
 582  *   PMC hardware is started.  Conversely, this pointer is unset atomically
 583  *   only after the PMC hardware is stopped.
 584  *
 585  *   We ensure that everything needed for the operation of an
 586  *   interrupt handler is available without it needing to acquire any
 587  *   locks.  We also ensure that a PMC's software state is destroyed only
 588  *   after the PMC is taken off hardware (on all CPUs).
 589  *
 590  * - Context-switch handling with process-private PMCs needs more
 591  *   care.
 592  *
 593  *   A given process may be the target of multiple PMCs.  For example,
 594  *   PMCATTACH and PMCDETACH may be requested by a process on one CPU
 595  *   while the target process is running on another.  A PMC could also
 596  *   be getting released because its owner is exiting.  We tackle
 597  *   these situations in the following manner:
 598  *
 599  *   - each target process structure 'pmc_process' has an array
 600  *     of 'struct pmc *' pointers, one for each hardware PMC.
 601  *
 602  *   - At context switch IN time, each "target" PMC in RUNNING state
 603  *     gets started on hardware and a pointer to each PMC is copied into
 604  *     the per-cpu phw array.  The 'runcount' for the PMC is
 605  *     incremented.
 606  *
 607  *   - At context switch OUT time, all process-virtual PMCs are stopped
 608  *     on hardware.  The saved value is added to the PMCs value field
 609  *     only if the PMC is in a non-deleted state (the PMCs state could
 610  *     have changed during the current time slice).
 611  *
 612  *     Note that since in-between a switch IN on a processor and a switch
 613  *     OUT, the PMC could have been released on another CPU.  Therefore
 614  *     context switch OUT always looks at the hardware state to turn
 615  *     OFF PMCs and will update a PMC's saved value only if reachable
 616  *     from the target process record.
 617  *
 618  *   - OP PMCRELEASE could be called on a PMC at any time (the PMC could
 619  *     be attached to many processes at the time of the call and could
 620  *     be active on multiple CPUs).
 621  *
 622  *     We prevent further scheduling of the PMC by marking it as in
 623  *     state 'DELETED'.  If the runcount of the PMC is non-zero then
 624  *     this PMC is currently running on a CPU somewhere.  The thread
 625  *     doing the PMCRELEASE operation waits by repeatedly doing a
 626  *     pause() till the runcount comes to zero.
 627  *
 628  * The contents of a PMC descriptor (struct pmc) are protected using
 629  * a spin-mutex.  In order to save space, we use a mutex pool.
 630  *
 631  * In terms of lock types used by witness(4), we use:
 632  * - Type "pmc-sx", used by the global SX lock.
 633  * - Type "pmc-sleep", for sleep mutexes used by logger threads.
 634  * - Type "pmc-per-proc", for protecting PMC owner descriptors.
 635  * - Type "pmc-leaf", used for all other spin mutexes.
 636  */
 637
 638 /*
 639  * save the cpu binding of the current kthread
 640  */
 641
 642 static void
 643 pmc_save_cpu_binding(struct pmc_binding *pb)
 644 {
 645         PMCDBG(CPU,BND,2, "%s", "save-cpu");
 646         thread_lock(curthread);
 647         pb->pb_bound = sched_is_bound(curthread);
 648         pb->pb_cpu   = curthread->td_oncpu;
 649         thread_unlock(curthread);
 650         PMCDBG(CPU,BND,2, "save-cpu cpu=%d", pb->pb_cpu);
 651 }
 652
 653 /*
 654  * restore the cpu binding of the current thread
 655  */
 656
 657 static void
 658 pmc_restore_cpu_binding(struct pmc_binding *pb)
 659 {
 660         PMCDBG(CPU,BND,2, "restore-cpu curcpu=%d restore=%d",
 661             curthread->td_oncpu, pb->pb_cpu);
 662         thread_lock(curthread);
 663         if (pb->pb_bound)
 664                 sched_bind(curthread, pb->pb_cpu);
 665         else
 666                 sched_unbind(curthread);
 667         thread_unlock(curthread);
 668         PMCDBG(CPU,BND,2, "%s", "restore-cpu done");
 669 }
 670
 671 /*
 672  * move execution over the specified cpu and bind it there.
 673  */
 674
 675 static void
 676 pmc_select_cpu(int cpu)
 677 {
 678         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 679             ("[pmc,%d] bad cpu number %d", __LINE__, cpu));
 680
 681         /* Never move to an inactive CPU. */
 682         KASSERT(pmc_cpu_is_active(cpu), ("[pmc,%d] selecting inactive "
 683             "CPU %d", __LINE__, cpu));
 684
 685         PMCDBG(CPU,SEL,2, "select-cpu cpu=%d", cpu);
 686         thread_lock(curthread);
 687         sched_bind(curthread, cpu);
 688         thread_unlock(curthread);
 689
 690         KASSERT(curthread->td_oncpu == cpu,
 691             ("[pmc,%d] CPU not bound [cpu=%d, curr=%d]", __LINE__,
 692                 cpu, curthread->td_oncpu));
 693
 694         PMCDBG(CPU,SEL,2, "select-cpu cpu=%d ok", cpu);
 695 }
 696
 697 /*
 698  * Force a context switch.
 699  *
 700  * We do this by pause'ing for 1 tick -- invoking mi_switch() is not
 701  * guaranteed to force a context switch.
 702  */
 703
 704 static void
 705 pmc_force_context_switch(void)
 706 {
 707
 708         pause("pmcctx", 1);
 709 }
 710
 711 /*
 712  * Get the file name for an executable.  This is a simple wrapper
 713  * around vn_fullpath(9).
 714  */
 715
 716 static void
 717 pmc_getfilename(struct vnode *v, char **fullpath, char **freepath)
 718 {
 719
 720         *fullpath = "unknown";
 721         *freepath = NULL;
 722         vn_fullpath(curthread, v, fullpath, freepath);
 723 }
 724
 725 /*
 726  * remove an process owning PMCs
 727  */
 728
 729 void
 730 pmc_remove_owner(struct pmc_owner *po)
 731 {
 732         struct pmc *pm, *tmp;
 733
 734         sx_assert(&pmc_sx, SX_XLOCKED);
 735
 736         PMCDBG(OWN,ORM,1, "remove-owner po=%p", po);
 737
 738         /* Remove descriptor from the owner hash table */
 739         LIST_REMOVE(po, po_next);
 740
 741         /* release all owned PMC descriptors */
 742         LIST_FOREACH_SAFE(pm, &po->po_pmcs, pm_next, tmp) {
 743                 PMCDBG(OWN,ORM,2, "pmc=%p", pm);
 744                 KASSERT(pm->pm_owner == po,
 745                     ("[pmc,%d] owner %p != po %p", __LINE__, pm->pm_owner, po));
 746
 747                 pmc_release_pmc_descriptor(pm); /* will unlink from the list */
 748         }
 749
 750         KASSERT(po->po_sscount == 0,
 751             ("[pmc,%d] SS count not zero", __LINE__));
 752         KASSERT(LIST_EMPTY(&po->po_pmcs),
 753             ("[pmc,%d] PMC list not empty", __LINE__));
 754
 755         /* de-configure the log file if present */
 756         if (po->po_flags & PMC_PO_OWNS_LOGFILE)
 757                 pmclog_deconfigure_log(po);
 758 }
 759
 760 /*
 761  * remove an owner process record if all conditions are met.
 762  */
 763
 764 static void
 765 pmc_maybe_remove_owner(struct pmc_owner *po)
 766 {
 767
 768         PMCDBG(OWN,OMR,1, "maybe-remove-owner po=%p", po);
 769
 770         /*
 771          * Remove owner record if
 772          * - this process does not own any PMCs
 773          * - this process has not allocated a system-wide sampling buffer
 774          */
 775
 776         if (LIST_EMPTY(&po->po_pmcs) &&
 777             ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)) {
 778                 pmc_remove_owner(po);
 779                 pmc_destroy_owner_descriptor(po);
 780         }
 781 }
 782
 783 /*
 784  * Add an association between a target process and a PMC.
 785  */
 786
 787 static void
 788 pmc_link_target_process(struct pmc *pm, struct pmc_process *pp)
 789 {
 790         int ri;
 791         struct pmc_target *pt;
 792
 793         sx_assert(&pmc_sx, SX_XLOCKED);
 794
 795         KASSERT(pm != NULL && pp != NULL,
 796             ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
 797         KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)),
 798             ("[pmc,%d] Attaching a non-process-virtual pmc=%p to pid=%d",
 799                 __LINE__, pm, pp->pp_proc->p_pid));
 800         KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= ((int) md->pmd_npmc - 1),
 801             ("[pmc,%d] Illegal reference count %d for process record %p",
 802                 __LINE__, pp->pp_refcnt, (void *) pp));
 803
 804         ri = PMC_TO_ROWINDEX(pm);
 805
 806         PMCDBG(PRC,TLK,1, "link-target pmc=%p ri=%d pmc-process=%p",
 807             pm, ri, pp);
 808
 809 #ifdef  DEBUG
 810         LIST_FOREACH(pt, &pm->pm_targets, pt_next)
 811             if (pt->pt_process == pp)
 812                     KASSERT(0, ("[pmc,%d] pp %p already in pmc %p targets",
 813                                 __LINE__, pp, pm));
 814 #endif
 815
 816         pt = malloc(sizeof(struct pmc_target), M_PMC, M_WAITOK|M_ZERO);
 817         pt->pt_process = pp;
 818
 819         LIST_INSERT_HEAD(&pm->pm_targets, pt, pt_next);
 820
 821         atomic_store_rel_ptr((uintptr_t *)&pp->pp_pmcs[ri].pp_pmc,
 822             (uintptr_t)pm);
 823
 824         if (pm->pm_owner->po_owner == pp->pp_proc)
 825                 pm->pm_flags |= PMC_F_ATTACHED_TO_OWNER;
 826
 827         /*
 828          * Initialize the per-process values at this row index.
 829          */
 830         pp->pp_pmcs[ri].pp_pmcval = PMC_TO_MODE(pm) == PMC_MODE_TS ?
 831             pm->pm_sc.pm_reloadcount : 0;
 832
 833         pp->pp_refcnt++;
 834
 835 }
 836
 837 /*
 838  * Removes the association between a target process and a PMC.
 839  */
 840
 841 static void
 842 pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp)
 843 {
 844         int ri;
 845         struct proc *p;
 846         struct pmc_target *ptgt;
 847
 848         sx_assert(&pmc_sx, SX_XLOCKED);
 849
 850         KASSERT(pm != NULL && pp != NULL,
 851             ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
 852
 853         KASSERT(pp->pp_refcnt >= 1 && pp->pp_refcnt <= (int) md->pmd_npmc,
 854             ("[pmc,%d] Illegal ref count %d on process record %p",
 855                 __LINE__, pp->pp_refcnt, (void *) pp));
 856
 857         ri = PMC_TO_ROWINDEX(pm);
 858
 859         PMCDBG(PRC,TUL,1, "unlink-target pmc=%p ri=%d pmc-process=%p",
 860             pm, ri, pp);
 861
 862         KASSERT(pp->pp_pmcs[ri].pp_pmc == pm,
 863             ("[pmc,%d] PMC ri %d mismatch pmc %p pp->[ri] %p", __LINE__,
 864                 ri, pm, pp->pp_pmcs[ri].pp_pmc));
 865
 866         pp->pp_pmcs[ri].pp_pmc = NULL;
 867         pp->pp_pmcs[ri].pp_pmcval = (pmc_value_t) 0;
 868
 869         /* Remove owner-specific flags */
 870         if (pm->pm_owner->po_owner == pp->pp_proc) {
 871                 pp->pp_flags &= ~PMC_PP_ENABLE_MSR_ACCESS;
 872                 pm->pm_flags &= ~PMC_F_ATTACHED_TO_OWNER;
 873         }
 874
 875         pp->pp_refcnt--;
 876
 877         /* Remove the target process from the PMC structure */
 878         LIST_FOREACH(ptgt, &pm->pm_targets, pt_next)
 879                 if (ptgt->pt_process == pp)
 880                         break;
 881
 882         KASSERT(ptgt != NULL, ("[pmc,%d] process %p (pp: %p) not found "
 883                     "in pmc %p", __LINE__, pp->pp_proc, pp, pm));
 884
 885         LIST_REMOVE(ptgt, pt_next);
 886         free(ptgt, M_PMC);
 887
 888         /* if the PMC now lacks targets, send the owner a SIGIO */
 889         if (LIST_EMPTY(&pm->pm_targets)) {
 890                 p = pm->pm_owner->po_owner;
 891                 PROC_LOCK(p);
 892                 psignal(p, SIGIO);
 893                 PROC_UNLOCK(p);
 894
 895                 PMCDBG(PRC,SIG,2, "signalling proc=%p signal=%d", p,
 896                     SIGIO);
 897         }
 898 }
 899
 900 /*
 901  * Check if PMC 'pm' may be attached to target process 't'.
 902  */
 903
 904 static int
 905 pmc_can_attach(struct pmc *pm, struct proc *t)
 906 {
 907         struct proc *o;         /* pmc owner */
 908         struct ucred *oc, *tc;  /* owner, target credentials */
 909         int decline_attach, i;
 910
 911         /*
 912          * A PMC's owner can always attach that PMC to itself.
 913          */
 914
 915         if ((o = pm->pm_owner->po_owner) == t)
 916                 return 0;
 917
 918         PROC_LOCK(o);
 919         oc = o->p_ucred;
 920         crhold(oc);
 921         PROC_UNLOCK(o);
 922
 923         PROC_LOCK(t);
 924         tc = t->p_ucred;
 925         crhold(tc);
 926         PROC_UNLOCK(t);
 927
 928         /*
 929          * The effective uid of the PMC owner should match at least one
 930          * of the {effective,real,saved} uids of the target process.
 931          */
 932
 933         decline_attach = oc->cr_uid != tc->cr_uid &&
 934             oc->cr_uid != tc->cr_svuid &&
 935             oc->cr_uid != tc->cr_ruid;
 936
 937         /*
 938          * Every one of the target's group ids, must be in the owner's
 939          * group list.
 940          */
 941         for (i = 0; !decline_attach && i < tc->cr_ngroups; i++)
 942                 decline_attach = !groupmember(tc->cr_groups[i], oc);
 943
 944         /* check the read and saved gids too */
 945         if (decline_attach == 0)
 946                 decline_attach = !groupmember(tc->cr_rgid, oc) ||
 947                     !groupmember(tc->cr_svgid, oc);
 948
 949         crfree(tc);
 950         crfree(oc);
 951
 952         return !decline_attach;
 953 }
 954
 955 /*
 956  * Attach a process to a PMC.
 957  */
 958
 959 static int
 960 pmc_attach_one_process(struct proc *p, struct pmc *pm)
 961 {
 962         int ri;
 963         char *fullpath, *freepath;
 964         struct pmc_process      *pp;
 965
 966         sx_assert(&pmc_sx, SX_XLOCKED);
 967
 968         PMCDBG(PRC,ATT,2, "attach-one pm=%p ri=%d proc=%p (%d, %s)", pm,
 969             PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
 970
 971         /*
 972          * Locate the process descriptor corresponding to process 'p',
 973          * allocating space as needed.
 974          *
 975          * Verify that rowindex 'pm_rowindex' is free in the process
 976          * descriptor.
 977          *
 978          * If not, allocate space for a descriptor and link the
 979          * process descriptor and PMC.
 980          */
 981         ri = PMC_TO_ROWINDEX(pm);
 982
 983         if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_ALLOCATE)) == NULL)
 984                 return ENOMEM;
 985
 986         if (pp->pp_pmcs[ri].pp_pmc == pm) /* already present at slot [ri] */
 987                 return EEXIST;
 988
 989         if (pp->pp_pmcs[ri].pp_pmc != NULL)
 990                 return EBUSY;
 991
 992         pmc_link_target_process(pm, pp);
 993
 994         if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) &&
 995             (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) == 0)
 996                 pm->pm_flags |= PMC_F_NEEDS_LOGFILE;
 997
 998         pm->pm_flags |= PMC_F_ATTACH_DONE; /* mark as attached */
 999
1000         /* issue an attach event to a configured log file */
1001         if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE) {
1002                 pmc_getfilename(p->p_textvp, &fullpath, &freepath);
1003                 if (p->p_flag & P_KTHREAD) {
1004                         fullpath = kernelname;
1005                         freepath = NULL;
1006                 } else
1007                         pmclog_process_pmcattach(pm, p->p_pid, fullpath);
1008                 if (freepath)
1009                         free(freepath, M_TEMP);
1010                 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1011                         pmc_log_process_mappings(pm->pm_owner, p);
1012         }
1013         /* mark process as using HWPMCs */
1014         PROC_LOCK(p);
1015         p->p_flag |= P_HWPMC;
1016         PROC_UNLOCK(p);
1017
1018         return 0;
1019 }
1020
1021 /*
1022  * Attach a process and optionally its children
1023  */
1024
1025 static int
1026 pmc_attach_process(struct proc *p, struct pmc *pm)
1027 {
1028         int error;
1029         struct proc *top;
1030
1031         sx_assert(&pmc_sx, SX_XLOCKED);
1032
1033         PMCDBG(PRC,ATT,1, "attach pm=%p ri=%d proc=%p (%d, %s)", pm,
1034             PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
1035
1036
1037         /*
1038          * If this PMC successfully allowed a GETMSR operation
1039          * in the past, disallow further ATTACHes.
1040          */
1041
1042         if ((pm->pm_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0)
1043                 return EPERM;
1044
1045         if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
1046                 return pmc_attach_one_process(p, pm);
1047
1048         /*
1049          * Traverse all child processes, attaching them to
1050          * this PMC.
1051          */
1052
1053         sx_slock(&proctree_lock);
1054
1055         top = p;
1056
1057         for (;;) {
1058                 if ((error = pmc_attach_one_process(p, pm)) != 0)
1059                         break;
1060                 if (!LIST_EMPTY(&p->p_children))
1061                         p = LIST_FIRST(&p->p_children);
1062                 else for (;;) {
1063                         if (p == top)
1064                                 goto done;
1065                         if (LIST_NEXT(p, p_sibling)) {
1066                                 p = LIST_NEXT(p, p_sibling);
1067                                 break;
1068                         }
1069                         p = p->p_pptr;
1070                 }
1071         }
1072
1073         if (error)
1074                 (void) pmc_detach_process(top, pm);
1075
1076  done:
1077         sx_sunlock(&proctree_lock);
1078         return error;
1079 }
1080
1081 /*
1082  * Detach a process from a PMC.  If there are no other PMCs tracking
1083  * this process, remove the process structure from its hash table.  If
1084  * 'flags' contains PMC_FLAG_REMOVE, then free the process structure.
1085  */
1086
1087 static int
1088 pmc_detach_one_process(struct proc *p, struct pmc *pm, int flags)
1089 {
1090         int ri;
1091         struct pmc_process *pp;
1092
1093         sx_assert(&pmc_sx, SX_XLOCKED);
1094
1095         KASSERT(pm != NULL,
1096             ("[pmc,%d] null pm pointer", __LINE__));
1097
1098         ri = PMC_TO_ROWINDEX(pm);
1099
1100         PMCDBG(PRC,ATT,2, "detach-one pm=%p ri=%d proc=%p (%d, %s) flags=0x%x",
1101             pm, ri, p, p->p_pid, p->p_comm, flags);
1102
1103         if ((pp = pmc_find_process_descriptor(p, 0)) == NULL)
1104                 return ESRCH;
1105
1106         if (pp->pp_pmcs[ri].pp_pmc != pm)
1107                 return EINVAL;
1108
1109         pmc_unlink_target_process(pm, pp);
1110
1111         /* Issue a detach entry if a log file is configured */
1112         if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE)
1113                 pmclog_process_pmcdetach(pm, p->p_pid);
1114
1115         /*
1116          * If there are no PMCs targetting this process, we remove its
1117          * descriptor from the target hash table and unset the P_HWPMC
1118          * flag in the struct proc.
1119          */
1120         KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= (int) md->pmd_npmc,
1121             ("[pmc,%d] Illegal refcnt %d for process struct %p",
1122                 __LINE__, pp->pp_refcnt, pp));
1123
1124         if (pp->pp_refcnt != 0) /* still a target of some PMC */
1125                 return 0;
1126
1127         pmc_remove_process_descriptor(pp);
1128
1129         if (flags & PMC_FLAG_REMOVE)
1130                 free(pp, M_PMC);
1131
1132         PROC_LOCK(p);
1133         p->p_flag &= ~P_HWPMC;
1134         PROC_UNLOCK(p);
1135
1136         return 0;
1137 }
1138
1139 /*
1140  * Detach a process and optionally its descendants from a PMC.
1141  */
1142
1143 static int
1144 pmc_detach_process(struct proc *p, struct pmc *pm)
1145 {
1146         struct proc *top;
1147
1148         sx_assert(&pmc_sx, SX_XLOCKED);
1149
1150         PMCDBG(PRC,ATT,1, "detach pm=%p ri=%d proc=%p (%d, %s)", pm,
1151             PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
1152
1153         if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
1154                 return pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE);
1155
1156         /*
1157          * Traverse all children, detaching them from this PMC.  We
1158          * ignore errors since we could be detaching a PMC from a
1159          * partially attached proc tree.
1160          */
1161
1162         sx_slock(&proctree_lock);
1163
1164         top = p;
1165
1166         for (;;) {
1167                 (void) pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE);
1168
1169                 if (!LIST_EMPTY(&p->p_children))
1170                         p = LIST_FIRST(&p->p_children);
1171                 else for (;;) {
1172                         if (p == top)
1173                                 goto done;
1174                         if (LIST_NEXT(p, p_sibling)) {
1175                                 p = LIST_NEXT(p, p_sibling);
1176                                 break;
1177                         }
1178                         p = p->p_pptr;
1179                 }
1180         }
1181
1182  done:
1183         sx_sunlock(&proctree_lock);
1184
1185         if (LIST_EMPTY(&pm->pm_targets))
1186                 pm->pm_flags &= ~PMC_F_ATTACH_DONE;
1187
1188         return 0;
1189 }
1190
1191
1192 /*
1193  * Thread context switch IN
1194  */
1195
1196 static void
1197 pmc_process_csw_in(struct thread *td)
1198 {
1199         int cpu;
1200         unsigned int adjri, ri;
1201         struct pmc *pm;
1202         struct proc *p;
1203         struct pmc_cpu *pc;
1204         struct pmc_hw *phw;
1205         pmc_value_t newvalue;
1206         struct pmc_process *pp;
1207         struct pmc_classdep *pcd;
1208
1209         p = td->td_proc;
1210
1211         if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE)) == NULL)
1212                 return;
1213
1214         KASSERT(pp->pp_proc == td->td_proc,
1215             ("[pmc,%d] not my thread state", __LINE__));
1216
1217         critical_enter(); /* no preemption from this point */
1218
1219         cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */
1220
1221         PMCDBG(CSW,SWI,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p,
1222             p->p_pid, p->p_comm, pp);
1223
1224         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
1225             ("[pmc,%d] wierd CPU id %d", __LINE__, cpu));
1226
1227         pc = pmc_pcpu[cpu];
1228
1229         for (ri = 0; ri < md->pmd_npmc; ri++) {
1230
1231                 if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL)
1232                         continue;
1233
1234                 KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)),
1235                     ("[pmc,%d] Target PMC in non-virtual mode (%d)",
1236                         __LINE__, PMC_TO_MODE(pm)));
1237
1238                 KASSERT(PMC_TO_ROWINDEX(pm) == ri,
1239                     ("[pmc,%d] Row index mismatch pmc %d != ri %d",
1240                         __LINE__, PMC_TO_ROWINDEX(pm), ri));
1241
1242                 /*
1243                  * Only PMCs that are marked as 'RUNNING' need
1244                  * be placed on hardware.
1245                  */
1246
1247                 if (pm->pm_state != PMC_STATE_RUNNING)
1248                         continue;
1249
1250                 /* increment PMC runcount */
1251                 atomic_add_rel_32(&pm->pm_runcount, 1);
1252
1253                 /* configure the HWPMC we are going to use. */
1254                 pcd = pmc_ri_to_classdep(md, ri, &adjri);
1255                 pcd->pcd_config_pmc(cpu, adjri, pm);
1256
1257                 phw = pc->pc_hwpmcs[ri];
1258
1259                 KASSERT(phw != NULL,
1260                     ("[pmc,%d] null hw pointer", __LINE__));
1261
1262                 KASSERT(phw->phw_pmc == pm,
1263                     ("[pmc,%d] hw->pmc %p != pmc %p", __LINE__,
1264                         phw->phw_pmc, pm));
1265
1266                 /*
1267                  * Write out saved value and start the PMC.
1268                  *
1269                  * Sampling PMCs use a per-process value, while
1270                  * counting mode PMCs use a per-pmc value that is
1271                  * inherited across descendants.
1272                  */
1273                 if (PMC_TO_MODE(pm) == PMC_MODE_TS) {
1274                         mtx_pool_lock_spin(pmc_mtxpool, pm);
1275                         newvalue = PMC_PCPU_SAVED(cpu,ri) =
1276                             pp->pp_pmcs[ri].pp_pmcval;
1277                         mtx_pool_unlock_spin(pmc_mtxpool, pm);
1278                 } else {
1279                         KASSERT(PMC_TO_MODE(pm) == PMC_MODE_TC,
1280                             ("[pmc,%d] illegal mode=%d", __LINE__,
1281                             PMC_TO_MODE(pm)));
1282                         mtx_pool_lock_spin(pmc_mtxpool, pm);
1283                         newvalue = PMC_PCPU_SAVED(cpu, ri) =
1284                             pm->pm_gv.pm_savedvalue;
1285                         mtx_pool_unlock_spin(pmc_mtxpool, pm);
1286                 }
1287
1288                 PMCDBG(CSW,SWI,1,"cpu=%d ri=%d new=%jd", cpu, ri, newvalue);
1289
1290                 pcd->pcd_write_pmc(cpu, adjri, newvalue);
1291                 pcd->pcd_start_pmc(cpu, adjri);
1292         }
1293
1294         /*
1295          * perform any other architecture/cpu dependent thread
1296          * switch-in actions.
1297          */
1298
1299         (void) (*md->pmd_switch_in)(pc, pp);
1300
1301         critical_exit();
1302
1303 }
1304
1305 /*
1306  * Thread context switch OUT.
1307  */
1308
1309 static void
1310 pmc_process_csw_out(struct thread *td)
1311 {
1312         int cpu;
1313         int64_t tmp;
1314         struct pmc *pm;
1315         struct proc *p;
1316         enum pmc_mode mode;
1317         struct pmc_cpu *pc;
1318         pmc_value_t newvalue;
1319         unsigned int adjri, ri;
1320         struct pmc_process *pp;
1321         struct pmc_classdep *pcd;
1322
1323
1324         /*
1325          * Locate our process descriptor; this may be NULL if
1326          * this process is exiting and we have already removed
1327          * the process from the target process table.
1328          *
1329          * Note that due to kernel preemption, multiple
1330          * context switches may happen while the process is
1331          * exiting.
1332          *
1333          * Note also that if the target process cannot be
1334          * found we still need to deconfigure any PMCs that
1335          * are currently running on hardware.
1336          */
1337
1338         p = td->td_proc;
1339         pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE);
1340
1341         /*
1342          * save PMCs
1343          */
1344
1345         critical_enter();
1346
1347         cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */
1348
1349         PMCDBG(CSW,SWO,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p,
1350             p->p_pid, p->p_comm, pp);
1351
1352         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
1353             ("[pmc,%d wierd CPU id %d", __LINE__, cpu));
1354
1355         pc = pmc_pcpu[cpu];
1356
1357         /*
1358          * When a PMC gets unlinked from a target PMC, it will
1359          * be removed from the target's pp_pmc[] array.
1360          *
1361          * However, on a MP system, the target could have been
1362          * executing on another CPU at the time of the unlink.
1363          * So, at context switch OUT time, we need to look at
1364          * the hardware to determine if a PMC is scheduled on
1365          * it.
1366          */
1367
1368         for (ri = 0; ri < md->pmd_npmc; ri++) {
1369
1370                 pcd = pmc_ri_to_classdep(md, ri, &adjri);
1371                 pm  = NULL;
1372                 (void) (*pcd->pcd_get_config)(cpu, adjri, &pm);
1373
1374                 if (pm == NULL) /* nothing at this row index */
1375                         continue;
1376
1377                 mode = PMC_TO_MODE(pm);
1378                 if (!PMC_IS_VIRTUAL_MODE(mode))
1379                         continue; /* not a process virtual PMC */
1380
1381                 KASSERT(PMC_TO_ROWINDEX(pm) == ri,
1382                     ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
1383                         __LINE__, PMC_TO_ROWINDEX(pm), ri));
1384
1385                 /* Stop hardware if not already stopped */
1386                 if (pm->pm_stalled == 0)
1387                         pcd->pcd_stop_pmc(cpu, adjri);
1388
1389                 /* reduce this PMC's runcount */
1390                 atomic_subtract_rel_32(&pm->pm_runcount, 1);
1391
1392                 /*
1393                  * If this PMC is associated with this process,
1394                  * save the reading.
1395                  */
1396
1397                 if (pp != NULL && pp->pp_pmcs[ri].pp_pmc != NULL) {
1398
1399                         KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
1400                             ("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__,
1401                                 pm, ri, pp->pp_pmcs[ri].pp_pmc));
1402
1403                         KASSERT(pp->pp_refcnt > 0,
1404                             ("[pmc,%d] pp refcnt = %d", __LINE__,
1405                                 pp->pp_refcnt));
1406
1407                         pcd->pcd_read_pmc(cpu, adjri, &newvalue);
1408
1409                         tmp = newvalue - PMC_PCPU_SAVED(cpu,ri);
1410
1411                         PMCDBG(CSW,SWI,1,"cpu=%d ri=%d tmp=%jd", cpu, ri,
1412                             tmp);
1413
1414                         if (mode == PMC_MODE_TS) {
1415
1416                                 /*
1417                                  * For sampling process-virtual PMCs,
1418                                  * we expect the count to be
1419                                  * decreasing as the 'value'
1420                                  * programmed into the PMC is the
1421                                  * number of events to be seen till
1422                                  * the next sampling interrupt.
1423                                  */
1424                                 if (tmp < 0)
1425                                         tmp += pm->pm_sc.pm_reloadcount;
1426                                 mtx_pool_lock_spin(pmc_mtxpool, pm);
1427                                 pp->pp_pmcs[ri].pp_pmcval -= tmp;
1428                                 if ((int64_t) pp->pp_pmcs[ri].pp_pmcval < 0)
1429                                         pp->pp_pmcs[ri].pp_pmcval +=
1430                                             pm->pm_sc.pm_reloadcount;
1431                                 mtx_pool_unlock_spin(pmc_mtxpool, pm);
1432
1433                         } else {
1434
1435                                 /*
1436                                  * For counting process-virtual PMCs,
1437                                  * we expect the count to be
1438                                  * increasing monotonically, modulo a 64
1439                                  * bit wraparound.
1440                                  */
1441                                 KASSERT((int64_t) tmp >= 0,
1442                                     ("[pmc,%d] negative increment cpu=%d "
1443                                      "ri=%d newvalue=%jx saved=%jx "
1444                                      "incr=%jx", __LINE__, cpu, ri,
1445                                      newvalue, PMC_PCPU_SAVED(cpu,ri), tmp));
1446
1447                                 mtx_pool_lock_spin(pmc_mtxpool, pm);
1448                                 pm->pm_gv.pm_savedvalue += tmp;
1449                                 pp->pp_pmcs[ri].pp_pmcval += tmp;
1450                                 mtx_pool_unlock_spin(pmc_mtxpool, pm);
1451
1452                                 if (pm->pm_flags & PMC_F_LOG_PROCCSW)
1453                                         pmclog_process_proccsw(pm, pp, tmp);
1454                         }
1455                 }
1456
1457                 /* mark hardware as free */
1458                 pcd->pcd_config_pmc(cpu, adjri, NULL);
1459         }
1460
1461         /*
1462          * perform any other architecture/cpu dependent thread
1463          * switch out functions.
1464          */
1465
1466         (void) (*md->pmd_switch_out)(pc, pp);
1467
1468         critical_exit();
1469 }
1470
1471 /*
1472  * Log a KLD operation.
1473  */
1474
1475 static void
1476 pmc_process_kld_load(struct pmckern_map_in *pkm)
1477 {
1478         struct pmc_owner *po;
1479
1480         sx_assert(&pmc_sx, SX_LOCKED);
1481
1482         /*
1483          * Notify owners of system sampling PMCs about KLD operations.
1484          */
1485
1486         LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
1487             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
1488                 pmclog_process_map_in(po, (pid_t) -1, pkm->pm_address,
1489                     (char *) pkm->pm_file);
1490
1491         /*
1492          * TODO: Notify owners of (all) process-sampling PMCs too.
1493          */
1494
1495         return;
1496 }
1497
1498 static void
1499 pmc_process_kld_unload(struct pmckern_map_out *pkm)
1500 {
1501         struct pmc_owner *po;
1502
1503         sx_assert(&pmc_sx, SX_LOCKED);
1504
1505         LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
1506             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
1507                 pmclog_process_map_out(po, (pid_t) -1,
1508                     pkm->pm_address, pkm->pm_address + pkm->pm_size);
1509
1510         /*
1511          * TODO: Notify owners of process-sampling PMCs.
1512          */
1513 }
1514
1515 /*
1516  * A mapping change for a process.
1517  */
1518
1519 static void
1520 pmc_process_mmap(struct thread *td, struct pmckern_map_in *pkm)
1521 {
1522         int ri;
1523         pid_t pid;
1524         char *fullpath, *freepath;
1525         const struct pmc *pm;
1526         struct pmc_owner *po;
1527         const struct pmc_process *pp;
1528
1529         freepath = fullpath = NULL;
1530         pmc_getfilename((struct vnode *) pkm->pm_file, &fullpath, &freepath);
1531
1532         pid = td->td_proc->p_pid;
1533
1534         /* Inform owners of all system-wide sampling PMCs. */
1535         LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
1536             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
1537                 pmclog_process_map_in(po, pid, pkm->pm_address, fullpath);
1538
1539         if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL)
1540                 goto done;
1541
1542         /*
1543          * Inform sampling PMC owners tracking this process.
1544          */
1545         for (ri = 0; ri < md->pmd_npmc; ri++)
1546                 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL &&
1547                     PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1548                         pmclog_process_map_in(pm->pm_owner,
1549                             pid, pkm->pm_address, fullpath);
1550
1551   done:
1552         if (freepath)
1553                 free(freepath, M_TEMP);
1554 }
1555
1556
1557 /*
1558  * Log an munmap request.
1559  */
1560
1561 static void
1562 pmc_process_munmap(struct thread *td, struct pmckern_map_out *pkm)
1563 {
1564         int ri;
1565         pid_t pid;
1566         struct pmc_owner *po;
1567         const struct pmc *pm;
1568         const struct pmc_process *pp;
1569
1570         pid = td->td_proc->p_pid;
1571
1572         LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
1573             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
1574                 pmclog_process_map_out(po, pid, pkm->pm_address,
1575                     pkm->pm_address + pkm->pm_size);
1576
1577         if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL)
1578                 return;
1579
1580         for (ri = 0; ri < md->pmd_npmc; ri++)
1581                 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL &&
1582                     PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1583                         pmclog_process_map_out(pm->pm_owner, pid,
1584                             pkm->pm_address, pkm->pm_address + pkm->pm_size);
1585 }
1586
1587 /*
1588  * Log mapping information about the kernel.
1589  */
1590
1591 static void
1592 pmc_log_kernel_mappings(struct pmc *pm)
1593 {
1594         struct pmc_owner *po;
1595         struct pmckern_map_in *km, *kmbase;
1596
1597         sx_assert(&pmc_sx, SX_LOCKED);
1598         KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
1599             ("[pmc,%d] non-sampling PMC (%p) desires mapping information",
1600                 __LINE__, (void *) pm));
1601
1602         po = pm->pm_owner;
1603
1604         if (po->po_flags & PMC_PO_INITIAL_MAPPINGS_DONE)
1605                 return;
1606
1607         /*
1608          * Log the current set of kernel modules.
1609          */
1610         kmbase = linker_hwpmc_list_objects();
1611         for (km = kmbase; km->pm_file != NULL; km++) {
1612                 PMCDBG(LOG,REG,1,"%s %p", (char *) km->pm_file,
1613                     (void *) km->pm_address);
1614                 pmclog_process_map_in(po, (pid_t) -1, km->pm_address,
1615                     km->pm_file);
1616         }
1617         free(kmbase, M_LINKER);
1618
1619         po->po_flags |= PMC_PO_INITIAL_MAPPINGS_DONE;
1620 }
1621
1622 /*
1623  * Log the mappings for a single process.
1624  */
1625
1626 static void
1627 pmc_log_process_mappings(struct pmc_owner *po, struct proc *p)
1628 {
1629         int locked;
1630         vm_map_t map;
1631         struct vnode *vp;
1632         struct vmspace *vm;
1633         vm_map_entry_t entry;
1634         vm_offset_t last_end;
1635         u_int last_timestamp;
1636         struct vnode *last_vp;
1637         vm_offset_t start_addr;
1638         vm_object_t obj, lobj, tobj;
1639         char *fullpath, *freepath;
1640
1641         last_vp = NULL;
1642         last_end = (vm_offset_t) 0;
1643         fullpath = freepath = NULL;
1644
1645         if ((vm = vmspace_acquire_ref(p)) == NULL)
1646                 return;
1647
1648         map = &vm->vm_map;
1649         vm_map_lock_read(map);
1650
1651         for (entry = map->header.next; entry != &map->header; entry = entry->next) {
1652
1653                 if (entry == NULL) {
1654                         PMCDBG(LOG,OPS,2, "hwpmc: vm_map entry unexpectedly "
1655                             "NULL! pid=%d vm_map=%p\n", p->p_pid, map);
1656                         break;
1657                 }
1658
1659                 /*
1660                  * We only care about executable map entries.
1661                  */
1662                 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) ||
1663                     !(entry->protection & VM_PROT_EXECUTE) ||
1664                     (entry->object.vm_object == NULL)) {
1665                         continue;
1666                 }
1667
1668                 obj = entry->object.vm_object;
1669                 VM_OBJECT_LOCK(obj);
1670
1671                 /*
1672                  * Walk the backing_object list to find the base
1673                  * (non-shadowed) vm_object.
1674                  */
1675                 for (lobj = tobj = obj; tobj != NULL; tobj = tobj->backing_object) {
1676                         if (tobj != obj)
1677                                 VM_OBJECT_LOCK(tobj);
1678                         if (lobj != obj)
1679                                 VM_OBJECT_UNLOCK(lobj);
1680                         lobj = tobj;
1681                 }
1682
1683                 /*
1684                  * At this point lobj is the base vm_object and it is locked.
1685                  */
1686                 if (lobj == NULL) {
1687                         PMCDBG(LOG,OPS,2, "hwpmc: lobj unexpectedly NULL! pid=%d "
1688                             "vm_map=%p vm_obj=%p\n", p->p_pid, map, obj);
1689                         VM_OBJECT_UNLOCK(obj);
1690                         continue;
1691                 }
1692
1693                 if (lobj->type != OBJT_VNODE || lobj->handle == NULL) {
1694                         if (lobj != obj)
1695                                 VM_OBJECT_UNLOCK(lobj);
1696                         VM_OBJECT_UNLOCK(obj);
1697                         continue;
1698                 }
1699
1700                 /*
1701                  * Skip contiguous regions that point to the same
1702                  * vnode, so we don't emit redundant MAP-IN
1703                  * directives.
1704                  */
1705                 if (entry->start == last_end && lobj->handle == last_vp) {
1706                         last_end = entry->end;
1707                         if (lobj != obj)
1708                                 VM_OBJECT_UNLOCK(lobj);
1709                         VM_OBJECT_UNLOCK(obj);
1710                         continue;
1711                 }
1712
1713                 /*
1714                  * We don't want to keep the proc's vm_map or this
1715                  * vm_object locked while we walk the pathname, since
1716                  * vn_fullpath() can sleep.  However, if we drop the
1717                  * lock, it's possible for concurrent activity to
1718                  * modify the vm_map list.  To protect against this,
1719                  * we save the vm_map timestamp before we release the
1720                  * lock, and check it after we reacquire the lock
1721                  * below.
1722                  */
1723                 start_addr = entry->start;
1724                 last_end = entry->end;
1725                 last_timestamp = map->timestamp;
1726                 vm_map_unlock_read(map);
1727
1728                 vp = lobj->handle;
1729                 vref(vp);
1730                 if (lobj != obj)
1731                         VM_OBJECT_UNLOCK(lobj);
1732
1733                 VM_OBJECT_UNLOCK(obj);
1734
1735                 freepath = NULL;
1736                 pmc_getfilename(vp, &fullpath, &freepath);
1737                 last_vp = vp;
1738
1739                 locked = VFS_LOCK_GIANT(vp->v_mount);
1740                 vrele(vp);
1741                 VFS_UNLOCK_GIANT(locked);
1742
1743                 vp = NULL;
1744                 pmclog_process_map_in(po, p->p_pid, start_addr, fullpath);
1745                 if (freepath)
1746                         free(freepath, M_TEMP);
1747
1748                 vm_map_lock_read(map);
1749
1750                 /*
1751                  * If our saved timestamp doesn't match, this means
1752                  * that the vm_map was modified out from under us and
1753                  * we can't trust our current "entry" pointer.  Do a
1754                  * new lookup for this entry.  If there is no entry
1755                  * for this address range, vm_map_lookup_entry() will
1756                  * return the previous one, so we always want to go to
1757                  * entry->next on the next loop iteration.
1758                  *
1759                  * There is an edge condition here that can occur if
1760                  * there is no entry at or before this address.  In
1761                  * this situation, vm_map_lookup_entry returns
1762                  * &map->header, which would cause our loop to abort
1763                  * without processing the rest of the map.  However,
1764                  * in practice this will never happen for process
1765                  * vm_map.  This is because the executable's text
1766                  * segment is the first mapping in the proc's address
1767                  * space, and this mapping is never removed until the
1768                  * process exits, so there will always be a non-header
1769                  * entry at or before the requested address for
1770                  * vm_map_lookup_entry to return.
1771                  */
1772                 if (map->timestamp != last_timestamp)
1773                         vm_map_lookup_entry(map, last_end - 1, &entry);
1774         }
1775
1776         vm_map_unlock_read(map);
1777         vmspace_free(vm);
1778         return;
1779 }
1780
1781 /*
1782  * Log mappings for all processes in the system.
1783  */
1784
1785 static void
1786 pmc_log_all_process_mappings(struct pmc_owner *po)
1787 {
1788         struct proc *p, *top;
1789
1790         sx_assert(&pmc_sx, SX_XLOCKED);
1791
1792         if ((p = pfind(1)) == NULL)
1793                 panic("[pmc,%d] Cannot find init", __LINE__);
1794
1795         PROC_UNLOCK(p);
1796
1797         sx_slock(&proctree_lock);
1798
1799         top = p;
1800
1801         for (;;) {
1802                 pmc_log_process_mappings(po, p);
1803                 if (!LIST_EMPTY(&p->p_children))
1804                         p = LIST_FIRST(&p->p_children);
1805                 else for (;;) {
1806                         if (p == top)
1807                                 goto done;
1808                         if (LIST_NEXT(p, p_sibling)) {
1809                                 p = LIST_NEXT(p, p_sibling);
1810                                 break;
1811                         }
1812                         p = p->p_pptr;
1813                 }
1814         }
1815  done:
1816         sx_sunlock(&proctree_lock);
1817 }
1818
1819 /*
1820  * The 'hook' invoked from the kernel proper
1821  */
1822
1823
1824 #ifdef  DEBUG
1825 const char *pmc_hooknames[] = {
1826         /* these strings correspond to PMC_FN_* in <sys/pmckern.h> */
1827         "",
1828         "EXEC",
1829         "CSW-IN",
1830         "CSW-OUT",
1831         "SAMPLE",
1832         "KLDLOAD",
1833         "KLDUNLOAD",
1834         "MMAP",
1835         "MUNMAP",
1836         "CALLCHAIN"
1837 };
1838 #endif
1839
1840 static int
1841 pmc_hook_handler(struct thread *td, int function, void *arg)
1842 {
1843
1844         PMCDBG(MOD,PMH,1, "hook td=%p func=%d \"%s\" arg=%p", td, function,
1845             pmc_hooknames[function], arg);
1846
1847         switch (function)
1848         {
1849
1850         /*
1851          * Process exec()
1852          */
1853
1854         case PMC_FN_PROCESS_EXEC:
1855         {
1856                 char *fullpath, *freepath;
1857                 unsigned int ri;
1858                 int is_using_hwpmcs;
1859                 struct pmc *pm;
1860                 struct proc *p;
1861                 struct pmc_owner *po;
1862                 struct pmc_process *pp;
1863                 struct pmckern_procexec *pk;
1864
1865                 sx_assert(&pmc_sx, SX_XLOCKED);
1866
1867                 p = td->td_proc;
1868                 pmc_getfilename(p->p_textvp, &fullpath, &freepath);
1869
1870                 pk = (struct pmckern_procexec *) arg;
1871
1872                 /* Inform owners of SS mode PMCs of the exec event. */
1873                 LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
1874                     if (po->po_flags & PMC_PO_OWNS_LOGFILE)
1875                             pmclog_process_procexec(po, PMC_ID_INVALID,
1876                                 p->p_pid, pk->pm_entryaddr, fullpath);
1877
1878                 PROC_LOCK(p);
1879                 is_using_hwpmcs = p->p_flag & P_HWPMC;
1880                 PROC_UNLOCK(p);
1881
1882                 if (!is_using_hwpmcs) {
1883                         if (freepath)
1884                                 free(freepath, M_TEMP);
1885                         break;
1886                 }
1887
1888                 /*
1889                  * PMCs are not inherited across an exec():  remove any
1890                  * PMCs that this process is the owner of.
1891                  */
1892
1893                 if ((po = pmc_find_owner_descriptor(p)) != NULL) {
1894                         pmc_remove_owner(po);
1895                         pmc_destroy_owner_descriptor(po);
1896                 }
1897
1898                 /*
1899                  * If the process being exec'ed is not the target of any
1900                  * PMC, we are done.
1901                  */
1902                 if ((pp = pmc_find_process_descriptor(p, 0)) == NULL) {
1903                         if (freepath)
1904                                 free(freepath, M_TEMP);
1905                         break;
1906                 }
1907
1908                 /*
1909                  * Log the exec event to all monitoring owners.  Skip
1910                  * owners who have already recieved the event because
1911                  * they had system sampling PMCs active.
1912                  */
1913                 for (ri = 0; ri < md->pmd_npmc; ri++)
1914                         if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) {
1915                                 po = pm->pm_owner;
1916                                 if (po->po_sscount == 0 &&
1917                                     po->po_flags & PMC_PO_OWNS_LOGFILE)
1918                                         pmclog_process_procexec(po, pm->pm_id,
1919                                             p->p_pid, pk->pm_entryaddr,
1920                                             fullpath);
1921                         }
1922
1923                 if (freepath)
1924                         free(freepath, M_TEMP);
1925
1926
1927                 PMCDBG(PRC,EXC,1, "exec proc=%p (%d, %s) cred-changed=%d",
1928                     p, p->p_pid, p->p_comm, pk->pm_credentialschanged);
1929
1930                 if (pk->pm_credentialschanged == 0) /* no change */
1931                         break;
1932
1933                 /*
1934                  * If the newly exec()'ed process has a different credential
1935                  * than before, allow it to be the target of a PMC only if
1936                  * the PMC's owner has sufficient priviledge.
1937                  */
1938
1939                 for (ri = 0; ri < md->pmd_npmc; ri++)
1940                         if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL)
1941                                 if (pmc_can_attach(pm, td->td_proc) != 0)
1942                                         pmc_detach_one_process(td->td_proc,
1943                                             pm, PMC_FLAG_NONE);
1944
1945                 KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= (int) md->pmd_npmc,
1946                     ("[pmc,%d] Illegal ref count %d on pp %p", __LINE__,
1947                         pp->pp_refcnt, pp));
1948
1949                 /*
1950                  * If this process is no longer the target of any
1951                  * PMCs, we can remove the process entry and free
1952                  * up space.
1953                  */
1954
1955                 if (pp->pp_refcnt == 0) {
1956                         pmc_remove_process_descriptor(pp);
1957                         free(pp, M_PMC);
1958                         break;
1959                 }
1960
1961         }
1962         break;
1963
1964         case PMC_FN_CSW_IN:
1965                 pmc_process_csw_in(td);
1966                 break;
1967
1968         case PMC_FN_CSW_OUT:
1969                 pmc_process_csw_out(td);
1970                 break;
1971
1972         /*
1973          * Process accumulated PC samples.
1974          *
1975          * This function is expected to be called by hardclock() for
1976          * each CPU that has accumulated PC samples.
1977          *
1978          * This function is to be executed on the CPU whose samples
1979          * are being processed.
1980          */
1981         case PMC_FN_DO_SAMPLES:
1982
1983                 /*
1984                  * Clear the cpu specific bit in the CPU mask before
1985                  * do the rest of the processing.  If the NMI handler
1986                  * gets invoked after the "atomic_clear_int()" call
1987                  * below but before "pmc_process_samples()" gets
1988                  * around to processing the interrupt, then we will
1989                  * come back here at the next hardclock() tick (and
1990                  * may find nothing to do if "pmc_process_samples()"
1991                  * had already processed the interrupt).  We don't
1992                  * lose the interrupt sample.
1993                  */
1994                 atomic_clear_int(&pmc_cpumask, (1 << PCPU_GET(cpuid)));
1995                 pmc_process_samples(PCPU_GET(cpuid));
1996                 break;
1997
1998
1999         case PMC_FN_KLD_LOAD:
2000                 sx_assert(&pmc_sx, SX_LOCKED);
2001                 pmc_process_kld_load((struct pmckern_map_in *) arg);
2002                 break;
2003
2004         case PMC_FN_KLD_UNLOAD:
2005                 sx_assert(&pmc_sx, SX_LOCKED);
2006                 pmc_process_kld_unload((struct pmckern_map_out *) arg);
2007                 break;
2008
2009         case PMC_FN_MMAP:
2010                 sx_assert(&pmc_sx, SX_LOCKED);
2011                 pmc_process_mmap(td, (struct pmckern_map_in *) arg);
2012                 break;
2013
2014         case PMC_FN_MUNMAP:
2015                 sx_assert(&pmc_sx, SX_LOCKED);
2016                 pmc_process_munmap(td, (struct pmckern_map_out *) arg);
2017                 break;
2018
2019         case PMC_FN_USER_CALLCHAIN:
2020                 /*
2021                  * Record a call chain.
2022                  */
2023                 KASSERT(td == curthread, ("[pmc,%d] td != curthread",
2024                     __LINE__));
2025                 pmc_capture_user_callchain(PCPU_GET(cpuid),
2026                     (struct trapframe *) arg);
2027                 td->td_pflags &= ~TDP_CALLCHAIN;
2028                 break;
2029
2030         default:
2031 #ifdef  DEBUG
2032                 KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function));
2033 #endif
2034                 break;
2035
2036         }
2037
2038         return 0;
2039 }
2040
2041 /*
2042  * allocate a 'struct pmc_owner' descriptor in the owner hash table.
2043  */
2044
2045 static struct pmc_owner *
2046 pmc_allocate_owner_descriptor(struct proc *p)
2047 {
2048         uint32_t hindex;
2049         struct pmc_owner *po;
2050         struct pmc_ownerhash *poh;
2051
2052         hindex = PMC_HASH_PTR(p, pmc_ownerhashmask);
2053         poh = &pmc_ownerhash[hindex];
2054
2055         /* allocate space for N pointers and one descriptor struct */
2056         po = malloc(sizeof(struct pmc_owner), M_PMC, M_WAITOK|M_ZERO);
2057         po->po_sscount = po->po_error = po->po_flags = po->po_logprocmaps = 0;
2058         po->po_file  = NULL;
2059         po->po_owner = p;
2060         po->po_kthread = NULL;
2061         LIST_INIT(&po->po_pmcs);
2062         LIST_INSERT_HEAD(poh, po, po_next); /* insert into hash table */
2063
2064         TAILQ_INIT(&po->po_logbuffers);
2065         mtx_init(&po->po_mtx, "pmc-owner-mtx", "pmc-per-proc", MTX_SPIN);
2066
2067         PMCDBG(OWN,ALL,1, "allocate-owner proc=%p (%d, %s) pmc-owner=%p",
2068             p, p->p_pid, p->p_comm, po);
2069
2070         return po;
2071 }
2072
2073 static void
2074 pmc_destroy_owner_descriptor(struct pmc_owner *po)
2075 {
2076
2077         PMCDBG(OWN,REL,1, "destroy-owner po=%p proc=%p (%d, %s)",
2078             po, po->po_owner, po->po_owner->p_pid, po->po_owner->p_comm);
2079
2080         mtx_destroy(&po->po_mtx);
2081         free(po, M_PMC);
2082 }
2083
2084 /*
2085  * find the descriptor corresponding to process 'p', adding or removing it
2086  * as specified by 'mode'.
2087  */
2088
2089 static struct pmc_process *
2090 pmc_find_process_descriptor(struct proc *p, uint32_t mode)
2091 {
2092         uint32_t hindex;
2093         struct pmc_process *pp, *ppnew;
2094         struct pmc_processhash *pph;
2095
2096         hindex = PMC_HASH_PTR(p, pmc_processhashmask);
2097         pph = &pmc_processhash[hindex];
2098
2099         ppnew = NULL;
2100
2101         /*
2102          * Pre-allocate memory in the FIND_ALLOCATE case since we
2103          * cannot call malloc(9) once we hold a spin lock.
2104          */
2105         if (mode & PMC_FLAG_ALLOCATE)
2106                 ppnew = malloc(sizeof(struct pmc_process) + md->pmd_npmc *
2107                     sizeof(struct pmc_targetstate), M_PMC, M_WAITOK|M_ZERO);
2108
2109         mtx_lock_spin(&pmc_processhash_mtx);
2110         LIST_FOREACH(pp, pph, pp_next)
2111             if (pp->pp_proc == p)
2112                     break;
2113
2114         if ((mode & PMC_FLAG_REMOVE) && pp != NULL)
2115                 LIST_REMOVE(pp, pp_next);
2116
2117         if ((mode & PMC_FLAG_ALLOCATE) && pp == NULL &&
2118             ppnew != NULL) {
2119                 ppnew->pp_proc = p;
2120                 LIST_INSERT_HEAD(pph, ppnew, pp_next);
2121                 pp = ppnew;
2122                 ppnew = NULL;
2123         }
2124         mtx_unlock_spin(&pmc_processhash_mtx);
2125
2126         if (pp != NULL && ppnew != NULL)
2127                 free(ppnew, M_PMC);
2128
2129         return pp;
2130 }
2131
2132 /*
2133  * remove a process descriptor from the process hash table.
2134  */
2135
2136 static void
2137 pmc_remove_process_descriptor(struct pmc_process *pp)
2138 {
2139         KASSERT(pp->pp_refcnt == 0,
2140             ("[pmc,%d] Removing process descriptor %p with count %d",
2141                 __LINE__, pp, pp->pp_refcnt));
2142
2143         mtx_lock_spin(&pmc_processhash_mtx);
2144         LIST_REMOVE(pp, pp_next);
2145         mtx_unlock_spin(&pmc_processhash_mtx);
2146 }
2147
2148
2149 /*
2150  * find an owner descriptor corresponding to proc 'p'
2151  */
2152
2153 static struct pmc_owner *
2154 pmc_find_owner_descriptor(struct proc *p)
2155 {
2156         uint32_t hindex;
2157         struct pmc_owner *po;
2158         struct pmc_ownerhash *poh;
2159
2160         hindex = PMC_HASH_PTR(p, pmc_ownerhashmask);
2161         poh = &pmc_ownerhash[hindex];
2162
2163         po = NULL;
2164         LIST_FOREACH(po, poh, po_next)
2165             if (po->po_owner == p)
2166                     break;
2167
2168         PMCDBG(OWN,FND,1, "find-owner proc=%p (%d, %s) hindex=0x%x -> "
2169             "pmc-owner=%p", p, p->p_pid, p->p_comm, hindex, po);
2170
2171         return po;
2172 }
2173
2174 /*
2175  * pmc_allocate_pmc_descriptor
2176  *
2177  * Allocate a pmc descriptor and initialize its
2178  * fields.
2179  */
2180
2181 static struct pmc *
2182 pmc_allocate_pmc_descriptor(void)
2183 {
2184         struct pmc *pmc;
2185
2186         pmc = malloc(sizeof(struct pmc), M_PMC, M_WAITOK|M_ZERO);
2187
2188         if (pmc != NULL) {
2189                 pmc->pm_owner = NULL;
2190                 LIST_INIT(&pmc->pm_targets);
2191         }
2192
2193         PMCDBG(PMC,ALL,1, "allocate-pmc -> pmc=%p", pmc);
2194
2195         return pmc;
2196 }
2197
2198 /*
2199  * Destroy a pmc descriptor.
2200  */
2201
2202 static void
2203 pmc_destroy_pmc_descriptor(struct pmc *pm)
2204 {
2205         (void) pm;
2206
2207 #ifdef  DEBUG
2208         KASSERT(pm->pm_state == PMC_STATE_DELETED ||
2209             pm->pm_state == PMC_STATE_FREE,
2210             ("[pmc,%d] destroying non-deleted PMC", __LINE__));
2211         KASSERT(LIST_EMPTY(&pm->pm_targets),
2212             ("[pmc,%d] destroying pmc with targets", __LINE__));
2213         KASSERT(pm->pm_owner == NULL,
2214             ("[pmc,%d] destroying pmc attached to an owner", __LINE__));
2215         KASSERT(pm->pm_runcount == 0,
2216             ("[pmc,%d] pmc has non-zero run count %d", __LINE__,
2217                 pm->pm_runcount));
2218 #endif
2219 }
2220
2221 static void
2222 pmc_wait_for_pmc_idle(struct pmc *pm)
2223 {
2224 #ifdef  DEBUG
2225         volatile int maxloop;
2226
2227         maxloop = 100 * pmc_cpu_max();
2228 #endif
2229
2230         /*
2231          * Loop (with a forced context switch) till the PMC's runcount
2232          * comes down to zero.
2233          */
2234         while (atomic_load_acq_32(&pm->pm_runcount) > 0) {
2235 #ifdef  DEBUG
2236                 maxloop--;
2237                 KASSERT(maxloop > 0,
2238                     ("[pmc,%d] (ri%d, rc%d) waiting too long for "
2239                         "pmc to be free", __LINE__,
2240                         PMC_TO_ROWINDEX(pm), pm->pm_runcount));
2241 #endif
2242                 pmc_force_context_switch();
2243         }
2244 }
2245
2246 /*
2247  * This function does the following things:
2248  *
2249  *  - detaches the PMC from hardware
2250  *  - unlinks all target threads that were attached to it
2251  *  - removes the PMC from its owner's list
2252  *  - destroy's the PMC private mutex
2253  *
2254  * Once this function completes, the given pmc pointer can be safely
2255  * FREE'd by the caller.
2256  */
2257
2258 static void
2259 pmc_release_pmc_descriptor(struct pmc *pm)
2260 {
2261         enum pmc_mode mode;
2262         struct pmc_hw *phw;
2263         u_int adjri, ri, cpu;
2264         struct pmc_owner *po;
2265         struct pmc_binding pb;
2266         struct pmc_process *pp;
2267         struct pmc_classdep *pcd;
2268         struct pmc_target *ptgt, *tmp;
2269
2270         sx_assert(&pmc_sx, SX_XLOCKED);
2271
2272         KASSERT(pm, ("[pmc,%d] null pmc", __LINE__));
2273
2274         ri   = PMC_TO_ROWINDEX(pm);
2275         pcd  = pmc_ri_to_classdep(md, ri, &adjri);
2276         mode = PMC_TO_MODE(pm);
2277
2278         PMCDBG(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri,
2279             mode);
2280
2281         /*
2282          * First, we take the PMC off hardware.
2283          */
2284         cpu = 0;
2285         if (PMC_IS_SYSTEM_MODE(mode)) {
2286
2287                 /*
2288                  * A system mode PMC runs on a specific CPU.  Switch
2289                  * to this CPU and turn hardware off.
2290                  */
2291                 pmc_save_cpu_binding(&pb);
2292
2293                 cpu = PMC_TO_CPU(pm);
2294
2295                 pmc_select_cpu(cpu);
2296
2297                 /* switch off non-stalled CPUs */
2298                 if (pm->pm_state == PMC_STATE_RUNNING &&
2299                     pm->pm_stalled == 0) {
2300
2301                         phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
2302
2303                         KASSERT(phw->phw_pmc == pm,
2304                             ("[pmc, %d] pmc ptr ri(%d) hw(%p) pm(%p)",
2305                                 __LINE__, ri, phw->phw_pmc, pm));
2306                         PMCDBG(PMC,REL,2, "stopping cpu=%d ri=%d", cpu, ri);
2307
2308                         critical_enter();
2309                         pcd->pcd_stop_pmc(cpu, adjri);
2310                         critical_exit();
2311                 }
2312
2313                 PMCDBG(PMC,REL,2, "decfg cpu=%d ri=%d", cpu, ri);
2314
2315                 critical_enter();
2316                 pcd->pcd_config_pmc(cpu, adjri, NULL);
2317                 critical_exit();
2318
2319                 /* adjust the global and process count of SS mode PMCs */
2320                 if (mode == PMC_MODE_SS && pm->pm_state == PMC_STATE_RUNNING) {
2321                         po = pm->pm_owner;
2322                         po->po_sscount--;
2323                         if (po->po_sscount == 0) {
2324                                 atomic_subtract_rel_int(&pmc_ss_count, 1);
2325                                 LIST_REMOVE(po, po_ssnext);
2326                         }
2327                 }
2328
2329                 pm->pm_state = PMC_STATE_DELETED;
2330
2331                 pmc_restore_cpu_binding(&pb);
2332
2333                 /*
2334                  * We could have references to this PMC structure in
2335                  * the per-cpu sample queues.  Wait for the queue to
2336                  * drain.
2337                  */
2338                 pmc_wait_for_pmc_idle(pm);
2339
2340         } else if (PMC_IS_VIRTUAL_MODE(mode)) {
2341
2342                 /*
2343                  * A virtual PMC could be running on multiple CPUs at
2344                  * a given instant.
2345                  *
2346                  * By marking its state as DELETED, we ensure that
2347                  * this PMC is never further scheduled on hardware.
2348                  *
2349                  * Then we wait till all CPUs are done with this PMC.
2350                  */
2351                 pm->pm_state = PMC_STATE_DELETED;
2352
2353
2354                 /* Wait for the PMCs runcount to come to zero. */
2355                 pmc_wait_for_pmc_idle(pm);
2356
2357                 /*
2358                  * At this point the PMC is off all CPUs and cannot be
2359                  * freshly scheduled onto a CPU.  It is now safe to
2360                  * unlink all targets from this PMC.  If a
2361                  * process-record's refcount falls to zero, we remove
2362                  * it from the hash table.  The module-wide SX lock
2363                  * protects us from races.
2364                  */
2365                 LIST_FOREACH_SAFE(ptgt, &pm->pm_targets, pt_next, tmp) {
2366                         pp = ptgt->pt_process;
2367                         pmc_unlink_target_process(pm, pp); /* frees 'ptgt' */
2368
2369                         PMCDBG(PMC,REL,3, "pp->refcnt=%d", pp->pp_refcnt);
2370
2371                         /*
2372                          * If the target process record shows that no
2373                          * PMCs are attached to it, reclaim its space.
2374                          */
2375
2376                         if (pp->pp_refcnt == 0) {
2377                                 pmc_remove_process_descriptor(pp);
2378                                 free(pp, M_PMC);
2379                         }
2380                 }
2381
2382                 cpu = curthread->td_oncpu; /* setup cpu for pmd_release() */
2383
2384         }
2385
2386         /*
2387          * Release any MD resources
2388          */
2389         (void) pcd->pcd_release_pmc(cpu, adjri, pm);
2390
2391         /*
2392          * Update row disposition
2393          */
2394
2395         if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm)))
2396                 PMC_UNMARK_ROW_STANDALONE(ri);
2397         else
2398                 PMC_UNMARK_ROW_THREAD(ri);
2399
2400         /* unlink from the owner's list */
2401         if (pm->pm_owner) {
2402                 LIST_REMOVE(pm, pm_next);
2403                 pm->pm_owner = NULL;
2404         }
2405
2406         pmc_destroy_pmc_descriptor(pm);
2407 }
2408
2409 /*
2410  * Register an owner and a pmc.
2411  */
2412
2413 static int
2414 pmc_register_owner(struct proc *p, struct pmc *pmc)
2415 {
2416         struct pmc_owner *po;
2417
2418         sx_assert(&pmc_sx, SX_XLOCKED);
2419
2420         if ((po = pmc_find_owner_descriptor(p)) == NULL)
2421                 if ((po = pmc_allocate_owner_descriptor(p)) == NULL)
2422                         return ENOMEM;
2423
2424         KASSERT(pmc->pm_owner == NULL,
2425             ("[pmc,%d] attempting to own an initialized PMC", __LINE__));
2426         pmc->pm_owner  = po;
2427
2428         LIST_INSERT_HEAD(&po->po_pmcs, pmc, pm_next);
2429
2430         PROC_LOCK(p);
2431         p->p_flag |= P_HWPMC;
2432         PROC_UNLOCK(p);
2433
2434         if (po->po_flags & PMC_PO_OWNS_LOGFILE)
2435                 pmclog_process_pmcallocate(pmc);
2436
2437         PMCDBG(PMC,REG,1, "register-owner pmc-owner=%p pmc=%p",
2438             po, pmc);
2439
2440         return 0;
2441 }
2442
2443 /*
2444  * Return the current row disposition:
2445  * == 0 => FREE
2446  *  > 0 => PROCESS MODE
2447  *  < 0 => SYSTEM MODE
2448  */
2449
2450 int
2451 pmc_getrowdisp(int ri)
2452 {
2453         return pmc_pmcdisp[ri];
2454 }
2455
2456 /*
2457  * Check if a PMC at row index 'ri' can be allocated to the current
2458  * process.
2459  *
2460  * Allocation can fail if:
2461  *   - the current process is already being profiled by a PMC at index 'ri',
2462  *     attached to it via OP_PMCATTACH.
2463  *   - the current process has already allocated a PMC at index 'ri'
2464  *     via OP_ALLOCATE.
2465  */
2466
2467 static int
2468 pmc_can_allocate_rowindex(struct proc *p, unsigned int ri, int cpu)
2469 {
2470         enum pmc_mode mode;
2471         struct pmc *pm;
2472         struct pmc_owner *po;
2473         struct pmc_process *pp;
2474
2475         PMCDBG(PMC,ALR,1, "can-allocate-rowindex proc=%p (%d, %s) ri=%d "
2476             "cpu=%d", p, p->p_pid, p->p_comm, ri, cpu);
2477
2478         /*
2479          * We shouldn't have already allocated a process-mode PMC at
2480          * row index 'ri'.
2481          *
2482          * We shouldn't have allocated a system-wide PMC on the same
2483          * CPU and same RI.
2484          */
2485         if ((po = pmc_find_owner_descriptor(p)) != NULL)
2486                 LIST_FOREACH(pm, &po->po_pmcs, pm_next) {
2487                     if (PMC_TO_ROWINDEX(pm) == ri) {
2488                             mode = PMC_TO_MODE(pm);
2489                             if (PMC_IS_VIRTUAL_MODE(mode))
2490                                     return EEXIST;
2491                             if (PMC_IS_SYSTEM_MODE(mode) &&
2492                                 (int) PMC_TO_CPU(pm) == cpu)
2493                                     return EEXIST;
2494                     }
2495                 }
2496
2497         /*
2498          * We also shouldn't be the target of any PMC at this index
2499          * since otherwise a PMC_ATTACH to ourselves will fail.
2500          */
2501         if ((pp = pmc_find_process_descriptor(p, 0)) != NULL)
2502                 if (pp->pp_pmcs[ri].pp_pmc)
2503                         return EEXIST;
2504
2505         PMCDBG(PMC,ALR,2, "can-allocate-rowindex proc=%p (%d, %s) ri=%d ok",
2506             p, p->p_pid, p->p_comm, ri);
2507
2508         return 0;
2509 }
2510
2511 /*
2512  * Check if a given PMC at row index 'ri' can be currently used in
2513  * mode 'mode'.
2514  */
2515
2516 static int
2517 pmc_can_allocate_row(int ri, enum pmc_mode mode)
2518 {
2519         enum pmc_disp   disp;
2520
2521         sx_assert(&pmc_sx, SX_XLOCKED);
2522
2523         PMCDBG(PMC,ALR,1, "can-allocate-row ri=%d mode=%d", ri, mode);
2524
2525         if (PMC_IS_SYSTEM_MODE(mode))
2526                 disp = PMC_DISP_STANDALONE;
2527         else
2528                 disp = PMC_DISP_THREAD;
2529
2530         /*
2531          * check disposition for PMC row 'ri':
2532          *
2533          * Expected disposition         Row-disposition         Result
2534          *
2535          * STANDALONE                   STANDALONE or FREE      proceed
2536          * STANDALONE                   THREAD                  fail
2537          * THREAD                       THREAD or FREE          proceed
2538          * THREAD                       STANDALONE              fail
2539          */
2540
2541         if (!PMC_ROW_DISP_IS_FREE(ri) &&
2542             !(disp == PMC_DISP_THREAD && PMC_ROW_DISP_IS_THREAD(ri)) &&
2543             !(disp == PMC_DISP_STANDALONE && PMC_ROW_DISP_IS_STANDALONE(ri)))
2544                 return EBUSY;
2545
2546         /*
2547          * All OK
2548          */
2549
2550         PMCDBG(PMC,ALR,2, "can-allocate-row ri=%d mode=%d ok", ri, mode);
2551
2552         return 0;
2553
2554 }
2555
2556 /*
2557  * Find a PMC descriptor with user handle 'pmcid' for thread 'td'.
2558  */
2559
2560 static struct pmc *
2561 pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, pmc_id_t pmcid)
2562 {
2563         struct pmc *pm;
2564
2565         KASSERT(PMC_ID_TO_ROWINDEX(pmcid) < md->pmd_npmc,
2566             ("[pmc,%d] Illegal pmc index %d (max %d)", __LINE__,
2567                 PMC_ID_TO_ROWINDEX(pmcid), md->pmd_npmc));
2568
2569         LIST_FOREACH(pm, &po->po_pmcs, pm_next)
2570             if (pm->pm_id == pmcid)
2571                     return pm;
2572
2573         return NULL;
2574 }
2575
2576 static int
2577 pmc_find_pmc(pmc_id_t pmcid, struct pmc **pmc)
2578 {
2579
2580         struct pmc *pm;
2581         struct pmc_owner *po;
2582
2583         PMCDBG(PMC,FND,1, "find-pmc id=%d", pmcid);
2584
2585         if ((po = pmc_find_owner_descriptor(curthread->td_proc)) == NULL)
2586                 return ESRCH;
2587
2588         if ((pm = pmc_find_pmc_descriptor_in_process(po, pmcid)) == NULL)
2589                 return EINVAL;
2590
2591         PMCDBG(PMC,FND,2, "find-pmc id=%d -> pmc=%p", pmcid, pm);
2592
2593         *pmc = pm;
2594         return 0;
2595 }
2596
2597 /*
2598  * Start a PMC.
2599  */
2600
2601 static int
2602 pmc_start(struct pmc *pm)
2603 {
2604         enum pmc_mode mode;
2605         struct pmc_owner *po;
2606         struct pmc_binding pb;
2607         struct pmc_classdep *pcd;
2608         int adjri, error, cpu, ri;
2609
2610         KASSERT(pm != NULL,
2611             ("[pmc,%d] null pm", __LINE__));
2612
2613         mode = PMC_TO_MODE(pm);
2614         ri   = PMC_TO_ROWINDEX(pm);
2615         pcd  = pmc_ri_to_classdep(md, ri, &adjri);
2616
2617         error = 0;
2618
2619         PMCDBG(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri);
2620
2621         po = pm->pm_owner;
2622
2623         /*
2624          * Disallow PMCSTART if a logfile is required but has not been
2625          * configured yet.
2626          */
2627         if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) &&
2628             (po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
2629                 return (EDOOFUS);       /* programming error */
2630
2631         /*
2632          * If this is a sampling mode PMC, log mapping information for
2633          * the kernel modules that are currently loaded.
2634          */
2635         if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
2636             pmc_log_kernel_mappings(pm);
2637
2638         if (PMC_IS_VIRTUAL_MODE(mode)) {
2639
2640                 /*
2641                  * If a PMCATTACH has never been done on this PMC,
2642                  * attach it to its owner process.
2643                  */
2644
2645                 if (LIST_EMPTY(&pm->pm_targets))
2646                         error = (pm->pm_flags & PMC_F_ATTACH_DONE) ? ESRCH :
2647                             pmc_attach_process(po->po_owner, pm);
2648
2649                 /*
2650                  * If the PMC is attached to its owner, then force a context
2651                  * switch to ensure that the MD state gets set correctly.
2652                  */
2653
2654                 if (error == 0) {
2655                         pm->pm_state = PMC_STATE_RUNNING;
2656                         if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER)
2657                                 pmc_force_context_switch();
2658                 }
2659
2660                 return (error);
2661         }
2662
2663
2664         /*
2665          * A system-wide PMC.
2666          *
2667          * Add the owner to the global list if this is a system-wide
2668          * sampling PMC.
2669          */
2670
2671         if (mode == PMC_MODE_SS) {
2672                 if (po->po_sscount == 0) {
2673                         LIST_INSERT_HEAD(&pmc_ss_owners, po, po_ssnext);
2674                         atomic_add_rel_int(&pmc_ss_count, 1);
2675                         PMCDBG(PMC,OPS,1, "po=%p in global list", po);
2676                 }
2677                 po->po_sscount++;
2678
2679                 /*
2680                  * Log mapping information for all existing processes in the
2681                  * system.  Subsequent mappings are logged as they happen;
2682                  * see pmc_process_mmap().
2683                  */
2684                 if (po->po_logprocmaps == 0) {
2685                         pmc_log_all_process_mappings(po);
2686                         po->po_logprocmaps = 1;
2687                 }
2688         }
2689
2690         /*
2691          * Move to the CPU associated with this
2692          * PMC, and start the hardware.
2693          */
2694
2695         pmc_save_cpu_binding(&pb);
2696
2697         cpu = PMC_TO_CPU(pm);
2698
2699         if (!pmc_cpu_is_active(cpu))
2700                 return (ENXIO);
2701
2702         pmc_select_cpu(cpu);
2703
2704         /*
2705          * global PMCs are configured at allocation time
2706          * so write out the initial value and start the PMC.
2707          */
2708
2709         pm->pm_state = PMC_STATE_RUNNING;
2710
2711         critical_enter();
2712         if ((error = pcd->pcd_write_pmc(cpu, adjri,
2713                  PMC_IS_SAMPLING_MODE(mode) ?
2714                  pm->pm_sc.pm_reloadcount :
2715                  pm->pm_sc.pm_initial)) == 0)
2716                 error = pcd->pcd_start_pmc(cpu, adjri);
2717         critical_exit();
2718
2719         pmc_restore_cpu_binding(&pb);
2720
2721         return (error);
2722 }
2723
2724 /*
2725  * Stop a PMC.
2726  */
2727
2728 static int
2729 pmc_stop(struct pmc *pm)
2730 {
2731         struct pmc_owner *po;
2732         struct pmc_binding pb;
2733         struct pmc_classdep *pcd;
2734         int adjri, cpu, error, ri;
2735
2736         KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__));
2737
2738         PMCDBG(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm,
2739             PMC_TO_MODE(pm), PMC_TO_ROWINDEX(pm));
2740
2741         pm->pm_state = PMC_STATE_STOPPED;
2742
2743         /*
2744          * If the PMC is a virtual mode one, changing the state to
2745          * non-RUNNING is enough to ensure that the PMC never gets
2746          * scheduled.
2747          *
2748          * If this PMC is current running on a CPU, then it will
2749          * handled correctly at the time its target process is context
2750          * switched out.
2751          */
2752
2753         if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
2754                 return 0;
2755
2756         /*
2757          * A system-mode PMC.  Move to the CPU associated with
2758          * this PMC, and stop the hardware.  We update the
2759          * 'initial count' so that a subsequent PMCSTART will
2760          * resume counting from the current hardware count.
2761          */
2762
2763         pmc_save_cpu_binding(&pb);
2764
2765         cpu = PMC_TO_CPU(pm);
2766
2767         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
2768             ("[pmc,%d] illegal cpu=%d", __LINE__, cpu));
2769
2770         if (!pmc_cpu_is_active(cpu))
2771                 return ENXIO;
2772
2773         pmc_select_cpu(cpu);
2774
2775         ri = PMC_TO_ROWINDEX(pm);
2776         pcd = pmc_ri_to_classdep(md, ri, &adjri);
2777
2778         critical_enter();
2779         if ((error = pcd->pcd_stop_pmc(cpu, adjri)) == 0)
2780                 error = pcd->pcd_read_pmc(cpu, adjri, &pm->pm_sc.pm_initial);
2781         critical_exit();
2782
2783         pmc_restore_cpu_binding(&pb);
2784
2785         po = pm->pm_owner;
2786
2787         /* remove this owner from the global list of SS PMC owners */
2788         if (PMC_TO_MODE(pm) == PMC_MODE_SS) {
2789                 po->po_sscount--;
2790                 if (po->po_sscount == 0) {
2791                         atomic_subtract_rel_int(&pmc_ss_count, 1);
2792                         LIST_REMOVE(po, po_ssnext);
2793                         PMCDBG(PMC,OPS,2,"po=%p removed from global list", po);
2794                 }
2795         }
2796
2797         return (error);
2798 }
2799
2800
2801 #ifdef  DEBUG
2802 static const char *pmc_op_to_name[] = {
2803 #undef  __PMC_OP
2804 #define __PMC_OP(N, D)  #N ,
2805         __PMC_OPS()
2806         NULL
2807 };
2808 #endif
2809
2810 /*
2811  * The syscall interface
2812  */
2813
2814 #define PMC_GET_SX_XLOCK(...) do {              \
2815         sx_xlock(&pmc_sx);                      \
2816         if (pmc_hook == NULL) {                 \
2817                 sx_xunlock(&pmc_sx);            \
2818                 return __VA_ARGS__;             \
2819         }                                       \
2820 } while (0)
2821
2822 #define PMC_DOWNGRADE_SX() do {                 \
2823         sx_downgrade(&pmc_sx);                  \
2824         is_sx_downgraded = 1;                   \
2825 } while (0)
2826
2827 static int
2828 pmc_syscall_handler(struct thread *td, void *syscall_args)
2829 {
2830         int error, is_sx_downgraded, is_sx_locked, op;
2831         struct pmc_syscall_args *c;
2832         void *arg;
2833
2834         PMC_GET_SX_XLOCK(ENOSYS);
2835
2836         DROP_GIANT();
2837
2838         is_sx_downgraded = 0;
2839         is_sx_locked = 1;
2840
2841         c = (struct pmc_syscall_args *) syscall_args;
2842
2843         op = c->pmop_code;
2844         arg = c->pmop_data;
2845
2846         PMCDBG(MOD,PMS,1, "syscall op=%d \"%s\" arg=%p", op,
2847             pmc_op_to_name[op], arg);
2848
2849         error = 0;
2850         atomic_add_int(&pmc_stats.pm_syscalls, 1);
2851
2852         switch(op)
2853         {
2854
2855
2856         /*
2857          * Configure a log file.
2858          *
2859          * XXX This OP will be reworked.
2860          */
2861
2862         case PMC_OP_CONFIGURELOG:
2863         {
2864                 struct proc *p;
2865                 struct pmc *pm;
2866                 struct pmc_owner *po;
2867                 struct pmc_op_configurelog cl;
2868
2869                 sx_assert(&pmc_sx, SX_XLOCKED);
2870
2871                 if ((error = copyin(arg, &cl, sizeof(cl))) != 0)
2872                         break;
2873
2874                 /* mark this process as owning a log file */
2875                 p = td->td_proc;
2876                 if ((po = pmc_find_owner_descriptor(p)) == NULL)
2877                         if ((po = pmc_allocate_owner_descriptor(p)) == NULL) {
2878                                 error = ENOMEM;
2879                                 break;
2880                         }
2881
2882                 /*
2883                  * If a valid fd was passed in, try to configure that,
2884                  * otherwise if 'fd' was less than zero and there was
2885                  * a log file configured, flush its buffers and
2886                  * de-configure it.
2887                  */
2888                 if (cl.pm_logfd >= 0) {
2889                         sx_xunlock(&pmc_sx);
2890                         is_sx_locked = 0;
2891                         error = pmclog_configure_log(md, po, cl.pm_logfd);
2892                 } else if (po->po_flags & PMC_PO_OWNS_LOGFILE) {
2893                         pmclog_process_closelog(po);
2894                         error = pmclog_flush(po);
2895                         if (error == 0) {
2896                                 LIST_FOREACH(pm, &po->po_pmcs, pm_next)
2897                                     if (pm->pm_flags & PMC_F_NEEDS_LOGFILE &&
2898                                         pm->pm_state == PMC_STATE_RUNNING)
2899                                             pmc_stop(pm);
2900                                 error = pmclog_deconfigure_log(po);
2901                         }
2902                 } else
2903                         error = EINVAL;
2904
2905                 if (error)
2906                         break;
2907         }
2908         break;
2909
2910
2911         /*
2912          * Flush a log file.
2913          */
2914
2915         case PMC_OP_FLUSHLOG:
2916         {
2917                 struct pmc_owner *po;
2918
2919                 sx_assert(&pmc_sx, SX_XLOCKED);
2920
2921                 if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
2922                         error = EINVAL;
2923                         break;
2924                 }
2925
2926                 error = pmclog_flush(po);
2927         }
2928         break;
2929
2930         /*
2931          * Retrieve hardware configuration.
2932          */
2933
2934         case PMC_OP_GETCPUINFO: /* CPU information */
2935         {
2936                 struct pmc_op_getcpuinfo gci;
2937                 struct pmc_classinfo *pci;
2938                 struct pmc_classdep *pcd;
2939                 int cl;
2940
2941                 gci.pm_cputype = md->pmd_cputype;
2942                 gci.pm_ncpu    = pmc_cpu_max();
2943                 gci.pm_npmc    = md->pmd_npmc;
2944                 gci.pm_nclass  = md->pmd_nclass;
2945                 pci = gci.pm_classes;
2946                 pcd = md->pmd_classdep;
2947                 for (cl = 0; cl < md->pmd_nclass; cl++, pci++, pcd++) {
2948                         pci->pm_caps  = pcd->pcd_caps;
2949                         pci->pm_class = pcd->pcd_class;
2950                         pci->pm_width = pcd->pcd_width;
2951                         pci->pm_num   = pcd->pcd_num;
2952                 }
2953                 error = copyout(&gci, arg, sizeof(gci));
2954         }
2955         break;
2956
2957
2958         /*
2959          * Get module statistics
2960          */
2961
2962         case PMC_OP_GETDRIVERSTATS:
2963         {
2964                 struct pmc_op_getdriverstats gms;
2965
2966                 bcopy(&pmc_stats, &gms, sizeof(gms));
2967                 error = copyout(&gms, arg, sizeof(gms));
2968         }
2969         break;
2970
2971
2972         /*
2973          * Retrieve module version number
2974          */
2975
2976         case PMC_OP_GETMODULEVERSION:
2977         {
2978                 uint32_t cv, modv;
2979
2980                 /* retrieve the client's idea of the ABI version */
2981                 if ((error = copyin(arg, &cv, sizeof(uint32_t))) != 0)
2982                         break;
2983                 /* don't service clients newer than our driver */
2984                 modv = PMC_VERSION;
2985                 if ((cv & 0xFFFF0000) > (modv & 0xFFFF0000)) {
2986                         error = EPROGMISMATCH;
2987                         break;
2988                 }
2989                 error = copyout(&modv, arg, sizeof(int));
2990         }
2991         break;
2992
2993
2994         /*
2995          * Retrieve the state of all the PMCs on a given
2996          * CPU.
2997          */
2998
2999         case PMC_OP_GETPMCINFO:
3000         {
3001                 int ari;
3002                 struct pmc *pm;
3003                 size_t pmcinfo_size;
3004                 uint32_t cpu, n, npmc;
3005                 struct pmc_owner *po;
3006                 struct pmc_binding pb;
3007                 struct pmc_classdep *pcd;
3008                 struct pmc_info *p, *pmcinfo;
3009                 struct pmc_op_getpmcinfo *gpi;
3010
3011                 PMC_DOWNGRADE_SX();
3012
3013                 gpi = (struct pmc_op_getpmcinfo *) arg;
3014
3015                 if ((error = copyin(&gpi->pm_cpu, &cpu, sizeof(cpu))) != 0)
3016                         break;
3017
3018                 if (cpu >= pmc_cpu_max()) {
3019                         error = EINVAL;
3020                         break;
3021                 }
3022
3023                 if (!pmc_cpu_is_active(cpu)) {
3024                         error = ENXIO;
3025                         break;
3026                 }
3027
3028                 /* switch to CPU 'cpu' */
3029                 pmc_save_cpu_binding(&pb);
3030                 pmc_select_cpu(cpu);
3031
3032                 npmc = md->pmd_npmc;
3033
3034                 pmcinfo_size = npmc * sizeof(struct pmc_info);
3035                 pmcinfo = malloc(pmcinfo_size, M_PMC, M_WAITOK);
3036
3037                 p = pmcinfo;
3038
3039                 for (n = 0; n < md->pmd_npmc; n++, p++) {
3040
3041                         pcd = pmc_ri_to_classdep(md, n, &ari);
3042
3043                         KASSERT(pcd != NULL,
3044                             ("[pmc,%d] null pcd ri=%d", __LINE__, n));
3045
3046                         if ((error = pcd->pcd_describe(cpu, ari, p, &pm)) != 0)
3047                                 break;
3048
3049                         if (PMC_ROW_DISP_IS_STANDALONE(n))
3050                                 p->pm_rowdisp = PMC_DISP_STANDALONE;
3051                         else if (PMC_ROW_DISP_IS_THREAD(n))
3052                                 p->pm_rowdisp = PMC_DISP_THREAD;
3053                         else
3054                                 p->pm_rowdisp = PMC_DISP_FREE;
3055
3056                         p->pm_ownerpid = -1;
3057
3058                         if (pm == NULL) /* no PMC associated */
3059                                 continue;
3060
3061                         po = pm->pm_owner;
3062
3063                         KASSERT(po->po_owner != NULL,
3064                             ("[pmc,%d] pmc_owner had a null proc pointer",
3065                                 __LINE__));
3066
3067                         p->pm_ownerpid = po->po_owner->p_pid;
3068                         p->pm_mode     = PMC_TO_MODE(pm);
3069                         p->pm_event    = pm->pm_event;
3070                         p->pm_flags    = pm->pm_flags;
3071
3072                         if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
3073                                 p->pm_reloadcount =
3074                                     pm->pm_sc.pm_reloadcount;
3075                 }
3076
3077                 pmc_restore_cpu_binding(&pb);
3078
3079                 /* now copy out the PMC info collected */
3080                 if (error == 0)
3081                         error = copyout(pmcinfo, &gpi->pm_pmcs, pmcinfo_size);
3082
3083                 free(pmcinfo, M_PMC);
3084         }
3085         break;
3086
3087
3088         /*
3089          * Set the administrative state of a PMC.  I.e. whether
3090          * the PMC is to be used or not.
3091          */
3092
3093         case PMC_OP_PMCADMIN:
3094         {
3095                 int cpu, ri;
3096                 enum pmc_state request;
3097                 struct pmc_cpu *pc;
3098                 struct pmc_hw *phw;
3099                 struct pmc_op_pmcadmin pma;
3100                 struct pmc_binding pb;
3101
3102                 sx_assert(&pmc_sx, SX_XLOCKED);
3103
3104                 KASSERT(td == curthread,
3105                     ("[pmc,%d] td != curthread", __LINE__));
3106
3107                 error = priv_check(td, PRIV_PMC_MANAGE);
3108                 if (error)
3109                         break;
3110
3111                 if ((error = copyin(arg, &pma, sizeof(pma))) != 0)
3112                         break;
3113
3114                 cpu = pma.pm_cpu;
3115
3116                 if (cpu < 0 || cpu >= (int) pmc_cpu_max()) {
3117                         error = EINVAL;
3118                         break;
3119                 }
3120
3121                 if (!pmc_cpu_is_active(cpu)) {
3122                         error = ENXIO;
3123                         break;
3124                 }
3125
3126                 request = pma.pm_state;
3127
3128                 if (request != PMC_STATE_DISABLED &&
3129                     request != PMC_STATE_FREE) {
3130                         error = EINVAL;
3131                         break;
3132                 }
3133
3134                 ri = pma.pm_pmc; /* pmc id == row index */
3135                 if (ri < 0 || ri >= (int) md->pmd_npmc) {
3136                         error = EINVAL;
3137                         break;
3138                 }
3139
3140                 /*
3141                  * We can't disable a PMC with a row-index allocated
3142                  * for process virtual PMCs.
3143                  */
3144
3145                 if (PMC_ROW_DISP_IS_THREAD(ri) &&
3146                     request == PMC_STATE_DISABLED) {
3147                         error = EBUSY;
3148                         break;
3149                 }
3150
3151                 /*
3152                  * otherwise, this PMC on this CPU is either free or
3153                  * in system-wide mode.
3154                  */
3155
3156                 pmc_save_cpu_binding(&pb);
3157                 pmc_select_cpu(cpu);
3158
3159                 pc  = pmc_pcpu[cpu];
3160                 phw = pc->pc_hwpmcs[ri];
3161
3162                 /*
3163                  * XXX do we need some kind of 'forced' disable?
3164                  */
3165
3166                 if (phw->phw_pmc == NULL) {
3167                         if (request == PMC_STATE_DISABLED &&
3168                             (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED)) {
3169                                 phw->phw_state &= ~PMC_PHW_FLAG_IS_ENABLED;
3170                                 PMC_MARK_ROW_STANDALONE(ri);
3171                         } else if (request == PMC_STATE_FREE &&
3172                             (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0) {
3173                                 phw->phw_state |=  PMC_PHW_FLAG_IS_ENABLED;
3174                                 PMC_UNMARK_ROW_STANDALONE(ri);
3175                         }
3176                         /* other cases are a no-op */
3177                 } else
3178                         error = EBUSY;
3179
3180                 pmc_restore_cpu_binding(&pb);
3181         }
3182         break;
3183
3184
3185         /*
3186          * Allocate a PMC.
3187          */
3188
3189         case PMC_OP_PMCALLOCATE:
3190         {
3191                 int adjri, n;
3192                 u_int cpu;
3193                 uint32_t caps;
3194                 struct pmc *pmc;
3195                 enum pmc_mode mode;
3196                 struct pmc_hw *phw;
3197                 struct pmc_binding pb;
3198                 struct pmc_classdep *pcd;
3199                 struct pmc_op_pmcallocate pa;
3200
3201                 if ((error = copyin(arg, &pa, sizeof(pa))) != 0)
3202                         break;
3203
3204                 caps = pa.pm_caps;
3205                 mode = pa.pm_mode;
3206                 cpu  = pa.pm_cpu;
3207
3208                 if ((mode != PMC_MODE_SS  &&  mode != PMC_MODE_SC  &&
3209                      mode != PMC_MODE_TS  &&  mode != PMC_MODE_TC) ||
3210                     (cpu != (u_int) PMC_CPU_ANY && cpu >= pmc_cpu_max())) {
3211                         error = EINVAL;
3212                         break;
3213                 }
3214
3215                 /*
3216                  * Virtual PMCs should only ask for a default CPU.
3217                  * System mode PMCs need to specify a non-default CPU.
3218                  */
3219
3220                 if ((PMC_IS_VIRTUAL_MODE(mode) && cpu != (u_int) PMC_CPU_ANY) ||
3221                     (PMC_IS_SYSTEM_MODE(mode) && cpu == (u_int) PMC_CPU_ANY)) {
3222                         error = EINVAL;
3223                         break;
3224                 }
3225
3226                 /*
3227                  * Check that an inactive CPU is not being asked for.
3228                  */
3229
3230                 if (PMC_IS_SYSTEM_MODE(mode) && !pmc_cpu_is_active(cpu)) {
3231                         error = ENXIO;
3232                         break;
3233                 }
3234
3235                 /*
3236                  * Refuse an allocation for a system-wide PMC if this
3237                  * process has been jailed, or if this process lacks
3238                  * super-user credentials and the sysctl tunable
3239                  * 'security.bsd.unprivileged_syspmcs' is zero.
3240                  */
3241
3242                 if (PMC_IS_SYSTEM_MODE(mode)) {
3243                         if (jailed(curthread->td_ucred)) {
3244                                 error = EPERM;
3245                                 break;
3246                         }
3247                         if (!pmc_unprivileged_syspmcs) {
3248                                 error = priv_check(curthread,
3249                                     PRIV_PMC_SYSTEM);
3250                                 if (error)
3251                                         break;
3252                         }
3253                 }
3254
3255                 if (error)
3256                         break;
3257
3258                 /*
3259                  * Look for valid values for 'pm_flags'
3260                  */
3261
3262                 if ((pa.pm_flags & ~(PMC_F_DESCENDANTS | PMC_F_LOG_PROCCSW |
3263                     PMC_F_LOG_PROCEXIT | PMC_F_CALLCHAIN)) != 0) {
3264                         error = EINVAL;
3265                         break;
3266                 }
3267
3268                 /* process logging options are not allowed for system PMCs */
3269                 if (PMC_IS_SYSTEM_MODE(mode) && (pa.pm_flags &
3270                     (PMC_F_LOG_PROCCSW | PMC_F_LOG_PROCEXIT))) {
3271                         error = EINVAL;
3272                         break;
3273                 }
3274
3275                 /*
3276                  * All sampling mode PMCs need to be able to interrupt the
3277                  * CPU.
3278                  */
3279                 if (PMC_IS_SAMPLING_MODE(mode))
3280                         caps |= PMC_CAP_INTERRUPT;
3281
3282                 /* A valid class specifier should have been passed in. */
3283                 for (n = 0; n < md->pmd_nclass; n++)
3284                         if (md->pmd_classdep[n].pcd_class == pa.pm_class)
3285                                 break;
3286                 if (n == md->pmd_nclass) {
3287                         error = EINVAL;
3288                         break;
3289                 }
3290
3291                 /* The requested PMC capabilities should be feasible. */
3292                 if ((md->pmd_classdep[n].pcd_caps & caps) != caps) {
3293                         error = EOPNOTSUPP;
3294                         break;
3295                 }
3296
3297                 PMCDBG(PMC,ALL,2, "event=%d caps=0x%x mode=%d cpu=%d",
3298                     pa.pm_ev, caps, mode, cpu);
3299
3300                 pmc = pmc_allocate_pmc_descriptor();
3301                 pmc->pm_id    = PMC_ID_MAKE_ID(cpu,pa.pm_mode,pa.pm_class,
3302                     PMC_ID_INVALID);
3303                 pmc->pm_event = pa.pm_ev;
3304                 pmc->pm_state = PMC_STATE_FREE;
3305                 pmc->pm_caps  = caps;
3306                 pmc->pm_flags = pa.pm_flags;
3307
3308                 /* switch thread to CPU 'cpu' */
3309                 pmc_save_cpu_binding(&pb);
3310
3311 #define PMC_IS_SHAREABLE_PMC(cpu, n)                            \
3312         (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_state &           \
3313          PMC_PHW_FLAG_IS_SHAREABLE)
3314 #define PMC_IS_UNALLOCATED(cpu, n)                              \
3315         (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_pmc == NULL)
3316
3317                 if (PMC_IS_SYSTEM_MODE(mode)) {
3318                         pmc_select_cpu(cpu);
3319                         for (n = 0; n < (int) md->pmd_npmc; n++) {
3320                                 pcd = pmc_ri_to_classdep(md, n, &adjri);
3321                                 if (pmc_can_allocate_row(n, mode) == 0 &&
3322                                     pmc_can_allocate_rowindex(
3323                                             curthread->td_proc, n, cpu) == 0 &&
3324                                     (PMC_IS_UNALLOCATED(cpu, n) ||
3325                                      PMC_IS_SHAREABLE_PMC(cpu, n)) &&
3326                                     pcd->pcd_allocate_pmc(cpu, adjri, pmc,
3327                                         &pa) == 0)
3328                                         break;
3329                         }
3330                 } else {
3331                         /* Process virtual mode */
3332                         for (n = 0; n < (int) md->pmd_npmc; n++) {
3333                                 pcd = pmc_ri_to_classdep(md, n, &adjri);
3334                                 if (pmc_can_allocate_row(n, mode) == 0 &&
3335                                     pmc_can_allocate_rowindex(
3336                                             curthread->td_proc, n,
3337                                             PMC_CPU_ANY) == 0 &&
3338                                     pcd->pcd_allocate_pmc(curthread->td_oncpu,
3339                                         adjri, pmc, &pa) == 0)
3340                                         break;
3341                         }
3342                 }
3343
3344 #undef  PMC_IS_UNALLOCATED
3345 #undef  PMC_IS_SHAREABLE_PMC
3346
3347                 pmc_restore_cpu_binding(&pb);
3348
3349                 if (n == (int) md->pmd_npmc) {
3350                         pmc_destroy_pmc_descriptor(pmc);
3351                         free(pmc, M_PMC);
3352                         pmc = NULL;
3353                         error = EINVAL;
3354                         break;
3355                 }
3356
3357                 /* Fill in the correct value in the ID field */
3358                 pmc->pm_id = PMC_ID_MAKE_ID(cpu,mode,pa.pm_class,n);
3359
3360                 PMCDBG(PMC,ALL,2, "ev=%d class=%d mode=%d n=%d -> pmcid=%x",
3361                     pmc->pm_event, pa.pm_class, mode, n, pmc->pm_id);
3362
3363                 /* Process mode PMCs with logging enabled need log files */
3364                 if (pmc->pm_flags & (PMC_F_LOG_PROCEXIT | PMC_F_LOG_PROCCSW))
3365                         pmc->pm_flags |= PMC_F_NEEDS_LOGFILE;
3366
3367                 /* All system mode sampling PMCs require a log file */
3368                 if (PMC_IS_SAMPLING_MODE(mode) && PMC_IS_SYSTEM_MODE(mode))
3369                         pmc->pm_flags |= PMC_F_NEEDS_LOGFILE;
3370
3371                 /*
3372                  * Configure global pmc's immediately
3373                  */
3374
3375                 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pmc))) {
3376
3377                         pmc_save_cpu_binding(&pb);
3378                         pmc_select_cpu(cpu);
3379
3380                         phw = pmc_pcpu[cpu]->pc_hwpmcs[n];
3381                         pcd = pmc_ri_to_classdep(md, n, &adjri);
3382
3383                         if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0 ||
3384                             (error = pcd->pcd_config_pmc(cpu, adjri, pmc)) != 0) {
3385                                 (void) pcd->pcd_release_pmc(cpu, adjri, pmc);
3386                                 pmc_destroy_pmc_descriptor(pmc);
3387                                 free(pmc, M_PMC);
3388                                 pmc = NULL;
3389                                 pmc_restore_cpu_binding(&pb);
3390                                 error = EPERM;
3391                                 break;
3392                         }
3393
3394                         pmc_restore_cpu_binding(&pb);
3395                 }
3396
3397                 pmc->pm_state    = PMC_STATE_ALLOCATED;
3398
3399                 /*
3400                  * mark row disposition
3401                  */
3402
3403                 if (PMC_IS_SYSTEM_MODE(mode))
3404                         PMC_MARK_ROW_STANDALONE(n);
3405                 else
3406                         PMC_MARK_ROW_THREAD(n);
3407
3408                 /*
3409                  * Register this PMC with the current thread as its owner.
3410                  */
3411
3412                 if ((error =
3413                     pmc_register_owner(curthread->td_proc, pmc)) != 0) {
3414                         pmc_release_pmc_descriptor(pmc);
3415                         free(pmc, M_PMC);
3416                         pmc = NULL;
3417                         break;
3418                 }
3419
3420                 /*
3421                  * Return the allocated index.
3422                  */
3423
3424                 pa.pm_pmcid = pmc->pm_id;
3425
3426                 error = copyout(&pa, arg, sizeof(pa));
3427         }
3428         break;
3429
3430
3431         /*
3432          * Attach a PMC to a process.
3433          */
3434
3435         case PMC_OP_PMCATTACH:
3436         {
3437                 struct pmc *pm;
3438                 struct proc *p;
3439                 struct pmc_op_pmcattach a;
3440
3441                 sx_assert(&pmc_sx, SX_XLOCKED);
3442
3443                 if ((error = copyin(arg, &a, sizeof(a))) != 0)
3444                         break;
3445
3446                 if (a.pm_pid < 0) {
3447                         error = EINVAL;
3448                         break;
3449                 } else if (a.pm_pid == 0)
3450                         a.pm_pid = td->td_proc->p_pid;
3451
3452                 if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0)
3453                         break;
3454
3455                 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
3456                         error = EINVAL;
3457                         break;
3458                 }
3459
3460                 /* PMCs may be (re)attached only when allocated or stopped */
3461                 if (pm->pm_state == PMC_STATE_RUNNING) {
3462                         error = EBUSY;
3463                         break;
3464                 } else if (pm->pm_state != PMC_STATE_ALLOCATED &&
3465                     pm->pm_state != PMC_STATE_STOPPED) {
3466                         error = EINVAL;
3467                         break;
3468                 }
3469
3470                 /* lookup pid */
3471                 if ((p = pfind(a.pm_pid)) == NULL) {
3472                         error = ESRCH;
3473                         break;
3474                 }
3475
3476                 /*
3477                  * Ignore processes that are working on exiting.
3478                  */
3479                 if (p->p_flag & P_WEXIT) {
3480                         error = ESRCH;
3481                         PROC_UNLOCK(p); /* pfind() returns a locked process */
3482                         break;
3483                 }
3484
3485                 /*
3486                  * we are allowed to attach a PMC to a process if
3487                  * we can debug it.
3488                  */
3489                 error = p_candebug(curthread, p);
3490
3491                 PROC_UNLOCK(p);
3492
3493                 if (error == 0)
3494                         error = pmc_attach_process(p, pm);
3495         }
3496         break;
3497
3498
3499         /*
3500          * Detach an attached PMC from a process.
3501          */
3502
3503         case PMC_OP_PMCDETACH:
3504         {
3505                 struct pmc *pm;
3506                 struct proc *p;
3507                 struct pmc_op_pmcattach a;
3508
3509                 if ((error = copyin(arg, &a, sizeof(a))) != 0)
3510                         break;
3511
3512                 if (a.pm_pid < 0) {
3513                         error = EINVAL;
3514                         break;
3515                 } else if (a.pm_pid == 0)
3516                         a.pm_pid = td->td_proc->p_pid;
3517
3518                 if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0)
3519                         break;
3520
3521                 if ((p = pfind(a.pm_pid)) == NULL) {
3522                         error = ESRCH;
3523                         break;
3524                 }
3525
3526                 /*
3527                  * Treat processes that are in the process of exiting
3528                  * as if they were not present.
3529                  */
3530
3531                 if (p->p_flag & P_WEXIT)
3532                         error = ESRCH;
3533
3534                 PROC_UNLOCK(p); /* pfind() returns a locked process */
3535
3536                 if (error == 0)
3537                         error = pmc_detach_process(p, pm);
3538         }
3539         break;
3540
3541
3542         /*
3543          * Retrieve the MSR number associated with the counter
3544          * 'pmc_id'.  This allows processes to directly use RDPMC
3545          * instructions to read their PMCs, without the overhead of a
3546          * system call.
3547          */
3548
3549         case PMC_OP_PMCGETMSR:
3550         {
3551                 int adjri, ri;
3552                 struct pmc *pm;
3553                 struct pmc_target *pt;
3554                 struct pmc_op_getmsr gm;
3555                 struct pmc_classdep *pcd;
3556
3557                 PMC_DOWNGRADE_SX();
3558
3559                 if ((error = copyin(arg, &gm, sizeof(gm))) != 0)
3560                         break;
3561
3562                 if ((error = pmc_find_pmc(gm.pm_pmcid, &pm)) != 0)
3563                         break;
3564
3565                 /*
3566                  * The allocated PMC has to be a process virtual PMC,
3567                  * i.e., of type MODE_T[CS].  Global PMCs can only be
3568                  * read using the PMCREAD operation since they may be
3569                  * allocated on a different CPU than the one we could
3570                  * be running on at the time of the RDPMC instruction.
3571                  *
3572                  * The GETMSR operation is not allowed for PMCs that
3573                  * are inherited across processes.
3574                  */
3575
3576                 if (!PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) ||
3577                     (pm->pm_flags & PMC_F_DESCENDANTS)) {
3578                         error = EINVAL;
3579                         break;
3580                 }
3581
3582                 /*
3583                  * It only makes sense to use a RDPMC (or its
3584                  * equivalent instruction on non-x86 architectures) on
3585                  * a process that has allocated and attached a PMC to
3586                  * itself.  Conversely the PMC is only allowed to have
3587                  * one process attached to it -- its owner.
3588                  */
3589
3590                 if ((pt = LIST_FIRST(&pm->pm_targets)) == NULL ||
3591                     LIST_NEXT(pt, pt_next) != NULL ||
3592                     pt->pt_process->pp_proc != pm->pm_owner->po_owner) {
3593                         error = EINVAL;
3594                         break;
3595                 }
3596
3597                 ri = PMC_TO_ROWINDEX(pm);
3598                 pcd = pmc_ri_to_classdep(md, ri, &adjri);
3599
3600                 /* PMC class has no 'GETMSR' support */
3601                 if (pcd->pcd_get_msr == NULL) {
3602                         error = ENOSYS;
3603                         break;
3604                 }
3605
3606                 if ((error = (*pcd->pcd_get_msr)(adjri, &gm.pm_msr)) < 0)
3607                         break;
3608
3609                 if ((error = copyout(&gm, arg, sizeof(gm))) < 0)
3610                         break;
3611
3612                 /*
3613                  * Mark our process as using MSRs.  Update machine
3614                  * state using a forced context switch.
3615                  */
3616
3617                 pt->pt_process->pp_flags |= PMC_PP_ENABLE_MSR_ACCESS;
3618                 pmc_force_context_switch();
3619
3620         }
3621         break;
3622
3623         /*
3624          * Release an allocated PMC
3625          */
3626
3627         case PMC_OP_PMCRELEASE:
3628         {
3629                 pmc_id_t pmcid;
3630                 struct pmc *pm;
3631                 struct pmc_owner *po;
3632                 struct pmc_op_simple sp;
3633
3634                 /*
3635                  * Find PMC pointer for the named PMC.
3636                  *
3637                  * Use pmc_release_pmc_descriptor() to switch off the
3638                  * PMC, remove all its target threads, and remove the
3639                  * PMC from its owner's list.
3640                  *
3641                  * Remove the owner record if this is the last PMC
3642                  * owned.
3643                  *
3644                  * Free up space.
3645                  */
3646
3647                 if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
3648                         break;
3649
3650                 pmcid = sp.pm_pmcid;
3651
3652                 if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
3653                         break;
3654
3655                 po = pm->pm_owner;
3656                 pmc_release_pmc_descriptor(pm);
3657                 pmc_maybe_remove_owner(po);
3658
3659                 free(pm, M_PMC);
3660         }
3661         break;
3662
3663
3664         /*
3665          * Read and/or write a PMC.
3666          */
3667
3668         case PMC_OP_PMCRW:
3669         {
3670                 int adjri;
3671                 struct pmc *pm;
3672                 uint32_t cpu, ri;
3673                 pmc_value_t oldvalue;
3674                 struct pmc_binding pb;
3675                 struct pmc_op_pmcrw prw;
3676                 struct pmc_classdep *pcd;
3677                 struct pmc_op_pmcrw *pprw;
3678
3679                 PMC_DOWNGRADE_SX();
3680
3681                 if ((error = copyin(arg, &prw, sizeof(prw))) != 0)
3682                         break;
3683
3684                 ri = 0;
3685                 PMCDBG(PMC,OPS,1, "rw id=%d flags=0x%x", prw.pm_pmcid,
3686                     prw.pm_flags);
3687
3688                 /* must have at least one flag set */
3689                 if ((prw.pm_flags & (PMC_F_OLDVALUE|PMC_F_NEWVALUE)) == 0) {
3690                         error = EINVAL;
3691                         break;
3692                 }
3693
3694                 /* locate pmc descriptor */
3695                 if ((error = pmc_find_pmc(prw.pm_pmcid, &pm)) != 0)
3696                         break;
3697
3698                 /* Can't read a PMC that hasn't been started. */
3699                 if (pm->pm_state != PMC_STATE_ALLOCATED &&
3700                     pm->pm_state != PMC_STATE_STOPPED &&
3701                     pm->pm_state != PMC_STATE_RUNNING) {
3702                         error = EINVAL;
3703                         break;
3704                 }
3705
3706                 /* writing a new value is allowed only for 'STOPPED' pmcs */
3707                 if (pm->pm_state == PMC_STATE_RUNNING &&
3708                     (prw.pm_flags & PMC_F_NEWVALUE)) {
3709                         error = EBUSY;
3710                         break;
3711                 }
3712
3713                 if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) {
3714
3715                         /*
3716                          * If this PMC is attached to its owner (i.e.,
3717                          * the process requesting this operation) and
3718                          * is running, then attempt to get an
3719                          * upto-date reading from hardware for a READ.
3720                          * Writes are only allowed when the PMC is
3721                          * stopped, so only update the saved value
3722                          * field.
3723                          *
3724                          * If the PMC is not running, or is not
3725                          * attached to its owner, read/write to the
3726                          * savedvalue field.
3727                          */
3728
3729                         ri = PMC_TO_ROWINDEX(pm);
3730                         pcd = pmc_ri_to_classdep(md, ri, &adjri);
3731
3732                         mtx_pool_lock_spin(pmc_mtxpool, pm);
3733                         cpu = curthread->td_oncpu;
3734
3735                         if (prw.pm_flags & PMC_F_OLDVALUE) {
3736                                 if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) &&
3737                                     (pm->pm_state == PMC_STATE_RUNNING))
3738                                         error = (*pcd->pcd_read_pmc)(cpu, adjri,
3739                                             &oldvalue);
3740                                 else
3741                                         oldvalue = pm->pm_gv.pm_savedvalue;
3742                         }
3743                         if (prw.pm_flags & PMC_F_NEWVALUE)
3744                                 pm->pm_gv.pm_savedvalue = prw.pm_value;
3745
3746                         mtx_pool_unlock_spin(pmc_mtxpool, pm);
3747
3748                 } else { /* System mode PMCs */
3749                         cpu = PMC_TO_CPU(pm);
3750                         ri  = PMC_TO_ROWINDEX(pm);
3751                         pcd = pmc_ri_to_classdep(md, ri, &adjri);
3752
3753                         if (!pmc_cpu_is_active(cpu)) {
3754                                 error = ENXIO;
3755                                 break;
3756                         }
3757
3758                         /* move this thread to CPU 'cpu' */
3759                         pmc_save_cpu_binding(&pb);
3760                         pmc_select_cpu(cpu);
3761
3762                         critical_enter();
3763                         /* save old value */
3764                         if (prw.pm_flags & PMC_F_OLDVALUE)
3765                                 if ((error = (*pcd->pcd_read_pmc)(cpu, adjri,
3766                                          &oldvalue)))
3767                                         goto error;
3768                         /* write out new value */
3769                         if (prw.pm_flags & PMC_F_NEWVALUE)
3770                                 error = (*pcd->pcd_write_pmc)(cpu, adjri,
3771                                     prw.pm_value);
3772                 error:
3773                         critical_exit();
3774                         pmc_restore_cpu_binding(&pb);
3775                         if (error)
3776                                 break;
3777                 }
3778
3779                 pprw = (struct pmc_op_pmcrw *) arg;
3780
3781 #ifdef  DEBUG
3782                 if (prw.pm_flags & PMC_F_NEWVALUE)
3783                         PMCDBG(PMC,OPS,2, "rw id=%d new %jx -> old %jx",
3784                             ri, prw.pm_value, oldvalue);
3785                 else if (prw.pm_flags & PMC_F_OLDVALUE)
3786                         PMCDBG(PMC,OPS,2, "rw id=%d -> old %jx", ri, oldvalue);
3787 #endif
3788
3789                 /* return old value if requested */
3790                 if (prw.pm_flags & PMC_F_OLDVALUE)
3791                         if ((error = copyout(&oldvalue, &pprw->pm_value,
3792                                  sizeof(prw.pm_value))))
3793                                 break;
3794
3795         }
3796         break;
3797
3798
3799         /*
3800          * Set the sampling rate for a sampling mode PMC and the
3801          * initial count for a counting mode PMC.
3802          */
3803
3804         case PMC_OP_PMCSETCOUNT:
3805         {
3806                 struct pmc *pm;
3807                 struct pmc_op_pmcsetcount sc;
3808
3809                 PMC_DOWNGRADE_SX();
3810
3811                 if ((error = copyin(arg, &sc, sizeof(sc))) != 0)
3812                         break;
3813
3814                 if ((error = pmc_find_pmc(sc.pm_pmcid, &pm)) != 0)
3815                         break;
3816
3817                 if (pm->pm_state == PMC_STATE_RUNNING) {
3818                         error = EBUSY;
3819                         break;
3820                 }
3821
3822                 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
3823                         pm->pm_sc.pm_reloadcount = sc.pm_count;
3824                 else
3825                         pm->pm_sc.pm_initial = sc.pm_count;
3826         }
3827         break;
3828
3829
3830         /*
3831          * Start a PMC.
3832          */
3833
3834         case PMC_OP_PMCSTART:
3835         {
3836                 pmc_id_t pmcid;
3837                 struct pmc *pm;
3838                 struct pmc_op_simple sp;
3839
3840                 sx_assert(&pmc_sx, SX_XLOCKED);
3841
3842                 if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
3843                         break;
3844
3845                 pmcid = sp.pm_pmcid;
3846
3847                 if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
3848                         break;
3849
3850                 KASSERT(pmcid == pm->pm_id,
3851                     ("[pmc,%d] pmcid %x != id %x", __LINE__,
3852                         pm->pm_id, pmcid));
3853
3854                 if (pm->pm_state == PMC_STATE_RUNNING) /* already running */
3855                         break;
3856                 else if (pm->pm_state != PMC_STATE_STOPPED &&
3857                     pm->pm_state != PMC_STATE_ALLOCATED) {
3858                         error = EINVAL;
3859                         break;
3860                 }
3861
3862                 error = pmc_start(pm);
3863         }
3864         break;
3865
3866
3867         /*
3868          * Stop a PMC.
3869          */
3870
3871         case PMC_OP_PMCSTOP:
3872         {
3873                 pmc_id_t pmcid;
3874                 struct pmc *pm;
3875                 struct pmc_op_simple sp;
3876
3877                 PMC_DOWNGRADE_SX();
3878
3879                 if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
3880                         break;
3881
3882                 pmcid = sp.pm_pmcid;
3883
3884                 /*
3885                  * Mark the PMC as inactive and invoke the MD stop
3886                  * routines if needed.
3887                  */
3888
3889                 if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
3890                         break;
3891
3892                 KASSERT(pmcid == pm->pm_id,
3893                     ("[pmc,%d] pmc id %x != pmcid %x", __LINE__,
3894                         pm->pm_id, pmcid));
3895
3896                 if (pm->pm_state == PMC_STATE_STOPPED) /* already stopped */
3897                         break;
3898                 else if (pm->pm_state != PMC_STATE_RUNNING) {
3899                         error = EINVAL;
3900                         break;
3901                 }
3902
3903                 error = pmc_stop(pm);
3904         }
3905         break;
3906
3907
3908         /*
3909          * Write a user supplied value to the log file.
3910          */
3911
3912         case PMC_OP_WRITELOG:
3913         {
3914                 struct pmc_op_writelog wl;
3915                 struct pmc_owner *po;
3916
3917                 PMC_DOWNGRADE_SX();
3918
3919                 if ((error = copyin(arg, &wl, sizeof(wl))) != 0)
3920                         break;
3921
3922                 if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
3923                         error = EINVAL;
3924                         break;
3925                 }
3926
3927                 if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) {
3928                         error = EINVAL;
3929                         break;
3930                 }
3931
3932                 error = pmclog_process_userlog(po, &wl);
3933         }
3934         break;
3935
3936
3937         default:
3938                 error = EINVAL;
3939                 break;
3940         }
3941
3942         if (is_sx_locked != 0) {
3943                 if (is_sx_downgraded)
3944                         sx_sunlock(&pmc_sx);
3945                 else
3946                         sx_xunlock(&pmc_sx);
3947         }
3948
3949         if (error)
3950                 atomic_add_int(&pmc_stats.pm_syscall_errors, 1);
3951
3952         PICKUP_GIANT();
3953
3954         return error;
3955 }
3956
3957 /*
3958  * Helper functions
3959  */
3960
3961
3962 /*
3963  * Mark the thread as needing callchain capture and post an AST.  The
3964  * actual callchain capture will be done in a context where it is safe
3965  * to take page faults.
3966  */
3967
3968 static void
3969 pmc_post_callchain_callback(void)
3970 {
3971         struct thread *td;
3972
3973         td = curthread;
3974
3975         /*
3976          * If there is multiple PMCs for the same interrupt ignore new post
3977          */
3978         if (td->td_pflags & TDP_CALLCHAIN)
3979                 return;
3980
3981         /*
3982          * Mark this thread as needing callchain capture.
3983          * `td->td_pflags' will be safe to touch because this thread
3984          * was in user space when it was interrupted.
3985          */
3986         td->td_pflags |= TDP_CALLCHAIN;
3987
3988         /*
3989          * Don't let this thread migrate between CPUs until callchain
3990          * capture completes.
3991          */
3992         sched_pin();
3993
3994         return;
3995 }
3996
3997 /*
3998  * Interrupt processing.
3999  *
4000  * Find a free slot in the per-cpu array of samples and capture the
4001  * current callchain there.  If a sample was successfully added, a bit
4002  * is set in mask 'pmc_cpumask' denoting that the DO_SAMPLES hook
4003  * needs to be invoked from the clock handler.
4004  *
4005  * This function is meant to be called from an NMI handler.  It cannot
4006  * use any of the locking primitives supplied by the OS.
4007  */
4008
4009 int
4010 pmc_process_interrupt(int cpu, struct pmc *pm, struct trapframe *tf,
4011     int inuserspace)
4012 {
4013         int error, callchaindepth;
4014         struct thread *td;
4015         struct pmc_sample *ps;
4016         struct pmc_samplebuffer *psb;
4017
4018         error = 0;
4019
4020         /*
4021          * Allocate space for a sample buffer.
4022          */
4023         psb = pmc_pcpu[cpu]->pc_sb;
4024
4025         ps = psb->ps_write;
4026         if (ps->ps_nsamples) {  /* in use, reader hasn't caught up */
4027                 pm->pm_stalled = 1;
4028                 atomic_add_int(&pmc_stats.pm_intr_bufferfull, 1);
4029                 PMCDBG(SAM,INT,1,"(spc) cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d",
4030                     cpu, pm, (void *) tf, inuserspace,
4031                     (int) (psb->ps_write - psb->ps_samples),
4032                     (int) (psb->ps_read - psb->ps_samples));
4033                 error = ENOMEM;
4034                 goto done;
4035         }
4036
4037
4038         /* Fill in entry. */
4039         PMCDBG(SAM,INT,1,"cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d", cpu, pm,
4040             (void *) tf, inuserspace,
4041             (int) (psb->ps_write - psb->ps_samples),
4042             (int) (psb->ps_read - psb->ps_samples));
4043
4044         KASSERT(pm->pm_runcount >= 0,
4045             ("[pmc,%d] pm=%p runcount %d", __LINE__, (void *) pm,
4046                 pm->pm_runcount));
4047
4048         atomic_add_rel_32(&pm->pm_runcount, 1); /* hold onto PMC */
4049         ps->ps_pmc = pm;
4050         if ((td = curthread) && td->td_proc)
4051                 ps->ps_pid = td->td_proc->p_pid;
4052         else
4053                 ps->ps_pid = -1;
4054         ps->ps_cpu = cpu;
4055         ps->ps_td = td;
4056         ps->ps_flags = inuserspace ? PMC_CC_F_USERSPACE : 0;
4057
4058         callchaindepth = (pm->pm_flags & PMC_F_CALLCHAIN) ?
4059             pmc_callchaindepth : 1;
4060
4061         if (callchaindepth == 1)
4062                 ps->ps_pc[0] = PMC_TRAPFRAME_TO_PC(tf);
4063         else {
4064                 /*
4065                  * Kernel stack traversals can be done immediately,
4066                  * while we defer to an AST for user space traversals.
4067                  */
4068                 if (!inuserspace)
4069                         callchaindepth =
4070                             pmc_save_kernel_callchain(ps->ps_pc,
4071                                 callchaindepth, tf);
4072                 else {
4073                         pmc_post_callchain_callback();
4074                         callchaindepth = PMC_SAMPLE_INUSE;
4075                 }
4076         }
4077
4078         ps->ps_nsamples = callchaindepth;       /* mark entry as in use */
4079
4080         /* increment write pointer, modulo ring buffer size */
4081         ps++;
4082         if (ps == psb->ps_fence)
4083                 psb->ps_write = psb->ps_samples;
4084         else
4085                 psb->ps_write = ps;
4086
4087  done:
4088         /* mark CPU as needing processing */
4089         atomic_set_rel_int(&pmc_cpumask, (1 << cpu));
4090
4091         return (error);
4092 }
4093
4094 /*
4095  * Capture a user call chain.  This function will be called from ast()
4096  * before control returns to userland and before the process gets
4097  * rescheduled.
4098  */
4099
4100 static void
4101 pmc_capture_user_callchain(int cpu, struct trapframe *tf)
4102 {
4103         int i;
4104         struct pmc *pm;
4105         struct thread *td;
4106         struct pmc_sample *ps;
4107         struct pmc_samplebuffer *psb;
4108 #ifdef  INVARIANTS
4109         int ncallchains;
4110 #endif
4111
4112         sched_unpin();  /* Can migrate safely now. */
4113
4114         psb = pmc_pcpu[cpu]->pc_sb;
4115         td = curthread;
4116
4117         KASSERT(td->td_pflags & TDP_CALLCHAIN,
4118             ("[pmc,%d] Retrieving callchain for thread that doesn't want it",
4119                 __LINE__));
4120
4121 #ifdef  INVARIANTS
4122         ncallchains = 0;
4123 #endif
4124
4125         /*
4126          * Iterate through all deferred callchain requests.
4127          */
4128
4129         ps = psb->ps_samples;
4130         for (i = 0; i < pmc_nsamples; i++, ps++) {
4131
4132                 if (ps->ps_nsamples != PMC_SAMPLE_INUSE)
4133                         continue;
4134                 if (ps->ps_td != td)
4135                         continue;
4136
4137                 KASSERT(ps->ps_cpu == cpu,
4138                     ("[pmc,%d] cpu mismatch ps_cpu=%d pcpu=%d", __LINE__,
4139                         ps->ps_cpu, PCPU_GET(cpuid)));
4140
4141                 pm = ps->ps_pmc;
4142
4143                 KASSERT(pm->pm_flags & PMC_F_CALLCHAIN,
4144                     ("[pmc,%d] Retrieving callchain for PMC that doesn't "
4145                         "want it", __LINE__));
4146
4147                 KASSERT(pm->pm_runcount > 0,
4148                     ("[pmc,%d] runcount %d", __LINE__, pm->pm_runcount));
4149
4150                 /*
4151                  * Retrieve the callchain and mark the sample buffer
4152                  * as 'processable' by the timer tick sweep code.
4153                  */
4154                 ps->ps_nsamples = pmc_save_user_callchain(ps->ps_pc,
4155                     pmc_callchaindepth, tf);
4156
4157 #ifdef  INVARIANTS
4158                 ncallchains++;
4159 #endif
4160
4161         }
4162
4163         KASSERT(ncallchains > 0,
4164             ("[pmc,%d] cpu %d didn't find a sample to collect", __LINE__,
4165                 cpu));
4166
4167         return;
4168 }
4169
4170
4171 /*
4172  * Process saved PC samples.
4173  */
4174
4175 static void
4176 pmc_process_samples(int cpu)
4177 {
4178         struct pmc *pm;
4179         int adjri, n;
4180         struct thread *td;
4181         struct pmc_owner *po;
4182         struct pmc_sample *ps;
4183         struct pmc_classdep *pcd;
4184         struct pmc_samplebuffer *psb;
4185
4186         KASSERT(PCPU_GET(cpuid) == cpu,
4187             ("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__,
4188                 PCPU_GET(cpuid), cpu));
4189
4190         psb = pmc_pcpu[cpu]->pc_sb;
4191
4192         for (n = 0; n < pmc_nsamples; n++) { /* bound on #iterations */
4193
4194                 ps = psb->ps_read;
4195                 if (ps->ps_nsamples == PMC_SAMPLE_FREE)
4196                         break;
4197                 if (ps->ps_nsamples == PMC_SAMPLE_INUSE) {
4198                         /* Need a rescan at a later time. */
4199                         atomic_set_rel_int(&pmc_cpumask, (1 << cpu));
4200                         break;
4201                 }
4202
4203                 pm = ps->ps_pmc;
4204
4205                 KASSERT(pm->pm_runcount > 0,
4206                     ("[pmc,%d] pm=%p runcount %d", __LINE__, (void *) pm,
4207                         pm->pm_runcount));
4208
4209                 po = pm->pm_owner;
4210
4211                 KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
4212                     ("[pmc,%d] pmc=%p non-sampling mode=%d", __LINE__,
4213                         pm, PMC_TO_MODE(pm)));
4214
4215                 /* Ignore PMCs that have been switched off */
4216                 if (pm->pm_state != PMC_STATE_RUNNING)
4217                         goto entrydone;
4218
4219                 PMCDBG(SAM,OPS,1,"cpu=%d pm=%p n=%d fl=%x wr=%d rd=%d", cpu,
4220                     pm, ps->ps_nsamples, ps->ps_flags,
4221                     (int) (psb->ps_write - psb->ps_samples),
4222                     (int) (psb->ps_read - psb->ps_samples));
4223
4224                 /*
4225                  * If this is a process-mode PMC that is attached to
4226                  * its owner, and if the PC is in user mode, update
4227                  * profiling statistics like timer-based profiling
4228                  * would have done.
4229                  */
4230                 if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) {
4231                         if (ps->ps_flags & PMC_CC_F_USERSPACE) {
4232                                 td = FIRST_THREAD_IN_PROC(po->po_owner);
4233                                 addupc_intr(td, ps->ps_pc[0], 1);
4234                         }
4235                         goto entrydone;
4236                 }
4237
4238                 /*
4239                  * Otherwise, this is either a sampling mode PMC that
4240                  * is attached to a different process than its owner,
4241                  * or a system-wide sampling PMC.  Dispatch a log
4242                  * entry to the PMC's owner process.
4243                  */
4244
4245                 pmclog_process_callchain(pm, ps);
4246
4247         entrydone:
4248                 ps->ps_nsamples = 0;    /* mark entry as free */
4249                 atomic_subtract_rel_32(&pm->pm_runcount, 1);
4250
4251                 /* increment read pointer, modulo sample size */
4252                 if (++ps == psb->ps_fence)
4253                         psb->ps_read = psb->ps_samples;
4254                 else
4255                         psb->ps_read = ps;
4256         }
4257
4258         atomic_add_int(&pmc_stats.pm_log_sweeps, 1);
4259
4260         /* Do not re-enable stalled PMCs if we failed to process any samples */
4261         if (n == 0)
4262                 return;
4263
4264         /*
4265          * Restart any stalled sampling PMCs on this CPU.
4266          *
4267          * If the NMI handler sets the pm_stalled field of a PMC after
4268          * the check below, we'll end up processing the stalled PMC at
4269          * the next hardclock tick.
4270          */
4271         for (n = 0; n < md->pmd_npmc; n++) {
4272                 pcd = pmc_ri_to_classdep(md, n, &adjri);
4273                 KASSERT(pcd != NULL,
4274                     ("[pmc,%d] null pcd ri=%d", __LINE__, n));
4275                 (void) (*pcd->pcd_get_config)(cpu,adjri,&pm);
4276
4277                 if (pm == NULL ||                        /* !cfg'ed */
4278                     pm->pm_state != PMC_STATE_RUNNING || /* !active */
4279                     !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || /* !sampling */
4280                     pm->pm_stalled == 0) /* !stalled */
4281                         continue;
4282
4283                 pm->pm_stalled = 0;
4284                 (*pcd->pcd_start_pmc)(cpu, adjri);
4285         }
4286 }
4287
4288 /*
4289  * Event handlers.
4290  */
4291
4292 /*
4293  * Handle a process exit.
4294  *
4295  * Remove this process from all hash tables.  If this process
4296  * owned any PMCs, turn off those PMCs and deallocate them,
4297  * removing any associations with target processes.
4298  *
4299  * This function will be called by the last 'thread' of a
4300  * process.
4301  *
4302  * XXX This eventhandler gets called early in the exit process.
4303  * Consider using a 'hook' invocation from thread_exit() or equivalent
4304  * spot.  Another negative is that kse_exit doesn't seem to call
4305  * exit1() [??].
4306  *
4307  */
4308
4309 static void
4310 pmc_process_exit(void *arg __unused, struct proc *p)
4311 {
4312         struct pmc *pm;
4313         int adjri, cpu;
4314         unsigned int ri;
4315         int is_using_hwpmcs;
4316         struct pmc_owner *po;
4317         struct pmc_process *pp;
4318         struct pmc_classdep *pcd;
4319         pmc_value_t newvalue, tmp;
4320
4321         PROC_LOCK(p);
4322         is_using_hwpmcs = p->p_flag & P_HWPMC;
4323         PROC_UNLOCK(p);
4324
4325         /*
4326          * Log a sysexit event to all SS PMC owners.
4327          */
4328         LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
4329             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
4330                     pmclog_process_sysexit(po, p->p_pid);
4331
4332         if (!is_using_hwpmcs)
4333                 return;
4334
4335         PMC_GET_SX_XLOCK();
4336         PMCDBG(PRC,EXT,1,"process-exit proc=%p (%d, %s)", p, p->p_pid,
4337             p->p_comm);
4338
4339         /*
4340          * Since this code is invoked by the last thread in an exiting
4341          * process, we would have context switched IN at some prior
4342          * point.  However, with PREEMPTION, kernel mode context
4343          * switches may happen any time, so we want to disable a
4344          * context switch OUT till we get any PMCs targetting this
4345          * process off the hardware.
4346          *
4347          * We also need to atomically remove this process'
4348          * entry from our target process hash table, using
4349          * PMC_FLAG_REMOVE.
4350          */
4351         PMCDBG(PRC,EXT,1, "process-exit proc=%p (%d, %s)", p, p->p_pid,
4352             p->p_comm);
4353
4354         critical_enter(); /* no preemption */
4355
4356         cpu = curthread->td_oncpu;
4357
4358         if ((pp = pmc_find_process_descriptor(p,
4359                  PMC_FLAG_REMOVE)) != NULL) {
4360
4361                 PMCDBG(PRC,EXT,2,
4362                     "process-exit proc=%p pmc-process=%p", p, pp);
4363
4364                 /*
4365                  * The exiting process could the target of
4366                  * some PMCs which will be running on
4367                  * currently executing CPU.
4368                  *
4369                  * We need to turn these PMCs off like we
4370                  * would do at context switch OUT time.
4371                  */
4372                 for (ri = 0; ri < md->pmd_npmc; ri++) {
4373
4374                         /*
4375                          * Pick up the pmc pointer from hardware
4376                          * state similar to the CSW_OUT code.
4377                          */
4378                         pm = NULL;
4379
4380                         pcd = pmc_ri_to_classdep(md, ri, &adjri);
4381
4382                         (void) (*pcd->pcd_get_config)(cpu, adjri, &pm);
4383
4384                         PMCDBG(PRC,EXT,2, "ri=%d pm=%p", ri, pm);
4385
4386                         if (pm == NULL ||
4387                             !PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
4388                                 continue;
4389
4390                         PMCDBG(PRC,EXT,2, "ppmcs[%d]=%p pm=%p "
4391                             "state=%d", ri, pp->pp_pmcs[ri].pp_pmc,
4392                             pm, pm->pm_state);
4393
4394                         KASSERT(PMC_TO_ROWINDEX(pm) == ri,
4395                             ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
4396                                 __LINE__, PMC_TO_ROWINDEX(pm), ri));
4397
4398                         KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
4399                             ("[pmc,%d] pm %p != pp_pmcs[%d] %p",
4400                                 __LINE__, pm, ri, pp->pp_pmcs[ri].pp_pmc));
4401
4402                         (void) pcd->pcd_stop_pmc(cpu, adjri);
4403
4404                         KASSERT(pm->pm_runcount > 0,
4405                             ("[pmc,%d] bad runcount ri %d rc %d",
4406                                 __LINE__, ri, pm->pm_runcount));
4407
4408                         /* Stop hardware only if it is actually running */
4409                         if (pm->pm_state == PMC_STATE_RUNNING &&
4410                             pm->pm_stalled == 0) {
4411                                 pcd->pcd_read_pmc(cpu, adjri, &newvalue);
4412                                 tmp = newvalue -
4413                                     PMC_PCPU_SAVED(cpu,ri);
4414
4415                                 mtx_pool_lock_spin(pmc_mtxpool, pm);
4416                                 pm->pm_gv.pm_savedvalue += tmp;
4417                                 pp->pp_pmcs[ri].pp_pmcval += tmp;
4418                                 mtx_pool_unlock_spin(pmc_mtxpool, pm);
4419                         }
4420
4421                         atomic_subtract_rel_32(&pm->pm_runcount,1);
4422
4423                         KASSERT((int) pm->pm_runcount >= 0,
4424                             ("[pmc,%d] runcount is %d", __LINE__, ri));
4425
4426                         (void) pcd->pcd_config_pmc(cpu, adjri, NULL);
4427                 }
4428
4429                 /*
4430                  * Inform the MD layer of this pseudo "context switch
4431                  * out"
4432                  */
4433                 (void) md->pmd_switch_out(pmc_pcpu[cpu], pp);
4434
4435                 critical_exit(); /* ok to be pre-empted now */
4436
4437                 /*
4438                  * Unlink this process from the PMCs that are
4439                  * targetting it.  This will send a signal to
4440                  * all PMC owner's whose PMCs are orphaned.
4441                  *
4442                  * Log PMC value at exit time if requested.
4443                  */
4444                 for (ri = 0; ri < md->pmd_npmc; ri++)
4445                         if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) {
4446                                 if (pm->pm_flags & PMC_F_NEEDS_LOGFILE &&
4447                                     PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm)))
4448                                         pmclog_process_procexit(pm, pp);
4449                                 pmc_unlink_target_process(pm, pp);
4450                         }
4451                 free(pp, M_PMC);
4452
4453         } else
4454                 critical_exit(); /* pp == NULL */
4455
4456
4457         /*
4458          * If the process owned PMCs, free them up and free up
4459          * memory.
4460          */
4461         if ((po = pmc_find_owner_descriptor(p)) != NULL) {
4462                 pmc_remove_owner(po);
4463                 pmc_destroy_owner_descriptor(po);
4464         }
4465
4466         sx_xunlock(&pmc_sx);
4467 }
4468
4469 /*
4470  * Handle a process fork.
4471  *
4472  * If the parent process 'p1' is under HWPMC monitoring, then copy
4473  * over any attached PMCs that have 'do_descendants' semantics.
4474  */
4475
4476 static void
4477 pmc_process_fork(void *arg __unused, struct proc *p1, struct proc *newproc,
4478     int flags)
4479 {
4480         int is_using_hwpmcs;
4481         unsigned int ri;
4482         uint32_t do_descendants;
4483         struct pmc *pm;
4484         struct pmc_owner *po;
4485         struct pmc_process *ppnew, *ppold;
4486
4487         (void) flags;           /* unused parameter */
4488
4489         PROC_LOCK(p1);
4490         is_using_hwpmcs = p1->p_flag & P_HWPMC;
4491         PROC_UNLOCK(p1);
4492
4493         /*
4494          * If there are system-wide sampling PMCs active, we need to
4495          * log all fork events to their owner's logs.
4496          */
4497
4498         LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
4499             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
4500                     pmclog_process_procfork(po, p1->p_pid, newproc->p_pid);
4501
4502         if (!is_using_hwpmcs)
4503                 return;
4504
4505         PMC_GET_SX_XLOCK();
4506         PMCDBG(PMC,FRK,1, "process-fork proc=%p (%d, %s) -> %p", p1,
4507             p1->p_pid, p1->p_comm, newproc);
4508
4509         /*
4510          * If the parent process (curthread->td_proc) is a
4511          * target of any PMCs, look for PMCs that are to be
4512          * inherited, and link these into the new process
4513          * descriptor.
4514          */
4515         if ((ppold = pmc_find_process_descriptor(curthread->td_proc,
4516                  PMC_FLAG_NONE)) == NULL)
4517                 goto done;              /* nothing to do */
4518
4519         do_descendants = 0;
4520         for (ri = 0; ri < md->pmd_npmc; ri++)
4521                 if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL)
4522                         do_descendants |= pm->pm_flags & PMC_F_DESCENDANTS;
4523         if (do_descendants == 0) /* nothing to do */
4524                 goto done;
4525
4526         /* allocate a descriptor for the new process  */
4527         if ((ppnew = pmc_find_process_descriptor(newproc,
4528                  PMC_FLAG_ALLOCATE)) == NULL)
4529                 goto done;
4530
4531         /*
4532          * Run through all PMCs that were targeting the old process
4533          * and which specified F_DESCENDANTS and attach them to the
4534          * new process.
4535          *
4536          * Log the fork event to all owners of PMCs attached to this
4537          * process, if not already logged.
4538          */
4539         for (ri = 0; ri < md->pmd_npmc; ri++)
4540                 if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL &&
4541                     (pm->pm_flags & PMC_F_DESCENDANTS)) {
4542                         pmc_link_target_process(pm, ppnew);
4543                         po = pm->pm_owner;
4544                         if (po->po_sscount == 0 &&
4545                             po->po_flags & PMC_PO_OWNS_LOGFILE)
4546                                 pmclog_process_procfork(po, p1->p_pid,
4547                                     newproc->p_pid);
4548                 }
4549
4550         /*
4551          * Now mark the new process as being tracked by this driver.
4552          */
4553         PROC_LOCK(newproc);
4554         newproc->p_flag |= P_HWPMC;
4555         PROC_UNLOCK(newproc);
4556
4557  done:
4558         sx_xunlock(&pmc_sx);
4559 }
4560
4561
4562 /*
4563  * initialization
4564  */
4565
4566 static const char *pmc_name_of_pmcclass[] = {
4567 #undef  __PMC_CLASS
4568 #define __PMC_CLASS(N) #N ,
4569         __PMC_CLASSES()
4570 };
4571
4572 static int
4573 pmc_initialize(void)
4574 {
4575         int c, cpu, error, n, ri;
4576         unsigned int maxcpu;
4577         struct pmc_binding pb;
4578         struct pmc_sample *ps;
4579         struct pmc_classdep *pcd;
4580         struct pmc_samplebuffer *sb;
4581
4582         md = NULL;
4583         error = 0;
4584
4585 #ifdef  DEBUG
4586         /* parse debug flags first */
4587         if (TUNABLE_STR_FETCH(PMC_SYSCTL_NAME_PREFIX "debugflags",
4588                 pmc_debugstr, sizeof(pmc_debugstr)))
4589                 pmc_debugflags_parse(pmc_debugstr,
4590                     pmc_debugstr+strlen(pmc_debugstr));
4591 #endif
4592
4593         PMCDBG(MOD,INI,0, "PMC Initialize (version %x)", PMC_VERSION);
4594
4595         /* check kernel version */
4596         if (pmc_kernel_version != PMC_VERSION) {
4597                 if (pmc_kernel_version == 0)
4598                         printf("hwpmc: this kernel has not been compiled with "
4599                             "'options HWPMC_HOOKS'.\n");
4600                 else
4601                         printf("hwpmc: kernel version (0x%x) does not match "
4602                             "module version (0x%x).\n", pmc_kernel_version,
4603                             PMC_VERSION);
4604                 return EPROGMISMATCH;
4605         }
4606
4607         /*
4608          * check sysctl parameters
4609          */
4610
4611         if (pmc_hashsize <= 0) {
4612                 (void) printf("hwpmc: tunable \"hashsize\"=%d must be "
4613                     "greater than zero.\n", pmc_hashsize);
4614                 pmc_hashsize = PMC_HASH_SIZE;
4615         }
4616
4617         if (pmc_nsamples <= 0 || pmc_nsamples > 65535) {
4618                 (void) printf("hwpmc: tunable \"nsamples\"=%d out of "
4619                     "range.\n", pmc_nsamples);
4620                 pmc_nsamples = PMC_NSAMPLES;
4621         }
4622
4623         if (pmc_callchaindepth <= 0 ||
4624             pmc_callchaindepth > PMC_CALLCHAIN_DEPTH_MAX) {
4625                 (void) printf("hwpmc: tunable \"callchaindepth\"=%d out of "
4626                     "range.\n", pmc_callchaindepth);
4627                 pmc_callchaindepth = PMC_CALLCHAIN_DEPTH;
4628         }
4629
4630         md = pmc_md_initialize();
4631
4632         if (md == NULL)
4633                 return (ENOSYS);
4634
4635         KASSERT(md->pmd_nclass >= 1 && md->pmd_npmc >= 1,
4636             ("[pmc,%d] no classes or pmcs", __LINE__));
4637
4638         /* Compute the map from row-indices to classdep pointers. */
4639         pmc_rowindex_to_classdep = malloc(sizeof(struct pmc_classdep *) *
4640             md->pmd_npmc, M_PMC, M_WAITOK|M_ZERO);
4641
4642         for (n = 0; n < md->pmd_npmc; n++)
4643                 pmc_rowindex_to_classdep[n] = NULL;
4644         for (ri = c = 0; c < md->pmd_nclass; c++) {
4645                 pcd = &md->pmd_classdep[c];
4646                 for (n = 0; n < pcd->pcd_num; n++, ri++)
4647                         pmc_rowindex_to_classdep[ri] = pcd;
4648         }
4649
4650         KASSERT(ri == md->pmd_npmc,
4651             ("[pmc,%d] npmc miscomputed: ri=%d, md->npmc=%d", __LINE__,
4652             ri, md->pmd_npmc));
4653
4654         maxcpu = pmc_cpu_max();
4655
4656         /* allocate space for the per-cpu array */
4657         pmc_pcpu = malloc(maxcpu * sizeof(struct pmc_cpu *), M_PMC,
4658             M_WAITOK|M_ZERO);
4659
4660         /* per-cpu 'saved values' for managing process-mode PMCs */
4661         pmc_pcpu_saved = malloc(sizeof(pmc_value_t) * maxcpu * md->pmd_npmc,
4662             M_PMC, M_WAITOK);
4663
4664         /* Perform CPU-dependent initialization. */
4665         pmc_save_cpu_binding(&pb);
4666         error = 0;
4667         for (cpu = 0; error == 0 && cpu < maxcpu; cpu++) {
4668                 if (!pmc_cpu_is_active(cpu))
4669                         continue;
4670                 pmc_select_cpu(cpu);
4671                 pmc_pcpu[cpu] = malloc(sizeof(struct pmc_cpu) +
4672                     md->pmd_npmc * sizeof(struct pmc_hw *), M_PMC,
4673                     M_WAITOK|M_ZERO);
4674                 if (md->pmd_pcpu_init)
4675                         error = md->pmd_pcpu_init(md, cpu);
4676                 for (n = 0; error == 0 && n < md->pmd_nclass; n++)
4677                         error = md->pmd_classdep[n].pcd_pcpu_init(md, cpu);
4678         }
4679         pmc_restore_cpu_binding(&pb);
4680
4681         if (error)
4682                 return (error);
4683
4684         /* allocate space for the sample array */
4685         for (cpu = 0; cpu < maxcpu; cpu++) {
4686                 if (!pmc_cpu_is_active(cpu))
4687                         continue;
4688
4689                 sb = malloc(sizeof(struct pmc_samplebuffer) +
4690                     pmc_nsamples * sizeof(struct pmc_sample), M_PMC,
4691                     M_WAITOK|M_ZERO);
4692                 sb->ps_read = sb->ps_write = sb->ps_samples;
4693                 sb->ps_fence = sb->ps_samples + pmc_nsamples;
4694
4695                 KASSERT(pmc_pcpu[cpu] != NULL,
4696                     ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
4697
4698                 sb->ps_callchains = malloc(pmc_callchaindepth * pmc_nsamples *
4699                     sizeof(uintptr_t), M_PMC, M_WAITOK|M_ZERO);
4700
4701                 for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
4702                         ps->ps_pc = sb->ps_callchains +
4703                             (n * pmc_callchaindepth);
4704
4705                 pmc_pcpu[cpu]->pc_sb = sb;
4706         }
4707
4708         /* allocate space for the row disposition array */
4709         pmc_pmcdisp = malloc(sizeof(enum pmc_mode) * md->pmd_npmc,
4710             M_PMC, M_WAITOK|M_ZERO);
4711
4712         KASSERT(pmc_pmcdisp != NULL,
4713             ("[pmc,%d] pmcdisp allocation returned NULL", __LINE__));
4714
4715         /* mark all PMCs as available */
4716         for (n = 0; n < (int) md->pmd_npmc; n++)
4717                 PMC_MARK_ROW_FREE(n);
4718
4719         /* allocate thread hash tables */
4720         pmc_ownerhash = hashinit(pmc_hashsize, M_PMC,
4721             &pmc_ownerhashmask);
4722
4723         pmc_processhash = hashinit(pmc_hashsize, M_PMC,
4724             &pmc_processhashmask);
4725         mtx_init(&pmc_processhash_mtx, "pmc-process-hash", "pmc-leaf",
4726             MTX_SPIN);
4727
4728         LIST_INIT(&pmc_ss_owners);
4729         pmc_ss_count = 0;
4730
4731         /* allocate a pool of spin mutexes */
4732         pmc_mtxpool = mtx_pool_create("pmc-leaf", pmc_mtxpool_size,
4733             MTX_SPIN);
4734
4735         PMCDBG(MOD,INI,1, "pmc_ownerhash=%p, mask=0x%lx "
4736             "targethash=%p mask=0x%lx", pmc_ownerhash, pmc_ownerhashmask,
4737             pmc_processhash, pmc_processhashmask);
4738
4739         /* register process {exit,fork,exec} handlers */
4740         pmc_exit_tag = EVENTHANDLER_REGISTER(process_exit,
4741             pmc_process_exit, NULL, EVENTHANDLER_PRI_ANY);
4742         pmc_fork_tag = EVENTHANDLER_REGISTER(process_fork,
4743             pmc_process_fork, NULL, EVENTHANDLER_PRI_ANY);
4744
4745         /* initialize logging */
4746         pmclog_initialize();
4747
4748         /* set hook functions */
4749         pmc_intr = md->pmd_intr;
4750         pmc_hook = pmc_hook_handler;
4751
4752         if (error == 0) {
4753                 printf(PMC_MODULE_NAME ":");
4754                 for (n = 0; n < (int) md->pmd_nclass; n++) {
4755                         pcd = &md->pmd_classdep[n];
4756                         printf(" %s/%d/%d/0x%b",
4757                             pmc_name_of_pmcclass[pcd->pcd_class],
4758                             pcd->pcd_num,
4759                             pcd->pcd_width,
4760                             pcd->pcd_caps,
4761                             "\20"
4762                             "\1INT\2USR\3SYS\4EDG\5THR"
4763                             "\6REA\7WRI\10INV\11QUA\12PRC"
4764                             "\13TAG\14CSC");
4765                 }
4766                 printf("\n");
4767         }
4768
4769         return (error);
4770 }
4771
4772 /* prepare to be unloaded */
4773 static void
4774 pmc_cleanup(void)
4775 {
4776         int c, cpu;
4777         unsigned int maxcpu;
4778         struct pmc_ownerhash *ph;
4779         struct pmc_owner *po, *tmp;
4780         struct pmc_binding pb;
4781 #ifdef  DEBUG
4782         struct pmc_processhash *prh;
4783 #endif
4784
4785         PMCDBG(MOD,INI,0, "%s", "cleanup");
4786
4787         /* switch off sampling */
4788         atomic_store_rel_int(&pmc_cpumask, 0);
4789         pmc_intr = NULL;
4790
4791         sx_xlock(&pmc_sx);
4792         if (pmc_hook == NULL) { /* being unloaded already */
4793                 sx_xunlock(&pmc_sx);
4794                 return;
4795         }
4796
4797         pmc_hook = NULL; /* prevent new threads from entering module */
4798
4799         /* deregister event handlers */
4800         EVENTHANDLER_DEREGISTER(process_fork, pmc_fork_tag);
4801         EVENTHANDLER_DEREGISTER(process_exit, pmc_exit_tag);
4802
4803         /* send SIGBUS to all owner threads, free up allocations */
4804         if (pmc_ownerhash)
4805                 for (ph = pmc_ownerhash;
4806                      ph <= &pmc_ownerhash[pmc_ownerhashmask];
4807                      ph++) {
4808                         LIST_FOREACH_SAFE(po, ph, po_next, tmp) {
4809                                 pmc_remove_owner(po);
4810
4811                                 /* send SIGBUS to owner processes */
4812                                 PMCDBG(MOD,INI,2, "cleanup signal proc=%p "
4813                                     "(%d, %s)", po->po_owner,
4814                                     po->po_owner->p_pid,
4815                                     po->po_owner->p_comm);
4816
4817                                 PROC_LOCK(po->po_owner);
4818                                 psignal(po->po_owner, SIGBUS);
4819                                 PROC_UNLOCK(po->po_owner);
4820
4821                                 pmc_destroy_owner_descriptor(po);
4822                         }
4823                 }
4824
4825         /* reclaim allocated data structures */
4826         if (pmc_mtxpool)
4827                 mtx_pool_destroy(&pmc_mtxpool);
4828
4829         mtx_destroy(&pmc_processhash_mtx);
4830         if (pmc_processhash) {
4831 #ifdef  DEBUG
4832                 struct pmc_process *pp;
4833
4834                 PMCDBG(MOD,INI,3, "%s", "destroy process hash");
4835                 for (prh = pmc_processhash;
4836                      prh <= &pmc_processhash[pmc_processhashmask];
4837                      prh++)
4838                         LIST_FOREACH(pp, prh, pp_next)
4839                             PMCDBG(MOD,INI,3, "pid=%d", pp->pp_proc->p_pid);
4840 #endif
4841
4842                 hashdestroy(pmc_processhash, M_PMC, pmc_processhashmask);
4843                 pmc_processhash = NULL;
4844         }
4845
4846         if (pmc_ownerhash) {
4847                 PMCDBG(MOD,INI,3, "%s", "destroy owner hash");
4848                 hashdestroy(pmc_ownerhash, M_PMC, pmc_ownerhashmask);
4849                 pmc_ownerhash = NULL;
4850         }
4851
4852         KASSERT(LIST_EMPTY(&pmc_ss_owners),
4853             ("[pmc,%d] Global SS owner list not empty", __LINE__));
4854         KASSERT(pmc_ss_count == 0,
4855             ("[pmc,%d] Global SS count not empty", __LINE__));
4856
4857         /* do processor and pmc-class dependent cleanup */
4858         maxcpu = pmc_cpu_max();
4859
4860         PMCDBG(MOD,INI,3, "%s", "md cleanup");
4861         if (md) {
4862                 pmc_save_cpu_binding(&pb);
4863                 for (cpu = 0; cpu < maxcpu; cpu++) {
4864                         PMCDBG(MOD,INI,1,"pmc-cleanup cpu=%d pcs=%p",
4865                             cpu, pmc_pcpu[cpu]);
4866                         if (!pmc_cpu_is_active(cpu) || pmc_pcpu[cpu] == NULL)
4867                                 continue;
4868                         pmc_select_cpu(cpu);
4869                         for (c = 0; c < md->pmd_nclass; c++)
4870                                 md->pmd_classdep[c].pcd_pcpu_fini(md, cpu);
4871                         if (md->pmd_pcpu_fini)
4872                                 md->pmd_pcpu_fini(md, cpu);
4873                 }
4874
4875                 pmc_md_finalize(md);
4876
4877                 free(md, M_PMC);
4878                 md = NULL;
4879                 pmc_restore_cpu_binding(&pb);
4880         }
4881
4882         /* Free per-cpu descriptors. */
4883         for (cpu = 0; cpu < maxcpu; cpu++) {
4884                 if (!pmc_cpu_is_active(cpu))
4885                         continue;
4886                 KASSERT(pmc_pcpu[cpu]->pc_sb != NULL,
4887                     ("[pmc,%d] Null cpu sample buffer cpu=%d", __LINE__,
4888                         cpu));
4889                 free(pmc_pcpu[cpu]->pc_sb->ps_callchains, M_PMC);
4890                 free(pmc_pcpu[cpu]->pc_sb, M_PMC);
4891                 free(pmc_pcpu[cpu], M_PMC);
4892         }
4893
4894         free(pmc_pcpu, M_PMC);
4895         pmc_pcpu = NULL;
4896
4897         free(pmc_pcpu_saved, M_PMC);
4898         pmc_pcpu_saved = NULL;
4899
4900         if (pmc_pmcdisp) {
4901                 free(pmc_pmcdisp, M_PMC);
4902                 pmc_pmcdisp = NULL;
4903         }
4904
4905         if (pmc_rowindex_to_classdep) {
4906                 free(pmc_rowindex_to_classdep, M_PMC);
4907                 pmc_rowindex_to_classdep = NULL;
4908         }
4909
4910         pmclog_shutdown();
4911
4912         sx_xunlock(&pmc_sx);    /* we are done */
4913 }
4914
4915 /*
4916  * The function called at load/unload.
4917  */
4918
4919 static int
4920 load (struct module *module __unused, int cmd, void *arg __unused)
4921 {
4922         int error;
4923
4924         error = 0;
4925
4926         switch (cmd) {
4927         case MOD_LOAD :
4928                 /* initialize the subsystem */
4929                 error = pmc_initialize();
4930                 if (error != 0)
4931                         break;
4932                 PMCDBG(MOD,INI,1, "syscall=%d maxcpu=%d",
4933                     pmc_syscall_num, pmc_cpu_max());
4934                 break;
4935
4936
4937         case MOD_UNLOAD :
4938         case MOD_SHUTDOWN:
4939                 pmc_cleanup();
4940                 PMCDBG(MOD,INI,1, "%s", "unloaded");
4941                 break;
4942
4943         default :
4944                 error = EINVAL; /* XXX should panic(9) */
4945                 break;
4946         }
4947
4948         return error;
4949 }
4950
4951 /* memory pool */
4952 MALLOC_DEFINE(M_PMC, "pmc", "Memory space for the PMC module");