sys/dev/hwpmc/hwpmc_mod.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2003-2008 Joseph Koshy
   5  * Copyright (c) 2007 The FreeBSD Foundation
   6  * Copyright (c) 2018 Matthew Macy
   7  * All rights reserved.
   8  *
   9  * Portions of this software were developed by A. Joseph Koshy under
  10  * sponsorship from the FreeBSD Foundation and Google, Inc.
  11  *
  12  * Redistribution and use in source and binary forms, with or without
  13  * modification, are permitted provided that the following conditions
  14  * are met:
  15  * 1. Redistributions of source code must retain the above copyright
  16  *    notice, this list of conditions and the following disclaimer.
  17  * 2. Redistributions in binary form must reproduce the above copyright
  18  *    notice, this list of conditions and the following disclaimer in the
  19  *    documentation and/or other materials provided with the distribution.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  24  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  25  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  30  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  31  * SUCH DAMAGE.
  32  *
  33  */
  34
  35 #include <sys/cdefs.h>
  36 __FBSDID("$FreeBSD$");
  37
  38 #include <sys/param.h>
  39 #include <sys/eventhandler.h>
  40 #include <sys/gtaskqueue.h>
  41 #include <sys/jail.h>
  42 #include <sys/kernel.h>
  43 #include <sys/kthread.h>
  44 #include <sys/limits.h>
  45 #include <sys/lock.h>
  46 #include <sys/malloc.h>
  47 #include <sys/module.h>
  48 #include <sys/mount.h>
  49 #include <sys/mutex.h>
  50 #include <sys/pmc.h>
  51 #include <sys/pmckern.h>
  52 #include <sys/pmclog.h>
  53 #include <sys/priv.h>
  54 #include <sys/proc.h>
  55 #include <sys/queue.h>
  56 #include <sys/resourcevar.h>
  57 #include <sys/rwlock.h>
  58 #include <sys/sched.h>
  59 #include <sys/signalvar.h>
  60 #include <sys/smp.h>
  61 #include <sys/sx.h>
  62 #include <sys/sysctl.h>
  63 #include <sys/sysent.h>
  64 #include <sys/systm.h>
  65 #include <sys/vnode.h>
  66
  67 #include <sys/linker.h>         /* needs to be after <sys/malloc.h> */
  68
  69 #include <machine/atomic.h>
  70 #include <machine/md_var.h>
  71
  72 #include <vm/vm.h>
  73 #include <vm/vm_extern.h>
  74 #include <vm/pmap.h>
  75 #include <vm/vm_map.h>
  76 #include <vm/vm_object.h>
  77
  78 #include "hwpmc_soft.h"
  79
  80 #ifdef NUMA
  81 #define NDOMAINS vm_ndomains
  82 #else
  83 #define NDOMAINS 1
  84 #define malloc_domain(size, type, domain, flags) malloc((size), (type), (flags))
  85 #define free_domain(addr, type) free(addr, type)
  86 #endif
  87
  88 /*
  89  * Types
  90  */
  91
  92 enum pmc_flags {
  93         PMC_FLAG_NONE     = 0x00, /* do nothing */
  94         PMC_FLAG_REMOVE   = 0x01, /* atomically remove entry from hash */
  95         PMC_FLAG_ALLOCATE = 0x02, /* add entry to hash if not found */
  96         PMC_FLAG_NOWAIT   = 0x04, /* do not wait for mallocs */
  97 };
  98
  99 /*
 100  * The offset in sysent where the syscall is allocated.
 101  */
 102
 103 static int pmc_syscall_num = NO_SYSCALL;
 104 struct pmc_cpu          **pmc_pcpu;      /* per-cpu state */
 105 pmc_value_t             *pmc_pcpu_saved; /* saved PMC values: CSW handling */
 106
 107 #define PMC_PCPU_SAVED(C,R)     pmc_pcpu_saved[(R) + md->pmd_npmc*(C)]
 108
 109 struct mtx_pool         *pmc_mtxpool;
 110 static int              *pmc_pmcdisp;    /* PMC row dispositions */
 111
 112 #define PMC_ROW_DISP_IS_FREE(R)         (pmc_pmcdisp[(R)] == 0)
 113 #define PMC_ROW_DISP_IS_THREAD(R)       (pmc_pmcdisp[(R)] > 0)
 114 #define PMC_ROW_DISP_IS_STANDALONE(R)   (pmc_pmcdisp[(R)] < 0)
 115
 116 #define PMC_MARK_ROW_FREE(R) do {                                         \
 117         pmc_pmcdisp[(R)] = 0;                                             \
 118 } while (0)
 119
 120 #define PMC_MARK_ROW_STANDALONE(R) do {                                   \
 121         KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \
 122                     __LINE__));                                           \
 123         atomic_add_int(&pmc_pmcdisp[(R)], -1);                            \
 124         KASSERT(pmc_pmcdisp[(R)] >= (-pmc_cpu_max_active()),              \
 125                 ("[pmc,%d] row disposition error", __LINE__));            \
 126 } while (0)
 127
 128 #define PMC_UNMARK_ROW_STANDALONE(R) do {                                 \
 129         atomic_add_int(&pmc_pmcdisp[(R)], 1);                             \
 130         KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \
 131                     __LINE__));                                           \
 132 } while (0)
 133
 134 #define PMC_MARK_ROW_THREAD(R) do {                                       \
 135         KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \
 136                     __LINE__));                                           \
 137         atomic_add_int(&pmc_pmcdisp[(R)], 1);                             \
 138 } while (0)
 139
 140 #define PMC_UNMARK_ROW_THREAD(R) do {                                     \
 141         atomic_add_int(&pmc_pmcdisp[(R)], -1);                            \
 142         KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \
 143                     __LINE__));                                           \
 144 } while (0)
 145
 146
 147 /* various event handlers */
 148 static eventhandler_tag pmc_exit_tag, pmc_fork_tag, pmc_kld_load_tag,
 149     pmc_kld_unload_tag;
 150
 151 /* Module statistics */
 152 struct pmc_driverstats pmc_stats;
 153
 154
 155 /* Machine/processor dependent operations */
 156 static struct pmc_mdep  *md;
 157
 158 /*
 159  * Hash tables mapping owner processes and target threads to PMCs.
 160  */
 161
 162 struct mtx pmc_processhash_mtx;         /* spin mutex */
 163 static u_long pmc_processhashmask;
 164 static LIST_HEAD(pmc_processhash, pmc_process)  *pmc_processhash;
 165
 166 /*
 167  * Hash table of PMC owner descriptors.  This table is protected by
 168  * the shared PMC "sx" lock.
 169  */
 170
 171 static u_long pmc_ownerhashmask;
 172 static LIST_HEAD(pmc_ownerhash, pmc_owner)      *pmc_ownerhash;
 173
 174 /*
 175  * List of PMC owners with system-wide sampling PMCs.
 176  */
 177
 178 static CK_LIST_HEAD(, pmc_owner)                        pmc_ss_owners;
 179
 180 /*
 181  * List of free thread entries. This is protected by the spin
 182  * mutex.
 183  */
 184 static struct mtx pmc_threadfreelist_mtx;       /* spin mutex */
 185 static LIST_HEAD(, pmc_thread)                  pmc_threadfreelist;
 186 static int pmc_threadfreelist_entries=0;
 187 #define THREADENTRY_SIZE                                                \
 188 (sizeof(struct pmc_thread) + (md->pmd_npmc * sizeof(struct pmc_threadpmcstate)))
 189
 190 /*
 191  * Task to free thread descriptors
 192  */
 193 static struct grouptask free_gtask;
 194
 195 /*
 196  * A map of row indices to classdep structures.
 197  */
 198 static struct pmc_classdep **pmc_rowindex_to_classdep;
 199
 200 /*
 201  * Prototypes
 202  */
 203
 204 #ifdef  HWPMC_DEBUG
 205 static int      pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS);
 206 static int      pmc_debugflags_parse(char *newstr, char *fence);
 207 #endif
 208
 209 static int      load(struct module *module, int cmd, void *arg);
 210 static int      pmc_add_sample(int ring, struct pmc *pm, struct trapframe *tf);
 211 static void     pmc_add_thread_descriptors_from_proc(struct proc *p,
 212     struct pmc_process *pp);
 213 static int      pmc_attach_process(struct proc *p, struct pmc *pm);
 214 static struct pmc *pmc_allocate_pmc_descriptor(void);
 215 static struct pmc_owner *pmc_allocate_owner_descriptor(struct proc *p);
 216 static int      pmc_attach_one_process(struct proc *p, struct pmc *pm);
 217 static int      pmc_can_allocate_rowindex(struct proc *p, unsigned int ri,
 218     int cpu);
 219 static int      pmc_can_attach(struct pmc *pm, struct proc *p);
 220 static void     pmc_capture_user_callchain(int cpu, int soft, struct trapframe *tf);
 221 static void     pmc_cleanup(void);
 222 static int      pmc_detach_process(struct proc *p, struct pmc *pm);
 223 static int      pmc_detach_one_process(struct proc *p, struct pmc *pm,
 224     int flags);
 225 static void     pmc_destroy_owner_descriptor(struct pmc_owner *po);
 226 static void     pmc_destroy_pmc_descriptor(struct pmc *pm);
 227 static void     pmc_destroy_process_descriptor(struct pmc_process *pp);
 228 static struct pmc_owner *pmc_find_owner_descriptor(struct proc *p);
 229 static int      pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm);
 230 static struct pmc *pmc_find_pmc_descriptor_in_process(struct pmc_owner *po,
 231     pmc_id_t pmc);
 232 static struct pmc_process *pmc_find_process_descriptor(struct proc *p,
 233     uint32_t mode);
 234 static struct pmc_thread *pmc_find_thread_descriptor(struct pmc_process *pp,
 235     struct thread *td, uint32_t mode);
 236 static void     pmc_force_context_switch(void);
 237 static void     pmc_link_target_process(struct pmc *pm,
 238     struct pmc_process *pp);
 239 static void     pmc_log_all_process_mappings(struct pmc_owner *po);
 240 static void     pmc_log_kernel_mappings(struct pmc *pm);
 241 static void     pmc_log_process_mappings(struct pmc_owner *po, struct proc *p);
 242 static void     pmc_maybe_remove_owner(struct pmc_owner *po);
 243 static void     pmc_process_csw_in(struct thread *td);
 244 static void     pmc_process_csw_out(struct thread *td);
 245 static void     pmc_process_exit(void *arg, struct proc *p);
 246 static void     pmc_process_fork(void *arg, struct proc *p1,
 247     struct proc *p2, int n);
 248 static void     pmc_process_samples(int cpu, int soft);
 249 static void     pmc_release_pmc_descriptor(struct pmc *pmc);
 250 static void     pmc_process_thread_add(struct thread *td);
 251 static void     pmc_process_thread_delete(struct thread *td);
 252 static void     pmc_process_thread_userret(struct thread *td);
 253 static void     pmc_remove_owner(struct pmc_owner *po);
 254 static void     pmc_remove_process_descriptor(struct pmc_process *pp);
 255 static void     pmc_restore_cpu_binding(struct pmc_binding *pb);
 256 static void     pmc_save_cpu_binding(struct pmc_binding *pb);
 257 static void     pmc_select_cpu(int cpu);
 258 static int      pmc_start(struct pmc *pm);
 259 static int      pmc_stop(struct pmc *pm);
 260 static int      pmc_syscall_handler(struct thread *td, void *syscall_args);
 261 static struct pmc_thread *pmc_thread_descriptor_pool_alloc(void);
 262 static void     pmc_thread_descriptor_pool_drain(void);
 263 static void     pmc_thread_descriptor_pool_free(struct pmc_thread *pt);
 264 static void     pmc_unlink_target_process(struct pmc *pmc,
 265     struct pmc_process *pp);
 266 static int generic_switch_in(struct pmc_cpu *pc, struct pmc_process *pp);
 267 static int generic_switch_out(struct pmc_cpu *pc, struct pmc_process *pp);
 268 static struct pmc_mdep *pmc_generic_cpu_initialize(void);
 269 static void pmc_generic_cpu_finalize(struct pmc_mdep *md);
 270 static void pmc_post_callchain_callback(void);
 271 static void pmc_process_threadcreate(struct thread *td);
 272 static void pmc_process_threadexit(struct thread *td);
 273 static void pmc_process_proccreate(struct proc *p);
 274 static void pmc_process_allproc(struct pmc *pm);
 275
 276 /*
 277  * Kernel tunables and sysctl(8) interface.
 278  */
 279
 280 SYSCTL_DECL(_kern_hwpmc);
 281 SYSCTL_NODE(_kern_hwpmc, OID_AUTO, stats, CTLFLAG_RW, 0, "HWPMC stats");
 282
 283
 284 /* Stats. */
 285 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, intr_ignored, CTLFLAG_RW,
 286                                    &pmc_stats.pm_intr_ignored, "# of interrupts ignored");
 287 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, intr_processed, CTLFLAG_RW,
 288                                    &pmc_stats.pm_intr_processed, "# of interrupts processed");
 289 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, intr_bufferfull, CTLFLAG_RW,
 290                                    &pmc_stats.pm_intr_bufferfull, "# of interrupts where buffer was full");
 291 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, syscalls, CTLFLAG_RW,
 292                                    &pmc_stats.pm_syscalls, "# of syscalls");
 293 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, syscall_errors, CTLFLAG_RW,
 294                                    &pmc_stats.pm_syscall_errors, "# of syscall_errors");
 295 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, buffer_requests, CTLFLAG_RW,
 296                                    &pmc_stats.pm_buffer_requests, "# of buffer requests");
 297 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, buffer_requests_failed, CTLFLAG_RW,
 298                                    &pmc_stats.pm_buffer_requests_failed, "# of buffer requests which failed");
 299 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, log_sweeps, CTLFLAG_RW,
 300                                    &pmc_stats.pm_log_sweeps, "# of ?");
 301 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, merges, CTLFLAG_RW,
 302                                    &pmc_stats.pm_merges, "# of times kernel stack was found for user trace");
 303 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, overwrites, CTLFLAG_RW,
 304                                    &pmc_stats.pm_overwrites, "# of times a sample was overwritten before being logged");
 305
 306 static int pmc_callchaindepth = PMC_CALLCHAIN_DEPTH;
 307 SYSCTL_INT(_kern_hwpmc, OID_AUTO, callchaindepth, CTLFLAG_RDTUN,
 308     &pmc_callchaindepth, 0, "depth of call chain records");
 309
 310 char pmc_cpuid[64];
 311 SYSCTL_STRING(_kern_hwpmc, OID_AUTO, cpuid, CTLFLAG_RD,
 312         pmc_cpuid, 0, "cpu version string");
 313 #ifdef  HWPMC_DEBUG
 314 struct pmc_debugflags pmc_debugflags = PMC_DEBUG_DEFAULT_FLAGS;
 315 char    pmc_debugstr[PMC_DEBUG_STRSIZE];
 316 TUNABLE_STR(PMC_SYSCTL_NAME_PREFIX "debugflags", pmc_debugstr,
 317     sizeof(pmc_debugstr));
 318 SYSCTL_PROC(_kern_hwpmc, OID_AUTO, debugflags,
 319     CTLTYPE_STRING | CTLFLAG_RWTUN | CTLFLAG_NOFETCH,
 320     0, 0, pmc_debugflags_sysctl_handler, "A", "debug flags");
 321 #endif
 322
 323
 324 /*
 325  * kern.hwpmc.hashrows -- determines the number of rows in the
 326  * of the hash table used to look up threads
 327  */
 328
 329 static int pmc_hashsize = PMC_HASH_SIZE;
 330 SYSCTL_INT(_kern_hwpmc, OID_AUTO, hashsize, CTLFLAG_RDTUN,
 331     &pmc_hashsize, 0, "rows in hash tables");
 332
 333 /*
 334  * kern.hwpmc.nsamples --- number of PC samples/callchain stacks per CPU
 335  */
 336
 337 static int pmc_nsamples = PMC_NSAMPLES;
 338 SYSCTL_INT(_kern_hwpmc, OID_AUTO, nsamples, CTLFLAG_RDTUN,
 339     &pmc_nsamples, 0, "number of PC samples per CPU");
 340
 341
 342 /*
 343  * kern.hwpmc.mtxpoolsize -- number of mutexes in the mutex pool.
 344  */
 345
 346 static int pmc_mtxpool_size = PMC_MTXPOOL_SIZE;
 347 SYSCTL_INT(_kern_hwpmc, OID_AUTO, mtxpoolsize, CTLFLAG_RDTUN,
 348     &pmc_mtxpool_size, 0, "size of spin mutex pool");
 349
 350
 351 /*
 352  * kern.hwpmc.threadfreelist_entries -- number of free entries
 353  */
 354
 355 SYSCTL_INT(_kern_hwpmc, OID_AUTO, threadfreelist_entries, CTLFLAG_RD,
 356     &pmc_threadfreelist_entries, 0, "number of avalable thread entries");
 357
 358
 359 /*
 360  * kern.hwpmc.threadfreelist_max -- maximum number of free entries
 361  */
 362
 363 static int pmc_threadfreelist_max = PMC_THREADLIST_MAX;
 364 SYSCTL_INT(_kern_hwpmc, OID_AUTO, threadfreelist_max, CTLFLAG_RW,
 365     &pmc_threadfreelist_max, 0,
 366     "maximum number of available thread entries before freeing some");
 367
 368
 369 /*
 370  * security.bsd.unprivileged_syspmcs -- allow non-root processes to
 371  * allocate system-wide PMCs.
 372  *
 373  * Allowing unprivileged processes to allocate system PMCs is convenient
 374  * if system-wide measurements need to be taken concurrently with other
 375  * per-process measurements.  This feature is turned off by default.
 376  */
 377
 378 static int pmc_unprivileged_syspmcs = 0;
 379 SYSCTL_INT(_security_bsd, OID_AUTO, unprivileged_syspmcs, CTLFLAG_RWTUN,
 380     &pmc_unprivileged_syspmcs, 0,
 381     "allow unprivileged process to allocate system PMCs");
 382
 383 /*
 384  * Hash function.  Discard the lower 2 bits of the pointer since
 385  * these are always zero for our uses.  The hash multiplier is
 386  * round((2^LONG_BIT) * ((sqrt(5)-1)/2)).
 387  */
 388
 389 #if     LONG_BIT == 64
 390 #define _PMC_HM         11400714819323198486u
 391 #elif   LONG_BIT == 32
 392 #define _PMC_HM         2654435769u
 393 #else
 394 #error  Must know the size of 'long' to compile
 395 #endif
 396
 397 #define PMC_HASH_PTR(P,M)       ((((unsigned long) (P) >> 2) * _PMC_HM) & (M))
 398
 399 /*
 400  * Syscall structures
 401  */
 402
 403 /* The `sysent' for the new syscall */
 404 static struct sysent pmc_sysent = {
 405         .sy_narg =      2,
 406         .sy_call =      pmc_syscall_handler,
 407 };
 408
 409 static struct syscall_module_data pmc_syscall_mod = {
 410         .chainevh =     load,
 411         .chainarg =     NULL,
 412         .offset =       &pmc_syscall_num,
 413         .new_sysent =   &pmc_sysent,
 414         .old_sysent =   { .sy_narg = 0, .sy_call = NULL },
 415         .flags =        SY_THR_STATIC_KLD,
 416 };
 417
 418 static moduledata_t pmc_mod = {
 419         .name =         PMC_MODULE_NAME,
 420         .evhand =       syscall_module_handler,
 421         .priv =         &pmc_syscall_mod,
 422 };
 423
 424 #ifdef EARLY_AP_STARTUP
 425 DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SYSCALLS, SI_ORDER_ANY);
 426 #else
 427 DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SMP, SI_ORDER_ANY);
 428 #endif
 429 MODULE_VERSION(pmc, PMC_VERSION);
 430
 431 #ifdef  HWPMC_DEBUG
 432 enum pmc_dbgparse_state {
 433         PMCDS_WS,               /* in whitespace */
 434         PMCDS_MAJOR,            /* seen a major keyword */
 435         PMCDS_MINOR
 436 };
 437
 438 static int
 439 pmc_debugflags_parse(char *newstr, char *fence)
 440 {
 441         char c, *p, *q;
 442         struct pmc_debugflags *tmpflags;
 443         int error, found, *newbits, tmp;
 444         size_t kwlen;
 445
 446         tmpflags = malloc(sizeof(*tmpflags), M_PMC, M_WAITOK|M_ZERO);
 447
 448         p = newstr;
 449         error = 0;
 450
 451         for (; p < fence && (c = *p); p++) {
 452
 453                 /* skip white space */
 454                 if (c == ' ' || c == '\t')
 455                         continue;
 456
 457                 /* look for a keyword followed by "=" */
 458                 for (q = p; p < fence && (c = *p) && c != '='; p++)
 459                         ;
 460                 if (c != '=') {
 461                         error = EINVAL;
 462                         goto done;
 463                 }
 464
 465                 kwlen = p - q;
 466                 newbits = NULL;
 467
 468                 /* lookup flag group name */
 469 #define DBG_SET_FLAG_MAJ(S,F)                                           \
 470                 if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0)  \
 471                         newbits = &tmpflags->pdb_ ## F;
 472
 473                 DBG_SET_FLAG_MAJ("cpu",         CPU);
 474                 DBG_SET_FLAG_MAJ("csw",         CSW);
 475                 DBG_SET_FLAG_MAJ("logging",     LOG);
 476                 DBG_SET_FLAG_MAJ("module",      MOD);
 477                 DBG_SET_FLAG_MAJ("md",          MDP);
 478                 DBG_SET_FLAG_MAJ("owner",       OWN);
 479                 DBG_SET_FLAG_MAJ("pmc",         PMC);
 480                 DBG_SET_FLAG_MAJ("process",     PRC);
 481                 DBG_SET_FLAG_MAJ("sampling",    SAM);
 482
 483                 if (newbits == NULL) {
 484                         error = EINVAL;
 485                         goto done;
 486                 }
 487
 488                 p++;            /* skip the '=' */
 489
 490                 /* Now parse the individual flags */
 491                 tmp = 0;
 492         newflag:
 493                 for (q = p; p < fence && (c = *p); p++)
 494                         if (c == ' ' || c == '\t' || c == ',')
 495                                 break;
 496
 497                 /* p == fence or c == ws or c == "," or c == 0 */
 498
 499                 if ((kwlen = p - q) == 0) {
 500                         *newbits = tmp;
 501                         continue;
 502                 }
 503
 504                 found = 0;
 505 #define DBG_SET_FLAG_MIN(S,F)                                           \
 506                 if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0)  \
 507                         tmp |= found = (1 << PMC_DEBUG_MIN_ ## F)
 508
 509                 /* a '*' denotes all possible flags in the group */
 510                 if (kwlen == 1 && *q == '*')
 511                         tmp = found = ~0;
 512                 /* look for individual flag names */
 513                 DBG_SET_FLAG_MIN("allocaterow", ALR);
 514                 DBG_SET_FLAG_MIN("allocate",    ALL);
 515                 DBG_SET_FLAG_MIN("attach",      ATT);
 516                 DBG_SET_FLAG_MIN("bind",        BND);
 517                 DBG_SET_FLAG_MIN("config",      CFG);
 518                 DBG_SET_FLAG_MIN("exec",        EXC);
 519                 DBG_SET_FLAG_MIN("exit",        EXT);
 520                 DBG_SET_FLAG_MIN("find",        FND);
 521                 DBG_SET_FLAG_MIN("flush",       FLS);
 522                 DBG_SET_FLAG_MIN("fork",        FRK);
 523                 DBG_SET_FLAG_MIN("getbuf",      GTB);
 524                 DBG_SET_FLAG_MIN("hook",        PMH);
 525                 DBG_SET_FLAG_MIN("init",        INI);
 526                 DBG_SET_FLAG_MIN("intr",        INT);
 527                 DBG_SET_FLAG_MIN("linktarget",  TLK);
 528                 DBG_SET_FLAG_MIN("mayberemove", OMR);
 529                 DBG_SET_FLAG_MIN("ops",         OPS);
 530                 DBG_SET_FLAG_MIN("read",        REA);
 531                 DBG_SET_FLAG_MIN("register",    REG);
 532                 DBG_SET_FLAG_MIN("release",     REL);
 533                 DBG_SET_FLAG_MIN("remove",      ORM);
 534                 DBG_SET_FLAG_MIN("sample",      SAM);
 535                 DBG_SET_FLAG_MIN("scheduleio",  SIO);
 536                 DBG_SET_FLAG_MIN("select",      SEL);
 537                 DBG_SET_FLAG_MIN("signal",      SIG);
 538                 DBG_SET_FLAG_MIN("swi",         SWI);
 539                 DBG_SET_FLAG_MIN("swo",         SWO);
 540                 DBG_SET_FLAG_MIN("start",       STA);
 541                 DBG_SET_FLAG_MIN("stop",        STO);
 542                 DBG_SET_FLAG_MIN("syscall",     PMS);
 543                 DBG_SET_FLAG_MIN("unlinktarget", TUL);
 544                 DBG_SET_FLAG_MIN("write",       WRI);
 545                 if (found == 0) {
 546                         /* unrecognized flag name */
 547                         error = EINVAL;
 548                         goto done;
 549                 }
 550
 551                 if (c == 0 || c == ' ' || c == '\t') {  /* end of flag group */
 552                         *newbits = tmp;
 553                         continue;
 554                 }
 555
 556                 p++;
 557                 goto newflag;
 558         }
 559
 560         /* save the new flag set */
 561         bcopy(tmpflags, &pmc_debugflags, sizeof(pmc_debugflags));
 562
 563  done:
 564         free(tmpflags, M_PMC);
 565         return error;
 566 }
 567
 568 static int
 569 pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS)
 570 {
 571         char *fence, *newstr;
 572         int error;
 573         unsigned int n;
 574
 575         (void) arg1; (void) arg2; /* unused parameters */
 576
 577         n = sizeof(pmc_debugstr);
 578         newstr = malloc(n, M_PMC, M_WAITOK|M_ZERO);
 579         (void) strlcpy(newstr, pmc_debugstr, n);
 580
 581         error = sysctl_handle_string(oidp, newstr, n, req);
 582
 583         /* if there is a new string, parse and copy it */
 584         if (error == 0 && req->newptr != NULL) {
 585                 fence = newstr + (n < req->newlen ? n : req->newlen + 1);
 586                 if ((error = pmc_debugflags_parse(newstr, fence)) == 0)
 587                         (void) strlcpy(pmc_debugstr, newstr,
 588                             sizeof(pmc_debugstr));
 589         }
 590
 591         free(newstr, M_PMC);
 592
 593         return error;
 594 }
 595 #endif
 596
 597 /*
 598  * Map a row index to a classdep structure and return the adjusted row
 599  * index for the PMC class index.
 600  */
 601 static struct pmc_classdep *
 602 pmc_ri_to_classdep(struct pmc_mdep *md, int ri, int *adjri)
 603 {
 604         struct pmc_classdep *pcd;
 605
 606         (void) md;
 607
 608         KASSERT(ri >= 0 && ri < md->pmd_npmc,
 609             ("[pmc,%d] illegal row-index %d", __LINE__, ri));
 610
 611         pcd = pmc_rowindex_to_classdep[ri];
 612
 613         KASSERT(pcd != NULL,
 614             ("[pmc,%d] ri %d null pcd", __LINE__, ri));
 615
 616         *adjri = ri - pcd->pcd_ri;
 617
 618         KASSERT(*adjri >= 0 && *adjri < pcd->pcd_num,
 619             ("[pmc,%d] adjusted row-index %d", __LINE__, *adjri));
 620
 621         return (pcd);
 622 }
 623
 624 /*
 625  * Concurrency Control
 626  *
 627  * The driver manages the following data structures:
 628  *
 629  *   - target process descriptors, one per target process
 630  *   - owner process descriptors (and attached lists), one per owner process
 631  *   - lookup hash tables for owner and target processes
 632  *   - PMC descriptors (and attached lists)
 633  *   - per-cpu hardware state
 634  *   - the 'hook' variable through which the kernel calls into
 635  *     this module
 636  *   - the machine hardware state (managed by the MD layer)
 637  *
 638  * These data structures are accessed from:
 639  *
 640  * - thread context-switch code
 641  * - interrupt handlers (possibly on multiple cpus)
 642  * - kernel threads on multiple cpus running on behalf of user
 643  *   processes doing system calls
 644  * - this driver's private kernel threads
 645  *
 646  * = Locks and Locking strategy =
 647  *
 648  * The driver uses four locking strategies for its operation:
 649  *
 650  * - The global SX lock "pmc_sx" is used to protect internal
 651  *   data structures.
 652  *
 653  *   Calls into the module by syscall() start with this lock being
 654  *   held in exclusive mode.  Depending on the requested operation,
 655  *   the lock may be downgraded to 'shared' mode to allow more
 656  *   concurrent readers into the module.  Calls into the module from
 657  *   other parts of the kernel acquire the lock in shared mode.
 658  *
 659  *   This SX lock is held in exclusive mode for any operations that
 660  *   modify the linkages between the driver's internal data structures.
 661  *
 662  *   The 'pmc_hook' function pointer is also protected by this lock.
 663  *   It is only examined with the sx lock held in exclusive mode.  The
 664  *   kernel module is allowed to be unloaded only with the sx lock held
 665  *   in exclusive mode.  In normal syscall handling, after acquiring the
 666  *   pmc_sx lock we first check that 'pmc_hook' is non-null before
 667  *   proceeding.  This prevents races between the thread unloading the module
 668  *   and other threads seeking to use the module.
 669  *
 670  * - Lookups of target process structures and owner process structures
 671  *   cannot use the global "pmc_sx" SX lock because these lookups need
 672  *   to happen during context switches and in other critical sections
 673  *   where sleeping is not allowed.  We protect these lookup tables
 674  *   with their own private spin-mutexes, "pmc_processhash_mtx" and
 675  *   "pmc_ownerhash_mtx".
 676  *
 677  * - Interrupt handlers work in a lock free manner.  At interrupt
 678  *   time, handlers look at the PMC pointer (phw->phw_pmc) configured
 679  *   when the PMC was started.  If this pointer is NULL, the interrupt
 680  *   is ignored after updating driver statistics.  We ensure that this
 681  *   pointer is set (using an atomic operation if necessary) before the
 682  *   PMC hardware is started.  Conversely, this pointer is unset atomically
 683  *   only after the PMC hardware is stopped.
 684  *
 685  *   We ensure that everything needed for the operation of an
 686  *   interrupt handler is available without it needing to acquire any
 687  *   locks.  We also ensure that a PMC's software state is destroyed only
 688  *   after the PMC is taken off hardware (on all CPUs).
 689  *
 690  * - Context-switch handling with process-private PMCs needs more
 691  *   care.
 692  *
 693  *   A given process may be the target of multiple PMCs.  For example,
 694  *   PMCATTACH and PMCDETACH may be requested by a process on one CPU
 695  *   while the target process is running on another.  A PMC could also
 696  *   be getting released because its owner is exiting.  We tackle
 697  *   these situations in the following manner:
 698  *
 699  *   - each target process structure 'pmc_process' has an array
 700  *     of 'struct pmc *' pointers, one for each hardware PMC.
 701  *
 702  *   - At context switch IN time, each "target" PMC in RUNNING state
 703  *     gets started on hardware and a pointer to each PMC is copied into
 704  *     the per-cpu phw array.  The 'runcount' for the PMC is
 705  *     incremented.
 706  *
 707  *   - At context switch OUT time, all process-virtual PMCs are stopped
 708  *     on hardware.  The saved value is added to the PMCs value field
 709  *     only if the PMC is in a non-deleted state (the PMCs state could
 710  *     have changed during the current time slice).
 711  *
 712  *     Note that since in-between a switch IN on a processor and a switch
 713  *     OUT, the PMC could have been released on another CPU.  Therefore
 714  *     context switch OUT always looks at the hardware state to turn
 715  *     OFF PMCs and will update a PMC's saved value only if reachable
 716  *     from the target process record.
 717  *
 718  *   - OP PMCRELEASE could be called on a PMC at any time (the PMC could
 719  *     be attached to many processes at the time of the call and could
 720  *     be active on multiple CPUs).
 721  *
 722  *     We prevent further scheduling of the PMC by marking it as in
 723  *     state 'DELETED'.  If the runcount of the PMC is non-zero then
 724  *     this PMC is currently running on a CPU somewhere.  The thread
 725  *     doing the PMCRELEASE operation waits by repeatedly doing a
 726  *     pause() till the runcount comes to zero.
 727  *
 728  * The contents of a PMC descriptor (struct pmc) are protected using
 729  * a spin-mutex.  In order to save space, we use a mutex pool.
 730  *
 731  * In terms of lock types used by witness(4), we use:
 732  * - Type "pmc-sx", used by the global SX lock.
 733  * - Type "pmc-sleep", for sleep mutexes used by logger threads.
 734  * - Type "pmc-per-proc", for protecting PMC owner descriptors.
 735  * - Type "pmc-leaf", used for all other spin mutexes.
 736  */
 737
 738 /*
 739  * save the cpu binding of the current kthread
 740  */
 741
 742 static void
 743 pmc_save_cpu_binding(struct pmc_binding *pb)
 744 {
 745         PMCDBG0(CPU,BND,2, "save-cpu");
 746         thread_lock(curthread);
 747         pb->pb_bound = sched_is_bound(curthread);
 748         pb->pb_cpu   = curthread->td_oncpu;
 749         thread_unlock(curthread);
 750         PMCDBG1(CPU,BND,2, "save-cpu cpu=%d", pb->pb_cpu);
 751 }
 752
 753 /*
 754  * restore the cpu binding of the current thread
 755  */
 756
 757 static void
 758 pmc_restore_cpu_binding(struct pmc_binding *pb)
 759 {
 760         PMCDBG2(CPU,BND,2, "restore-cpu curcpu=%d restore=%d",
 761             curthread->td_oncpu, pb->pb_cpu);
 762         thread_lock(curthread);
 763         if (pb->pb_bound)
 764                 sched_bind(curthread, pb->pb_cpu);
 765         else
 766                 sched_unbind(curthread);
 767         thread_unlock(curthread);
 768         PMCDBG0(CPU,BND,2, "restore-cpu done");
 769 }
 770
 771 /*
 772  * move execution over the specified cpu and bind it there.
 773  */
 774
 775 static void
 776 pmc_select_cpu(int cpu)
 777 {
 778         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 779             ("[pmc,%d] bad cpu number %d", __LINE__, cpu));
 780
 781         /* Never move to an inactive CPU. */
 782         KASSERT(pmc_cpu_is_active(cpu), ("[pmc,%d] selecting inactive "
 783             "CPU %d", __LINE__, cpu));
 784
 785         PMCDBG1(CPU,SEL,2, "select-cpu cpu=%d", cpu);
 786         thread_lock(curthread);
 787         sched_bind(curthread, cpu);
 788         thread_unlock(curthread);
 789
 790         KASSERT(curthread->td_oncpu == cpu,
 791             ("[pmc,%d] CPU not bound [cpu=%d, curr=%d]", __LINE__,
 792                 cpu, curthread->td_oncpu));
 793
 794         PMCDBG1(CPU,SEL,2, "select-cpu cpu=%d ok", cpu);
 795 }
 796
 797 /*
 798  * Force a context switch.
 799  *
 800  * We do this by pause'ing for 1 tick -- invoking mi_switch() is not
 801  * guaranteed to force a context switch.
 802  */
 803
 804 static void
 805 pmc_force_context_switch(void)
 806 {
 807
 808         pause("pmcctx", 1);
 809 }
 810
 811 uint64_t
 812 pmc_rdtsc(void)
 813 {
 814 #if defined(__i386__) || defined(__amd64__)
 815         if (__predict_true(amd_feature & AMDID_RDTSCP))
 816                 return rdtscp();
 817         else
 818                 return rdtsc();
 819 #else
 820         return get_cyclecount();
 821 #endif
 822 }
 823
 824 /*
 825  * Get the file name for an executable.  This is a simple wrapper
 826  * around vn_fullpath(9).
 827  */
 828
 829 static void
 830 pmc_getfilename(struct vnode *v, char **fullpath, char **freepath)
 831 {
 832
 833         *fullpath = "unknown";
 834         *freepath = NULL;
 835         vn_fullpath(curthread, v, fullpath, freepath);
 836 }
 837
 838 /*
 839  * remove an process owning PMCs
 840  */
 841
 842 void
 843 pmc_remove_owner(struct pmc_owner *po)
 844 {
 845         struct pmc *pm, *tmp;
 846
 847         sx_assert(&pmc_sx, SX_XLOCKED);
 848
 849         PMCDBG1(OWN,ORM,1, "remove-owner po=%p", po);
 850
 851         /* Remove descriptor from the owner hash table */
 852         LIST_REMOVE(po, po_next);
 853
 854         /* release all owned PMC descriptors */
 855         LIST_FOREACH_SAFE(pm, &po->po_pmcs, pm_next, tmp) {
 856                 PMCDBG1(OWN,ORM,2, "pmc=%p", pm);
 857                 KASSERT(pm->pm_owner == po,
 858                     ("[pmc,%d] owner %p != po %p", __LINE__, pm->pm_owner, po));
 859
 860                 pmc_release_pmc_descriptor(pm); /* will unlink from the list */
 861                 pmc_destroy_pmc_descriptor(pm);
 862         }
 863
 864         KASSERT(po->po_sscount == 0,
 865             ("[pmc,%d] SS count not zero", __LINE__));
 866         KASSERT(LIST_EMPTY(&po->po_pmcs),
 867             ("[pmc,%d] PMC list not empty", __LINE__));
 868
 869         /* de-configure the log file if present */
 870         if (po->po_flags & PMC_PO_OWNS_LOGFILE)
 871                 pmclog_deconfigure_log(po);
 872 }
 873
 874 /*
 875  * remove an owner process record if all conditions are met.
 876  */
 877
 878 static void
 879 pmc_maybe_remove_owner(struct pmc_owner *po)
 880 {
 881
 882         PMCDBG1(OWN,OMR,1, "maybe-remove-owner po=%p", po);
 883
 884         /*
 885          * Remove owner record if
 886          * - this process does not own any PMCs
 887          * - this process has not allocated a system-wide sampling buffer
 888          */
 889
 890         if (LIST_EMPTY(&po->po_pmcs) &&
 891             ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)) {
 892                 pmc_remove_owner(po);
 893                 pmc_destroy_owner_descriptor(po);
 894         }
 895 }
 896
 897 /*
 898  * Add an association between a target process and a PMC.
 899  */
 900
 901 static void
 902 pmc_link_target_process(struct pmc *pm, struct pmc_process *pp)
 903 {
 904         int ri;
 905         struct pmc_target *pt;
 906 #ifdef INVARIANTS
 907         struct pmc_thread *pt_td;
 908 #endif
 909
 910         sx_assert(&pmc_sx, SX_XLOCKED);
 911
 912         KASSERT(pm != NULL && pp != NULL,
 913             ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
 914         KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)),
 915             ("[pmc,%d] Attaching a non-process-virtual pmc=%p to pid=%d",
 916                 __LINE__, pm, pp->pp_proc->p_pid));
 917         KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= ((int) md->pmd_npmc - 1),
 918             ("[pmc,%d] Illegal reference count %d for process record %p",
 919                 __LINE__, pp->pp_refcnt, (void *) pp));
 920
 921         ri = PMC_TO_ROWINDEX(pm);
 922
 923         PMCDBG3(PRC,TLK,1, "link-target pmc=%p ri=%d pmc-process=%p",
 924             pm, ri, pp);
 925
 926 #ifdef  HWPMC_DEBUG
 927         LIST_FOREACH(pt, &pm->pm_targets, pt_next)
 928             if (pt->pt_process == pp)
 929                     KASSERT(0, ("[pmc,%d] pp %p already in pmc %p targets",
 930                                 __LINE__, pp, pm));
 931 #endif
 932
 933         pt = malloc(sizeof(struct pmc_target), M_PMC, M_WAITOK|M_ZERO);
 934         pt->pt_process = pp;
 935
 936         LIST_INSERT_HEAD(&pm->pm_targets, pt, pt_next);
 937
 938         atomic_store_rel_ptr((uintptr_t *)&pp->pp_pmcs[ri].pp_pmc,
 939             (uintptr_t)pm);
 940
 941         if (pm->pm_owner->po_owner == pp->pp_proc)
 942                 pm->pm_flags |= PMC_F_ATTACHED_TO_OWNER;
 943
 944         /*
 945          * Initialize the per-process values at this row index.
 946          */
 947         pp->pp_pmcs[ri].pp_pmcval = PMC_TO_MODE(pm) == PMC_MODE_TS ?
 948             pm->pm_sc.pm_reloadcount : 0;
 949
 950         pp->pp_refcnt++;
 951
 952 #ifdef INVARIANTS
 953         /* Confirm that the per-thread values at this row index are cleared. */
 954         if (PMC_TO_MODE(pm) == PMC_MODE_TS) {
 955                 mtx_lock_spin(pp->pp_tdslock);
 956                 LIST_FOREACH(pt_td, &pp->pp_tds, pt_next) {
 957                         KASSERT(pt_td->pt_pmcs[ri].pt_pmcval == (pmc_value_t) 0,
 958                             ("[pmc,%d] pt_pmcval not cleared for pid=%d at "
 959                             "ri=%d", __LINE__, pp->pp_proc->p_pid, ri));
 960                 }
 961                 mtx_unlock_spin(pp->pp_tdslock);
 962         }
 963 #endif
 964 }
 965
 966 /*
 967  * Removes the association between a target process and a PMC.
 968  */
 969
 970 static void
 971 pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp)
 972 {
 973         int ri;
 974         struct proc *p;
 975         struct pmc_target *ptgt;
 976         struct pmc_thread *pt;
 977
 978         sx_assert(&pmc_sx, SX_XLOCKED);
 979
 980         KASSERT(pm != NULL && pp != NULL,
 981             ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
 982
 983         KASSERT(pp->pp_refcnt >= 1 && pp->pp_refcnt <= (int) md->pmd_npmc,
 984             ("[pmc,%d] Illegal ref count %d on process record %p",
 985                 __LINE__, pp->pp_refcnt, (void *) pp));
 986
 987         ri = PMC_TO_ROWINDEX(pm);
 988
 989         PMCDBG3(PRC,TUL,1, "unlink-target pmc=%p ri=%d pmc-process=%p",
 990             pm, ri, pp);
 991
 992         KASSERT(pp->pp_pmcs[ri].pp_pmc == pm,
 993             ("[pmc,%d] PMC ri %d mismatch pmc %p pp->[ri] %p", __LINE__,
 994                 ri, pm, pp->pp_pmcs[ri].pp_pmc));
 995
 996         pp->pp_pmcs[ri].pp_pmc = NULL;
 997         pp->pp_pmcs[ri].pp_pmcval = (pmc_value_t) 0;
 998
 999         /* Clear the per-thread values at this row index. */
1000         if (PMC_TO_MODE(pm) == PMC_MODE_TS) {
1001                 mtx_lock_spin(pp->pp_tdslock);
1002                 LIST_FOREACH(pt, &pp->pp_tds, pt_next)
1003                         pt->pt_pmcs[ri].pt_pmcval = (pmc_value_t) 0;
1004                 mtx_unlock_spin(pp->pp_tdslock);
1005         }
1006
1007         /* Remove owner-specific flags */
1008         if (pm->pm_owner->po_owner == pp->pp_proc) {
1009                 pp->pp_flags &= ~PMC_PP_ENABLE_MSR_ACCESS;
1010                 pm->pm_flags &= ~PMC_F_ATTACHED_TO_OWNER;
1011         }
1012
1013         pp->pp_refcnt--;
1014
1015         /* Remove the target process from the PMC structure */
1016         LIST_FOREACH(ptgt, &pm->pm_targets, pt_next)
1017                 if (ptgt->pt_process == pp)
1018                         break;
1019
1020         KASSERT(ptgt != NULL, ("[pmc,%d] process %p (pp: %p) not found "
1021                     "in pmc %p", __LINE__, pp->pp_proc, pp, pm));
1022
1023         LIST_REMOVE(ptgt, pt_next);
1024         free(ptgt, M_PMC);
1025
1026         /* if the PMC now lacks targets, send the owner a SIGIO */
1027         if (LIST_EMPTY(&pm->pm_targets)) {
1028                 p = pm->pm_owner->po_owner;
1029                 PROC_LOCK(p);
1030                 kern_psignal(p, SIGIO);
1031                 PROC_UNLOCK(p);
1032
1033                 PMCDBG2(PRC,SIG,2, "signalling proc=%p signal=%d", p,
1034                     SIGIO);
1035         }
1036 }
1037
1038 /*
1039  * Check if PMC 'pm' may be attached to target process 't'.
1040  */
1041
1042 static int
1043 pmc_can_attach(struct pmc *pm, struct proc *t)
1044 {
1045         struct proc *o;         /* pmc owner */
1046         struct ucred *oc, *tc;  /* owner, target credentials */
1047         int decline_attach, i;
1048
1049         /*
1050          * A PMC's owner can always attach that PMC to itself.
1051          */
1052
1053         if ((o = pm->pm_owner->po_owner) == t)
1054                 return 0;
1055
1056         PROC_LOCK(o);
1057         oc = o->p_ucred;
1058         crhold(oc);
1059         PROC_UNLOCK(o);
1060
1061         PROC_LOCK(t);
1062         tc = t->p_ucred;
1063         crhold(tc);
1064         PROC_UNLOCK(t);
1065
1066         /*
1067          * The effective uid of the PMC owner should match at least one
1068          * of the {effective,real,saved} uids of the target process.
1069          */
1070
1071         decline_attach = oc->cr_uid != tc->cr_uid &&
1072             oc->cr_uid != tc->cr_svuid &&
1073             oc->cr_uid != tc->cr_ruid;
1074
1075         /*
1076          * Every one of the target's group ids, must be in the owner's
1077          * group list.
1078          */
1079         for (i = 0; !decline_attach && i < tc->cr_ngroups; i++)
1080                 decline_attach = !groupmember(tc->cr_groups[i], oc);
1081
1082         /* check the read and saved gids too */
1083         if (decline_attach == 0)
1084                 decline_attach = !groupmember(tc->cr_rgid, oc) ||
1085                     !groupmember(tc->cr_svgid, oc);
1086
1087         crfree(tc);
1088         crfree(oc);
1089
1090         return !decline_attach;
1091 }
1092
1093 /*
1094  * Attach a process to a PMC.
1095  */
1096
1097 static int
1098 pmc_attach_one_process(struct proc *p, struct pmc *pm)
1099 {
1100         int ri, error;
1101         char *fullpath, *freepath;
1102         struct pmc_process      *pp;
1103
1104         sx_assert(&pmc_sx, SX_XLOCKED);
1105
1106         PMCDBG5(PRC,ATT,2, "attach-one pm=%p ri=%d proc=%p (%d, %s)", pm,
1107             PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
1108
1109         /*
1110          * Locate the process descriptor corresponding to process 'p',
1111          * allocating space as needed.
1112          *
1113          * Verify that rowindex 'pm_rowindex' is free in the process
1114          * descriptor.
1115          *
1116          * If not, allocate space for a descriptor and link the
1117          * process descriptor and PMC.
1118          */
1119         ri = PMC_TO_ROWINDEX(pm);
1120
1121         /* mark process as using HWPMCs */
1122         PROC_LOCK(p);
1123         p->p_flag |= P_HWPMC;
1124         PROC_UNLOCK(p);
1125
1126         if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_ALLOCATE)) == NULL) {
1127                 error = ENOMEM;
1128                 goto fail;
1129         }
1130
1131         if (pp->pp_pmcs[ri].pp_pmc == pm) {/* already present at slot [ri] */
1132                 error = EEXIST;
1133                 goto fail;
1134         }
1135
1136         if (pp->pp_pmcs[ri].pp_pmc != NULL) {
1137                 error = EBUSY;
1138                 goto fail;
1139         }
1140
1141         pmc_link_target_process(pm, pp);
1142
1143         if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) &&
1144             (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) == 0)
1145                 pm->pm_flags |= PMC_F_NEEDS_LOGFILE;
1146
1147         pm->pm_flags |= PMC_F_ATTACH_DONE; /* mark as attached */
1148
1149         /* issue an attach event to a configured log file */
1150         if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE) {
1151                 if (p->p_flag & P_KPROC) {
1152                         fullpath = kernelname;
1153                         freepath = NULL;
1154                 } else {
1155                         pmc_getfilename(p->p_textvp, &fullpath, &freepath);
1156                         pmclog_process_pmcattach(pm, p->p_pid, fullpath);
1157                 }
1158                 free(freepath, M_TEMP);
1159                 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1160                         pmc_log_process_mappings(pm->pm_owner, p);
1161         }
1162
1163         return (0);
1164  fail:
1165         PROC_LOCK(p);
1166         p->p_flag &= ~P_HWPMC;
1167         PROC_UNLOCK(p);
1168         return (error);
1169 }
1170
1171 /*
1172  * Attach a process and optionally its children
1173  */
1174
1175 static int
1176 pmc_attach_process(struct proc *p, struct pmc *pm)
1177 {
1178         int error;
1179         struct proc *top;
1180
1181         sx_assert(&pmc_sx, SX_XLOCKED);
1182
1183         PMCDBG5(PRC,ATT,1, "attach pm=%p ri=%d proc=%p (%d, %s)", pm,
1184             PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
1185
1186
1187         /*
1188          * If this PMC successfully allowed a GETMSR operation
1189          * in the past, disallow further ATTACHes.
1190          */
1191
1192         if ((pm->pm_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0)
1193                 return EPERM;
1194
1195         if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
1196                 return pmc_attach_one_process(p, pm);
1197
1198         /*
1199          * Traverse all child processes, attaching them to
1200          * this PMC.
1201          */
1202
1203         sx_slock(&proctree_lock);
1204
1205         top = p;
1206
1207         for (;;) {
1208                 if ((error = pmc_attach_one_process(p, pm)) != 0)
1209                         break;
1210                 if (!LIST_EMPTY(&p->p_children))
1211                         p = LIST_FIRST(&p->p_children);
1212                 else for (;;) {
1213                         if (p == top)
1214                                 goto done;
1215                         if (LIST_NEXT(p, p_sibling)) {
1216                                 p = LIST_NEXT(p, p_sibling);
1217                                 break;
1218                         }
1219                         p = p->p_pptr;
1220                 }
1221         }
1222
1223         if (error)
1224                 (void) pmc_detach_process(top, pm);
1225
1226  done:
1227         sx_sunlock(&proctree_lock);
1228         return error;
1229 }
1230
1231 /*
1232  * Detach a process from a PMC.  If there are no other PMCs tracking
1233  * this process, remove the process structure from its hash table.  If
1234  * 'flags' contains PMC_FLAG_REMOVE, then free the process structure.
1235  */
1236
1237 static int
1238 pmc_detach_one_process(struct proc *p, struct pmc *pm, int flags)
1239 {
1240         int ri;
1241         struct pmc_process *pp;
1242
1243         sx_assert(&pmc_sx, SX_XLOCKED);
1244
1245         KASSERT(pm != NULL,
1246             ("[pmc,%d] null pm pointer", __LINE__));
1247
1248         ri = PMC_TO_ROWINDEX(pm);
1249
1250         PMCDBG6(PRC,ATT,2, "detach-one pm=%p ri=%d proc=%p (%d, %s) flags=0x%x",
1251             pm, ri, p, p->p_pid, p->p_comm, flags);
1252
1253         if ((pp = pmc_find_process_descriptor(p, 0)) == NULL)
1254                 return ESRCH;
1255
1256         if (pp->pp_pmcs[ri].pp_pmc != pm)
1257                 return EINVAL;
1258
1259         pmc_unlink_target_process(pm, pp);
1260
1261         /* Issue a detach entry if a log file is configured */
1262         if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE)
1263                 pmclog_process_pmcdetach(pm, p->p_pid);
1264
1265         /*
1266          * If there are no PMCs targeting this process, we remove its
1267          * descriptor from the target hash table and unset the P_HWPMC
1268          * flag in the struct proc.
1269          */
1270         KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= (int) md->pmd_npmc,
1271             ("[pmc,%d] Illegal refcnt %d for process struct %p",
1272                 __LINE__, pp->pp_refcnt, pp));
1273
1274         if (pp->pp_refcnt != 0) /* still a target of some PMC */
1275                 return 0;
1276
1277         pmc_remove_process_descriptor(pp);
1278
1279         if (flags & PMC_FLAG_REMOVE)
1280                 pmc_destroy_process_descriptor(pp);
1281
1282         PROC_LOCK(p);
1283         p->p_flag &= ~P_HWPMC;
1284         PROC_UNLOCK(p);
1285
1286         return 0;
1287 }
1288
1289 /*
1290  * Detach a process and optionally its descendants from a PMC.
1291  */
1292
1293 static int
1294 pmc_detach_process(struct proc *p, struct pmc *pm)
1295 {
1296         struct proc *top;
1297
1298         sx_assert(&pmc_sx, SX_XLOCKED);
1299
1300         PMCDBG5(PRC,ATT,1, "detach pm=%p ri=%d proc=%p (%d, %s)", pm,
1301             PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
1302
1303         if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
1304                 return pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE);
1305
1306         /*
1307          * Traverse all children, detaching them from this PMC.  We
1308          * ignore errors since we could be detaching a PMC from a
1309          * partially attached proc tree.
1310          */
1311
1312         sx_slock(&proctree_lock);
1313
1314         top = p;
1315
1316         for (;;) {
1317                 (void) pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE);
1318
1319                 if (!LIST_EMPTY(&p->p_children))
1320                         p = LIST_FIRST(&p->p_children);
1321                 else for (;;) {
1322                         if (p == top)
1323                                 goto done;
1324                         if (LIST_NEXT(p, p_sibling)) {
1325                                 p = LIST_NEXT(p, p_sibling);
1326                                 break;
1327                         }
1328                         p = p->p_pptr;
1329                 }
1330         }
1331
1332  done:
1333         sx_sunlock(&proctree_lock);
1334
1335         if (LIST_EMPTY(&pm->pm_targets))
1336                 pm->pm_flags &= ~PMC_F_ATTACH_DONE;
1337
1338         return 0;
1339 }
1340
1341
1342 /*
1343  * Thread context switch IN
1344  */
1345
1346 static void
1347 pmc_process_csw_in(struct thread *td)
1348 {
1349         int cpu;
1350         unsigned int adjri, ri;
1351         struct pmc *pm;
1352         struct proc *p;
1353         struct pmc_cpu *pc;
1354         struct pmc_hw *phw;
1355         pmc_value_t newvalue;
1356         struct pmc_process *pp;
1357         struct pmc_thread *pt;
1358         struct pmc_classdep *pcd;
1359
1360         p = td->td_proc;
1361         pt = NULL;
1362         if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE)) == NULL)
1363                 return;
1364
1365         KASSERT(pp->pp_proc == td->td_proc,
1366             ("[pmc,%d] not my thread state", __LINE__));
1367
1368         critical_enter(); /* no preemption from this point */
1369
1370         cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */
1371
1372         PMCDBG5(CSW,SWI,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p,
1373             p->p_pid, p->p_comm, pp);
1374
1375         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
1376             ("[pmc,%d] weird CPU id %d", __LINE__, cpu));
1377
1378         pc = pmc_pcpu[cpu];
1379
1380         for (ri = 0; ri < md->pmd_npmc; ri++) {
1381
1382                 if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL)
1383                         continue;
1384
1385                 KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)),
1386                     ("[pmc,%d] Target PMC in non-virtual mode (%d)",
1387                         __LINE__, PMC_TO_MODE(pm)));
1388
1389                 KASSERT(PMC_TO_ROWINDEX(pm) == ri,
1390                     ("[pmc,%d] Row index mismatch pmc %d != ri %d",
1391                         __LINE__, PMC_TO_ROWINDEX(pm), ri));
1392
1393                 /*
1394                  * Only PMCs that are marked as 'RUNNING' need
1395                  * be placed on hardware.
1396                  */
1397
1398                 if (pm->pm_state != PMC_STATE_RUNNING)
1399                         continue;
1400
1401                 /* increment PMC runcount */
1402                 counter_u64_add(pm->pm_runcount, 1);
1403
1404                 /* configure the HWPMC we are going to use. */
1405                 pcd = pmc_ri_to_classdep(md, ri, &adjri);
1406                 pcd->pcd_config_pmc(cpu, adjri, pm);
1407
1408                 phw = pc->pc_hwpmcs[ri];
1409
1410                 KASSERT(phw != NULL,
1411                     ("[pmc,%d] null hw pointer", __LINE__));
1412
1413                 KASSERT(phw->phw_pmc == pm,
1414                     ("[pmc,%d] hw->pmc %p != pmc %p", __LINE__,
1415                         phw->phw_pmc, pm));
1416
1417                 /*
1418                  * Write out saved value and start the PMC.
1419                  *
1420                  * Sampling PMCs use a per-thread value, while
1421                  * counting mode PMCs use a per-pmc value that is
1422                  * inherited across descendants.
1423                  */
1424                 if (PMC_TO_MODE(pm) == PMC_MODE_TS) {
1425                         if (pt == NULL)
1426                                 pt = pmc_find_thread_descriptor(pp, td,
1427                                     PMC_FLAG_NONE);
1428
1429                         KASSERT(pt != NULL,
1430                             ("[pmc,%d] No thread found for td=%p", __LINE__,
1431                             td));
1432
1433                         mtx_pool_lock_spin(pmc_mtxpool, pm);
1434
1435                         /*
1436                          * If we have a thread descriptor, use the per-thread
1437                          * counter in the descriptor. If not, we will use
1438                          * a per-process counter.
1439                          *
1440                          * TODO: Remove the per-process "safety net" once
1441                          * we have thoroughly tested that we don't hit the
1442                          * above assert.
1443                          */
1444                         if (pt != NULL) {
1445                                 if (pt->pt_pmcs[ri].pt_pmcval > 0)
1446                                         newvalue = pt->pt_pmcs[ri].pt_pmcval;
1447                                 else
1448                                         newvalue = pm->pm_sc.pm_reloadcount;
1449                         } else {
1450                                 /*
1451                                  * Use the saved value calculated after the most
1452                                  * recent time a thread using the shared counter
1453                                  * switched out. Reset the saved count in case
1454                                  * another thread from this process switches in
1455                                  * before any threads switch out.
1456                                  */
1457
1458                                 newvalue = pp->pp_pmcs[ri].pp_pmcval;
1459                                 pp->pp_pmcs[ri].pp_pmcval =
1460                                     pm->pm_sc.pm_reloadcount;
1461                         }
1462                         mtx_pool_unlock_spin(pmc_mtxpool, pm);
1463                         KASSERT(newvalue > 0 && newvalue <=
1464                             pm->pm_sc.pm_reloadcount,
1465                             ("[pmc,%d] pmcval outside of expected range cpu=%d "
1466                             "ri=%d pmcval=%jx pm_reloadcount=%jx", __LINE__,
1467                             cpu, ri, newvalue, pm->pm_sc.pm_reloadcount));
1468                 } else {
1469                         KASSERT(PMC_TO_MODE(pm) == PMC_MODE_TC,
1470                             ("[pmc,%d] illegal mode=%d", __LINE__,
1471                             PMC_TO_MODE(pm)));
1472                         mtx_pool_lock_spin(pmc_mtxpool, pm);
1473                         newvalue = PMC_PCPU_SAVED(cpu, ri) =
1474                             pm->pm_gv.pm_savedvalue;
1475                         mtx_pool_unlock_spin(pmc_mtxpool, pm);
1476                 }
1477
1478                 PMCDBG3(CSW,SWI,1,"cpu=%d ri=%d new=%jd", cpu, ri, newvalue);
1479
1480                 pcd->pcd_write_pmc(cpu, adjri, newvalue);
1481
1482                 /* If a sampling mode PMC, reset stalled state. */
1483                 if (PMC_TO_MODE(pm) == PMC_MODE_TS)
1484                         pm->pm_pcpu_state[cpu].pps_stalled = 0;
1485
1486                 /* Indicate that we desire this to run. */
1487                 pm->pm_pcpu_state[cpu].pps_cpustate = 1;
1488
1489                 /* Start the PMC. */
1490                 pcd->pcd_start_pmc(cpu, adjri);
1491         }
1492
1493         /*
1494          * perform any other architecture/cpu dependent thread
1495          * switch-in actions.
1496          */
1497
1498         (void) (*md->pmd_switch_in)(pc, pp);
1499
1500         critical_exit();
1501
1502 }
1503
1504 /*
1505  * Thread context switch OUT.
1506  */
1507
1508 static void
1509 pmc_process_csw_out(struct thread *td)
1510 {
1511         int cpu;
1512         int64_t tmp;
1513         struct pmc *pm;
1514         struct proc *p;
1515         enum pmc_mode mode;
1516         struct pmc_cpu *pc;
1517         pmc_value_t newvalue;
1518         unsigned int adjri, ri;
1519         struct pmc_process *pp;
1520         struct pmc_thread *pt = NULL;
1521         struct pmc_classdep *pcd;
1522
1523
1524         /*
1525          * Locate our process descriptor; this may be NULL if
1526          * this process is exiting and we have already removed
1527          * the process from the target process table.
1528          *
1529          * Note that due to kernel preemption, multiple
1530          * context switches may happen while the process is
1531          * exiting.
1532          *
1533          * Note also that if the target process cannot be
1534          * found we still need to deconfigure any PMCs that
1535          * are currently running on hardware.
1536          */
1537
1538         p = td->td_proc;
1539         pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE);
1540
1541         /*
1542          * save PMCs
1543          */
1544
1545         critical_enter();
1546
1547         cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */
1548
1549         PMCDBG5(CSW,SWO,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p,
1550             p->p_pid, p->p_comm, pp);
1551
1552         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
1553             ("[pmc,%d weird CPU id %d", __LINE__, cpu));
1554
1555         pc = pmc_pcpu[cpu];
1556
1557         /*
1558          * When a PMC gets unlinked from a target PMC, it will
1559          * be removed from the target's pp_pmc[] array.
1560          *
1561          * However, on a MP system, the target could have been
1562          * executing on another CPU at the time of the unlink.
1563          * So, at context switch OUT time, we need to look at
1564          * the hardware to determine if a PMC is scheduled on
1565          * it.
1566          */
1567
1568         for (ri = 0; ri < md->pmd_npmc; ri++) {
1569
1570                 pcd = pmc_ri_to_classdep(md, ri, &adjri);
1571                 pm  = NULL;
1572                 (void) (*pcd->pcd_get_config)(cpu, adjri, &pm);
1573
1574                 if (pm == NULL) /* nothing at this row index */
1575                         continue;
1576
1577                 mode = PMC_TO_MODE(pm);
1578                 if (!PMC_IS_VIRTUAL_MODE(mode))
1579                         continue; /* not a process virtual PMC */
1580
1581                 KASSERT(PMC_TO_ROWINDEX(pm) == ri,
1582                     ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
1583                         __LINE__, PMC_TO_ROWINDEX(pm), ri));
1584
1585                 /*
1586                  * Change desired state, and then stop if not stalled.
1587                  * This two-step dance should avoid race conditions where
1588                  * an interrupt re-enables the PMC after this code has
1589                  * already checked the pm_stalled flag.
1590                  */
1591                 pm->pm_pcpu_state[cpu].pps_cpustate = 0;
1592                 if (pm->pm_pcpu_state[cpu].pps_stalled == 0)
1593                         pcd->pcd_stop_pmc(cpu, adjri);
1594
1595                 /* reduce this PMC's runcount */
1596                 counter_u64_add(pm->pm_runcount, -1);
1597
1598                 /*
1599                  * If this PMC is associated with this process,
1600                  * save the reading.
1601                  */
1602
1603                 if (pm->pm_state != PMC_STATE_DELETED && pp != NULL &&
1604                     pp->pp_pmcs[ri].pp_pmc != NULL) {
1605                         KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
1606                             ("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__,
1607                                 pm, ri, pp->pp_pmcs[ri].pp_pmc));
1608
1609                         KASSERT(pp->pp_refcnt > 0,
1610                             ("[pmc,%d] pp refcnt = %d", __LINE__,
1611                                 pp->pp_refcnt));
1612
1613                         pcd->pcd_read_pmc(cpu, adjri, &newvalue);
1614
1615                         if (mode == PMC_MODE_TS) {
1616                                 PMCDBG3(CSW,SWO,1,"cpu=%d ri=%d val=%jd (samp)",
1617                                     cpu, ri, newvalue);
1618
1619                                 if (pt == NULL)
1620                                         pt = pmc_find_thread_descriptor(pp, td,
1621                                             PMC_FLAG_NONE);
1622
1623                                 KASSERT(pt != NULL,
1624                                     ("[pmc,%d] No thread found for td=%p",
1625                                     __LINE__, td));
1626
1627                                 mtx_pool_lock_spin(pmc_mtxpool, pm);
1628
1629                                 /*
1630                                  * If we have a thread descriptor, save the
1631                                  * per-thread counter in the descriptor. If not,
1632                                  * we will update the per-process counter.
1633                                  *
1634                                  * TODO: Remove the per-process "safety net"
1635                                  * once we have thoroughly tested that we
1636                                  * don't hit the above assert.
1637                                  */
1638                                 if (pt != NULL)
1639                                         pt->pt_pmcs[ri].pt_pmcval = newvalue;
1640                                 else {
1641                                         /*
1642                                          * For sampling process-virtual PMCs,
1643                                          * newvalue is the number of events to
1644                                          * be seen until the next sampling
1645                                          * interrupt. We can just add the events
1646                                          * left from this invocation to the
1647                                          * counter, then adjust in case we
1648                                          * overflow our range.
1649                                          *
1650                                          * (Recall that we reload the counter
1651                                          * every time we use it.)
1652                                          */
1653                                         pp->pp_pmcs[ri].pp_pmcval += newvalue;
1654                                         if (pp->pp_pmcs[ri].pp_pmcval >
1655                                             pm->pm_sc.pm_reloadcount)
1656                                                 pp->pp_pmcs[ri].pp_pmcval -=
1657                                                     pm->pm_sc.pm_reloadcount;
1658                                 }
1659                                 mtx_pool_unlock_spin(pmc_mtxpool, pm);
1660                         } else {
1661                                 tmp = newvalue - PMC_PCPU_SAVED(cpu,ri);
1662
1663                                 PMCDBG3(CSW,SWO,1,"cpu=%d ri=%d tmp=%jd (count)",
1664                                     cpu, ri, tmp);
1665
1666                                 /*
1667                                  * For counting process-virtual PMCs,
1668                                  * we expect the count to be
1669                                  * increasing monotonically, modulo a 64
1670                                  * bit wraparound.
1671                                  */
1672                                 KASSERT(tmp >= 0,
1673                                     ("[pmc,%d] negative increment cpu=%d "
1674                                      "ri=%d newvalue=%jx saved=%jx "
1675                                      "incr=%jx", __LINE__, cpu, ri,
1676                                      newvalue, PMC_PCPU_SAVED(cpu,ri), tmp));
1677
1678                                 mtx_pool_lock_spin(pmc_mtxpool, pm);
1679                                 pm->pm_gv.pm_savedvalue += tmp;
1680                                 pp->pp_pmcs[ri].pp_pmcval += tmp;
1681                                 mtx_pool_unlock_spin(pmc_mtxpool, pm);
1682
1683                                 if (pm->pm_flags & PMC_F_LOG_PROCCSW)
1684                                         pmclog_process_proccsw(pm, pp, tmp, td);
1685                         }
1686                 }
1687
1688                 /* mark hardware as free */
1689                 pcd->pcd_config_pmc(cpu, adjri, NULL);
1690         }
1691
1692         /*
1693          * perform any other architecture/cpu dependent thread
1694          * switch out functions.
1695          */
1696
1697         (void) (*md->pmd_switch_out)(pc, pp);
1698
1699         critical_exit();
1700 }
1701
1702 /*
1703  * A new thread for a process.
1704  */
1705 static void
1706 pmc_process_thread_add(struct thread *td)
1707 {
1708         struct pmc_process *pmc;
1709
1710         pmc = pmc_find_process_descriptor(td->td_proc, PMC_FLAG_NONE);
1711         if (pmc != NULL)
1712                 pmc_find_thread_descriptor(pmc, td, PMC_FLAG_ALLOCATE);
1713 }
1714
1715 /*
1716  * A thread delete for a process.
1717  */
1718 static void
1719 pmc_process_thread_delete(struct thread *td)
1720 {
1721         struct pmc_process *pmc;
1722
1723         pmc = pmc_find_process_descriptor(td->td_proc, PMC_FLAG_NONE);
1724         if (pmc != NULL)
1725                 pmc_thread_descriptor_pool_free(pmc_find_thread_descriptor(pmc,
1726                     td, PMC_FLAG_REMOVE));
1727 }
1728
1729 /*
1730  * A userret() call for a thread.
1731  */
1732 static void
1733 pmc_process_thread_userret(struct thread *td)
1734 {
1735         sched_pin();
1736         pmc_capture_user_callchain(curcpu, PMC_UR, td->td_frame);
1737         sched_unpin();
1738 }
1739
1740 /*
1741  * A mapping change for a process.
1742  */
1743
1744 static void
1745 pmc_process_mmap(struct thread *td, struct pmckern_map_in *pkm)
1746 {
1747         int ri;
1748         pid_t pid;
1749         char *fullpath, *freepath;
1750         const struct pmc *pm;
1751         struct pmc_owner *po;
1752         const struct pmc_process *pp;
1753
1754         freepath = fullpath = NULL;
1755         MPASS(!in_epoch());
1756         pmc_getfilename((struct vnode *) pkm->pm_file, &fullpath, &freepath);
1757
1758         pid = td->td_proc->p_pid;
1759
1760         epoch_enter_preempt(global_epoch_preempt);
1761         /* Inform owners of all system-wide sampling PMCs. */
1762         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
1763             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
1764                         pmclog_process_map_in(po, pid, pkm->pm_address, fullpath);
1765
1766         if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL)
1767                 goto done;
1768
1769         /*
1770          * Inform sampling PMC owners tracking this process.
1771          */
1772         for (ri = 0; ri < md->pmd_npmc; ri++)
1773                 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL &&
1774                     PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1775                         pmclog_process_map_in(pm->pm_owner,
1776                             pid, pkm->pm_address, fullpath);
1777
1778   done:
1779         if (freepath)
1780                 free(freepath, M_TEMP);
1781         epoch_exit_preempt(global_epoch_preempt);
1782 }
1783
1784
1785 /*
1786  * Log an munmap request.
1787  */
1788
1789 static void
1790 pmc_process_munmap(struct thread *td, struct pmckern_map_out *pkm)
1791 {
1792         int ri;
1793         pid_t pid;
1794         struct pmc_owner *po;
1795         const struct pmc *pm;
1796         const struct pmc_process *pp;
1797
1798         pid = td->td_proc->p_pid;
1799
1800         epoch_enter_preempt(global_epoch_preempt);
1801         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
1802             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
1803                 pmclog_process_map_out(po, pid, pkm->pm_address,
1804                     pkm->pm_address + pkm->pm_size);
1805         epoch_exit_preempt(global_epoch_preempt);
1806
1807         if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL)
1808                 return;
1809
1810         for (ri = 0; ri < md->pmd_npmc; ri++)
1811                 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL &&
1812                     PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1813                         pmclog_process_map_out(pm->pm_owner, pid,
1814                             pkm->pm_address, pkm->pm_address + pkm->pm_size);
1815 }
1816
1817 /*
1818  * Log mapping information about the kernel.
1819  */
1820
1821 static void
1822 pmc_log_kernel_mappings(struct pmc *pm)
1823 {
1824         struct pmc_owner *po;
1825         struct pmckern_map_in *km, *kmbase;
1826
1827         MPASS(in_epoch() || sx_xlocked(&pmc_sx));
1828         KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
1829             ("[pmc,%d] non-sampling PMC (%p) desires mapping information",
1830                 __LINE__, (void *) pm));
1831
1832         po = pm->pm_owner;
1833
1834         if (po->po_flags & PMC_PO_INITIAL_MAPPINGS_DONE)
1835                 return;
1836         if (PMC_TO_MODE(pm) == PMC_MODE_SS)
1837                 pmc_process_allproc(pm);
1838         /*
1839          * Log the current set of kernel modules.
1840          */
1841         kmbase = linker_hwpmc_list_objects();
1842         for (km = kmbase; km->pm_file != NULL; km++) {
1843                 PMCDBG2(LOG,REG,1,"%s %p", (char *) km->pm_file,
1844                     (void *) km->pm_address);
1845                 pmclog_process_map_in(po, (pid_t) -1, km->pm_address,
1846                     km->pm_file);
1847         }
1848         free(kmbase, M_LINKER);
1849
1850         po->po_flags |= PMC_PO_INITIAL_MAPPINGS_DONE;
1851 }
1852
1853 /*
1854  * Log the mappings for a single process.
1855  */
1856
1857 static void
1858 pmc_log_process_mappings(struct pmc_owner *po, struct proc *p)
1859 {
1860         vm_map_t map;
1861         struct vnode *vp;
1862         struct vmspace *vm;
1863         vm_map_entry_t entry;
1864         vm_offset_t last_end;
1865         u_int last_timestamp;
1866         struct vnode *last_vp;
1867         vm_offset_t start_addr;
1868         vm_object_t obj, lobj, tobj;
1869         char *fullpath, *freepath;
1870
1871         last_vp = NULL;
1872         last_end = (vm_offset_t) 0;
1873         fullpath = freepath = NULL;
1874
1875         if ((vm = vmspace_acquire_ref(p)) == NULL)
1876                 return;
1877
1878         map = &vm->vm_map;
1879         vm_map_lock_read(map);
1880
1881         for (entry = map->header.next; entry != &map->header; entry = entry->next) {
1882
1883                 if (entry == NULL) {
1884                         PMCDBG2(LOG,OPS,2, "hwpmc: vm_map entry unexpectedly "
1885                             "NULL! pid=%d vm_map=%p\n", p->p_pid, map);
1886                         break;
1887                 }
1888
1889                 /*
1890                  * We only care about executable map entries.
1891                  */
1892                 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) ||
1893                     !(entry->protection & VM_PROT_EXECUTE) ||
1894                     (entry->object.vm_object == NULL)) {
1895                         continue;
1896                 }
1897
1898                 obj = entry->object.vm_object;
1899                 VM_OBJECT_RLOCK(obj);
1900
1901                 /*
1902                  * Walk the backing_object list to find the base
1903                  * (non-shadowed) vm_object.
1904                  */
1905                 for (lobj = tobj = obj; tobj != NULL; tobj = tobj->backing_object) {
1906                         if (tobj != obj)
1907                                 VM_OBJECT_RLOCK(tobj);
1908                         if (lobj != obj)
1909                                 VM_OBJECT_RUNLOCK(lobj);
1910                         lobj = tobj;
1911                 }
1912
1913                 /*
1914                  * At this point lobj is the base vm_object and it is locked.
1915                  */
1916                 if (lobj == NULL) {
1917                         PMCDBG3(LOG,OPS,2, "hwpmc: lobj unexpectedly NULL! pid=%d "
1918                             "vm_map=%p vm_obj=%p\n", p->p_pid, map, obj);
1919                         VM_OBJECT_RUNLOCK(obj);
1920                         continue;
1921                 }
1922
1923                 vp = vm_object_vnode(lobj);
1924                 if (vp == NULL) {
1925                         if (lobj != obj)
1926                                 VM_OBJECT_RUNLOCK(lobj);
1927                         VM_OBJECT_RUNLOCK(obj);
1928                         continue;
1929                 }
1930
1931                 /*
1932                  * Skip contiguous regions that point to the same
1933                  * vnode, so we don't emit redundant MAP-IN
1934                  * directives.
1935                  */
1936                 if (entry->start == last_end && vp == last_vp) {
1937                         last_end = entry->end;
1938                         if (lobj != obj)
1939                                 VM_OBJECT_RUNLOCK(lobj);
1940                         VM_OBJECT_RUNLOCK(obj);
1941                         continue;
1942                 }
1943
1944                 /*
1945                  * We don't want to keep the proc's vm_map or this
1946                  * vm_object locked while we walk the pathname, since
1947                  * vn_fullpath() can sleep.  However, if we drop the
1948                  * lock, it's possible for concurrent activity to
1949                  * modify the vm_map list.  To protect against this,
1950                  * we save the vm_map timestamp before we release the
1951                  * lock, and check it after we reacquire the lock
1952                  * below.
1953                  */
1954                 start_addr = entry->start;
1955                 last_end = entry->end;
1956                 last_timestamp = map->timestamp;
1957                 vm_map_unlock_read(map);
1958
1959                 vref(vp);
1960                 if (lobj != obj)
1961                         VM_OBJECT_RUNLOCK(lobj);
1962
1963                 VM_OBJECT_RUNLOCK(obj);
1964
1965                 freepath = NULL;
1966                 pmc_getfilename(vp, &fullpath, &freepath);
1967                 last_vp = vp;
1968
1969                 vrele(vp);
1970
1971                 vp = NULL;
1972                 pmclog_process_map_in(po, p->p_pid, start_addr, fullpath);
1973                 if (freepath)
1974                         free(freepath, M_TEMP);
1975
1976                 vm_map_lock_read(map);
1977
1978                 /*
1979                  * If our saved timestamp doesn't match, this means
1980                  * that the vm_map was modified out from under us and
1981                  * we can't trust our current "entry" pointer.  Do a
1982                  * new lookup for this entry.  If there is no entry
1983                  * for this address range, vm_map_lookup_entry() will
1984                  * return the previous one, so we always want to go to
1985                  * entry->next on the next loop iteration.
1986                  *
1987                  * There is an edge condition here that can occur if
1988                  * there is no entry at or before this address.  In
1989                  * this situation, vm_map_lookup_entry returns
1990                  * &map->header, which would cause our loop to abort
1991                  * without processing the rest of the map.  However,
1992                  * in practice this will never happen for process
1993                  * vm_map.  This is because the executable's text
1994                  * segment is the first mapping in the proc's address
1995                  * space, and this mapping is never removed until the
1996                  * process exits, so there will always be a non-header
1997                  * entry at or before the requested address for
1998                  * vm_map_lookup_entry to return.
1999                  */
2000                 if (map->timestamp != last_timestamp)
2001                         vm_map_lookup_entry(map, last_end - 1, &entry);
2002         }
2003
2004         vm_map_unlock_read(map);
2005         vmspace_free(vm);
2006         return;
2007 }
2008
2009 /*
2010  * Log mappings for all processes in the system.
2011  */
2012
2013 static void
2014 pmc_log_all_process_mappings(struct pmc_owner *po)
2015 {
2016         struct proc *p, *top;
2017
2018         sx_assert(&pmc_sx, SX_XLOCKED);
2019
2020         if ((p = pfind(1)) == NULL)
2021                 panic("[pmc,%d] Cannot find init", __LINE__);
2022
2023         PROC_UNLOCK(p);
2024
2025         sx_slock(&proctree_lock);
2026
2027         top = p;
2028
2029         for (;;) {
2030                 pmc_log_process_mappings(po, p);
2031                 if (!LIST_EMPTY(&p->p_children))
2032                         p = LIST_FIRST(&p->p_children);
2033                 else for (;;) {
2034                         if (p == top)
2035                                 goto done;
2036                         if (LIST_NEXT(p, p_sibling)) {
2037                                 p = LIST_NEXT(p, p_sibling);
2038                                 break;
2039                         }
2040                         p = p->p_pptr;
2041                 }
2042         }
2043  done:
2044         sx_sunlock(&proctree_lock);
2045 }
2046
2047 /*
2048  * The 'hook' invoked from the kernel proper
2049  */
2050
2051
2052 #ifdef  HWPMC_DEBUG
2053 const char *pmc_hooknames[] = {
2054         /* these strings correspond to PMC_FN_* in <sys/pmckern.h> */
2055         "",
2056         "EXEC",
2057         "CSW-IN",
2058         "CSW-OUT",
2059         "SAMPLE",
2060         "UNUSED1",
2061         "UNUSED2",
2062         "MMAP",
2063         "MUNMAP",
2064         "CALLCHAIN-NMI",
2065         "CALLCHAIN-SOFT",
2066         "SOFTSAMPLING",
2067         "THR-CREATE",
2068         "THR-EXIT",
2069         "THR-USERRET",
2070         "THR-CREATE-LOG",
2071         "THR-EXIT-LOG",
2072         "PROC-CREATE-LOG"
2073 };
2074 #endif
2075
2076 static int
2077 pmc_hook_handler(struct thread *td, int function, void *arg)
2078 {
2079         int cpu;
2080
2081         PMCDBG4(MOD,PMH,1, "hook td=%p func=%d \"%s\" arg=%p", td, function,
2082             pmc_hooknames[function], arg);
2083
2084         switch (function)
2085         {
2086
2087         /*
2088          * Process exec()
2089          */
2090
2091         case PMC_FN_PROCESS_EXEC:
2092         {
2093                 char *fullpath, *freepath;
2094                 unsigned int ri;
2095                 int is_using_hwpmcs;
2096                 struct pmc *pm;
2097                 struct proc *p;
2098                 struct pmc_owner *po;
2099                 struct pmc_process *pp;
2100                 struct pmckern_procexec *pk;
2101
2102                 sx_assert(&pmc_sx, SX_XLOCKED);
2103
2104                 p = td->td_proc;
2105                 pmc_getfilename(p->p_textvp, &fullpath, &freepath);
2106
2107                 pk = (struct pmckern_procexec *) arg;
2108
2109                 epoch_enter_preempt(global_epoch_preempt);
2110                 /* Inform owners of SS mode PMCs of the exec event. */
2111                 CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
2112                     if (po->po_flags & PMC_PO_OWNS_LOGFILE)
2113                             pmclog_process_procexec(po, PMC_ID_INVALID,
2114                                 p->p_pid, pk->pm_entryaddr, fullpath);
2115                 epoch_exit_preempt(global_epoch_preempt);
2116
2117                 PROC_LOCK(p);
2118                 is_using_hwpmcs = p->p_flag & P_HWPMC;
2119                 PROC_UNLOCK(p);
2120
2121                 if (!is_using_hwpmcs) {
2122                         if (freepath)
2123                                 free(freepath, M_TEMP);
2124                         break;
2125                 }
2126
2127                 /*
2128                  * PMCs are not inherited across an exec():  remove any
2129                  * PMCs that this process is the owner of.
2130                  */
2131
2132                 if ((po = pmc_find_owner_descriptor(p)) != NULL) {
2133                         pmc_remove_owner(po);
2134                         pmc_destroy_owner_descriptor(po);
2135                 }
2136
2137                 /*
2138                  * If the process being exec'ed is not the target of any
2139                  * PMC, we are done.
2140                  */
2141                 if ((pp = pmc_find_process_descriptor(p, 0)) == NULL) {
2142                         if (freepath)
2143                                 free(freepath, M_TEMP);
2144                         break;
2145                 }
2146
2147                 /*
2148                  * Log the exec event to all monitoring owners.  Skip
2149                  * owners who have already received the event because
2150                  * they had system sampling PMCs active.
2151                  */
2152                 for (ri = 0; ri < md->pmd_npmc; ri++)
2153                         if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) {
2154                                 po = pm->pm_owner;
2155                                 if (po->po_sscount == 0 &&
2156                                     po->po_flags & PMC_PO_OWNS_LOGFILE)
2157                                         pmclog_process_procexec(po, pm->pm_id,
2158                                             p->p_pid, pk->pm_entryaddr,
2159                                             fullpath);
2160                         }
2161
2162                 if (freepath)
2163                         free(freepath, M_TEMP);
2164
2165
2166                 PMCDBG4(PRC,EXC,1, "exec proc=%p (%d, %s) cred-changed=%d",
2167                     p, p->p_pid, p->p_comm, pk->pm_credentialschanged);
2168
2169                 if (pk->pm_credentialschanged == 0) /* no change */
2170                         break;
2171
2172                 /*
2173                  * If the newly exec()'ed process has a different credential
2174                  * than before, allow it to be the target of a PMC only if
2175                  * the PMC's owner has sufficient privilege.
2176                  */
2177
2178                 for (ri = 0; ri < md->pmd_npmc; ri++)
2179                         if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL)
2180                                 if (pmc_can_attach(pm, td->td_proc) != 0)
2181                                         pmc_detach_one_process(td->td_proc,
2182                                             pm, PMC_FLAG_NONE);
2183
2184                 KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= (int) md->pmd_npmc,
2185                     ("[pmc,%d] Illegal ref count %d on pp %p", __LINE__,
2186                         pp->pp_refcnt, pp));
2187
2188                 /*
2189                  * If this process is no longer the target of any
2190                  * PMCs, we can remove the process entry and free
2191                  * up space.
2192                  */
2193
2194                 if (pp->pp_refcnt == 0) {
2195                         pmc_remove_process_descriptor(pp);
2196                         pmc_destroy_process_descriptor(pp);
2197                         break;
2198                 }
2199
2200         }
2201         break;
2202
2203         case PMC_FN_CSW_IN:
2204                 pmc_process_csw_in(td);
2205                 break;
2206
2207         case PMC_FN_CSW_OUT:
2208                 pmc_process_csw_out(td);
2209                 break;
2210
2211         /*
2212          * Process accumulated PC samples.
2213          *
2214          * This function is expected to be called by hardclock() for
2215          * each CPU that has accumulated PC samples.
2216          *
2217          * This function is to be executed on the CPU whose samples
2218          * are being processed.
2219          */
2220         case PMC_FN_DO_SAMPLES:
2221
2222                 /*
2223                  * Clear the cpu specific bit in the CPU mask before
2224                  * do the rest of the processing.  If the NMI handler
2225                  * gets invoked after the "atomic_clear_int()" call
2226                  * below but before "pmc_process_samples()" gets
2227                  * around to processing the interrupt, then we will
2228                  * come back here at the next hardclock() tick (and
2229                  * may find nothing to do if "pmc_process_samples()"
2230                  * had already processed the interrupt).  We don't
2231                  * lose the interrupt sample.
2232                  */
2233                 DPCPU_SET(pmc_sampled, 0);
2234                 cpu = PCPU_GET(cpuid);
2235                 pmc_process_samples(cpu, PMC_HR);
2236                 pmc_process_samples(cpu, PMC_SR);
2237                 pmc_process_samples(cpu, PMC_UR);
2238                 break;
2239
2240         case PMC_FN_MMAP:
2241                 pmc_process_mmap(td, (struct pmckern_map_in *) arg);
2242                 break;
2243
2244         case PMC_FN_MUNMAP:
2245                 MPASS(in_epoch() || sx_xlocked(&pmc_sx));
2246                 pmc_process_munmap(td, (struct pmckern_map_out *) arg);
2247                 break;
2248
2249         case PMC_FN_PROC_CREATE_LOG:
2250                 pmc_process_proccreate((struct proc *)arg);
2251                 break;
2252
2253         case PMC_FN_USER_CALLCHAIN:
2254                 /*
2255                  * Record a call chain.
2256                  */
2257                 KASSERT(td == curthread, ("[pmc,%d] td != curthread",
2258                     __LINE__));
2259
2260                 pmc_capture_user_callchain(PCPU_GET(cpuid), PMC_HR,
2261                     (struct trapframe *) arg);
2262
2263                 KASSERT(td->td_pinned == 1,
2264                         ("[pmc,%d] invalid td_pinned value", __LINE__));
2265                 sched_unpin();  /* Can migrate safely now. */
2266
2267                 td->td_pflags &= ~TDP_CALLCHAIN;
2268                 break;
2269
2270         case PMC_FN_USER_CALLCHAIN_SOFT:
2271                 /*
2272                  * Record a call chain.
2273                  */
2274                 KASSERT(td == curthread, ("[pmc,%d] td != curthread",
2275                     __LINE__));
2276
2277                 cpu = PCPU_GET(cpuid);
2278                 pmc_capture_user_callchain(cpu, PMC_SR,
2279                     (struct trapframe *) arg);
2280
2281                 KASSERT(td->td_pinned == 1,
2282                     ("[pmc,%d] invalid td_pinned value", __LINE__));
2283
2284                 sched_unpin();  /* Can migrate safely now. */
2285
2286                 td->td_pflags &= ~TDP_CALLCHAIN;
2287                 break;
2288
2289         case PMC_FN_SOFT_SAMPLING:
2290                 /*
2291                  * Call soft PMC sampling intr.
2292                  */
2293                 pmc_soft_intr((struct pmckern_soft *) arg);
2294                 break;
2295
2296         case PMC_FN_THR_CREATE:
2297                 pmc_process_thread_add(td);
2298                 pmc_process_threadcreate(td);
2299                 break;
2300
2301         case PMC_FN_THR_CREATE_LOG:
2302                 pmc_process_threadcreate(td);
2303                 break;
2304
2305         case PMC_FN_THR_EXIT:
2306                 KASSERT(td == curthread, ("[pmc,%d] td != curthread",
2307                     __LINE__));
2308                 pmc_process_thread_delete(td);
2309                 pmc_process_threadexit(td);
2310                 break;
2311         case PMC_FN_THR_EXIT_LOG:
2312                 pmc_process_threadexit(td);
2313                 break;
2314         case PMC_FN_THR_USERRET:
2315                 KASSERT(td == curthread, ("[pmc,%d] td != curthread",
2316                     __LINE__));
2317                 pmc_process_thread_userret(td);
2318                 break;
2319
2320         default:
2321 #ifdef  HWPMC_DEBUG
2322                 KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function));
2323 #endif
2324                 break;
2325
2326         }
2327
2328         return 0;
2329 }
2330
2331 /*
2332  * allocate a 'struct pmc_owner' descriptor in the owner hash table.
2333  */
2334
2335 static struct pmc_owner *
2336 pmc_allocate_owner_descriptor(struct proc *p)
2337 {
2338         uint32_t hindex;
2339         struct pmc_owner *po;
2340         struct pmc_ownerhash *poh;
2341
2342         hindex = PMC_HASH_PTR(p, pmc_ownerhashmask);
2343         poh = &pmc_ownerhash[hindex];
2344
2345         /* allocate space for N pointers and one descriptor struct */
2346         po = malloc(sizeof(struct pmc_owner), M_PMC, M_WAITOK|M_ZERO);
2347         po->po_owner = p;
2348         LIST_INSERT_HEAD(poh, po, po_next); /* insert into hash table */
2349
2350         TAILQ_INIT(&po->po_logbuffers);
2351         mtx_init(&po->po_mtx, "pmc-owner-mtx", "pmc-per-proc", MTX_SPIN);
2352
2353         PMCDBG4(OWN,ALL,1, "allocate-owner proc=%p (%d, %s) pmc-owner=%p",
2354             p, p->p_pid, p->p_comm, po);
2355
2356         return po;
2357 }
2358
2359 static void
2360 pmc_destroy_owner_descriptor(struct pmc_owner *po)
2361 {
2362
2363         PMCDBG4(OWN,REL,1, "destroy-owner po=%p proc=%p (%d, %s)",
2364             po, po->po_owner, po->po_owner->p_pid, po->po_owner->p_comm);
2365
2366         mtx_destroy(&po->po_mtx);
2367         free(po, M_PMC);
2368 }
2369
2370 /*
2371  * Allocate a thread descriptor from the free pool.
2372  *
2373  * NOTE: This *can* return NULL.
2374  */
2375 static struct pmc_thread *
2376 pmc_thread_descriptor_pool_alloc(void)
2377 {
2378         struct pmc_thread *pt;
2379
2380         mtx_lock_spin(&pmc_threadfreelist_mtx);
2381         if ((pt = LIST_FIRST(&pmc_threadfreelist)) != NULL) {
2382                 LIST_REMOVE(pt, pt_next);
2383                 pmc_threadfreelist_entries--;
2384         }
2385         mtx_unlock_spin(&pmc_threadfreelist_mtx);
2386
2387         return (pt);
2388 }
2389
2390 /*
2391  * Add a thread descriptor to the free pool. We use this instead of free()
2392  * to maintain a cache of free entries. Additionally, we can safely call
2393  * this function when we cannot call free(), such as in a critical section.
2394  *
2395  */
2396 static void
2397 pmc_thread_descriptor_pool_free(struct pmc_thread *pt)
2398 {
2399
2400         if (pt == NULL)
2401                 return;
2402
2403         memset(pt, 0, THREADENTRY_SIZE);
2404         mtx_lock_spin(&pmc_threadfreelist_mtx);
2405         LIST_INSERT_HEAD(&pmc_threadfreelist, pt, pt_next);
2406         pmc_threadfreelist_entries++;
2407         if (pmc_threadfreelist_entries > pmc_threadfreelist_max)
2408                 GROUPTASK_ENQUEUE(&free_gtask);
2409         mtx_unlock_spin(&pmc_threadfreelist_mtx);
2410 }
2411
2412 /*
2413  * A callout to manage the free list.
2414  */
2415 static void
2416 pmc_thread_descriptor_pool_free_task(void *arg __unused)
2417 {
2418         struct pmc_thread *pt;
2419         LIST_HEAD(, pmc_thread) tmplist;
2420         int delta;
2421
2422         LIST_INIT(&tmplist);
2423         /* Determine what changes, if any, we need to make. */
2424         mtx_lock_spin(&pmc_threadfreelist_mtx);
2425         delta = pmc_threadfreelist_entries - pmc_threadfreelist_max;
2426         while (delta > 0 &&
2427                    (pt = LIST_FIRST(&pmc_threadfreelist)) != NULL) {
2428                 delta--;
2429                 LIST_REMOVE(pt, pt_next);
2430                 LIST_INSERT_HEAD(&tmplist, pt, pt_next);
2431         }
2432         mtx_unlock_spin(&pmc_threadfreelist_mtx);
2433
2434         /* If there are entries to free, free them. */
2435         while (!LIST_EMPTY(&tmplist)) {
2436                 pt = LIST_FIRST(&tmplist);
2437                 LIST_REMOVE(pt, pt_next);
2438                 free(pt, M_PMC);
2439         }
2440 }
2441
2442 /*
2443  * Drain the thread free pool, freeing all allocations.
2444  */
2445 static void
2446 pmc_thread_descriptor_pool_drain()
2447 {
2448         struct pmc_thread *pt, *next;
2449
2450         LIST_FOREACH_SAFE(pt, &pmc_threadfreelist, pt_next, next) {
2451                 LIST_REMOVE(pt, pt_next);
2452                 free(pt, M_PMC);
2453         }
2454 }
2455
2456 /*
2457  * find the descriptor corresponding to thread 'td', adding or removing it
2458  * as specified by 'mode'.
2459  *
2460  * Note that this supports additional mode flags in addition to those
2461  * supported by pmc_find_process_descriptor():
2462  * PMC_FLAG_NOWAIT: Causes the function to not wait for mallocs.
2463  *     This makes it safe to call while holding certain other locks.
2464  */
2465
2466 static struct pmc_thread *
2467 pmc_find_thread_descriptor(struct pmc_process *pp, struct thread *td,
2468     uint32_t mode)
2469 {
2470         struct pmc_thread *pt = NULL, *ptnew = NULL;
2471         int wait_flag;
2472
2473         KASSERT(td != NULL, ("[pmc,%d] called to add NULL td", __LINE__));
2474
2475         /*
2476          * Pre-allocate memory in the PMC_FLAG_ALLOCATE case prior to
2477          * acquiring the lock.
2478          */
2479         if (mode & PMC_FLAG_ALLOCATE) {
2480                 if ((ptnew = pmc_thread_descriptor_pool_alloc()) == NULL) {
2481                         wait_flag = M_WAITOK;
2482                         if ((mode & PMC_FLAG_NOWAIT) || in_epoch())
2483                                 wait_flag = M_NOWAIT;
2484
2485                         ptnew = malloc(THREADENTRY_SIZE, M_PMC,
2486                             wait_flag|M_ZERO);
2487                 }
2488         }
2489
2490         mtx_lock_spin(pp->pp_tdslock);
2491
2492         LIST_FOREACH(pt, &pp->pp_tds, pt_next)
2493                 if (pt->pt_td == td)
2494                         break;
2495
2496         if ((mode & PMC_FLAG_REMOVE) && pt != NULL)
2497                 LIST_REMOVE(pt, pt_next);
2498
2499         if ((mode & PMC_FLAG_ALLOCATE) && pt == NULL && ptnew != NULL) {
2500                 pt = ptnew;
2501                 ptnew = NULL;
2502                 pt->pt_td = td;
2503                 LIST_INSERT_HEAD(&pp->pp_tds, pt, pt_next);
2504         }
2505
2506         mtx_unlock_spin(pp->pp_tdslock);
2507
2508         if (ptnew != NULL) {
2509                 free(ptnew, M_PMC);
2510         }
2511
2512         return pt;
2513 }
2514
2515 /*
2516  * Try to add thread descriptors for each thread in a process.
2517  */
2518
2519 static void
2520 pmc_add_thread_descriptors_from_proc(struct proc *p, struct pmc_process *pp)
2521 {
2522         struct thread *curtd;
2523         struct pmc_thread **tdlist;
2524         int i, tdcnt, tdlistsz;
2525
2526         KASSERT(!PROC_LOCKED(p), ("[pmc,%d] proc unexpectedly locked",
2527             __LINE__));
2528         tdcnt = 32;
2529  restart:
2530         tdlistsz = roundup2(tdcnt, 32);
2531
2532         tdcnt = 0;
2533         tdlist = malloc(sizeof(struct pmc_thread*) * tdlistsz, M_TEMP, M_WAITOK);
2534
2535         PROC_LOCK(p);
2536         FOREACH_THREAD_IN_PROC(p, curtd)
2537                 tdcnt++;
2538         if (tdcnt >= tdlistsz) {
2539                 PROC_UNLOCK(p);
2540                 free(tdlist, M_TEMP);
2541                 goto restart;
2542         }
2543         /*
2544          * Try to add each thread to the list without sleeping. If unable,
2545          * add to a queue to retry after dropping the process lock.
2546          */
2547         tdcnt = 0;
2548         FOREACH_THREAD_IN_PROC(p, curtd) {
2549                 tdlist[tdcnt] = pmc_find_thread_descriptor(pp, curtd,
2550                                                    PMC_FLAG_ALLOCATE|PMC_FLAG_NOWAIT);
2551                 if (tdlist[tdcnt] == NULL) {
2552                         PROC_UNLOCK(p);
2553                         for (i = 0; i <= tdcnt; i++)
2554                                 pmc_thread_descriptor_pool_free(tdlist[i]);
2555                         free(tdlist, M_TEMP);
2556                         goto restart;
2557                 }
2558                 tdcnt++;
2559         }
2560         PROC_UNLOCK(p);
2561         free(tdlist, M_TEMP);
2562 }
2563
2564 /*
2565  * find the descriptor corresponding to process 'p', adding or removing it
2566  * as specified by 'mode'.
2567  */
2568
2569 static struct pmc_process *
2570 pmc_find_process_descriptor(struct proc *p, uint32_t mode)
2571 {
2572         uint32_t hindex;
2573         struct pmc_process *pp, *ppnew;
2574         struct pmc_processhash *pph;
2575
2576         hindex = PMC_HASH_PTR(p, pmc_processhashmask);
2577         pph = &pmc_processhash[hindex];
2578
2579         ppnew = NULL;
2580
2581         /*
2582          * Pre-allocate memory in the PMC_FLAG_ALLOCATE case since we
2583          * cannot call malloc(9) once we hold a spin lock.
2584          */
2585         if (mode & PMC_FLAG_ALLOCATE)
2586                 ppnew = malloc(sizeof(struct pmc_process) + md->pmd_npmc *
2587                     sizeof(struct pmc_targetstate), M_PMC, M_WAITOK|M_ZERO);
2588
2589         mtx_lock_spin(&pmc_processhash_mtx);
2590         LIST_FOREACH(pp, pph, pp_next)
2591             if (pp->pp_proc == p)
2592                     break;
2593
2594         if ((mode & PMC_FLAG_REMOVE) && pp != NULL)
2595                 LIST_REMOVE(pp, pp_next);
2596
2597         if ((mode & PMC_FLAG_ALLOCATE) && pp == NULL &&
2598             ppnew != NULL) {
2599                 ppnew->pp_proc = p;
2600                 LIST_INIT(&ppnew->pp_tds);
2601                 ppnew->pp_tdslock = mtx_pool_find(pmc_mtxpool, ppnew);
2602                 LIST_INSERT_HEAD(pph, ppnew, pp_next);
2603                 mtx_unlock_spin(&pmc_processhash_mtx);
2604                 pp = ppnew;
2605                 ppnew = NULL;
2606
2607                 /* Add thread descriptors for this process' current threads. */
2608                 pmc_add_thread_descriptors_from_proc(p, pp);
2609         }
2610         else
2611                 mtx_unlock_spin(&pmc_processhash_mtx);
2612
2613         if (ppnew != NULL)
2614                 free(ppnew, M_PMC);
2615
2616         return pp;
2617 }
2618
2619 /*
2620  * remove a process descriptor from the process hash table.
2621  */
2622
2623 static void
2624 pmc_remove_process_descriptor(struct pmc_process *pp)
2625 {
2626         KASSERT(pp->pp_refcnt == 0,
2627             ("[pmc,%d] Removing process descriptor %p with count %d",
2628                 __LINE__, pp, pp->pp_refcnt));
2629
2630         mtx_lock_spin(&pmc_processhash_mtx);
2631         LIST_REMOVE(pp, pp_next);
2632         mtx_unlock_spin(&pmc_processhash_mtx);
2633 }
2634
2635 /*
2636  * destroy a process descriptor.
2637  */
2638
2639 static void
2640 pmc_destroy_process_descriptor(struct pmc_process *pp)
2641 {
2642         struct pmc_thread *pmc_td;
2643
2644         while ((pmc_td = LIST_FIRST(&pp->pp_tds)) != NULL) {
2645                 LIST_REMOVE(pmc_td, pt_next);
2646                 pmc_thread_descriptor_pool_free(pmc_td);
2647         }
2648         free(pp, M_PMC);
2649 }
2650
2651
2652 /*
2653  * find an owner descriptor corresponding to proc 'p'
2654  */
2655
2656 static struct pmc_owner *
2657 pmc_find_owner_descriptor(struct proc *p)
2658 {
2659         uint32_t hindex;
2660         struct pmc_owner *po;
2661         struct pmc_ownerhash *poh;
2662
2663         hindex = PMC_HASH_PTR(p, pmc_ownerhashmask);
2664         poh = &pmc_ownerhash[hindex];
2665
2666         po = NULL;
2667         LIST_FOREACH(po, poh, po_next)
2668             if (po->po_owner == p)
2669                     break;
2670
2671         PMCDBG5(OWN,FND,1, "find-owner proc=%p (%d, %s) hindex=0x%x -> "
2672             "pmc-owner=%p", p, p->p_pid, p->p_comm, hindex, po);
2673
2674         return po;
2675 }
2676
2677 /*
2678  * pmc_allocate_pmc_descriptor
2679  *
2680  * Allocate a pmc descriptor and initialize its
2681  * fields.
2682  */
2683
2684 static struct pmc *
2685 pmc_allocate_pmc_descriptor(void)
2686 {
2687         struct pmc *pmc;
2688
2689         pmc = malloc(sizeof(struct pmc), M_PMC, M_WAITOK|M_ZERO);
2690         pmc->pm_runcount = counter_u64_alloc(M_WAITOK);
2691         pmc->pm_pcpu_state = malloc(sizeof(struct pmc_pcpu_state)*mp_ncpus, M_PMC, M_WAITOK|M_ZERO);
2692         PMCDBG1(PMC,ALL,1, "allocate-pmc -> pmc=%p", pmc);
2693
2694         return pmc;
2695 }
2696
2697 /*
2698  * Destroy a pmc descriptor.
2699  */
2700
2701 static void
2702 pmc_destroy_pmc_descriptor(struct pmc *pm)
2703 {
2704
2705         KASSERT(pm->pm_state == PMC_STATE_DELETED ||
2706             pm->pm_state == PMC_STATE_FREE,
2707             ("[pmc,%d] destroying non-deleted PMC", __LINE__));
2708         KASSERT(LIST_EMPTY(&pm->pm_targets),
2709             ("[pmc,%d] destroying pmc with targets", __LINE__));
2710         KASSERT(pm->pm_owner == NULL,
2711             ("[pmc,%d] destroying pmc attached to an owner", __LINE__));
2712         KASSERT(counter_u64_fetch(pm->pm_runcount) == 0,
2713             ("[pmc,%d] pmc has non-zero run count %ld", __LINE__,
2714                  (unsigned long)counter_u64_fetch(pm->pm_runcount)));
2715
2716         counter_u64_free(pm->pm_runcount);
2717         free(pm->pm_pcpu_state, M_PMC);
2718         free(pm, M_PMC);
2719 }
2720
2721 static void
2722 pmc_wait_for_pmc_idle(struct pmc *pm)
2723 {
2724 #ifdef HWPMC_DEBUG
2725         volatile int maxloop;
2726
2727         maxloop = 100 * pmc_cpu_max();
2728 #endif
2729         /*
2730          * Loop (with a forced context switch) till the PMC's runcount
2731          * comes down to zero.
2732          */
2733         pmclog_flush(pm->pm_owner, 1);
2734         while (counter_u64_fetch(pm->pm_runcount) > 0) {
2735                 pmclog_flush(pm->pm_owner, 1);
2736 #ifdef HWPMC_DEBUG
2737                 maxloop--;
2738                 KASSERT(maxloop > 0,
2739                     ("[pmc,%d] (ri%d, rc%ld) waiting too long for "
2740                         "pmc to be free", __LINE__,
2741                          PMC_TO_ROWINDEX(pm), (unsigned long)counter_u64_fetch(pm->pm_runcount)));
2742 #endif
2743                 pmc_force_context_switch();
2744         }
2745 }
2746
2747 /*
2748  * This function does the following things:
2749  *
2750  *  - detaches the PMC from hardware
2751  *  - unlinks all target threads that were attached to it
2752  *  - removes the PMC from its owner's list
2753  *  - destroys the PMC private mutex
2754  *
2755  * Once this function completes, the given pmc pointer can be freed by
2756  * calling pmc_destroy_pmc_descriptor().
2757  */
2758
2759 static void
2760 pmc_release_pmc_descriptor(struct pmc *pm)
2761 {
2762         enum pmc_mode mode;
2763         struct pmc_hw *phw;
2764         u_int adjri, ri, cpu;
2765         struct pmc_owner *po;
2766         struct pmc_binding pb;
2767         struct pmc_process *pp;
2768         struct pmc_classdep *pcd;
2769         struct pmc_target *ptgt, *tmp;
2770
2771         sx_assert(&pmc_sx, SX_XLOCKED);
2772
2773         KASSERT(pm, ("[pmc,%d] null pmc", __LINE__));
2774
2775         ri   = PMC_TO_ROWINDEX(pm);
2776         pcd  = pmc_ri_to_classdep(md, ri, &adjri);
2777         mode = PMC_TO_MODE(pm);
2778
2779         PMCDBG3(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri,
2780             mode);
2781
2782         /*
2783          * First, we take the PMC off hardware.
2784          */
2785         cpu = 0;
2786         if (PMC_IS_SYSTEM_MODE(mode)) {
2787
2788                 /*
2789                  * A system mode PMC runs on a specific CPU.  Switch
2790                  * to this CPU and turn hardware off.
2791                  */
2792                 pmc_save_cpu_binding(&pb);
2793
2794                 cpu = PMC_TO_CPU(pm);
2795
2796                 pmc_select_cpu(cpu);
2797
2798                 /* switch off non-stalled CPUs */
2799                 pm->pm_pcpu_state[cpu].pps_cpustate = 0;
2800                 if (pm->pm_state == PMC_STATE_RUNNING &&
2801                         pm->pm_pcpu_state[cpu].pps_stalled == 0) {
2802
2803                         phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
2804
2805                         KASSERT(phw->phw_pmc == pm,
2806                             ("[pmc, %d] pmc ptr ri(%d) hw(%p) pm(%p)",
2807                                 __LINE__, ri, phw->phw_pmc, pm));
2808                         PMCDBG2(PMC,REL,2, "stopping cpu=%d ri=%d", cpu, ri);
2809
2810                         critical_enter();
2811                         pcd->pcd_stop_pmc(cpu, adjri);
2812                         critical_exit();
2813                 }
2814
2815                 PMCDBG2(PMC,REL,2, "decfg cpu=%d ri=%d", cpu, ri);
2816
2817                 critical_enter();
2818                 pcd->pcd_config_pmc(cpu, adjri, NULL);
2819                 critical_exit();
2820
2821                 /* adjust the global and process count of SS mode PMCs */
2822                 if (mode == PMC_MODE_SS && pm->pm_state == PMC_STATE_RUNNING) {
2823                         po = pm->pm_owner;
2824                         po->po_sscount--;
2825                         if (po->po_sscount == 0) {
2826                                 atomic_subtract_rel_int(&pmc_ss_count, 1);
2827                                 CK_LIST_REMOVE(po, po_ssnext);
2828                                 epoch_wait_preempt(global_epoch_preempt);
2829                         }
2830                 }
2831
2832                 pm->pm_state = PMC_STATE_DELETED;
2833
2834                 pmc_restore_cpu_binding(&pb);
2835
2836                 /*
2837                  * We could have references to this PMC structure in
2838                  * the per-cpu sample queues.  Wait for the queue to
2839                  * drain.
2840                  */
2841                 pmc_wait_for_pmc_idle(pm);
2842
2843         } else if (PMC_IS_VIRTUAL_MODE(mode)) {
2844
2845                 /*
2846                  * A virtual PMC could be running on multiple CPUs at
2847                  * a given instant.
2848                  *
2849                  * By marking its state as DELETED, we ensure that
2850                  * this PMC is never further scheduled on hardware.
2851                  *
2852                  * Then we wait till all CPUs are done with this PMC.
2853                  */
2854                 pm->pm_state = PMC_STATE_DELETED;
2855
2856
2857                 /* Wait for the PMCs runcount to come to zero. */
2858                 pmc_wait_for_pmc_idle(pm);
2859
2860                 /*
2861                  * At this point the PMC is off all CPUs and cannot be
2862                  * freshly scheduled onto a CPU.  It is now safe to
2863                  * unlink all targets from this PMC.  If a
2864                  * process-record's refcount falls to zero, we remove
2865                  * it from the hash table.  The module-wide SX lock
2866                  * protects us from races.
2867                  */
2868                 LIST_FOREACH_SAFE(ptgt, &pm->pm_targets, pt_next, tmp) {
2869                         pp = ptgt->pt_process;
2870                         pmc_unlink_target_process(pm, pp); /* frees 'ptgt' */
2871
2872                         PMCDBG1(PMC,REL,3, "pp->refcnt=%d", pp->pp_refcnt);
2873
2874                         /*
2875                          * If the target process record shows that no
2876                          * PMCs are attached to it, reclaim its space.
2877                          */
2878
2879                         if (pp->pp_refcnt == 0) {
2880                                 pmc_remove_process_descriptor(pp);
2881                                 pmc_destroy_process_descriptor(pp);
2882                         }
2883                 }
2884
2885                 cpu = curthread->td_oncpu; /* setup cpu for pmd_release() */
2886
2887         }
2888
2889         /*
2890          * Release any MD resources
2891          */
2892         (void) pcd->pcd_release_pmc(cpu, adjri, pm);
2893
2894         /*
2895          * Update row disposition
2896          */
2897
2898         if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm)))
2899                 PMC_UNMARK_ROW_STANDALONE(ri);
2900         else
2901                 PMC_UNMARK_ROW_THREAD(ri);
2902
2903         /* unlink from the owner's list */
2904         if (pm->pm_owner) {
2905                 LIST_REMOVE(pm, pm_next);
2906                 pm->pm_owner = NULL;
2907         }
2908 }
2909
2910 /*
2911  * Register an owner and a pmc.
2912  */
2913
2914 static int
2915 pmc_register_owner(struct proc *p, struct pmc *pmc)
2916 {
2917         struct pmc_owner *po;
2918
2919         sx_assert(&pmc_sx, SX_XLOCKED);
2920
2921         if ((po = pmc_find_owner_descriptor(p)) == NULL)
2922                 if ((po = pmc_allocate_owner_descriptor(p)) == NULL)
2923                         return ENOMEM;
2924
2925         KASSERT(pmc->pm_owner == NULL,
2926             ("[pmc,%d] attempting to own an initialized PMC", __LINE__));
2927         pmc->pm_owner  = po;
2928
2929         LIST_INSERT_HEAD(&po->po_pmcs, pmc, pm_next);
2930
2931         PROC_LOCK(p);
2932         p->p_flag |= P_HWPMC;
2933         PROC_UNLOCK(p);
2934
2935         if (po->po_flags & PMC_PO_OWNS_LOGFILE)
2936                 pmclog_process_pmcallocate(pmc);
2937
2938         PMCDBG2(PMC,REG,1, "register-owner pmc-owner=%p pmc=%p",
2939             po, pmc);
2940
2941         return 0;
2942 }
2943
2944 /*
2945  * Return the current row disposition:
2946  * == 0 => FREE
2947  *  > 0 => PROCESS MODE
2948  *  < 0 => SYSTEM MODE
2949  */
2950
2951 int
2952 pmc_getrowdisp(int ri)
2953 {
2954         return pmc_pmcdisp[ri];
2955 }
2956
2957 /*
2958  * Check if a PMC at row index 'ri' can be allocated to the current
2959  * process.
2960  *
2961  * Allocation can fail if:
2962  *   - the current process is already being profiled by a PMC at index 'ri',
2963  *     attached to it via OP_PMCATTACH.
2964  *   - the current process has already allocated a PMC at index 'ri'
2965  *     via OP_ALLOCATE.
2966  */
2967
2968 static int
2969 pmc_can_allocate_rowindex(struct proc *p, unsigned int ri, int cpu)
2970 {
2971         enum pmc_mode mode;
2972         struct pmc *pm;
2973         struct pmc_owner *po;
2974         struct pmc_process *pp;
2975
2976         PMCDBG5(PMC,ALR,1, "can-allocate-rowindex proc=%p (%d, %s) ri=%d "
2977             "cpu=%d", p, p->p_pid, p->p_comm, ri, cpu);
2978
2979         /*
2980          * We shouldn't have already allocated a process-mode PMC at
2981          * row index 'ri'.
2982          *
2983          * We shouldn't have allocated a system-wide PMC on the same
2984          * CPU and same RI.
2985          */
2986         if ((po = pmc_find_owner_descriptor(p)) != NULL)
2987                 LIST_FOREACH(pm, &po->po_pmcs, pm_next) {
2988                     if (PMC_TO_ROWINDEX(pm) == ri) {
2989                             mode = PMC_TO_MODE(pm);
2990                             if (PMC_IS_VIRTUAL_MODE(mode))
2991                                     return EEXIST;
2992                             if (PMC_IS_SYSTEM_MODE(mode) &&
2993                                 (int) PMC_TO_CPU(pm) == cpu)
2994                                     return EEXIST;
2995                     }
2996                 }
2997
2998         /*
2999          * We also shouldn't be the target of any PMC at this index
3000          * since otherwise a PMC_ATTACH to ourselves will fail.
3001          */
3002         if ((pp = pmc_find_process_descriptor(p, 0)) != NULL)
3003                 if (pp->pp_pmcs[ri].pp_pmc)
3004                         return EEXIST;
3005
3006         PMCDBG4(PMC,ALR,2, "can-allocate-rowindex proc=%p (%d, %s) ri=%d ok",
3007             p, p->p_pid, p->p_comm, ri);
3008
3009         return 0;
3010 }
3011
3012 /*
3013  * Check if a given PMC at row index 'ri' can be currently used in
3014  * mode 'mode'.
3015  */
3016
3017 static int
3018 pmc_can_allocate_row(int ri, enum pmc_mode mode)
3019 {
3020         enum pmc_disp   disp;
3021
3022         sx_assert(&pmc_sx, SX_XLOCKED);
3023
3024         PMCDBG2(PMC,ALR,1, "can-allocate-row ri=%d mode=%d", ri, mode);
3025
3026         if (PMC_IS_SYSTEM_MODE(mode))
3027                 disp = PMC_DISP_STANDALONE;
3028         else
3029                 disp = PMC_DISP_THREAD;
3030
3031         /*
3032          * check disposition for PMC row 'ri':
3033          *
3034          * Expected disposition         Row-disposition         Result
3035          *
3036          * STANDALONE                   STANDALONE or FREE      proceed
3037          * STANDALONE                   THREAD                  fail
3038          * THREAD                       THREAD or FREE          proceed
3039          * THREAD                       STANDALONE              fail
3040          */
3041
3042         if (!PMC_ROW_DISP_IS_FREE(ri) &&
3043             !(disp == PMC_DISP_THREAD && PMC_ROW_DISP_IS_THREAD(ri)) &&
3044             !(disp == PMC_DISP_STANDALONE && PMC_ROW_DISP_IS_STANDALONE(ri)))
3045                 return EBUSY;
3046
3047         /*
3048          * All OK
3049          */
3050
3051         PMCDBG2(PMC,ALR,2, "can-allocate-row ri=%d mode=%d ok", ri, mode);
3052
3053         return 0;
3054
3055 }
3056
3057 /*
3058  * Find a PMC descriptor with user handle 'pmcid' for thread 'td'.
3059  */
3060
3061 static struct pmc *
3062 pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, pmc_id_t pmcid)
3063 {
3064         struct pmc *pm;
3065
3066         KASSERT(PMC_ID_TO_ROWINDEX(pmcid) < md->pmd_npmc,
3067             ("[pmc,%d] Illegal pmc index %d (max %d)", __LINE__,
3068                 PMC_ID_TO_ROWINDEX(pmcid), md->pmd_npmc));
3069
3070         LIST_FOREACH(pm, &po->po_pmcs, pm_next)
3071             if (pm->pm_id == pmcid)
3072                     return pm;
3073
3074         return NULL;
3075 }
3076
3077 static int
3078 pmc_find_pmc(pmc_id_t pmcid, struct pmc **pmc)
3079 {
3080
3081         struct pmc *pm, *opm;
3082         struct pmc_owner *po;
3083         struct pmc_process *pp;
3084
3085         PMCDBG1(PMC,FND,1, "find-pmc id=%d", pmcid);
3086         if (PMC_ID_TO_ROWINDEX(pmcid) >= md->pmd_npmc)
3087                 return (EINVAL);
3088
3089         if ((po = pmc_find_owner_descriptor(curthread->td_proc)) == NULL) {
3090                 /*
3091                  * In case of PMC_F_DESCENDANTS child processes we will not find
3092                  * the current process in the owners hash list.  Find the owner
3093                  * process first and from there lookup the po.
3094                  */
3095                 if ((pp = pmc_find_process_descriptor(curthread->td_proc,
3096                     PMC_FLAG_NONE)) == NULL) {
3097                         return ESRCH;
3098                 } else {
3099                         opm = pp->pp_pmcs[PMC_ID_TO_ROWINDEX(pmcid)].pp_pmc;
3100                         if (opm == NULL)
3101                                 return ESRCH;
3102                         if ((opm->pm_flags & (PMC_F_ATTACHED_TO_OWNER|
3103                             PMC_F_DESCENDANTS)) != (PMC_F_ATTACHED_TO_OWNER|
3104                             PMC_F_DESCENDANTS))
3105                                 return ESRCH;
3106                         po = opm->pm_owner;
3107                 }
3108         }
3109
3110         if ((pm = pmc_find_pmc_descriptor_in_process(po, pmcid)) == NULL)
3111                 return EINVAL;
3112
3113         PMCDBG2(PMC,FND,2, "find-pmc id=%d -> pmc=%p", pmcid, pm);
3114
3115         *pmc = pm;
3116         return 0;
3117 }
3118
3119 /*
3120  * Start a PMC.
3121  */
3122
3123 static int
3124 pmc_start(struct pmc *pm)
3125 {
3126         enum pmc_mode mode;
3127         struct pmc_owner *po;
3128         struct pmc_binding pb;
3129         struct pmc_classdep *pcd;
3130         int adjri, error, cpu, ri;
3131
3132         KASSERT(pm != NULL,
3133             ("[pmc,%d] null pm", __LINE__));
3134
3135         mode = PMC_TO_MODE(pm);
3136         ri   = PMC_TO_ROWINDEX(pm);
3137         pcd  = pmc_ri_to_classdep(md, ri, &adjri);
3138
3139         error = 0;
3140
3141         PMCDBG3(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri);
3142
3143         po = pm->pm_owner;
3144
3145         /*
3146          * Disallow PMCSTART if a logfile is required but has not been
3147          * configured yet.
3148          */
3149         if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) &&
3150             (po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
3151                 return (EDOOFUS);       /* programming error */
3152
3153         /*
3154          * If this is a sampling mode PMC, log mapping information for
3155          * the kernel modules that are currently loaded.
3156          */
3157         if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
3158             pmc_log_kernel_mappings(pm);
3159
3160         if (PMC_IS_VIRTUAL_MODE(mode)) {
3161
3162                 /*
3163                  * If a PMCATTACH has never been done on this PMC,
3164                  * attach it to its owner process.
3165                  */
3166
3167                 if (LIST_EMPTY(&pm->pm_targets))
3168                         error = (pm->pm_flags & PMC_F_ATTACH_DONE) ? ESRCH :
3169                             pmc_attach_process(po->po_owner, pm);
3170
3171                 /*
3172                  * If the PMC is attached to its owner, then force a context
3173                  * switch to ensure that the MD state gets set correctly.
3174                  */
3175
3176                 if (error == 0) {
3177                         pm->pm_state = PMC_STATE_RUNNING;
3178                         if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER)
3179                                 pmc_force_context_switch();
3180                 }
3181
3182                 return (error);
3183         }
3184
3185
3186         /*
3187          * A system-wide PMC.
3188          *
3189          * Add the owner to the global list if this is a system-wide
3190          * sampling PMC.
3191          */
3192
3193         if (mode == PMC_MODE_SS) {
3194                 /*
3195                  * Log mapping information for all existing processes in the
3196                  * system.  Subsequent mappings are logged as they happen;
3197                  * see pmc_process_mmap().
3198                  */
3199                 if (po->po_logprocmaps == 0) {
3200                         pmc_log_all_process_mappings(po);
3201                         po->po_logprocmaps = 1;
3202                 }
3203                 po->po_sscount++;
3204                 if (po->po_sscount == 1) {
3205                         atomic_add_rel_int(&pmc_ss_count, 1);
3206                         CK_LIST_INSERT_HEAD(&pmc_ss_owners, po, po_ssnext);
3207                         PMCDBG1(PMC,OPS,1, "po=%p in global list", po);
3208                 }
3209         }
3210
3211         /*
3212          * Move to the CPU associated with this
3213          * PMC, and start the hardware.
3214          */
3215
3216         pmc_save_cpu_binding(&pb);
3217
3218         cpu = PMC_TO_CPU(pm);
3219
3220         if (!pmc_cpu_is_active(cpu))
3221                 return (ENXIO);
3222
3223         pmc_select_cpu(cpu);
3224
3225         /*
3226          * global PMCs are configured at allocation time
3227          * so write out the initial value and start the PMC.
3228          */
3229
3230         pm->pm_state = PMC_STATE_RUNNING;
3231
3232         critical_enter();
3233         if ((error = pcd->pcd_write_pmc(cpu, adjri,
3234                  PMC_IS_SAMPLING_MODE(mode) ?
3235                  pm->pm_sc.pm_reloadcount :
3236                  pm->pm_sc.pm_initial)) == 0) {
3237                 /* If a sampling mode PMC, reset stalled state. */
3238                 if (PMC_IS_SAMPLING_MODE(mode))
3239                         pm->pm_pcpu_state[cpu].pps_stalled = 0;
3240
3241                 /* Indicate that we desire this to run. Start it. */
3242                 pm->pm_pcpu_state[cpu].pps_cpustate = 1;
3243                 error = pcd->pcd_start_pmc(cpu, adjri);
3244         }
3245         critical_exit();
3246
3247         pmc_restore_cpu_binding(&pb);
3248
3249         return (error);
3250 }
3251
3252 /*
3253  * Stop a PMC.
3254  */
3255
3256 static int
3257 pmc_stop(struct pmc *pm)
3258 {
3259         struct pmc_owner *po;
3260         struct pmc_binding pb;
3261         struct pmc_classdep *pcd;
3262         int adjri, cpu, error, ri;
3263
3264         KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__));
3265
3266         PMCDBG3(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm,
3267             PMC_TO_MODE(pm), PMC_TO_ROWINDEX(pm));
3268
3269         pm->pm_state = PMC_STATE_STOPPED;
3270
3271         /*
3272          * If the PMC is a virtual mode one, changing the state to
3273          * non-RUNNING is enough to ensure that the PMC never gets
3274          * scheduled.
3275          *
3276          * If this PMC is current running on a CPU, then it will
3277          * handled correctly at the time its target process is context
3278          * switched out.
3279          */
3280
3281         if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
3282                 return 0;
3283
3284         /*
3285          * A system-mode PMC.  Move to the CPU associated with
3286          * this PMC, and stop the hardware.  We update the
3287          * 'initial count' so that a subsequent PMCSTART will
3288          * resume counting from the current hardware count.
3289          */
3290
3291         pmc_save_cpu_binding(&pb);
3292
3293         cpu = PMC_TO_CPU(pm);
3294
3295         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
3296             ("[pmc,%d] illegal cpu=%d", __LINE__, cpu));
3297
3298         if (!pmc_cpu_is_active(cpu))
3299                 return ENXIO;
3300
3301         pmc_select_cpu(cpu);
3302
3303         ri = PMC_TO_ROWINDEX(pm);
3304         pcd = pmc_ri_to_classdep(md, ri, &adjri);
3305
3306         pm->pm_pcpu_state[cpu].pps_cpustate = 0;
3307         critical_enter();
3308         if ((error = pcd->pcd_stop_pmc(cpu, adjri)) == 0)
3309                 error = pcd->pcd_read_pmc(cpu, adjri, &pm->pm_sc.pm_initial);
3310         critical_exit();
3311
3312         pmc_restore_cpu_binding(&pb);
3313
3314         po = pm->pm_owner;
3315
3316         /* remove this owner from the global list of SS PMC owners */
3317         if (PMC_TO_MODE(pm) == PMC_MODE_SS) {
3318                 po->po_sscount--;
3319                 if (po->po_sscount == 0) {
3320                         atomic_subtract_rel_int(&pmc_ss_count, 1);
3321                         CK_LIST_REMOVE(po, po_ssnext);
3322                         epoch_wait_preempt(global_epoch_preempt);
3323                         PMCDBG1(PMC,OPS,2,"po=%p removed from global list", po);
3324                 }
3325         }
3326
3327         return (error);
3328 }
3329
3330 static struct pmc_classdep *
3331 pmc_class_to_classdep(enum pmc_class class)
3332 {
3333         int n;
3334
3335         for (n = 0; n < md->pmd_nclass; n++)
3336                 if (md->pmd_classdep[n].pcd_class == class)
3337                         return (&md->pmd_classdep[n]);
3338         return (NULL);
3339 }
3340
3341 #if defined(HWPMC_DEBUG) && defined(KTR)
3342 static const char *pmc_op_to_name[] = {
3343 #undef  __PMC_OP
3344 #define __PMC_OP(N, D)  #N ,
3345         __PMC_OPS()
3346         NULL
3347 };
3348 #endif
3349
3350 /*
3351  * The syscall interface
3352  */
3353
3354 #define PMC_GET_SX_XLOCK(...) do {              \
3355         sx_xlock(&pmc_sx);                      \
3356         if (pmc_hook == NULL) {                 \
3357                 sx_xunlock(&pmc_sx);            \
3358                 return __VA_ARGS__;             \
3359         }                                       \
3360 } while (0)
3361
3362 #define PMC_DOWNGRADE_SX() do {                 \
3363         sx_downgrade(&pmc_sx);                  \
3364         is_sx_downgraded = 1;                   \
3365 } while (0)
3366
3367 static int
3368 pmc_syscall_handler(struct thread *td, void *syscall_args)
3369 {
3370         int error, is_sx_downgraded, op;
3371         struct pmc_syscall_args *c;
3372         void *pmclog_proc_handle;
3373         void *arg;
3374
3375         c = (struct pmc_syscall_args *)syscall_args;
3376         op = c->pmop_code;
3377         arg = c->pmop_data;
3378         /* PMC isn't set up yet */
3379         if (pmc_hook == NULL)
3380                 return (EINVAL);
3381         if (op == PMC_OP_CONFIGURELOG) {
3382                 /*
3383                  * We cannot create the logging process inside
3384                  * pmclog_configure_log() because there is a LOR
3385                  * between pmc_sx and process structure locks.
3386                  * Instead, pre-create the process and ignite the loop
3387                  * if everything is fine, otherwise direct the process
3388                  * to exit.
3389                  */
3390                 error = pmclog_proc_create(td, &pmclog_proc_handle);
3391                 if (error != 0)
3392                         goto done_syscall;
3393         }
3394
3395         PMC_GET_SX_XLOCK(ENOSYS);
3396         is_sx_downgraded = 0;
3397         PMCDBG3(MOD,PMS,1, "syscall op=%d \"%s\" arg=%p", op,
3398             pmc_op_to_name[op], arg);
3399
3400         error = 0;
3401         counter_u64_add(pmc_stats.pm_syscalls, 1);
3402
3403         switch (op) {
3404
3405
3406         /*
3407          * Configure a log file.
3408          *
3409          * XXX This OP will be reworked.
3410          */
3411
3412         case PMC_OP_CONFIGURELOG:
3413         {
3414                 struct proc *p;
3415                 struct pmc *pm;
3416                 struct pmc_owner *po;
3417                 struct pmc_op_configurelog cl;
3418
3419                 if ((error = copyin(arg, &cl, sizeof(cl))) != 0) {
3420                         pmclog_proc_ignite(pmclog_proc_handle, NULL);
3421                         break;
3422                 }
3423
3424                 /* mark this process as owning a log file */
3425                 p = td->td_proc;
3426                 if ((po = pmc_find_owner_descriptor(p)) == NULL)
3427                         if ((po = pmc_allocate_owner_descriptor(p)) == NULL) {
3428                                 pmclog_proc_ignite(pmclog_proc_handle, NULL);
3429                                 error = ENOMEM;
3430                                 break;
3431                         }
3432
3433                 /*
3434                  * If a valid fd was passed in, try to configure that,
3435                  * otherwise if 'fd' was less than zero and there was
3436                  * a log file configured, flush its buffers and
3437                  * de-configure it.
3438                  */
3439                 if (cl.pm_logfd >= 0) {
3440                         error = pmclog_configure_log(md, po, cl.pm_logfd);
3441                         pmclog_proc_ignite(pmclog_proc_handle, error == 0 ?
3442                             po : NULL);
3443                 } else if (po->po_flags & PMC_PO_OWNS_LOGFILE) {
3444                         pmclog_proc_ignite(pmclog_proc_handle, NULL);
3445                         error = pmclog_close(po);
3446                         if (error == 0) {
3447                                 LIST_FOREACH(pm, &po->po_pmcs, pm_next)
3448                                     if (pm->pm_flags & PMC_F_NEEDS_LOGFILE &&
3449                                         pm->pm_state == PMC_STATE_RUNNING)
3450                                             pmc_stop(pm);
3451                                 error = pmclog_deconfigure_log(po);
3452                         }
3453                 } else {
3454                         pmclog_proc_ignite(pmclog_proc_handle, NULL);
3455                         error = EINVAL;
3456                 }
3457         }
3458         break;
3459
3460         /*
3461          * Flush a log file.
3462          */
3463
3464         case PMC_OP_FLUSHLOG:
3465         {
3466                 struct pmc_owner *po;
3467
3468                 sx_assert(&pmc_sx, SX_XLOCKED);
3469
3470                 if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
3471                         error = EINVAL;
3472                         break;
3473                 }
3474
3475                 error = pmclog_flush(po, 0);
3476         }
3477         break;
3478
3479         /*
3480          * Close a log file.
3481          */
3482
3483         case PMC_OP_CLOSELOG:
3484         {
3485                 struct pmc_owner *po;
3486
3487                 sx_assert(&pmc_sx, SX_XLOCKED);
3488
3489                 if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
3490                         error = EINVAL;
3491                         break;
3492                 }
3493
3494                 error = pmclog_close(po);
3495         }
3496         break;
3497
3498         /*
3499          * Retrieve hardware configuration.
3500          */
3501
3502         case PMC_OP_GETCPUINFO: /* CPU information */
3503         {
3504                 struct pmc_op_getcpuinfo gci;
3505                 struct pmc_classinfo *pci;
3506                 struct pmc_classdep *pcd;
3507                 int cl;
3508
3509                 gci.pm_cputype = md->pmd_cputype;
3510                 gci.pm_ncpu    = pmc_cpu_max();
3511                 gci.pm_npmc    = md->pmd_npmc;
3512                 gci.pm_nclass  = md->pmd_nclass;
3513                 pci = gci.pm_classes;
3514                 pcd = md->pmd_classdep;
3515                 for (cl = 0; cl < md->pmd_nclass; cl++, pci++, pcd++) {
3516                         pci->pm_caps  = pcd->pcd_caps;
3517                         pci->pm_class = pcd->pcd_class;
3518                         pci->pm_width = pcd->pcd_width;
3519                         pci->pm_num   = pcd->pcd_num;
3520                 }
3521                 error = copyout(&gci, arg, sizeof(gci));
3522         }
3523         break;
3524
3525         /*
3526          * Retrieve soft events list.
3527          */
3528         case PMC_OP_GETDYNEVENTINFO:
3529         {
3530                 enum pmc_class                  cl;
3531                 enum pmc_event                  ev;
3532                 struct pmc_op_getdyneventinfo   *gei;
3533                 struct pmc_dyn_event_descr      dev;
3534                 struct pmc_soft                 *ps;
3535                 uint32_t                        nevent;
3536
3537                 sx_assert(&pmc_sx, SX_LOCKED);
3538
3539                 gei = (struct pmc_op_getdyneventinfo *) arg;
3540
3541                 if ((error = copyin(&gei->pm_class, &cl, sizeof(cl))) != 0)
3542                         break;
3543
3544                 /* Only SOFT class is dynamic. */
3545                 if (cl != PMC_CLASS_SOFT) {
3546                         error = EINVAL;
3547                         break;
3548                 }
3549
3550                 nevent = 0;
3551                 for (ev = PMC_EV_SOFT_FIRST; (int)ev <= PMC_EV_SOFT_LAST; ev++) {
3552                         ps = pmc_soft_ev_acquire(ev);
3553                         if (ps == NULL)
3554                                 continue;
3555                         bcopy(&ps->ps_ev, &dev, sizeof(dev));
3556                         pmc_soft_ev_release(ps);
3557
3558                         error = copyout(&dev,
3559                             &gei->pm_events[nevent],
3560                             sizeof(struct pmc_dyn_event_descr));
3561                         if (error != 0)
3562                                 break;
3563                         nevent++;
3564                 }
3565                 if (error != 0)
3566                         break;
3567
3568                 error = copyout(&nevent, &gei->pm_nevent,
3569                     sizeof(nevent));
3570         }
3571         break;
3572
3573         /*
3574          * Get module statistics
3575          */
3576
3577         case PMC_OP_GETDRIVERSTATS:
3578         {
3579                 struct pmc_op_getdriverstats gms;
3580 #define CFETCH(a, b, field) a.field = counter_u64_fetch(b.field)
3581                 CFETCH(gms, pmc_stats, pm_intr_ignored);
3582                 CFETCH(gms, pmc_stats, pm_intr_processed);
3583                 CFETCH(gms, pmc_stats, pm_intr_bufferfull);
3584                 CFETCH(gms, pmc_stats, pm_syscalls);
3585                 CFETCH(gms, pmc_stats, pm_syscall_errors);
3586                 CFETCH(gms, pmc_stats, pm_buffer_requests);
3587                 CFETCH(gms, pmc_stats, pm_buffer_requests_failed);
3588                 CFETCH(gms, pmc_stats, pm_log_sweeps);
3589 #undef CFETCH
3590                 error = copyout(&gms, arg, sizeof(gms));
3591         }
3592         break;
3593
3594
3595         /*
3596          * Retrieve module version number
3597          */
3598
3599         case PMC_OP_GETMODULEVERSION:
3600         {
3601                 uint32_t cv, modv;
3602
3603                 /* retrieve the client's idea of the ABI version */
3604                 if ((error = copyin(arg, &cv, sizeof(uint32_t))) != 0)
3605                         break;
3606                 /* don't service clients newer than our driver */
3607                 modv = PMC_VERSION;
3608                 if ((cv & 0xFFFF0000) > (modv & 0xFFFF0000)) {
3609                         error = EPROGMISMATCH;
3610                         break;
3611                 }
3612                 error = copyout(&modv, arg, sizeof(int));
3613         }
3614         break;
3615
3616
3617         /*
3618          * Retrieve the state of all the PMCs on a given
3619          * CPU.
3620          */
3621
3622         case PMC_OP_GETPMCINFO:
3623         {
3624                 int ari;
3625                 struct pmc *pm;
3626                 size_t pmcinfo_size;
3627                 uint32_t cpu, n, npmc;
3628                 struct pmc_owner *po;
3629                 struct pmc_binding pb;
3630                 struct pmc_classdep *pcd;
3631                 struct pmc_info *p, *pmcinfo;
3632                 struct pmc_op_getpmcinfo *gpi;
3633
3634                 PMC_DOWNGRADE_SX();
3635
3636                 gpi = (struct pmc_op_getpmcinfo *) arg;
3637
3638                 if ((error = copyin(&gpi->pm_cpu, &cpu, sizeof(cpu))) != 0)
3639                         break;
3640
3641                 if (cpu >= pmc_cpu_max()) {
3642                         error = EINVAL;
3643                         break;
3644                 }
3645
3646                 if (!pmc_cpu_is_active(cpu)) {
3647                         error = ENXIO;
3648                         break;
3649                 }
3650
3651                 /* switch to CPU 'cpu' */
3652                 pmc_save_cpu_binding(&pb);
3653                 pmc_select_cpu(cpu);
3654
3655                 npmc = md->pmd_npmc;
3656
3657                 pmcinfo_size = npmc * sizeof(struct pmc_info);
3658                 pmcinfo = malloc(pmcinfo_size, M_PMC, M_WAITOK);
3659
3660                 p = pmcinfo;
3661
3662                 for (n = 0; n < md->pmd_npmc; n++, p++) {
3663
3664                         pcd = pmc_ri_to_classdep(md, n, &ari);
3665
3666                         KASSERT(pcd != NULL,
3667                             ("[pmc,%d] null pcd ri=%d", __LINE__, n));
3668
3669                         if ((error = pcd->pcd_describe(cpu, ari, p, &pm)) != 0)
3670                                 break;
3671
3672                         if (PMC_ROW_DISP_IS_STANDALONE(n))
3673                                 p->pm_rowdisp = PMC_DISP_STANDALONE;
3674                         else if (PMC_ROW_DISP_IS_THREAD(n))
3675                                 p->pm_rowdisp = PMC_DISP_THREAD;
3676                         else
3677                                 p->pm_rowdisp = PMC_DISP_FREE;
3678
3679                         p->pm_ownerpid = -1;
3680
3681                         if (pm == NULL) /* no PMC associated */
3682                                 continue;
3683
3684                         po = pm->pm_owner;
3685
3686                         KASSERT(po->po_owner != NULL,
3687                             ("[pmc,%d] pmc_owner had a null proc pointer",
3688                                 __LINE__));
3689
3690                         p->pm_ownerpid = po->po_owner->p_pid;
3691                         p->pm_mode     = PMC_TO_MODE(pm);
3692                         p->pm_event    = pm->pm_event;
3693                         p->pm_flags    = pm->pm_flags;
3694
3695                         if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
3696                                 p->pm_reloadcount =
3697                                     pm->pm_sc.pm_reloadcount;
3698                 }
3699
3700                 pmc_restore_cpu_binding(&pb);
3701
3702                 /* now copy out the PMC info collected */
3703                 if (error == 0)
3704                         error = copyout(pmcinfo, &gpi->pm_pmcs, pmcinfo_size);
3705
3706                 free(pmcinfo, M_PMC);
3707         }
3708         break;
3709
3710
3711         /*
3712          * Set the administrative state of a PMC.  I.e. whether
3713          * the PMC is to be used or not.
3714          */
3715
3716         case PMC_OP_PMCADMIN:
3717         {
3718                 int cpu, ri;
3719                 enum pmc_state request;
3720                 struct pmc_cpu *pc;
3721                 struct pmc_hw *phw;
3722                 struct pmc_op_pmcadmin pma;
3723                 struct pmc_binding pb;
3724
3725                 sx_assert(&pmc_sx, SX_XLOCKED);
3726
3727                 KASSERT(td == curthread,
3728                     ("[pmc,%d] td != curthread", __LINE__));
3729
3730                 error = priv_check(td, PRIV_PMC_MANAGE);
3731                 if (error)
3732                         break;
3733
3734                 if ((error = copyin(arg, &pma, sizeof(pma))) != 0)
3735                         break;
3736
3737                 cpu = pma.pm_cpu;
3738
3739                 if (cpu < 0 || cpu >= (int) pmc_cpu_max()) {
3740                         error = EINVAL;
3741                         break;
3742                 }
3743
3744                 if (!pmc_cpu_is_active(cpu)) {
3745                         error = ENXIO;
3746                         break;
3747                 }
3748
3749                 request = pma.pm_state;
3750
3751                 if (request != PMC_STATE_DISABLED &&
3752                     request != PMC_STATE_FREE) {
3753                         error = EINVAL;
3754                         break;
3755                 }
3756
3757                 ri = pma.pm_pmc; /* pmc id == row index */
3758                 if (ri < 0 || ri >= (int) md->pmd_npmc) {
3759                         error = EINVAL;
3760                         break;
3761                 }
3762
3763                 /*
3764                  * We can't disable a PMC with a row-index allocated
3765                  * for process virtual PMCs.
3766                  */
3767
3768                 if (PMC_ROW_DISP_IS_THREAD(ri) &&
3769                     request == PMC_STATE_DISABLED) {
3770                         error = EBUSY;
3771                         break;
3772                 }
3773
3774                 /*
3775                  * otherwise, this PMC on this CPU is either free or
3776                  * in system-wide mode.
3777                  */
3778
3779                 pmc_save_cpu_binding(&pb);
3780                 pmc_select_cpu(cpu);
3781
3782                 pc  = pmc_pcpu[cpu];
3783                 phw = pc->pc_hwpmcs[ri];
3784
3785                 /*
3786                  * XXX do we need some kind of 'forced' disable?
3787                  */
3788
3789                 if (phw->phw_pmc == NULL) {
3790                         if (request == PMC_STATE_DISABLED &&
3791                             (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED)) {
3792                                 phw->phw_state &= ~PMC_PHW_FLAG_IS_ENABLED;
3793                                 PMC_MARK_ROW_STANDALONE(ri);
3794                         } else if (request == PMC_STATE_FREE &&
3795                             (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0) {
3796                                 phw->phw_state |=  PMC_PHW_FLAG_IS_ENABLED;
3797                                 PMC_UNMARK_ROW_STANDALONE(ri);
3798                         }
3799                         /* other cases are a no-op */
3800                 } else
3801                         error = EBUSY;
3802
3803                 pmc_restore_cpu_binding(&pb);
3804         }
3805         break;
3806
3807
3808         /*
3809          * Allocate a PMC.
3810          */
3811
3812         case PMC_OP_PMCALLOCATE:
3813         {
3814                 int adjri, n;
3815                 u_int cpu;
3816                 uint32_t caps;
3817                 struct pmc *pmc;
3818                 enum pmc_mode mode;
3819                 struct pmc_hw *phw;
3820                 struct pmc_binding pb;
3821                 struct pmc_classdep *pcd;
3822                 struct pmc_op_pmcallocate pa;
3823
3824                 if ((error = copyin(arg, &pa, sizeof(pa))) != 0)
3825                         break;
3826
3827                 caps = pa.pm_caps;
3828                 mode = pa.pm_mode;
3829                 cpu  = pa.pm_cpu;
3830
3831                 if ((mode != PMC_MODE_SS  &&  mode != PMC_MODE_SC  &&
3832                      mode != PMC_MODE_TS  &&  mode != PMC_MODE_TC) ||
3833                     (cpu != (u_int) PMC_CPU_ANY && cpu >= pmc_cpu_max())) {
3834                         error = EINVAL;
3835                         break;
3836                 }
3837
3838                 /*
3839                  * Virtual PMCs should only ask for a default CPU.
3840                  * System mode PMCs need to specify a non-default CPU.
3841                  */
3842
3843                 if ((PMC_IS_VIRTUAL_MODE(mode) && cpu != (u_int) PMC_CPU_ANY) ||
3844                     (PMC_IS_SYSTEM_MODE(mode) && cpu == (u_int) PMC_CPU_ANY)) {
3845                         error = EINVAL;
3846                         break;
3847                 }
3848
3849                 /*
3850                  * Check that an inactive CPU is not being asked for.
3851                  */
3852
3853                 if (PMC_IS_SYSTEM_MODE(mode) && !pmc_cpu_is_active(cpu)) {
3854                         error = ENXIO;
3855                         break;
3856                 }
3857
3858                 /*
3859                  * Refuse an allocation for a system-wide PMC if this
3860                  * process has been jailed, or if this process lacks
3861                  * super-user credentials and the sysctl tunable
3862                  * 'security.bsd.unprivileged_syspmcs' is zero.
3863                  */
3864
3865                 if (PMC_IS_SYSTEM_MODE(mode)) {
3866                         if (jailed(curthread->td_ucred)) {
3867                                 error = EPERM;
3868                                 break;
3869                         }
3870                         if (!pmc_unprivileged_syspmcs) {
3871                                 error = priv_check(curthread,
3872                                     PRIV_PMC_SYSTEM);
3873                                 if (error)
3874                                         break;
3875                         }
3876                 }
3877
3878                 /*
3879                  * Look for valid values for 'pm_flags'
3880                  */
3881
3882                 if ((pa.pm_flags & ~(PMC_F_DESCENDANTS | PMC_F_LOG_PROCCSW |
3883                     PMC_F_LOG_PROCEXIT | PMC_F_CALLCHAIN |
3884                     PMC_F_USERCALLCHAIN)) != 0) {
3885                         error = EINVAL;
3886                         break;
3887                 }
3888
3889                 /* PMC_F_USERCALLCHAIN is only valid with PMC_F_CALLCHAIN */
3890                 if ((pa.pm_flags & (PMC_F_CALLCHAIN | PMC_F_USERCALLCHAIN)) ==
3891                     PMC_F_USERCALLCHAIN) {
3892                         error = EINVAL;
3893                         break;
3894                 }
3895
3896                 /* PMC_F_USERCALLCHAIN is only valid for sampling mode */
3897                 if (pa.pm_flags & PMC_F_USERCALLCHAIN &&
3898                         mode != PMC_MODE_TS && mode != PMC_MODE_SS) {
3899                         error = EINVAL;
3900                         break;
3901                 }
3902
3903                 /* process logging options are not allowed for system PMCs */
3904                 if (PMC_IS_SYSTEM_MODE(mode) && (pa.pm_flags &
3905                     (PMC_F_LOG_PROCCSW | PMC_F_LOG_PROCEXIT))) {
3906                         error = EINVAL;
3907                         break;
3908                 }
3909
3910                 /*
3911                  * All sampling mode PMCs need to be able to interrupt the
3912                  * CPU.
3913                  */
3914                 if (PMC_IS_SAMPLING_MODE(mode))
3915                         caps |= PMC_CAP_INTERRUPT;
3916
3917                 /* A valid class specifier should have been passed in. */
3918                 pcd = pmc_class_to_classdep(pa.pm_class);
3919                 if (pcd == NULL) {
3920                         error = EINVAL;
3921                         break;
3922                 }
3923
3924                 /* The requested PMC capabilities should be feasible. */
3925                 if ((pcd->pcd_caps & caps) != caps) {
3926                         error = EOPNOTSUPP;
3927                         break;
3928                 }
3929
3930                 PMCDBG4(PMC,ALL,2, "event=%d caps=0x%x mode=%d cpu=%d",
3931                     pa.pm_ev, caps, mode, cpu);
3932
3933                 pmc = pmc_allocate_pmc_descriptor();
3934                 pmc->pm_id    = PMC_ID_MAKE_ID(cpu,pa.pm_mode,pa.pm_class,
3935                     PMC_ID_INVALID);
3936                 pmc->pm_event = pa.pm_ev;
3937                 pmc->pm_state = PMC_STATE_FREE;
3938                 pmc->pm_caps  = caps;
3939                 pmc->pm_flags = pa.pm_flags;
3940
3941                 /* XXX set lower bound on sampling for process counters */
3942                 if (PMC_IS_SAMPLING_MODE(mode))
3943                         pmc->pm_sc.pm_reloadcount = pa.pm_count;
3944                 else
3945                         pmc->pm_sc.pm_initial = pa.pm_count;
3946
3947                 /* switch thread to CPU 'cpu' */
3948                 pmc_save_cpu_binding(&pb);
3949
3950 #define PMC_IS_SHAREABLE_PMC(cpu, n)                            \
3951         (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_state &           \
3952          PMC_PHW_FLAG_IS_SHAREABLE)
3953 #define PMC_IS_UNALLOCATED(cpu, n)                              \
3954         (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_pmc == NULL)
3955
3956                 if (PMC_IS_SYSTEM_MODE(mode)) {
3957                         pmc_select_cpu(cpu);
3958                         for (n = pcd->pcd_ri; n < (int) md->pmd_npmc; n++) {
3959                                 pcd = pmc_ri_to_classdep(md, n, &adjri);
3960                                 if (pmc_can_allocate_row(n, mode) == 0 &&
3961                                     pmc_can_allocate_rowindex(
3962                                             curthread->td_proc, n, cpu) == 0 &&
3963                                     (PMC_IS_UNALLOCATED(cpu, n) ||
3964                                      PMC_IS_SHAREABLE_PMC(cpu, n)) &&
3965                                     pcd->pcd_allocate_pmc(cpu, adjri, pmc,
3966                                         &pa) == 0)
3967                                         break;
3968                         }
3969                 } else {
3970                         /* Process virtual mode */
3971                         for (n = pcd->pcd_ri; n < (int) md->pmd_npmc; n++) {
3972                                 pcd = pmc_ri_to_classdep(md, n, &adjri);
3973                                 if (pmc_can_allocate_row(n, mode) == 0 &&
3974                                     pmc_can_allocate_rowindex(
3975                                             curthread->td_proc, n,
3976                                             PMC_CPU_ANY) == 0 &&
3977                                     pcd->pcd_allocate_pmc(curthread->td_oncpu,
3978                                         adjri, pmc, &pa) == 0)
3979                                         break;
3980                         }
3981                 }
3982
3983 #undef  PMC_IS_UNALLOCATED
3984 #undef  PMC_IS_SHAREABLE_PMC
3985
3986                 pmc_restore_cpu_binding(&pb);
3987
3988                 if (n == (int) md->pmd_npmc) {
3989                         pmc_destroy_pmc_descriptor(pmc);
3990                         pmc = NULL;
3991                         error = EINVAL;
3992                         break;
3993                 }
3994
3995                 /* Fill in the correct value in the ID field */
3996                 pmc->pm_id = PMC_ID_MAKE_ID(cpu,mode,pa.pm_class,n);
3997
3998                 PMCDBG5(PMC,ALL,2, "ev=%d class=%d mode=%d n=%d -> pmcid=%x",
3999                     pmc->pm_event, pa.pm_class, mode, n, pmc->pm_id);
4000
4001                 /* Process mode PMCs with logging enabled need log files */
4002                 if (pmc->pm_flags & (PMC_F_LOG_PROCEXIT | PMC_F_LOG_PROCCSW))
4003                         pmc->pm_flags |= PMC_F_NEEDS_LOGFILE;
4004
4005                 /* All system mode sampling PMCs require a log file */
4006                 if (PMC_IS_SAMPLING_MODE(mode) && PMC_IS_SYSTEM_MODE(mode))
4007                         pmc->pm_flags |= PMC_F_NEEDS_LOGFILE;
4008
4009                 /*
4010                  * Configure global pmc's immediately
4011                  */
4012
4013                 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pmc))) {
4014
4015                         pmc_save_cpu_binding(&pb);
4016                         pmc_select_cpu(cpu);
4017
4018                         phw = pmc_pcpu[cpu]->pc_hwpmcs[n];
4019                         pcd = pmc_ri_to_classdep(md, n, &adjri);
4020
4021                         if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0 ||
4022                             (error = pcd->pcd_config_pmc(cpu, adjri, pmc)) != 0) {
4023                                 (void) pcd->pcd_release_pmc(cpu, adjri, pmc);
4024                                 pmc_destroy_pmc_descriptor(pmc);
4025                                 pmc = NULL;
4026                                 pmc_restore_cpu_binding(&pb);
4027                                 error = EPERM;
4028                                 break;
4029                         }
4030
4031                         pmc_restore_cpu_binding(&pb);
4032                 }
4033
4034                 pmc->pm_state    = PMC_STATE_ALLOCATED;
4035                 pmc->pm_class   = pa.pm_class;
4036
4037                 /*
4038                  * mark row disposition
4039                  */
4040
4041                 if (PMC_IS_SYSTEM_MODE(mode))
4042                         PMC_MARK_ROW_STANDALONE(n);
4043                 else
4044                         PMC_MARK_ROW_THREAD(n);
4045
4046                 /*
4047                  * Register this PMC with the current thread as its owner.
4048                  */
4049
4050                 if ((error =
4051                     pmc_register_owner(curthread->td_proc, pmc)) != 0) {
4052                         pmc_release_pmc_descriptor(pmc);
4053                         pmc_destroy_pmc_descriptor(pmc);
4054                         pmc = NULL;
4055                         break;
4056                 }
4057
4058
4059                 /*
4060                  * Return the allocated index.
4061                  */
4062
4063                 pa.pm_pmcid = pmc->pm_id;
4064
4065                 error = copyout(&pa, arg, sizeof(pa));
4066         }
4067         break;
4068
4069
4070         /*
4071          * Attach a PMC to a process.
4072          */
4073
4074         case PMC_OP_PMCATTACH:
4075         {
4076                 struct pmc *pm;
4077                 struct proc *p;
4078                 struct pmc_op_pmcattach a;
4079
4080                 sx_assert(&pmc_sx, SX_XLOCKED);
4081
4082                 if ((error = copyin(arg, &a, sizeof(a))) != 0)
4083                         break;
4084
4085                 if (a.pm_pid < 0) {
4086                         error = EINVAL;
4087                         break;
4088                 } else if (a.pm_pid == 0)
4089                         a.pm_pid = td->td_proc->p_pid;
4090
4091                 if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0)
4092                         break;
4093
4094                 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
4095                         error = EINVAL;
4096                         break;
4097                 }
4098
4099                 /* PMCs may be (re)attached only when allocated or stopped */
4100                 if (pm->pm_state == PMC_STATE_RUNNING) {
4101                         error = EBUSY;
4102                         break;
4103                 } else if (pm->pm_state != PMC_STATE_ALLOCATED &&
4104                     pm->pm_state != PMC_STATE_STOPPED) {
4105                         error = EINVAL;
4106                         break;
4107                 }
4108
4109                 /* lookup pid */
4110                 if ((p = pfind(a.pm_pid)) == NULL) {
4111                         error = ESRCH;
4112                         break;
4113                 }
4114
4115                 /*
4116                  * Ignore processes that are working on exiting.
4117                  */
4118                 if (p->p_flag & P_WEXIT) {
4119                         error = ESRCH;
4120                         PROC_UNLOCK(p); /* pfind() returns a locked process */
4121                         break;
4122                 }
4123
4124                 /*
4125                  * we are allowed to attach a PMC to a process if
4126                  * we can debug it.
4127                  */
4128                 error = p_candebug(curthread, p);
4129
4130                 PROC_UNLOCK(p);
4131
4132                 if (error == 0)
4133                         error = pmc_attach_process(p, pm);
4134         }
4135         break;
4136
4137
4138         /*
4139          * Detach an attached PMC from a process.
4140          */
4141
4142         case PMC_OP_PMCDETACH:
4143         {
4144                 struct pmc *pm;
4145                 struct proc *p;
4146                 struct pmc_op_pmcattach a;
4147
4148                 if ((error = copyin(arg, &a, sizeof(a))) != 0)
4149                         break;
4150
4151                 if (a.pm_pid < 0) {
4152                         error = EINVAL;
4153                         break;
4154                 } else if (a.pm_pid == 0)
4155                         a.pm_pid = td->td_proc->p_pid;
4156
4157                 if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0)
4158                         break;
4159
4160                 if ((p = pfind(a.pm_pid)) == NULL) {
4161                         error = ESRCH;
4162                         break;
4163                 }
4164
4165                 /*
4166                  * Treat processes that are in the process of exiting
4167                  * as if they were not present.
4168                  */
4169
4170                 if (p->p_flag & P_WEXIT)
4171                         error = ESRCH;
4172
4173                 PROC_UNLOCK(p); /* pfind() returns a locked process */
4174
4175                 if (error == 0)
4176                         error = pmc_detach_process(p, pm);
4177         }
4178         break;
4179
4180
4181         /*
4182          * Retrieve the MSR number associated with the counter
4183          * 'pmc_id'.  This allows processes to directly use RDPMC
4184          * instructions to read their PMCs, without the overhead of a
4185          * system call.
4186          */
4187
4188         case PMC_OP_PMCGETMSR:
4189         {
4190                 int adjri, ri;
4191                 struct pmc *pm;
4192                 struct pmc_target *pt;
4193                 struct pmc_op_getmsr gm;
4194                 struct pmc_classdep *pcd;
4195
4196                 PMC_DOWNGRADE_SX();
4197
4198                 if ((error = copyin(arg, &gm, sizeof(gm))) != 0)
4199                         break;
4200
4201                 if ((error = pmc_find_pmc(gm.pm_pmcid, &pm)) != 0)
4202                         break;
4203
4204                 /*
4205                  * The allocated PMC has to be a process virtual PMC,
4206                  * i.e., of type MODE_T[CS].  Global PMCs can only be
4207                  * read using the PMCREAD operation since they may be
4208                  * allocated on a different CPU than the one we could
4209                  * be running on at the time of the RDPMC instruction.
4210                  *
4211                  * The GETMSR operation is not allowed for PMCs that
4212                  * are inherited across processes.
4213                  */
4214
4215                 if (!PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) ||
4216                     (pm->pm_flags & PMC_F_DESCENDANTS)) {
4217                         error = EINVAL;
4218                         break;
4219                 }
4220
4221                 /*
4222                  * It only makes sense to use a RDPMC (or its
4223                  * equivalent instruction on non-x86 architectures) on
4224                  * a process that has allocated and attached a PMC to
4225                  * itself.  Conversely the PMC is only allowed to have
4226                  * one process attached to it -- its owner.
4227                  */
4228
4229                 if ((pt = LIST_FIRST(&pm->pm_targets)) == NULL ||
4230                     LIST_NEXT(pt, pt_next) != NULL ||
4231                     pt->pt_process->pp_proc != pm->pm_owner->po_owner) {
4232                         error = EINVAL;
4233                         break;
4234                 }
4235
4236                 ri = PMC_TO_ROWINDEX(pm);
4237                 pcd = pmc_ri_to_classdep(md, ri, &adjri);
4238
4239                 /* PMC class has no 'GETMSR' support */
4240                 if (pcd->pcd_get_msr == NULL) {
4241                         error = ENOSYS;
4242                         break;
4243                 }
4244
4245                 if ((error = (*pcd->pcd_get_msr)(adjri, &gm.pm_msr)) < 0)
4246                         break;
4247
4248                 if ((error = copyout(&gm, arg, sizeof(gm))) < 0)
4249                         break;
4250
4251                 /*
4252                  * Mark our process as using MSRs.  Update machine
4253                  * state using a forced context switch.
4254                  */
4255
4256                 pt->pt_process->pp_flags |= PMC_PP_ENABLE_MSR_ACCESS;
4257                 pmc_force_context_switch();
4258
4259         }
4260         break;
4261
4262         /*
4263          * Release an allocated PMC
4264          */
4265
4266         case PMC_OP_PMCRELEASE:
4267         {
4268                 pmc_id_t pmcid;
4269                 struct pmc *pm;
4270                 struct pmc_owner *po;
4271                 struct pmc_op_simple sp;
4272
4273                 /*
4274                  * Find PMC pointer for the named PMC.
4275                  *
4276                  * Use pmc_release_pmc_descriptor() to switch off the
4277                  * PMC, remove all its target threads, and remove the
4278                  * PMC from its owner's list.
4279                  *
4280                  * Remove the owner record if this is the last PMC
4281                  * owned.
4282                  *
4283                  * Free up space.
4284                  */
4285
4286                 if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
4287                         break;
4288
4289                 pmcid = sp.pm_pmcid;
4290
4291                 if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
4292                         break;
4293
4294                 po = pm->pm_owner;
4295                 pmc_release_pmc_descriptor(pm);
4296                 pmc_maybe_remove_owner(po);
4297                 pmc_destroy_pmc_descriptor(pm);
4298         }
4299         break;
4300
4301
4302         /*
4303          * Read and/or write a PMC.
4304          */
4305
4306         case PMC_OP_PMCRW:
4307         {
4308                 int adjri;
4309                 struct pmc *pm;
4310                 uint32_t cpu, ri;
4311                 pmc_value_t oldvalue;
4312                 struct pmc_binding pb;
4313                 struct pmc_op_pmcrw prw;
4314                 struct pmc_classdep *pcd;
4315                 struct pmc_op_pmcrw *pprw;
4316
4317                 PMC_DOWNGRADE_SX();
4318
4319                 if ((error = copyin(arg, &prw, sizeof(prw))) != 0)
4320                         break;
4321
4322                 ri = 0;
4323                 PMCDBG2(PMC,OPS,1, "rw id=%d flags=0x%x", prw.pm_pmcid,
4324                     prw.pm_flags);
4325
4326                 /* must have at least one flag set */
4327                 if ((prw.pm_flags & (PMC_F_OLDVALUE|PMC_F_NEWVALUE)) == 0) {
4328                         error = EINVAL;
4329                         break;
4330                 }
4331
4332                 /* locate pmc descriptor */
4333                 if ((error = pmc_find_pmc(prw.pm_pmcid, &pm)) != 0)
4334                         break;
4335
4336                 /* Can't read a PMC that hasn't been started. */
4337                 if (pm->pm_state != PMC_STATE_ALLOCATED &&
4338                     pm->pm_state != PMC_STATE_STOPPED &&
4339                     pm->pm_state != PMC_STATE_RUNNING) {
4340                         error = EINVAL;
4341                         break;
4342                 }
4343
4344                 /* writing a new value is allowed only for 'STOPPED' pmcs */
4345                 if (pm->pm_state == PMC_STATE_RUNNING &&
4346                     (prw.pm_flags & PMC_F_NEWVALUE)) {
4347                         error = EBUSY;
4348                         break;
4349                 }
4350
4351                 if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) {
4352
4353                         /*
4354                          * If this PMC is attached to its owner (i.e.,
4355                          * the process requesting this operation) and
4356                          * is running, then attempt to get an
4357                          * upto-date reading from hardware for a READ.
4358                          * Writes are only allowed when the PMC is
4359                          * stopped, so only update the saved value
4360                          * field.
4361                          *
4362                          * If the PMC is not running, or is not
4363                          * attached to its owner, read/write to the
4364                          * savedvalue field.
4365                          */
4366
4367                         ri = PMC_TO_ROWINDEX(pm);
4368                         pcd = pmc_ri_to_classdep(md, ri, &adjri);
4369
4370                         mtx_pool_lock_spin(pmc_mtxpool, pm);
4371                         cpu = curthread->td_oncpu;
4372
4373                         if (prw.pm_flags & PMC_F_OLDVALUE) {
4374                                 if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) &&
4375                                     (pm->pm_state == PMC_STATE_RUNNING))
4376                                         error = (*pcd->pcd_read_pmc)(cpu, adjri,
4377                                             &oldvalue);
4378                                 else
4379                                         oldvalue = pm->pm_gv.pm_savedvalue;
4380                         }
4381                         if (prw.pm_flags & PMC_F_NEWVALUE)
4382                                 pm->pm_gv.pm_savedvalue = prw.pm_value;
4383
4384                         mtx_pool_unlock_spin(pmc_mtxpool, pm);
4385
4386                 } else { /* System mode PMCs */
4387                         cpu = PMC_TO_CPU(pm);
4388                         ri  = PMC_TO_ROWINDEX(pm);
4389                         pcd = pmc_ri_to_classdep(md, ri, &adjri);
4390
4391                         if (!pmc_cpu_is_active(cpu)) {
4392                                 error = ENXIO;
4393                                 break;
4394                         }
4395
4396                         /* move this thread to CPU 'cpu' */
4397                         pmc_save_cpu_binding(&pb);
4398                         pmc_select_cpu(cpu);
4399
4400                         critical_enter();
4401                         /* save old value */
4402                         if (prw.pm_flags & PMC_F_OLDVALUE)
4403                                 if ((error = (*pcd->pcd_read_pmc)(cpu, adjri,
4404                                          &oldvalue)))
4405                                         goto error;
4406                         /* write out new value */
4407                         if (prw.pm_flags & PMC_F_NEWVALUE)
4408                                 error = (*pcd->pcd_write_pmc)(cpu, adjri,
4409                                     prw.pm_value);
4410                 error:
4411                         critical_exit();
4412                         pmc_restore_cpu_binding(&pb);
4413                         if (error)
4414                                 break;
4415                 }
4416
4417                 pprw = (struct pmc_op_pmcrw *) arg;
4418
4419 #ifdef  HWPMC_DEBUG
4420                 if (prw.pm_flags & PMC_F_NEWVALUE)
4421                         PMCDBG3(PMC,OPS,2, "rw id=%d new %jx -> old %jx",
4422                             ri, prw.pm_value, oldvalue);
4423                 else if (prw.pm_flags & PMC_F_OLDVALUE)
4424                         PMCDBG2(PMC,OPS,2, "rw id=%d -> old %jx", ri, oldvalue);
4425 #endif
4426
4427                 /* return old value if requested */
4428                 if (prw.pm_flags & PMC_F_OLDVALUE)
4429                         if ((error = copyout(&oldvalue, &pprw->pm_value,
4430                                  sizeof(prw.pm_value))))
4431                                 break;
4432
4433         }
4434         break;
4435
4436
4437         /*
4438          * Set the sampling rate for a sampling mode PMC and the
4439          * initial count for a counting mode PMC.
4440          */
4441
4442         case PMC_OP_PMCSETCOUNT:
4443         {
4444                 struct pmc *pm;
4445                 struct pmc_op_pmcsetcount sc;
4446
4447                 PMC_DOWNGRADE_SX();
4448
4449                 if ((error = copyin(arg, &sc, sizeof(sc))) != 0)
4450                         break;
4451
4452                 if ((error = pmc_find_pmc(sc.pm_pmcid, &pm)) != 0)
4453                         break;
4454
4455                 if (pm->pm_state == PMC_STATE_RUNNING) {
4456                         error = EBUSY;
4457                         break;
4458                 }
4459
4460                 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
4461                         pm->pm_sc.pm_reloadcount = sc.pm_count;
4462                 else
4463                         pm->pm_sc.pm_initial = sc.pm_count;
4464         }
4465         break;
4466
4467
4468         /*
4469          * Start a PMC.
4470          */
4471
4472         case PMC_OP_PMCSTART:
4473         {
4474                 pmc_id_t pmcid;
4475                 struct pmc *pm;
4476                 struct pmc_op_simple sp;
4477
4478                 sx_assert(&pmc_sx, SX_XLOCKED);
4479
4480                 if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
4481                         break;
4482
4483                 pmcid = sp.pm_pmcid;
4484
4485                 if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
4486                         break;
4487
4488                 KASSERT(pmcid == pm->pm_id,
4489                     ("[pmc,%d] pmcid %x != id %x", __LINE__,
4490                         pm->pm_id, pmcid));
4491
4492                 if (pm->pm_state == PMC_STATE_RUNNING) /* already running */
4493                         break;
4494                 else if (pm->pm_state != PMC_STATE_STOPPED &&
4495                     pm->pm_state != PMC_STATE_ALLOCATED) {
4496                         error = EINVAL;
4497                         break;
4498                 }
4499
4500                 error = pmc_start(pm);
4501         }
4502         break;
4503
4504
4505         /*
4506          * Stop a PMC.
4507          */
4508
4509         case PMC_OP_PMCSTOP:
4510         {
4511                 pmc_id_t pmcid;
4512                 struct pmc *pm;
4513                 struct pmc_op_simple sp;
4514
4515                 PMC_DOWNGRADE_SX();
4516
4517                 if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
4518                         break;
4519
4520                 pmcid = sp.pm_pmcid;
4521
4522                 /*
4523                  * Mark the PMC as inactive and invoke the MD stop
4524                  * routines if needed.
4525                  */
4526
4527                 if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
4528                         break;
4529
4530                 KASSERT(pmcid == pm->pm_id,
4531                     ("[pmc,%d] pmc id %x != pmcid %x", __LINE__,
4532                         pm->pm_id, pmcid));
4533
4534                 if (pm->pm_state == PMC_STATE_STOPPED) /* already stopped */
4535                         break;
4536                 else if (pm->pm_state != PMC_STATE_RUNNING) {
4537                         error = EINVAL;
4538                         break;
4539                 }
4540
4541                 error = pmc_stop(pm);
4542         }
4543         break;
4544
4545
4546         /*
4547          * Write a user supplied value to the log file.
4548          */
4549
4550         case PMC_OP_WRITELOG:
4551         {
4552                 struct pmc_op_writelog wl;
4553                 struct pmc_owner *po;
4554
4555                 PMC_DOWNGRADE_SX();
4556
4557                 if ((error = copyin(arg, &wl, sizeof(wl))) != 0)
4558                         break;
4559
4560                 if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
4561                         error = EINVAL;
4562                         break;
4563                 }
4564
4565                 if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) {
4566                         error = EINVAL;
4567                         break;
4568                 }
4569
4570                 error = pmclog_process_userlog(po, &wl);
4571         }
4572         break;
4573
4574
4575         default:
4576                 error = EINVAL;
4577                 break;
4578         }
4579
4580         if (is_sx_downgraded)
4581                 sx_sunlock(&pmc_sx);
4582         else
4583                 sx_xunlock(&pmc_sx);
4584 done_syscall:
4585         if (error)
4586                 counter_u64_add(pmc_stats.pm_syscall_errors, 1);
4587
4588         return (error);
4589 }
4590
4591 /*
4592  * Helper functions
4593  */
4594
4595
4596 /*
4597  * Mark the thread as needing callchain capture and post an AST.  The
4598  * actual callchain capture will be done in a context where it is safe
4599  * to take page faults.
4600  */
4601
4602 static void
4603 pmc_post_callchain_callback(void)
4604 {
4605         struct thread *td;
4606
4607         td = curthread;
4608
4609         /*
4610          * If there is multiple PMCs for the same interrupt ignore new post
4611          */
4612         if (td->td_pflags & TDP_CALLCHAIN)
4613                 return;
4614
4615         /*
4616          * Mark this thread as needing callchain capture.
4617          * `td->td_pflags' will be safe to touch because this thread
4618          * was in user space when it was interrupted.
4619          */
4620         td->td_pflags |= TDP_CALLCHAIN;
4621
4622         /*
4623          * Don't let this thread migrate between CPUs until callchain
4624          * capture completes.
4625          */
4626         sched_pin();
4627
4628         return;
4629 }
4630
4631 /*
4632  * Find a free slot in the per-cpu array of samples and capture the
4633  * current callchain there.  If a sample was successfully added, a bit
4634  * is set in mask 'pmc_cpumask' denoting that the DO_SAMPLES hook
4635  * needs to be invoked from the clock handler.
4636  *
4637  * This function is meant to be called from an NMI handler.  It cannot
4638  * use any of the locking primitives supplied by the OS.
4639  */
4640
4641 static int
4642 pmc_add_sample(int ring, struct pmc *pm, struct trapframe *tf)
4643 {
4644         int error, cpu, callchaindepth, inuserspace;
4645         struct thread *td;
4646         struct pmc_sample *ps;
4647         struct pmc_samplebuffer *psb;
4648
4649         error = 0;
4650
4651         /*
4652          * Allocate space for a sample buffer.
4653          */
4654         cpu = curcpu;
4655         psb = pmc_pcpu[cpu]->pc_sb[ring];
4656         inuserspace = TRAPF_USERMODE(tf);
4657         ps = psb->ps_write;
4658         if (ps->ps_nsamples == PMC_SAMPLE_INUSE) {
4659                 counter_u64_add(ps->ps_pmc->pm_runcount, -1);
4660                 counter_u64_add(pmc_stats.pm_overwrites, 1);
4661                 ps->ps_nsamples = 0;
4662         } else if (ps->ps_nsamples) {   /* in use, reader hasn't caught up */
4663                 pm->pm_pcpu_state[cpu].pps_stalled = 1;
4664                 counter_u64_add(pmc_stats.pm_intr_bufferfull, 1);
4665                 PMCDBG6(SAM,INT,1,"(spc) cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d",
4666                     cpu, pm, (void *) tf, inuserspace,
4667                     (int) (psb->ps_write - psb->ps_samples),
4668                     (int) (psb->ps_read - psb->ps_samples));
4669                 callchaindepth = 1;
4670                 error = ENOMEM;
4671                 goto done;
4672         }
4673
4674         /* Fill in entry. */
4675         PMCDBG6(SAM,INT,1,"cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d", cpu, pm,
4676             (void *) tf, inuserspace,
4677             (int) (psb->ps_write - psb->ps_samples),
4678             (int) (psb->ps_read - psb->ps_samples));
4679
4680         KASSERT(counter_u64_fetch(pm->pm_runcount) >= 0,
4681             ("[pmc,%d] pm=%p runcount %ld", __LINE__, (void *) pm,
4682                  (unsigned long)counter_u64_fetch(pm->pm_runcount)));
4683
4684         counter_u64_add(pm->pm_runcount, 1);    /* hold onto PMC */
4685
4686         td = curthread;
4687         ps->ps_pmc = pm;
4688         ps->ps_td = td;
4689         ps->ps_pid = td->td_proc->p_pid;
4690         ps->ps_tid = td->td_tid;
4691         ps->ps_tsc = pmc_rdtsc();
4692
4693         ps->ps_cpu = cpu;
4694         ps->ps_flags = inuserspace ? PMC_CC_F_USERSPACE : 0;
4695
4696         callchaindepth = (pm->pm_flags & PMC_F_CALLCHAIN) ?
4697             pmc_callchaindepth : 1;
4698
4699         if (callchaindepth == 1)
4700                 ps->ps_pc[0] = PMC_TRAPFRAME_TO_PC(tf);
4701         else {
4702                 /*
4703                  * Kernel stack traversals can be done immediately,
4704                  * while we defer to an AST for user space traversals.
4705                  */
4706                 if (!inuserspace) {
4707                         callchaindepth =
4708                             pmc_save_kernel_callchain(ps->ps_pc,
4709                                 callchaindepth, tf);
4710                 } else {
4711                         pmc_post_callchain_callback();
4712                         callchaindepth = PMC_SAMPLE_INUSE;
4713                 }
4714         }
4715
4716         ps->ps_nsamples = callchaindepth;       /* mark entry as in use */
4717         if (ring == PMC_UR) {
4718                 ps->ps_nsamples_actual = callchaindepth;        /* mark entry as in use */
4719                 ps->ps_nsamples = PMC_SAMPLE_INUSE;
4720         } else
4721                 ps->ps_nsamples = callchaindepth;       /* mark entry as in use */
4722         /* increment write pointer, modulo ring buffer size */
4723         ps++;
4724         if (ps == psb->ps_fence)
4725                 psb->ps_write = psb->ps_samples;
4726         else
4727                 psb->ps_write = ps;
4728
4729  done:
4730         /* mark CPU as needing processing */
4731         if (callchaindepth != PMC_SAMPLE_INUSE)
4732                 DPCPU_SET(pmc_sampled, 1);
4733
4734         return (error);
4735 }
4736
4737 /*
4738  * Interrupt processing.
4739  *
4740  * This function is meant to be called from an NMI handler.  It cannot
4741  * use any of the locking primitives supplied by the OS.
4742  */
4743
4744 int
4745 pmc_process_interrupt(int ring, struct pmc *pm, struct trapframe *tf)
4746 {
4747         struct thread *td;
4748
4749         td = curthread;
4750         if ((pm->pm_flags & PMC_F_USERCALLCHAIN) &&
4751             (td->td_proc->p_flag & P_KPROC) == 0 &&
4752             !TRAPF_USERMODE(tf)) {
4753                 atomic_add_int(&td->td_pmcpend, 1);
4754                 return (pmc_add_sample(PMC_UR, pm, tf));
4755         }
4756         return (pmc_add_sample(ring, pm, tf));
4757 }
4758
4759 /*
4760  * Capture a user call chain.  This function will be called from ast()
4761  * before control returns to userland and before the process gets
4762  * rescheduled.
4763  */
4764
4765 static void
4766 pmc_capture_user_callchain(int cpu, int ring, struct trapframe *tf)
4767 {
4768         struct pmc *pm;
4769         struct thread *td;
4770         struct pmc_sample *ps, *ps_end;
4771         struct pmc_samplebuffer *psb;
4772         int nsamples, nrecords, pass;
4773 #ifdef  INVARIANTS
4774         int ncallchains;
4775         int nfree;
4776 #endif
4777
4778         psb = pmc_pcpu[cpu]->pc_sb[ring];
4779         td = curthread;
4780
4781         KASSERT(td->td_pflags & TDP_CALLCHAIN,
4782             ("[pmc,%d] Retrieving callchain for thread that doesn't want it",
4783                 __LINE__));
4784
4785 #ifdef  INVARIANTS
4786         ncallchains = 0;
4787         nfree = 0;
4788 #endif
4789         nrecords = INT_MAX;
4790         pass = 0;
4791  restart:
4792         if (ring == PMC_UR)
4793                 nrecords = atomic_readandclear_32(&td->td_pmcpend);
4794
4795         /*
4796          * Iterate through all deferred callchain requests.
4797          * Walk from the current read pointer to the current
4798          * write pointer.
4799          */
4800
4801         ps = psb->ps_read;
4802         ps_end = psb->ps_write;
4803         do {
4804 #ifdef  INVARIANTS
4805                 if (ps->ps_nsamples == PMC_SAMPLE_FREE) {
4806                         nfree++;
4807                         goto next;
4808                 }
4809
4810                 if ((ps->ps_pmc == NULL) ||
4811                     (ps->ps_pmc->pm_state != PMC_STATE_RUNNING))
4812                         nfree++;
4813 #endif
4814                 if (ps->ps_nsamples != PMC_SAMPLE_INUSE)
4815                         goto next;
4816                 if (ps->ps_td != td)
4817                         goto next;
4818
4819                 KASSERT(ps->ps_cpu == cpu,
4820                     ("[pmc,%d] cpu mismatch ps_cpu=%d pcpu=%d", __LINE__,
4821                         ps->ps_cpu, PCPU_GET(cpuid)));
4822
4823                 pm = ps->ps_pmc;
4824
4825                 KASSERT(pm->pm_flags & PMC_F_CALLCHAIN,
4826                     ("[pmc,%d] Retrieving callchain for PMC that doesn't "
4827                         "want it", __LINE__));
4828
4829                 KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
4830                     ("[pmc,%d] runcount %ld", __LINE__, (unsigned long)counter_u64_fetch(pm->pm_runcount)));
4831
4832                 if (ring == PMC_UR) {
4833                         nsamples = ps->ps_nsamples_actual;
4834                         counter_u64_add(pmc_stats.pm_merges, 1);
4835                 } else
4836                         nsamples = 0;
4837
4838                 /*
4839                  * Retrieve the callchain and mark the sample buffer
4840                  * as 'processable' by the timer tick sweep code.
4841                  */
4842
4843 #ifdef INVARIANTS
4844                 ncallchains++;
4845 #endif
4846
4847                 if (__predict_true(nsamples < pmc_callchaindepth - 1))
4848                         nsamples += pmc_save_user_callchain(ps->ps_pc + nsamples,
4849                        pmc_callchaindepth - nsamples - 1, tf);
4850                 wmb();
4851                 ps->ps_nsamples = nsamples;
4852                 if (nrecords-- == 1)
4853                         break;
4854 next:
4855                 /* increment the pointer, modulo sample ring size */
4856                 if (++ps == psb->ps_fence)
4857                         ps = psb->ps_samples;
4858         } while (ps != ps_end);
4859         if (__predict_false(ring == PMC_UR && td->td_pmcpend)) {
4860                 if (pass == 0) {
4861                         pass = 1;
4862                         goto restart;
4863                 }
4864                 /* only collect samples for this part once */
4865                 td->td_pmcpend = 0;
4866         }
4867
4868 #ifdef INVARIANTS
4869         if (ring == PMC_HR)
4870                 KASSERT(ncallchains > 0 || nfree > 0,
4871                     ("[pmc,%d] cpu %d didn't find a sample to collect", __LINE__,
4872                             cpu));
4873 #endif
4874
4875         /* mark CPU as needing processing */
4876         DPCPU_SET(pmc_sampled, 1);
4877 }
4878
4879
4880 static void
4881 pmc_flush_ring(int cpu, int ring)
4882 {
4883         struct pmc *pm;
4884         struct pmc_sample *ps;
4885         struct pmc_samplebuffer *psb;
4886         int n;
4887
4888         psb = pmc_pcpu[cpu]->pc_sb[ring];
4889
4890         for (n = 0; n < pmc_nsamples; n++) { /* bound on #iterations */
4891
4892                 ps = psb->ps_read;
4893                 if (ps->ps_nsamples == PMC_SAMPLE_FREE)
4894                         goto next;
4895                 pm = ps->ps_pmc;
4896                 counter_u64_add(pm->pm_runcount, -1);
4897                 ps->ps_nsamples = PMC_SAMPLE_FREE;
4898                 /* increment read pointer, modulo sample size */
4899         next:
4900                 if (++ps == psb->ps_fence)
4901                         psb->ps_read = psb->ps_samples;
4902                 else
4903                         psb->ps_read = ps;
4904         }
4905 }
4906
4907 void
4908 pmc_flush_samples(int cpu)
4909 {
4910         int n;
4911
4912         for (n = 0; n < PMC_NUM_SR; n++)
4913                 pmc_flush_ring(cpu, n);
4914 }
4915
4916
4917 /*
4918  * Process saved PC samples.
4919  */
4920
4921 static void
4922 pmc_process_samples(int cpu, int ring)
4923 {
4924         struct pmc *pm;
4925         int adjri, n;
4926         struct thread *td;
4927         struct pmc_owner *po;
4928         struct pmc_sample *ps;
4929         struct pmc_classdep *pcd;
4930         struct pmc_samplebuffer *psb;
4931
4932         KASSERT(PCPU_GET(cpuid) == cpu,
4933             ("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__,
4934                 PCPU_GET(cpuid), cpu));
4935
4936         psb = pmc_pcpu[cpu]->pc_sb[ring];
4937
4938         for (n = 0; n < pmc_nsamples; n++) { /* bound on #iterations */
4939
4940                 ps = psb->ps_read;
4941                 if (ps->ps_nsamples == PMC_SAMPLE_FREE)
4942                         break;
4943
4944                 pm = ps->ps_pmc;
4945
4946                 KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
4947                     ("[pmc,%d] pm=%p runcount %ld", __LINE__, (void *) pm,
4948                          (unsigned long)counter_u64_fetch(pm->pm_runcount)));
4949
4950                 po = pm->pm_owner;
4951
4952                 KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
4953                     ("[pmc,%d] pmc=%p non-sampling mode=%d", __LINE__,
4954                         pm, PMC_TO_MODE(pm)));
4955
4956                 /* Ignore PMCs that have been switched off */
4957                 if (pm->pm_state != PMC_STATE_RUNNING)
4958                         goto entrydone;
4959
4960                 /* If there is a pending AST wait for completion */
4961                 if (ps->ps_nsamples == PMC_SAMPLE_INUSE) {
4962                         /* Need a rescan at a later time. */
4963                         DPCPU_SET(pmc_sampled, 1);
4964                         break;
4965                 }
4966
4967                 PMCDBG6(SAM,OPS,1,"cpu=%d pm=%p n=%d fl=%x wr=%d rd=%d", cpu,
4968                     pm, ps->ps_nsamples, ps->ps_flags,
4969                     (int) (psb->ps_write - psb->ps_samples),
4970                     (int) (psb->ps_read - psb->ps_samples));
4971
4972                 /*
4973                  * If this is a process-mode PMC that is attached to
4974                  * its owner, and if the PC is in user mode, update
4975                  * profiling statistics like timer-based profiling
4976                  * would have done.
4977                  *
4978                  * Otherwise, this is either a sampling-mode PMC that
4979                  * is attached to a different process than its owner,
4980                  * or a system-wide sampling PMC. Dispatch a log
4981                  * entry to the PMC's owner process.
4982                  */
4983                 if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) {
4984                         if (ps->ps_flags & PMC_CC_F_USERSPACE) {
4985                                 td = FIRST_THREAD_IN_PROC(po->po_owner);
4986                                 addupc_intr(td, ps->ps_pc[0], 1);
4987                         }
4988                 } else
4989                         pmclog_process_callchain(pm, ps);
4990
4991         entrydone:
4992                 ps->ps_nsamples = 0; /* mark entry as free */
4993                 counter_u64_add(pm->pm_runcount, -1);
4994
4995                 /* increment read pointer, modulo sample size */
4996                 if (++ps == psb->ps_fence)
4997                         psb->ps_read = psb->ps_samples;
4998                 else
4999                         psb->ps_read = ps;
5000         }
5001
5002         counter_u64_add(pmc_stats.pm_log_sweeps, 1);
5003
5004         /* Do not re-enable stalled PMCs if we failed to process any samples */
5005         if (n == 0)
5006                 return;
5007
5008         /*
5009          * Restart any stalled sampling PMCs on this CPU.
5010          *
5011          * If the NMI handler sets the pm_stalled field of a PMC after
5012          * the check below, we'll end up processing the stalled PMC at
5013          * the next hardclock tick.
5014          */
5015         for (n = 0; n < md->pmd_npmc; n++) {
5016                 pcd = pmc_ri_to_classdep(md, n, &adjri);
5017                 KASSERT(pcd != NULL,
5018                     ("[pmc,%d] null pcd ri=%d", __LINE__, n));
5019                 (void) (*pcd->pcd_get_config)(cpu,adjri,&pm);
5020
5021                 if (pm == NULL ||                        /* !cfg'ed */
5022                     pm->pm_state != PMC_STATE_RUNNING || /* !active */
5023                     !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || /* !sampling */
5024                         !pm->pm_pcpu_state[cpu].pps_cpustate  || /* !desired */
5025                     !pm->pm_pcpu_state[cpu].pps_stalled) /* !stalled */
5026                         continue;
5027
5028                 pm->pm_pcpu_state[cpu].pps_stalled = 0;
5029                 (*pcd->pcd_start_pmc)(cpu, adjri);
5030         }
5031 }
5032
5033 /*
5034  * Event handlers.
5035  */
5036
5037 /*
5038  * Handle a process exit.
5039  *
5040  * Remove this process from all hash tables.  If this process
5041  * owned any PMCs, turn off those PMCs and deallocate them,
5042  * removing any associations with target processes.
5043  *
5044  * This function will be called by the last 'thread' of a
5045  * process.
5046  *
5047  * XXX This eventhandler gets called early in the exit process.
5048  * Consider using a 'hook' invocation from thread_exit() or equivalent
5049  * spot.  Another negative is that kse_exit doesn't seem to call
5050  * exit1() [??].
5051  *
5052  */
5053
5054 static void
5055 pmc_process_exit(void *arg __unused, struct proc *p)
5056 {
5057         struct pmc *pm;
5058         int adjri, cpu;
5059         unsigned int ri;
5060         int is_using_hwpmcs;
5061         struct pmc_owner *po;
5062         struct pmc_process *pp;
5063         struct pmc_classdep *pcd;
5064         pmc_value_t newvalue, tmp;
5065
5066         PROC_LOCK(p);
5067         is_using_hwpmcs = p->p_flag & P_HWPMC;
5068         PROC_UNLOCK(p);
5069
5070         /*
5071          * Log a sysexit event to all SS PMC owners.
5072          */
5073         epoch_enter_preempt(global_epoch_preempt);
5074         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5075             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
5076                     pmclog_process_sysexit(po, p->p_pid);
5077         epoch_exit_preempt(global_epoch_preempt);
5078
5079         if (!is_using_hwpmcs)
5080                 return;
5081
5082         PMC_GET_SX_XLOCK();
5083         PMCDBG3(PRC,EXT,1,"process-exit proc=%p (%d, %s)", p, p->p_pid,
5084             p->p_comm);
5085
5086         /*
5087          * Since this code is invoked by the last thread in an exiting
5088          * process, we would have context switched IN at some prior
5089          * point.  However, with PREEMPTION, kernel mode context
5090          * switches may happen any time, so we want to disable a
5091          * context switch OUT till we get any PMCs targeting this
5092          * process off the hardware.
5093          *
5094          * We also need to atomically remove this process'
5095          * entry from our target process hash table, using
5096          * PMC_FLAG_REMOVE.
5097          */
5098         PMCDBG3(PRC,EXT,1, "process-exit proc=%p (%d, %s)", p, p->p_pid,
5099             p->p_comm);
5100
5101         critical_enter(); /* no preemption */
5102
5103         cpu = curthread->td_oncpu;
5104
5105         if ((pp = pmc_find_process_descriptor(p,
5106                  PMC_FLAG_REMOVE)) != NULL) {
5107
5108                 PMCDBG2(PRC,EXT,2,
5109                     "process-exit proc=%p pmc-process=%p", p, pp);
5110
5111                 /*
5112                  * The exiting process could the target of
5113                  * some PMCs which will be running on
5114                  * currently executing CPU.
5115                  *
5116                  * We need to turn these PMCs off like we
5117                  * would do at context switch OUT time.
5118                  */
5119                 for (ri = 0; ri < md->pmd_npmc; ri++) {
5120
5121                         /*
5122                          * Pick up the pmc pointer from hardware
5123                          * state similar to the CSW_OUT code.
5124                          */
5125                         pm = NULL;
5126
5127                         pcd = pmc_ri_to_classdep(md, ri, &adjri);
5128
5129                         (void) (*pcd->pcd_get_config)(cpu, adjri, &pm);
5130
5131                         PMCDBG2(PRC,EXT,2, "ri=%d pm=%p", ri, pm);
5132
5133                         if (pm == NULL ||
5134                             !PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
5135                                 continue;
5136
5137                         PMCDBG4(PRC,EXT,2, "ppmcs[%d]=%p pm=%p "
5138                             "state=%d", ri, pp->pp_pmcs[ri].pp_pmc,
5139                             pm, pm->pm_state);
5140
5141                         KASSERT(PMC_TO_ROWINDEX(pm) == ri,
5142                             ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
5143                                 __LINE__, PMC_TO_ROWINDEX(pm), ri));
5144
5145                         KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
5146                             ("[pmc,%d] pm %p != pp_pmcs[%d] %p",
5147                                 __LINE__, pm, ri, pp->pp_pmcs[ri].pp_pmc));
5148
5149                         KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
5150                             ("[pmc,%d] bad runcount ri %d rc %ld",
5151                                  __LINE__, ri, (unsigned long)counter_u64_fetch(pm->pm_runcount)));
5152
5153                         /*
5154                          * Change desired state, and then stop if not
5155                          * stalled. This two-step dance should avoid
5156                          * race conditions where an interrupt re-enables
5157                          * the PMC after this code has already checked
5158                          * the pm_stalled flag.
5159                          */
5160                         if (pm->pm_pcpu_state[cpu].pps_cpustate) {
5161                                 pm->pm_pcpu_state[cpu].pps_cpustate = 0;
5162                                 if (!pm->pm_pcpu_state[cpu].pps_stalled) {
5163                                         (void) pcd->pcd_stop_pmc(cpu, adjri);
5164
5165                                         if (PMC_TO_MODE(pm) == PMC_MODE_TC) {
5166                                                 pcd->pcd_read_pmc(cpu, adjri,
5167                                                     &newvalue);
5168                                                 tmp = newvalue -
5169                                                     PMC_PCPU_SAVED(cpu,ri);
5170
5171                                                 mtx_pool_lock_spin(pmc_mtxpool,
5172                                                     pm);
5173                                                 pm->pm_gv.pm_savedvalue += tmp;
5174                                                 pp->pp_pmcs[ri].pp_pmcval +=
5175                                                     tmp;
5176                                                 mtx_pool_unlock_spin(
5177                                                     pmc_mtxpool, pm);
5178                                         }
5179                                 }
5180                         }
5181
5182                         counter_u64_add(pm->pm_runcount, -1);
5183
5184                         KASSERT((int) counter_u64_fetch(pm->pm_runcount) >= 0,
5185                             ("[pmc,%d] runcount is %d", __LINE__, ri));
5186
5187                         (void) pcd->pcd_config_pmc(cpu, adjri, NULL);
5188                 }
5189
5190                 /*
5191                  * Inform the MD layer of this pseudo "context switch
5192                  * out"
5193                  */
5194                 (void) md->pmd_switch_out(pmc_pcpu[cpu], pp);
5195
5196                 critical_exit(); /* ok to be pre-empted now */
5197
5198                 /*
5199                  * Unlink this process from the PMCs that are
5200                  * targeting it.  This will send a signal to
5201                  * all PMC owner's whose PMCs are orphaned.
5202                  *
5203                  * Log PMC value at exit time if requested.
5204                  */
5205                 for (ri = 0; ri < md->pmd_npmc; ri++)
5206                         if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) {
5207                                 if (pm->pm_flags & PMC_F_NEEDS_LOGFILE &&
5208                                     PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm)))
5209                                         pmclog_process_procexit(pm, pp);
5210                                 pmc_unlink_target_process(pm, pp);
5211                         }
5212                 free(pp, M_PMC);
5213
5214         } else
5215                 critical_exit(); /* pp == NULL */
5216
5217
5218         /*
5219          * If the process owned PMCs, free them up and free up
5220          * memory.
5221          */
5222         if ((po = pmc_find_owner_descriptor(p)) != NULL) {
5223                 pmc_remove_owner(po);
5224                 pmc_destroy_owner_descriptor(po);
5225         }
5226
5227         sx_xunlock(&pmc_sx);
5228 }
5229
5230 /*
5231  * Handle a process fork.
5232  *
5233  * If the parent process 'p1' is under HWPMC monitoring, then copy
5234  * over any attached PMCs that have 'do_descendants' semantics.
5235  */
5236
5237 static void
5238 pmc_process_fork(void *arg __unused, struct proc *p1, struct proc *newproc,
5239     int flags)
5240 {
5241         int is_using_hwpmcs;
5242         unsigned int ri;
5243         uint32_t do_descendants;
5244         struct pmc *pm;
5245         struct pmc_owner *po;
5246         struct pmc_process *ppnew, *ppold;
5247
5248         (void) flags;           /* unused parameter */
5249
5250         PROC_LOCK(p1);
5251         is_using_hwpmcs = p1->p_flag & P_HWPMC;
5252         PROC_UNLOCK(p1);
5253
5254         /*
5255          * If there are system-wide sampling PMCs active, we need to
5256          * log all fork events to their owner's logs.
5257          */
5258         epoch_enter_preempt(global_epoch_preempt);
5259         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5260             if (po->po_flags & PMC_PO_OWNS_LOGFILE) {
5261                     pmclog_process_procfork(po, p1->p_pid, newproc->p_pid);
5262                         pmclog_process_proccreate(po, newproc, 1);
5263                 }
5264         epoch_exit_preempt(global_epoch_preempt);
5265
5266         if (!is_using_hwpmcs)
5267                 return;
5268
5269         PMC_GET_SX_XLOCK();
5270         PMCDBG4(PMC,FRK,1, "process-fork proc=%p (%d, %s) -> %p", p1,
5271             p1->p_pid, p1->p_comm, newproc);
5272
5273         /*
5274          * If the parent process (curthread->td_proc) is a
5275          * target of any PMCs, look for PMCs that are to be
5276          * inherited, and link these into the new process
5277          * descriptor.
5278          */
5279         if ((ppold = pmc_find_process_descriptor(curthread->td_proc,
5280                  PMC_FLAG_NONE)) == NULL)
5281                 goto done;              /* nothing to do */
5282
5283         do_descendants = 0;
5284         for (ri = 0; ri < md->pmd_npmc; ri++)
5285                 if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL)
5286                         do_descendants |= pm->pm_flags & PMC_F_DESCENDANTS;
5287         if (do_descendants == 0) /* nothing to do */
5288                 goto done;
5289
5290         /*
5291          * Now mark the new process as being tracked by this driver.
5292          */
5293         PROC_LOCK(newproc);
5294         newproc->p_flag |= P_HWPMC;
5295         PROC_UNLOCK(newproc);
5296
5297         /* allocate a descriptor for the new process  */
5298         if ((ppnew = pmc_find_process_descriptor(newproc,
5299                  PMC_FLAG_ALLOCATE)) == NULL)
5300                 goto done;
5301
5302         /*
5303          * Run through all PMCs that were targeting the old process
5304          * and which specified F_DESCENDANTS and attach them to the
5305          * new process.
5306          *
5307          * Log the fork event to all owners of PMCs attached to this
5308          * process, if not already logged.
5309          */
5310         for (ri = 0; ri < md->pmd_npmc; ri++)
5311                 if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL &&
5312                     (pm->pm_flags & PMC_F_DESCENDANTS)) {
5313                         pmc_link_target_process(pm, ppnew);
5314                         po = pm->pm_owner;
5315                         if (po->po_sscount == 0 &&
5316                             po->po_flags & PMC_PO_OWNS_LOGFILE)
5317                                 pmclog_process_procfork(po, p1->p_pid,
5318                                     newproc->p_pid);
5319                 }
5320
5321  done:
5322         sx_xunlock(&pmc_sx);
5323 }
5324
5325 static void
5326 pmc_process_threadcreate(struct thread *td)
5327 {
5328         struct pmc_owner *po;
5329
5330         epoch_enter_preempt(global_epoch_preempt);
5331         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5332             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
5333                         pmclog_process_threadcreate(po, td, 1);
5334         epoch_exit_preempt(global_epoch_preempt);
5335 }
5336
5337 static void
5338 pmc_process_threadexit(struct thread *td)
5339 {
5340         struct pmc_owner *po;
5341
5342         epoch_enter_preempt(global_epoch_preempt);
5343         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5344             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
5345                         pmclog_process_threadexit(po, td);
5346         epoch_exit_preempt(global_epoch_preempt);
5347 }
5348
5349 static void
5350 pmc_process_proccreate(struct proc *p)
5351 {
5352         struct pmc_owner *po;
5353
5354         epoch_enter_preempt(global_epoch_preempt);
5355         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5356             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
5357                         pmclog_process_proccreate(po, p, 1 /* sync */);
5358         epoch_exit_preempt(global_epoch_preempt);
5359 }
5360
5361 static void
5362 pmc_process_allproc(struct pmc *pm)
5363 {
5364         struct pmc_owner *po;
5365         struct thread *td;
5366         struct proc *p;
5367
5368         po = pm->pm_owner;
5369         if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
5370                 return;
5371         sx_slock(&allproc_lock);
5372         FOREACH_PROC_IN_SYSTEM(p) {
5373                 pmclog_process_proccreate(po, p, 0 /* sync */);
5374                 PROC_LOCK(p);
5375                 FOREACH_THREAD_IN_PROC(p, td)
5376                         pmclog_process_threadcreate(po, td, 0 /* sync */);
5377                 PROC_UNLOCK(p);
5378         }
5379         sx_sunlock(&allproc_lock);
5380         pmclog_flush(po, 0);
5381 }
5382
5383 static void
5384 pmc_kld_load(void *arg __unused, linker_file_t lf)
5385 {
5386         struct pmc_owner *po;
5387
5388         /*
5389          * Notify owners of system sampling PMCs about KLD operations.
5390          */
5391         epoch_enter_preempt(global_epoch_preempt);
5392         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5393                 if (po->po_flags & PMC_PO_OWNS_LOGFILE)
5394                         pmclog_process_map_in(po, (pid_t) -1,
5395                             (uintfptr_t) lf->address, lf->filename);
5396         epoch_exit_preempt(global_epoch_preempt);
5397
5398         /*
5399          * TODO: Notify owners of (all) process-sampling PMCs too.
5400          */
5401 }
5402
5403 static void
5404 pmc_kld_unload(void *arg __unused, const char *filename __unused,
5405     caddr_t address, size_t size)
5406 {
5407         struct pmc_owner *po;
5408
5409         epoch_enter_preempt(global_epoch_preempt);
5410         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5411                 if (po->po_flags & PMC_PO_OWNS_LOGFILE)
5412                         pmclog_process_map_out(po, (pid_t) -1,
5413                             (uintfptr_t) address, (uintfptr_t) address + size);
5414         epoch_exit_preempt(global_epoch_preempt);
5415
5416         /*
5417          * TODO: Notify owners of process-sampling PMCs.
5418          */
5419 }
5420
5421 /*
5422  * initialization
5423  */
5424 static const char *
5425 pmc_name_of_pmcclass(enum pmc_class class)
5426 {
5427
5428         switch (class) {
5429 #undef  __PMC_CLASS
5430 #define __PMC_CLASS(S,V,D)                                              \
5431         case PMC_CLASS_##S:                                             \
5432                 return #S;
5433         __PMC_CLASSES();
5434         default:
5435                 return ("<unknown>");
5436         }
5437 }
5438
5439 /*
5440  * Base class initializer: allocate structure and set default classes.
5441  */
5442 struct pmc_mdep *
5443 pmc_mdep_alloc(int nclasses)
5444 {
5445         struct pmc_mdep *md;
5446         int     n;
5447
5448         /* SOFT + md classes */
5449         n = 1 + nclasses;
5450         md = malloc(sizeof(struct pmc_mdep) + n *
5451             sizeof(struct pmc_classdep), M_PMC, M_WAITOK|M_ZERO);
5452         md->pmd_nclass = n;
5453
5454         /* Add base class. */
5455         pmc_soft_initialize(md);
5456         return md;
5457 }
5458
5459 void
5460 pmc_mdep_free(struct pmc_mdep *md)
5461 {
5462         pmc_soft_finalize(md);
5463         free(md, M_PMC);
5464 }
5465
5466 static int
5467 generic_switch_in(struct pmc_cpu *pc, struct pmc_process *pp)
5468 {
5469         (void) pc; (void) pp;
5470
5471         return (0);
5472 }
5473
5474 static int
5475 generic_switch_out(struct pmc_cpu *pc, struct pmc_process *pp)
5476 {
5477         (void) pc; (void) pp;
5478
5479         return (0);
5480 }
5481
5482 static struct pmc_mdep *
5483 pmc_generic_cpu_initialize(void)
5484 {
5485         struct pmc_mdep *md;
5486
5487         md = pmc_mdep_alloc(0);
5488
5489         md->pmd_cputype    = PMC_CPU_GENERIC;
5490
5491         md->pmd_pcpu_init  = NULL;
5492         md->pmd_pcpu_fini  = NULL;
5493         md->pmd_switch_in  = generic_switch_in;
5494         md->pmd_switch_out = generic_switch_out;
5495
5496         return (md);
5497 }
5498
5499 static void
5500 pmc_generic_cpu_finalize(struct pmc_mdep *md)
5501 {
5502         (void) md;
5503 }
5504
5505
5506 static int
5507 pmc_initialize(void)
5508 {
5509         int c, cpu, error, n, ri;
5510         unsigned int maxcpu, domain;
5511         struct pcpu *pc;
5512         struct pmc_binding pb;
5513         struct pmc_sample *ps;
5514         struct pmc_classdep *pcd;
5515         struct pmc_samplebuffer *sb;
5516
5517         md = NULL;
5518         error = 0;
5519
5520         pmc_stats.pm_intr_ignored = counter_u64_alloc(M_WAITOK);
5521         pmc_stats.pm_intr_processed = counter_u64_alloc(M_WAITOK);
5522         pmc_stats.pm_intr_bufferfull = counter_u64_alloc(M_WAITOK);
5523         pmc_stats.pm_syscalls = counter_u64_alloc(M_WAITOK);
5524         pmc_stats.pm_syscall_errors = counter_u64_alloc(M_WAITOK);
5525         pmc_stats.pm_buffer_requests = counter_u64_alloc(M_WAITOK);
5526         pmc_stats.pm_buffer_requests_failed = counter_u64_alloc(M_WAITOK);
5527         pmc_stats.pm_log_sweeps = counter_u64_alloc(M_WAITOK);
5528         pmc_stats.pm_merges = counter_u64_alloc(M_WAITOK);
5529         pmc_stats.pm_overwrites = counter_u64_alloc(M_WAITOK);
5530
5531 #ifdef  HWPMC_DEBUG
5532         /* parse debug flags first */
5533         if (TUNABLE_STR_FETCH(PMC_SYSCTL_NAME_PREFIX "debugflags",
5534                 pmc_debugstr, sizeof(pmc_debugstr)))
5535                 pmc_debugflags_parse(pmc_debugstr,
5536                     pmc_debugstr+strlen(pmc_debugstr));
5537 #endif
5538
5539         PMCDBG1(MOD,INI,0, "PMC Initialize (version %x)", PMC_VERSION);
5540
5541         /* check kernel version */
5542         if (pmc_kernel_version != PMC_VERSION) {
5543                 if (pmc_kernel_version == 0)
5544                         printf("hwpmc: this kernel has not been compiled with "
5545                             "'options HWPMC_HOOKS'.\n");
5546                 else
5547                         printf("hwpmc: kernel version (0x%x) does not match "
5548                             "module version (0x%x).\n", pmc_kernel_version,
5549                             PMC_VERSION);
5550                 return EPROGMISMATCH;
5551         }
5552
5553         /*
5554          * check sysctl parameters
5555          */
5556
5557         if (pmc_hashsize <= 0) {
5558                 (void) printf("hwpmc: tunable \"hashsize\"=%d must be "
5559                     "greater than zero.\n", pmc_hashsize);
5560                 pmc_hashsize = PMC_HASH_SIZE;
5561         }
5562
5563         if (pmc_nsamples <= 0 || pmc_nsamples > 65535) {
5564                 (void) printf("hwpmc: tunable \"nsamples\"=%d out of "
5565                     "range.\n", pmc_nsamples);
5566                 pmc_nsamples = PMC_NSAMPLES;
5567         }
5568
5569         if (pmc_callchaindepth <= 0 ||
5570             pmc_callchaindepth > PMC_CALLCHAIN_DEPTH_MAX) {
5571                 (void) printf("hwpmc: tunable \"callchaindepth\"=%d out of "
5572                     "range - using %d.\n", pmc_callchaindepth,
5573                     PMC_CALLCHAIN_DEPTH_MAX);
5574                 pmc_callchaindepth = PMC_CALLCHAIN_DEPTH_MAX;
5575         }
5576
5577         md = pmc_md_initialize();
5578         if (md == NULL) {
5579                 /* Default to generic CPU. */
5580                 md = pmc_generic_cpu_initialize();
5581                 if (md == NULL)
5582                         return (ENOSYS);
5583         }
5584
5585         KASSERT(md->pmd_nclass >= 1 && md->pmd_npmc >= 1,
5586             ("[pmc,%d] no classes or pmcs", __LINE__));
5587
5588         /* Compute the map from row-indices to classdep pointers. */
5589         pmc_rowindex_to_classdep = malloc(sizeof(struct pmc_classdep *) *
5590             md->pmd_npmc, M_PMC, M_WAITOK|M_ZERO);
5591
5592         for (n = 0; n < md->pmd_npmc; n++)
5593                 pmc_rowindex_to_classdep[n] = NULL;
5594         for (ri = c = 0; c < md->pmd_nclass; c++) {
5595                 pcd = &md->pmd_classdep[c];
5596                 for (n = 0; n < pcd->pcd_num; n++, ri++)
5597                         pmc_rowindex_to_classdep[ri] = pcd;
5598         }
5599
5600         KASSERT(ri == md->pmd_npmc,
5601             ("[pmc,%d] npmc miscomputed: ri=%d, md->npmc=%d", __LINE__,
5602             ri, md->pmd_npmc));
5603
5604         maxcpu = pmc_cpu_max();
5605
5606         /* allocate space for the per-cpu array */
5607         pmc_pcpu = malloc(maxcpu * sizeof(struct pmc_cpu *), M_PMC,
5608             M_WAITOK|M_ZERO);
5609
5610         /* per-cpu 'saved values' for managing process-mode PMCs */
5611         pmc_pcpu_saved = malloc(sizeof(pmc_value_t) * maxcpu * md->pmd_npmc,
5612             M_PMC, M_WAITOK);
5613
5614         /* Perform CPU-dependent initialization. */
5615         pmc_save_cpu_binding(&pb);
5616         error = 0;
5617         for (cpu = 0; error == 0 && cpu < maxcpu; cpu++) {
5618                 if (!pmc_cpu_is_active(cpu))
5619                         continue;
5620                 pmc_select_cpu(cpu);
5621                 pmc_pcpu[cpu] = malloc(sizeof(struct pmc_cpu) +
5622                     md->pmd_npmc * sizeof(struct pmc_hw *), M_PMC,
5623                     M_WAITOK|M_ZERO);
5624                 if (md->pmd_pcpu_init)
5625                         error = md->pmd_pcpu_init(md, cpu);
5626                 for (n = 0; error == 0 && n < md->pmd_nclass; n++)
5627                         error = md->pmd_classdep[n].pcd_pcpu_init(md, cpu);
5628         }
5629         pmc_restore_cpu_binding(&pb);
5630
5631         if (error)
5632                 return (error);
5633
5634         /* allocate space for the sample array */
5635         for (cpu = 0; cpu < maxcpu; cpu++) {
5636                 if (!pmc_cpu_is_active(cpu))
5637                         continue;
5638                 pc = pcpu_find(cpu);
5639                 domain = pc->pc_domain;
5640                 sb = malloc_domain(sizeof(struct pmc_samplebuffer) +
5641                         pmc_nsamples * sizeof(struct pmc_sample), M_PMC, domain,
5642                     M_WAITOK|M_ZERO);
5643                 sb->ps_read = sb->ps_write = sb->ps_samples;
5644                 sb->ps_fence = sb->ps_samples + pmc_nsamples;
5645
5646                 KASSERT(pmc_pcpu[cpu] != NULL,
5647                     ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
5648
5649                 sb->ps_callchains = malloc_domain(pmc_callchaindepth * pmc_nsamples *
5650                         sizeof(uintptr_t), M_PMC, domain, M_WAITOK|M_ZERO);
5651
5652                 for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
5653                         ps->ps_pc = sb->ps_callchains +
5654                             (n * pmc_callchaindepth);
5655
5656                 pmc_pcpu[cpu]->pc_sb[PMC_HR] = sb;
5657
5658                 sb = malloc_domain(sizeof(struct pmc_samplebuffer) +
5659                         pmc_nsamples * sizeof(struct pmc_sample), M_PMC, domain,
5660                     M_WAITOK|M_ZERO);
5661                 sb->ps_read = sb->ps_write = sb->ps_samples;
5662                 sb->ps_fence = sb->ps_samples + pmc_nsamples;
5663
5664                 KASSERT(pmc_pcpu[cpu] != NULL,
5665                     ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
5666
5667                 sb->ps_callchains = malloc_domain(pmc_callchaindepth * pmc_nsamples *
5668                         sizeof(uintptr_t), M_PMC, domain, M_WAITOK|M_ZERO);
5669
5670                 for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
5671                         ps->ps_pc = sb->ps_callchains +
5672                             (n * pmc_callchaindepth);
5673
5674                 pmc_pcpu[cpu]->pc_sb[PMC_SR] = sb;
5675
5676                 sb = malloc_domain(sizeof(struct pmc_samplebuffer) +
5677                         pmc_nsamples * sizeof(struct pmc_sample), M_PMC, domain,
5678                     M_WAITOK|M_ZERO);
5679                 sb->ps_read = sb->ps_write = sb->ps_samples;
5680                 sb->ps_fence = sb->ps_samples + pmc_nsamples;
5681
5682                 KASSERT(pmc_pcpu[cpu] != NULL,
5683                     ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
5684
5685                 sb->ps_callchains = malloc_domain(pmc_callchaindepth * pmc_nsamples *
5686                     sizeof(uintptr_t), M_PMC, domain, M_WAITOK|M_ZERO);
5687
5688                 for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
5689                         ps->ps_pc = sb->ps_callchains +
5690                             (n * pmc_callchaindepth);
5691
5692                 pmc_pcpu[cpu]->pc_sb[PMC_UR] = sb;
5693         }
5694
5695         /* allocate space for the row disposition array */
5696         pmc_pmcdisp = malloc(sizeof(enum pmc_mode) * md->pmd_npmc,
5697             M_PMC, M_WAITOK|M_ZERO);
5698
5699         /* mark all PMCs as available */
5700         for (n = 0; n < (int) md->pmd_npmc; n++)
5701                 PMC_MARK_ROW_FREE(n);
5702
5703         /* allocate thread hash tables */
5704         pmc_ownerhash = hashinit(pmc_hashsize, M_PMC,
5705             &pmc_ownerhashmask);
5706
5707         pmc_processhash = hashinit(pmc_hashsize, M_PMC,
5708             &pmc_processhashmask);
5709         mtx_init(&pmc_processhash_mtx, "pmc-process-hash", "pmc-leaf",
5710             MTX_SPIN);
5711
5712         CK_LIST_INIT(&pmc_ss_owners);
5713         pmc_ss_count = 0;
5714
5715         /* allocate a pool of spin mutexes */
5716         pmc_mtxpool = mtx_pool_create("pmc-leaf", pmc_mtxpool_size,
5717             MTX_SPIN);
5718
5719         PMCDBG4(MOD,INI,1, "pmc_ownerhash=%p, mask=0x%lx "
5720             "targethash=%p mask=0x%lx", pmc_ownerhash, pmc_ownerhashmask,
5721             pmc_processhash, pmc_processhashmask);
5722
5723         /* Initialize a spin mutex for the thread free list. */
5724         mtx_init(&pmc_threadfreelist_mtx, "pmc-threadfreelist", "pmc-leaf",
5725             MTX_SPIN);
5726
5727         /*
5728          * Initialize the callout to monitor the thread free list.
5729          * This callout will also handle the initial population of the list.
5730          */
5731         taskqgroup_config_gtask_init(NULL, &free_gtask, pmc_thread_descriptor_pool_free_task, "thread descriptor pool free task");
5732
5733         /* register process {exit,fork,exec} handlers */
5734         pmc_exit_tag = EVENTHANDLER_REGISTER(process_exit,
5735             pmc_process_exit, NULL, EVENTHANDLER_PRI_ANY);
5736         pmc_fork_tag = EVENTHANDLER_REGISTER(process_fork,
5737             pmc_process_fork, NULL, EVENTHANDLER_PRI_ANY);
5738
5739         /* register kld event handlers */
5740         pmc_kld_load_tag = EVENTHANDLER_REGISTER(kld_load, pmc_kld_load,
5741             NULL, EVENTHANDLER_PRI_ANY);
5742         pmc_kld_unload_tag = EVENTHANDLER_REGISTER(kld_unload, pmc_kld_unload,
5743             NULL, EVENTHANDLER_PRI_ANY);
5744
5745         /* initialize logging */
5746         pmclog_initialize();
5747
5748         /* set hook functions */
5749         pmc_intr = md->pmd_intr;
5750         wmb();
5751         pmc_hook = pmc_hook_handler;
5752
5753         if (error == 0) {
5754                 printf(PMC_MODULE_NAME ":");
5755                 for (n = 0; n < (int) md->pmd_nclass; n++) {
5756                         pcd = &md->pmd_classdep[n];
5757                         printf(" %s/%d/%d/0x%b",
5758                             pmc_name_of_pmcclass(pcd->pcd_class),
5759                             pcd->pcd_num,
5760                             pcd->pcd_width,
5761                             pcd->pcd_caps,
5762                             "\20"
5763                             "\1INT\2USR\3SYS\4EDG\5THR"
5764                             "\6REA\7WRI\10INV\11QUA\12PRC"
5765                             "\13TAG\14CSC");
5766                 }
5767                 printf("\n");
5768         }
5769
5770         return (error);
5771 }
5772
5773 /* prepare to be unloaded */
5774 static void
5775 pmc_cleanup(void)
5776 {
5777         int c, cpu;
5778         unsigned int maxcpu;
5779         struct pmc_ownerhash *ph;
5780         struct pmc_owner *po, *tmp;
5781         struct pmc_binding pb;
5782 #ifdef  HWPMC_DEBUG
5783         struct pmc_processhash *prh;
5784 #endif
5785
5786         PMCDBG0(MOD,INI,0, "cleanup");
5787
5788         /* switch off sampling */
5789         CPU_FOREACH(cpu)
5790                 DPCPU_ID_SET(cpu, pmc_sampled, 0);
5791         pmc_intr = NULL;
5792
5793         sx_xlock(&pmc_sx);
5794         if (pmc_hook == NULL) { /* being unloaded already */
5795                 sx_xunlock(&pmc_sx);
5796                 return;
5797         }
5798
5799         pmc_hook = NULL; /* prevent new threads from entering module */
5800
5801         /* deregister event handlers */
5802         EVENTHANDLER_DEREGISTER(process_fork, pmc_fork_tag);
5803         EVENTHANDLER_DEREGISTER(process_exit, pmc_exit_tag);
5804         EVENTHANDLER_DEREGISTER(kld_load, pmc_kld_load_tag);
5805         EVENTHANDLER_DEREGISTER(kld_unload, pmc_kld_unload_tag);
5806
5807         /* send SIGBUS to all owner threads, free up allocations */
5808         if (pmc_ownerhash)
5809                 for (ph = pmc_ownerhash;
5810                      ph <= &pmc_ownerhash[pmc_ownerhashmask];
5811                      ph++) {
5812                         LIST_FOREACH_SAFE(po, ph, po_next, tmp) {
5813                                 pmc_remove_owner(po);
5814
5815                                 /* send SIGBUS to owner processes */
5816                                 PMCDBG3(MOD,INI,2, "cleanup signal proc=%p "
5817                                     "(%d, %s)", po->po_owner,
5818                                     po->po_owner->p_pid,
5819                                     po->po_owner->p_comm);
5820
5821                                 PROC_LOCK(po->po_owner);
5822                                 kern_psignal(po->po_owner, SIGBUS);
5823                                 PROC_UNLOCK(po->po_owner);
5824
5825                                 pmc_destroy_owner_descriptor(po);
5826                         }
5827                 }
5828
5829         /* reclaim allocated data structures */
5830         mtx_destroy(&pmc_threadfreelist_mtx);
5831         pmc_thread_descriptor_pool_drain();
5832
5833         if (pmc_mtxpool)
5834                 mtx_pool_destroy(&pmc_mtxpool);
5835
5836         mtx_destroy(&pmc_processhash_mtx);
5837         taskqgroup_config_gtask_deinit(&free_gtask);
5838         if (pmc_processhash) {
5839 #ifdef  HWPMC_DEBUG
5840                 struct pmc_process *pp;
5841
5842                 PMCDBG0(MOD,INI,3, "destroy process hash");
5843                 for (prh = pmc_processhash;
5844                      prh <= &pmc_processhash[pmc_processhashmask];
5845                      prh++)
5846                         LIST_FOREACH(pp, prh, pp_next)
5847                             PMCDBG1(MOD,INI,3, "pid=%d", pp->pp_proc->p_pid);
5848 #endif
5849
5850                 hashdestroy(pmc_processhash, M_PMC, pmc_processhashmask);
5851                 pmc_processhash = NULL;
5852         }
5853
5854         if (pmc_ownerhash) {
5855                 PMCDBG0(MOD,INI,3, "destroy owner hash");
5856                 hashdestroy(pmc_ownerhash, M_PMC, pmc_ownerhashmask);
5857                 pmc_ownerhash = NULL;
5858         }
5859
5860         KASSERT(CK_LIST_EMPTY(&pmc_ss_owners),
5861             ("[pmc,%d] Global SS owner list not empty", __LINE__));
5862         KASSERT(pmc_ss_count == 0,
5863             ("[pmc,%d] Global SS count not empty", __LINE__));
5864
5865         /* do processor and pmc-class dependent cleanup */
5866         maxcpu = pmc_cpu_max();
5867
5868         PMCDBG0(MOD,INI,3, "md cleanup");
5869         if (md) {
5870                 pmc_save_cpu_binding(&pb);
5871                 for (cpu = 0; cpu < maxcpu; cpu++) {
5872                         PMCDBG2(MOD,INI,1,"pmc-cleanup cpu=%d pcs=%p",
5873                             cpu, pmc_pcpu[cpu]);
5874                         if (!pmc_cpu_is_active(cpu) || pmc_pcpu[cpu] == NULL)
5875                                 continue;
5876                         pmc_select_cpu(cpu);
5877                         for (c = 0; c < md->pmd_nclass; c++)
5878                                 md->pmd_classdep[c].pcd_pcpu_fini(md, cpu);
5879                         if (md->pmd_pcpu_fini)
5880                                 md->pmd_pcpu_fini(md, cpu);
5881                 }
5882
5883                 if (md->pmd_cputype == PMC_CPU_GENERIC)
5884                         pmc_generic_cpu_finalize(md);
5885                 else
5886                         pmc_md_finalize(md);
5887
5888                 pmc_mdep_free(md);
5889                 md = NULL;
5890                 pmc_restore_cpu_binding(&pb);
5891         }
5892
5893         /* Free per-cpu descriptors. */
5894         for (cpu = 0; cpu < maxcpu; cpu++) {
5895                 if (!pmc_cpu_is_active(cpu))
5896                         continue;
5897                 KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_HR] != NULL,
5898                     ("[pmc,%d] Null hw cpu sample buffer cpu=%d", __LINE__,
5899                         cpu));
5900                 KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_SR] != NULL,
5901                     ("[pmc,%d] Null sw cpu sample buffer cpu=%d", __LINE__,
5902                         cpu));
5903                 KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_UR] != NULL,
5904                     ("[pmc,%d] Null userret cpu sample buffer cpu=%d", __LINE__,
5905                         cpu));
5906                 free_domain(pmc_pcpu[cpu]->pc_sb[PMC_HR]->ps_callchains, M_PMC);
5907                 free_domain(pmc_pcpu[cpu]->pc_sb[PMC_HR], M_PMC);
5908                 free_domain(pmc_pcpu[cpu]->pc_sb[PMC_SR]->ps_callchains, M_PMC);
5909                 free_domain(pmc_pcpu[cpu]->pc_sb[PMC_SR], M_PMC);
5910                 free_domain(pmc_pcpu[cpu]->pc_sb[PMC_UR]->ps_callchains, M_PMC);
5911                 free_domain(pmc_pcpu[cpu]->pc_sb[PMC_UR], M_PMC);
5912                 free_domain(pmc_pcpu[cpu], M_PMC);
5913         }
5914
5915         free(pmc_pcpu, M_PMC);
5916         pmc_pcpu = NULL;
5917
5918         free(pmc_pcpu_saved, M_PMC);
5919         pmc_pcpu_saved = NULL;
5920
5921         if (pmc_pmcdisp) {
5922                 free(pmc_pmcdisp, M_PMC);
5923                 pmc_pmcdisp = NULL;
5924         }
5925
5926         if (pmc_rowindex_to_classdep) {
5927                 free(pmc_rowindex_to_classdep, M_PMC);
5928                 pmc_rowindex_to_classdep = NULL;
5929         }
5930
5931         pmclog_shutdown();
5932         counter_u64_free(pmc_stats.pm_intr_ignored);
5933         counter_u64_free(pmc_stats.pm_intr_processed);
5934         counter_u64_free(pmc_stats.pm_intr_bufferfull);
5935         counter_u64_free(pmc_stats.pm_syscalls);
5936         counter_u64_free(pmc_stats.pm_syscall_errors);
5937         counter_u64_free(pmc_stats.pm_buffer_requests);
5938         counter_u64_free(pmc_stats.pm_buffer_requests_failed);
5939         counter_u64_free(pmc_stats.pm_log_sweeps);
5940         counter_u64_free(pmc_stats.pm_merges);
5941         counter_u64_free(pmc_stats.pm_overwrites);
5942         sx_xunlock(&pmc_sx);    /* we are done */
5943 }
5944
5945 /*
5946  * The function called at load/unload.
5947  */
5948
5949 static int
5950 load (struct module *module __unused, int cmd, void *arg __unused)
5951 {
5952         int error;
5953
5954         error = 0;
5955
5956         switch (cmd) {
5957         case MOD_LOAD :
5958                 /* initialize the subsystem */
5959                 error = pmc_initialize();
5960                 if (error != 0)
5961                         break;
5962                 PMCDBG2(MOD,INI,1, "syscall=%d maxcpu=%d",
5963                     pmc_syscall_num, pmc_cpu_max());
5964                 break;
5965
5966
5967         case MOD_UNLOAD :
5968         case MOD_SHUTDOWN:
5969                 pmc_cleanup();
5970                 PMCDBG0(MOD,INI,1, "unloaded");
5971                 break;
5972
5973         default :
5974                 error = EINVAL; /* XXX should panic(9) */
5975                 break;
5976         }
5977
5978         return error;
5979 }