sys/dev/hwpmc/hwpmc_mod.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2003-2008 Joseph Koshy
   5  * Copyright (c) 2007 The FreeBSD Foundation
   6  * Copyright (c) 2018 Matthew Macy
   7  * All rights reserved.
   8  *
   9  * Portions of this software were developed by A. Joseph Koshy under
  10  * sponsorship from the FreeBSD Foundation and Google, Inc.
  11  *
  12  * Redistribution and use in source and binary forms, with or without
  13  * modification, are permitted provided that the following conditions
  14  * are met:
  15  * 1. Redistributions of source code must retain the above copyright
  16  *    notice, this list of conditions and the following disclaimer.
  17  * 2. Redistributions in binary form must reproduce the above copyright
  18  *    notice, this list of conditions and the following disclaimer in the
  19  *    documentation and/or other materials provided with the distribution.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  24  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  25  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  30  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  31  * SUCH DAMAGE.
  32  *
  33  */
  34
  35 #include <sys/cdefs.h>
  36 __FBSDID("$FreeBSD$");
  37
  38 #include <sys/param.h>
  39 #include <sys/eventhandler.h>
  40 #include <sys/gtaskqueue.h>
  41 #include <sys/jail.h>
  42 #include <sys/kernel.h>
  43 #include <sys/kthread.h>
  44 #include <sys/limits.h>
  45 #include <sys/lock.h>
  46 #include <sys/malloc.h>
  47 #include <sys/module.h>
  48 #include <sys/mount.h>
  49 #include <sys/mutex.h>
  50 #include <sys/pmc.h>
  51 #include <sys/pmckern.h>
  52 #include <sys/pmclog.h>
  53 #include <sys/priv.h>
  54 #include <sys/proc.h>
  55 #include <sys/queue.h>
  56 #include <sys/resourcevar.h>
  57 #include <sys/rwlock.h>
  58 #include <sys/sched.h>
  59 #include <sys/signalvar.h>
  60 #include <sys/smp.h>
  61 #include <sys/sx.h>
  62 #include <sys/sysctl.h>
  63 #include <sys/sysent.h>
  64 #include <sys/systm.h>
  65 #include <sys/vnode.h>
  66
  67 #include <sys/linker.h>         /* needs to be after <sys/malloc.h> */
  68
  69 #include <machine/atomic.h>
  70 #include <machine/md_var.h>
  71
  72 #include <vm/vm.h>
  73 #include <vm/vm_extern.h>
  74 #include <vm/pmap.h>
  75 #include <vm/vm_map.h>
  76 #include <vm/vm_object.h>
  77
  78 #include "hwpmc_soft.h"
  79
  80 #ifdef NUMA
  81 #define NDOMAINS vm_ndomains
  82 #else
  83 #define NDOMAINS 1
  84 #define malloc_domain(size, type, domain, flags) malloc((size), (type), (flags))
  85 #define free_domain(addr, type) free(addr, type)
  86 #endif
  87
  88 /*
  89  * Types
  90  */
  91
  92 enum pmc_flags {
  93         PMC_FLAG_NONE     = 0x00, /* do nothing */
  94         PMC_FLAG_REMOVE   = 0x01, /* atomically remove entry from hash */
  95         PMC_FLAG_ALLOCATE = 0x02, /* add entry to hash if not found */
  96         PMC_FLAG_NOWAIT   = 0x04, /* do not wait for mallocs */
  97 };
  98
  99 /*
 100  * The offset in sysent where the syscall is allocated.
 101  */
 102
 103 static int pmc_syscall_num = NO_SYSCALL;
 104 struct pmc_cpu          **pmc_pcpu;      /* per-cpu state */
 105 pmc_value_t             *pmc_pcpu_saved; /* saved PMC values: CSW handling */
 106
 107 #define PMC_PCPU_SAVED(C,R)     pmc_pcpu_saved[(R) + md->pmd_npmc*(C)]
 108
 109 struct mtx_pool         *pmc_mtxpool;
 110 static int              *pmc_pmcdisp;    /* PMC row dispositions */
 111
 112 #define PMC_ROW_DISP_IS_FREE(R)         (pmc_pmcdisp[(R)] == 0)
 113 #define PMC_ROW_DISP_IS_THREAD(R)       (pmc_pmcdisp[(R)] > 0)
 114 #define PMC_ROW_DISP_IS_STANDALONE(R)   (pmc_pmcdisp[(R)] < 0)
 115
 116 #define PMC_MARK_ROW_FREE(R) do {                                         \
 117         pmc_pmcdisp[(R)] = 0;                                             \
 118 } while (0)
 119
 120 #define PMC_MARK_ROW_STANDALONE(R) do {                                   \
 121         KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \
 122                     __LINE__));                                           \
 123         atomic_add_int(&pmc_pmcdisp[(R)], -1);                            \
 124         KASSERT(pmc_pmcdisp[(R)] >= (-pmc_cpu_max_active()),              \
 125                 ("[pmc,%d] row disposition error", __LINE__));            \
 126 } while (0)
 127
 128 #define PMC_UNMARK_ROW_STANDALONE(R) do {                                 \
 129         atomic_add_int(&pmc_pmcdisp[(R)], 1);                             \
 130         KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \
 131                     __LINE__));                                           \
 132 } while (0)
 133
 134 #define PMC_MARK_ROW_THREAD(R) do {                                       \
 135         KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \
 136                     __LINE__));                                           \
 137         atomic_add_int(&pmc_pmcdisp[(R)], 1);                             \
 138 } while (0)
 139
 140 #define PMC_UNMARK_ROW_THREAD(R) do {                                     \
 141         atomic_add_int(&pmc_pmcdisp[(R)], -1);                            \
 142         KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \
 143                     __LINE__));                                           \
 144 } while (0)
 145
 146
 147 /* various event handlers */
 148 static eventhandler_tag pmc_exit_tag, pmc_fork_tag, pmc_kld_load_tag,
 149     pmc_kld_unload_tag;
 150
 151 /* Module statistics */
 152 struct pmc_driverstats pmc_stats;
 153
 154
 155 /* Machine/processor dependent operations */
 156 static struct pmc_mdep  *md;
 157
 158 /*
 159  * Hash tables mapping owner processes and target threads to PMCs.
 160  */
 161
 162 struct mtx pmc_processhash_mtx;         /* spin mutex */
 163 static u_long pmc_processhashmask;
 164 static LIST_HEAD(pmc_processhash, pmc_process)  *pmc_processhash;
 165
 166 /*
 167  * Hash table of PMC owner descriptors.  This table is protected by
 168  * the shared PMC "sx" lock.
 169  */
 170
 171 static u_long pmc_ownerhashmask;
 172 static LIST_HEAD(pmc_ownerhash, pmc_owner)      *pmc_ownerhash;
 173
 174 /*
 175  * List of PMC owners with system-wide sampling PMCs.
 176  */
 177
 178 static CK_LIST_HEAD(, pmc_owner)                        pmc_ss_owners;
 179
 180 /*
 181  * List of free thread entries. This is protected by the spin
 182  * mutex.
 183  */
 184 static struct mtx pmc_threadfreelist_mtx;       /* spin mutex */
 185 static LIST_HEAD(, pmc_thread)                  pmc_threadfreelist;
 186 static int pmc_threadfreelist_entries=0;
 187 #define THREADENTRY_SIZE                                                \
 188 (sizeof(struct pmc_thread) + (md->pmd_npmc * sizeof(struct pmc_threadpmcstate)))
 189
 190 /*
 191  * Task to free thread descriptors
 192  */
 193 static struct grouptask free_gtask;
 194
 195 /*
 196  * A map of row indices to classdep structures.
 197  */
 198 static struct pmc_classdep **pmc_rowindex_to_classdep;
 199
 200 /*
 201  * Prototypes
 202  */
 203
 204 #ifdef  HWPMC_DEBUG
 205 static int      pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS);
 206 static int      pmc_debugflags_parse(char *newstr, char *fence);
 207 #endif
 208
 209 static int      load(struct module *module, int cmd, void *arg);
 210 static int      pmc_add_sample(int cpu, int ring, struct pmc *pm,
 211     struct trapframe *tf, int inuserspace);
 212 static void     pmc_add_thread_descriptors_from_proc(struct proc *p,
 213     struct pmc_process *pp);
 214 static int      pmc_attach_process(struct proc *p, struct pmc *pm);
 215 static struct pmc *pmc_allocate_pmc_descriptor(void);
 216 static struct pmc_owner *pmc_allocate_owner_descriptor(struct proc *p);
 217 static int      pmc_attach_one_process(struct proc *p, struct pmc *pm);
 218 static int      pmc_can_allocate_rowindex(struct proc *p, unsigned int ri,
 219     int cpu);
 220 static int      pmc_can_attach(struct pmc *pm, struct proc *p);
 221 static void     pmc_capture_user_callchain(int cpu, int soft, struct trapframe *tf);
 222 static void     pmc_cleanup(void);
 223 static int      pmc_detach_process(struct proc *p, struct pmc *pm);
 224 static int      pmc_detach_one_process(struct proc *p, struct pmc *pm,
 225     int flags);
 226 static void     pmc_destroy_owner_descriptor(struct pmc_owner *po);
 227 static void     pmc_destroy_pmc_descriptor(struct pmc *pm);
 228 static void     pmc_destroy_process_descriptor(struct pmc_process *pp);
 229 static struct pmc_owner *pmc_find_owner_descriptor(struct proc *p);
 230 static int      pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm);
 231 static struct pmc *pmc_find_pmc_descriptor_in_process(struct pmc_owner *po,
 232     pmc_id_t pmc);
 233 static struct pmc_process *pmc_find_process_descriptor(struct proc *p,
 234     uint32_t mode);
 235 static struct pmc_thread *pmc_find_thread_descriptor(struct pmc_process *pp,
 236     struct thread *td, uint32_t mode);
 237 static void     pmc_force_context_switch(void);
 238 static void     pmc_link_target_process(struct pmc *pm,
 239     struct pmc_process *pp);
 240 static void     pmc_log_all_process_mappings(struct pmc_owner *po);
 241 static void     pmc_log_kernel_mappings(struct pmc *pm);
 242 static void     pmc_log_process_mappings(struct pmc_owner *po, struct proc *p);
 243 static void     pmc_maybe_remove_owner(struct pmc_owner *po);
 244 static void     pmc_process_csw_in(struct thread *td);
 245 static void     pmc_process_csw_out(struct thread *td);
 246 static void     pmc_process_exit(void *arg, struct proc *p);
 247 static void     pmc_process_fork(void *arg, struct proc *p1,
 248     struct proc *p2, int n);
 249 static void     pmc_process_samples(int cpu, int soft);
 250 static void     pmc_release_pmc_descriptor(struct pmc *pmc);
 251 static void     pmc_process_thread_add(struct thread *td);
 252 static void     pmc_process_thread_delete(struct thread *td);
 253 static void     pmc_process_thread_userret(struct thread *td);
 254 static void     pmc_remove_owner(struct pmc_owner *po);
 255 static void     pmc_remove_process_descriptor(struct pmc_process *pp);
 256 static void     pmc_restore_cpu_binding(struct pmc_binding *pb);
 257 static void     pmc_save_cpu_binding(struct pmc_binding *pb);
 258 static void     pmc_select_cpu(int cpu);
 259 static int      pmc_start(struct pmc *pm);
 260 static int      pmc_stop(struct pmc *pm);
 261 static int      pmc_syscall_handler(struct thread *td, void *syscall_args);
 262 static struct pmc_thread *pmc_thread_descriptor_pool_alloc(void);
 263 static void     pmc_thread_descriptor_pool_drain(void);
 264 static void     pmc_thread_descriptor_pool_free(struct pmc_thread *pt);
 265 static void     pmc_unlink_target_process(struct pmc *pmc,
 266     struct pmc_process *pp);
 267 static int generic_switch_in(struct pmc_cpu *pc, struct pmc_process *pp);
 268 static int generic_switch_out(struct pmc_cpu *pc, struct pmc_process *pp);
 269 static struct pmc_mdep *pmc_generic_cpu_initialize(void);
 270 static void pmc_generic_cpu_finalize(struct pmc_mdep *md);
 271 static void pmc_post_callchain_callback(void);
 272 static void pmc_process_threadcreate(struct thread *td);
 273 static void pmc_process_threadexit(struct thread *td);
 274 static void pmc_process_proccreate(struct proc *p);
 275 static void pmc_process_allproc(struct pmc *pm);
 276
 277 /*
 278  * Kernel tunables and sysctl(8) interface.
 279  */
 280
 281 SYSCTL_DECL(_kern_hwpmc);
 282 SYSCTL_NODE(_kern_hwpmc, OID_AUTO, stats, CTLFLAG_RW, 0, "HWPMC stats");
 283
 284
 285 /* Stats. */
 286 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, intr_ignored, CTLFLAG_RW,
 287                                    &pmc_stats.pm_intr_ignored, "# of interrupts ignored");
 288 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, intr_processed, CTLFLAG_RW,
 289                                    &pmc_stats.pm_intr_processed, "# of interrupts processed");
 290 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, intr_bufferfull, CTLFLAG_RW,
 291                                    &pmc_stats.pm_intr_bufferfull, "# of interrupts where buffer was full");
 292 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, syscalls, CTLFLAG_RW,
 293                                    &pmc_stats.pm_syscalls, "# of syscalls");
 294 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, syscall_errors, CTLFLAG_RW,
 295                                    &pmc_stats.pm_syscall_errors, "# of syscall_errors");
 296 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, buffer_requests, CTLFLAG_RW,
 297                                    &pmc_stats.pm_buffer_requests, "# of buffer requests");
 298 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, buffer_requests_failed, CTLFLAG_RW,
 299                                    &pmc_stats.pm_buffer_requests_failed, "# of buffer requests which failed");
 300 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, log_sweeps, CTLFLAG_RW,
 301                                    &pmc_stats.pm_log_sweeps, "# of ?");
 302 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, merges, CTLFLAG_RW,
 303                                    &pmc_stats.pm_merges, "# of times kernel stack was found for user trace");
 304 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, overwrites, CTLFLAG_RW,
 305                                    &pmc_stats.pm_overwrites, "# of times a sample was overwritten before being logged");
 306
 307 static int pmc_callchaindepth = PMC_CALLCHAIN_DEPTH;
 308 SYSCTL_INT(_kern_hwpmc, OID_AUTO, callchaindepth, CTLFLAG_RDTUN,
 309     &pmc_callchaindepth, 0, "depth of call chain records");
 310
 311 char pmc_cpuid[64];
 312 SYSCTL_STRING(_kern_hwpmc, OID_AUTO, cpuid, CTLFLAG_RD,
 313         pmc_cpuid, 0, "cpu version string");
 314 #ifdef  HWPMC_DEBUG
 315 struct pmc_debugflags pmc_debugflags = PMC_DEBUG_DEFAULT_FLAGS;
 316 char    pmc_debugstr[PMC_DEBUG_STRSIZE];
 317 TUNABLE_STR(PMC_SYSCTL_NAME_PREFIX "debugflags", pmc_debugstr,
 318     sizeof(pmc_debugstr));
 319 SYSCTL_PROC(_kern_hwpmc, OID_AUTO, debugflags,
 320     CTLTYPE_STRING | CTLFLAG_RWTUN | CTLFLAG_NOFETCH,
 321     0, 0, pmc_debugflags_sysctl_handler, "A", "debug flags");
 322 #endif
 323
 324
 325 /*
 326  * kern.hwpmc.hashrows -- determines the number of rows in the
 327  * of the hash table used to look up threads
 328  */
 329
 330 static int pmc_hashsize = PMC_HASH_SIZE;
 331 SYSCTL_INT(_kern_hwpmc, OID_AUTO, hashsize, CTLFLAG_RDTUN,
 332     &pmc_hashsize, 0, "rows in hash tables");
 333
 334 /*
 335  * kern.hwpmc.nsamples --- number of PC samples/callchain stacks per CPU
 336  */
 337
 338 static int pmc_nsamples = PMC_NSAMPLES;
 339 SYSCTL_INT(_kern_hwpmc, OID_AUTO, nsamples, CTLFLAG_RDTUN,
 340     &pmc_nsamples, 0, "number of PC samples per CPU");
 341
 342
 343 /*
 344  * kern.hwpmc.mtxpoolsize -- number of mutexes in the mutex pool.
 345  */
 346
 347 static int pmc_mtxpool_size = PMC_MTXPOOL_SIZE;
 348 SYSCTL_INT(_kern_hwpmc, OID_AUTO, mtxpoolsize, CTLFLAG_RDTUN,
 349     &pmc_mtxpool_size, 0, "size of spin mutex pool");
 350
 351
 352 /*
 353  * kern.hwpmc.threadfreelist_entries -- number of free entries
 354  */
 355
 356 SYSCTL_INT(_kern_hwpmc, OID_AUTO, threadfreelist_entries, CTLFLAG_RD,
 357     &pmc_threadfreelist_entries, 0, "number of avalable thread entries");
 358
 359
 360 /*
 361  * kern.hwpmc.threadfreelist_max -- maximum number of free entries
 362  */
 363
 364 static int pmc_threadfreelist_max = PMC_THREADLIST_MAX;
 365 SYSCTL_INT(_kern_hwpmc, OID_AUTO, threadfreelist_max, CTLFLAG_RW,
 366     &pmc_threadfreelist_max, 0,
 367     "maximum number of available thread entries before freeing some");
 368
 369
 370 /*
 371  * security.bsd.unprivileged_syspmcs -- allow non-root processes to
 372  * allocate system-wide PMCs.
 373  *
 374  * Allowing unprivileged processes to allocate system PMCs is convenient
 375  * if system-wide measurements need to be taken concurrently with other
 376  * per-process measurements.  This feature is turned off by default.
 377  */
 378
 379 static int pmc_unprivileged_syspmcs = 0;
 380 SYSCTL_INT(_security_bsd, OID_AUTO, unprivileged_syspmcs, CTLFLAG_RWTUN,
 381     &pmc_unprivileged_syspmcs, 0,
 382     "allow unprivileged process to allocate system PMCs");
 383
 384 /*
 385  * Hash function.  Discard the lower 2 bits of the pointer since
 386  * these are always zero for our uses.  The hash multiplier is
 387  * round((2^LONG_BIT) * ((sqrt(5)-1)/2)).
 388  */
 389
 390 #if     LONG_BIT == 64
 391 #define _PMC_HM         11400714819323198486u
 392 #elif   LONG_BIT == 32
 393 #define _PMC_HM         2654435769u
 394 #else
 395 #error  Must know the size of 'long' to compile
 396 #endif
 397
 398 #define PMC_HASH_PTR(P,M)       ((((unsigned long) (P) >> 2) * _PMC_HM) & (M))
 399
 400 /*
 401  * Syscall structures
 402  */
 403
 404 /* The `sysent' for the new syscall */
 405 static struct sysent pmc_sysent = {
 406         .sy_narg =      2,
 407         .sy_call =      pmc_syscall_handler,
 408 };
 409
 410 static struct syscall_module_data pmc_syscall_mod = {
 411         .chainevh =     load,
 412         .chainarg =     NULL,
 413         .offset =       &pmc_syscall_num,
 414         .new_sysent =   &pmc_sysent,
 415         .old_sysent =   { .sy_narg = 0, .sy_call = NULL },
 416         .flags =        SY_THR_STATIC_KLD,
 417 };
 418
 419 static moduledata_t pmc_mod = {
 420         .name =         PMC_MODULE_NAME,
 421         .evhand =       syscall_module_handler,
 422         .priv =         &pmc_syscall_mod,
 423 };
 424
 425 #ifdef EARLY_AP_STARTUP
 426 DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SYSCALLS, SI_ORDER_ANY);
 427 #else
 428 DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SMP, SI_ORDER_ANY);
 429 #endif
 430 MODULE_VERSION(pmc, PMC_VERSION);
 431
 432 #ifdef  HWPMC_DEBUG
 433 enum pmc_dbgparse_state {
 434         PMCDS_WS,               /* in whitespace */
 435         PMCDS_MAJOR,            /* seen a major keyword */
 436         PMCDS_MINOR
 437 };
 438
 439 static int
 440 pmc_debugflags_parse(char *newstr, char *fence)
 441 {
 442         char c, *p, *q;
 443         struct pmc_debugflags *tmpflags;
 444         int error, found, *newbits, tmp;
 445         size_t kwlen;
 446
 447         tmpflags = malloc(sizeof(*tmpflags), M_PMC, M_WAITOK|M_ZERO);
 448
 449         p = newstr;
 450         error = 0;
 451
 452         for (; p < fence && (c = *p); p++) {
 453
 454                 /* skip white space */
 455                 if (c == ' ' || c == '\t')
 456                         continue;
 457
 458                 /* look for a keyword followed by "=" */
 459                 for (q = p; p < fence && (c = *p) && c != '='; p++)
 460                         ;
 461                 if (c != '=') {
 462                         error = EINVAL;
 463                         goto done;
 464                 }
 465
 466                 kwlen = p - q;
 467                 newbits = NULL;
 468
 469                 /* lookup flag group name */
 470 #define DBG_SET_FLAG_MAJ(S,F)                                           \
 471                 if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0)  \
 472                         newbits = &tmpflags->pdb_ ## F;
 473
 474                 DBG_SET_FLAG_MAJ("cpu",         CPU);
 475                 DBG_SET_FLAG_MAJ("csw",         CSW);
 476                 DBG_SET_FLAG_MAJ("logging",     LOG);
 477                 DBG_SET_FLAG_MAJ("module",      MOD);
 478                 DBG_SET_FLAG_MAJ("md",          MDP);
 479                 DBG_SET_FLAG_MAJ("owner",       OWN);
 480                 DBG_SET_FLAG_MAJ("pmc",         PMC);
 481                 DBG_SET_FLAG_MAJ("process",     PRC);
 482                 DBG_SET_FLAG_MAJ("sampling",    SAM);
 483
 484                 if (newbits == NULL) {
 485                         error = EINVAL;
 486                         goto done;
 487                 }
 488
 489                 p++;            /* skip the '=' */
 490
 491                 /* Now parse the individual flags */
 492                 tmp = 0;
 493         newflag:
 494                 for (q = p; p < fence && (c = *p); p++)
 495                         if (c == ' ' || c == '\t' || c == ',')
 496                                 break;
 497
 498                 /* p == fence or c == ws or c == "," or c == 0 */
 499
 500                 if ((kwlen = p - q) == 0) {
 501                         *newbits = tmp;
 502                         continue;
 503                 }
 504
 505                 found = 0;
 506 #define DBG_SET_FLAG_MIN(S,F)                                           \
 507                 if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0)  \
 508                         tmp |= found = (1 << PMC_DEBUG_MIN_ ## F)
 509
 510                 /* a '*' denotes all possible flags in the group */
 511                 if (kwlen == 1 && *q == '*')
 512                         tmp = found = ~0;
 513                 /* look for individual flag names */
 514                 DBG_SET_FLAG_MIN("allocaterow", ALR);
 515                 DBG_SET_FLAG_MIN("allocate",    ALL);
 516                 DBG_SET_FLAG_MIN("attach",      ATT);
 517                 DBG_SET_FLAG_MIN("bind",        BND);
 518                 DBG_SET_FLAG_MIN("config",      CFG);
 519                 DBG_SET_FLAG_MIN("exec",        EXC);
 520                 DBG_SET_FLAG_MIN("exit",        EXT);
 521                 DBG_SET_FLAG_MIN("find",        FND);
 522                 DBG_SET_FLAG_MIN("flush",       FLS);
 523                 DBG_SET_FLAG_MIN("fork",        FRK);
 524                 DBG_SET_FLAG_MIN("getbuf",      GTB);
 525                 DBG_SET_FLAG_MIN("hook",        PMH);
 526                 DBG_SET_FLAG_MIN("init",        INI);
 527                 DBG_SET_FLAG_MIN("intr",        INT);
 528                 DBG_SET_FLAG_MIN("linktarget",  TLK);
 529                 DBG_SET_FLAG_MIN("mayberemove", OMR);
 530                 DBG_SET_FLAG_MIN("ops",         OPS);
 531                 DBG_SET_FLAG_MIN("read",        REA);
 532                 DBG_SET_FLAG_MIN("register",    REG);
 533                 DBG_SET_FLAG_MIN("release",     REL);
 534                 DBG_SET_FLAG_MIN("remove",      ORM);
 535                 DBG_SET_FLAG_MIN("sample",      SAM);
 536                 DBG_SET_FLAG_MIN("scheduleio",  SIO);
 537                 DBG_SET_FLAG_MIN("select",      SEL);
 538                 DBG_SET_FLAG_MIN("signal",      SIG);
 539                 DBG_SET_FLAG_MIN("swi",         SWI);
 540                 DBG_SET_FLAG_MIN("swo",         SWO);
 541                 DBG_SET_FLAG_MIN("start",       STA);
 542                 DBG_SET_FLAG_MIN("stop",        STO);
 543                 DBG_SET_FLAG_MIN("syscall",     PMS);
 544                 DBG_SET_FLAG_MIN("unlinktarget", TUL);
 545                 DBG_SET_FLAG_MIN("write",       WRI);
 546                 if (found == 0) {
 547                         /* unrecognized flag name */
 548                         error = EINVAL;
 549                         goto done;
 550                 }
 551
 552                 if (c == 0 || c == ' ' || c == '\t') {  /* end of flag group */
 553                         *newbits = tmp;
 554                         continue;
 555                 }
 556
 557                 p++;
 558                 goto newflag;
 559         }
 560
 561         /* save the new flag set */
 562         bcopy(tmpflags, &pmc_debugflags, sizeof(pmc_debugflags));
 563
 564  done:
 565         free(tmpflags, M_PMC);
 566         return error;
 567 }
 568
 569 static int
 570 pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS)
 571 {
 572         char *fence, *newstr;
 573         int error;
 574         unsigned int n;
 575
 576         (void) arg1; (void) arg2; /* unused parameters */
 577
 578         n = sizeof(pmc_debugstr);
 579         newstr = malloc(n, M_PMC, M_WAITOK|M_ZERO);
 580         (void) strlcpy(newstr, pmc_debugstr, n);
 581
 582         error = sysctl_handle_string(oidp, newstr, n, req);
 583
 584         /* if there is a new string, parse and copy it */
 585         if (error == 0 && req->newptr != NULL) {
 586                 fence = newstr + (n < req->newlen ? n : req->newlen + 1);
 587                 if ((error = pmc_debugflags_parse(newstr, fence)) == 0)
 588                         (void) strlcpy(pmc_debugstr, newstr,
 589                             sizeof(pmc_debugstr));
 590         }
 591
 592         free(newstr, M_PMC);
 593
 594         return error;
 595 }
 596 #endif
 597
 598 /*
 599  * Map a row index to a classdep structure and return the adjusted row
 600  * index for the PMC class index.
 601  */
 602 static struct pmc_classdep *
 603 pmc_ri_to_classdep(struct pmc_mdep *md, int ri, int *adjri)
 604 {
 605         struct pmc_classdep *pcd;
 606
 607         (void) md;
 608
 609         KASSERT(ri >= 0 && ri < md->pmd_npmc,
 610             ("[pmc,%d] illegal row-index %d", __LINE__, ri));
 611
 612         pcd = pmc_rowindex_to_classdep[ri];
 613
 614         KASSERT(pcd != NULL,
 615             ("[pmc,%d] ri %d null pcd", __LINE__, ri));
 616
 617         *adjri = ri - pcd->pcd_ri;
 618
 619         KASSERT(*adjri >= 0 && *adjri < pcd->pcd_num,
 620             ("[pmc,%d] adjusted row-index %d", __LINE__, *adjri));
 621
 622         return (pcd);
 623 }
 624
 625 /*
 626  * Concurrency Control
 627  *
 628  * The driver manages the following data structures:
 629  *
 630  *   - target process descriptors, one per target process
 631  *   - owner process descriptors (and attached lists), one per owner process
 632  *   - lookup hash tables for owner and target processes
 633  *   - PMC descriptors (and attached lists)
 634  *   - per-cpu hardware state
 635  *   - the 'hook' variable through which the kernel calls into
 636  *     this module
 637  *   - the machine hardware state (managed by the MD layer)
 638  *
 639  * These data structures are accessed from:
 640  *
 641  * - thread context-switch code
 642  * - interrupt handlers (possibly on multiple cpus)
 643  * - kernel threads on multiple cpus running on behalf of user
 644  *   processes doing system calls
 645  * - this driver's private kernel threads
 646  *
 647  * = Locks and Locking strategy =
 648  *
 649  * The driver uses four locking strategies for its operation:
 650  *
 651  * - The global SX lock "pmc_sx" is used to protect internal
 652  *   data structures.
 653  *
 654  *   Calls into the module by syscall() start with this lock being
 655  *   held in exclusive mode.  Depending on the requested operation,
 656  *   the lock may be downgraded to 'shared' mode to allow more
 657  *   concurrent readers into the module.  Calls into the module from
 658  *   other parts of the kernel acquire the lock in shared mode.
 659  *
 660  *   This SX lock is held in exclusive mode for any operations that
 661  *   modify the linkages between the driver's internal data structures.
 662  *
 663  *   The 'pmc_hook' function pointer is also protected by this lock.
 664  *   It is only examined with the sx lock held in exclusive mode.  The
 665  *   kernel module is allowed to be unloaded only with the sx lock held
 666  *   in exclusive mode.  In normal syscall handling, after acquiring the
 667  *   pmc_sx lock we first check that 'pmc_hook' is non-null before
 668  *   proceeding.  This prevents races between the thread unloading the module
 669  *   and other threads seeking to use the module.
 670  *
 671  * - Lookups of target process structures and owner process structures
 672  *   cannot use the global "pmc_sx" SX lock because these lookups need
 673  *   to happen during context switches and in other critical sections
 674  *   where sleeping is not allowed.  We protect these lookup tables
 675  *   with their own private spin-mutexes, "pmc_processhash_mtx" and
 676  *   "pmc_ownerhash_mtx".
 677  *
 678  * - Interrupt handlers work in a lock free manner.  At interrupt
 679  *   time, handlers look at the PMC pointer (phw->phw_pmc) configured
 680  *   when the PMC was started.  If this pointer is NULL, the interrupt
 681  *   is ignored after updating driver statistics.  We ensure that this
 682  *   pointer is set (using an atomic operation if necessary) before the
 683  *   PMC hardware is started.  Conversely, this pointer is unset atomically
 684  *   only after the PMC hardware is stopped.
 685  *
 686  *   We ensure that everything needed for the operation of an
 687  *   interrupt handler is available without it needing to acquire any
 688  *   locks.  We also ensure that a PMC's software state is destroyed only
 689  *   after the PMC is taken off hardware (on all CPUs).
 690  *
 691  * - Context-switch handling with process-private PMCs needs more
 692  *   care.
 693  *
 694  *   A given process may be the target of multiple PMCs.  For example,
 695  *   PMCATTACH and PMCDETACH may be requested by a process on one CPU
 696  *   while the target process is running on another.  A PMC could also
 697  *   be getting released because its owner is exiting.  We tackle
 698  *   these situations in the following manner:
 699  *
 700  *   - each target process structure 'pmc_process' has an array
 701  *     of 'struct pmc *' pointers, one for each hardware PMC.
 702  *
 703  *   - At context switch IN time, each "target" PMC in RUNNING state
 704  *     gets started on hardware and a pointer to each PMC is copied into
 705  *     the per-cpu phw array.  The 'runcount' for the PMC is
 706  *     incremented.
 707  *
 708  *   - At context switch OUT time, all process-virtual PMCs are stopped
 709  *     on hardware.  The saved value is added to the PMCs value field
 710  *     only if the PMC is in a non-deleted state (the PMCs state could
 711  *     have changed during the current time slice).
 712  *
 713  *     Note that since in-between a switch IN on a processor and a switch
 714  *     OUT, the PMC could have been released on another CPU.  Therefore
 715  *     context switch OUT always looks at the hardware state to turn
 716  *     OFF PMCs and will update a PMC's saved value only if reachable
 717  *     from the target process record.
 718  *
 719  *   - OP PMCRELEASE could be called on a PMC at any time (the PMC could
 720  *     be attached to many processes at the time of the call and could
 721  *     be active on multiple CPUs).
 722  *
 723  *     We prevent further scheduling of the PMC by marking it as in
 724  *     state 'DELETED'.  If the runcount of the PMC is non-zero then
 725  *     this PMC is currently running on a CPU somewhere.  The thread
 726  *     doing the PMCRELEASE operation waits by repeatedly doing a
 727  *     pause() till the runcount comes to zero.
 728  *
 729  * The contents of a PMC descriptor (struct pmc) are protected using
 730  * a spin-mutex.  In order to save space, we use a mutex pool.
 731  *
 732  * In terms of lock types used by witness(4), we use:
 733  * - Type "pmc-sx", used by the global SX lock.
 734  * - Type "pmc-sleep", for sleep mutexes used by logger threads.
 735  * - Type "pmc-per-proc", for protecting PMC owner descriptors.
 736  * - Type "pmc-leaf", used for all other spin mutexes.
 737  */
 738
 739 /*
 740  * save the cpu binding of the current kthread
 741  */
 742
 743 static void
 744 pmc_save_cpu_binding(struct pmc_binding *pb)
 745 {
 746         PMCDBG0(CPU,BND,2, "save-cpu");
 747         thread_lock(curthread);
 748         pb->pb_bound = sched_is_bound(curthread);
 749         pb->pb_cpu   = curthread->td_oncpu;
 750         thread_unlock(curthread);
 751         PMCDBG1(CPU,BND,2, "save-cpu cpu=%d", pb->pb_cpu);
 752 }
 753
 754 /*
 755  * restore the cpu binding of the current thread
 756  */
 757
 758 static void
 759 pmc_restore_cpu_binding(struct pmc_binding *pb)
 760 {
 761         PMCDBG2(CPU,BND,2, "restore-cpu curcpu=%d restore=%d",
 762             curthread->td_oncpu, pb->pb_cpu);
 763         thread_lock(curthread);
 764         if (pb->pb_bound)
 765                 sched_bind(curthread, pb->pb_cpu);
 766         else
 767                 sched_unbind(curthread);
 768         thread_unlock(curthread);
 769         PMCDBG0(CPU,BND,2, "restore-cpu done");
 770 }
 771
 772 /*
 773  * move execution over the specified cpu and bind it there.
 774  */
 775
 776 static void
 777 pmc_select_cpu(int cpu)
 778 {
 779         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 780             ("[pmc,%d] bad cpu number %d", __LINE__, cpu));
 781
 782         /* Never move to an inactive CPU. */
 783         KASSERT(pmc_cpu_is_active(cpu), ("[pmc,%d] selecting inactive "
 784             "CPU %d", __LINE__, cpu));
 785
 786         PMCDBG1(CPU,SEL,2, "select-cpu cpu=%d", cpu);
 787         thread_lock(curthread);
 788         sched_bind(curthread, cpu);
 789         thread_unlock(curthread);
 790
 791         KASSERT(curthread->td_oncpu == cpu,
 792             ("[pmc,%d] CPU not bound [cpu=%d, curr=%d]", __LINE__,
 793                 cpu, curthread->td_oncpu));
 794
 795         PMCDBG1(CPU,SEL,2, "select-cpu cpu=%d ok", cpu);
 796 }
 797
 798 /*
 799  * Force a context switch.
 800  *
 801  * We do this by pause'ing for 1 tick -- invoking mi_switch() is not
 802  * guaranteed to force a context switch.
 803  */
 804
 805 static void
 806 pmc_force_context_switch(void)
 807 {
 808
 809         pause("pmcctx", 1);
 810 }
 811
 812 /*
 813  * Get the file name for an executable.  This is a simple wrapper
 814  * around vn_fullpath(9).
 815  */
 816
 817 static void
 818 pmc_getfilename(struct vnode *v, char **fullpath, char **freepath)
 819 {
 820
 821         *fullpath = "unknown";
 822         *freepath = NULL;
 823         vn_fullpath(curthread, v, fullpath, freepath);
 824 }
 825
 826 /*
 827  * remove an process owning PMCs
 828  */
 829
 830 void
 831 pmc_remove_owner(struct pmc_owner *po)
 832 {
 833         struct pmc *pm, *tmp;
 834
 835         sx_assert(&pmc_sx, SX_XLOCKED);
 836
 837         PMCDBG1(OWN,ORM,1, "remove-owner po=%p", po);
 838
 839         /* Remove descriptor from the owner hash table */
 840         LIST_REMOVE(po, po_next);
 841
 842         /* release all owned PMC descriptors */
 843         LIST_FOREACH_SAFE(pm, &po->po_pmcs, pm_next, tmp) {
 844                 PMCDBG1(OWN,ORM,2, "pmc=%p", pm);
 845                 KASSERT(pm->pm_owner == po,
 846                     ("[pmc,%d] owner %p != po %p", __LINE__, pm->pm_owner, po));
 847
 848                 pmc_release_pmc_descriptor(pm); /* will unlink from the list */
 849                 pmc_destroy_pmc_descriptor(pm);
 850         }
 851
 852         KASSERT(po->po_sscount == 0,
 853             ("[pmc,%d] SS count not zero", __LINE__));
 854         KASSERT(LIST_EMPTY(&po->po_pmcs),
 855             ("[pmc,%d] PMC list not empty", __LINE__));
 856
 857         /* de-configure the log file if present */
 858         if (po->po_flags & PMC_PO_OWNS_LOGFILE)
 859                 pmclog_deconfigure_log(po);
 860 }
 861
 862 /*
 863  * remove an owner process record if all conditions are met.
 864  */
 865
 866 static void
 867 pmc_maybe_remove_owner(struct pmc_owner *po)
 868 {
 869
 870         PMCDBG1(OWN,OMR,1, "maybe-remove-owner po=%p", po);
 871
 872         /*
 873          * Remove owner record if
 874          * - this process does not own any PMCs
 875          * - this process has not allocated a system-wide sampling buffer
 876          */
 877
 878         if (LIST_EMPTY(&po->po_pmcs) &&
 879             ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)) {
 880                 pmc_remove_owner(po);
 881                 pmc_destroy_owner_descriptor(po);
 882         }
 883 }
 884
 885 /*
 886  * Add an association between a target process and a PMC.
 887  */
 888
 889 static void
 890 pmc_link_target_process(struct pmc *pm, struct pmc_process *pp)
 891 {
 892         int ri;
 893         struct pmc_target *pt;
 894 #ifdef INVARIANTS
 895         struct pmc_thread *pt_td;
 896 #endif
 897
 898         sx_assert(&pmc_sx, SX_XLOCKED);
 899
 900         KASSERT(pm != NULL && pp != NULL,
 901             ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
 902         KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)),
 903             ("[pmc,%d] Attaching a non-process-virtual pmc=%p to pid=%d",
 904                 __LINE__, pm, pp->pp_proc->p_pid));
 905         KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= ((int) md->pmd_npmc - 1),
 906             ("[pmc,%d] Illegal reference count %d for process record %p",
 907                 __LINE__, pp->pp_refcnt, (void *) pp));
 908
 909         ri = PMC_TO_ROWINDEX(pm);
 910
 911         PMCDBG3(PRC,TLK,1, "link-target pmc=%p ri=%d pmc-process=%p",
 912             pm, ri, pp);
 913
 914 #ifdef  HWPMC_DEBUG
 915         LIST_FOREACH(pt, &pm->pm_targets, pt_next)
 916             if (pt->pt_process == pp)
 917                     KASSERT(0, ("[pmc,%d] pp %p already in pmc %p targets",
 918                                 __LINE__, pp, pm));
 919 #endif
 920
 921         pt = malloc(sizeof(struct pmc_target), M_PMC, M_WAITOK|M_ZERO);
 922         pt->pt_process = pp;
 923
 924         LIST_INSERT_HEAD(&pm->pm_targets, pt, pt_next);
 925
 926         atomic_store_rel_ptr((uintptr_t *)&pp->pp_pmcs[ri].pp_pmc,
 927             (uintptr_t)pm);
 928
 929         if (pm->pm_owner->po_owner == pp->pp_proc)
 930                 pm->pm_flags |= PMC_F_ATTACHED_TO_OWNER;
 931
 932         /*
 933          * Initialize the per-process values at this row index.
 934          */
 935         pp->pp_pmcs[ri].pp_pmcval = PMC_TO_MODE(pm) == PMC_MODE_TS ?
 936             pm->pm_sc.pm_reloadcount : 0;
 937
 938         pp->pp_refcnt++;
 939
 940 #ifdef INVARIANTS
 941         /* Confirm that the per-thread values at this row index are cleared. */
 942         if (PMC_TO_MODE(pm) == PMC_MODE_TS) {
 943                 mtx_lock_spin(pp->pp_tdslock);
 944                 LIST_FOREACH(pt_td, &pp->pp_tds, pt_next) {
 945                         KASSERT(pt_td->pt_pmcs[ri].pt_pmcval == (pmc_value_t) 0,
 946                             ("[pmc,%d] pt_pmcval not cleared for pid=%d at "
 947                             "ri=%d", __LINE__, pp->pp_proc->p_pid, ri));
 948                 }
 949                 mtx_unlock_spin(pp->pp_tdslock);
 950         }
 951 #endif
 952 }
 953
 954 /*
 955  * Removes the association between a target process and a PMC.
 956  */
 957
 958 static void
 959 pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp)
 960 {
 961         int ri;
 962         struct proc *p;
 963         struct pmc_target *ptgt;
 964         struct pmc_thread *pt;
 965
 966         sx_assert(&pmc_sx, SX_XLOCKED);
 967
 968         KASSERT(pm != NULL && pp != NULL,
 969             ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
 970
 971         KASSERT(pp->pp_refcnt >= 1 && pp->pp_refcnt <= (int) md->pmd_npmc,
 972             ("[pmc,%d] Illegal ref count %d on process record %p",
 973                 __LINE__, pp->pp_refcnt, (void *) pp));
 974
 975         ri = PMC_TO_ROWINDEX(pm);
 976
 977         PMCDBG3(PRC,TUL,1, "unlink-target pmc=%p ri=%d pmc-process=%p",
 978             pm, ri, pp);
 979
 980         KASSERT(pp->pp_pmcs[ri].pp_pmc == pm,
 981             ("[pmc,%d] PMC ri %d mismatch pmc %p pp->[ri] %p", __LINE__,
 982                 ri, pm, pp->pp_pmcs[ri].pp_pmc));
 983
 984         pp->pp_pmcs[ri].pp_pmc = NULL;
 985         pp->pp_pmcs[ri].pp_pmcval = (pmc_value_t) 0;
 986
 987         /* Clear the per-thread values at this row index. */
 988         if (PMC_TO_MODE(pm) == PMC_MODE_TS) {
 989                 mtx_lock_spin(pp->pp_tdslock);
 990                 LIST_FOREACH(pt, &pp->pp_tds, pt_next)
 991                         pt->pt_pmcs[ri].pt_pmcval = (pmc_value_t) 0;
 992                 mtx_unlock_spin(pp->pp_tdslock);
 993         }
 994
 995         /* Remove owner-specific flags */
 996         if (pm->pm_owner->po_owner == pp->pp_proc) {
 997                 pp->pp_flags &= ~PMC_PP_ENABLE_MSR_ACCESS;
 998                 pm->pm_flags &= ~PMC_F_ATTACHED_TO_OWNER;
 999         }
1000
1001         pp->pp_refcnt--;
1002
1003         /* Remove the target process from the PMC structure */
1004         LIST_FOREACH(ptgt, &pm->pm_targets, pt_next)
1005                 if (ptgt->pt_process == pp)
1006                         break;
1007
1008         KASSERT(ptgt != NULL, ("[pmc,%d] process %p (pp: %p) not found "
1009                     "in pmc %p", __LINE__, pp->pp_proc, pp, pm));
1010
1011         LIST_REMOVE(ptgt, pt_next);
1012         free(ptgt, M_PMC);
1013
1014         /* if the PMC now lacks targets, send the owner a SIGIO */
1015         if (LIST_EMPTY(&pm->pm_targets)) {
1016                 p = pm->pm_owner->po_owner;
1017                 PROC_LOCK(p);
1018                 kern_psignal(p, SIGIO);
1019                 PROC_UNLOCK(p);
1020
1021                 PMCDBG2(PRC,SIG,2, "signalling proc=%p signal=%d", p,
1022                     SIGIO);
1023         }
1024 }
1025
1026 /*
1027  * Check if PMC 'pm' may be attached to target process 't'.
1028  */
1029
1030 static int
1031 pmc_can_attach(struct pmc *pm, struct proc *t)
1032 {
1033         struct proc *o;         /* pmc owner */
1034         struct ucred *oc, *tc;  /* owner, target credentials */
1035         int decline_attach, i;
1036
1037         /*
1038          * A PMC's owner can always attach that PMC to itself.
1039          */
1040
1041         if ((o = pm->pm_owner->po_owner) == t)
1042                 return 0;
1043
1044         PROC_LOCK(o);
1045         oc = o->p_ucred;
1046         crhold(oc);
1047         PROC_UNLOCK(o);
1048
1049         PROC_LOCK(t);
1050         tc = t->p_ucred;
1051         crhold(tc);
1052         PROC_UNLOCK(t);
1053
1054         /*
1055          * The effective uid of the PMC owner should match at least one
1056          * of the {effective,real,saved} uids of the target process.
1057          */
1058
1059         decline_attach = oc->cr_uid != tc->cr_uid &&
1060             oc->cr_uid != tc->cr_svuid &&
1061             oc->cr_uid != tc->cr_ruid;
1062
1063         /*
1064          * Every one of the target's group ids, must be in the owner's
1065          * group list.
1066          */
1067         for (i = 0; !decline_attach && i < tc->cr_ngroups; i++)
1068                 decline_attach = !groupmember(tc->cr_groups[i], oc);
1069
1070         /* check the read and saved gids too */
1071         if (decline_attach == 0)
1072                 decline_attach = !groupmember(tc->cr_rgid, oc) ||
1073                     !groupmember(tc->cr_svgid, oc);
1074
1075         crfree(tc);
1076         crfree(oc);
1077
1078         return !decline_attach;
1079 }
1080
1081 /*
1082  * Attach a process to a PMC.
1083  */
1084
1085 static int
1086 pmc_attach_one_process(struct proc *p, struct pmc *pm)
1087 {
1088         int ri, error;
1089         char *fullpath, *freepath;
1090         struct pmc_process      *pp;
1091
1092         sx_assert(&pmc_sx, SX_XLOCKED);
1093
1094         PMCDBG5(PRC,ATT,2, "attach-one pm=%p ri=%d proc=%p (%d, %s)", pm,
1095             PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
1096
1097         /*
1098          * Locate the process descriptor corresponding to process 'p',
1099          * allocating space as needed.
1100          *
1101          * Verify that rowindex 'pm_rowindex' is free in the process
1102          * descriptor.
1103          *
1104          * If not, allocate space for a descriptor and link the
1105          * process descriptor and PMC.
1106          */
1107         ri = PMC_TO_ROWINDEX(pm);
1108
1109         /* mark process as using HWPMCs */
1110         PROC_LOCK(p);
1111         p->p_flag |= P_HWPMC;
1112         PROC_UNLOCK(p);
1113
1114         if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_ALLOCATE)) == NULL) {
1115                 error = ENOMEM;
1116                 goto fail;
1117         }
1118
1119         if (pp->pp_pmcs[ri].pp_pmc == pm) {/* already present at slot [ri] */
1120                 error = EEXIST;
1121                 goto fail;
1122         }
1123
1124         if (pp->pp_pmcs[ri].pp_pmc != NULL) {
1125                 error = EBUSY;
1126                 goto fail;
1127         }
1128
1129         pmc_link_target_process(pm, pp);
1130
1131         if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) &&
1132             (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) == 0)
1133                 pm->pm_flags |= PMC_F_NEEDS_LOGFILE;
1134
1135         pm->pm_flags |= PMC_F_ATTACH_DONE; /* mark as attached */
1136
1137         /* issue an attach event to a configured log file */
1138         if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE) {
1139                 if (p->p_flag & P_KPROC) {
1140                         fullpath = kernelname;
1141                         freepath = NULL;
1142                 } else {
1143                         pmc_getfilename(p->p_textvp, &fullpath, &freepath);
1144                         pmclog_process_pmcattach(pm, p->p_pid, fullpath);
1145                 }
1146                 free(freepath, M_TEMP);
1147                 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1148                         pmc_log_process_mappings(pm->pm_owner, p);
1149         }
1150
1151         return (0);
1152  fail:
1153         PROC_LOCK(p);
1154         p->p_flag &= ~P_HWPMC;
1155         PROC_UNLOCK(p);
1156         return (error);
1157 }
1158
1159 /*
1160  * Attach a process and optionally its children
1161  */
1162
1163 static int
1164 pmc_attach_process(struct proc *p, struct pmc *pm)
1165 {
1166         int error;
1167         struct proc *top;
1168
1169         sx_assert(&pmc_sx, SX_XLOCKED);
1170
1171         PMCDBG5(PRC,ATT,1, "attach pm=%p ri=%d proc=%p (%d, %s)", pm,
1172             PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
1173
1174
1175         /*
1176          * If this PMC successfully allowed a GETMSR operation
1177          * in the past, disallow further ATTACHes.
1178          */
1179
1180         if ((pm->pm_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0)
1181                 return EPERM;
1182
1183         if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
1184                 return pmc_attach_one_process(p, pm);
1185
1186         /*
1187          * Traverse all child processes, attaching them to
1188          * this PMC.
1189          */
1190
1191         sx_slock(&proctree_lock);
1192
1193         top = p;
1194
1195         for (;;) {
1196                 if ((error = pmc_attach_one_process(p, pm)) != 0)
1197                         break;
1198                 if (!LIST_EMPTY(&p->p_children))
1199                         p = LIST_FIRST(&p->p_children);
1200                 else for (;;) {
1201                         if (p == top)
1202                                 goto done;
1203                         if (LIST_NEXT(p, p_sibling)) {
1204                                 p = LIST_NEXT(p, p_sibling);
1205                                 break;
1206                         }
1207                         p = p->p_pptr;
1208                 }
1209         }
1210
1211         if (error)
1212                 (void) pmc_detach_process(top, pm);
1213
1214  done:
1215         sx_sunlock(&proctree_lock);
1216         return error;
1217 }
1218
1219 /*
1220  * Detach a process from a PMC.  If there are no other PMCs tracking
1221  * this process, remove the process structure from its hash table.  If
1222  * 'flags' contains PMC_FLAG_REMOVE, then free the process structure.
1223  */
1224
1225 static int
1226 pmc_detach_one_process(struct proc *p, struct pmc *pm, int flags)
1227 {
1228         int ri;
1229         struct pmc_process *pp;
1230
1231         sx_assert(&pmc_sx, SX_XLOCKED);
1232
1233         KASSERT(pm != NULL,
1234             ("[pmc,%d] null pm pointer", __LINE__));
1235
1236         ri = PMC_TO_ROWINDEX(pm);
1237
1238         PMCDBG6(PRC,ATT,2, "detach-one pm=%p ri=%d proc=%p (%d, %s) flags=0x%x",
1239             pm, ri, p, p->p_pid, p->p_comm, flags);
1240
1241         if ((pp = pmc_find_process_descriptor(p, 0)) == NULL)
1242                 return ESRCH;
1243
1244         if (pp->pp_pmcs[ri].pp_pmc != pm)
1245                 return EINVAL;
1246
1247         pmc_unlink_target_process(pm, pp);
1248
1249         /* Issue a detach entry if a log file is configured */
1250         if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE)
1251                 pmclog_process_pmcdetach(pm, p->p_pid);
1252
1253         /*
1254          * If there are no PMCs targeting this process, we remove its
1255          * descriptor from the target hash table and unset the P_HWPMC
1256          * flag in the struct proc.
1257          */
1258         KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= (int) md->pmd_npmc,
1259             ("[pmc,%d] Illegal refcnt %d for process struct %p",
1260                 __LINE__, pp->pp_refcnt, pp));
1261
1262         if (pp->pp_refcnt != 0) /* still a target of some PMC */
1263                 return 0;
1264
1265         pmc_remove_process_descriptor(pp);
1266
1267         if (flags & PMC_FLAG_REMOVE)
1268                 pmc_destroy_process_descriptor(pp);
1269
1270         PROC_LOCK(p);
1271         p->p_flag &= ~P_HWPMC;
1272         PROC_UNLOCK(p);
1273
1274         return 0;
1275 }
1276
1277 /*
1278  * Detach a process and optionally its descendants from a PMC.
1279  */
1280
1281 static int
1282 pmc_detach_process(struct proc *p, struct pmc *pm)
1283 {
1284         struct proc *top;
1285
1286         sx_assert(&pmc_sx, SX_XLOCKED);
1287
1288         PMCDBG5(PRC,ATT,1, "detach pm=%p ri=%d proc=%p (%d, %s)", pm,
1289             PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
1290
1291         if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
1292                 return pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE);
1293
1294         /*
1295          * Traverse all children, detaching them from this PMC.  We
1296          * ignore errors since we could be detaching a PMC from a
1297          * partially attached proc tree.
1298          */
1299
1300         sx_slock(&proctree_lock);
1301
1302         top = p;
1303
1304         for (;;) {
1305                 (void) pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE);
1306
1307                 if (!LIST_EMPTY(&p->p_children))
1308                         p = LIST_FIRST(&p->p_children);
1309                 else for (;;) {
1310                         if (p == top)
1311                                 goto done;
1312                         if (LIST_NEXT(p, p_sibling)) {
1313                                 p = LIST_NEXT(p, p_sibling);
1314                                 break;
1315                         }
1316                         p = p->p_pptr;
1317                 }
1318         }
1319
1320  done:
1321         sx_sunlock(&proctree_lock);
1322
1323         if (LIST_EMPTY(&pm->pm_targets))
1324                 pm->pm_flags &= ~PMC_F_ATTACH_DONE;
1325
1326         return 0;
1327 }
1328
1329
1330 /*
1331  * Thread context switch IN
1332  */
1333
1334 static void
1335 pmc_process_csw_in(struct thread *td)
1336 {
1337         int cpu;
1338         unsigned int adjri, ri;
1339         struct pmc *pm;
1340         struct proc *p;
1341         struct pmc_cpu *pc;
1342         struct pmc_hw *phw;
1343         pmc_value_t newvalue;
1344         struct pmc_process *pp;
1345         struct pmc_thread *pt;
1346         struct pmc_classdep *pcd;
1347
1348         p = td->td_proc;
1349         pt = NULL;
1350         if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE)) == NULL)
1351                 return;
1352
1353         KASSERT(pp->pp_proc == td->td_proc,
1354             ("[pmc,%d] not my thread state", __LINE__));
1355
1356         critical_enter(); /* no preemption from this point */
1357
1358         cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */
1359
1360         PMCDBG5(CSW,SWI,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p,
1361             p->p_pid, p->p_comm, pp);
1362
1363         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
1364             ("[pmc,%d] weird CPU id %d", __LINE__, cpu));
1365
1366         pc = pmc_pcpu[cpu];
1367
1368         for (ri = 0; ri < md->pmd_npmc; ri++) {
1369
1370                 if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL)
1371                         continue;
1372
1373                 KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)),
1374                     ("[pmc,%d] Target PMC in non-virtual mode (%d)",
1375                         __LINE__, PMC_TO_MODE(pm)));
1376
1377                 KASSERT(PMC_TO_ROWINDEX(pm) == ri,
1378                     ("[pmc,%d] Row index mismatch pmc %d != ri %d",
1379                         __LINE__, PMC_TO_ROWINDEX(pm), ri));
1380
1381                 /*
1382                  * Only PMCs that are marked as 'RUNNING' need
1383                  * be placed on hardware.
1384                  */
1385
1386                 if (pm->pm_state != PMC_STATE_RUNNING)
1387                         continue;
1388
1389                 /* increment PMC runcount */
1390                 counter_u64_add(pm->pm_runcount, 1);
1391
1392                 /* configure the HWPMC we are going to use. */
1393                 pcd = pmc_ri_to_classdep(md, ri, &adjri);
1394                 pcd->pcd_config_pmc(cpu, adjri, pm);
1395
1396                 phw = pc->pc_hwpmcs[ri];
1397
1398                 KASSERT(phw != NULL,
1399                     ("[pmc,%d] null hw pointer", __LINE__));
1400
1401                 KASSERT(phw->phw_pmc == pm,
1402                     ("[pmc,%d] hw->pmc %p != pmc %p", __LINE__,
1403                         phw->phw_pmc, pm));
1404
1405                 /*
1406                  * Write out saved value and start the PMC.
1407                  *
1408                  * Sampling PMCs use a per-thread value, while
1409                  * counting mode PMCs use a per-pmc value that is
1410                  * inherited across descendants.
1411                  */
1412                 if (PMC_TO_MODE(pm) == PMC_MODE_TS) {
1413                         if (pt == NULL)
1414                                 pt = pmc_find_thread_descriptor(pp, td,
1415                                     PMC_FLAG_NONE);
1416
1417                         KASSERT(pt != NULL,
1418                             ("[pmc,%d] No thread found for td=%p", __LINE__,
1419                             td));
1420
1421                         mtx_pool_lock_spin(pmc_mtxpool, pm);
1422
1423                         /*
1424                          * If we have a thread descriptor, use the per-thread
1425                          * counter in the descriptor. If not, we will use
1426                          * a per-process counter.
1427                          *
1428                          * TODO: Remove the per-process "safety net" once
1429                          * we have thoroughly tested that we don't hit the
1430                          * above assert.
1431                          */
1432                         if (pt != NULL) {
1433                                 if (pt->pt_pmcs[ri].pt_pmcval > 0)
1434                                         newvalue = pt->pt_pmcs[ri].pt_pmcval;
1435                                 else
1436                                         newvalue = pm->pm_sc.pm_reloadcount;
1437                         } else {
1438                                 /*
1439                                  * Use the saved value calculated after the most
1440                                  * recent time a thread using the shared counter
1441                                  * switched out. Reset the saved count in case
1442                                  * another thread from this process switches in
1443                                  * before any threads switch out.
1444                                  */
1445
1446                                 newvalue = pp->pp_pmcs[ri].pp_pmcval;
1447                                 pp->pp_pmcs[ri].pp_pmcval =
1448                                     pm->pm_sc.pm_reloadcount;
1449                         }
1450                         mtx_pool_unlock_spin(pmc_mtxpool, pm);
1451                         KASSERT(newvalue > 0 && newvalue <=
1452                             pm->pm_sc.pm_reloadcount,
1453                             ("[pmc,%d] pmcval outside of expected range cpu=%d "
1454                             "ri=%d pmcval=%jx pm_reloadcount=%jx", __LINE__,
1455                             cpu, ri, newvalue, pm->pm_sc.pm_reloadcount));
1456                 } else {
1457                         KASSERT(PMC_TO_MODE(pm) == PMC_MODE_TC,
1458                             ("[pmc,%d] illegal mode=%d", __LINE__,
1459                             PMC_TO_MODE(pm)));
1460                         mtx_pool_lock_spin(pmc_mtxpool, pm);
1461                         newvalue = PMC_PCPU_SAVED(cpu, ri) =
1462                             pm->pm_gv.pm_savedvalue;
1463                         mtx_pool_unlock_spin(pmc_mtxpool, pm);
1464                 }
1465
1466                 PMCDBG3(CSW,SWI,1,"cpu=%d ri=%d new=%jd", cpu, ri, newvalue);
1467
1468                 pcd->pcd_write_pmc(cpu, adjri, newvalue);
1469
1470                 /* If a sampling mode PMC, reset stalled state. */
1471                 if (PMC_TO_MODE(pm) == PMC_MODE_TS)
1472                         pm->pm_pcpu_state[cpu].pps_stalled = 0;
1473
1474                 /* Indicate that we desire this to run. */
1475                 pm->pm_pcpu_state[cpu].pps_cpustate = 1;
1476
1477                 /* Start the PMC. */
1478                 pcd->pcd_start_pmc(cpu, adjri);
1479         }
1480
1481         /*
1482          * perform any other architecture/cpu dependent thread
1483          * switch-in actions.
1484          */
1485
1486         (void) (*md->pmd_switch_in)(pc, pp);
1487
1488         critical_exit();
1489
1490 }
1491
1492 /*
1493  * Thread context switch OUT.
1494  */
1495
1496 static void
1497 pmc_process_csw_out(struct thread *td)
1498 {
1499         int cpu;
1500         int64_t tmp;
1501         struct pmc *pm;
1502         struct proc *p;
1503         enum pmc_mode mode;
1504         struct pmc_cpu *pc;
1505         pmc_value_t newvalue;
1506         unsigned int adjri, ri;
1507         struct pmc_process *pp;
1508         struct pmc_thread *pt = NULL;
1509         struct pmc_classdep *pcd;
1510
1511
1512         /*
1513          * Locate our process descriptor; this may be NULL if
1514          * this process is exiting and we have already removed
1515          * the process from the target process table.
1516          *
1517          * Note that due to kernel preemption, multiple
1518          * context switches may happen while the process is
1519          * exiting.
1520          *
1521          * Note also that if the target process cannot be
1522          * found we still need to deconfigure any PMCs that
1523          * are currently running on hardware.
1524          */
1525
1526         p = td->td_proc;
1527         pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE);
1528
1529         /*
1530          * save PMCs
1531          */
1532
1533         critical_enter();
1534
1535         cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */
1536
1537         PMCDBG5(CSW,SWO,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p,
1538             p->p_pid, p->p_comm, pp);
1539
1540         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
1541             ("[pmc,%d weird CPU id %d", __LINE__, cpu));
1542
1543         pc = pmc_pcpu[cpu];
1544
1545         /*
1546          * When a PMC gets unlinked from a target PMC, it will
1547          * be removed from the target's pp_pmc[] array.
1548          *
1549          * However, on a MP system, the target could have been
1550          * executing on another CPU at the time of the unlink.
1551          * So, at context switch OUT time, we need to look at
1552          * the hardware to determine if a PMC is scheduled on
1553          * it.
1554          */
1555
1556         for (ri = 0; ri < md->pmd_npmc; ri++) {
1557
1558                 pcd = pmc_ri_to_classdep(md, ri, &adjri);
1559                 pm  = NULL;
1560                 (void) (*pcd->pcd_get_config)(cpu, adjri, &pm);
1561
1562                 if (pm == NULL) /* nothing at this row index */
1563                         continue;
1564
1565                 mode = PMC_TO_MODE(pm);
1566                 if (!PMC_IS_VIRTUAL_MODE(mode))
1567                         continue; /* not a process virtual PMC */
1568
1569                 KASSERT(PMC_TO_ROWINDEX(pm) == ri,
1570                     ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
1571                         __LINE__, PMC_TO_ROWINDEX(pm), ri));
1572
1573                 /*
1574                  * Change desired state, and then stop if not stalled.
1575                  * This two-step dance should avoid race conditions where
1576                  * an interrupt re-enables the PMC after this code has
1577                  * already checked the pm_stalled flag.
1578                  */
1579                 pm->pm_pcpu_state[cpu].pps_cpustate = 0;
1580                 if (pm->pm_pcpu_state[cpu].pps_stalled == 0)
1581                         pcd->pcd_stop_pmc(cpu, adjri);
1582
1583                 /* reduce this PMC's runcount */
1584                 counter_u64_add(pm->pm_runcount, -1);
1585
1586                 /*
1587                  * If this PMC is associated with this process,
1588                  * save the reading.
1589                  */
1590
1591                 if (pm->pm_state != PMC_STATE_DELETED && pp != NULL &&
1592                     pp->pp_pmcs[ri].pp_pmc != NULL) {
1593                         KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
1594                             ("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__,
1595                                 pm, ri, pp->pp_pmcs[ri].pp_pmc));
1596
1597                         KASSERT(pp->pp_refcnt > 0,
1598                             ("[pmc,%d] pp refcnt = %d", __LINE__,
1599                                 pp->pp_refcnt));
1600
1601                         pcd->pcd_read_pmc(cpu, adjri, &newvalue);
1602
1603                         if (mode == PMC_MODE_TS) {
1604                                 PMCDBG3(CSW,SWO,1,"cpu=%d ri=%d val=%jd (samp)",
1605                                     cpu, ri, newvalue);
1606
1607                                 if (pt == NULL)
1608                                         pt = pmc_find_thread_descriptor(pp, td,
1609                                             PMC_FLAG_NONE);
1610
1611                                 KASSERT(pt != NULL,
1612                                     ("[pmc,%d] No thread found for td=%p",
1613                                     __LINE__, td));
1614
1615                                 mtx_pool_lock_spin(pmc_mtxpool, pm);
1616
1617                                 /*
1618                                  * If we have a thread descriptor, save the
1619                                  * per-thread counter in the descriptor. If not,
1620                                  * we will update the per-process counter.
1621                                  *
1622                                  * TODO: Remove the per-process "safety net"
1623                                  * once we have thoroughly tested that we
1624                                  * don't hit the above assert.
1625                                  */
1626                                 if (pt != NULL)
1627                                         pt->pt_pmcs[ri].pt_pmcval = newvalue;
1628                                 else {
1629                                         /*
1630                                          * For sampling process-virtual PMCs,
1631                                          * newvalue is the number of events to
1632                                          * be seen until the next sampling
1633                                          * interrupt. We can just add the events
1634                                          * left from this invocation to the
1635                                          * counter, then adjust in case we
1636                                          * overflow our range.
1637                                          *
1638                                          * (Recall that we reload the counter
1639                                          * every time we use it.)
1640                                          */
1641                                         pp->pp_pmcs[ri].pp_pmcval += newvalue;
1642                                         if (pp->pp_pmcs[ri].pp_pmcval >
1643                                             pm->pm_sc.pm_reloadcount)
1644                                                 pp->pp_pmcs[ri].pp_pmcval -=
1645                                                     pm->pm_sc.pm_reloadcount;
1646                                 }
1647                                 mtx_pool_unlock_spin(pmc_mtxpool, pm);
1648                         } else {
1649                                 tmp = newvalue - PMC_PCPU_SAVED(cpu,ri);
1650
1651                                 PMCDBG3(CSW,SWO,1,"cpu=%d ri=%d tmp=%jd (count)",
1652                                     cpu, ri, tmp);
1653
1654                                 /*
1655                                  * For counting process-virtual PMCs,
1656                                  * we expect the count to be
1657                                  * increasing monotonically, modulo a 64
1658                                  * bit wraparound.
1659                                  */
1660                                 KASSERT(tmp >= 0,
1661                                     ("[pmc,%d] negative increment cpu=%d "
1662                                      "ri=%d newvalue=%jx saved=%jx "
1663                                      "incr=%jx", __LINE__, cpu, ri,
1664                                      newvalue, PMC_PCPU_SAVED(cpu,ri), tmp));
1665
1666                                 mtx_pool_lock_spin(pmc_mtxpool, pm);
1667                                 pm->pm_gv.pm_savedvalue += tmp;
1668                                 pp->pp_pmcs[ri].pp_pmcval += tmp;
1669                                 mtx_pool_unlock_spin(pmc_mtxpool, pm);
1670
1671                                 if (pm->pm_flags & PMC_F_LOG_PROCCSW)
1672                                         pmclog_process_proccsw(pm, pp, tmp, td);
1673                         }
1674                 }
1675
1676                 /* mark hardware as free */
1677                 pcd->pcd_config_pmc(cpu, adjri, NULL);
1678         }
1679
1680         /*
1681          * perform any other architecture/cpu dependent thread
1682          * switch out functions.
1683          */
1684
1685         (void) (*md->pmd_switch_out)(pc, pp);
1686
1687         critical_exit();
1688 }
1689
1690 /*
1691  * A new thread for a process.
1692  */
1693 static void
1694 pmc_process_thread_add(struct thread *td)
1695 {
1696         struct pmc_process *pmc;
1697
1698         pmc = pmc_find_process_descriptor(td->td_proc, PMC_FLAG_NONE);
1699         if (pmc != NULL)
1700                 pmc_find_thread_descriptor(pmc, td, PMC_FLAG_ALLOCATE);
1701 }
1702
1703 /*
1704  * A thread delete for a process.
1705  */
1706 static void
1707 pmc_process_thread_delete(struct thread *td)
1708 {
1709         struct pmc_process *pmc;
1710
1711         pmc = pmc_find_process_descriptor(td->td_proc, PMC_FLAG_NONE);
1712         if (pmc != NULL)
1713                 pmc_thread_descriptor_pool_free(pmc_find_thread_descriptor(pmc,
1714                     td, PMC_FLAG_REMOVE));
1715 }
1716
1717 /*
1718  * A userret() call for a thread.
1719  */
1720 static void
1721 pmc_process_thread_userret(struct thread *td)
1722 {
1723         sched_pin();
1724         pmc_capture_user_callchain(curcpu, PMC_UR, td->td_frame);
1725         sched_unpin();
1726 }
1727
1728 /*
1729  * A mapping change for a process.
1730  */
1731
1732 static void
1733 pmc_process_mmap(struct thread *td, struct pmckern_map_in *pkm)
1734 {
1735         int ri;
1736         pid_t pid;
1737         char *fullpath, *freepath;
1738         const struct pmc *pm;
1739         struct pmc_owner *po;
1740         const struct pmc_process *pp;
1741
1742         freepath = fullpath = NULL;
1743         MPASS(!in_epoch());
1744         pmc_getfilename((struct vnode *) pkm->pm_file, &fullpath, &freepath);
1745
1746         pid = td->td_proc->p_pid;
1747
1748         epoch_enter_preempt(global_epoch_preempt);
1749         /* Inform owners of all system-wide sampling PMCs. */
1750         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
1751             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
1752                         pmclog_process_map_in(po, pid, pkm->pm_address, fullpath);
1753
1754         if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL)
1755                 goto done;
1756
1757         /*
1758          * Inform sampling PMC owners tracking this process.
1759          */
1760         for (ri = 0; ri < md->pmd_npmc; ri++)
1761                 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL &&
1762                     PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1763                         pmclog_process_map_in(pm->pm_owner,
1764                             pid, pkm->pm_address, fullpath);
1765
1766   done:
1767         if (freepath)
1768                 free(freepath, M_TEMP);
1769         epoch_exit_preempt(global_epoch_preempt);
1770 }
1771
1772
1773 /*
1774  * Log an munmap request.
1775  */
1776
1777 static void
1778 pmc_process_munmap(struct thread *td, struct pmckern_map_out *pkm)
1779 {
1780         int ri;
1781         pid_t pid;
1782         struct pmc_owner *po;
1783         const struct pmc *pm;
1784         const struct pmc_process *pp;
1785
1786         pid = td->td_proc->p_pid;
1787
1788         epoch_enter_preempt(global_epoch_preempt);
1789         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
1790             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
1791                 pmclog_process_map_out(po, pid, pkm->pm_address,
1792                     pkm->pm_address + pkm->pm_size);
1793         epoch_exit_preempt(global_epoch_preempt);
1794
1795         if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL)
1796                 return;
1797
1798         for (ri = 0; ri < md->pmd_npmc; ri++)
1799                 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL &&
1800                     PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1801                         pmclog_process_map_out(pm->pm_owner, pid,
1802                             pkm->pm_address, pkm->pm_address + pkm->pm_size);
1803 }
1804
1805 /*
1806  * Log mapping information about the kernel.
1807  */
1808
1809 static void
1810 pmc_log_kernel_mappings(struct pmc *pm)
1811 {
1812         struct pmc_owner *po;
1813         struct pmckern_map_in *km, *kmbase;
1814
1815         MPASS(in_epoch() || sx_xlocked(&pmc_sx));
1816         KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
1817             ("[pmc,%d] non-sampling PMC (%p) desires mapping information",
1818                 __LINE__, (void *) pm));
1819
1820         po = pm->pm_owner;
1821
1822         if (po->po_flags & PMC_PO_INITIAL_MAPPINGS_DONE)
1823                 return;
1824
1825         /*
1826          * Log the current set of kernel modules.
1827          */
1828         kmbase = linker_hwpmc_list_objects();
1829         for (km = kmbase; km->pm_file != NULL; km++) {
1830                 PMCDBG2(LOG,REG,1,"%s %p", (char *) km->pm_file,
1831                     (void *) km->pm_address);
1832                 pmclog_process_map_in(po, (pid_t) -1, km->pm_address,
1833                     km->pm_file);
1834         }
1835         free(kmbase, M_LINKER);
1836
1837         po->po_flags |= PMC_PO_INITIAL_MAPPINGS_DONE;
1838 }
1839
1840 /*
1841  * Log the mappings for a single process.
1842  */
1843
1844 static void
1845 pmc_log_process_mappings(struct pmc_owner *po, struct proc *p)
1846 {
1847         vm_map_t map;
1848         struct vnode *vp;
1849         struct vmspace *vm;
1850         vm_map_entry_t entry;
1851         vm_offset_t last_end;
1852         u_int last_timestamp;
1853         struct vnode *last_vp;
1854         vm_offset_t start_addr;
1855         vm_object_t obj, lobj, tobj;
1856         char *fullpath, *freepath;
1857
1858         last_vp = NULL;
1859         last_end = (vm_offset_t) 0;
1860         fullpath = freepath = NULL;
1861
1862         if ((vm = vmspace_acquire_ref(p)) == NULL)
1863                 return;
1864
1865         map = &vm->vm_map;
1866         vm_map_lock_read(map);
1867
1868         for (entry = map->header.next; entry != &map->header; entry = entry->next) {
1869
1870                 if (entry == NULL) {
1871                         PMCDBG2(LOG,OPS,2, "hwpmc: vm_map entry unexpectedly "
1872                             "NULL! pid=%d vm_map=%p\n", p->p_pid, map);
1873                         break;
1874                 }
1875
1876                 /*
1877                  * We only care about executable map entries.
1878                  */
1879                 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) ||
1880                     !(entry->protection & VM_PROT_EXECUTE) ||
1881                     (entry->object.vm_object == NULL)) {
1882                         continue;
1883                 }
1884
1885                 obj = entry->object.vm_object;
1886                 VM_OBJECT_RLOCK(obj);
1887
1888                 /*
1889                  * Walk the backing_object list to find the base
1890                  * (non-shadowed) vm_object.
1891                  */
1892                 for (lobj = tobj = obj; tobj != NULL; tobj = tobj->backing_object) {
1893                         if (tobj != obj)
1894                                 VM_OBJECT_RLOCK(tobj);
1895                         if (lobj != obj)
1896                                 VM_OBJECT_RUNLOCK(lobj);
1897                         lobj = tobj;
1898                 }
1899
1900                 /*
1901                  * At this point lobj is the base vm_object and it is locked.
1902                  */
1903                 if (lobj == NULL) {
1904                         PMCDBG3(LOG,OPS,2, "hwpmc: lobj unexpectedly NULL! pid=%d "
1905                             "vm_map=%p vm_obj=%p\n", p->p_pid, map, obj);
1906                         VM_OBJECT_RUNLOCK(obj);
1907                         continue;
1908                 }
1909
1910                 vp = vm_object_vnode(lobj);
1911                 if (vp == NULL) {
1912                         if (lobj != obj)
1913                                 VM_OBJECT_RUNLOCK(lobj);
1914                         VM_OBJECT_RUNLOCK(obj);
1915                         continue;
1916                 }
1917
1918                 /*
1919                  * Skip contiguous regions that point to the same
1920                  * vnode, so we don't emit redundant MAP-IN
1921                  * directives.
1922                  */
1923                 if (entry->start == last_end && vp == last_vp) {
1924                         last_end = entry->end;
1925                         if (lobj != obj)
1926                                 VM_OBJECT_RUNLOCK(lobj);
1927                         VM_OBJECT_RUNLOCK(obj);
1928                         continue;
1929                 }
1930
1931                 /*
1932                  * We don't want to keep the proc's vm_map or this
1933                  * vm_object locked while we walk the pathname, since
1934                  * vn_fullpath() can sleep.  However, if we drop the
1935                  * lock, it's possible for concurrent activity to
1936                  * modify the vm_map list.  To protect against this,
1937                  * we save the vm_map timestamp before we release the
1938                  * lock, and check it after we reacquire the lock
1939                  * below.
1940                  */
1941                 start_addr = entry->start;
1942                 last_end = entry->end;
1943                 last_timestamp = map->timestamp;
1944                 vm_map_unlock_read(map);
1945
1946                 vref(vp);
1947                 if (lobj != obj)
1948                         VM_OBJECT_RUNLOCK(lobj);
1949
1950                 VM_OBJECT_RUNLOCK(obj);
1951
1952                 freepath = NULL;
1953                 pmc_getfilename(vp, &fullpath, &freepath);
1954                 last_vp = vp;
1955
1956                 vrele(vp);
1957
1958                 vp = NULL;
1959                 pmclog_process_map_in(po, p->p_pid, start_addr, fullpath);
1960                 if (freepath)
1961                         free(freepath, M_TEMP);
1962
1963                 vm_map_lock_read(map);
1964
1965                 /*
1966                  * If our saved timestamp doesn't match, this means
1967                  * that the vm_map was modified out from under us and
1968                  * we can't trust our current "entry" pointer.  Do a
1969                  * new lookup for this entry.  If there is no entry
1970                  * for this address range, vm_map_lookup_entry() will
1971                  * return the previous one, so we always want to go to
1972                  * entry->next on the next loop iteration.
1973                  *
1974                  * There is an edge condition here that can occur if
1975                  * there is no entry at or before this address.  In
1976                  * this situation, vm_map_lookup_entry returns
1977                  * &map->header, which would cause our loop to abort
1978                  * without processing the rest of the map.  However,
1979                  * in practice this will never happen for process
1980                  * vm_map.  This is because the executable's text
1981                  * segment is the first mapping in the proc's address
1982                  * space, and this mapping is never removed until the
1983                  * process exits, so there will always be a non-header
1984                  * entry at or before the requested address for
1985                  * vm_map_lookup_entry to return.
1986                  */
1987                 if (map->timestamp != last_timestamp)
1988                         vm_map_lookup_entry(map, last_end - 1, &entry);
1989         }
1990
1991         vm_map_unlock_read(map);
1992         vmspace_free(vm);
1993         return;
1994 }
1995
1996 /*
1997  * Log mappings for all processes in the system.
1998  */
1999
2000 static void
2001 pmc_log_all_process_mappings(struct pmc_owner *po)
2002 {
2003         struct proc *p, *top;
2004
2005         sx_assert(&pmc_sx, SX_XLOCKED);
2006
2007         if ((p = pfind(1)) == NULL)
2008                 panic("[pmc,%d] Cannot find init", __LINE__);
2009
2010         PROC_UNLOCK(p);
2011
2012         sx_slock(&proctree_lock);
2013
2014         top = p;
2015
2016         for (;;) {
2017                 pmc_log_process_mappings(po, p);
2018                 if (!LIST_EMPTY(&p->p_children))
2019                         p = LIST_FIRST(&p->p_children);
2020                 else for (;;) {
2021                         if (p == top)
2022                                 goto done;
2023                         if (LIST_NEXT(p, p_sibling)) {
2024                                 p = LIST_NEXT(p, p_sibling);
2025                                 break;
2026                         }
2027                         p = p->p_pptr;
2028                 }
2029         }
2030  done:
2031         sx_sunlock(&proctree_lock);
2032 }
2033
2034 /*
2035  * The 'hook' invoked from the kernel proper
2036  */
2037
2038
2039 #ifdef  HWPMC_DEBUG
2040 const char *pmc_hooknames[] = {
2041         /* these strings correspond to PMC_FN_* in <sys/pmckern.h> */
2042         "",
2043         "EXEC",
2044         "CSW-IN",
2045         "CSW-OUT",
2046         "SAMPLE",
2047         "UNUSED1",
2048         "UNUSED2",
2049         "MMAP",
2050         "MUNMAP",
2051         "CALLCHAIN-NMI",
2052         "CALLCHAIN-SOFT",
2053         "SOFTSAMPLING",
2054         "THR-CREATE",
2055         "THR-EXIT",
2056         "THR-USERRET",
2057         "THR-CREATE-LOG",
2058         "THR-EXIT-LOG",
2059         "PROC-CREATE-LOG"
2060 };
2061 #endif
2062
2063 static int
2064 pmc_hook_handler(struct thread *td, int function, void *arg)
2065 {
2066         int cpu;
2067
2068         PMCDBG4(MOD,PMH,1, "hook td=%p func=%d \"%s\" arg=%p", td, function,
2069             pmc_hooknames[function], arg);
2070
2071         switch (function)
2072         {
2073
2074         /*
2075          * Process exec()
2076          */
2077
2078         case PMC_FN_PROCESS_EXEC:
2079         {
2080                 char *fullpath, *freepath;
2081                 unsigned int ri;
2082                 int is_using_hwpmcs;
2083                 struct pmc *pm;
2084                 struct proc *p;
2085                 struct pmc_owner *po;
2086                 struct pmc_process *pp;
2087                 struct pmckern_procexec *pk;
2088
2089                 sx_assert(&pmc_sx, SX_XLOCKED);
2090
2091                 p = td->td_proc;
2092                 pmc_getfilename(p->p_textvp, &fullpath, &freepath);
2093
2094                 pk = (struct pmckern_procexec *) arg;
2095
2096                 epoch_enter_preempt(global_epoch_preempt);
2097                 /* Inform owners of SS mode PMCs of the exec event. */
2098                 CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
2099                     if (po->po_flags & PMC_PO_OWNS_LOGFILE)
2100                             pmclog_process_procexec(po, PMC_ID_INVALID,
2101                                 p->p_pid, pk->pm_entryaddr, fullpath);
2102                 epoch_exit_preempt(global_epoch_preempt);
2103
2104                 PROC_LOCK(p);
2105                 is_using_hwpmcs = p->p_flag & P_HWPMC;
2106                 PROC_UNLOCK(p);
2107
2108                 if (!is_using_hwpmcs) {
2109                         if (freepath)
2110                                 free(freepath, M_TEMP);
2111                         break;
2112                 }
2113
2114                 /*
2115                  * PMCs are not inherited across an exec():  remove any
2116                  * PMCs that this process is the owner of.
2117                  */
2118
2119                 if ((po = pmc_find_owner_descriptor(p)) != NULL) {
2120                         pmc_remove_owner(po);
2121                         pmc_destroy_owner_descriptor(po);
2122                 }
2123
2124                 /*
2125                  * If the process being exec'ed is not the target of any
2126                  * PMC, we are done.
2127                  */
2128                 if ((pp = pmc_find_process_descriptor(p, 0)) == NULL) {
2129                         if (freepath)
2130                                 free(freepath, M_TEMP);
2131                         break;
2132                 }
2133
2134                 /*
2135                  * Log the exec event to all monitoring owners.  Skip
2136                  * owners who have already received the event because
2137                  * they had system sampling PMCs active.
2138                  */
2139                 for (ri = 0; ri < md->pmd_npmc; ri++)
2140                         if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) {
2141                                 po = pm->pm_owner;
2142                                 if (po->po_sscount == 0 &&
2143                                     po->po_flags & PMC_PO_OWNS_LOGFILE)
2144                                         pmclog_process_procexec(po, pm->pm_id,
2145                                             p->p_pid, pk->pm_entryaddr,
2146                                             fullpath);
2147                         }
2148
2149                 if (freepath)
2150                         free(freepath, M_TEMP);
2151
2152
2153                 PMCDBG4(PRC,EXC,1, "exec proc=%p (%d, %s) cred-changed=%d",
2154                     p, p->p_pid, p->p_comm, pk->pm_credentialschanged);
2155
2156                 if (pk->pm_credentialschanged == 0) /* no change */
2157                         break;
2158
2159                 /*
2160                  * If the newly exec()'ed process has a different credential
2161                  * than before, allow it to be the target of a PMC only if
2162                  * the PMC's owner has sufficient privilege.
2163                  */
2164
2165                 for (ri = 0; ri < md->pmd_npmc; ri++)
2166                         if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL)
2167                                 if (pmc_can_attach(pm, td->td_proc) != 0)
2168                                         pmc_detach_one_process(td->td_proc,
2169                                             pm, PMC_FLAG_NONE);
2170
2171                 KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= (int) md->pmd_npmc,
2172                     ("[pmc,%d] Illegal ref count %d on pp %p", __LINE__,
2173                         pp->pp_refcnt, pp));
2174
2175                 /*
2176                  * If this process is no longer the target of any
2177                  * PMCs, we can remove the process entry and free
2178                  * up space.
2179                  */
2180
2181                 if (pp->pp_refcnt == 0) {
2182                         pmc_remove_process_descriptor(pp);
2183                         pmc_destroy_process_descriptor(pp);
2184                         break;
2185                 }
2186
2187         }
2188         break;
2189
2190         case PMC_FN_CSW_IN:
2191                 pmc_process_csw_in(td);
2192                 break;
2193
2194         case PMC_FN_CSW_OUT:
2195                 pmc_process_csw_out(td);
2196                 break;
2197
2198         /*
2199          * Process accumulated PC samples.
2200          *
2201          * This function is expected to be called by hardclock() for
2202          * each CPU that has accumulated PC samples.
2203          *
2204          * This function is to be executed on the CPU whose samples
2205          * are being processed.
2206          */
2207         case PMC_FN_DO_SAMPLES:
2208
2209                 /*
2210                  * Clear the cpu specific bit in the CPU mask before
2211                  * do the rest of the processing.  If the NMI handler
2212                  * gets invoked after the "atomic_clear_int()" call
2213                  * below but before "pmc_process_samples()" gets
2214                  * around to processing the interrupt, then we will
2215                  * come back here at the next hardclock() tick (and
2216                  * may find nothing to do if "pmc_process_samples()"
2217                  * had already processed the interrupt).  We don't
2218                  * lose the interrupt sample.
2219                  */
2220                 DPCPU_SET(pmc_sampled, 0);
2221                 cpu = PCPU_GET(cpuid);
2222                 pmc_process_samples(cpu, PMC_HR);
2223                 pmc_process_samples(cpu, PMC_SR);
2224                 pmc_process_samples(cpu, PMC_UR);
2225                 break;
2226
2227         case PMC_FN_MMAP:
2228                 pmc_process_mmap(td, (struct pmckern_map_in *) arg);
2229                 break;
2230
2231         case PMC_FN_MUNMAP:
2232                 MPASS(in_epoch() || sx_xlocked(&pmc_sx));
2233                 pmc_process_munmap(td, (struct pmckern_map_out *) arg);
2234                 break;
2235
2236         case PMC_FN_PROC_CREATE_LOG:
2237                 pmc_process_proccreate((struct proc *)arg);
2238                 break;
2239
2240         case PMC_FN_USER_CALLCHAIN:
2241                 /*
2242                  * Record a call chain.
2243                  */
2244                 KASSERT(td == curthread, ("[pmc,%d] td != curthread",
2245                     __LINE__));
2246
2247                 pmc_capture_user_callchain(PCPU_GET(cpuid), PMC_HR,
2248                     (struct trapframe *) arg);
2249
2250                 KASSERT(td->td_pinned == 1,
2251                         ("[pmc,%d] invalid td_pinned value", __LINE__));
2252                 sched_unpin();  /* Can migrate safely now. */
2253
2254                 td->td_pflags &= ~TDP_CALLCHAIN;
2255                 break;
2256
2257         case PMC_FN_USER_CALLCHAIN_SOFT:
2258                 /*
2259                  * Record a call chain.
2260                  */
2261                 KASSERT(td == curthread, ("[pmc,%d] td != curthread",
2262                     __LINE__));
2263
2264                 cpu = PCPU_GET(cpuid);
2265                 pmc_capture_user_callchain(cpu, PMC_SR,
2266                     (struct trapframe *) arg);
2267
2268                 KASSERT(td->td_pinned == 1,
2269                     ("[pmc,%d] invalid td_pinned value", __LINE__));
2270
2271                 sched_unpin();  /* Can migrate safely now. */
2272
2273                 td->td_pflags &= ~TDP_CALLCHAIN;
2274                 break;
2275
2276         case PMC_FN_SOFT_SAMPLING:
2277                 /*
2278                  * Call soft PMC sampling intr.
2279                  */
2280                 pmc_soft_intr((struct pmckern_soft *) arg);
2281                 break;
2282
2283         case PMC_FN_THR_CREATE:
2284                 pmc_process_thread_add(td);
2285                 pmc_process_threadcreate(td);
2286                 break;
2287
2288         case PMC_FN_THR_CREATE_LOG:
2289                 pmc_process_threadcreate(td);
2290                 break;
2291
2292         case PMC_FN_THR_EXIT:
2293                 KASSERT(td == curthread, ("[pmc,%d] td != curthread",
2294                     __LINE__));
2295                 pmc_process_thread_delete(td);
2296                 pmc_process_threadexit(td);
2297                 break;
2298         case PMC_FN_THR_EXIT_LOG:
2299                 pmc_process_threadexit(td);
2300                 break;
2301         case PMC_FN_THR_USERRET:
2302                 KASSERT(td == curthread, ("[pmc,%d] td != curthread",
2303                     __LINE__));
2304                 pmc_process_thread_userret(td);
2305                 break;
2306
2307         default:
2308 #ifdef  HWPMC_DEBUG
2309                 KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function));
2310 #endif
2311                 break;
2312
2313         }
2314
2315         return 0;
2316 }
2317
2318 /*
2319  * allocate a 'struct pmc_owner' descriptor in the owner hash table.
2320  */
2321
2322 static struct pmc_owner *
2323 pmc_allocate_owner_descriptor(struct proc *p)
2324 {
2325         uint32_t hindex;
2326         struct pmc_owner *po;
2327         struct pmc_ownerhash *poh;
2328
2329         hindex = PMC_HASH_PTR(p, pmc_ownerhashmask);
2330         poh = &pmc_ownerhash[hindex];
2331
2332         /* allocate space for N pointers and one descriptor struct */
2333         po = malloc(sizeof(struct pmc_owner), M_PMC, M_WAITOK|M_ZERO);
2334         po->po_owner = p;
2335         LIST_INSERT_HEAD(poh, po, po_next); /* insert into hash table */
2336
2337         TAILQ_INIT(&po->po_logbuffers);
2338         mtx_init(&po->po_mtx, "pmc-owner-mtx", "pmc-per-proc", MTX_SPIN);
2339
2340         PMCDBG4(OWN,ALL,1, "allocate-owner proc=%p (%d, %s) pmc-owner=%p",
2341             p, p->p_pid, p->p_comm, po);
2342
2343         return po;
2344 }
2345
2346 static void
2347 pmc_destroy_owner_descriptor(struct pmc_owner *po)
2348 {
2349
2350         PMCDBG4(OWN,REL,1, "destroy-owner po=%p proc=%p (%d, %s)",
2351             po, po->po_owner, po->po_owner->p_pid, po->po_owner->p_comm);
2352
2353         mtx_destroy(&po->po_mtx);
2354         free(po, M_PMC);
2355 }
2356
2357 /*
2358  * Allocate a thread descriptor from the free pool.
2359  *
2360  * NOTE: This *can* return NULL.
2361  */
2362 static struct pmc_thread *
2363 pmc_thread_descriptor_pool_alloc(void)
2364 {
2365         struct pmc_thread *pt;
2366
2367         mtx_lock_spin(&pmc_threadfreelist_mtx);
2368         if ((pt = LIST_FIRST(&pmc_threadfreelist)) != NULL) {
2369                 LIST_REMOVE(pt, pt_next);
2370                 pmc_threadfreelist_entries--;
2371         }
2372         mtx_unlock_spin(&pmc_threadfreelist_mtx);
2373
2374         return (pt);
2375 }
2376
2377 /*
2378  * Add a thread descriptor to the free pool. We use this instead of free()
2379  * to maintain a cache of free entries. Additionally, we can safely call
2380  * this function when we cannot call free(), such as in a critical section.
2381  *
2382  */
2383 static void
2384 pmc_thread_descriptor_pool_free(struct pmc_thread *pt)
2385 {
2386
2387         if (pt == NULL)
2388                 return;
2389
2390         memset(pt, 0, THREADENTRY_SIZE);
2391         mtx_lock_spin(&pmc_threadfreelist_mtx);
2392         LIST_INSERT_HEAD(&pmc_threadfreelist, pt, pt_next);
2393         pmc_threadfreelist_entries++;
2394         if (pmc_threadfreelist_entries > pmc_threadfreelist_max)
2395                 GROUPTASK_ENQUEUE(&free_gtask);
2396         mtx_unlock_spin(&pmc_threadfreelist_mtx);
2397 }
2398
2399 /*
2400  * A callout to manage the free list.
2401  */
2402 static void
2403 pmc_thread_descriptor_pool_free_task(void *arg __unused)
2404 {
2405         struct pmc_thread *pt;
2406         LIST_HEAD(, pmc_thread) tmplist;
2407         int delta;
2408
2409         LIST_INIT(&tmplist);
2410         /* Determine what changes, if any, we need to make. */
2411         mtx_lock_spin(&pmc_threadfreelist_mtx);
2412         delta = pmc_threadfreelist_entries - pmc_threadfreelist_max;
2413         while (delta > 0 &&
2414                    (pt = LIST_FIRST(&pmc_threadfreelist)) != NULL) {
2415                 delta--;
2416                 LIST_REMOVE(pt, pt_next);
2417                 LIST_INSERT_HEAD(&tmplist, pt, pt_next);
2418         }
2419         mtx_unlock_spin(&pmc_threadfreelist_mtx);
2420
2421         /* If there are entries to free, free them. */
2422         while (!LIST_EMPTY(&tmplist)) {
2423                 pt = LIST_FIRST(&tmplist);
2424                 LIST_REMOVE(pt, pt_next);
2425                 free(pt, M_PMC);
2426         }
2427 }
2428
2429 /*
2430  * Drain the thread free pool, freeing all allocations.
2431  */
2432 static void
2433 pmc_thread_descriptor_pool_drain()
2434 {
2435         struct pmc_thread *pt, *next;
2436
2437         LIST_FOREACH_SAFE(pt, &pmc_threadfreelist, pt_next, next) {
2438                 LIST_REMOVE(pt, pt_next);
2439                 free(pt, M_PMC);
2440         }
2441 }
2442
2443 /*
2444  * find the descriptor corresponding to thread 'td', adding or removing it
2445  * as specified by 'mode'.
2446  *
2447  * Note that this supports additional mode flags in addition to those
2448  * supported by pmc_find_process_descriptor():
2449  * PMC_FLAG_NOWAIT: Causes the function to not wait for mallocs.
2450  *     This makes it safe to call while holding certain other locks.
2451  */
2452
2453 static struct pmc_thread *
2454 pmc_find_thread_descriptor(struct pmc_process *pp, struct thread *td,
2455     uint32_t mode)
2456 {
2457         struct pmc_thread *pt = NULL, *ptnew = NULL;
2458         int wait_flag;
2459
2460         KASSERT(td != NULL, ("[pmc,%d] called to add NULL td", __LINE__));
2461
2462         /*
2463          * Pre-allocate memory in the PMC_FLAG_ALLOCATE case prior to
2464          * acquiring the lock.
2465          */
2466         if (mode & PMC_FLAG_ALLOCATE) {
2467                 if ((ptnew = pmc_thread_descriptor_pool_alloc()) == NULL) {
2468                         wait_flag = M_WAITOK;
2469                         if ((mode & PMC_FLAG_NOWAIT) || in_epoch())
2470                                 wait_flag = M_NOWAIT;
2471
2472                         ptnew = malloc(THREADENTRY_SIZE, M_PMC,
2473                             wait_flag|M_ZERO);
2474                 }
2475         }
2476
2477         mtx_lock_spin(pp->pp_tdslock);
2478
2479         LIST_FOREACH(pt, &pp->pp_tds, pt_next)
2480                 if (pt->pt_td == td)
2481                         break;
2482
2483         if ((mode & PMC_FLAG_REMOVE) && pt != NULL)
2484                 LIST_REMOVE(pt, pt_next);
2485
2486         if ((mode & PMC_FLAG_ALLOCATE) && pt == NULL && ptnew != NULL) {
2487                 pt = ptnew;
2488                 ptnew = NULL;
2489                 pt->pt_td = td;
2490                 LIST_INSERT_HEAD(&pp->pp_tds, pt, pt_next);
2491         }
2492
2493         mtx_unlock_spin(pp->pp_tdslock);
2494
2495         if (ptnew != NULL) {
2496                 free(ptnew, M_PMC);
2497         }
2498
2499         return pt;
2500 }
2501
2502 /*
2503  * Try to add thread descriptors for each thread in a process.
2504  */
2505
2506 static void
2507 pmc_add_thread_descriptors_from_proc(struct proc *p, struct pmc_process *pp)
2508 {
2509         struct thread *curtd;
2510         struct pmc_thread **tdlist;
2511         int i, tdcnt, tdlistsz;
2512
2513         KASSERT(!PROC_LOCKED(p), ("[pmc,%d] proc unexpectedly locked",
2514             __LINE__));
2515         tdcnt = 32;
2516  restart:
2517         tdlistsz = roundup2(tdcnt, 32);
2518
2519         tdcnt = 0;
2520         tdlist = malloc(sizeof(struct pmc_thread*) * tdlistsz, M_TEMP, M_WAITOK);
2521
2522         PROC_LOCK(p);
2523         FOREACH_THREAD_IN_PROC(p, curtd)
2524                 tdcnt++;
2525         if (tdcnt >= tdlistsz) {
2526                 PROC_UNLOCK(p);
2527                 free(tdlist, M_TEMP);
2528                 goto restart;
2529         }
2530         /*
2531          * Try to add each thread to the list without sleeping. If unable,
2532          * add to a queue to retry after dropping the process lock.
2533          */
2534         tdcnt = 0;
2535         FOREACH_THREAD_IN_PROC(p, curtd) {
2536                 tdlist[tdcnt] = pmc_find_thread_descriptor(pp, curtd,
2537                                                    PMC_FLAG_ALLOCATE|PMC_FLAG_NOWAIT);
2538                 if (tdlist[tdcnt] == NULL) {
2539                         PROC_UNLOCK(p);
2540                         for (i = 0; i <= tdcnt; i++)
2541                                 pmc_thread_descriptor_pool_free(tdlist[i]);
2542                         free(tdlist, M_TEMP);
2543                         goto restart;
2544                 }
2545                 tdcnt++;
2546         }
2547         PROC_UNLOCK(p);
2548         free(tdlist, M_TEMP);
2549 }
2550
2551 /*
2552  * find the descriptor corresponding to process 'p', adding or removing it
2553  * as specified by 'mode'.
2554  */
2555
2556 static struct pmc_process *
2557 pmc_find_process_descriptor(struct proc *p, uint32_t mode)
2558 {
2559         uint32_t hindex;
2560         struct pmc_process *pp, *ppnew;
2561         struct pmc_processhash *pph;
2562
2563         hindex = PMC_HASH_PTR(p, pmc_processhashmask);
2564         pph = &pmc_processhash[hindex];
2565
2566         ppnew = NULL;
2567
2568         /*
2569          * Pre-allocate memory in the PMC_FLAG_ALLOCATE case since we
2570          * cannot call malloc(9) once we hold a spin lock.
2571          */
2572         if (mode & PMC_FLAG_ALLOCATE)
2573                 ppnew = malloc(sizeof(struct pmc_process) + md->pmd_npmc *
2574                     sizeof(struct pmc_targetstate), M_PMC, M_WAITOK|M_ZERO);
2575
2576         mtx_lock_spin(&pmc_processhash_mtx);
2577         LIST_FOREACH(pp, pph, pp_next)
2578             if (pp->pp_proc == p)
2579                     break;
2580
2581         if ((mode & PMC_FLAG_REMOVE) && pp != NULL)
2582                 LIST_REMOVE(pp, pp_next);
2583
2584         if ((mode & PMC_FLAG_ALLOCATE) && pp == NULL &&
2585             ppnew != NULL) {
2586                 ppnew->pp_proc = p;
2587                 LIST_INIT(&ppnew->pp_tds);
2588                 ppnew->pp_tdslock = mtx_pool_find(pmc_mtxpool, ppnew);
2589                 LIST_INSERT_HEAD(pph, ppnew, pp_next);
2590                 mtx_unlock_spin(&pmc_processhash_mtx);
2591                 pp = ppnew;
2592                 ppnew = NULL;
2593
2594                 /* Add thread descriptors for this process' current threads. */
2595                 pmc_add_thread_descriptors_from_proc(p, pp);
2596         }
2597         else
2598                 mtx_unlock_spin(&pmc_processhash_mtx);
2599
2600         if (ppnew != NULL)
2601                 free(ppnew, M_PMC);
2602
2603         return pp;
2604 }
2605
2606 /*
2607  * remove a process descriptor from the process hash table.
2608  */
2609
2610 static void
2611 pmc_remove_process_descriptor(struct pmc_process *pp)
2612 {
2613         KASSERT(pp->pp_refcnt == 0,
2614             ("[pmc,%d] Removing process descriptor %p with count %d",
2615                 __LINE__, pp, pp->pp_refcnt));
2616
2617         mtx_lock_spin(&pmc_processhash_mtx);
2618         LIST_REMOVE(pp, pp_next);
2619         mtx_unlock_spin(&pmc_processhash_mtx);
2620 }
2621
2622 /*
2623  * destroy a process descriptor.
2624  */
2625
2626 static void
2627 pmc_destroy_process_descriptor(struct pmc_process *pp)
2628 {
2629         struct pmc_thread *pmc_td;
2630
2631         while ((pmc_td = LIST_FIRST(&pp->pp_tds)) != NULL) {
2632                 LIST_REMOVE(pmc_td, pt_next);
2633                 pmc_thread_descriptor_pool_free(pmc_td);
2634         }
2635         free(pp, M_PMC);
2636 }
2637
2638
2639 /*
2640  * find an owner descriptor corresponding to proc 'p'
2641  */
2642
2643 static struct pmc_owner *
2644 pmc_find_owner_descriptor(struct proc *p)
2645 {
2646         uint32_t hindex;
2647         struct pmc_owner *po;
2648         struct pmc_ownerhash *poh;
2649
2650         hindex = PMC_HASH_PTR(p, pmc_ownerhashmask);
2651         poh = &pmc_ownerhash[hindex];
2652
2653         po = NULL;
2654         LIST_FOREACH(po, poh, po_next)
2655             if (po->po_owner == p)
2656                     break;
2657
2658         PMCDBG5(OWN,FND,1, "find-owner proc=%p (%d, %s) hindex=0x%x -> "
2659             "pmc-owner=%p", p, p->p_pid, p->p_comm, hindex, po);
2660
2661         return po;
2662 }
2663
2664 /*
2665  * pmc_allocate_pmc_descriptor
2666  *
2667  * Allocate a pmc descriptor and initialize its
2668  * fields.
2669  */
2670
2671 static struct pmc *
2672 pmc_allocate_pmc_descriptor(void)
2673 {
2674         struct pmc *pmc;
2675
2676         pmc = malloc(sizeof(struct pmc), M_PMC, M_WAITOK|M_ZERO);
2677         pmc->pm_runcount = counter_u64_alloc(M_WAITOK);
2678         pmc->pm_pcpu_state = malloc(sizeof(struct pmc_pcpu_state)*mp_ncpus, M_PMC, M_WAITOK|M_ZERO);
2679         PMCDBG1(PMC,ALL,1, "allocate-pmc -> pmc=%p", pmc);
2680
2681         return pmc;
2682 }
2683
2684 /*
2685  * Destroy a pmc descriptor.
2686  */
2687
2688 static void
2689 pmc_destroy_pmc_descriptor(struct pmc *pm)
2690 {
2691
2692         KASSERT(pm->pm_state == PMC_STATE_DELETED ||
2693             pm->pm_state == PMC_STATE_FREE,
2694             ("[pmc,%d] destroying non-deleted PMC", __LINE__));
2695         KASSERT(LIST_EMPTY(&pm->pm_targets),
2696             ("[pmc,%d] destroying pmc with targets", __LINE__));
2697         KASSERT(pm->pm_owner == NULL,
2698             ("[pmc,%d] destroying pmc attached to an owner", __LINE__));
2699         KASSERT(counter_u64_fetch(pm->pm_runcount) == 0,
2700             ("[pmc,%d] pmc has non-zero run count %ld", __LINE__,
2701                  (unsigned long)counter_u64_fetch(pm->pm_runcount)));
2702
2703         counter_u64_free(pm->pm_runcount);
2704         free(pm->pm_pcpu_state, M_PMC);
2705         free(pm, M_PMC);
2706 }
2707
2708 static void
2709 pmc_wait_for_pmc_idle(struct pmc *pm)
2710 {
2711 #ifdef HWPMC_DEBUG
2712         volatile int maxloop;
2713
2714         maxloop = 100 * pmc_cpu_max();
2715 #endif
2716         /*
2717          * Loop (with a forced context switch) till the PMC's runcount
2718          * comes down to zero.
2719          */
2720         pmclog_flush(pm->pm_owner, 1);
2721         while (counter_u64_fetch(pm->pm_runcount) > 0) {
2722                 pmclog_flush(pm->pm_owner, 1);
2723 #ifdef HWPMC_DEBUG
2724                 maxloop--;
2725                 KASSERT(maxloop > 0,
2726                     ("[pmc,%d] (ri%d, rc%ld) waiting too long for "
2727                         "pmc to be free", __LINE__,
2728                          PMC_TO_ROWINDEX(pm), (unsigned long)counter_u64_fetch(pm->pm_runcount)));
2729 #endif
2730                 pmc_force_context_switch();
2731         }
2732 }
2733
2734 /*
2735  * This function does the following things:
2736  *
2737  *  - detaches the PMC from hardware
2738  *  - unlinks all target threads that were attached to it
2739  *  - removes the PMC from its owner's list
2740  *  - destroys the PMC private mutex
2741  *
2742  * Once this function completes, the given pmc pointer can be freed by
2743  * calling pmc_destroy_pmc_descriptor().
2744  */
2745
2746 static void
2747 pmc_release_pmc_descriptor(struct pmc *pm)
2748 {
2749         enum pmc_mode mode;
2750         struct pmc_hw *phw;
2751         u_int adjri, ri, cpu;
2752         struct pmc_owner *po;
2753         struct pmc_binding pb;
2754         struct pmc_process *pp;
2755         struct pmc_classdep *pcd;
2756         struct pmc_target *ptgt, *tmp;
2757
2758         sx_assert(&pmc_sx, SX_XLOCKED);
2759
2760         KASSERT(pm, ("[pmc,%d] null pmc", __LINE__));
2761
2762         ri   = PMC_TO_ROWINDEX(pm);
2763         pcd  = pmc_ri_to_classdep(md, ri, &adjri);
2764         mode = PMC_TO_MODE(pm);
2765
2766         PMCDBG3(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri,
2767             mode);
2768
2769         /*
2770          * First, we take the PMC off hardware.
2771          */
2772         cpu = 0;
2773         if (PMC_IS_SYSTEM_MODE(mode)) {
2774
2775                 /*
2776                  * A system mode PMC runs on a specific CPU.  Switch
2777                  * to this CPU and turn hardware off.
2778                  */
2779                 pmc_save_cpu_binding(&pb);
2780
2781                 cpu = PMC_TO_CPU(pm);
2782
2783                 pmc_select_cpu(cpu);
2784
2785                 /* switch off non-stalled CPUs */
2786                 pm->pm_pcpu_state[cpu].pps_cpustate = 0;
2787                 if (pm->pm_state == PMC_STATE_RUNNING &&
2788                         pm->pm_pcpu_state[cpu].pps_stalled == 0) {
2789
2790                         phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
2791
2792                         KASSERT(phw->phw_pmc == pm,
2793                             ("[pmc, %d] pmc ptr ri(%d) hw(%p) pm(%p)",
2794                                 __LINE__, ri, phw->phw_pmc, pm));
2795                         PMCDBG2(PMC,REL,2, "stopping cpu=%d ri=%d", cpu, ri);
2796
2797                         critical_enter();
2798                         pcd->pcd_stop_pmc(cpu, adjri);
2799                         critical_exit();
2800                 }
2801
2802                 PMCDBG2(PMC,REL,2, "decfg cpu=%d ri=%d", cpu, ri);
2803
2804                 critical_enter();
2805                 pcd->pcd_config_pmc(cpu, adjri, NULL);
2806                 critical_exit();
2807
2808                 /* adjust the global and process count of SS mode PMCs */
2809                 if (mode == PMC_MODE_SS && pm->pm_state == PMC_STATE_RUNNING) {
2810                         po = pm->pm_owner;
2811                         po->po_sscount--;
2812                         if (po->po_sscount == 0) {
2813                                 atomic_subtract_rel_int(&pmc_ss_count, 1);
2814                                 CK_LIST_REMOVE(po, po_ssnext);
2815                                 epoch_wait_preempt(global_epoch_preempt);
2816                         }
2817                 }
2818
2819                 pm->pm_state = PMC_STATE_DELETED;
2820
2821                 pmc_restore_cpu_binding(&pb);
2822
2823                 /*
2824                  * We could have references to this PMC structure in
2825                  * the per-cpu sample queues.  Wait for the queue to
2826                  * drain.
2827                  */
2828                 pmc_wait_for_pmc_idle(pm);
2829
2830         } else if (PMC_IS_VIRTUAL_MODE(mode)) {
2831
2832                 /*
2833                  * A virtual PMC could be running on multiple CPUs at
2834                  * a given instant.
2835                  *
2836                  * By marking its state as DELETED, we ensure that
2837                  * this PMC is never further scheduled on hardware.
2838                  *
2839                  * Then we wait till all CPUs are done with this PMC.
2840                  */
2841                 pm->pm_state = PMC_STATE_DELETED;
2842
2843
2844                 /* Wait for the PMCs runcount to come to zero. */
2845                 pmc_wait_for_pmc_idle(pm);
2846
2847                 /*
2848                  * At this point the PMC is off all CPUs and cannot be
2849                  * freshly scheduled onto a CPU.  It is now safe to
2850                  * unlink all targets from this PMC.  If a
2851                  * process-record's refcount falls to zero, we remove
2852                  * it from the hash table.  The module-wide SX lock
2853                  * protects us from races.
2854                  */
2855                 LIST_FOREACH_SAFE(ptgt, &pm->pm_targets, pt_next, tmp) {
2856                         pp = ptgt->pt_process;
2857                         pmc_unlink_target_process(pm, pp); /* frees 'ptgt' */
2858
2859                         PMCDBG1(PMC,REL,3, "pp->refcnt=%d", pp->pp_refcnt);
2860
2861                         /*
2862                          * If the target process record shows that no
2863                          * PMCs are attached to it, reclaim its space.
2864                          */
2865
2866                         if (pp->pp_refcnt == 0) {
2867                                 pmc_remove_process_descriptor(pp);
2868                                 pmc_destroy_process_descriptor(pp);
2869                         }
2870                 }
2871
2872                 cpu = curthread->td_oncpu; /* setup cpu for pmd_release() */
2873
2874         }
2875
2876         /*
2877          * Release any MD resources
2878          */
2879         (void) pcd->pcd_release_pmc(cpu, adjri, pm);
2880
2881         /*
2882          * Update row disposition
2883          */
2884
2885         if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm)))
2886                 PMC_UNMARK_ROW_STANDALONE(ri);
2887         else
2888                 PMC_UNMARK_ROW_THREAD(ri);
2889
2890         /* unlink from the owner's list */
2891         if (pm->pm_owner) {
2892                 LIST_REMOVE(pm, pm_next);
2893                 pm->pm_owner = NULL;
2894         }
2895 }
2896
2897 /*
2898  * Register an owner and a pmc.
2899  */
2900
2901 static int
2902 pmc_register_owner(struct proc *p, struct pmc *pmc)
2903 {
2904         struct pmc_owner *po;
2905
2906         sx_assert(&pmc_sx, SX_XLOCKED);
2907
2908         if ((po = pmc_find_owner_descriptor(p)) == NULL)
2909                 if ((po = pmc_allocate_owner_descriptor(p)) == NULL)
2910                         return ENOMEM;
2911
2912         KASSERT(pmc->pm_owner == NULL,
2913             ("[pmc,%d] attempting to own an initialized PMC", __LINE__));
2914         pmc->pm_owner  = po;
2915
2916         LIST_INSERT_HEAD(&po->po_pmcs, pmc, pm_next);
2917
2918         PROC_LOCK(p);
2919         p->p_flag |= P_HWPMC;
2920         PROC_UNLOCK(p);
2921
2922         if (po->po_flags & PMC_PO_OWNS_LOGFILE)
2923                 pmclog_process_pmcallocate(pmc);
2924
2925         PMCDBG2(PMC,REG,1, "register-owner pmc-owner=%p pmc=%p",
2926             po, pmc);
2927
2928         return 0;
2929 }
2930
2931 /*
2932  * Return the current row disposition:
2933  * == 0 => FREE
2934  *  > 0 => PROCESS MODE
2935  *  < 0 => SYSTEM MODE
2936  */
2937
2938 int
2939 pmc_getrowdisp(int ri)
2940 {
2941         return pmc_pmcdisp[ri];
2942 }
2943
2944 /*
2945  * Check if a PMC at row index 'ri' can be allocated to the current
2946  * process.
2947  *
2948  * Allocation can fail if:
2949  *   - the current process is already being profiled by a PMC at index 'ri',
2950  *     attached to it via OP_PMCATTACH.
2951  *   - the current process has already allocated a PMC at index 'ri'
2952  *     via OP_ALLOCATE.
2953  */
2954
2955 static int
2956 pmc_can_allocate_rowindex(struct proc *p, unsigned int ri, int cpu)
2957 {
2958         enum pmc_mode mode;
2959         struct pmc *pm;
2960         struct pmc_owner *po;
2961         struct pmc_process *pp;
2962
2963         PMCDBG5(PMC,ALR,1, "can-allocate-rowindex proc=%p (%d, %s) ri=%d "
2964             "cpu=%d", p, p->p_pid, p->p_comm, ri, cpu);
2965
2966         /*
2967          * We shouldn't have already allocated a process-mode PMC at
2968          * row index 'ri'.
2969          *
2970          * We shouldn't have allocated a system-wide PMC on the same
2971          * CPU and same RI.
2972          */
2973         if ((po = pmc_find_owner_descriptor(p)) != NULL)
2974                 LIST_FOREACH(pm, &po->po_pmcs, pm_next) {
2975                     if (PMC_TO_ROWINDEX(pm) == ri) {
2976                             mode = PMC_TO_MODE(pm);
2977                             if (PMC_IS_VIRTUAL_MODE(mode))
2978                                     return EEXIST;
2979                             if (PMC_IS_SYSTEM_MODE(mode) &&
2980                                 (int) PMC_TO_CPU(pm) == cpu)
2981                                     return EEXIST;
2982                     }
2983                 }
2984
2985         /*
2986          * We also shouldn't be the target of any PMC at this index
2987          * since otherwise a PMC_ATTACH to ourselves will fail.
2988          */
2989         if ((pp = pmc_find_process_descriptor(p, 0)) != NULL)
2990                 if (pp->pp_pmcs[ri].pp_pmc)
2991                         return EEXIST;
2992
2993         PMCDBG4(PMC,ALR,2, "can-allocate-rowindex proc=%p (%d, %s) ri=%d ok",
2994             p, p->p_pid, p->p_comm, ri);
2995
2996         return 0;
2997 }
2998
2999 /*
3000  * Check if a given PMC at row index 'ri' can be currently used in
3001  * mode 'mode'.
3002  */
3003
3004 static int
3005 pmc_can_allocate_row(int ri, enum pmc_mode mode)
3006 {
3007         enum pmc_disp   disp;
3008
3009         sx_assert(&pmc_sx, SX_XLOCKED);
3010
3011         PMCDBG2(PMC,ALR,1, "can-allocate-row ri=%d mode=%d", ri, mode);
3012
3013         if (PMC_IS_SYSTEM_MODE(mode))
3014                 disp = PMC_DISP_STANDALONE;
3015         else
3016                 disp = PMC_DISP_THREAD;
3017
3018         /*
3019          * check disposition for PMC row 'ri':
3020          *
3021          * Expected disposition         Row-disposition         Result
3022          *
3023          * STANDALONE                   STANDALONE or FREE      proceed
3024          * STANDALONE                   THREAD                  fail
3025          * THREAD                       THREAD or FREE          proceed
3026          * THREAD                       STANDALONE              fail
3027          */
3028
3029         if (!PMC_ROW_DISP_IS_FREE(ri) &&
3030             !(disp == PMC_DISP_THREAD && PMC_ROW_DISP_IS_THREAD(ri)) &&
3031             !(disp == PMC_DISP_STANDALONE && PMC_ROW_DISP_IS_STANDALONE(ri)))
3032                 return EBUSY;
3033
3034         /*
3035          * All OK
3036          */
3037
3038         PMCDBG2(PMC,ALR,2, "can-allocate-row ri=%d mode=%d ok", ri, mode);
3039
3040         return 0;
3041
3042 }
3043
3044 /*
3045  * Find a PMC descriptor with user handle 'pmcid' for thread 'td'.
3046  */
3047
3048 static struct pmc *
3049 pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, pmc_id_t pmcid)
3050 {
3051         struct pmc *pm;
3052
3053         KASSERT(PMC_ID_TO_ROWINDEX(pmcid) < md->pmd_npmc,
3054             ("[pmc,%d] Illegal pmc index %d (max %d)", __LINE__,
3055                 PMC_ID_TO_ROWINDEX(pmcid), md->pmd_npmc));
3056
3057         LIST_FOREACH(pm, &po->po_pmcs, pm_next)
3058             if (pm->pm_id == pmcid)
3059                     return pm;
3060
3061         return NULL;
3062 }
3063
3064 static int
3065 pmc_find_pmc(pmc_id_t pmcid, struct pmc **pmc)
3066 {
3067
3068         struct pmc *pm, *opm;
3069         struct pmc_owner *po;
3070         struct pmc_process *pp;
3071
3072         PMCDBG1(PMC,FND,1, "find-pmc id=%d", pmcid);
3073         if (PMC_ID_TO_ROWINDEX(pmcid) >= md->pmd_npmc)
3074                 return (EINVAL);
3075
3076         if ((po = pmc_find_owner_descriptor(curthread->td_proc)) == NULL) {
3077                 /*
3078                  * In case of PMC_F_DESCENDANTS child processes we will not find
3079                  * the current process in the owners hash list.  Find the owner
3080                  * process first and from there lookup the po.
3081                  */
3082                 if ((pp = pmc_find_process_descriptor(curthread->td_proc,
3083                     PMC_FLAG_NONE)) == NULL) {
3084                         return ESRCH;
3085                 } else {
3086                         opm = pp->pp_pmcs[PMC_ID_TO_ROWINDEX(pmcid)].pp_pmc;
3087                         if (opm == NULL)
3088                                 return ESRCH;
3089                         if ((opm->pm_flags & (PMC_F_ATTACHED_TO_OWNER|
3090                             PMC_F_DESCENDANTS)) != (PMC_F_ATTACHED_TO_OWNER|
3091                             PMC_F_DESCENDANTS))
3092                                 return ESRCH;
3093                         po = opm->pm_owner;
3094                 }
3095         }
3096
3097         if ((pm = pmc_find_pmc_descriptor_in_process(po, pmcid)) == NULL)
3098                 return EINVAL;
3099
3100         PMCDBG2(PMC,FND,2, "find-pmc id=%d -> pmc=%p", pmcid, pm);
3101
3102         *pmc = pm;
3103         return 0;
3104 }
3105
3106 /*
3107  * Start a PMC.
3108  */
3109
3110 static int
3111 pmc_start(struct pmc *pm)
3112 {
3113         enum pmc_mode mode;
3114         struct pmc_owner *po;
3115         struct pmc_binding pb;
3116         struct pmc_classdep *pcd;
3117         int adjri, error, cpu, ri;
3118
3119         KASSERT(pm != NULL,
3120             ("[pmc,%d] null pm", __LINE__));
3121
3122         mode = PMC_TO_MODE(pm);
3123         ri   = PMC_TO_ROWINDEX(pm);
3124         pcd  = pmc_ri_to_classdep(md, ri, &adjri);
3125
3126         error = 0;
3127
3128         PMCDBG3(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri);
3129
3130         po = pm->pm_owner;
3131
3132         /*
3133          * Disallow PMCSTART if a logfile is required but has not been
3134          * configured yet.
3135          */
3136         if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) &&
3137             (po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
3138                 return (EDOOFUS);       /* programming error */
3139
3140         /*
3141          * If this is a sampling mode PMC, log mapping information for
3142          * the kernel modules that are currently loaded.
3143          */
3144         if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
3145             pmc_log_kernel_mappings(pm);
3146
3147         if (PMC_IS_VIRTUAL_MODE(mode)) {
3148
3149                 /*
3150                  * If a PMCATTACH has never been done on this PMC,
3151                  * attach it to its owner process.
3152                  */
3153
3154                 if (LIST_EMPTY(&pm->pm_targets))
3155                         error = (pm->pm_flags & PMC_F_ATTACH_DONE) ? ESRCH :
3156                             pmc_attach_process(po->po_owner, pm);
3157
3158                 /*
3159                  * If the PMC is attached to its owner, then force a context
3160                  * switch to ensure that the MD state gets set correctly.
3161                  */
3162
3163                 if (error == 0) {
3164                         pm->pm_state = PMC_STATE_RUNNING;
3165                         if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER)
3166                                 pmc_force_context_switch();
3167                 }
3168
3169                 return (error);
3170         }
3171
3172
3173         /*
3174          * A system-wide PMC.
3175          *
3176          * Add the owner to the global list if this is a system-wide
3177          * sampling PMC.
3178          */
3179
3180         if (mode == PMC_MODE_SS) {
3181                 /*
3182                  * Log mapping information for all existing processes in the
3183                  * system.  Subsequent mappings are logged as they happen;
3184                  * see pmc_process_mmap().
3185                  */
3186                 if (po->po_logprocmaps == 0) {
3187                         pmc_log_all_process_mappings(po);
3188                         po->po_logprocmaps = 1;
3189                 }
3190                 po->po_sscount++;
3191                 if (po->po_sscount == 1) {
3192                         atomic_add_rel_int(&pmc_ss_count, 1);
3193                         CK_LIST_INSERT_HEAD(&pmc_ss_owners, po, po_ssnext);
3194                         PMCDBG1(PMC,OPS,1, "po=%p in global list", po);
3195                 }
3196         }
3197
3198         /*
3199          * Move to the CPU associated with this
3200          * PMC, and start the hardware.
3201          */
3202
3203         pmc_save_cpu_binding(&pb);
3204
3205         cpu = PMC_TO_CPU(pm);
3206
3207         if (!pmc_cpu_is_active(cpu))
3208                 return (ENXIO);
3209
3210         pmc_select_cpu(cpu);
3211
3212         /*
3213          * global PMCs are configured at allocation time
3214          * so write out the initial value and start the PMC.
3215          */
3216
3217         pm->pm_state = PMC_STATE_RUNNING;
3218
3219         critical_enter();
3220         if ((error = pcd->pcd_write_pmc(cpu, adjri,
3221                  PMC_IS_SAMPLING_MODE(mode) ?
3222                  pm->pm_sc.pm_reloadcount :
3223                  pm->pm_sc.pm_initial)) == 0) {
3224                 /* If a sampling mode PMC, reset stalled state. */
3225                 if (PMC_IS_SAMPLING_MODE(mode))
3226                         pm->pm_pcpu_state[cpu].pps_stalled = 0;
3227
3228                 /* Indicate that we desire this to run. Start it. */
3229                 pm->pm_pcpu_state[cpu].pps_cpustate = 1;
3230                 error = pcd->pcd_start_pmc(cpu, adjri);
3231         }
3232         critical_exit();
3233
3234         pmc_restore_cpu_binding(&pb);
3235
3236         return (error);
3237 }
3238
3239 /*
3240  * Stop a PMC.
3241  */
3242
3243 static int
3244 pmc_stop(struct pmc *pm)
3245 {
3246         struct pmc_owner *po;
3247         struct pmc_binding pb;
3248         struct pmc_classdep *pcd;
3249         int adjri, cpu, error, ri;
3250
3251         KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__));
3252
3253         PMCDBG3(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm,
3254             PMC_TO_MODE(pm), PMC_TO_ROWINDEX(pm));
3255
3256         pm->pm_state = PMC_STATE_STOPPED;
3257
3258         /*
3259          * If the PMC is a virtual mode one, changing the state to
3260          * non-RUNNING is enough to ensure that the PMC never gets
3261          * scheduled.
3262          *
3263          * If this PMC is current running on a CPU, then it will
3264          * handled correctly at the time its target process is context
3265          * switched out.
3266          */
3267
3268         if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
3269                 return 0;
3270
3271         /*
3272          * A system-mode PMC.  Move to the CPU associated with
3273          * this PMC, and stop the hardware.  We update the
3274          * 'initial count' so that a subsequent PMCSTART will
3275          * resume counting from the current hardware count.
3276          */
3277
3278         pmc_save_cpu_binding(&pb);
3279
3280         cpu = PMC_TO_CPU(pm);
3281
3282         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
3283             ("[pmc,%d] illegal cpu=%d", __LINE__, cpu));
3284
3285         if (!pmc_cpu_is_active(cpu))
3286                 return ENXIO;
3287
3288         pmc_select_cpu(cpu);
3289
3290         ri = PMC_TO_ROWINDEX(pm);
3291         pcd = pmc_ri_to_classdep(md, ri, &adjri);
3292
3293         pm->pm_pcpu_state[cpu].pps_cpustate = 0;
3294         critical_enter();
3295         if ((error = pcd->pcd_stop_pmc(cpu, adjri)) == 0)
3296                 error = pcd->pcd_read_pmc(cpu, adjri, &pm->pm_sc.pm_initial);
3297         critical_exit();
3298
3299         pmc_restore_cpu_binding(&pb);
3300
3301         po = pm->pm_owner;
3302
3303         /* remove this owner from the global list of SS PMC owners */
3304         if (PMC_TO_MODE(pm) == PMC_MODE_SS) {
3305                 po->po_sscount--;
3306                 if (po->po_sscount == 0) {
3307                         atomic_subtract_rel_int(&pmc_ss_count, 1);
3308                         CK_LIST_REMOVE(po, po_ssnext);
3309                         epoch_wait_preempt(global_epoch_preempt);
3310                         PMCDBG1(PMC,OPS,2,"po=%p removed from global list", po);
3311                 }
3312         }
3313
3314         return (error);
3315 }
3316
3317 static struct pmc_classdep *
3318 pmc_class_to_classdep(enum pmc_class class)
3319 {
3320         int n;
3321
3322         for (n = 0; n < md->pmd_nclass; n++)
3323                 if (md->pmd_classdep[n].pcd_class == class)
3324                         return (&md->pmd_classdep[n]);
3325         return (NULL);
3326 }
3327
3328 #if defined(HWPMC_DEBUG) && defined(KTR)
3329 static const char *pmc_op_to_name[] = {
3330 #undef  __PMC_OP
3331 #define __PMC_OP(N, D)  #N ,
3332         __PMC_OPS()
3333         NULL
3334 };
3335 #endif
3336
3337 /*
3338  * The syscall interface
3339  */
3340
3341 #define PMC_GET_SX_XLOCK(...) do {              \
3342         sx_xlock(&pmc_sx);                      \
3343         if (pmc_hook == NULL) {                 \
3344                 sx_xunlock(&pmc_sx);            \
3345                 return __VA_ARGS__;             \
3346         }                                       \
3347 } while (0)
3348
3349 #define PMC_DOWNGRADE_SX() do {                 \
3350         sx_downgrade(&pmc_sx);                  \
3351         is_sx_downgraded = 1;                   \
3352 } while (0)
3353
3354 static int
3355 pmc_syscall_handler(struct thread *td, void *syscall_args)
3356 {
3357         int error, is_sx_downgraded, op;
3358         struct pmc_syscall_args *c;
3359         void *pmclog_proc_handle;
3360         void *arg;
3361
3362         c = (struct pmc_syscall_args *)syscall_args;
3363         op = c->pmop_code;
3364         arg = c->pmop_data;
3365         /* PMC isn't set up yet */
3366         if (pmc_hook == NULL)
3367                 return (EINVAL);
3368         if (op == PMC_OP_CONFIGURELOG) {
3369                 /*
3370                  * We cannot create the logging process inside
3371                  * pmclog_configure_log() because there is a LOR
3372                  * between pmc_sx and process structure locks.
3373                  * Instead, pre-create the process and ignite the loop
3374                  * if everything is fine, otherwise direct the process
3375                  * to exit.
3376                  */
3377                 error = pmclog_proc_create(td, &pmclog_proc_handle);
3378                 if (error != 0)
3379                         goto done_syscall;
3380         }
3381
3382         PMC_GET_SX_XLOCK(ENOSYS);
3383         is_sx_downgraded = 0;
3384         PMCDBG3(MOD,PMS,1, "syscall op=%d \"%s\" arg=%p", op,
3385             pmc_op_to_name[op], arg);
3386
3387         error = 0;
3388         counter_u64_add(pmc_stats.pm_syscalls, 1);
3389
3390         switch (op) {
3391
3392
3393         /*
3394          * Configure a log file.
3395          *
3396          * XXX This OP will be reworked.
3397          */
3398
3399         case PMC_OP_CONFIGURELOG:
3400         {
3401                 struct proc *p;
3402                 struct pmc *pm;
3403                 struct pmc_owner *po;
3404                 struct pmc_op_configurelog cl;
3405
3406                 if ((error = copyin(arg, &cl, sizeof(cl))) != 0) {
3407                         pmclog_proc_ignite(pmclog_proc_handle, NULL);
3408                         break;
3409                 }
3410
3411                 /* mark this process as owning a log file */
3412                 p = td->td_proc;
3413                 if ((po = pmc_find_owner_descriptor(p)) == NULL)
3414                         if ((po = pmc_allocate_owner_descriptor(p)) == NULL) {
3415                                 pmclog_proc_ignite(pmclog_proc_handle, NULL);
3416                                 error = ENOMEM;
3417                                 break;
3418                         }
3419
3420                 /*
3421                  * If a valid fd was passed in, try to configure that,
3422                  * otherwise if 'fd' was less than zero and there was
3423                  * a log file configured, flush its buffers and
3424                  * de-configure it.
3425                  */
3426                 if (cl.pm_logfd >= 0) {
3427                         error = pmclog_configure_log(md, po, cl.pm_logfd);
3428                         pmclog_proc_ignite(pmclog_proc_handle, error == 0 ?
3429                             po : NULL);
3430                 } else if (po->po_flags & PMC_PO_OWNS_LOGFILE) {
3431                         pmclog_proc_ignite(pmclog_proc_handle, NULL);
3432                         error = pmclog_close(po);
3433                         if (error == 0) {
3434                                 LIST_FOREACH(pm, &po->po_pmcs, pm_next)
3435                                     if (pm->pm_flags & PMC_F_NEEDS_LOGFILE &&
3436                                         pm->pm_state == PMC_STATE_RUNNING)
3437                                             pmc_stop(pm);
3438                                 error = pmclog_deconfigure_log(po);
3439                         }
3440                 } else {
3441                         pmclog_proc_ignite(pmclog_proc_handle, NULL);
3442                         error = EINVAL;
3443                 }
3444         }
3445         break;
3446
3447         /*
3448          * Flush a log file.
3449          */
3450
3451         case PMC_OP_FLUSHLOG:
3452         {
3453                 struct pmc_owner *po;
3454
3455                 sx_assert(&pmc_sx, SX_XLOCKED);
3456
3457                 if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
3458                         error = EINVAL;
3459                         break;
3460                 }
3461
3462                 error = pmclog_flush(po, 0);
3463         }
3464         break;
3465
3466         /*
3467          * Close a log file.
3468          */
3469
3470         case PMC_OP_CLOSELOG:
3471         {
3472                 struct pmc_owner *po;
3473
3474                 sx_assert(&pmc_sx, SX_XLOCKED);
3475
3476                 if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
3477                         error = EINVAL;
3478                         break;
3479                 }
3480
3481                 error = pmclog_close(po);
3482         }
3483         break;
3484
3485         /*
3486          * Retrieve hardware configuration.
3487          */
3488
3489         case PMC_OP_GETCPUINFO: /* CPU information */
3490         {
3491                 struct pmc_op_getcpuinfo gci;
3492                 struct pmc_classinfo *pci;
3493                 struct pmc_classdep *pcd;
3494                 int cl;
3495
3496                 gci.pm_cputype = md->pmd_cputype;
3497                 gci.pm_ncpu    = pmc_cpu_max();
3498                 gci.pm_npmc    = md->pmd_npmc;
3499                 gci.pm_nclass  = md->pmd_nclass;
3500                 pci = gci.pm_classes;
3501                 pcd = md->pmd_classdep;
3502                 for (cl = 0; cl < md->pmd_nclass; cl++, pci++, pcd++) {
3503                         pci->pm_caps  = pcd->pcd_caps;
3504                         pci->pm_class = pcd->pcd_class;
3505                         pci->pm_width = pcd->pcd_width;
3506                         pci->pm_num   = pcd->pcd_num;
3507                 }
3508                 error = copyout(&gci, arg, sizeof(gci));
3509         }
3510         break;
3511
3512         /*
3513          * Retrieve soft events list.
3514          */
3515         case PMC_OP_GETDYNEVENTINFO:
3516         {
3517                 enum pmc_class                  cl;
3518                 enum pmc_event                  ev;
3519                 struct pmc_op_getdyneventinfo   *gei;
3520                 struct pmc_dyn_event_descr      dev;
3521                 struct pmc_soft                 *ps;
3522                 uint32_t                        nevent;
3523
3524                 sx_assert(&pmc_sx, SX_LOCKED);
3525
3526                 gei = (struct pmc_op_getdyneventinfo *) arg;
3527
3528                 if ((error = copyin(&gei->pm_class, &cl, sizeof(cl))) != 0)
3529                         break;
3530
3531                 /* Only SOFT class is dynamic. */
3532                 if (cl != PMC_CLASS_SOFT) {
3533                         error = EINVAL;
3534                         break;
3535                 }
3536
3537                 nevent = 0;
3538                 for (ev = PMC_EV_SOFT_FIRST; (int)ev <= PMC_EV_SOFT_LAST; ev++) {
3539                         ps = pmc_soft_ev_acquire(ev);
3540                         if (ps == NULL)
3541                                 continue;
3542                         bcopy(&ps->ps_ev, &dev, sizeof(dev));
3543                         pmc_soft_ev_release(ps);
3544
3545                         error = copyout(&dev,
3546                             &gei->pm_events[nevent],
3547                             sizeof(struct pmc_dyn_event_descr));
3548                         if (error != 0)
3549                                 break;
3550                         nevent++;
3551                 }
3552                 if (error != 0)
3553                         break;
3554
3555                 error = copyout(&nevent, &gei->pm_nevent,
3556                     sizeof(nevent));
3557         }
3558         break;
3559
3560         /*
3561          * Get module statistics
3562          */
3563
3564         case PMC_OP_GETDRIVERSTATS:
3565         {
3566                 struct pmc_op_getdriverstats gms;
3567 #define CFETCH(a, b, field) a.field = counter_u64_fetch(b.field)
3568                 CFETCH(gms, pmc_stats, pm_intr_ignored);
3569                 CFETCH(gms, pmc_stats, pm_intr_processed);
3570                 CFETCH(gms, pmc_stats, pm_intr_bufferfull);
3571                 CFETCH(gms, pmc_stats, pm_syscalls);
3572                 CFETCH(gms, pmc_stats, pm_syscall_errors);
3573                 CFETCH(gms, pmc_stats, pm_buffer_requests);
3574                 CFETCH(gms, pmc_stats, pm_buffer_requests_failed);
3575                 CFETCH(gms, pmc_stats, pm_log_sweeps);
3576 #undef CFETCH
3577                 error = copyout(&gms, arg, sizeof(gms));
3578         }
3579         break;
3580
3581
3582         /*
3583          * Retrieve module version number
3584          */
3585
3586         case PMC_OP_GETMODULEVERSION:
3587         {
3588                 uint32_t cv, modv;
3589
3590                 /* retrieve the client's idea of the ABI version */
3591                 if ((error = copyin(arg, &cv, sizeof(uint32_t))) != 0)
3592                         break;
3593                 /* don't service clients newer than our driver */
3594                 modv = PMC_VERSION;
3595                 if ((cv & 0xFFFF0000) > (modv & 0xFFFF0000)) {
3596                         error = EPROGMISMATCH;
3597                         break;
3598                 }
3599                 error = copyout(&modv, arg, sizeof(int));
3600         }
3601         break;
3602
3603
3604         /*
3605          * Retrieve the state of all the PMCs on a given
3606          * CPU.
3607          */
3608
3609         case PMC_OP_GETPMCINFO:
3610         {
3611                 int ari;
3612                 struct pmc *pm;
3613                 size_t pmcinfo_size;
3614                 uint32_t cpu, n, npmc;
3615                 struct pmc_owner *po;
3616                 struct pmc_binding pb;
3617                 struct pmc_classdep *pcd;
3618                 struct pmc_info *p, *pmcinfo;
3619                 struct pmc_op_getpmcinfo *gpi;
3620
3621                 PMC_DOWNGRADE_SX();
3622
3623                 gpi = (struct pmc_op_getpmcinfo *) arg;
3624
3625                 if ((error = copyin(&gpi->pm_cpu, &cpu, sizeof(cpu))) != 0)
3626                         break;
3627
3628                 if (cpu >= pmc_cpu_max()) {
3629                         error = EINVAL;
3630                         break;
3631                 }
3632
3633                 if (!pmc_cpu_is_active(cpu)) {
3634                         error = ENXIO;
3635                         break;
3636                 }
3637
3638                 /* switch to CPU 'cpu' */
3639                 pmc_save_cpu_binding(&pb);
3640                 pmc_select_cpu(cpu);
3641
3642                 npmc = md->pmd_npmc;
3643
3644                 pmcinfo_size = npmc * sizeof(struct pmc_info);
3645                 pmcinfo = malloc(pmcinfo_size, M_PMC, M_WAITOK);
3646
3647                 p = pmcinfo;
3648
3649                 for (n = 0; n < md->pmd_npmc; n++, p++) {
3650
3651                         pcd = pmc_ri_to_classdep(md, n, &ari);
3652
3653                         KASSERT(pcd != NULL,
3654                             ("[pmc,%d] null pcd ri=%d", __LINE__, n));
3655
3656                         if ((error = pcd->pcd_describe(cpu, ari, p, &pm)) != 0)
3657                                 break;
3658
3659                         if (PMC_ROW_DISP_IS_STANDALONE(n))
3660                                 p->pm_rowdisp = PMC_DISP_STANDALONE;
3661                         else if (PMC_ROW_DISP_IS_THREAD(n))
3662                                 p->pm_rowdisp = PMC_DISP_THREAD;
3663                         else
3664                                 p->pm_rowdisp = PMC_DISP_FREE;
3665
3666                         p->pm_ownerpid = -1;
3667
3668                         if (pm == NULL) /* no PMC associated */
3669                                 continue;
3670
3671                         po = pm->pm_owner;
3672
3673                         KASSERT(po->po_owner != NULL,
3674                             ("[pmc,%d] pmc_owner had a null proc pointer",
3675                                 __LINE__));
3676
3677                         p->pm_ownerpid = po->po_owner->p_pid;
3678                         p->pm_mode     = PMC_TO_MODE(pm);
3679                         p->pm_event    = pm->pm_event;
3680                         p->pm_flags    = pm->pm_flags;
3681
3682                         if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
3683                                 p->pm_reloadcount =
3684                                     pm->pm_sc.pm_reloadcount;
3685                 }
3686
3687                 pmc_restore_cpu_binding(&pb);
3688
3689                 /* now copy out the PMC info collected */
3690                 if (error == 0)
3691                         error = copyout(pmcinfo, &gpi->pm_pmcs, pmcinfo_size);
3692
3693                 free(pmcinfo, M_PMC);
3694         }
3695         break;
3696
3697
3698         /*
3699          * Set the administrative state of a PMC.  I.e. whether
3700          * the PMC is to be used or not.
3701          */
3702
3703         case PMC_OP_PMCADMIN:
3704         {
3705                 int cpu, ri;
3706                 enum pmc_state request;
3707                 struct pmc_cpu *pc;
3708                 struct pmc_hw *phw;
3709                 struct pmc_op_pmcadmin pma;
3710                 struct pmc_binding pb;
3711
3712                 sx_assert(&pmc_sx, SX_XLOCKED);
3713
3714                 KASSERT(td == curthread,
3715                     ("[pmc,%d] td != curthread", __LINE__));
3716
3717                 error = priv_check(td, PRIV_PMC_MANAGE);
3718                 if (error)
3719                         break;
3720
3721                 if ((error = copyin(arg, &pma, sizeof(pma))) != 0)
3722                         break;
3723
3724                 cpu = pma.pm_cpu;
3725
3726                 if (cpu < 0 || cpu >= (int) pmc_cpu_max()) {
3727                         error = EINVAL;
3728                         break;
3729                 }
3730
3731                 if (!pmc_cpu_is_active(cpu)) {
3732                         error = ENXIO;
3733                         break;
3734                 }
3735
3736                 request = pma.pm_state;
3737
3738                 if (request != PMC_STATE_DISABLED &&
3739                     request != PMC_STATE_FREE) {
3740                         error = EINVAL;
3741                         break;
3742                 }
3743
3744                 ri = pma.pm_pmc; /* pmc id == row index */
3745                 if (ri < 0 || ri >= (int) md->pmd_npmc) {
3746                         error = EINVAL;
3747                         break;
3748                 }
3749
3750                 /*
3751                  * We can't disable a PMC with a row-index allocated
3752                  * for process virtual PMCs.
3753                  */
3754
3755                 if (PMC_ROW_DISP_IS_THREAD(ri) &&
3756                     request == PMC_STATE_DISABLED) {
3757                         error = EBUSY;
3758                         break;
3759                 }
3760
3761                 /*
3762                  * otherwise, this PMC on this CPU is either free or
3763                  * in system-wide mode.
3764                  */
3765
3766                 pmc_save_cpu_binding(&pb);
3767                 pmc_select_cpu(cpu);
3768
3769                 pc  = pmc_pcpu[cpu];
3770                 phw = pc->pc_hwpmcs[ri];
3771
3772                 /*
3773                  * XXX do we need some kind of 'forced' disable?
3774                  */
3775
3776                 if (phw->phw_pmc == NULL) {
3777                         if (request == PMC_STATE_DISABLED &&
3778                             (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED)) {
3779                                 phw->phw_state &= ~PMC_PHW_FLAG_IS_ENABLED;
3780                                 PMC_MARK_ROW_STANDALONE(ri);
3781                         } else if (request == PMC_STATE_FREE &&
3782                             (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0) {
3783                                 phw->phw_state |=  PMC_PHW_FLAG_IS_ENABLED;
3784                                 PMC_UNMARK_ROW_STANDALONE(ri);
3785                         }
3786                         /* other cases are a no-op */
3787                 } else
3788                         error = EBUSY;
3789
3790                 pmc_restore_cpu_binding(&pb);
3791         }
3792         break;
3793
3794
3795         /*
3796          * Allocate a PMC.
3797          */
3798
3799         case PMC_OP_PMCALLOCATE:
3800         {
3801                 int adjri, n;
3802                 u_int cpu;
3803                 uint32_t caps;
3804                 struct pmc *pmc;
3805                 enum pmc_mode mode;
3806                 struct pmc_hw *phw;
3807                 struct pmc_binding pb;
3808                 struct pmc_classdep *pcd;
3809                 struct pmc_op_pmcallocate pa;
3810
3811                 if ((error = copyin(arg, &pa, sizeof(pa))) != 0)
3812                         break;
3813
3814                 caps = pa.pm_caps;
3815                 mode = pa.pm_mode;
3816                 cpu  = pa.pm_cpu;
3817
3818                 if ((mode != PMC_MODE_SS  &&  mode != PMC_MODE_SC  &&
3819                      mode != PMC_MODE_TS  &&  mode != PMC_MODE_TC) ||
3820                     (cpu != (u_int) PMC_CPU_ANY && cpu >= pmc_cpu_max())) {
3821                         error = EINVAL;
3822                         break;
3823                 }
3824
3825                 /*
3826                  * Virtual PMCs should only ask for a default CPU.
3827                  * System mode PMCs need to specify a non-default CPU.
3828                  */
3829
3830                 if ((PMC_IS_VIRTUAL_MODE(mode) && cpu != (u_int) PMC_CPU_ANY) ||
3831                     (PMC_IS_SYSTEM_MODE(mode) && cpu == (u_int) PMC_CPU_ANY)) {
3832                         error = EINVAL;
3833                         break;
3834                 }
3835
3836                 /*
3837                  * Check that an inactive CPU is not being asked for.
3838                  */
3839
3840                 if (PMC_IS_SYSTEM_MODE(mode) && !pmc_cpu_is_active(cpu)) {
3841                         error = ENXIO;
3842                         break;
3843                 }
3844
3845                 /*
3846                  * Refuse an allocation for a system-wide PMC if this
3847                  * process has been jailed, or if this process lacks
3848                  * super-user credentials and the sysctl tunable
3849                  * 'security.bsd.unprivileged_syspmcs' is zero.
3850                  */
3851
3852                 if (PMC_IS_SYSTEM_MODE(mode)) {
3853                         if (jailed(curthread->td_ucred)) {
3854                                 error = EPERM;
3855                                 break;
3856                         }
3857                         if (!pmc_unprivileged_syspmcs) {
3858                                 error = priv_check(curthread,
3859                                     PRIV_PMC_SYSTEM);
3860                                 if (error)
3861                                         break;
3862                         }
3863                 }
3864
3865                 /*
3866                  * Look for valid values for 'pm_flags'
3867                  */
3868
3869                 if ((pa.pm_flags & ~(PMC_F_DESCENDANTS | PMC_F_LOG_PROCCSW |
3870                     PMC_F_LOG_PROCEXIT | PMC_F_CALLCHAIN |
3871                     PMC_F_USERCALLCHAIN)) != 0) {
3872                         error = EINVAL;
3873                         break;
3874                 }
3875
3876                 /* PMC_F_USERCALLCHAIN is only valid with PMC_F_CALLCHAIN */
3877                 if ((pa.pm_flags & (PMC_F_CALLCHAIN | PMC_F_USERCALLCHAIN)) ==
3878                     PMC_F_USERCALLCHAIN) {
3879                         error = EINVAL;
3880                         break;
3881                 }
3882
3883                 /* PMC_F_USERCALLCHAIN is only valid for sampling mode */
3884                 if (pa.pm_flags & PMC_F_USERCALLCHAIN &&
3885                         mode != PMC_MODE_TS && mode != PMC_MODE_SS) {
3886                         error = EINVAL;
3887                         break;
3888                 }
3889
3890                 /* process logging options are not allowed for system PMCs */
3891                 if (PMC_IS_SYSTEM_MODE(mode) && (pa.pm_flags &
3892                     (PMC_F_LOG_PROCCSW | PMC_F_LOG_PROCEXIT))) {
3893                         error = EINVAL;
3894                         break;
3895                 }
3896
3897                 /*
3898                  * All sampling mode PMCs need to be able to interrupt the
3899                  * CPU.
3900                  */
3901                 if (PMC_IS_SAMPLING_MODE(mode))
3902                         caps |= PMC_CAP_INTERRUPT;
3903
3904                 /* A valid class specifier should have been passed in. */
3905                 pcd = pmc_class_to_classdep(pa.pm_class);
3906                 if (pcd == NULL) {
3907                         error = EINVAL;
3908                         break;
3909                 }
3910
3911                 /* The requested PMC capabilities should be feasible. */
3912                 if ((pcd->pcd_caps & caps) != caps) {
3913                         error = EOPNOTSUPP;
3914                         break;
3915                 }
3916
3917                 PMCDBG4(PMC,ALL,2, "event=%d caps=0x%x mode=%d cpu=%d",
3918                     pa.pm_ev, caps, mode, cpu);
3919
3920                 pmc = pmc_allocate_pmc_descriptor();
3921                 pmc->pm_id    = PMC_ID_MAKE_ID(cpu,pa.pm_mode,pa.pm_class,
3922                     PMC_ID_INVALID);
3923                 pmc->pm_event = pa.pm_ev;
3924                 pmc->pm_state = PMC_STATE_FREE;
3925                 pmc->pm_caps  = caps;
3926                 pmc->pm_flags = pa.pm_flags;
3927
3928                 /* switch thread to CPU 'cpu' */
3929                 pmc_save_cpu_binding(&pb);
3930
3931 #define PMC_IS_SHAREABLE_PMC(cpu, n)                            \
3932         (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_state &           \
3933          PMC_PHW_FLAG_IS_SHAREABLE)
3934 #define PMC_IS_UNALLOCATED(cpu, n)                              \
3935         (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_pmc == NULL)
3936
3937                 if (PMC_IS_SYSTEM_MODE(mode)) {
3938                         pmc_select_cpu(cpu);
3939                         for (n = pcd->pcd_ri; n < (int) md->pmd_npmc; n++) {
3940                                 pcd = pmc_ri_to_classdep(md, n, &adjri);
3941                                 if (pmc_can_allocate_row(n, mode) == 0 &&
3942                                     pmc_can_allocate_rowindex(
3943                                             curthread->td_proc, n, cpu) == 0 &&
3944                                     (PMC_IS_UNALLOCATED(cpu, n) ||
3945                                      PMC_IS_SHAREABLE_PMC(cpu, n)) &&
3946                                     pcd->pcd_allocate_pmc(cpu, adjri, pmc,
3947                                         &pa) == 0)
3948                                         break;
3949                         }
3950                 } else {
3951                         /* Process virtual mode */
3952                         for (n = pcd->pcd_ri; n < (int) md->pmd_npmc; n++) {
3953                                 pcd = pmc_ri_to_classdep(md, n, &adjri);
3954                                 if (pmc_can_allocate_row(n, mode) == 0 &&
3955                                     pmc_can_allocate_rowindex(
3956                                             curthread->td_proc, n,
3957                                             PMC_CPU_ANY) == 0 &&
3958                                     pcd->pcd_allocate_pmc(curthread->td_oncpu,
3959                                         adjri, pmc, &pa) == 0)
3960                                         break;
3961                         }
3962                 }
3963
3964 #undef  PMC_IS_UNALLOCATED
3965 #undef  PMC_IS_SHAREABLE_PMC
3966
3967                 pmc_restore_cpu_binding(&pb);
3968
3969                 if (n == (int) md->pmd_npmc) {
3970                         pmc_destroy_pmc_descriptor(pmc);
3971                         pmc = NULL;
3972                         error = EINVAL;
3973                         break;
3974                 }
3975
3976                 /* Fill in the correct value in the ID field */
3977                 pmc->pm_id = PMC_ID_MAKE_ID(cpu,mode,pa.pm_class,n);
3978
3979                 PMCDBG5(PMC,ALL,2, "ev=%d class=%d mode=%d n=%d -> pmcid=%x",
3980                     pmc->pm_event, pa.pm_class, mode, n, pmc->pm_id);
3981
3982                 /* Process mode PMCs with logging enabled need log files */
3983                 if (pmc->pm_flags & (PMC_F_LOG_PROCEXIT | PMC_F_LOG_PROCCSW))
3984                         pmc->pm_flags |= PMC_F_NEEDS_LOGFILE;
3985
3986                 /* All system mode sampling PMCs require a log file */
3987                 if (PMC_IS_SAMPLING_MODE(mode) && PMC_IS_SYSTEM_MODE(mode))
3988                         pmc->pm_flags |= PMC_F_NEEDS_LOGFILE;
3989
3990                 /*
3991                  * Configure global pmc's immediately
3992                  */
3993
3994                 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pmc))) {
3995
3996                         pmc_save_cpu_binding(&pb);
3997                         pmc_select_cpu(cpu);
3998
3999                         phw = pmc_pcpu[cpu]->pc_hwpmcs[n];
4000                         pcd = pmc_ri_to_classdep(md, n, &adjri);
4001
4002                         if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0 ||
4003                             (error = pcd->pcd_config_pmc(cpu, adjri, pmc)) != 0) {
4004                                 (void) pcd->pcd_release_pmc(cpu, adjri, pmc);
4005                                 pmc_destroy_pmc_descriptor(pmc);
4006                                 pmc = NULL;
4007                                 pmc_restore_cpu_binding(&pb);
4008                                 error = EPERM;
4009                                 break;
4010                         }
4011
4012                         pmc_restore_cpu_binding(&pb);
4013                 }
4014
4015                 pmc->pm_state    = PMC_STATE_ALLOCATED;
4016                 pmc->pm_class   = pa.pm_class;
4017
4018                 /*
4019                  * mark row disposition
4020                  */
4021
4022                 if (PMC_IS_SYSTEM_MODE(mode))
4023                         PMC_MARK_ROW_STANDALONE(n);
4024                 else
4025                         PMC_MARK_ROW_THREAD(n);
4026
4027                 /*
4028                  * Register this PMC with the current thread as its owner.
4029                  */
4030
4031                 if ((error =
4032                     pmc_register_owner(curthread->td_proc, pmc)) != 0) {
4033                         pmc_release_pmc_descriptor(pmc);
4034                         pmc_destroy_pmc_descriptor(pmc);
4035                         pmc = NULL;
4036                         break;
4037                 }
4038                 if (mode == PMC_MODE_SS)
4039                         pmc_process_allproc(pmc);
4040
4041                 /*
4042                  * Return the allocated index.
4043                  */
4044
4045                 pa.pm_pmcid = pmc->pm_id;
4046
4047                 error = copyout(&pa, arg, sizeof(pa));
4048         }
4049         break;
4050
4051
4052         /*
4053          * Attach a PMC to a process.
4054          */
4055
4056         case PMC_OP_PMCATTACH:
4057         {
4058                 struct pmc *pm;
4059                 struct proc *p;
4060                 struct pmc_op_pmcattach a;
4061
4062                 sx_assert(&pmc_sx, SX_XLOCKED);
4063
4064                 if ((error = copyin(arg, &a, sizeof(a))) != 0)
4065                         break;
4066
4067                 if (a.pm_pid < 0) {
4068                         error = EINVAL;
4069                         break;
4070                 } else if (a.pm_pid == 0)
4071                         a.pm_pid = td->td_proc->p_pid;
4072
4073                 if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0)
4074                         break;
4075
4076                 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
4077                         error = EINVAL;
4078                         break;
4079                 }
4080
4081                 /* PMCs may be (re)attached only when allocated or stopped */
4082                 if (pm->pm_state == PMC_STATE_RUNNING) {
4083                         error = EBUSY;
4084                         break;
4085                 } else if (pm->pm_state != PMC_STATE_ALLOCATED &&
4086                     pm->pm_state != PMC_STATE_STOPPED) {
4087                         error = EINVAL;
4088                         break;
4089                 }
4090
4091                 /* lookup pid */
4092                 if ((p = pfind(a.pm_pid)) == NULL) {
4093                         error = ESRCH;
4094                         break;
4095                 }
4096
4097                 /*
4098                  * Ignore processes that are working on exiting.
4099                  */
4100                 if (p->p_flag & P_WEXIT) {
4101                         error = ESRCH;
4102                         PROC_UNLOCK(p); /* pfind() returns a locked process */
4103                         break;
4104                 }
4105
4106                 /*
4107                  * we are allowed to attach a PMC to a process if
4108                  * we can debug it.
4109                  */
4110                 error = p_candebug(curthread, p);
4111
4112                 PROC_UNLOCK(p);
4113
4114                 if (error == 0)
4115                         error = pmc_attach_process(p, pm);
4116         }
4117         break;
4118
4119
4120         /*
4121          * Detach an attached PMC from a process.
4122          */
4123
4124         case PMC_OP_PMCDETACH:
4125         {
4126                 struct pmc *pm;
4127                 struct proc *p;
4128                 struct pmc_op_pmcattach a;
4129
4130                 if ((error = copyin(arg, &a, sizeof(a))) != 0)
4131                         break;
4132
4133                 if (a.pm_pid < 0) {
4134                         error = EINVAL;
4135                         break;
4136                 } else if (a.pm_pid == 0)
4137                         a.pm_pid = td->td_proc->p_pid;
4138
4139                 if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0)
4140                         break;
4141
4142                 if ((p = pfind(a.pm_pid)) == NULL) {
4143                         error = ESRCH;
4144                         break;
4145                 }
4146
4147                 /*
4148                  * Treat processes that are in the process of exiting
4149                  * as if they were not present.
4150                  */
4151
4152                 if (p->p_flag & P_WEXIT)
4153                         error = ESRCH;
4154
4155                 PROC_UNLOCK(p); /* pfind() returns a locked process */
4156
4157                 if (error == 0)
4158                         error = pmc_detach_process(p, pm);
4159         }
4160         break;
4161
4162
4163         /*
4164          * Retrieve the MSR number associated with the counter
4165          * 'pmc_id'.  This allows processes to directly use RDPMC
4166          * instructions to read their PMCs, without the overhead of a
4167          * system call.
4168          */
4169
4170         case PMC_OP_PMCGETMSR:
4171         {
4172                 int adjri, ri;
4173                 struct pmc *pm;
4174                 struct pmc_target *pt;
4175                 struct pmc_op_getmsr gm;
4176                 struct pmc_classdep *pcd;
4177
4178                 PMC_DOWNGRADE_SX();
4179
4180                 if ((error = copyin(arg, &gm, sizeof(gm))) != 0)
4181                         break;
4182
4183                 if ((error = pmc_find_pmc(gm.pm_pmcid, &pm)) != 0)
4184                         break;
4185
4186                 /*
4187                  * The allocated PMC has to be a process virtual PMC,
4188                  * i.e., of type MODE_T[CS].  Global PMCs can only be
4189                  * read using the PMCREAD operation since they may be
4190                  * allocated on a different CPU than the one we could
4191                  * be running on at the time of the RDPMC instruction.
4192                  *
4193                  * The GETMSR operation is not allowed for PMCs that
4194                  * are inherited across processes.
4195                  */
4196
4197                 if (!PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) ||
4198                     (pm->pm_flags & PMC_F_DESCENDANTS)) {
4199                         error = EINVAL;
4200                         break;
4201                 }
4202
4203                 /*
4204                  * It only makes sense to use a RDPMC (or its
4205                  * equivalent instruction on non-x86 architectures) on
4206                  * a process that has allocated and attached a PMC to
4207                  * itself.  Conversely the PMC is only allowed to have
4208                  * one process attached to it -- its owner.
4209                  */
4210
4211                 if ((pt = LIST_FIRST(&pm->pm_targets)) == NULL ||
4212                     LIST_NEXT(pt, pt_next) != NULL ||
4213                     pt->pt_process->pp_proc != pm->pm_owner->po_owner) {
4214                         error = EINVAL;
4215                         break;
4216                 }
4217
4218                 ri = PMC_TO_ROWINDEX(pm);
4219                 pcd = pmc_ri_to_classdep(md, ri, &adjri);
4220
4221                 /* PMC class has no 'GETMSR' support */
4222                 if (pcd->pcd_get_msr == NULL) {
4223                         error = ENOSYS;
4224                         break;
4225                 }
4226
4227                 if ((error = (*pcd->pcd_get_msr)(adjri, &gm.pm_msr)) < 0)
4228                         break;
4229
4230                 if ((error = copyout(&gm, arg, sizeof(gm))) < 0)
4231                         break;
4232
4233                 /*
4234                  * Mark our process as using MSRs.  Update machine
4235                  * state using a forced context switch.
4236                  */
4237
4238                 pt->pt_process->pp_flags |= PMC_PP_ENABLE_MSR_ACCESS;
4239                 pmc_force_context_switch();
4240
4241         }
4242         break;
4243
4244         /*
4245          * Release an allocated PMC
4246          */
4247
4248         case PMC_OP_PMCRELEASE:
4249         {
4250                 pmc_id_t pmcid;
4251                 struct pmc *pm;
4252                 struct pmc_owner *po;
4253                 struct pmc_op_simple sp;
4254
4255                 /*
4256                  * Find PMC pointer for the named PMC.
4257                  *
4258                  * Use pmc_release_pmc_descriptor() to switch off the
4259                  * PMC, remove all its target threads, and remove the
4260                  * PMC from its owner's list.
4261                  *
4262                  * Remove the owner record if this is the last PMC
4263                  * owned.
4264                  *
4265                  * Free up space.
4266                  */
4267
4268                 if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
4269                         break;
4270
4271                 pmcid = sp.pm_pmcid;
4272
4273                 if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
4274                         break;
4275
4276                 po = pm->pm_owner;
4277                 pmc_release_pmc_descriptor(pm);
4278                 pmc_maybe_remove_owner(po);
4279                 pmc_destroy_pmc_descriptor(pm);
4280         }
4281         break;
4282
4283
4284         /*
4285          * Read and/or write a PMC.
4286          */
4287
4288         case PMC_OP_PMCRW:
4289         {
4290                 int adjri;
4291                 struct pmc *pm;
4292                 uint32_t cpu, ri;
4293                 pmc_value_t oldvalue;
4294                 struct pmc_binding pb;
4295                 struct pmc_op_pmcrw prw;
4296                 struct pmc_classdep *pcd;
4297                 struct pmc_op_pmcrw *pprw;
4298
4299                 PMC_DOWNGRADE_SX();
4300
4301                 if ((error = copyin(arg, &prw, sizeof(prw))) != 0)
4302                         break;
4303
4304                 ri = 0;
4305                 PMCDBG2(PMC,OPS,1, "rw id=%d flags=0x%x", prw.pm_pmcid,
4306                     prw.pm_flags);
4307
4308                 /* must have at least one flag set */
4309                 if ((prw.pm_flags & (PMC_F_OLDVALUE|PMC_F_NEWVALUE)) == 0) {
4310                         error = EINVAL;
4311                         break;
4312                 }
4313
4314                 /* locate pmc descriptor */
4315                 if ((error = pmc_find_pmc(prw.pm_pmcid, &pm)) != 0)
4316                         break;
4317
4318                 /* Can't read a PMC that hasn't been started. */
4319                 if (pm->pm_state != PMC_STATE_ALLOCATED &&
4320                     pm->pm_state != PMC_STATE_STOPPED &&
4321                     pm->pm_state != PMC_STATE_RUNNING) {
4322                         error = EINVAL;
4323                         break;
4324                 }
4325
4326                 /* writing a new value is allowed only for 'STOPPED' pmcs */
4327                 if (pm->pm_state == PMC_STATE_RUNNING &&
4328                     (prw.pm_flags & PMC_F_NEWVALUE)) {
4329                         error = EBUSY;
4330                         break;
4331                 }
4332
4333                 if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) {
4334
4335                         /*
4336                          * If this PMC is attached to its owner (i.e.,
4337                          * the process requesting this operation) and
4338                          * is running, then attempt to get an
4339                          * upto-date reading from hardware for a READ.
4340                          * Writes are only allowed when the PMC is
4341                          * stopped, so only update the saved value
4342                          * field.
4343                          *
4344                          * If the PMC is not running, or is not
4345                          * attached to its owner, read/write to the
4346                          * savedvalue field.
4347                          */
4348
4349                         ri = PMC_TO_ROWINDEX(pm);
4350                         pcd = pmc_ri_to_classdep(md, ri, &adjri);
4351
4352                         mtx_pool_lock_spin(pmc_mtxpool, pm);
4353                         cpu = curthread->td_oncpu;
4354
4355                         if (prw.pm_flags & PMC_F_OLDVALUE) {
4356                                 if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) &&
4357                                     (pm->pm_state == PMC_STATE_RUNNING))
4358                                         error = (*pcd->pcd_read_pmc)(cpu, adjri,
4359                                             &oldvalue);
4360                                 else
4361                                         oldvalue = pm->pm_gv.pm_savedvalue;
4362                         }
4363                         if (prw.pm_flags & PMC_F_NEWVALUE)
4364                                 pm->pm_gv.pm_savedvalue = prw.pm_value;
4365
4366                         mtx_pool_unlock_spin(pmc_mtxpool, pm);
4367
4368                 } else { /* System mode PMCs */
4369                         cpu = PMC_TO_CPU(pm);
4370                         ri  = PMC_TO_ROWINDEX(pm);
4371                         pcd = pmc_ri_to_classdep(md, ri, &adjri);
4372
4373                         if (!pmc_cpu_is_active(cpu)) {
4374                                 error = ENXIO;
4375                                 break;
4376                         }
4377
4378                         /* move this thread to CPU 'cpu' */
4379                         pmc_save_cpu_binding(&pb);
4380                         pmc_select_cpu(cpu);
4381
4382                         critical_enter();
4383                         /* save old value */
4384                         if (prw.pm_flags & PMC_F_OLDVALUE)
4385                                 if ((error = (*pcd->pcd_read_pmc)(cpu, adjri,
4386                                          &oldvalue)))
4387                                         goto error;
4388                         /* write out new value */
4389                         if (prw.pm_flags & PMC_F_NEWVALUE)
4390                                 error = (*pcd->pcd_write_pmc)(cpu, adjri,
4391                                     prw.pm_value);
4392                 error:
4393                         critical_exit();
4394                         pmc_restore_cpu_binding(&pb);
4395                         if (error)
4396                                 break;
4397                 }
4398
4399                 pprw = (struct pmc_op_pmcrw *) arg;
4400
4401 #ifdef  HWPMC_DEBUG
4402                 if (prw.pm_flags & PMC_F_NEWVALUE)
4403                         PMCDBG3(PMC,OPS,2, "rw id=%d new %jx -> old %jx",
4404                             ri, prw.pm_value, oldvalue);
4405                 else if (prw.pm_flags & PMC_F_OLDVALUE)
4406                         PMCDBG2(PMC,OPS,2, "rw id=%d -> old %jx", ri, oldvalue);
4407 #endif
4408
4409                 /* return old value if requested */
4410                 if (prw.pm_flags & PMC_F_OLDVALUE)
4411                         if ((error = copyout(&oldvalue, &pprw->pm_value,
4412                                  sizeof(prw.pm_value))))
4413                                 break;
4414
4415         }
4416         break;
4417
4418
4419         /*
4420          * Set the sampling rate for a sampling mode PMC and the
4421          * initial count for a counting mode PMC.
4422          */
4423
4424         case PMC_OP_PMCSETCOUNT:
4425         {
4426                 struct pmc *pm;
4427                 struct pmc_op_pmcsetcount sc;
4428
4429                 PMC_DOWNGRADE_SX();
4430
4431                 if ((error = copyin(arg, &sc, sizeof(sc))) != 0)
4432                         break;
4433
4434                 if ((error = pmc_find_pmc(sc.pm_pmcid, &pm)) != 0)
4435                         break;
4436
4437                 if (pm->pm_state == PMC_STATE_RUNNING) {
4438                         error = EBUSY;
4439                         break;
4440                 }
4441
4442                 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
4443                         pm->pm_sc.pm_reloadcount = sc.pm_count;
4444                 else
4445                         pm->pm_sc.pm_initial = sc.pm_count;
4446         }
4447         break;
4448
4449
4450         /*
4451          * Start a PMC.
4452          */
4453
4454         case PMC_OP_PMCSTART:
4455         {
4456                 pmc_id_t pmcid;
4457                 struct pmc *pm;
4458                 struct pmc_op_simple sp;
4459
4460                 sx_assert(&pmc_sx, SX_XLOCKED);
4461
4462                 if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
4463                         break;
4464
4465                 pmcid = sp.pm_pmcid;
4466
4467                 if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
4468                         break;
4469
4470                 KASSERT(pmcid == pm->pm_id,
4471                     ("[pmc,%d] pmcid %x != id %x", __LINE__,
4472                         pm->pm_id, pmcid));
4473
4474                 if (pm->pm_state == PMC_STATE_RUNNING) /* already running */
4475                         break;
4476                 else if (pm->pm_state != PMC_STATE_STOPPED &&
4477                     pm->pm_state != PMC_STATE_ALLOCATED) {
4478                         error = EINVAL;
4479                         break;
4480                 }
4481
4482                 error = pmc_start(pm);
4483         }
4484         break;
4485
4486
4487         /*
4488          * Stop a PMC.
4489          */
4490
4491         case PMC_OP_PMCSTOP:
4492         {
4493                 pmc_id_t pmcid;
4494                 struct pmc *pm;
4495                 struct pmc_op_simple sp;
4496
4497                 PMC_DOWNGRADE_SX();
4498
4499                 if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
4500                         break;
4501
4502                 pmcid = sp.pm_pmcid;
4503
4504                 /*
4505                  * Mark the PMC as inactive and invoke the MD stop
4506                  * routines if needed.
4507                  */
4508
4509                 if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
4510                         break;
4511
4512                 KASSERT(pmcid == pm->pm_id,
4513                     ("[pmc,%d] pmc id %x != pmcid %x", __LINE__,
4514                         pm->pm_id, pmcid));
4515
4516                 if (pm->pm_state == PMC_STATE_STOPPED) /* already stopped */
4517                         break;
4518                 else if (pm->pm_state != PMC_STATE_RUNNING) {
4519                         error = EINVAL;
4520                         break;
4521                 }
4522
4523                 error = pmc_stop(pm);
4524         }
4525         break;
4526
4527
4528         /*
4529          * Write a user supplied value to the log file.
4530          */
4531
4532         case PMC_OP_WRITELOG:
4533         {
4534                 struct pmc_op_writelog wl;
4535                 struct pmc_owner *po;
4536
4537                 PMC_DOWNGRADE_SX();
4538
4539                 if ((error = copyin(arg, &wl, sizeof(wl))) != 0)
4540                         break;
4541
4542                 if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
4543                         error = EINVAL;
4544                         break;
4545                 }
4546
4547                 if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) {
4548                         error = EINVAL;
4549                         break;
4550                 }
4551
4552                 error = pmclog_process_userlog(po, &wl);
4553         }
4554         break;
4555
4556
4557         default:
4558                 error = EINVAL;
4559                 break;
4560         }
4561
4562         if (is_sx_downgraded)
4563                 sx_sunlock(&pmc_sx);
4564         else
4565                 sx_xunlock(&pmc_sx);
4566 done_syscall:
4567         if (error)
4568                 counter_u64_add(pmc_stats.pm_syscall_errors, 1);
4569
4570         return (error);
4571 }
4572
4573 /*
4574  * Helper functions
4575  */
4576
4577
4578 /*
4579  * Mark the thread as needing callchain capture and post an AST.  The
4580  * actual callchain capture will be done in a context where it is safe
4581  * to take page faults.
4582  */
4583
4584 static void
4585 pmc_post_callchain_callback(void)
4586 {
4587         struct thread *td;
4588
4589         td = curthread;
4590
4591         /*
4592          * If there is multiple PMCs for the same interrupt ignore new post
4593          */
4594         if (td->td_pflags & TDP_CALLCHAIN)
4595                 return;
4596
4597         /*
4598          * Mark this thread as needing callchain capture.
4599          * `td->td_pflags' will be safe to touch because this thread
4600          * was in user space when it was interrupted.
4601          */
4602         td->td_pflags |= TDP_CALLCHAIN;
4603
4604         /*
4605          * Don't let this thread migrate between CPUs until callchain
4606          * capture completes.
4607          */
4608         sched_pin();
4609
4610         return;
4611 }
4612
4613 /*
4614  * Find a free slot in the per-cpu array of samples and capture the
4615  * current callchain there.  If a sample was successfully added, a bit
4616  * is set in mask 'pmc_cpumask' denoting that the DO_SAMPLES hook
4617  * needs to be invoked from the clock handler.
4618  *
4619  * This function is meant to be called from an NMI handler.  It cannot
4620  * use any of the locking primitives supplied by the OS.
4621  */
4622
4623 static int
4624 pmc_add_sample(int cpu, int ring, struct pmc *pm, struct trapframe *tf,
4625     int inuserspace)
4626 {
4627         int error, callchaindepth;
4628         struct thread *td;
4629         struct pmc_sample *ps;
4630         struct pmc_samplebuffer *psb;
4631
4632         error = 0;
4633
4634         /*
4635          * Allocate space for a sample buffer.
4636          */
4637         psb = pmc_pcpu[cpu]->pc_sb[ring];
4638
4639         ps = psb->ps_write;
4640         if (ps->ps_nsamples == PMC_SAMPLE_INUSE) {
4641                 counter_u64_add(ps->ps_pmc->pm_runcount, -1);
4642                 counter_u64_add(pmc_stats.pm_overwrites, 1);
4643                 ps->ps_nsamples = 0;
4644         } else if (ps->ps_nsamples) {   /* in use, reader hasn't caught up */
4645                 pm->pm_pcpu_state[cpu].pps_stalled = 1;
4646                 counter_u64_add(pmc_stats.pm_intr_bufferfull, 1);
4647                 PMCDBG6(SAM,INT,1,"(spc) cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d",
4648                     cpu, pm, (void *) tf, inuserspace,
4649                     (int) (psb->ps_write - psb->ps_samples),
4650                     (int) (psb->ps_read - psb->ps_samples));
4651                 callchaindepth = 1;
4652                 error = ENOMEM;
4653                 goto done;
4654         }
4655
4656         /* Fill in entry. */
4657         PMCDBG6(SAM,INT,1,"cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d", cpu, pm,
4658             (void *) tf, inuserspace,
4659             (int) (psb->ps_write - psb->ps_samples),
4660             (int) (psb->ps_read - psb->ps_samples));
4661
4662         KASSERT(counter_u64_fetch(pm->pm_runcount) >= 0,
4663             ("[pmc,%d] pm=%p runcount %ld", __LINE__, (void *) pm,
4664                  (unsigned long)counter_u64_fetch(pm->pm_runcount)));
4665
4666         counter_u64_add(pm->pm_runcount, 1);    /* hold onto PMC */
4667
4668         td = curthread;
4669         ps->ps_pmc = pm;
4670         ps->ps_td = td;
4671         ps->ps_pid = td->td_proc->p_pid;
4672         ps->ps_tid = td->td_tid;
4673         ps->ps_cpu = cpu;
4674         ps->ps_flags = inuserspace ? PMC_CC_F_USERSPACE : 0;
4675
4676         callchaindepth = (pm->pm_flags & PMC_F_CALLCHAIN) ?
4677             pmc_callchaindepth : 1;
4678
4679         if (callchaindepth == 1)
4680                 ps->ps_pc[0] = PMC_TRAPFRAME_TO_PC(tf);
4681         else {
4682                 /*
4683                  * Kernel stack traversals can be done immediately,
4684                  * while we defer to an AST for user space traversals.
4685                  */
4686                 if (!inuserspace) {
4687                         callchaindepth =
4688                             pmc_save_kernel_callchain(ps->ps_pc,
4689                                 callchaindepth, tf);
4690                 } else {
4691                         pmc_post_callchain_callback();
4692                         callchaindepth = PMC_SAMPLE_INUSE;
4693                 }
4694         }
4695
4696         ps->ps_nsamples = callchaindepth;       /* mark entry as in use */
4697         if (ring == PMC_UR) {
4698                 ps->ps_nsamples_actual = callchaindepth;        /* mark entry as in use */
4699                 ps->ps_nsamples = PMC_SAMPLE_INUSE;
4700         } else
4701                 ps->ps_nsamples = callchaindepth;       /* mark entry as in use */
4702         /* increment write pointer, modulo ring buffer size */
4703         ps++;
4704         if (ps == psb->ps_fence)
4705                 psb->ps_write = psb->ps_samples;
4706         else
4707                 psb->ps_write = ps;
4708
4709  done:
4710         /* mark CPU as needing processing */
4711         if (callchaindepth != PMC_SAMPLE_INUSE)
4712                 DPCPU_SET(pmc_sampled, 1);
4713
4714         return (error);
4715 }
4716
4717 /*
4718  * Interrupt processing.
4719  *
4720  * This function is meant to be called from an NMI handler.  It cannot
4721  * use any of the locking primitives supplied by the OS.
4722  */
4723
4724 int
4725 pmc_process_interrupt(int cpu, int ring, struct pmc *pm, struct trapframe *tf,
4726     int inuserspace)
4727 {
4728         struct thread *td;
4729
4730         td = curthread;
4731         if ((pm->pm_flags & PMC_F_USERCALLCHAIN) &&
4732            (td->td_proc->p_flag & P_KPROC) == 0 &&
4733            !inuserspace) {
4734                 atomic_add_int(&curthread->td_pmcpend, 1);
4735                 return (pmc_add_sample(cpu, PMC_UR, pm, tf, 0));
4736         }
4737         return (pmc_add_sample(cpu, ring, pm, tf, inuserspace));
4738 }
4739
4740 /*
4741  * Capture a user call chain.  This function will be called from ast()
4742  * before control returns to userland and before the process gets
4743  * rescheduled.
4744  */
4745
4746 static void
4747 pmc_capture_user_callchain(int cpu, int ring, struct trapframe *tf)
4748 {
4749         struct pmc *pm;
4750         struct thread *td;
4751         struct pmc_sample *ps, *ps_end;
4752         struct pmc_samplebuffer *psb;
4753         int nsamples, nrecords, pass;
4754 #ifdef  INVARIANTS
4755         int ncallchains;
4756         int nfree;
4757 #endif
4758
4759         psb = pmc_pcpu[cpu]->pc_sb[ring];
4760         td = curthread;
4761
4762         KASSERT(td->td_pflags & TDP_CALLCHAIN,
4763             ("[pmc,%d] Retrieving callchain for thread that doesn't want it",
4764                 __LINE__));
4765
4766 #ifdef  INVARIANTS
4767         ncallchains = 0;
4768         nfree = 0;
4769 #endif
4770         nrecords = INT_MAX;
4771         pass = 0;
4772  restart:
4773         if (ring == PMC_UR)
4774                 nrecords = atomic_readandclear_32(&td->td_pmcpend);
4775
4776         /*
4777          * Iterate through all deferred callchain requests.
4778          * Walk from the current read pointer to the current
4779          * write pointer.
4780          */
4781
4782         ps = psb->ps_read;
4783         ps_end = psb->ps_write;
4784         do {
4785 #ifdef  INVARIANTS
4786                 if (ps->ps_nsamples == PMC_SAMPLE_FREE) {
4787                         nfree++;
4788                         goto next;
4789                 }
4790
4791                 if ((ps->ps_pmc == NULL) ||
4792                     (ps->ps_pmc->pm_state != PMC_STATE_RUNNING))
4793                         nfree++;
4794 #endif
4795                 if (ps->ps_nsamples != PMC_SAMPLE_INUSE)
4796                         goto next;
4797                 if (ps->ps_td != td)
4798                         goto next;
4799
4800                 KASSERT(ps->ps_cpu == cpu,
4801                     ("[pmc,%d] cpu mismatch ps_cpu=%d pcpu=%d", __LINE__,
4802                         ps->ps_cpu, PCPU_GET(cpuid)));
4803
4804                 pm = ps->ps_pmc;
4805
4806                 KASSERT(pm->pm_flags & PMC_F_CALLCHAIN,
4807                     ("[pmc,%d] Retrieving callchain for PMC that doesn't "
4808                         "want it", __LINE__));
4809
4810                 KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
4811                     ("[pmc,%d] runcount %ld", __LINE__, (unsigned long)counter_u64_fetch(pm->pm_runcount)));
4812
4813                 if (ring == PMC_UR) {
4814                         nsamples = ps->ps_nsamples_actual;
4815                         counter_u64_add(pmc_stats.pm_merges, 1);
4816                 } else
4817                         nsamples = 0;
4818
4819                 /*
4820                  * Retrieve the callchain and mark the sample buffer
4821                  * as 'processable' by the timer tick sweep code.
4822                  */
4823
4824 #ifdef INVARIANTS
4825                 ncallchains++;
4826 #endif
4827
4828                 if (__predict_true(nsamples < pmc_callchaindepth - 1))
4829                         nsamples += pmc_save_user_callchain(ps->ps_pc + nsamples,
4830                        pmc_callchaindepth - nsamples - 1, tf);
4831                 wmb();
4832                 ps->ps_nsamples = nsamples;
4833                 if (nrecords-- == 1)
4834                         break;
4835 next:
4836                 /* increment the pointer, modulo sample ring size */
4837                 if (++ps == psb->ps_fence)
4838                         ps = psb->ps_samples;
4839         } while (ps != ps_end);
4840         if (__predict_false(ring == PMC_UR && td->td_pmcpend)) {
4841                 if (pass == 0) {
4842                         pass = 1;
4843                         goto restart;
4844                 }
4845                 /* only collect samples for this part once */
4846                 td->td_pmcpend = 0;
4847         }
4848
4849 #ifdef INVARIANTS
4850         if (ring == PMC_HR)
4851                 KASSERT(ncallchains > 0 || nfree > 0,
4852                     ("[pmc,%d] cpu %d didn't find a sample to collect", __LINE__,
4853                             cpu));
4854 #endif
4855
4856         /* mark CPU as needing processing */
4857         DPCPU_SET(pmc_sampled, 1);
4858 }
4859
4860
4861 static void
4862 pmc_flush_ring(int cpu, int ring)
4863 {
4864         struct pmc *pm;
4865         struct pmc_sample *ps;
4866         struct pmc_samplebuffer *psb;
4867         int n;
4868
4869         psb = pmc_pcpu[cpu]->pc_sb[ring];
4870
4871         for (n = 0; n < pmc_nsamples; n++) { /* bound on #iterations */
4872
4873                 ps = psb->ps_read;
4874                 if (ps->ps_nsamples == PMC_SAMPLE_FREE)
4875                         goto next;
4876                 pm = ps->ps_pmc;
4877                 counter_u64_add(pm->pm_runcount, -1);
4878                 ps->ps_nsamples = PMC_SAMPLE_FREE;
4879                 /* increment read pointer, modulo sample size */
4880         next:
4881                 if (++ps == psb->ps_fence)
4882                         psb->ps_read = psb->ps_samples;
4883                 else
4884                         psb->ps_read = ps;
4885         }
4886 }
4887
4888 void
4889 pmc_flush_samples(int cpu)
4890 {
4891         int n;
4892
4893         for (n = 0; n < PMC_NUM_SR; n++)
4894                 pmc_flush_ring(cpu, n);
4895 }
4896
4897
4898 /*
4899  * Process saved PC samples.
4900  */
4901
4902 static void
4903 pmc_process_samples(int cpu, int ring)
4904 {
4905         struct pmc *pm;
4906         int adjri, n;
4907         struct thread *td;
4908         struct pmc_owner *po;
4909         struct pmc_sample *ps;
4910         struct pmc_classdep *pcd;
4911         struct pmc_samplebuffer *psb;
4912
4913         KASSERT(PCPU_GET(cpuid) == cpu,
4914             ("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__,
4915                 PCPU_GET(cpuid), cpu));
4916
4917         psb = pmc_pcpu[cpu]->pc_sb[ring];
4918
4919         for (n = 0; n < pmc_nsamples; n++) { /* bound on #iterations */
4920
4921                 ps = psb->ps_read;
4922                 if (ps->ps_nsamples == PMC_SAMPLE_FREE)
4923                         break;
4924
4925                 pm = ps->ps_pmc;
4926
4927                 KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
4928                     ("[pmc,%d] pm=%p runcount %ld", __LINE__, (void *) pm,
4929                          (unsigned long)counter_u64_fetch(pm->pm_runcount)));
4930
4931                 po = pm->pm_owner;
4932
4933                 KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
4934                     ("[pmc,%d] pmc=%p non-sampling mode=%d", __LINE__,
4935                         pm, PMC_TO_MODE(pm)));
4936
4937                 /* Ignore PMCs that have been switched off */
4938                 if (pm->pm_state != PMC_STATE_RUNNING)
4939                         goto entrydone;
4940
4941                 /* If there is a pending AST wait for completion */
4942                 if (ps->ps_nsamples == PMC_SAMPLE_INUSE) {
4943                         /* Need a rescan at a later time. */
4944                         DPCPU_SET(pmc_sampled, 1);
4945                         break;
4946                 }
4947
4948                 PMCDBG6(SAM,OPS,1,"cpu=%d pm=%p n=%d fl=%x wr=%d rd=%d", cpu,
4949                     pm, ps->ps_nsamples, ps->ps_flags,
4950                     (int) (psb->ps_write - psb->ps_samples),
4951                     (int) (psb->ps_read - psb->ps_samples));
4952
4953                 /*
4954                  * If this is a process-mode PMC that is attached to
4955                  * its owner, and if the PC is in user mode, update
4956                  * profiling statistics like timer-based profiling
4957                  * would have done.
4958                  *
4959                  * Otherwise, this is either a sampling-mode PMC that
4960                  * is attached to a different process than its owner,
4961                  * or a system-wide sampling PMC. Dispatch a log
4962                  * entry to the PMC's owner process.
4963                  */
4964                 if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) {
4965                         if (ps->ps_flags & PMC_CC_F_USERSPACE) {
4966                                 td = FIRST_THREAD_IN_PROC(po->po_owner);
4967                                 addupc_intr(td, ps->ps_pc[0], 1);
4968                         }
4969                 } else
4970                         pmclog_process_callchain(pm, ps);
4971
4972         entrydone:
4973                 ps->ps_nsamples = 0; /* mark entry as free */
4974                 counter_u64_add(pm->pm_runcount, -1);
4975
4976                 /* increment read pointer, modulo sample size */
4977                 if (++ps == psb->ps_fence)
4978                         psb->ps_read = psb->ps_samples;
4979                 else
4980                         psb->ps_read = ps;
4981         }
4982
4983         counter_u64_add(pmc_stats.pm_log_sweeps, 1);
4984
4985         /* Do not re-enable stalled PMCs if we failed to process any samples */
4986         if (n == 0)
4987                 return;
4988
4989         /*
4990          * Restart any stalled sampling PMCs on this CPU.
4991          *
4992          * If the NMI handler sets the pm_stalled field of a PMC after
4993          * the check below, we'll end up processing the stalled PMC at
4994          * the next hardclock tick.
4995          */
4996         for (n = 0; n < md->pmd_npmc; n++) {
4997                 pcd = pmc_ri_to_classdep(md, n, &adjri);
4998                 KASSERT(pcd != NULL,
4999                     ("[pmc,%d] null pcd ri=%d", __LINE__, n));
5000                 (void) (*pcd->pcd_get_config)(cpu,adjri,&pm);
5001
5002                 if (pm == NULL ||                        /* !cfg'ed */
5003                     pm->pm_state != PMC_STATE_RUNNING || /* !active */
5004                     !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || /* !sampling */
5005                         !pm->pm_pcpu_state[cpu].pps_cpustate  || /* !desired */
5006                     !pm->pm_pcpu_state[cpu].pps_stalled) /* !stalled */
5007                         continue;
5008
5009                 pm->pm_pcpu_state[cpu].pps_stalled = 0;
5010                 (*pcd->pcd_start_pmc)(cpu, adjri);
5011         }
5012 }
5013
5014 /*
5015  * Event handlers.
5016  */
5017
5018 /*
5019  * Handle a process exit.
5020  *
5021  * Remove this process from all hash tables.  If this process
5022  * owned any PMCs, turn off those PMCs and deallocate them,
5023  * removing any associations with target processes.
5024  *
5025  * This function will be called by the last 'thread' of a
5026  * process.
5027  *
5028  * XXX This eventhandler gets called early in the exit process.
5029  * Consider using a 'hook' invocation from thread_exit() or equivalent
5030  * spot.  Another negative is that kse_exit doesn't seem to call
5031  * exit1() [??].
5032  *
5033  */
5034
5035 static void
5036 pmc_process_exit(void *arg __unused, struct proc *p)
5037 {
5038         struct pmc *pm;
5039         int adjri, cpu;
5040         unsigned int ri;
5041         int is_using_hwpmcs;
5042         struct pmc_owner *po;
5043         struct pmc_process *pp;
5044         struct pmc_classdep *pcd;
5045         pmc_value_t newvalue, tmp;
5046
5047         PROC_LOCK(p);
5048         is_using_hwpmcs = p->p_flag & P_HWPMC;
5049         PROC_UNLOCK(p);
5050
5051         /*
5052          * Log a sysexit event to all SS PMC owners.
5053          */
5054         epoch_enter_preempt(global_epoch_preempt);
5055         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5056             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
5057                     pmclog_process_sysexit(po, p->p_pid);
5058         epoch_exit_preempt(global_epoch_preempt);
5059
5060         if (!is_using_hwpmcs)
5061                 return;
5062
5063         PMC_GET_SX_XLOCK();
5064         PMCDBG3(PRC,EXT,1,"process-exit proc=%p (%d, %s)", p, p->p_pid,
5065             p->p_comm);
5066
5067         /*
5068          * Since this code is invoked by the last thread in an exiting
5069          * process, we would have context switched IN at some prior
5070          * point.  However, with PREEMPTION, kernel mode context
5071          * switches may happen any time, so we want to disable a
5072          * context switch OUT till we get any PMCs targeting this
5073          * process off the hardware.
5074          *
5075          * We also need to atomically remove this process'
5076          * entry from our target process hash table, using
5077          * PMC_FLAG_REMOVE.
5078          */
5079         PMCDBG3(PRC,EXT,1, "process-exit proc=%p (%d, %s)", p, p->p_pid,
5080             p->p_comm);
5081
5082         critical_enter(); /* no preemption */
5083
5084         cpu = curthread->td_oncpu;
5085
5086         if ((pp = pmc_find_process_descriptor(p,
5087                  PMC_FLAG_REMOVE)) != NULL) {
5088
5089                 PMCDBG2(PRC,EXT,2,
5090                     "process-exit proc=%p pmc-process=%p", p, pp);
5091
5092                 /*
5093                  * The exiting process could the target of
5094                  * some PMCs which will be running on
5095                  * currently executing CPU.
5096                  *
5097                  * We need to turn these PMCs off like we
5098                  * would do at context switch OUT time.
5099                  */
5100                 for (ri = 0; ri < md->pmd_npmc; ri++) {
5101
5102                         /*
5103                          * Pick up the pmc pointer from hardware
5104                          * state similar to the CSW_OUT code.
5105                          */
5106                         pm = NULL;
5107
5108                         pcd = pmc_ri_to_classdep(md, ri, &adjri);
5109
5110                         (void) (*pcd->pcd_get_config)(cpu, adjri, &pm);
5111
5112                         PMCDBG2(PRC,EXT,2, "ri=%d pm=%p", ri, pm);
5113
5114                         if (pm == NULL ||
5115                             !PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
5116                                 continue;
5117
5118                         PMCDBG4(PRC,EXT,2, "ppmcs[%d]=%p pm=%p "
5119                             "state=%d", ri, pp->pp_pmcs[ri].pp_pmc,
5120                             pm, pm->pm_state);
5121
5122                         KASSERT(PMC_TO_ROWINDEX(pm) == ri,
5123                             ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
5124                                 __LINE__, PMC_TO_ROWINDEX(pm), ri));
5125
5126                         KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
5127                             ("[pmc,%d] pm %p != pp_pmcs[%d] %p",
5128                                 __LINE__, pm, ri, pp->pp_pmcs[ri].pp_pmc));
5129
5130                         KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
5131                             ("[pmc,%d] bad runcount ri %d rc %ld",
5132                                  __LINE__, ri, (unsigned long)counter_u64_fetch(pm->pm_runcount)));
5133
5134                         /*
5135                          * Change desired state, and then stop if not
5136                          * stalled. This two-step dance should avoid
5137                          * race conditions where an interrupt re-enables
5138                          * the PMC after this code has already checked
5139                          * the pm_stalled flag.
5140                          */
5141                         if (pm->pm_pcpu_state[cpu].pps_cpustate) {
5142                                 pm->pm_pcpu_state[cpu].pps_cpustate = 0;
5143                                 if (!pm->pm_pcpu_state[cpu].pps_stalled) {
5144                                         (void) pcd->pcd_stop_pmc(cpu, adjri);
5145
5146                                         if (PMC_TO_MODE(pm) == PMC_MODE_TC) {
5147                                                 pcd->pcd_read_pmc(cpu, adjri,
5148                                                     &newvalue);
5149                                                 tmp = newvalue -
5150                                                     PMC_PCPU_SAVED(cpu,ri);
5151
5152                                                 mtx_pool_lock_spin(pmc_mtxpool,
5153                                                     pm);
5154                                                 pm->pm_gv.pm_savedvalue += tmp;
5155                                                 pp->pp_pmcs[ri].pp_pmcval +=
5156                                                     tmp;
5157                                                 mtx_pool_unlock_spin(
5158                                                     pmc_mtxpool, pm);
5159                                         }
5160                                 }
5161                         }
5162
5163                         counter_u64_add(pm->pm_runcount, -1);
5164
5165                         KASSERT((int) counter_u64_fetch(pm->pm_runcount) >= 0,
5166                             ("[pmc,%d] runcount is %d", __LINE__, ri));
5167
5168                         (void) pcd->pcd_config_pmc(cpu, adjri, NULL);
5169                 }
5170
5171                 /*
5172                  * Inform the MD layer of this pseudo "context switch
5173                  * out"
5174                  */
5175                 (void) md->pmd_switch_out(pmc_pcpu[cpu], pp);
5176
5177                 critical_exit(); /* ok to be pre-empted now */
5178
5179                 /*
5180                  * Unlink this process from the PMCs that are
5181                  * targeting it.  This will send a signal to
5182                  * all PMC owner's whose PMCs are orphaned.
5183                  *
5184                  * Log PMC value at exit time if requested.
5185                  */
5186                 for (ri = 0; ri < md->pmd_npmc; ri++)
5187                         if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) {
5188                                 if (pm->pm_flags & PMC_F_NEEDS_LOGFILE &&
5189                                     PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm)))
5190                                         pmclog_process_procexit(pm, pp);
5191                                 pmc_unlink_target_process(pm, pp);
5192                         }
5193                 free(pp, M_PMC);
5194
5195         } else
5196                 critical_exit(); /* pp == NULL */
5197
5198
5199         /*
5200          * If the process owned PMCs, free them up and free up
5201          * memory.
5202          */
5203         if ((po = pmc_find_owner_descriptor(p)) != NULL) {
5204                 pmc_remove_owner(po);
5205                 pmc_destroy_owner_descriptor(po);
5206         }
5207
5208         sx_xunlock(&pmc_sx);
5209 }
5210
5211 /*
5212  * Handle a process fork.
5213  *
5214  * If the parent process 'p1' is under HWPMC monitoring, then copy
5215  * over any attached PMCs that have 'do_descendants' semantics.
5216  */
5217
5218 static void
5219 pmc_process_fork(void *arg __unused, struct proc *p1, struct proc *newproc,
5220     int flags)
5221 {
5222         int is_using_hwpmcs;
5223         unsigned int ri;
5224         uint32_t do_descendants;
5225         struct pmc *pm;
5226         struct pmc_owner *po;
5227         struct pmc_process *ppnew, *ppold;
5228
5229         (void) flags;           /* unused parameter */
5230
5231         PROC_LOCK(p1);
5232         is_using_hwpmcs = p1->p_flag & P_HWPMC;
5233         PROC_UNLOCK(p1);
5234
5235         /*
5236          * If there are system-wide sampling PMCs active, we need to
5237          * log all fork events to their owner's logs.
5238          */
5239         epoch_enter_preempt(global_epoch_preempt);
5240         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5241             if (po->po_flags & PMC_PO_OWNS_LOGFILE) {
5242                     pmclog_process_procfork(po, p1->p_pid, newproc->p_pid);
5243                         pmclog_process_proccreate(po, newproc, 1);
5244                 }
5245         epoch_exit_preempt(global_epoch_preempt);
5246
5247         if (!is_using_hwpmcs)
5248                 return;
5249
5250         PMC_GET_SX_XLOCK();
5251         PMCDBG4(PMC,FRK,1, "process-fork proc=%p (%d, %s) -> %p", p1,
5252             p1->p_pid, p1->p_comm, newproc);
5253
5254         /*
5255          * If the parent process (curthread->td_proc) is a
5256          * target of any PMCs, look for PMCs that are to be
5257          * inherited, and link these into the new process
5258          * descriptor.
5259          */
5260         if ((ppold = pmc_find_process_descriptor(curthread->td_proc,
5261                  PMC_FLAG_NONE)) == NULL)
5262                 goto done;              /* nothing to do */
5263
5264         do_descendants = 0;
5265         for (ri = 0; ri < md->pmd_npmc; ri++)
5266                 if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL)
5267                         do_descendants |= pm->pm_flags & PMC_F_DESCENDANTS;
5268         if (do_descendants == 0) /* nothing to do */
5269                 goto done;
5270
5271         /*
5272          * Now mark the new process as being tracked by this driver.
5273          */
5274         PROC_LOCK(newproc);
5275         newproc->p_flag |= P_HWPMC;
5276         PROC_UNLOCK(newproc);
5277
5278         /* allocate a descriptor for the new process  */
5279         if ((ppnew = pmc_find_process_descriptor(newproc,
5280                  PMC_FLAG_ALLOCATE)) == NULL)
5281                 goto done;
5282
5283         /*
5284          * Run through all PMCs that were targeting the old process
5285          * and which specified F_DESCENDANTS and attach them to the
5286          * new process.
5287          *
5288          * Log the fork event to all owners of PMCs attached to this
5289          * process, if not already logged.
5290          */
5291         for (ri = 0; ri < md->pmd_npmc; ri++)
5292                 if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL &&
5293                     (pm->pm_flags & PMC_F_DESCENDANTS)) {
5294                         pmc_link_target_process(pm, ppnew);
5295                         po = pm->pm_owner;
5296                         if (po->po_sscount == 0 &&
5297                             po->po_flags & PMC_PO_OWNS_LOGFILE)
5298                                 pmclog_process_procfork(po, p1->p_pid,
5299                                     newproc->p_pid);
5300                 }
5301
5302  done:
5303         sx_xunlock(&pmc_sx);
5304 }
5305
5306 static void
5307 pmc_process_threadcreate(struct thread *td)
5308 {
5309         struct pmc_owner *po;
5310
5311         epoch_enter_preempt(global_epoch_preempt);
5312         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5313             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
5314                         pmclog_process_threadcreate(po, td, 1);
5315         epoch_exit_preempt(global_epoch_preempt);
5316 }
5317
5318 static void
5319 pmc_process_threadexit(struct thread *td)
5320 {
5321         struct pmc_owner *po;
5322
5323         epoch_enter_preempt(global_epoch_preempt);
5324         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5325             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
5326                         pmclog_process_threadexit(po, td);
5327         epoch_exit_preempt(global_epoch_preempt);
5328 }
5329
5330 static void
5331 pmc_process_proccreate(struct proc *p)
5332 {
5333         struct pmc_owner *po;
5334
5335         epoch_enter_preempt(global_epoch_preempt);
5336         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5337             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
5338                         pmclog_process_proccreate(po, p, 1 /* sync */);
5339         epoch_exit_preempt(global_epoch_preempt);
5340 }
5341
5342 static void
5343 pmc_process_allproc(struct pmc *pm)
5344 {
5345         struct pmc_owner *po;
5346         struct thread *td;
5347         struct proc *p;
5348
5349         po = pm->pm_owner;
5350         if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
5351                 return;
5352         sx_slock(&allproc_lock);
5353         FOREACH_PROC_IN_SYSTEM(p) {
5354                 pmclog_process_proccreate(po, p, 0 /* sync */);
5355                 PROC_LOCK(p);
5356                 FOREACH_THREAD_IN_PROC(p, td)
5357                         pmclog_process_threadcreate(po, td, 0 /* sync */);
5358                 PROC_UNLOCK(p);
5359         }
5360         sx_sunlock(&allproc_lock);
5361         pmclog_flush(po, 0);
5362 }
5363
5364 static void
5365 pmc_kld_load(void *arg __unused, linker_file_t lf)
5366 {
5367         struct pmc_owner *po;
5368
5369         /*
5370          * Notify owners of system sampling PMCs about KLD operations.
5371          */
5372         epoch_enter_preempt(global_epoch_preempt);
5373         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5374                 if (po->po_flags & PMC_PO_OWNS_LOGFILE)
5375                         pmclog_process_map_in(po, (pid_t) -1,
5376                             (uintfptr_t) lf->address, lf->filename);
5377         epoch_exit_preempt(global_epoch_preempt);
5378
5379         /*
5380          * TODO: Notify owners of (all) process-sampling PMCs too.
5381          */
5382 }
5383
5384 static void
5385 pmc_kld_unload(void *arg __unused, const char *filename __unused,
5386     caddr_t address, size_t size)
5387 {
5388         struct pmc_owner *po;
5389
5390         epoch_enter_preempt(global_epoch_preempt);
5391         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5392                 if (po->po_flags & PMC_PO_OWNS_LOGFILE)
5393                         pmclog_process_map_out(po, (pid_t) -1,
5394                             (uintfptr_t) address, (uintfptr_t) address + size);
5395         epoch_exit_preempt(global_epoch_preempt);
5396
5397         /*
5398          * TODO: Notify owners of process-sampling PMCs.
5399          */
5400 }
5401
5402 /*
5403  * initialization
5404  */
5405 static const char *
5406 pmc_name_of_pmcclass(enum pmc_class class)
5407 {
5408
5409         switch (class) {
5410 #undef  __PMC_CLASS
5411 #define __PMC_CLASS(S,V,D)                                              \
5412         case PMC_CLASS_##S:                                             \
5413                 return #S;
5414         __PMC_CLASSES();
5415         default:
5416                 return ("<unknown>");
5417         }
5418 }
5419
5420 /*
5421  * Base class initializer: allocate structure and set default classes.
5422  */
5423 struct pmc_mdep *
5424 pmc_mdep_alloc(int nclasses)
5425 {
5426         struct pmc_mdep *md;
5427         int     n;
5428
5429         /* SOFT + md classes */
5430         n = 1 + nclasses;
5431         md = malloc(sizeof(struct pmc_mdep) + n *
5432             sizeof(struct pmc_classdep), M_PMC, M_WAITOK|M_ZERO);
5433         md->pmd_nclass = n;
5434
5435         /* Add base class. */
5436         pmc_soft_initialize(md);
5437         return md;
5438 }
5439
5440 void
5441 pmc_mdep_free(struct pmc_mdep *md)
5442 {
5443         pmc_soft_finalize(md);
5444         free(md, M_PMC);
5445 }
5446
5447 static int
5448 generic_switch_in(struct pmc_cpu *pc, struct pmc_process *pp)
5449 {
5450         (void) pc; (void) pp;
5451
5452         return (0);
5453 }
5454
5455 static int
5456 generic_switch_out(struct pmc_cpu *pc, struct pmc_process *pp)
5457 {
5458         (void) pc; (void) pp;
5459
5460         return (0);
5461 }
5462
5463 static struct pmc_mdep *
5464 pmc_generic_cpu_initialize(void)
5465 {
5466         struct pmc_mdep *md;
5467
5468         md = pmc_mdep_alloc(0);
5469
5470         md->pmd_cputype    = PMC_CPU_GENERIC;
5471
5472         md->pmd_pcpu_init  = NULL;
5473         md->pmd_pcpu_fini  = NULL;
5474         md->pmd_switch_in  = generic_switch_in;
5475         md->pmd_switch_out = generic_switch_out;
5476
5477         return (md);
5478 }
5479
5480 static void
5481 pmc_generic_cpu_finalize(struct pmc_mdep *md)
5482 {
5483         (void) md;
5484 }
5485
5486
5487 static int
5488 pmc_initialize(void)
5489 {
5490         int c, cpu, error, n, ri;
5491         unsigned int maxcpu, domain;
5492         struct pcpu *pc;
5493         struct pmc_binding pb;
5494         struct pmc_sample *ps;
5495         struct pmc_classdep *pcd;
5496         struct pmc_samplebuffer *sb;
5497
5498         md = NULL;
5499         error = 0;
5500
5501         pmc_stats.pm_intr_ignored = counter_u64_alloc(M_WAITOK);
5502         pmc_stats.pm_intr_processed = counter_u64_alloc(M_WAITOK);
5503         pmc_stats.pm_intr_bufferfull = counter_u64_alloc(M_WAITOK);
5504         pmc_stats.pm_syscalls = counter_u64_alloc(M_WAITOK);
5505         pmc_stats.pm_syscall_errors = counter_u64_alloc(M_WAITOK);
5506         pmc_stats.pm_buffer_requests = counter_u64_alloc(M_WAITOK);
5507         pmc_stats.pm_buffer_requests_failed = counter_u64_alloc(M_WAITOK);
5508         pmc_stats.pm_log_sweeps = counter_u64_alloc(M_WAITOK);
5509         pmc_stats.pm_merges = counter_u64_alloc(M_WAITOK);
5510         pmc_stats.pm_overwrites = counter_u64_alloc(M_WAITOK);
5511
5512 #ifdef  HWPMC_DEBUG
5513         /* parse debug flags first */
5514         if (TUNABLE_STR_FETCH(PMC_SYSCTL_NAME_PREFIX "debugflags",
5515                 pmc_debugstr, sizeof(pmc_debugstr)))
5516                 pmc_debugflags_parse(pmc_debugstr,
5517                     pmc_debugstr+strlen(pmc_debugstr));
5518 #endif
5519
5520         PMCDBG1(MOD,INI,0, "PMC Initialize (version %x)", PMC_VERSION);
5521
5522         /* check kernel version */
5523         if (pmc_kernel_version != PMC_VERSION) {
5524                 if (pmc_kernel_version == 0)
5525                         printf("hwpmc: this kernel has not been compiled with "
5526                             "'options HWPMC_HOOKS'.\n");
5527                 else
5528                         printf("hwpmc: kernel version (0x%x) does not match "
5529                             "module version (0x%x).\n", pmc_kernel_version,
5530                             PMC_VERSION);
5531                 return EPROGMISMATCH;
5532         }
5533
5534         /*
5535          * check sysctl parameters
5536          */
5537
5538         if (pmc_hashsize <= 0) {
5539                 (void) printf("hwpmc: tunable \"hashsize\"=%d must be "
5540                     "greater than zero.\n", pmc_hashsize);
5541                 pmc_hashsize = PMC_HASH_SIZE;
5542         }
5543
5544         if (pmc_nsamples <= 0 || pmc_nsamples > 65535) {
5545                 (void) printf("hwpmc: tunable \"nsamples\"=%d out of "
5546                     "range.\n", pmc_nsamples);
5547                 pmc_nsamples = PMC_NSAMPLES;
5548         }
5549
5550         if (pmc_callchaindepth <= 0 ||
5551             pmc_callchaindepth > PMC_CALLCHAIN_DEPTH_MAX) {
5552                 (void) printf("hwpmc: tunable \"callchaindepth\"=%d out of "
5553                     "range - using %d.\n", pmc_callchaindepth,
5554                     PMC_CALLCHAIN_DEPTH_MAX);
5555                 pmc_callchaindepth = PMC_CALLCHAIN_DEPTH_MAX;
5556         }
5557
5558         md = pmc_md_initialize();
5559         if (md == NULL) {
5560                 /* Default to generic CPU. */
5561                 md = pmc_generic_cpu_initialize();
5562                 if (md == NULL)
5563                         return (ENOSYS);
5564         }
5565
5566         KASSERT(md->pmd_nclass >= 1 && md->pmd_npmc >= 1,
5567             ("[pmc,%d] no classes or pmcs", __LINE__));
5568
5569         /* Compute the map from row-indices to classdep pointers. */
5570         pmc_rowindex_to_classdep = malloc(sizeof(struct pmc_classdep *) *
5571             md->pmd_npmc, M_PMC, M_WAITOK|M_ZERO);
5572
5573         for (n = 0; n < md->pmd_npmc; n++)
5574                 pmc_rowindex_to_classdep[n] = NULL;
5575         for (ri = c = 0; c < md->pmd_nclass; c++) {
5576                 pcd = &md->pmd_classdep[c];
5577                 for (n = 0; n < pcd->pcd_num; n++, ri++)
5578                         pmc_rowindex_to_classdep[ri] = pcd;
5579         }
5580
5581         KASSERT(ri == md->pmd_npmc,
5582             ("[pmc,%d] npmc miscomputed: ri=%d, md->npmc=%d", __LINE__,
5583             ri, md->pmd_npmc));
5584
5585         maxcpu = pmc_cpu_max();
5586
5587         /* allocate space for the per-cpu array */
5588         pmc_pcpu = malloc(maxcpu * sizeof(struct pmc_cpu *), M_PMC,
5589             M_WAITOK|M_ZERO);
5590
5591         /* per-cpu 'saved values' for managing process-mode PMCs */
5592         pmc_pcpu_saved = malloc(sizeof(pmc_value_t) * maxcpu * md->pmd_npmc,
5593             M_PMC, M_WAITOK);
5594
5595         /* Perform CPU-dependent initialization. */
5596         pmc_save_cpu_binding(&pb);
5597         error = 0;
5598         for (cpu = 0; error == 0 && cpu < maxcpu; cpu++) {
5599                 if (!pmc_cpu_is_active(cpu))
5600                         continue;
5601                 pmc_select_cpu(cpu);
5602                 pmc_pcpu[cpu] = malloc(sizeof(struct pmc_cpu) +
5603                     md->pmd_npmc * sizeof(struct pmc_hw *), M_PMC,
5604                     M_WAITOK|M_ZERO);
5605                 if (md->pmd_pcpu_init)
5606                         error = md->pmd_pcpu_init(md, cpu);
5607                 for (n = 0; error == 0 && n < md->pmd_nclass; n++)
5608                         error = md->pmd_classdep[n].pcd_pcpu_init(md, cpu);
5609         }
5610         pmc_restore_cpu_binding(&pb);
5611
5612         if (error)
5613                 return (error);
5614
5615         /* allocate space for the sample array */
5616         for (cpu = 0; cpu < maxcpu; cpu++) {
5617                 if (!pmc_cpu_is_active(cpu))
5618                         continue;
5619                 pc = pcpu_find(cpu);
5620                 domain = pc->pc_domain;
5621                 sb = malloc_domain(sizeof(struct pmc_samplebuffer) +
5622                         pmc_nsamples * sizeof(struct pmc_sample), M_PMC, domain,
5623                     M_WAITOK|M_ZERO);
5624                 sb->ps_read = sb->ps_write = sb->ps_samples;
5625                 sb->ps_fence = sb->ps_samples + pmc_nsamples;
5626
5627                 KASSERT(pmc_pcpu[cpu] != NULL,
5628                     ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
5629
5630                 sb->ps_callchains = malloc_domain(pmc_callchaindepth * pmc_nsamples *
5631                         sizeof(uintptr_t), M_PMC, domain, M_WAITOK|M_ZERO);
5632
5633                 for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
5634                         ps->ps_pc = sb->ps_callchains +
5635                             (n * pmc_callchaindepth);
5636
5637                 pmc_pcpu[cpu]->pc_sb[PMC_HR] = sb;
5638
5639                 sb = malloc_domain(sizeof(struct pmc_samplebuffer) +
5640                         pmc_nsamples * sizeof(struct pmc_sample), M_PMC, domain,
5641                     M_WAITOK|M_ZERO);
5642                 sb->ps_read = sb->ps_write = sb->ps_samples;
5643                 sb->ps_fence = sb->ps_samples + pmc_nsamples;
5644
5645                 KASSERT(pmc_pcpu[cpu] != NULL,
5646                     ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
5647
5648                 sb->ps_callchains = malloc_domain(pmc_callchaindepth * pmc_nsamples *
5649                         sizeof(uintptr_t), M_PMC, domain, M_WAITOK|M_ZERO);
5650
5651                 for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
5652                         ps->ps_pc = sb->ps_callchains +
5653                             (n * pmc_callchaindepth);
5654
5655                 pmc_pcpu[cpu]->pc_sb[PMC_SR] = sb;
5656
5657                 sb = malloc_domain(sizeof(struct pmc_samplebuffer) +
5658                         pmc_nsamples * sizeof(struct pmc_sample), M_PMC, domain,
5659                     M_WAITOK|M_ZERO);
5660                 sb->ps_read = sb->ps_write = sb->ps_samples;
5661                 sb->ps_fence = sb->ps_samples + pmc_nsamples;
5662
5663                 KASSERT(pmc_pcpu[cpu] != NULL,
5664                     ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
5665
5666                 sb->ps_callchains = malloc_domain(pmc_callchaindepth * pmc_nsamples *
5667                     sizeof(uintptr_t), M_PMC, domain, M_WAITOK|M_ZERO);
5668
5669                 for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
5670                         ps->ps_pc = sb->ps_callchains +
5671                             (n * pmc_callchaindepth);
5672
5673                 pmc_pcpu[cpu]->pc_sb[PMC_UR] = sb;
5674         }
5675
5676         /* allocate space for the row disposition array */
5677         pmc_pmcdisp = malloc(sizeof(enum pmc_mode) * md->pmd_npmc,
5678             M_PMC, M_WAITOK|M_ZERO);
5679
5680         /* mark all PMCs as available */
5681         for (n = 0; n < (int) md->pmd_npmc; n++)
5682                 PMC_MARK_ROW_FREE(n);
5683
5684         /* allocate thread hash tables */
5685         pmc_ownerhash = hashinit(pmc_hashsize, M_PMC,
5686             &pmc_ownerhashmask);
5687
5688         pmc_processhash = hashinit(pmc_hashsize, M_PMC,
5689             &pmc_processhashmask);
5690         mtx_init(&pmc_processhash_mtx, "pmc-process-hash", "pmc-leaf",
5691             MTX_SPIN);
5692
5693         CK_LIST_INIT(&pmc_ss_owners);
5694         pmc_ss_count = 0;
5695
5696         /* allocate a pool of spin mutexes */
5697         pmc_mtxpool = mtx_pool_create("pmc-leaf", pmc_mtxpool_size,
5698             MTX_SPIN);
5699
5700         PMCDBG4(MOD,INI,1, "pmc_ownerhash=%p, mask=0x%lx "
5701             "targethash=%p mask=0x%lx", pmc_ownerhash, pmc_ownerhashmask,
5702             pmc_processhash, pmc_processhashmask);
5703
5704         /* Initialize a spin mutex for the thread free list. */
5705         mtx_init(&pmc_threadfreelist_mtx, "pmc-threadfreelist", "pmc-leaf",
5706             MTX_SPIN);
5707
5708         /*
5709          * Initialize the callout to monitor the thread free list.
5710          * This callout will also handle the initial population of the list.
5711          */
5712         taskqgroup_config_gtask_init(NULL, &free_gtask, pmc_thread_descriptor_pool_free_task, "thread descriptor pool free task");
5713
5714         /* register process {exit,fork,exec} handlers */
5715         pmc_exit_tag = EVENTHANDLER_REGISTER(process_exit,
5716             pmc_process_exit, NULL, EVENTHANDLER_PRI_ANY);
5717         pmc_fork_tag = EVENTHANDLER_REGISTER(process_fork,
5718             pmc_process_fork, NULL, EVENTHANDLER_PRI_ANY);
5719
5720         /* register kld event handlers */
5721         pmc_kld_load_tag = EVENTHANDLER_REGISTER(kld_load, pmc_kld_load,
5722             NULL, EVENTHANDLER_PRI_ANY);
5723         pmc_kld_unload_tag = EVENTHANDLER_REGISTER(kld_unload, pmc_kld_unload,
5724             NULL, EVENTHANDLER_PRI_ANY);
5725
5726         /* initialize logging */
5727         pmclog_initialize();
5728
5729         /* set hook functions */
5730         pmc_intr = md->pmd_intr;
5731         wmb();
5732         pmc_hook = pmc_hook_handler;
5733
5734         if (error == 0) {
5735                 printf(PMC_MODULE_NAME ":");
5736                 for (n = 0; n < (int) md->pmd_nclass; n++) {
5737                         pcd = &md->pmd_classdep[n];
5738                         printf(" %s/%d/%d/0x%b",
5739                             pmc_name_of_pmcclass(pcd->pcd_class),
5740                             pcd->pcd_num,
5741                             pcd->pcd_width,
5742                             pcd->pcd_caps,
5743                             "\20"
5744                             "\1INT\2USR\3SYS\4EDG\5THR"
5745                             "\6REA\7WRI\10INV\11QUA\12PRC"
5746                             "\13TAG\14CSC");
5747                 }
5748                 printf("\n");
5749         }
5750
5751         return (error);
5752 }
5753
5754 /* prepare to be unloaded */
5755 static void
5756 pmc_cleanup(void)
5757 {
5758         int c, cpu;
5759         unsigned int maxcpu;
5760         struct pmc_ownerhash *ph;
5761         struct pmc_owner *po, *tmp;
5762         struct pmc_binding pb;
5763 #ifdef  HWPMC_DEBUG
5764         struct pmc_processhash *prh;
5765 #endif
5766
5767         PMCDBG0(MOD,INI,0, "cleanup");
5768
5769         /* switch off sampling */
5770         CPU_FOREACH(cpu)
5771                 DPCPU_ID_SET(cpu, pmc_sampled, 0);
5772         pmc_intr = NULL;
5773
5774         sx_xlock(&pmc_sx);
5775         if (pmc_hook == NULL) { /* being unloaded already */
5776                 sx_xunlock(&pmc_sx);
5777                 return;
5778         }
5779
5780         pmc_hook = NULL; /* prevent new threads from entering module */
5781
5782         /* deregister event handlers */
5783         EVENTHANDLER_DEREGISTER(process_fork, pmc_fork_tag);
5784         EVENTHANDLER_DEREGISTER(process_exit, pmc_exit_tag);
5785         EVENTHANDLER_DEREGISTER(kld_load, pmc_kld_load_tag);
5786         EVENTHANDLER_DEREGISTER(kld_unload, pmc_kld_unload_tag);
5787
5788         /* send SIGBUS to all owner threads, free up allocations */
5789         if (pmc_ownerhash)
5790                 for (ph = pmc_ownerhash;
5791                      ph <= &pmc_ownerhash[pmc_ownerhashmask];
5792                      ph++) {
5793                         LIST_FOREACH_SAFE(po, ph, po_next, tmp) {
5794                                 pmc_remove_owner(po);
5795
5796                                 /* send SIGBUS to owner processes */
5797                                 PMCDBG3(MOD,INI,2, "cleanup signal proc=%p "
5798                                     "(%d, %s)", po->po_owner,
5799                                     po->po_owner->p_pid,
5800                                     po->po_owner->p_comm);
5801
5802                                 PROC_LOCK(po->po_owner);
5803                                 kern_psignal(po->po_owner, SIGBUS);
5804                                 PROC_UNLOCK(po->po_owner);
5805
5806                                 pmc_destroy_owner_descriptor(po);
5807                         }
5808                 }
5809
5810         /* reclaim allocated data structures */
5811         mtx_destroy(&pmc_threadfreelist_mtx);
5812         pmc_thread_descriptor_pool_drain();
5813
5814         if (pmc_mtxpool)
5815                 mtx_pool_destroy(&pmc_mtxpool);
5816
5817         mtx_destroy(&pmc_processhash_mtx);
5818         taskqgroup_config_gtask_deinit(&free_gtask);
5819         if (pmc_processhash) {
5820 #ifdef  HWPMC_DEBUG
5821                 struct pmc_process *pp;
5822
5823                 PMCDBG0(MOD,INI,3, "destroy process hash");
5824                 for (prh = pmc_processhash;
5825                      prh <= &pmc_processhash[pmc_processhashmask];
5826                      prh++)
5827                         LIST_FOREACH(pp, prh, pp_next)
5828                             PMCDBG1(MOD,INI,3, "pid=%d", pp->pp_proc->p_pid);
5829 #endif
5830
5831                 hashdestroy(pmc_processhash, M_PMC, pmc_processhashmask);
5832                 pmc_processhash = NULL;
5833         }
5834
5835         if (pmc_ownerhash) {
5836                 PMCDBG0(MOD,INI,3, "destroy owner hash");
5837                 hashdestroy(pmc_ownerhash, M_PMC, pmc_ownerhashmask);
5838                 pmc_ownerhash = NULL;
5839         }
5840
5841         KASSERT(CK_LIST_EMPTY(&pmc_ss_owners),
5842             ("[pmc,%d] Global SS owner list not empty", __LINE__));
5843         KASSERT(pmc_ss_count == 0,
5844             ("[pmc,%d] Global SS count not empty", __LINE__));
5845
5846         /* do processor and pmc-class dependent cleanup */
5847         maxcpu = pmc_cpu_max();
5848
5849         PMCDBG0(MOD,INI,3, "md cleanup");
5850         if (md) {
5851                 pmc_save_cpu_binding(&pb);
5852                 for (cpu = 0; cpu < maxcpu; cpu++) {
5853                         PMCDBG2(MOD,INI,1,"pmc-cleanup cpu=%d pcs=%p",
5854                             cpu, pmc_pcpu[cpu]);
5855                         if (!pmc_cpu_is_active(cpu) || pmc_pcpu[cpu] == NULL)
5856                                 continue;
5857                         pmc_select_cpu(cpu);
5858                         for (c = 0; c < md->pmd_nclass; c++)
5859                                 md->pmd_classdep[c].pcd_pcpu_fini(md, cpu);
5860                         if (md->pmd_pcpu_fini)
5861                                 md->pmd_pcpu_fini(md, cpu);
5862                 }
5863
5864                 if (md->pmd_cputype == PMC_CPU_GENERIC)
5865                         pmc_generic_cpu_finalize(md);
5866                 else
5867                         pmc_md_finalize(md);
5868
5869                 pmc_mdep_free(md);
5870                 md = NULL;
5871                 pmc_restore_cpu_binding(&pb);
5872         }
5873
5874         /* Free per-cpu descriptors. */
5875         for (cpu = 0; cpu < maxcpu; cpu++) {
5876                 if (!pmc_cpu_is_active(cpu))
5877                         continue;
5878                 KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_HR] != NULL,
5879                     ("[pmc,%d] Null hw cpu sample buffer cpu=%d", __LINE__,
5880                         cpu));
5881                 KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_SR] != NULL,
5882                     ("[pmc,%d] Null sw cpu sample buffer cpu=%d", __LINE__,
5883                         cpu));
5884                 KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_UR] != NULL,
5885                     ("[pmc,%d] Null userret cpu sample buffer cpu=%d", __LINE__,
5886                         cpu));
5887                 free_domain(pmc_pcpu[cpu]->pc_sb[PMC_HR]->ps_callchains, M_PMC);
5888                 free_domain(pmc_pcpu[cpu]->pc_sb[PMC_HR], M_PMC);
5889                 free_domain(pmc_pcpu[cpu]->pc_sb[PMC_SR]->ps_callchains, M_PMC);
5890                 free_domain(pmc_pcpu[cpu]->pc_sb[PMC_SR], M_PMC);
5891                 free_domain(pmc_pcpu[cpu]->pc_sb[PMC_UR]->ps_callchains, M_PMC);
5892                 free_domain(pmc_pcpu[cpu]->pc_sb[PMC_UR], M_PMC);
5893                 free_domain(pmc_pcpu[cpu], M_PMC);
5894         }
5895
5896         free(pmc_pcpu, M_PMC);
5897         pmc_pcpu = NULL;
5898
5899         free(pmc_pcpu_saved, M_PMC);
5900         pmc_pcpu_saved = NULL;
5901
5902         if (pmc_pmcdisp) {
5903                 free(pmc_pmcdisp, M_PMC);
5904                 pmc_pmcdisp = NULL;
5905         }
5906
5907         if (pmc_rowindex_to_classdep) {
5908                 free(pmc_rowindex_to_classdep, M_PMC);
5909                 pmc_rowindex_to_classdep = NULL;
5910         }
5911
5912         pmclog_shutdown();
5913         counter_u64_free(pmc_stats.pm_intr_ignored);
5914         counter_u64_free(pmc_stats.pm_intr_processed);
5915         counter_u64_free(pmc_stats.pm_intr_bufferfull);
5916         counter_u64_free(pmc_stats.pm_syscalls);
5917         counter_u64_free(pmc_stats.pm_syscall_errors);
5918         counter_u64_free(pmc_stats.pm_buffer_requests);
5919         counter_u64_free(pmc_stats.pm_buffer_requests_failed);
5920         counter_u64_free(pmc_stats.pm_log_sweeps);
5921         counter_u64_free(pmc_stats.pm_merges);
5922         counter_u64_free(pmc_stats.pm_overwrites);
5923         sx_xunlock(&pmc_sx);    /* we are done */
5924 }
5925
5926 /*
5927  * The function called at load/unload.
5928  */
5929
5930 static int
5931 load (struct module *module __unused, int cmd, void *arg __unused)
5932 {
5933         int error;
5934
5935         error = 0;
5936
5937         switch (cmd) {
5938         case MOD_LOAD :
5939                 /* initialize the subsystem */
5940                 error = pmc_initialize();
5941                 if (error != 0)
5942                         break;
5943                 PMCDBG2(MOD,INI,1, "syscall=%d maxcpu=%d",
5944                     pmc_syscall_num, pmc_cpu_max());
5945                 break;
5946
5947
5948         case MOD_UNLOAD :
5949         case MOD_SHUTDOWN:
5950                 pmc_cleanup();
5951                 PMCDBG0(MOD,INI,1, "unloaded");
5952                 break;
5953
5954         default :
5955                 error = EINVAL; /* XXX should panic(9) */
5956                 break;
5957         }
5958
5959         return error;
5960 }