sys/dev/hwpmc/hwpmc_mod.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2003-2008 Joseph Koshy
   5  * Copyright (c) 2007 The FreeBSD Foundation
   6  * Copyright (c) 2018 Matthew Macy
   7  * All rights reserved.
   8  *
   9  * Portions of this software were developed by A. Joseph Koshy under
  10  * sponsorship from the FreeBSD Foundation and Google, Inc.
  11  *
  12  * Redistribution and use in source and binary forms, with or without
  13  * modification, are permitted provided that the following conditions
  14  * are met:
  15  * 1. Redistributions of source code must retain the above copyright
  16  *    notice, this list of conditions and the following disclaimer.
  17  * 2. Redistributions in binary form must reproduce the above copyright
  18  *    notice, this list of conditions and the following disclaimer in the
  19  *    documentation and/or other materials provided with the distribution.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  24  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  25  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  30  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  31  * SUCH DAMAGE.
  32  *
  33  */
  34
  35 #include <sys/cdefs.h>
  36 __FBSDID("$FreeBSD$");
  37
  38 #include <sys/param.h>
  39 #include <sys/systm.h>
  40 #include <sys/domainset.h>
  41 #include <sys/eventhandler.h>
  42 #include <sys/jail.h>
  43 #include <sys/kernel.h>
  44 #include <sys/kthread.h>
  45 #include <sys/limits.h>
  46 #include <sys/lock.h>
  47 #include <sys/malloc.h>
  48 #include <sys/module.h>
  49 #include <sys/mount.h>
  50 #include <sys/mutex.h>
  51 #include <sys/pmc.h>
  52 #include <sys/pmckern.h>
  53 #include <sys/pmclog.h>
  54 #include <sys/priv.h>
  55 #include <sys/proc.h>
  56 #include <sys/queue.h>
  57 #include <sys/resourcevar.h>
  58 #include <sys/rwlock.h>
  59 #include <sys/sched.h>
  60 #include <sys/signalvar.h>
  61 #include <sys/smp.h>
  62 #include <sys/sx.h>
  63 #include <sys/sysctl.h>
  64 #include <sys/sysent.h>
  65 #include <sys/syslog.h>
  66 #include <sys/taskqueue.h>
  67 #include <sys/vnode.h>
  68
  69 #include <sys/linker.h>         /* needs to be after <sys/malloc.h> */
  70
  71 #include <machine/atomic.h>
  72 #include <machine/md_var.h>
  73
  74 #include <vm/vm.h>
  75 #include <vm/vm_extern.h>
  76 #include <vm/pmap.h>
  77 #include <vm/vm_map.h>
  78 #include <vm/vm_object.h>
  79
  80 #include "hwpmc_soft.h"
  81
  82 #define PMC_EPOCH_ENTER() struct epoch_tracker pmc_et; epoch_enter_preempt(global_epoch_preempt, &pmc_et)
  83 #define PMC_EPOCH_EXIT() epoch_exit_preempt(global_epoch_preempt, &pmc_et)
  84
  85 /*
  86  * Types
  87  */
  88
  89 enum pmc_flags {
  90         PMC_FLAG_NONE     = 0x00, /* do nothing */
  91         PMC_FLAG_REMOVE   = 0x01, /* atomically remove entry from hash */
  92         PMC_FLAG_ALLOCATE = 0x02, /* add entry to hash if not found */
  93         PMC_FLAG_NOWAIT   = 0x04, /* do not wait for mallocs */
  94 };
  95
  96 /*
  97  * The offset in sysent where the syscall is allocated.
  98  */
  99
 100 static int pmc_syscall_num = NO_SYSCALL;
 101 struct pmc_cpu          **pmc_pcpu;      /* per-cpu state */
 102 pmc_value_t             *pmc_pcpu_saved; /* saved PMC values: CSW handling */
 103
 104 #define PMC_PCPU_SAVED(C,R)     pmc_pcpu_saved[(R) + md->pmd_npmc*(C)]
 105
 106 struct mtx_pool         *pmc_mtxpool;
 107 static int              *pmc_pmcdisp;    /* PMC row dispositions */
 108
 109 #define PMC_ROW_DISP_IS_FREE(R)         (pmc_pmcdisp[(R)] == 0)
 110 #define PMC_ROW_DISP_IS_THREAD(R)       (pmc_pmcdisp[(R)] > 0)
 111 #define PMC_ROW_DISP_IS_STANDALONE(R)   (pmc_pmcdisp[(R)] < 0)
 112
 113 #define PMC_MARK_ROW_FREE(R) do {                                         \
 114         pmc_pmcdisp[(R)] = 0;                                             \
 115 } while (0)
 116
 117 #define PMC_MARK_ROW_STANDALONE(R) do {                                   \
 118         KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \
 119                     __LINE__));                                           \
 120         atomic_add_int(&pmc_pmcdisp[(R)], -1);                            \
 121         KASSERT(pmc_pmcdisp[(R)] >= (-pmc_cpu_max_active()),              \
 122                 ("[pmc,%d] row disposition error", __LINE__));            \
 123 } while (0)
 124
 125 #define PMC_UNMARK_ROW_STANDALONE(R) do {                                 \
 126         atomic_add_int(&pmc_pmcdisp[(R)], 1);                             \
 127         KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \
 128                     __LINE__));                                           \
 129 } while (0)
 130
 131 #define PMC_MARK_ROW_THREAD(R) do {                                       \
 132         KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \
 133                     __LINE__));                                           \
 134         atomic_add_int(&pmc_pmcdisp[(R)], 1);                             \
 135 } while (0)
 136
 137 #define PMC_UNMARK_ROW_THREAD(R) do {                                     \
 138         atomic_add_int(&pmc_pmcdisp[(R)], -1);                            \
 139         KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \
 140                     __LINE__));                                           \
 141 } while (0)
 142
 143
 144 /* various event handlers */
 145 static eventhandler_tag pmc_exit_tag, pmc_fork_tag, pmc_kld_load_tag,
 146     pmc_kld_unload_tag;
 147
 148 /* Module statistics */
 149 struct pmc_driverstats pmc_stats;
 150
 151
 152 /* Machine/processor dependent operations */
 153 static struct pmc_mdep  *md;
 154
 155 /*
 156  * Hash tables mapping owner processes and target threads to PMCs.
 157  */
 158
 159 struct mtx pmc_processhash_mtx;         /* spin mutex */
 160 static u_long pmc_processhashmask;
 161 static LIST_HEAD(pmc_processhash, pmc_process)  *pmc_processhash;
 162
 163 /*
 164  * Hash table of PMC owner descriptors.  This table is protected by
 165  * the shared PMC "sx" lock.
 166  */
 167
 168 static u_long pmc_ownerhashmask;
 169 static LIST_HEAD(pmc_ownerhash, pmc_owner)      *pmc_ownerhash;
 170
 171 /*
 172  * List of PMC owners with system-wide sampling PMCs.
 173  */
 174
 175 static CK_LIST_HEAD(, pmc_owner)                        pmc_ss_owners;
 176
 177 /*
 178  * List of free thread entries. This is protected by the spin
 179  * mutex.
 180  */
 181 static struct mtx pmc_threadfreelist_mtx;       /* spin mutex */
 182 static LIST_HEAD(, pmc_thread)                  pmc_threadfreelist;
 183 static int pmc_threadfreelist_entries=0;
 184 #define THREADENTRY_SIZE                                                \
 185 (sizeof(struct pmc_thread) + (md->pmd_npmc * sizeof(struct pmc_threadpmcstate)))
 186
 187 /*
 188  * Task to free thread descriptors
 189  */
 190 static struct task free_task;
 191
 192 /*
 193  * A map of row indices to classdep structures.
 194  */
 195 static struct pmc_classdep **pmc_rowindex_to_classdep;
 196
 197 /*
 198  * Prototypes
 199  */
 200
 201 #ifdef  HWPMC_DEBUG
 202 static int      pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS);
 203 static int      pmc_debugflags_parse(char *newstr, char *fence);
 204 #endif
 205
 206 static int      load(struct module *module, int cmd, void *arg);
 207 static int      pmc_add_sample(ring_type_t ring, struct pmc *pm, struct trapframe *tf);
 208 static void     pmc_add_thread_descriptors_from_proc(struct proc *p,
 209     struct pmc_process *pp);
 210 static int      pmc_attach_process(struct proc *p, struct pmc *pm);
 211 static struct pmc *pmc_allocate_pmc_descriptor(void);
 212 static struct pmc_owner *pmc_allocate_owner_descriptor(struct proc *p);
 213 static int      pmc_attach_one_process(struct proc *p, struct pmc *pm);
 214 static int      pmc_can_allocate_rowindex(struct proc *p, unsigned int ri,
 215     int cpu);
 216 static int      pmc_can_attach(struct pmc *pm, struct proc *p);
 217 static void     pmc_capture_user_callchain(int cpu, int soft, struct trapframe *tf);
 218 static void     pmc_cleanup(void);
 219 static int      pmc_detach_process(struct proc *p, struct pmc *pm);
 220 static int      pmc_detach_one_process(struct proc *p, struct pmc *pm,
 221     int flags);
 222 static void     pmc_destroy_owner_descriptor(struct pmc_owner *po);
 223 static void     pmc_destroy_pmc_descriptor(struct pmc *pm);
 224 static void     pmc_destroy_process_descriptor(struct pmc_process *pp);
 225 static struct pmc_owner *pmc_find_owner_descriptor(struct proc *p);
 226 static int      pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm);
 227 static struct pmc *pmc_find_pmc_descriptor_in_process(struct pmc_owner *po,
 228     pmc_id_t pmc);
 229 static struct pmc_process *pmc_find_process_descriptor(struct proc *p,
 230     uint32_t mode);
 231 static struct pmc_thread *pmc_find_thread_descriptor(struct pmc_process *pp,
 232     struct thread *td, uint32_t mode);
 233 static void     pmc_force_context_switch(void);
 234 static void     pmc_link_target_process(struct pmc *pm,
 235     struct pmc_process *pp);
 236 static void     pmc_log_all_process_mappings(struct pmc_owner *po);
 237 static void     pmc_log_kernel_mappings(struct pmc *pm);
 238 static void     pmc_log_process_mappings(struct pmc_owner *po, struct proc *p);
 239 static void     pmc_maybe_remove_owner(struct pmc_owner *po);
 240 static void     pmc_process_csw_in(struct thread *td);
 241 static void     pmc_process_csw_out(struct thread *td);
 242 static void     pmc_process_exit(void *arg, struct proc *p);
 243 static void     pmc_process_fork(void *arg, struct proc *p1,
 244     struct proc *p2, int n);
 245 static void     pmc_process_samples(int cpu, ring_type_t soft);
 246 static void     pmc_release_pmc_descriptor(struct pmc *pmc);
 247 static void     pmc_process_thread_add(struct thread *td);
 248 static void     pmc_process_thread_delete(struct thread *td);
 249 static void     pmc_process_thread_userret(struct thread *td);
 250 static void     pmc_remove_owner(struct pmc_owner *po);
 251 static void     pmc_remove_process_descriptor(struct pmc_process *pp);
 252 static void     pmc_restore_cpu_binding(struct pmc_binding *pb);
 253 static void     pmc_save_cpu_binding(struct pmc_binding *pb);
 254 static void     pmc_select_cpu(int cpu);
 255 static int      pmc_start(struct pmc *pm);
 256 static int      pmc_stop(struct pmc *pm);
 257 static int      pmc_syscall_handler(struct thread *td, void *syscall_args);
 258 static struct pmc_thread *pmc_thread_descriptor_pool_alloc(void);
 259 static void     pmc_thread_descriptor_pool_drain(void);
 260 static void     pmc_thread_descriptor_pool_free(struct pmc_thread *pt);
 261 static void     pmc_unlink_target_process(struct pmc *pmc,
 262     struct pmc_process *pp);
 263 static int generic_switch_in(struct pmc_cpu *pc, struct pmc_process *pp);
 264 static int generic_switch_out(struct pmc_cpu *pc, struct pmc_process *pp);
 265 static struct pmc_mdep *pmc_generic_cpu_initialize(void);
 266 static void pmc_generic_cpu_finalize(struct pmc_mdep *md);
 267 static void pmc_post_callchain_callback(void);
 268 static void pmc_process_threadcreate(struct thread *td);
 269 static void pmc_process_threadexit(struct thread *td);
 270 static void pmc_process_proccreate(struct proc *p);
 271 static void pmc_process_allproc(struct pmc *pm);
 272
 273 /*
 274  * Kernel tunables and sysctl(8) interface.
 275  */
 276
 277 SYSCTL_DECL(_kern_hwpmc);
 278 SYSCTL_NODE(_kern_hwpmc, OID_AUTO, stats, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
 279     "HWPMC stats");
 280
 281
 282 /* Stats. */
 283 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, intr_ignored, CTLFLAG_RW,
 284                                    &pmc_stats.pm_intr_ignored, "# of interrupts ignored");
 285 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, intr_processed, CTLFLAG_RW,
 286                                    &pmc_stats.pm_intr_processed, "# of interrupts processed");
 287 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, intr_bufferfull, CTLFLAG_RW,
 288                                    &pmc_stats.pm_intr_bufferfull, "# of interrupts where buffer was full");
 289 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, syscalls, CTLFLAG_RW,
 290                                    &pmc_stats.pm_syscalls, "# of syscalls");
 291 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, syscall_errors, CTLFLAG_RW,
 292                                    &pmc_stats.pm_syscall_errors, "# of syscall_errors");
 293 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, buffer_requests, CTLFLAG_RW,
 294                                    &pmc_stats.pm_buffer_requests, "# of buffer requests");
 295 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, buffer_requests_failed, CTLFLAG_RW,
 296                                    &pmc_stats.pm_buffer_requests_failed, "# of buffer requests which failed");
 297 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, log_sweeps, CTLFLAG_RW,
 298                                    &pmc_stats.pm_log_sweeps, "# of ?");
 299 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, merges, CTLFLAG_RW,
 300                                    &pmc_stats.pm_merges, "# of times kernel stack was found for user trace");
 301 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, overwrites, CTLFLAG_RW,
 302                                    &pmc_stats.pm_overwrites, "# of times a sample was overwritten before being logged");
 303
 304 static int pmc_callchaindepth = PMC_CALLCHAIN_DEPTH;
 305 SYSCTL_INT(_kern_hwpmc, OID_AUTO, callchaindepth, CTLFLAG_RDTUN,
 306     &pmc_callchaindepth, 0, "depth of call chain records");
 307
 308 char pmc_cpuid[PMC_CPUID_LEN];
 309 SYSCTL_STRING(_kern_hwpmc, OID_AUTO, cpuid, CTLFLAG_RD,
 310         pmc_cpuid, 0, "cpu version string");
 311 #ifdef  HWPMC_DEBUG
 312 struct pmc_debugflags pmc_debugflags = PMC_DEBUG_DEFAULT_FLAGS;
 313 char    pmc_debugstr[PMC_DEBUG_STRSIZE];
 314 TUNABLE_STR(PMC_SYSCTL_NAME_PREFIX "debugflags", pmc_debugstr,
 315     sizeof(pmc_debugstr));
 316 SYSCTL_PROC(_kern_hwpmc, OID_AUTO, debugflags,
 317     CTLTYPE_STRING | CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_NEEDGIANT,
 318     0, 0, pmc_debugflags_sysctl_handler, "A",
 319     "debug flags");
 320 #endif
 321
 322
 323 /*
 324  * kern.hwpmc.hashrows -- determines the number of rows in the
 325  * of the hash table used to look up threads
 326  */
 327
 328 static int pmc_hashsize = PMC_HASH_SIZE;
 329 SYSCTL_INT(_kern_hwpmc, OID_AUTO, hashsize, CTLFLAG_RDTUN,
 330     &pmc_hashsize, 0, "rows in hash tables");
 331
 332 /*
 333  * kern.hwpmc.nsamples --- number of PC samples/callchain stacks per CPU
 334  */
 335
 336 static int pmc_nsamples = PMC_NSAMPLES;
 337 SYSCTL_INT(_kern_hwpmc, OID_AUTO, nsamples, CTLFLAG_RDTUN,
 338     &pmc_nsamples, 0, "number of PC samples per CPU");
 339
 340 static uint64_t pmc_sample_mask = PMC_NSAMPLES-1;
 341
 342 /*
 343  * kern.hwpmc.mtxpoolsize -- number of mutexes in the mutex pool.
 344  */
 345
 346 static int pmc_mtxpool_size = PMC_MTXPOOL_SIZE;
 347 SYSCTL_INT(_kern_hwpmc, OID_AUTO, mtxpoolsize, CTLFLAG_RDTUN,
 348     &pmc_mtxpool_size, 0, "size of spin mutex pool");
 349
 350
 351 /*
 352  * kern.hwpmc.threadfreelist_entries -- number of free entries
 353  */
 354
 355 SYSCTL_INT(_kern_hwpmc, OID_AUTO, threadfreelist_entries, CTLFLAG_RD,
 356     &pmc_threadfreelist_entries, 0, "number of avalable thread entries");
 357
 358
 359 /*
 360  * kern.hwpmc.threadfreelist_max -- maximum number of free entries
 361  */
 362
 363 static int pmc_threadfreelist_max = PMC_THREADLIST_MAX;
 364 SYSCTL_INT(_kern_hwpmc, OID_AUTO, threadfreelist_max, CTLFLAG_RW,
 365     &pmc_threadfreelist_max, 0,
 366     "maximum number of available thread entries before freeing some");
 367
 368
 369 /*
 370  * security.bsd.unprivileged_syspmcs -- allow non-root processes to
 371  * allocate system-wide PMCs.
 372  *
 373  * Allowing unprivileged processes to allocate system PMCs is convenient
 374  * if system-wide measurements need to be taken concurrently with other
 375  * per-process measurements.  This feature is turned off by default.
 376  */
 377
 378 static int pmc_unprivileged_syspmcs = 0;
 379 SYSCTL_INT(_security_bsd, OID_AUTO, unprivileged_syspmcs, CTLFLAG_RWTUN,
 380     &pmc_unprivileged_syspmcs, 0,
 381     "allow unprivileged process to allocate system PMCs");
 382
 383 /*
 384  * Hash function.  Discard the lower 2 bits of the pointer since
 385  * these are always zero for our uses.  The hash multiplier is
 386  * round((2^LONG_BIT) * ((sqrt(5)-1)/2)).
 387  */
 388
 389 #if     LONG_BIT == 64
 390 #define _PMC_HM         11400714819323198486u
 391 #elif   LONG_BIT == 32
 392 #define _PMC_HM         2654435769u
 393 #else
 394 #error  Must know the size of 'long' to compile
 395 #endif
 396
 397 #define PMC_HASH_PTR(P,M)       ((((unsigned long) (P) >> 2) * _PMC_HM) & (M))
 398
 399 /*
 400  * Syscall structures
 401  */
 402
 403 /* The `sysent' for the new syscall */
 404 static struct sysent pmc_sysent = {
 405         .sy_narg =      2,
 406         .sy_call =      pmc_syscall_handler,
 407 };
 408
 409 static struct syscall_module_data pmc_syscall_mod = {
 410         .chainevh =     load,
 411         .chainarg =     NULL,
 412         .offset =       &pmc_syscall_num,
 413         .new_sysent =   &pmc_sysent,
 414         .old_sysent =   { .sy_narg = 0, .sy_call = NULL },
 415         .flags =        SY_THR_STATIC_KLD,
 416 };
 417
 418 static moduledata_t pmc_mod = {
 419         .name =         PMC_MODULE_NAME,
 420         .evhand =       syscall_module_handler,
 421         .priv =         &pmc_syscall_mod,
 422 };
 423
 424 #ifdef EARLY_AP_STARTUP
 425 DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SYSCALLS, SI_ORDER_ANY);
 426 #else
 427 DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SMP, SI_ORDER_ANY);
 428 #endif
 429 MODULE_VERSION(pmc, PMC_VERSION);
 430
 431 #ifdef  HWPMC_DEBUG
 432 enum pmc_dbgparse_state {
 433         PMCDS_WS,               /* in whitespace */
 434         PMCDS_MAJOR,            /* seen a major keyword */
 435         PMCDS_MINOR
 436 };
 437
 438 static int
 439 pmc_debugflags_parse(char *newstr, char *fence)
 440 {
 441         char c, *p, *q;
 442         struct pmc_debugflags *tmpflags;
 443         int error, found, *newbits, tmp;
 444         size_t kwlen;
 445
 446         tmpflags = malloc(sizeof(*tmpflags), M_PMC, M_WAITOK|M_ZERO);
 447
 448         p = newstr;
 449         error = 0;
 450
 451         for (; p < fence && (c = *p); p++) {
 452
 453                 /* skip white space */
 454                 if (c == ' ' || c == '\t')
 455                         continue;
 456
 457                 /* look for a keyword followed by "=" */
 458                 for (q = p; p < fence && (c = *p) && c != '='; p++)
 459                         ;
 460                 if (c != '=') {
 461                         error = EINVAL;
 462                         goto done;
 463                 }
 464
 465                 kwlen = p - q;
 466                 newbits = NULL;
 467
 468                 /* lookup flag group name */
 469 #define DBG_SET_FLAG_MAJ(S,F)                                           \
 470                 if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0)  \
 471                         newbits = &tmpflags->pdb_ ## F;
 472
 473                 DBG_SET_FLAG_MAJ("cpu",         CPU);
 474                 DBG_SET_FLAG_MAJ("csw",         CSW);
 475                 DBG_SET_FLAG_MAJ("logging",     LOG);
 476                 DBG_SET_FLAG_MAJ("module",      MOD);
 477                 DBG_SET_FLAG_MAJ("md",          MDP);
 478                 DBG_SET_FLAG_MAJ("owner",       OWN);
 479                 DBG_SET_FLAG_MAJ("pmc",         PMC);
 480                 DBG_SET_FLAG_MAJ("process",     PRC);
 481                 DBG_SET_FLAG_MAJ("sampling",    SAM);
 482
 483                 if (newbits == NULL) {
 484                         error = EINVAL;
 485                         goto done;
 486                 }
 487
 488                 p++;            /* skip the '=' */
 489
 490                 /* Now parse the individual flags */
 491                 tmp = 0;
 492         newflag:
 493                 for (q = p; p < fence && (c = *p); p++)
 494                         if (c == ' ' || c == '\t' || c == ',')
 495                                 break;
 496
 497                 /* p == fence or c == ws or c == "," or c == 0 */
 498
 499                 if ((kwlen = p - q) == 0) {
 500                         *newbits = tmp;
 501                         continue;
 502                 }
 503
 504                 found = 0;
 505 #define DBG_SET_FLAG_MIN(S,F)                                           \
 506                 if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0)  \
 507                         tmp |= found = (1 << PMC_DEBUG_MIN_ ## F)
 508
 509                 /* a '*' denotes all possible flags in the group */
 510                 if (kwlen == 1 && *q == '*')
 511                         tmp = found = ~0;
 512                 /* look for individual flag names */
 513                 DBG_SET_FLAG_MIN("allocaterow", ALR);
 514                 DBG_SET_FLAG_MIN("allocate",    ALL);
 515                 DBG_SET_FLAG_MIN("attach",      ATT);
 516                 DBG_SET_FLAG_MIN("bind",        BND);
 517                 DBG_SET_FLAG_MIN("config",      CFG);
 518                 DBG_SET_FLAG_MIN("exec",        EXC);
 519                 DBG_SET_FLAG_MIN("exit",        EXT);
 520                 DBG_SET_FLAG_MIN("find",        FND);
 521                 DBG_SET_FLAG_MIN("flush",       FLS);
 522                 DBG_SET_FLAG_MIN("fork",        FRK);
 523                 DBG_SET_FLAG_MIN("getbuf",      GTB);
 524                 DBG_SET_FLAG_MIN("hook",        PMH);
 525                 DBG_SET_FLAG_MIN("init",        INI);
 526                 DBG_SET_FLAG_MIN("intr",        INT);
 527                 DBG_SET_FLAG_MIN("linktarget",  TLK);
 528                 DBG_SET_FLAG_MIN("mayberemove", OMR);
 529                 DBG_SET_FLAG_MIN("ops",         OPS);
 530                 DBG_SET_FLAG_MIN("read",        REA);
 531                 DBG_SET_FLAG_MIN("register",    REG);
 532                 DBG_SET_FLAG_MIN("release",     REL);
 533                 DBG_SET_FLAG_MIN("remove",      ORM);
 534                 DBG_SET_FLAG_MIN("sample",      SAM);
 535                 DBG_SET_FLAG_MIN("scheduleio",  SIO);
 536                 DBG_SET_FLAG_MIN("select",      SEL);
 537                 DBG_SET_FLAG_MIN("signal",      SIG);
 538                 DBG_SET_FLAG_MIN("swi",         SWI);
 539                 DBG_SET_FLAG_MIN("swo",         SWO);
 540                 DBG_SET_FLAG_MIN("start",       STA);
 541                 DBG_SET_FLAG_MIN("stop",        STO);
 542                 DBG_SET_FLAG_MIN("syscall",     PMS);
 543                 DBG_SET_FLAG_MIN("unlinktarget", TUL);
 544                 DBG_SET_FLAG_MIN("write",       WRI);
 545                 if (found == 0) {
 546                         /* unrecognized flag name */
 547                         error = EINVAL;
 548                         goto done;
 549                 }
 550
 551                 if (c == 0 || c == ' ' || c == '\t') {  /* end of flag group */
 552                         *newbits = tmp;
 553                         continue;
 554                 }
 555
 556                 p++;
 557                 goto newflag;
 558         }
 559
 560         /* save the new flag set */
 561         bcopy(tmpflags, &pmc_debugflags, sizeof(pmc_debugflags));
 562
 563  done:
 564         free(tmpflags, M_PMC);
 565         return error;
 566 }
 567
 568 static int
 569 pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS)
 570 {
 571         char *fence, *newstr;
 572         int error;
 573         unsigned int n;
 574
 575         (void) arg1; (void) arg2; /* unused parameters */
 576
 577         n = sizeof(pmc_debugstr);
 578         newstr = malloc(n, M_PMC, M_WAITOK|M_ZERO);
 579         (void) strlcpy(newstr, pmc_debugstr, n);
 580
 581         error = sysctl_handle_string(oidp, newstr, n, req);
 582
 583         /* if there is a new string, parse and copy it */
 584         if (error == 0 && req->newptr != NULL) {
 585                 fence = newstr + (n < req->newlen ? n : req->newlen + 1);
 586                 if ((error = pmc_debugflags_parse(newstr, fence)) == 0)
 587                         (void) strlcpy(pmc_debugstr, newstr,
 588                             sizeof(pmc_debugstr));
 589         }
 590
 591         free(newstr, M_PMC);
 592
 593         return error;
 594 }
 595 #endif
 596
 597 /*
 598  * Map a row index to a classdep structure and return the adjusted row
 599  * index for the PMC class index.
 600  */
 601 static struct pmc_classdep *
 602 pmc_ri_to_classdep(struct pmc_mdep *md, int ri, int *adjri)
 603 {
 604         struct pmc_classdep *pcd;
 605
 606         (void) md;
 607
 608         KASSERT(ri >= 0 && ri < md->pmd_npmc,
 609             ("[pmc,%d] illegal row-index %d", __LINE__, ri));
 610
 611         pcd = pmc_rowindex_to_classdep[ri];
 612
 613         KASSERT(pcd != NULL,
 614             ("[pmc,%d] ri %d null pcd", __LINE__, ri));
 615
 616         *adjri = ri - pcd->pcd_ri;
 617
 618         KASSERT(*adjri >= 0 && *adjri < pcd->pcd_num,
 619             ("[pmc,%d] adjusted row-index %d", __LINE__, *adjri));
 620
 621         return (pcd);
 622 }
 623
 624 /*
 625  * Concurrency Control
 626  *
 627  * The driver manages the following data structures:
 628  *
 629  *   - target process descriptors, one per target process
 630  *   - owner process descriptors (and attached lists), one per owner process
 631  *   - lookup hash tables for owner and target processes
 632  *   - PMC descriptors (and attached lists)
 633  *   - per-cpu hardware state
 634  *   - the 'hook' variable through which the kernel calls into
 635  *     this module
 636  *   - the machine hardware state (managed by the MD layer)
 637  *
 638  * These data structures are accessed from:
 639  *
 640  * - thread context-switch code
 641  * - interrupt handlers (possibly on multiple cpus)
 642  * - kernel threads on multiple cpus running on behalf of user
 643  *   processes doing system calls
 644  * - this driver's private kernel threads
 645  *
 646  * = Locks and Locking strategy =
 647  *
 648  * The driver uses four locking strategies for its operation:
 649  *
 650  * - The global SX lock "pmc_sx" is used to protect internal
 651  *   data structures.
 652  *
 653  *   Calls into the module by syscall() start with this lock being
 654  *   held in exclusive mode.  Depending on the requested operation,
 655  *   the lock may be downgraded to 'shared' mode to allow more
 656  *   concurrent readers into the module.  Calls into the module from
 657  *   other parts of the kernel acquire the lock in shared mode.
 658  *
 659  *   This SX lock is held in exclusive mode for any operations that
 660  *   modify the linkages between the driver's internal data structures.
 661  *
 662  *   The 'pmc_hook' function pointer is also protected by this lock.
 663  *   It is only examined with the sx lock held in exclusive mode.  The
 664  *   kernel module is allowed to be unloaded only with the sx lock held
 665  *   in exclusive mode.  In normal syscall handling, after acquiring the
 666  *   pmc_sx lock we first check that 'pmc_hook' is non-null before
 667  *   proceeding.  This prevents races between the thread unloading the module
 668  *   and other threads seeking to use the module.
 669  *
 670  * - Lookups of target process structures and owner process structures
 671  *   cannot use the global "pmc_sx" SX lock because these lookups need
 672  *   to happen during context switches and in other critical sections
 673  *   where sleeping is not allowed.  We protect these lookup tables
 674  *   with their own private spin-mutexes, "pmc_processhash_mtx" and
 675  *   "pmc_ownerhash_mtx".
 676  *
 677  * - Interrupt handlers work in a lock free manner.  At interrupt
 678  *   time, handlers look at the PMC pointer (phw->phw_pmc) configured
 679  *   when the PMC was started.  If this pointer is NULL, the interrupt
 680  *   is ignored after updating driver statistics.  We ensure that this
 681  *   pointer is set (using an atomic operation if necessary) before the
 682  *   PMC hardware is started.  Conversely, this pointer is unset atomically
 683  *   only after the PMC hardware is stopped.
 684  *
 685  *   We ensure that everything needed for the operation of an
 686  *   interrupt handler is available without it needing to acquire any
 687  *   locks.  We also ensure that a PMC's software state is destroyed only
 688  *   after the PMC is taken off hardware (on all CPUs).
 689  *
 690  * - Context-switch handling with process-private PMCs needs more
 691  *   care.
 692  *
 693  *   A given process may be the target of multiple PMCs.  For example,
 694  *   PMCATTACH and PMCDETACH may be requested by a process on one CPU
 695  *   while the target process is running on another.  A PMC could also
 696  *   be getting released because its owner is exiting.  We tackle
 697  *   these situations in the following manner:
 698  *
 699  *   - each target process structure 'pmc_process' has an array
 700  *     of 'struct pmc *' pointers, one for each hardware PMC.
 701  *
 702  *   - At context switch IN time, each "target" PMC in RUNNING state
 703  *     gets started on hardware and a pointer to each PMC is copied into
 704  *     the per-cpu phw array.  The 'runcount' for the PMC is
 705  *     incremented.
 706  *
 707  *   - At context switch OUT time, all process-virtual PMCs are stopped
 708  *     on hardware.  The saved value is added to the PMCs value field
 709  *     only if the PMC is in a non-deleted state (the PMCs state could
 710  *     have changed during the current time slice).
 711  *
 712  *     Note that since in-between a switch IN on a processor and a switch
 713  *     OUT, the PMC could have been released on another CPU.  Therefore
 714  *     context switch OUT always looks at the hardware state to turn
 715  *     OFF PMCs and will update a PMC's saved value only if reachable
 716  *     from the target process record.
 717  *
 718  *   - OP PMCRELEASE could be called on a PMC at any time (the PMC could
 719  *     be attached to many processes at the time of the call and could
 720  *     be active on multiple CPUs).
 721  *
 722  *     We prevent further scheduling of the PMC by marking it as in
 723  *     state 'DELETED'.  If the runcount of the PMC is non-zero then
 724  *     this PMC is currently running on a CPU somewhere.  The thread
 725  *     doing the PMCRELEASE operation waits by repeatedly doing a
 726  *     pause() till the runcount comes to zero.
 727  *
 728  * The contents of a PMC descriptor (struct pmc) are protected using
 729  * a spin-mutex.  In order to save space, we use a mutex pool.
 730  *
 731  * In terms of lock types used by witness(4), we use:
 732  * - Type "pmc-sx", used by the global SX lock.
 733  * - Type "pmc-sleep", for sleep mutexes used by logger threads.
 734  * - Type "pmc-per-proc", for protecting PMC owner descriptors.
 735  * - Type "pmc-leaf", used for all other spin mutexes.
 736  */
 737
 738 /*
 739  * save the cpu binding of the current kthread
 740  */
 741
 742 static void
 743 pmc_save_cpu_binding(struct pmc_binding *pb)
 744 {
 745         PMCDBG0(CPU,BND,2, "save-cpu");
 746         thread_lock(curthread);
 747         pb->pb_bound = sched_is_bound(curthread);
 748         pb->pb_cpu   = curthread->td_oncpu;
 749         thread_unlock(curthread);
 750         PMCDBG1(CPU,BND,2, "save-cpu cpu=%d", pb->pb_cpu);
 751 }
 752
 753 /*
 754  * restore the cpu binding of the current thread
 755  */
 756
 757 static void
 758 pmc_restore_cpu_binding(struct pmc_binding *pb)
 759 {
 760         PMCDBG2(CPU,BND,2, "restore-cpu curcpu=%d restore=%d",
 761             curthread->td_oncpu, pb->pb_cpu);
 762         thread_lock(curthread);
 763         if (pb->pb_bound)
 764                 sched_bind(curthread, pb->pb_cpu);
 765         else
 766                 sched_unbind(curthread);
 767         thread_unlock(curthread);
 768         PMCDBG0(CPU,BND,2, "restore-cpu done");
 769 }
 770
 771 /*
 772  * move execution over the specified cpu and bind it there.
 773  */
 774
 775 static void
 776 pmc_select_cpu(int cpu)
 777 {
 778         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 779             ("[pmc,%d] bad cpu number %d", __LINE__, cpu));
 780
 781         /* Never move to an inactive CPU. */
 782         KASSERT(pmc_cpu_is_active(cpu), ("[pmc,%d] selecting inactive "
 783             "CPU %d", __LINE__, cpu));
 784
 785         PMCDBG1(CPU,SEL,2, "select-cpu cpu=%d", cpu);
 786         thread_lock(curthread);
 787         sched_bind(curthread, cpu);
 788         thread_unlock(curthread);
 789
 790         KASSERT(curthread->td_oncpu == cpu,
 791             ("[pmc,%d] CPU not bound [cpu=%d, curr=%d]", __LINE__,
 792                 cpu, curthread->td_oncpu));
 793
 794         PMCDBG1(CPU,SEL,2, "select-cpu cpu=%d ok", cpu);
 795 }
 796
 797 /*
 798  * Force a context switch.
 799  *
 800  * We do this by pause'ing for 1 tick -- invoking mi_switch() is not
 801  * guaranteed to force a context switch.
 802  */
 803
 804 static void
 805 pmc_force_context_switch(void)
 806 {
 807
 808         pause("pmcctx", 1);
 809 }
 810
 811 uint64_t
 812 pmc_rdtsc(void)
 813 {
 814 #if defined(__i386__) || defined(__amd64__)
 815         if (__predict_true(amd_feature & AMDID_RDTSCP))
 816                 return rdtscp();
 817         else
 818                 return rdtsc();
 819 #else
 820         return get_cyclecount();
 821 #endif
 822 }
 823
 824 /*
 825  * Get the file name for an executable.  This is a simple wrapper
 826  * around vn_fullpath(9).
 827  */
 828
 829 static void
 830 pmc_getfilename(struct vnode *v, char **fullpath, char **freepath)
 831 {
 832
 833         *fullpath = "unknown";
 834         *freepath = NULL;
 835         vn_fullpath(v, fullpath, freepath);
 836 }
 837
 838 /*
 839  * remove an process owning PMCs
 840  */
 841
 842 void
 843 pmc_remove_owner(struct pmc_owner *po)
 844 {
 845         struct pmc *pm, *tmp;
 846
 847         sx_assert(&pmc_sx, SX_XLOCKED);
 848
 849         PMCDBG1(OWN,ORM,1, "remove-owner po=%p", po);
 850
 851         /* Remove descriptor from the owner hash table */
 852         LIST_REMOVE(po, po_next);
 853
 854         /* release all owned PMC descriptors */
 855         LIST_FOREACH_SAFE(pm, &po->po_pmcs, pm_next, tmp) {
 856                 PMCDBG1(OWN,ORM,2, "pmc=%p", pm);
 857                 KASSERT(pm->pm_owner == po,
 858                     ("[pmc,%d] owner %p != po %p", __LINE__, pm->pm_owner, po));
 859
 860                 pmc_release_pmc_descriptor(pm); /* will unlink from the list */
 861                 pmc_destroy_pmc_descriptor(pm);
 862         }
 863
 864         KASSERT(po->po_sscount == 0,
 865             ("[pmc,%d] SS count not zero", __LINE__));
 866         KASSERT(LIST_EMPTY(&po->po_pmcs),
 867             ("[pmc,%d] PMC list not empty", __LINE__));
 868
 869         /* de-configure the log file if present */
 870         if (po->po_flags & PMC_PO_OWNS_LOGFILE)
 871                 pmclog_deconfigure_log(po);
 872 }
 873
 874 /*
 875  * remove an owner process record if all conditions are met.
 876  */
 877
 878 static void
 879 pmc_maybe_remove_owner(struct pmc_owner *po)
 880 {
 881
 882         PMCDBG1(OWN,OMR,1, "maybe-remove-owner po=%p", po);
 883
 884         /*
 885          * Remove owner record if
 886          * - this process does not own any PMCs
 887          * - this process has not allocated a system-wide sampling buffer
 888          */
 889
 890         if (LIST_EMPTY(&po->po_pmcs) &&
 891             ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)) {
 892                 pmc_remove_owner(po);
 893                 pmc_destroy_owner_descriptor(po);
 894         }
 895 }
 896
 897 /*
 898  * Add an association between a target process and a PMC.
 899  */
 900
 901 static void
 902 pmc_link_target_process(struct pmc *pm, struct pmc_process *pp)
 903 {
 904         int ri;
 905         struct pmc_target *pt;
 906 #ifdef INVARIANTS
 907         struct pmc_thread *pt_td;
 908 #endif
 909
 910         sx_assert(&pmc_sx, SX_XLOCKED);
 911
 912         KASSERT(pm != NULL && pp != NULL,
 913             ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
 914         KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)),
 915             ("[pmc,%d] Attaching a non-process-virtual pmc=%p to pid=%d",
 916                 __LINE__, pm, pp->pp_proc->p_pid));
 917         KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= ((int) md->pmd_npmc - 1),
 918             ("[pmc,%d] Illegal reference count %d for process record %p",
 919                 __LINE__, pp->pp_refcnt, (void *) pp));
 920
 921         ri = PMC_TO_ROWINDEX(pm);
 922
 923         PMCDBG3(PRC,TLK,1, "link-target pmc=%p ri=%d pmc-process=%p",
 924             pm, ri, pp);
 925
 926 #ifdef  HWPMC_DEBUG
 927         LIST_FOREACH(pt, &pm->pm_targets, pt_next)
 928             if (pt->pt_process == pp)
 929                     KASSERT(0, ("[pmc,%d] pp %p already in pmc %p targets",
 930                                 __LINE__, pp, pm));
 931 #endif
 932
 933         pt = malloc(sizeof(struct pmc_target), M_PMC, M_WAITOK|M_ZERO);
 934         pt->pt_process = pp;
 935
 936         LIST_INSERT_HEAD(&pm->pm_targets, pt, pt_next);
 937
 938         atomic_store_rel_ptr((uintptr_t *)&pp->pp_pmcs[ri].pp_pmc,
 939             (uintptr_t)pm);
 940
 941         if (pm->pm_owner->po_owner == pp->pp_proc)
 942                 pm->pm_flags |= PMC_F_ATTACHED_TO_OWNER;
 943
 944         /*
 945          * Initialize the per-process values at this row index.
 946          */
 947         pp->pp_pmcs[ri].pp_pmcval = PMC_TO_MODE(pm) == PMC_MODE_TS ?
 948             pm->pm_sc.pm_reloadcount : 0;
 949
 950         pp->pp_refcnt++;
 951
 952 #ifdef INVARIANTS
 953         /* Confirm that the per-thread values at this row index are cleared. */
 954         if (PMC_TO_MODE(pm) == PMC_MODE_TS) {
 955                 mtx_lock_spin(pp->pp_tdslock);
 956                 LIST_FOREACH(pt_td, &pp->pp_tds, pt_next) {
 957                         KASSERT(pt_td->pt_pmcs[ri].pt_pmcval == (pmc_value_t) 0,
 958                             ("[pmc,%d] pt_pmcval not cleared for pid=%d at "
 959                             "ri=%d", __LINE__, pp->pp_proc->p_pid, ri));
 960                 }
 961                 mtx_unlock_spin(pp->pp_tdslock);
 962         }
 963 #endif
 964 }
 965
 966 /*
 967  * Removes the association between a target process and a PMC.
 968  */
 969
 970 static void
 971 pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp)
 972 {
 973         int ri;
 974         struct proc *p;
 975         struct pmc_target *ptgt;
 976         struct pmc_thread *pt;
 977
 978         sx_assert(&pmc_sx, SX_XLOCKED);
 979
 980         KASSERT(pm != NULL && pp != NULL,
 981             ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
 982
 983         KASSERT(pp->pp_refcnt >= 1 && pp->pp_refcnt <= (int) md->pmd_npmc,
 984             ("[pmc,%d] Illegal ref count %d on process record %p",
 985                 __LINE__, pp->pp_refcnt, (void *) pp));
 986
 987         ri = PMC_TO_ROWINDEX(pm);
 988
 989         PMCDBG3(PRC,TUL,1, "unlink-target pmc=%p ri=%d pmc-process=%p",
 990             pm, ri, pp);
 991
 992         KASSERT(pp->pp_pmcs[ri].pp_pmc == pm,
 993             ("[pmc,%d] PMC ri %d mismatch pmc %p pp->[ri] %p", __LINE__,
 994                 ri, pm, pp->pp_pmcs[ri].pp_pmc));
 995
 996         pp->pp_pmcs[ri].pp_pmc = NULL;
 997         pp->pp_pmcs[ri].pp_pmcval = (pmc_value_t) 0;
 998
 999         /* Clear the per-thread values at this row index. */
1000         if (PMC_TO_MODE(pm) == PMC_MODE_TS) {
1001                 mtx_lock_spin(pp->pp_tdslock);
1002                 LIST_FOREACH(pt, &pp->pp_tds, pt_next)
1003                         pt->pt_pmcs[ri].pt_pmcval = (pmc_value_t) 0;
1004                 mtx_unlock_spin(pp->pp_tdslock);
1005         }
1006
1007         /* Remove owner-specific flags */
1008         if (pm->pm_owner->po_owner == pp->pp_proc) {
1009                 pp->pp_flags &= ~PMC_PP_ENABLE_MSR_ACCESS;
1010                 pm->pm_flags &= ~PMC_F_ATTACHED_TO_OWNER;
1011         }
1012
1013         pp->pp_refcnt--;
1014
1015         /* Remove the target process from the PMC structure */
1016         LIST_FOREACH(ptgt, &pm->pm_targets, pt_next)
1017                 if (ptgt->pt_process == pp)
1018                         break;
1019
1020         KASSERT(ptgt != NULL, ("[pmc,%d] process %p (pp: %p) not found "
1021                     "in pmc %p", __LINE__, pp->pp_proc, pp, pm));
1022
1023         LIST_REMOVE(ptgt, pt_next);
1024         free(ptgt, M_PMC);
1025
1026         /* if the PMC now lacks targets, send the owner a SIGIO */
1027         if (LIST_EMPTY(&pm->pm_targets)) {
1028                 p = pm->pm_owner->po_owner;
1029                 PROC_LOCK(p);
1030                 kern_psignal(p, SIGIO);
1031                 PROC_UNLOCK(p);
1032
1033                 PMCDBG2(PRC,SIG,2, "signalling proc=%p signal=%d", p,
1034                     SIGIO);
1035         }
1036 }
1037
1038 /*
1039  * Check if PMC 'pm' may be attached to target process 't'.
1040  */
1041
1042 static int
1043 pmc_can_attach(struct pmc *pm, struct proc *t)
1044 {
1045         struct proc *o;         /* pmc owner */
1046         struct ucred *oc, *tc;  /* owner, target credentials */
1047         int decline_attach, i;
1048
1049         /*
1050          * A PMC's owner can always attach that PMC to itself.
1051          */
1052
1053         if ((o = pm->pm_owner->po_owner) == t)
1054                 return 0;
1055
1056         PROC_LOCK(o);
1057         oc = o->p_ucred;
1058         crhold(oc);
1059         PROC_UNLOCK(o);
1060
1061         PROC_LOCK(t);
1062         tc = t->p_ucred;
1063         crhold(tc);
1064         PROC_UNLOCK(t);
1065
1066         /*
1067          * The effective uid of the PMC owner should match at least one
1068          * of the {effective,real,saved} uids of the target process.
1069          */
1070
1071         decline_attach = oc->cr_uid != tc->cr_uid &&
1072             oc->cr_uid != tc->cr_svuid &&
1073             oc->cr_uid != tc->cr_ruid;
1074
1075         /*
1076          * Every one of the target's group ids, must be in the owner's
1077          * group list.
1078          */
1079         for (i = 0; !decline_attach && i < tc->cr_ngroups; i++)
1080                 decline_attach = !groupmember(tc->cr_groups[i], oc);
1081
1082         /* check the read and saved gids too */
1083         if (decline_attach == 0)
1084                 decline_attach = !groupmember(tc->cr_rgid, oc) ||
1085                     !groupmember(tc->cr_svgid, oc);
1086
1087         crfree(tc);
1088         crfree(oc);
1089
1090         return !decline_attach;
1091 }
1092
1093 /*
1094  * Attach a process to a PMC.
1095  */
1096
1097 static int
1098 pmc_attach_one_process(struct proc *p, struct pmc *pm)
1099 {
1100         int ri, error;
1101         char *fullpath, *freepath;
1102         struct pmc_process      *pp;
1103
1104         sx_assert(&pmc_sx, SX_XLOCKED);
1105
1106         PMCDBG5(PRC,ATT,2, "attach-one pm=%p ri=%d proc=%p (%d, %s)", pm,
1107             PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
1108
1109         /*
1110          * Locate the process descriptor corresponding to process 'p',
1111          * allocating space as needed.
1112          *
1113          * Verify that rowindex 'pm_rowindex' is free in the process
1114          * descriptor.
1115          *
1116          * If not, allocate space for a descriptor and link the
1117          * process descriptor and PMC.
1118          */
1119         ri = PMC_TO_ROWINDEX(pm);
1120
1121         /* mark process as using HWPMCs */
1122         PROC_LOCK(p);
1123         p->p_flag |= P_HWPMC;
1124         PROC_UNLOCK(p);
1125
1126         if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_ALLOCATE)) == NULL) {
1127                 error = ENOMEM;
1128                 goto fail;
1129         }
1130
1131         if (pp->pp_pmcs[ri].pp_pmc == pm) {/* already present at slot [ri] */
1132                 error = EEXIST;
1133                 goto fail;
1134         }
1135
1136         if (pp->pp_pmcs[ri].pp_pmc != NULL) {
1137                 error = EBUSY;
1138                 goto fail;
1139         }
1140
1141         pmc_link_target_process(pm, pp);
1142
1143         if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) &&
1144             (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) == 0)
1145                 pm->pm_flags |= PMC_F_NEEDS_LOGFILE;
1146
1147         pm->pm_flags |= PMC_F_ATTACH_DONE; /* mark as attached */
1148
1149         /* issue an attach event to a configured log file */
1150         if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE) {
1151                 if (p->p_flag & P_KPROC) {
1152                         fullpath = kernelname;
1153                         freepath = NULL;
1154                 } else {
1155                         pmc_getfilename(p->p_textvp, &fullpath, &freepath);
1156                         pmclog_process_pmcattach(pm, p->p_pid, fullpath);
1157                 }
1158                 free(freepath, M_TEMP);
1159                 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1160                         pmc_log_process_mappings(pm->pm_owner, p);
1161         }
1162
1163         return (0);
1164  fail:
1165         PROC_LOCK(p);
1166         p->p_flag &= ~P_HWPMC;
1167         PROC_UNLOCK(p);
1168         return (error);
1169 }
1170
1171 /*
1172  * Attach a process and optionally its children
1173  */
1174
1175 static int
1176 pmc_attach_process(struct proc *p, struct pmc *pm)
1177 {
1178         int error;
1179         struct proc *top;
1180
1181         sx_assert(&pmc_sx, SX_XLOCKED);
1182
1183         PMCDBG5(PRC,ATT,1, "attach pm=%p ri=%d proc=%p (%d, %s)", pm,
1184             PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
1185
1186
1187         /*
1188          * If this PMC successfully allowed a GETMSR operation
1189          * in the past, disallow further ATTACHes.
1190          */
1191
1192         if ((pm->pm_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0)
1193                 return EPERM;
1194
1195         if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
1196                 return pmc_attach_one_process(p, pm);
1197
1198         /*
1199          * Traverse all child processes, attaching them to
1200          * this PMC.
1201          */
1202
1203         sx_slock(&proctree_lock);
1204
1205         top = p;
1206
1207         for (;;) {
1208                 if ((error = pmc_attach_one_process(p, pm)) != 0)
1209                         break;
1210                 if (!LIST_EMPTY(&p->p_children))
1211                         p = LIST_FIRST(&p->p_children);
1212                 else for (;;) {
1213                         if (p == top)
1214                                 goto done;
1215                         if (LIST_NEXT(p, p_sibling)) {
1216                                 p = LIST_NEXT(p, p_sibling);
1217                                 break;
1218                         }
1219                         p = p->p_pptr;
1220                 }
1221         }
1222
1223         if (error)
1224                 (void) pmc_detach_process(top, pm);
1225
1226  done:
1227         sx_sunlock(&proctree_lock);
1228         return error;
1229 }
1230
1231 /*
1232  * Detach a process from a PMC.  If there are no other PMCs tracking
1233  * this process, remove the process structure from its hash table.  If
1234  * 'flags' contains PMC_FLAG_REMOVE, then free the process structure.
1235  */
1236
1237 static int
1238 pmc_detach_one_process(struct proc *p, struct pmc *pm, int flags)
1239 {
1240         int ri;
1241         struct pmc_process *pp;
1242
1243         sx_assert(&pmc_sx, SX_XLOCKED);
1244
1245         KASSERT(pm != NULL,
1246             ("[pmc,%d] null pm pointer", __LINE__));
1247
1248         ri = PMC_TO_ROWINDEX(pm);
1249
1250         PMCDBG6(PRC,ATT,2, "detach-one pm=%p ri=%d proc=%p (%d, %s) flags=0x%x",
1251             pm, ri, p, p->p_pid, p->p_comm, flags);
1252
1253         if ((pp = pmc_find_process_descriptor(p, 0)) == NULL)
1254                 return ESRCH;
1255
1256         if (pp->pp_pmcs[ri].pp_pmc != pm)
1257                 return EINVAL;
1258
1259         pmc_unlink_target_process(pm, pp);
1260
1261         /* Issue a detach entry if a log file is configured */
1262         if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE)
1263                 pmclog_process_pmcdetach(pm, p->p_pid);
1264
1265         /*
1266          * If there are no PMCs targeting this process, we remove its
1267          * descriptor from the target hash table and unset the P_HWPMC
1268          * flag in the struct proc.
1269          */
1270         KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= (int) md->pmd_npmc,
1271             ("[pmc,%d] Illegal refcnt %d for process struct %p",
1272                 __LINE__, pp->pp_refcnt, pp));
1273
1274         if (pp->pp_refcnt != 0) /* still a target of some PMC */
1275                 return 0;
1276
1277         pmc_remove_process_descriptor(pp);
1278
1279         if (flags & PMC_FLAG_REMOVE)
1280                 pmc_destroy_process_descriptor(pp);
1281
1282         PROC_LOCK(p);
1283         p->p_flag &= ~P_HWPMC;
1284         PROC_UNLOCK(p);
1285
1286         return 0;
1287 }
1288
1289 /*
1290  * Detach a process and optionally its descendants from a PMC.
1291  */
1292
1293 static int
1294 pmc_detach_process(struct proc *p, struct pmc *pm)
1295 {
1296         struct proc *top;
1297
1298         sx_assert(&pmc_sx, SX_XLOCKED);
1299
1300         PMCDBG5(PRC,ATT,1, "detach pm=%p ri=%d proc=%p (%d, %s)", pm,
1301             PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
1302
1303         if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
1304                 return pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE);
1305
1306         /*
1307          * Traverse all children, detaching them from this PMC.  We
1308          * ignore errors since we could be detaching a PMC from a
1309          * partially attached proc tree.
1310          */
1311
1312         sx_slock(&proctree_lock);
1313
1314         top = p;
1315
1316         for (;;) {
1317                 (void) pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE);
1318
1319                 if (!LIST_EMPTY(&p->p_children))
1320                         p = LIST_FIRST(&p->p_children);
1321                 else for (;;) {
1322                         if (p == top)
1323                                 goto done;
1324                         if (LIST_NEXT(p, p_sibling)) {
1325                                 p = LIST_NEXT(p, p_sibling);
1326                                 break;
1327                         }
1328                         p = p->p_pptr;
1329                 }
1330         }
1331
1332  done:
1333         sx_sunlock(&proctree_lock);
1334
1335         if (LIST_EMPTY(&pm->pm_targets))
1336                 pm->pm_flags &= ~PMC_F_ATTACH_DONE;
1337
1338         return 0;
1339 }
1340
1341
1342 /*
1343  * Thread context switch IN
1344  */
1345
1346 static void
1347 pmc_process_csw_in(struct thread *td)
1348 {
1349         int cpu;
1350         unsigned int adjri, ri;
1351         struct pmc *pm;
1352         struct proc *p;
1353         struct pmc_cpu *pc;
1354         struct pmc_hw *phw;
1355         pmc_value_t newvalue;
1356         struct pmc_process *pp;
1357         struct pmc_thread *pt;
1358         struct pmc_classdep *pcd;
1359
1360         p = td->td_proc;
1361         pt = NULL;
1362         if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE)) == NULL)
1363                 return;
1364
1365         KASSERT(pp->pp_proc == td->td_proc,
1366             ("[pmc,%d] not my thread state", __LINE__));
1367
1368         critical_enter(); /* no preemption from this point */
1369
1370         cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */
1371
1372         PMCDBG5(CSW,SWI,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p,
1373             p->p_pid, p->p_comm, pp);
1374
1375         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
1376             ("[pmc,%d] weird CPU id %d", __LINE__, cpu));
1377
1378         pc = pmc_pcpu[cpu];
1379
1380         for (ri = 0; ri < md->pmd_npmc; ri++) {
1381
1382                 if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL)
1383                         continue;
1384
1385                 KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)),
1386                     ("[pmc,%d] Target PMC in non-virtual mode (%d)",
1387                         __LINE__, PMC_TO_MODE(pm)));
1388
1389                 KASSERT(PMC_TO_ROWINDEX(pm) == ri,
1390                     ("[pmc,%d] Row index mismatch pmc %d != ri %d",
1391                         __LINE__, PMC_TO_ROWINDEX(pm), ri));
1392
1393                 /*
1394                  * Only PMCs that are marked as 'RUNNING' need
1395                  * be placed on hardware.
1396                  */
1397
1398                 if (pm->pm_state != PMC_STATE_RUNNING)
1399                         continue;
1400
1401                 KASSERT(counter_u64_fetch(pm->pm_runcount) >= 0,
1402             ("[pmc,%d] pm=%p runcount %ld", __LINE__, (void *) pm,
1403                  (unsigned long)counter_u64_fetch(pm->pm_runcount)));
1404
1405                 /* increment PMC runcount */
1406                 counter_u64_add(pm->pm_runcount, 1);
1407
1408                 /* configure the HWPMC we are going to use. */
1409                 pcd = pmc_ri_to_classdep(md, ri, &adjri);
1410                 pcd->pcd_config_pmc(cpu, adjri, pm);
1411
1412                 phw = pc->pc_hwpmcs[ri];
1413
1414                 KASSERT(phw != NULL,
1415                     ("[pmc,%d] null hw pointer", __LINE__));
1416
1417                 KASSERT(phw->phw_pmc == pm,
1418                     ("[pmc,%d] hw->pmc %p != pmc %p", __LINE__,
1419                         phw->phw_pmc, pm));
1420
1421                 /*
1422                  * Write out saved value and start the PMC.
1423                  *
1424                  * Sampling PMCs use a per-thread value, while
1425                  * counting mode PMCs use a per-pmc value that is
1426                  * inherited across descendants.
1427                  */
1428                 if (PMC_TO_MODE(pm) == PMC_MODE_TS) {
1429                         if (pt == NULL)
1430                                 pt = pmc_find_thread_descriptor(pp, td,
1431                                     PMC_FLAG_NONE);
1432
1433                         KASSERT(pt != NULL,
1434                             ("[pmc,%d] No thread found for td=%p", __LINE__,
1435                             td));
1436
1437                         mtx_pool_lock_spin(pmc_mtxpool, pm);
1438
1439                         /*
1440                          * If we have a thread descriptor, use the per-thread
1441                          * counter in the descriptor. If not, we will use
1442                          * a per-process counter.
1443                          *
1444                          * TODO: Remove the per-process "safety net" once
1445                          * we have thoroughly tested that we don't hit the
1446                          * above assert.
1447                          */
1448                         if (pt != NULL) {
1449                                 if (pt->pt_pmcs[ri].pt_pmcval > 0)
1450                                         newvalue = pt->pt_pmcs[ri].pt_pmcval;
1451                                 else
1452                                         newvalue = pm->pm_sc.pm_reloadcount;
1453                         } else {
1454                                 /*
1455                                  * Use the saved value calculated after the most
1456                                  * recent time a thread using the shared counter
1457                                  * switched out. Reset the saved count in case
1458                                  * another thread from this process switches in
1459                                  * before any threads switch out.
1460                                  */
1461
1462                                 newvalue = pp->pp_pmcs[ri].pp_pmcval;
1463                                 pp->pp_pmcs[ri].pp_pmcval =
1464                                     pm->pm_sc.pm_reloadcount;
1465                         }
1466                         mtx_pool_unlock_spin(pmc_mtxpool, pm);
1467                         KASSERT(newvalue > 0 && newvalue <=
1468                             pm->pm_sc.pm_reloadcount,
1469                             ("[pmc,%d] pmcval outside of expected range cpu=%d "
1470                             "ri=%d pmcval=%jx pm_reloadcount=%jx", __LINE__,
1471                             cpu, ri, newvalue, pm->pm_sc.pm_reloadcount));
1472                 } else {
1473                         KASSERT(PMC_TO_MODE(pm) == PMC_MODE_TC,
1474                             ("[pmc,%d] illegal mode=%d", __LINE__,
1475                             PMC_TO_MODE(pm)));
1476                         mtx_pool_lock_spin(pmc_mtxpool, pm);
1477                         newvalue = PMC_PCPU_SAVED(cpu, ri) =
1478                             pm->pm_gv.pm_savedvalue;
1479                         mtx_pool_unlock_spin(pmc_mtxpool, pm);
1480                 }
1481
1482                 PMCDBG3(CSW,SWI,1,"cpu=%d ri=%d new=%jd", cpu, ri, newvalue);
1483
1484                 pcd->pcd_write_pmc(cpu, adjri, newvalue);
1485
1486                 /* If a sampling mode PMC, reset stalled state. */
1487                 if (PMC_TO_MODE(pm) == PMC_MODE_TS)
1488                         pm->pm_pcpu_state[cpu].pps_stalled = 0;
1489
1490                 /* Indicate that we desire this to run. */
1491                 pm->pm_pcpu_state[cpu].pps_cpustate = 1;
1492
1493                 /* Start the PMC. */
1494                 pcd->pcd_start_pmc(cpu, adjri);
1495         }
1496
1497         /*
1498          * perform any other architecture/cpu dependent thread
1499          * switch-in actions.
1500          */
1501
1502         (void) (*md->pmd_switch_in)(pc, pp);
1503
1504         critical_exit();
1505
1506 }
1507
1508 /*
1509  * Thread context switch OUT.
1510  */
1511
1512 static void
1513 pmc_process_csw_out(struct thread *td)
1514 {
1515         int cpu;
1516         int64_t tmp;
1517         struct pmc *pm;
1518         struct proc *p;
1519         enum pmc_mode mode;
1520         struct pmc_cpu *pc;
1521         pmc_value_t newvalue;
1522         unsigned int adjri, ri;
1523         struct pmc_process *pp;
1524         struct pmc_thread *pt = NULL;
1525         struct pmc_classdep *pcd;
1526
1527
1528         /*
1529          * Locate our process descriptor; this may be NULL if
1530          * this process is exiting and we have already removed
1531          * the process from the target process table.
1532          *
1533          * Note that due to kernel preemption, multiple
1534          * context switches may happen while the process is
1535          * exiting.
1536          *
1537          * Note also that if the target process cannot be
1538          * found we still need to deconfigure any PMCs that
1539          * are currently running on hardware.
1540          */
1541
1542         p = td->td_proc;
1543         pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE);
1544
1545         /*
1546          * save PMCs
1547          */
1548
1549         critical_enter();
1550
1551         cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */
1552
1553         PMCDBG5(CSW,SWO,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p,
1554             p->p_pid, p->p_comm, pp);
1555
1556         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
1557             ("[pmc,%d weird CPU id %d", __LINE__, cpu));
1558
1559         pc = pmc_pcpu[cpu];
1560
1561         /*
1562          * When a PMC gets unlinked from a target PMC, it will
1563          * be removed from the target's pp_pmc[] array.
1564          *
1565          * However, on a MP system, the target could have been
1566          * executing on another CPU at the time of the unlink.
1567          * So, at context switch OUT time, we need to look at
1568          * the hardware to determine if a PMC is scheduled on
1569          * it.
1570          */
1571
1572         for (ri = 0; ri < md->pmd_npmc; ri++) {
1573
1574                 pcd = pmc_ri_to_classdep(md, ri, &adjri);
1575                 pm  = NULL;
1576                 (void) (*pcd->pcd_get_config)(cpu, adjri, &pm);
1577
1578                 if (pm == NULL) /* nothing at this row index */
1579                         continue;
1580
1581                 mode = PMC_TO_MODE(pm);
1582                 if (!PMC_IS_VIRTUAL_MODE(mode))
1583                         continue; /* not a process virtual PMC */
1584
1585                 KASSERT(PMC_TO_ROWINDEX(pm) == ri,
1586                     ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
1587                         __LINE__, PMC_TO_ROWINDEX(pm), ri));
1588
1589                 /*
1590                  * Change desired state, and then stop if not stalled.
1591                  * This two-step dance should avoid race conditions where
1592                  * an interrupt re-enables the PMC after this code has
1593                  * already checked the pm_stalled flag.
1594                  */
1595                 pm->pm_pcpu_state[cpu].pps_cpustate = 0;
1596                 if (pm->pm_pcpu_state[cpu].pps_stalled == 0)
1597                         pcd->pcd_stop_pmc(cpu, adjri);
1598
1599                 KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
1600                         ("[pmc,%d] pm=%p runcount %ld", __LINE__, (void *) pm,
1601                          (unsigned long)counter_u64_fetch(pm->pm_runcount)));
1602
1603                 /* reduce this PMC's runcount */
1604                 counter_u64_add(pm->pm_runcount, -1);
1605
1606                 /*
1607                  * If this PMC is associated with this process,
1608                  * save the reading.
1609                  */
1610
1611                 if (pm->pm_state != PMC_STATE_DELETED && pp != NULL &&
1612                     pp->pp_pmcs[ri].pp_pmc != NULL) {
1613                         KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
1614                             ("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__,
1615                                 pm, ri, pp->pp_pmcs[ri].pp_pmc));
1616
1617                         KASSERT(pp->pp_refcnt > 0,
1618                             ("[pmc,%d] pp refcnt = %d", __LINE__,
1619                                 pp->pp_refcnt));
1620
1621                         pcd->pcd_read_pmc(cpu, adjri, &newvalue);
1622
1623                         if (mode == PMC_MODE_TS) {
1624                                 PMCDBG3(CSW,SWO,1,"cpu=%d ri=%d val=%jd (samp)",
1625                                     cpu, ri, newvalue);
1626
1627                                 if (pt == NULL)
1628                                         pt = pmc_find_thread_descriptor(pp, td,
1629                                             PMC_FLAG_NONE);
1630
1631                                 KASSERT(pt != NULL,
1632                                     ("[pmc,%d] No thread found for td=%p",
1633                                     __LINE__, td));
1634
1635                                 mtx_pool_lock_spin(pmc_mtxpool, pm);
1636
1637                                 /*
1638                                  * If we have a thread descriptor, save the
1639                                  * per-thread counter in the descriptor. If not,
1640                                  * we will update the per-process counter.
1641                                  *
1642                                  * TODO: Remove the per-process "safety net"
1643                                  * once we have thoroughly tested that we
1644                                  * don't hit the above assert.
1645                                  */
1646                                 if (pt != NULL)
1647                                         pt->pt_pmcs[ri].pt_pmcval = newvalue;
1648                                 else {
1649                                         /*
1650                                          * For sampling process-virtual PMCs,
1651                                          * newvalue is the number of events to
1652                                          * be seen until the next sampling
1653                                          * interrupt. We can just add the events
1654                                          * left from this invocation to the
1655                                          * counter, then adjust in case we
1656                                          * overflow our range.
1657                                          *
1658                                          * (Recall that we reload the counter
1659                                          * every time we use it.)
1660                                          */
1661                                         pp->pp_pmcs[ri].pp_pmcval += newvalue;
1662                                         if (pp->pp_pmcs[ri].pp_pmcval >
1663                                             pm->pm_sc.pm_reloadcount)
1664                                                 pp->pp_pmcs[ri].pp_pmcval -=
1665                                                     pm->pm_sc.pm_reloadcount;
1666                                 }
1667                                 mtx_pool_unlock_spin(pmc_mtxpool, pm);
1668                         } else {
1669                                 tmp = newvalue - PMC_PCPU_SAVED(cpu,ri);
1670
1671                                 PMCDBG3(CSW,SWO,1,"cpu=%d ri=%d tmp=%jd (count)",
1672                                     cpu, ri, tmp);
1673
1674                                 /*
1675                                  * For counting process-virtual PMCs,
1676                                  * we expect the count to be
1677                                  * increasing monotonically, modulo a 64
1678                                  * bit wraparound.
1679                                  */
1680                                 KASSERT(tmp >= 0,
1681                                     ("[pmc,%d] negative increment cpu=%d "
1682                                      "ri=%d newvalue=%jx saved=%jx "
1683                                      "incr=%jx", __LINE__, cpu, ri,
1684                                      newvalue, PMC_PCPU_SAVED(cpu,ri), tmp));
1685
1686                                 mtx_pool_lock_spin(pmc_mtxpool, pm);
1687                                 pm->pm_gv.pm_savedvalue += tmp;
1688                                 pp->pp_pmcs[ri].pp_pmcval += tmp;
1689                                 mtx_pool_unlock_spin(pmc_mtxpool, pm);
1690
1691                                 if (pm->pm_flags & PMC_F_LOG_PROCCSW)
1692                                         pmclog_process_proccsw(pm, pp, tmp, td);
1693                         }
1694                 }
1695
1696                 /* mark hardware as free */
1697                 pcd->pcd_config_pmc(cpu, adjri, NULL);
1698         }
1699
1700         /*
1701          * perform any other architecture/cpu dependent thread
1702          * switch out functions.
1703          */
1704
1705         (void) (*md->pmd_switch_out)(pc, pp);
1706
1707         critical_exit();
1708 }
1709
1710 /*
1711  * A new thread for a process.
1712  */
1713 static void
1714 pmc_process_thread_add(struct thread *td)
1715 {
1716         struct pmc_process *pmc;
1717
1718         pmc = pmc_find_process_descriptor(td->td_proc, PMC_FLAG_NONE);
1719         if (pmc != NULL)
1720                 pmc_find_thread_descriptor(pmc, td, PMC_FLAG_ALLOCATE);
1721 }
1722
1723 /*
1724  * A thread delete for a process.
1725  */
1726 static void
1727 pmc_process_thread_delete(struct thread *td)
1728 {
1729         struct pmc_process *pmc;
1730
1731         pmc = pmc_find_process_descriptor(td->td_proc, PMC_FLAG_NONE);
1732         if (pmc != NULL)
1733                 pmc_thread_descriptor_pool_free(pmc_find_thread_descriptor(pmc,
1734                     td, PMC_FLAG_REMOVE));
1735 }
1736
1737 /*
1738  * A userret() call for a thread.
1739  */
1740 static void
1741 pmc_process_thread_userret(struct thread *td)
1742 {
1743         sched_pin();
1744         pmc_capture_user_callchain(curcpu, PMC_UR, td->td_frame);
1745         sched_unpin();
1746 }
1747
1748 /*
1749  * A mapping change for a process.
1750  */
1751
1752 static void
1753 pmc_process_mmap(struct thread *td, struct pmckern_map_in *pkm)
1754 {
1755         int ri;
1756         pid_t pid;
1757         char *fullpath, *freepath;
1758         const struct pmc *pm;
1759         struct pmc_owner *po;
1760         const struct pmc_process *pp;
1761
1762         freepath = fullpath = NULL;
1763         MPASS(!in_epoch(global_epoch_preempt));
1764         pmc_getfilename((struct vnode *) pkm->pm_file, &fullpath, &freepath);
1765
1766         pid = td->td_proc->p_pid;
1767
1768         PMC_EPOCH_ENTER();
1769         /* Inform owners of all system-wide sampling PMCs. */
1770         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
1771             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
1772                         pmclog_process_map_in(po, pid, pkm->pm_address, fullpath);
1773
1774         if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL)
1775                 goto done;
1776
1777         /*
1778          * Inform sampling PMC owners tracking this process.
1779          */
1780         for (ri = 0; ri < md->pmd_npmc; ri++)
1781                 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL &&
1782                     PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1783                         pmclog_process_map_in(pm->pm_owner,
1784                             pid, pkm->pm_address, fullpath);
1785
1786   done:
1787         if (freepath)
1788                 free(freepath, M_TEMP);
1789         PMC_EPOCH_EXIT();
1790 }
1791
1792
1793 /*
1794  * Log an munmap request.
1795  */
1796
1797 static void
1798 pmc_process_munmap(struct thread *td, struct pmckern_map_out *pkm)
1799 {
1800         int ri;
1801         pid_t pid;
1802         struct pmc_owner *po;
1803         const struct pmc *pm;
1804         const struct pmc_process *pp;
1805
1806         pid = td->td_proc->p_pid;
1807
1808         PMC_EPOCH_ENTER();
1809         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
1810             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
1811                 pmclog_process_map_out(po, pid, pkm->pm_address,
1812                     pkm->pm_address + pkm->pm_size);
1813         PMC_EPOCH_EXIT();
1814
1815         if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL)
1816                 return;
1817
1818         for (ri = 0; ri < md->pmd_npmc; ri++)
1819                 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL &&
1820                     PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1821                         pmclog_process_map_out(pm->pm_owner, pid,
1822                             pkm->pm_address, pkm->pm_address + pkm->pm_size);
1823 }
1824
1825 /*
1826  * Log mapping information about the kernel.
1827  */
1828
1829 static void
1830 pmc_log_kernel_mappings(struct pmc *pm)
1831 {
1832         struct pmc_owner *po;
1833         struct pmckern_map_in *km, *kmbase;
1834
1835         MPASS(in_epoch(global_epoch_preempt) || sx_xlocked(&pmc_sx));
1836         KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
1837             ("[pmc,%d] non-sampling PMC (%p) desires mapping information",
1838                 __LINE__, (void *) pm));
1839
1840         po = pm->pm_owner;
1841
1842         if (po->po_flags & PMC_PO_INITIAL_MAPPINGS_DONE)
1843                 return;
1844         if (PMC_TO_MODE(pm) == PMC_MODE_SS)
1845                 pmc_process_allproc(pm);
1846         /*
1847          * Log the current set of kernel modules.
1848          */
1849         kmbase = linker_hwpmc_list_objects();
1850         for (km = kmbase; km->pm_file != NULL; km++) {
1851                 PMCDBG2(LOG,REG,1,"%s %p", (char *) km->pm_file,
1852                     (void *) km->pm_address);
1853                 pmclog_process_map_in(po, (pid_t) -1, km->pm_address,
1854                     km->pm_file);
1855         }
1856         free(kmbase, M_LINKER);
1857
1858         po->po_flags |= PMC_PO_INITIAL_MAPPINGS_DONE;
1859 }
1860
1861 /*
1862  * Log the mappings for a single process.
1863  */
1864
1865 static void
1866 pmc_log_process_mappings(struct pmc_owner *po, struct proc *p)
1867 {
1868         vm_map_t map;
1869         struct vnode *vp;
1870         struct vmspace *vm;
1871         vm_map_entry_t entry;
1872         vm_offset_t last_end;
1873         u_int last_timestamp;
1874         struct vnode *last_vp;
1875         vm_offset_t start_addr;
1876         vm_object_t obj, lobj, tobj;
1877         char *fullpath, *freepath;
1878
1879         last_vp = NULL;
1880         last_end = (vm_offset_t) 0;
1881         fullpath = freepath = NULL;
1882
1883         if ((vm = vmspace_acquire_ref(p)) == NULL)
1884                 return;
1885
1886         map = &vm->vm_map;
1887         vm_map_lock_read(map);
1888
1889         VM_MAP_ENTRY_FOREACH(entry, map) {
1890
1891                 if (entry == NULL) {
1892                         PMCDBG2(LOG,OPS,2, "hwpmc: vm_map entry unexpectedly "
1893                             "NULL! pid=%d vm_map=%p\n", p->p_pid, map);
1894                         break;
1895                 }
1896
1897                 /*
1898                  * We only care about executable map entries.
1899                  */
1900                 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) ||
1901                     !(entry->protection & VM_PROT_EXECUTE) ||
1902                     (entry->object.vm_object == NULL)) {
1903                         continue;
1904                 }
1905
1906                 obj = entry->object.vm_object;
1907                 VM_OBJECT_RLOCK(obj);
1908
1909                 /*
1910                  * Walk the backing_object list to find the base
1911                  * (non-shadowed) vm_object.
1912                  */
1913                 for (lobj = tobj = obj; tobj != NULL; tobj = tobj->backing_object) {
1914                         if (tobj != obj)
1915                                 VM_OBJECT_RLOCK(tobj);
1916                         if (lobj != obj)
1917                                 VM_OBJECT_RUNLOCK(lobj);
1918                         lobj = tobj;
1919                 }
1920
1921                 /*
1922                  * At this point lobj is the base vm_object and it is locked.
1923                  */
1924                 if (lobj == NULL) {
1925                         PMCDBG3(LOG,OPS,2, "hwpmc: lobj unexpectedly NULL! pid=%d "
1926                             "vm_map=%p vm_obj=%p\n", p->p_pid, map, obj);
1927                         VM_OBJECT_RUNLOCK(obj);
1928                         continue;
1929                 }
1930
1931                 vp = vm_object_vnode(lobj);
1932                 if (vp == NULL) {
1933                         if (lobj != obj)
1934                                 VM_OBJECT_RUNLOCK(lobj);
1935                         VM_OBJECT_RUNLOCK(obj);
1936                         continue;
1937                 }
1938
1939                 /*
1940                  * Skip contiguous regions that point to the same
1941                  * vnode, so we don't emit redundant MAP-IN
1942                  * directives.
1943                  */
1944                 if (entry->start == last_end && vp == last_vp) {
1945                         last_end = entry->end;
1946                         if (lobj != obj)
1947                                 VM_OBJECT_RUNLOCK(lobj);
1948                         VM_OBJECT_RUNLOCK(obj);
1949                         continue;
1950                 }
1951
1952                 /*
1953                  * We don't want to keep the proc's vm_map or this
1954                  * vm_object locked while we walk the pathname, since
1955                  * vn_fullpath() can sleep.  However, if we drop the
1956                  * lock, it's possible for concurrent activity to
1957                  * modify the vm_map list.  To protect against this,
1958                  * we save the vm_map timestamp before we release the
1959                  * lock, and check it after we reacquire the lock
1960                  * below.
1961                  */
1962                 start_addr = entry->start;
1963                 last_end = entry->end;
1964                 last_timestamp = map->timestamp;
1965                 vm_map_unlock_read(map);
1966
1967                 vref(vp);
1968                 if (lobj != obj)
1969                         VM_OBJECT_RUNLOCK(lobj);
1970
1971                 VM_OBJECT_RUNLOCK(obj);
1972
1973                 freepath = NULL;
1974                 pmc_getfilename(vp, &fullpath, &freepath);
1975                 last_vp = vp;
1976
1977                 vrele(vp);
1978
1979                 vp = NULL;
1980                 pmclog_process_map_in(po, p->p_pid, start_addr, fullpath);
1981                 if (freepath)
1982                         free(freepath, M_TEMP);
1983
1984                 vm_map_lock_read(map);
1985
1986                 /*
1987                  * If our saved timestamp doesn't match, this means
1988                  * that the vm_map was modified out from under us and
1989                  * we can't trust our current "entry" pointer.  Do a
1990                  * new lookup for this entry.  If there is no entry
1991                  * for this address range, vm_map_lookup_entry() will
1992                  * return the previous one, so we always want to go to
1993                  * the next entry on the next loop iteration.
1994                  *
1995                  * There is an edge condition here that can occur if
1996                  * there is no entry at or before this address.  In
1997                  * this situation, vm_map_lookup_entry returns
1998                  * &map->header, which would cause our loop to abort
1999                  * without processing the rest of the map.  However,
2000                  * in practice this will never happen for process
2001                  * vm_map.  This is because the executable's text
2002                  * segment is the first mapping in the proc's address
2003                  * space, and this mapping is never removed until the
2004                  * process exits, so there will always be a non-header
2005                  * entry at or before the requested address for
2006                  * vm_map_lookup_entry to return.
2007                  */
2008                 if (map->timestamp != last_timestamp)
2009                         vm_map_lookup_entry(map, last_end - 1, &entry);
2010         }
2011
2012         vm_map_unlock_read(map);
2013         vmspace_free(vm);
2014         return;
2015 }
2016
2017 /*
2018  * Log mappings for all processes in the system.
2019  */
2020
2021 static void
2022 pmc_log_all_process_mappings(struct pmc_owner *po)
2023 {
2024         struct proc *p, *top;
2025
2026         sx_assert(&pmc_sx, SX_XLOCKED);
2027
2028         if ((p = pfind(1)) == NULL)
2029                 panic("[pmc,%d] Cannot find init", __LINE__);
2030
2031         PROC_UNLOCK(p);
2032
2033         sx_slock(&proctree_lock);
2034
2035         top = p;
2036
2037         for (;;) {
2038                 pmc_log_process_mappings(po, p);
2039                 if (!LIST_EMPTY(&p->p_children))
2040                         p = LIST_FIRST(&p->p_children);
2041                 else for (;;) {
2042                         if (p == top)
2043                                 goto done;
2044                         if (LIST_NEXT(p, p_sibling)) {
2045                                 p = LIST_NEXT(p, p_sibling);
2046                                 break;
2047                         }
2048                         p = p->p_pptr;
2049                 }
2050         }
2051  done:
2052         sx_sunlock(&proctree_lock);
2053 }
2054
2055 /*
2056  * The 'hook' invoked from the kernel proper
2057  */
2058
2059
2060 #ifdef  HWPMC_DEBUG
2061 const char *pmc_hooknames[] = {
2062         /* these strings correspond to PMC_FN_* in <sys/pmckern.h> */
2063         "",
2064         "EXEC",
2065         "CSW-IN",
2066         "CSW-OUT",
2067         "SAMPLE",
2068         "UNUSED1",
2069         "UNUSED2",
2070         "MMAP",
2071         "MUNMAP",
2072         "CALLCHAIN-NMI",
2073         "CALLCHAIN-SOFT",
2074         "SOFTSAMPLING",
2075         "THR-CREATE",
2076         "THR-EXIT",
2077         "THR-USERRET",
2078         "THR-CREATE-LOG",
2079         "THR-EXIT-LOG",
2080         "PROC-CREATE-LOG"
2081 };
2082 #endif
2083
2084 static int
2085 pmc_hook_handler(struct thread *td, int function, void *arg)
2086 {
2087         int cpu;
2088
2089         PMCDBG4(MOD,PMH,1, "hook td=%p func=%d \"%s\" arg=%p", td, function,
2090             pmc_hooknames[function], arg);
2091
2092         switch (function)
2093         {
2094
2095         /*
2096          * Process exec()
2097          */
2098
2099         case PMC_FN_PROCESS_EXEC:
2100         {
2101                 char *fullpath, *freepath;
2102                 unsigned int ri;
2103                 int is_using_hwpmcs;
2104                 struct pmc *pm;
2105                 struct proc *p;
2106                 struct pmc_owner *po;
2107                 struct pmc_process *pp;
2108                 struct pmckern_procexec *pk;
2109
2110                 sx_assert(&pmc_sx, SX_XLOCKED);
2111
2112                 p = td->td_proc;
2113                 pmc_getfilename(p->p_textvp, &fullpath, &freepath);
2114
2115                 pk = (struct pmckern_procexec *) arg;
2116
2117                 PMC_EPOCH_ENTER();
2118                 /* Inform owners of SS mode PMCs of the exec event. */
2119                 CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
2120                     if (po->po_flags & PMC_PO_OWNS_LOGFILE)
2121                             pmclog_process_procexec(po, PMC_ID_INVALID,
2122                                 p->p_pid, pk->pm_entryaddr, fullpath);
2123                 PMC_EPOCH_EXIT();
2124
2125                 PROC_LOCK(p);
2126                 is_using_hwpmcs = p->p_flag & P_HWPMC;
2127                 PROC_UNLOCK(p);
2128
2129                 if (!is_using_hwpmcs) {
2130                         if (freepath)
2131                                 free(freepath, M_TEMP);
2132                         break;
2133                 }
2134
2135                 /*
2136                  * PMCs are not inherited across an exec():  remove any
2137                  * PMCs that this process is the owner of.
2138                  */
2139
2140                 if ((po = pmc_find_owner_descriptor(p)) != NULL) {
2141                         pmc_remove_owner(po);
2142                         pmc_destroy_owner_descriptor(po);
2143                 }
2144
2145                 /*
2146                  * If the process being exec'ed is not the target of any
2147                  * PMC, we are done.
2148                  */
2149                 if ((pp = pmc_find_process_descriptor(p, 0)) == NULL) {
2150                         if (freepath)
2151                                 free(freepath, M_TEMP);
2152                         break;
2153                 }
2154
2155                 /*
2156                  * Log the exec event to all monitoring owners.  Skip
2157                  * owners who have already received the event because
2158                  * they had system sampling PMCs active.
2159                  */
2160                 for (ri = 0; ri < md->pmd_npmc; ri++)
2161                         if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) {
2162                                 po = pm->pm_owner;
2163                                 if (po->po_sscount == 0 &&
2164                                     po->po_flags & PMC_PO_OWNS_LOGFILE)
2165                                         pmclog_process_procexec(po, pm->pm_id,
2166                                             p->p_pid, pk->pm_entryaddr,
2167                                             fullpath);
2168                         }
2169
2170                 if (freepath)
2171                         free(freepath, M_TEMP);
2172
2173
2174                 PMCDBG4(PRC,EXC,1, "exec proc=%p (%d, %s) cred-changed=%d",
2175                     p, p->p_pid, p->p_comm, pk->pm_credentialschanged);
2176
2177                 if (pk->pm_credentialschanged == 0) /* no change */
2178                         break;
2179
2180                 /*
2181                  * If the newly exec()'ed process has a different credential
2182                  * than before, allow it to be the target of a PMC only if
2183                  * the PMC's owner has sufficient privilege.
2184                  */
2185
2186                 for (ri = 0; ri < md->pmd_npmc; ri++)
2187                         if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL)
2188                                 if (pmc_can_attach(pm, td->td_proc) != 0)
2189                                         pmc_detach_one_process(td->td_proc,
2190                                             pm, PMC_FLAG_NONE);
2191
2192                 KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= (int) md->pmd_npmc,
2193                     ("[pmc,%d] Illegal ref count %d on pp %p", __LINE__,
2194                         pp->pp_refcnt, pp));
2195
2196                 /*
2197                  * If this process is no longer the target of any
2198                  * PMCs, we can remove the process entry and free
2199                  * up space.
2200                  */
2201
2202                 if (pp->pp_refcnt == 0) {
2203                         pmc_remove_process_descriptor(pp);
2204                         pmc_destroy_process_descriptor(pp);
2205                         break;
2206                 }
2207
2208         }
2209         break;
2210
2211         case PMC_FN_CSW_IN:
2212                 pmc_process_csw_in(td);
2213                 break;
2214
2215         case PMC_FN_CSW_OUT:
2216                 pmc_process_csw_out(td);
2217                 break;
2218
2219         /*
2220          * Process accumulated PC samples.
2221          *
2222          * This function is expected to be called by hardclock() for
2223          * each CPU that has accumulated PC samples.
2224          *
2225          * This function is to be executed on the CPU whose samples
2226          * are being processed.
2227          */
2228         case PMC_FN_DO_SAMPLES:
2229
2230                 /*
2231                  * Clear the cpu specific bit in the CPU mask before
2232                  * do the rest of the processing.  If the NMI handler
2233                  * gets invoked after the "atomic_clear_int()" call
2234                  * below but before "pmc_process_samples()" gets
2235                  * around to processing the interrupt, then we will
2236                  * come back here at the next hardclock() tick (and
2237                  * may find nothing to do if "pmc_process_samples()"
2238                  * had already processed the interrupt).  We don't
2239                  * lose the interrupt sample.
2240                  */
2241                 DPCPU_SET(pmc_sampled, 0);
2242                 cpu = PCPU_GET(cpuid);
2243                 pmc_process_samples(cpu, PMC_HR);
2244                 pmc_process_samples(cpu, PMC_SR);
2245                 pmc_process_samples(cpu, PMC_UR);
2246                 break;
2247
2248         case PMC_FN_MMAP:
2249                 pmc_process_mmap(td, (struct pmckern_map_in *) arg);
2250                 break;
2251
2252         case PMC_FN_MUNMAP:
2253                 MPASS(in_epoch(global_epoch_preempt) || sx_xlocked(&pmc_sx));
2254                 pmc_process_munmap(td, (struct pmckern_map_out *) arg);
2255                 break;
2256
2257         case PMC_FN_PROC_CREATE_LOG:
2258                 pmc_process_proccreate((struct proc *)arg);
2259                 break;
2260
2261         case PMC_FN_USER_CALLCHAIN:
2262                 /*
2263                  * Record a call chain.
2264                  */
2265                 KASSERT(td == curthread, ("[pmc,%d] td != curthread",
2266                     __LINE__));
2267
2268                 pmc_capture_user_callchain(PCPU_GET(cpuid), PMC_HR,
2269                     (struct trapframe *) arg);
2270
2271                 KASSERT(td->td_pinned == 1,
2272                         ("[pmc,%d] invalid td_pinned value", __LINE__));
2273                 sched_unpin();  /* Can migrate safely now. */
2274
2275                 td->td_pflags &= ~TDP_CALLCHAIN;
2276                 break;
2277
2278         case PMC_FN_USER_CALLCHAIN_SOFT:
2279                 /*
2280                  * Record a call chain.
2281                  */
2282                 KASSERT(td == curthread, ("[pmc,%d] td != curthread",
2283                     __LINE__));
2284
2285                 cpu = PCPU_GET(cpuid);
2286                 pmc_capture_user_callchain(cpu, PMC_SR,
2287                     (struct trapframe *) arg);
2288
2289                 KASSERT(td->td_pinned == 1,
2290                     ("[pmc,%d] invalid td_pinned value", __LINE__));
2291
2292                 sched_unpin();  /* Can migrate safely now. */
2293
2294                 td->td_pflags &= ~TDP_CALLCHAIN;
2295                 break;
2296
2297         case PMC_FN_SOFT_SAMPLING:
2298                 /*
2299                  * Call soft PMC sampling intr.
2300                  */
2301                 pmc_soft_intr((struct pmckern_soft *) arg);
2302                 break;
2303
2304         case PMC_FN_THR_CREATE:
2305                 pmc_process_thread_add(td);
2306                 pmc_process_threadcreate(td);
2307                 break;
2308
2309         case PMC_FN_THR_CREATE_LOG:
2310                 pmc_process_threadcreate(td);
2311                 break;
2312
2313         case PMC_FN_THR_EXIT:
2314                 KASSERT(td == curthread, ("[pmc,%d] td != curthread",
2315                     __LINE__));
2316                 pmc_process_thread_delete(td);
2317                 pmc_process_threadexit(td);
2318                 break;
2319         case PMC_FN_THR_EXIT_LOG:
2320                 pmc_process_threadexit(td);
2321                 break;
2322         case PMC_FN_THR_USERRET:
2323                 KASSERT(td == curthread, ("[pmc,%d] td != curthread",
2324                     __LINE__));
2325                 pmc_process_thread_userret(td);
2326                 break;
2327
2328         default:
2329 #ifdef  HWPMC_DEBUG
2330                 KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function));
2331 #endif
2332                 break;
2333
2334         }
2335
2336         return 0;
2337 }
2338
2339 /*
2340  * allocate a 'struct pmc_owner' descriptor in the owner hash table.
2341  */
2342
2343 static struct pmc_owner *
2344 pmc_allocate_owner_descriptor(struct proc *p)
2345 {
2346         uint32_t hindex;
2347         struct pmc_owner *po;
2348         struct pmc_ownerhash *poh;
2349
2350         hindex = PMC_HASH_PTR(p, pmc_ownerhashmask);
2351         poh = &pmc_ownerhash[hindex];
2352
2353         /* allocate space for N pointers and one descriptor struct */
2354         po = malloc(sizeof(struct pmc_owner), M_PMC, M_WAITOK|M_ZERO);
2355         po->po_owner = p;
2356         LIST_INSERT_HEAD(poh, po, po_next); /* insert into hash table */
2357
2358         TAILQ_INIT(&po->po_logbuffers);
2359         mtx_init(&po->po_mtx, "pmc-owner-mtx", "pmc-per-proc", MTX_SPIN);
2360
2361         PMCDBG4(OWN,ALL,1, "allocate-owner proc=%p (%d, %s) pmc-owner=%p",
2362             p, p->p_pid, p->p_comm, po);
2363
2364         return po;
2365 }
2366
2367 static void
2368 pmc_destroy_owner_descriptor(struct pmc_owner *po)
2369 {
2370
2371         PMCDBG4(OWN,REL,1, "destroy-owner po=%p proc=%p (%d, %s)",
2372             po, po->po_owner, po->po_owner->p_pid, po->po_owner->p_comm);
2373
2374         mtx_destroy(&po->po_mtx);
2375         free(po, M_PMC);
2376 }
2377
2378 /*
2379  * Allocate a thread descriptor from the free pool.
2380  *
2381  * NOTE: This *can* return NULL.
2382  */
2383 static struct pmc_thread *
2384 pmc_thread_descriptor_pool_alloc(void)
2385 {
2386         struct pmc_thread *pt;
2387
2388         mtx_lock_spin(&pmc_threadfreelist_mtx);
2389         if ((pt = LIST_FIRST(&pmc_threadfreelist)) != NULL) {
2390                 LIST_REMOVE(pt, pt_next);
2391                 pmc_threadfreelist_entries--;
2392         }
2393         mtx_unlock_spin(&pmc_threadfreelist_mtx);
2394
2395         return (pt);
2396 }
2397
2398 /*
2399  * Add a thread descriptor to the free pool. We use this instead of free()
2400  * to maintain a cache of free entries. Additionally, we can safely call
2401  * this function when we cannot call free(), such as in a critical section.
2402  *
2403  */
2404 static void
2405 pmc_thread_descriptor_pool_free(struct pmc_thread *pt)
2406 {
2407
2408         if (pt == NULL)
2409                 return;
2410
2411         memset(pt, 0, THREADENTRY_SIZE);
2412         mtx_lock_spin(&pmc_threadfreelist_mtx);
2413         LIST_INSERT_HEAD(&pmc_threadfreelist, pt, pt_next);
2414         pmc_threadfreelist_entries++;
2415         if (pmc_threadfreelist_entries > pmc_threadfreelist_max)
2416                 taskqueue_enqueue(taskqueue_fast, &free_task);
2417         mtx_unlock_spin(&pmc_threadfreelist_mtx);
2418 }
2419
2420 /*
2421  * An asynchronous task to manage the free list.
2422  */
2423 static void
2424 pmc_thread_descriptor_pool_free_task(void *arg __unused, int pending __unused)
2425 {
2426         struct pmc_thread *pt;
2427         LIST_HEAD(, pmc_thread) tmplist;
2428         int delta;
2429
2430         LIST_INIT(&tmplist);
2431
2432         /* Determine what changes, if any, we need to make. */
2433         mtx_lock_spin(&pmc_threadfreelist_mtx);
2434         delta = pmc_threadfreelist_entries - pmc_threadfreelist_max;
2435         while (delta > 0 && (pt = LIST_FIRST(&pmc_threadfreelist)) != NULL) {
2436                 delta--;
2437                 pmc_threadfreelist_entries--;
2438                 LIST_REMOVE(pt, pt_next);
2439                 LIST_INSERT_HEAD(&tmplist, pt, pt_next);
2440         }
2441         mtx_unlock_spin(&pmc_threadfreelist_mtx);
2442
2443         /* If there are entries to free, free them. */
2444         while (!LIST_EMPTY(&tmplist)) {
2445                 pt = LIST_FIRST(&tmplist);
2446                 LIST_REMOVE(pt, pt_next);
2447                 free(pt, M_PMC);
2448         }
2449 }
2450
2451 /*
2452  * Drain the thread free pool, freeing all allocations.
2453  */
2454 static void
2455 pmc_thread_descriptor_pool_drain()
2456 {
2457         struct pmc_thread *pt, *next;
2458
2459         LIST_FOREACH_SAFE(pt, &pmc_threadfreelist, pt_next, next) {
2460                 LIST_REMOVE(pt, pt_next);
2461                 free(pt, M_PMC);
2462         }
2463 }
2464
2465 /*
2466  * find the descriptor corresponding to thread 'td', adding or removing it
2467  * as specified by 'mode'.
2468  *
2469  * Note that this supports additional mode flags in addition to those
2470  * supported by pmc_find_process_descriptor():
2471  * PMC_FLAG_NOWAIT: Causes the function to not wait for mallocs.
2472  *     This makes it safe to call while holding certain other locks.
2473  */
2474
2475 static struct pmc_thread *
2476 pmc_find_thread_descriptor(struct pmc_process *pp, struct thread *td,
2477     uint32_t mode)
2478 {
2479         struct pmc_thread *pt = NULL, *ptnew = NULL;
2480         int wait_flag;
2481
2482         KASSERT(td != NULL, ("[pmc,%d] called to add NULL td", __LINE__));
2483
2484         /*
2485          * Pre-allocate memory in the PMC_FLAG_ALLOCATE case prior to
2486          * acquiring the lock.
2487          */
2488         if (mode & PMC_FLAG_ALLOCATE) {
2489                 if ((ptnew = pmc_thread_descriptor_pool_alloc()) == NULL) {
2490                         wait_flag = M_WAITOK;
2491                         if ((mode & PMC_FLAG_NOWAIT) || in_epoch(global_epoch_preempt))
2492                                 wait_flag = M_NOWAIT;
2493
2494                         ptnew = malloc(THREADENTRY_SIZE, M_PMC,
2495                             wait_flag|M_ZERO);
2496                 }
2497         }
2498
2499         mtx_lock_spin(pp->pp_tdslock);
2500
2501         LIST_FOREACH(pt, &pp->pp_tds, pt_next)
2502                 if (pt->pt_td == td)
2503                         break;
2504
2505         if ((mode & PMC_FLAG_REMOVE) && pt != NULL)
2506                 LIST_REMOVE(pt, pt_next);
2507
2508         if ((mode & PMC_FLAG_ALLOCATE) && pt == NULL && ptnew != NULL) {
2509                 pt = ptnew;
2510                 ptnew = NULL;
2511                 pt->pt_td = td;
2512                 LIST_INSERT_HEAD(&pp->pp_tds, pt, pt_next);
2513         }
2514
2515         mtx_unlock_spin(pp->pp_tdslock);
2516
2517         if (ptnew != NULL) {
2518                 free(ptnew, M_PMC);
2519         }
2520
2521         return pt;
2522 }
2523
2524 /*
2525  * Try to add thread descriptors for each thread in a process.
2526  */
2527
2528 static void
2529 pmc_add_thread_descriptors_from_proc(struct proc *p, struct pmc_process *pp)
2530 {
2531         struct thread *curtd;
2532         struct pmc_thread **tdlist;
2533         int i, tdcnt, tdlistsz;
2534
2535         KASSERT(!PROC_LOCKED(p), ("[pmc,%d] proc unexpectedly locked",
2536             __LINE__));
2537         tdcnt = 32;
2538  restart:
2539         tdlistsz = roundup2(tdcnt, 32);
2540
2541         tdcnt = 0;
2542         tdlist = malloc(sizeof(struct pmc_thread*) * tdlistsz, M_TEMP, M_WAITOK);
2543
2544         PROC_LOCK(p);
2545         FOREACH_THREAD_IN_PROC(p, curtd)
2546                 tdcnt++;
2547         if (tdcnt >= tdlistsz) {
2548                 PROC_UNLOCK(p);
2549                 free(tdlist, M_TEMP);
2550                 goto restart;
2551         }
2552         /*
2553          * Try to add each thread to the list without sleeping. If unable,
2554          * add to a queue to retry after dropping the process lock.
2555          */
2556         tdcnt = 0;
2557         FOREACH_THREAD_IN_PROC(p, curtd) {
2558                 tdlist[tdcnt] = pmc_find_thread_descriptor(pp, curtd,
2559                                                    PMC_FLAG_ALLOCATE|PMC_FLAG_NOWAIT);
2560                 if (tdlist[tdcnt] == NULL) {
2561                         PROC_UNLOCK(p);
2562                         for (i = 0; i <= tdcnt; i++)
2563                                 pmc_thread_descriptor_pool_free(tdlist[i]);
2564                         free(tdlist, M_TEMP);
2565                         goto restart;
2566                 }
2567                 tdcnt++;
2568         }
2569         PROC_UNLOCK(p);
2570         free(tdlist, M_TEMP);
2571 }
2572
2573 /*
2574  * find the descriptor corresponding to process 'p', adding or removing it
2575  * as specified by 'mode'.
2576  */
2577
2578 static struct pmc_process *
2579 pmc_find_process_descriptor(struct proc *p, uint32_t mode)
2580 {
2581         uint32_t hindex;
2582         struct pmc_process *pp, *ppnew;
2583         struct pmc_processhash *pph;
2584
2585         hindex = PMC_HASH_PTR(p, pmc_processhashmask);
2586         pph = &pmc_processhash[hindex];
2587
2588         ppnew = NULL;
2589
2590         /*
2591          * Pre-allocate memory in the PMC_FLAG_ALLOCATE case since we
2592          * cannot call malloc(9) once we hold a spin lock.
2593          */
2594         if (mode & PMC_FLAG_ALLOCATE)
2595                 ppnew = malloc(sizeof(struct pmc_process) + md->pmd_npmc *
2596                     sizeof(struct pmc_targetstate), M_PMC, M_WAITOK|M_ZERO);
2597
2598         mtx_lock_spin(&pmc_processhash_mtx);
2599         LIST_FOREACH(pp, pph, pp_next)
2600             if (pp->pp_proc == p)
2601                     break;
2602
2603         if ((mode & PMC_FLAG_REMOVE) && pp != NULL)
2604                 LIST_REMOVE(pp, pp_next);
2605
2606         if ((mode & PMC_FLAG_ALLOCATE) && pp == NULL &&
2607             ppnew != NULL) {
2608                 ppnew->pp_proc = p;
2609                 LIST_INIT(&ppnew->pp_tds);
2610                 ppnew->pp_tdslock = mtx_pool_find(pmc_mtxpool, ppnew);
2611                 LIST_INSERT_HEAD(pph, ppnew, pp_next);
2612                 mtx_unlock_spin(&pmc_processhash_mtx);
2613                 pp = ppnew;
2614                 ppnew = NULL;
2615
2616                 /* Add thread descriptors for this process' current threads. */
2617                 pmc_add_thread_descriptors_from_proc(p, pp);
2618         }
2619         else
2620                 mtx_unlock_spin(&pmc_processhash_mtx);
2621
2622         if (ppnew != NULL)
2623                 free(ppnew, M_PMC);
2624
2625         return pp;
2626 }
2627
2628 /*
2629  * remove a process descriptor from the process hash table.
2630  */
2631
2632 static void
2633 pmc_remove_process_descriptor(struct pmc_process *pp)
2634 {
2635         KASSERT(pp->pp_refcnt == 0,
2636             ("[pmc,%d] Removing process descriptor %p with count %d",
2637                 __LINE__, pp, pp->pp_refcnt));
2638
2639         mtx_lock_spin(&pmc_processhash_mtx);
2640         LIST_REMOVE(pp, pp_next);
2641         mtx_unlock_spin(&pmc_processhash_mtx);
2642 }
2643
2644 /*
2645  * destroy a process descriptor.
2646  */
2647
2648 static void
2649 pmc_destroy_process_descriptor(struct pmc_process *pp)
2650 {
2651         struct pmc_thread *pmc_td;
2652
2653         while ((pmc_td = LIST_FIRST(&pp->pp_tds)) != NULL) {
2654                 LIST_REMOVE(pmc_td, pt_next);
2655                 pmc_thread_descriptor_pool_free(pmc_td);
2656         }
2657         free(pp, M_PMC);
2658 }
2659
2660
2661 /*
2662  * find an owner descriptor corresponding to proc 'p'
2663  */
2664
2665 static struct pmc_owner *
2666 pmc_find_owner_descriptor(struct proc *p)
2667 {
2668         uint32_t hindex;
2669         struct pmc_owner *po;
2670         struct pmc_ownerhash *poh;
2671
2672         hindex = PMC_HASH_PTR(p, pmc_ownerhashmask);
2673         poh = &pmc_ownerhash[hindex];
2674
2675         po = NULL;
2676         LIST_FOREACH(po, poh, po_next)
2677             if (po->po_owner == p)
2678                     break;
2679
2680         PMCDBG5(OWN,FND,1, "find-owner proc=%p (%d, %s) hindex=0x%x -> "
2681             "pmc-owner=%p", p, p->p_pid, p->p_comm, hindex, po);
2682
2683         return po;
2684 }
2685
2686 /*
2687  * pmc_allocate_pmc_descriptor
2688  *
2689  * Allocate a pmc descriptor and initialize its
2690  * fields.
2691  */
2692
2693 static struct pmc *
2694 pmc_allocate_pmc_descriptor(void)
2695 {
2696         struct pmc *pmc;
2697
2698         pmc = malloc(sizeof(struct pmc), M_PMC, M_WAITOK|M_ZERO);
2699         pmc->pm_runcount = counter_u64_alloc(M_WAITOK);
2700         pmc->pm_pcpu_state = malloc(sizeof(struct pmc_pcpu_state)*mp_ncpus, M_PMC, M_WAITOK|M_ZERO);
2701         PMCDBG1(PMC,ALL,1, "allocate-pmc -> pmc=%p", pmc);
2702
2703         return pmc;
2704 }
2705
2706 /*
2707  * Destroy a pmc descriptor.
2708  */
2709
2710 static void
2711 pmc_destroy_pmc_descriptor(struct pmc *pm)
2712 {
2713
2714         KASSERT(pm->pm_state == PMC_STATE_DELETED ||
2715             pm->pm_state == PMC_STATE_FREE,
2716             ("[pmc,%d] destroying non-deleted PMC", __LINE__));
2717         KASSERT(LIST_EMPTY(&pm->pm_targets),
2718             ("[pmc,%d] destroying pmc with targets", __LINE__));
2719         KASSERT(pm->pm_owner == NULL,
2720             ("[pmc,%d] destroying pmc attached to an owner", __LINE__));
2721         KASSERT(counter_u64_fetch(pm->pm_runcount) == 0,
2722             ("[pmc,%d] pmc has non-zero run count %ld", __LINE__,
2723                  (unsigned long)counter_u64_fetch(pm->pm_runcount)));
2724
2725         counter_u64_free(pm->pm_runcount);
2726         free(pm->pm_pcpu_state, M_PMC);
2727         free(pm, M_PMC);
2728 }
2729
2730 static void
2731 pmc_wait_for_pmc_idle(struct pmc *pm)
2732 {
2733 #ifdef INVARIANTS
2734         volatile int maxloop;
2735
2736         maxloop = 100 * pmc_cpu_max();
2737 #endif
2738         /*
2739          * Loop (with a forced context switch) till the PMC's runcount
2740          * comes down to zero.
2741          */
2742         pmclog_flush(pm->pm_owner, 1);
2743         while (counter_u64_fetch(pm->pm_runcount) > 0) {
2744                 pmclog_flush(pm->pm_owner, 1);
2745 #ifdef INVARIANTS
2746                 maxloop--;
2747                 KASSERT(maxloop > 0,
2748                     ("[pmc,%d] (ri%d, rc%ld) waiting too long for "
2749                         "pmc to be free", __LINE__,
2750                          PMC_TO_ROWINDEX(pm), (unsigned long)counter_u64_fetch(pm->pm_runcount)));
2751 #endif
2752                 pmc_force_context_switch();
2753         }
2754 }
2755
2756 /*
2757  * This function does the following things:
2758  *
2759  *  - detaches the PMC from hardware
2760  *  - unlinks all target threads that were attached to it
2761  *  - removes the PMC from its owner's list
2762  *  - destroys the PMC private mutex
2763  *
2764  * Once this function completes, the given pmc pointer can be freed by
2765  * calling pmc_destroy_pmc_descriptor().
2766  */
2767
2768 static void
2769 pmc_release_pmc_descriptor(struct pmc *pm)
2770 {
2771         enum pmc_mode mode;
2772         struct pmc_hw *phw;
2773         u_int adjri, ri, cpu;
2774         struct pmc_owner *po;
2775         struct pmc_binding pb;
2776         struct pmc_process *pp;
2777         struct pmc_classdep *pcd;
2778         struct pmc_target *ptgt, *tmp;
2779
2780         sx_assert(&pmc_sx, SX_XLOCKED);
2781
2782         KASSERT(pm, ("[pmc,%d] null pmc", __LINE__));
2783
2784         ri   = PMC_TO_ROWINDEX(pm);
2785         pcd  = pmc_ri_to_classdep(md, ri, &adjri);
2786         mode = PMC_TO_MODE(pm);
2787
2788         PMCDBG3(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri,
2789             mode);
2790
2791         /*
2792          * First, we take the PMC off hardware.
2793          */
2794         cpu = 0;
2795         if (PMC_IS_SYSTEM_MODE(mode)) {
2796
2797                 /*
2798                  * A system mode PMC runs on a specific CPU.  Switch
2799                  * to this CPU and turn hardware off.
2800                  */
2801                 pmc_save_cpu_binding(&pb);
2802
2803                 cpu = PMC_TO_CPU(pm);
2804
2805                 pmc_select_cpu(cpu);
2806
2807                 /* switch off non-stalled CPUs */
2808                 pm->pm_pcpu_state[cpu].pps_cpustate = 0;
2809                 if (pm->pm_state == PMC_STATE_RUNNING &&
2810                         pm->pm_pcpu_state[cpu].pps_stalled == 0) {
2811
2812                         phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
2813
2814                         KASSERT(phw->phw_pmc == pm,
2815                             ("[pmc, %d] pmc ptr ri(%d) hw(%p) pm(%p)",
2816                                 __LINE__, ri, phw->phw_pmc, pm));
2817                         PMCDBG2(PMC,REL,2, "stopping cpu=%d ri=%d", cpu, ri);
2818
2819                         critical_enter();
2820                         pcd->pcd_stop_pmc(cpu, adjri);
2821                         critical_exit();
2822                 }
2823
2824                 PMCDBG2(PMC,REL,2, "decfg cpu=%d ri=%d", cpu, ri);
2825
2826                 critical_enter();
2827                 pcd->pcd_config_pmc(cpu, adjri, NULL);
2828                 critical_exit();
2829
2830                 /* adjust the global and process count of SS mode PMCs */
2831                 if (mode == PMC_MODE_SS && pm->pm_state == PMC_STATE_RUNNING) {
2832                         po = pm->pm_owner;
2833                         po->po_sscount--;
2834                         if (po->po_sscount == 0) {
2835                                 atomic_subtract_rel_int(&pmc_ss_count, 1);
2836                                 CK_LIST_REMOVE(po, po_ssnext);
2837                                 epoch_wait_preempt(global_epoch_preempt);
2838                         }
2839                 }
2840
2841                 pm->pm_state = PMC_STATE_DELETED;
2842
2843                 pmc_restore_cpu_binding(&pb);
2844
2845                 /*
2846                  * We could have references to this PMC structure in
2847                  * the per-cpu sample queues.  Wait for the queue to
2848                  * drain.
2849                  */
2850                 pmc_wait_for_pmc_idle(pm);
2851
2852         } else if (PMC_IS_VIRTUAL_MODE(mode)) {
2853
2854                 /*
2855                  * A virtual PMC could be running on multiple CPUs at
2856                  * a given instant.
2857                  *
2858                  * By marking its state as DELETED, we ensure that
2859                  * this PMC is never further scheduled on hardware.
2860                  *
2861                  * Then we wait till all CPUs are done with this PMC.
2862                  */
2863                 pm->pm_state = PMC_STATE_DELETED;
2864
2865
2866                 /* Wait for the PMCs runcount to come to zero. */
2867                 pmc_wait_for_pmc_idle(pm);
2868
2869                 /*
2870                  * At this point the PMC is off all CPUs and cannot be
2871                  * freshly scheduled onto a CPU.  It is now safe to
2872                  * unlink all targets from this PMC.  If a
2873                  * process-record's refcount falls to zero, we remove
2874                  * it from the hash table.  The module-wide SX lock
2875                  * protects us from races.
2876                  */
2877                 LIST_FOREACH_SAFE(ptgt, &pm->pm_targets, pt_next, tmp) {
2878                         pp = ptgt->pt_process;
2879                         pmc_unlink_target_process(pm, pp); /* frees 'ptgt' */
2880
2881                         PMCDBG1(PMC,REL,3, "pp->refcnt=%d", pp->pp_refcnt);
2882
2883                         /*
2884                          * If the target process record shows that no
2885                          * PMCs are attached to it, reclaim its space.
2886                          */
2887
2888                         if (pp->pp_refcnt == 0) {
2889                                 pmc_remove_process_descriptor(pp);
2890                                 pmc_destroy_process_descriptor(pp);
2891                         }
2892                 }
2893
2894                 cpu = curthread->td_oncpu; /* setup cpu for pmd_release() */
2895
2896         }
2897
2898         /*
2899          * Release any MD resources
2900          */
2901         (void) pcd->pcd_release_pmc(cpu, adjri, pm);
2902
2903         /*
2904          * Update row disposition
2905          */
2906
2907         if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm)))
2908                 PMC_UNMARK_ROW_STANDALONE(ri);
2909         else
2910                 PMC_UNMARK_ROW_THREAD(ri);
2911
2912         /* unlink from the owner's list */
2913         if (pm->pm_owner) {
2914                 LIST_REMOVE(pm, pm_next);
2915                 pm->pm_owner = NULL;
2916         }
2917 }
2918
2919 /*
2920  * Register an owner and a pmc.
2921  */
2922
2923 static int
2924 pmc_register_owner(struct proc *p, struct pmc *pmc)
2925 {
2926         struct pmc_owner *po;
2927
2928         sx_assert(&pmc_sx, SX_XLOCKED);
2929
2930         if ((po = pmc_find_owner_descriptor(p)) == NULL)
2931                 if ((po = pmc_allocate_owner_descriptor(p)) == NULL)
2932                         return ENOMEM;
2933
2934         KASSERT(pmc->pm_owner == NULL,
2935             ("[pmc,%d] attempting to own an initialized PMC", __LINE__));
2936         pmc->pm_owner  = po;
2937
2938         LIST_INSERT_HEAD(&po->po_pmcs, pmc, pm_next);
2939
2940         PROC_LOCK(p);
2941         p->p_flag |= P_HWPMC;
2942         PROC_UNLOCK(p);
2943
2944         if (po->po_flags & PMC_PO_OWNS_LOGFILE)
2945                 pmclog_process_pmcallocate(pmc);
2946
2947         PMCDBG2(PMC,REG,1, "register-owner pmc-owner=%p pmc=%p",
2948             po, pmc);
2949
2950         return 0;
2951 }
2952
2953 /*
2954  * Return the current row disposition:
2955  * == 0 => FREE
2956  *  > 0 => PROCESS MODE
2957  *  < 0 => SYSTEM MODE
2958  */
2959
2960 int
2961 pmc_getrowdisp(int ri)
2962 {
2963         return pmc_pmcdisp[ri];
2964 }
2965
2966 /*
2967  * Check if a PMC at row index 'ri' can be allocated to the current
2968  * process.
2969  *
2970  * Allocation can fail if:
2971  *   - the current process is already being profiled by a PMC at index 'ri',
2972  *     attached to it via OP_PMCATTACH.
2973  *   - the current process has already allocated a PMC at index 'ri'
2974  *     via OP_ALLOCATE.
2975  */
2976
2977 static int
2978 pmc_can_allocate_rowindex(struct proc *p, unsigned int ri, int cpu)
2979 {
2980         enum pmc_mode mode;
2981         struct pmc *pm;
2982         struct pmc_owner *po;
2983         struct pmc_process *pp;
2984
2985         PMCDBG5(PMC,ALR,1, "can-allocate-rowindex proc=%p (%d, %s) ri=%d "
2986             "cpu=%d", p, p->p_pid, p->p_comm, ri, cpu);
2987
2988         /*
2989          * We shouldn't have already allocated a process-mode PMC at
2990          * row index 'ri'.
2991          *
2992          * We shouldn't have allocated a system-wide PMC on the same
2993          * CPU and same RI.
2994          */
2995         if ((po = pmc_find_owner_descriptor(p)) != NULL)
2996                 LIST_FOREACH(pm, &po->po_pmcs, pm_next) {
2997                     if (PMC_TO_ROWINDEX(pm) == ri) {
2998                             mode = PMC_TO_MODE(pm);
2999                             if (PMC_IS_VIRTUAL_MODE(mode))
3000                                     return EEXIST;
3001                             if (PMC_IS_SYSTEM_MODE(mode) &&
3002                                 (int) PMC_TO_CPU(pm) == cpu)
3003                                     return EEXIST;
3004                     }
3005                 }
3006
3007         /*
3008          * We also shouldn't be the target of any PMC at this index
3009          * since otherwise a PMC_ATTACH to ourselves will fail.
3010          */
3011         if ((pp = pmc_find_process_descriptor(p, 0)) != NULL)
3012                 if (pp->pp_pmcs[ri].pp_pmc)
3013                         return EEXIST;
3014
3015         PMCDBG4(PMC,ALR,2, "can-allocate-rowindex proc=%p (%d, %s) ri=%d ok",
3016             p, p->p_pid, p->p_comm, ri);
3017
3018         return 0;
3019 }
3020
3021 /*
3022  * Check if a given PMC at row index 'ri' can be currently used in
3023  * mode 'mode'.
3024  */
3025
3026 static int
3027 pmc_can_allocate_row(int ri, enum pmc_mode mode)
3028 {
3029         enum pmc_disp   disp;
3030
3031         sx_assert(&pmc_sx, SX_XLOCKED);
3032
3033         PMCDBG2(PMC,ALR,1, "can-allocate-row ri=%d mode=%d", ri, mode);
3034
3035         if (PMC_IS_SYSTEM_MODE(mode))
3036                 disp = PMC_DISP_STANDALONE;
3037         else
3038                 disp = PMC_DISP_THREAD;
3039
3040         /*
3041          * check disposition for PMC row 'ri':
3042          *
3043          * Expected disposition         Row-disposition         Result
3044          *
3045          * STANDALONE                   STANDALONE or FREE      proceed
3046          * STANDALONE                   THREAD                  fail
3047          * THREAD                       THREAD or FREE          proceed
3048          * THREAD                       STANDALONE              fail
3049          */
3050
3051         if (!PMC_ROW_DISP_IS_FREE(ri) &&
3052             !(disp == PMC_DISP_THREAD && PMC_ROW_DISP_IS_THREAD(ri)) &&
3053             !(disp == PMC_DISP_STANDALONE && PMC_ROW_DISP_IS_STANDALONE(ri)))
3054                 return EBUSY;
3055
3056         /*
3057          * All OK
3058          */
3059
3060         PMCDBG2(PMC,ALR,2, "can-allocate-row ri=%d mode=%d ok", ri, mode);
3061
3062         return 0;
3063
3064 }
3065
3066 /*
3067  * Find a PMC descriptor with user handle 'pmcid' for thread 'td'.
3068  */
3069
3070 static struct pmc *
3071 pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, pmc_id_t pmcid)
3072 {
3073         struct pmc *pm;
3074
3075         KASSERT(PMC_ID_TO_ROWINDEX(pmcid) < md->pmd_npmc,
3076             ("[pmc,%d] Illegal pmc index %d (max %d)", __LINE__,
3077                 PMC_ID_TO_ROWINDEX(pmcid), md->pmd_npmc));
3078
3079         LIST_FOREACH(pm, &po->po_pmcs, pm_next)
3080             if (pm->pm_id == pmcid)
3081                     return pm;
3082
3083         return NULL;
3084 }
3085
3086 static int
3087 pmc_find_pmc(pmc_id_t pmcid, struct pmc **pmc)
3088 {
3089
3090         struct pmc *pm, *opm;
3091         struct pmc_owner *po;
3092         struct pmc_process *pp;
3093
3094         PMCDBG1(PMC,FND,1, "find-pmc id=%d", pmcid);
3095         if (PMC_ID_TO_ROWINDEX(pmcid) >= md->pmd_npmc)
3096                 return (EINVAL);
3097
3098         if ((po = pmc_find_owner_descriptor(curthread->td_proc)) == NULL) {
3099                 /*
3100                  * In case of PMC_F_DESCENDANTS child processes we will not find
3101                  * the current process in the owners hash list.  Find the owner
3102                  * process first and from there lookup the po.
3103                  */
3104                 if ((pp = pmc_find_process_descriptor(curthread->td_proc,
3105                     PMC_FLAG_NONE)) == NULL) {
3106                         return ESRCH;
3107                 } else {
3108                         opm = pp->pp_pmcs[PMC_ID_TO_ROWINDEX(pmcid)].pp_pmc;
3109                         if (opm == NULL)
3110                                 return ESRCH;
3111                         if ((opm->pm_flags & (PMC_F_ATTACHED_TO_OWNER|
3112                             PMC_F_DESCENDANTS)) != (PMC_F_ATTACHED_TO_OWNER|
3113                             PMC_F_DESCENDANTS))
3114                                 return ESRCH;
3115                         po = opm->pm_owner;
3116                 }
3117         }
3118
3119         if ((pm = pmc_find_pmc_descriptor_in_process(po, pmcid)) == NULL)
3120                 return EINVAL;
3121
3122         PMCDBG2(PMC,FND,2, "find-pmc id=%d -> pmc=%p", pmcid, pm);
3123
3124         *pmc = pm;
3125         return 0;
3126 }
3127
3128 /*
3129  * Start a PMC.
3130  */
3131
3132 static int
3133 pmc_start(struct pmc *pm)
3134 {
3135         enum pmc_mode mode;
3136         struct pmc_owner *po;
3137         struct pmc_binding pb;
3138         struct pmc_classdep *pcd;
3139         int adjri, error, cpu, ri;
3140
3141         KASSERT(pm != NULL,
3142             ("[pmc,%d] null pm", __LINE__));
3143
3144         mode = PMC_TO_MODE(pm);
3145         ri   = PMC_TO_ROWINDEX(pm);
3146         pcd  = pmc_ri_to_classdep(md, ri, &adjri);
3147
3148         error = 0;
3149
3150         PMCDBG3(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri);
3151
3152         po = pm->pm_owner;
3153
3154         /*
3155          * Disallow PMCSTART if a logfile is required but has not been
3156          * configured yet.
3157          */
3158         if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) &&
3159             (po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
3160                 return (EDOOFUS);       /* programming error */
3161
3162         /*
3163          * If this is a sampling mode PMC, log mapping information for
3164          * the kernel modules that are currently loaded.
3165          */
3166         if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
3167             pmc_log_kernel_mappings(pm);
3168
3169         if (PMC_IS_VIRTUAL_MODE(mode)) {
3170
3171                 /*
3172                  * If a PMCATTACH has never been done on this PMC,
3173                  * attach it to its owner process.
3174                  */
3175
3176                 if (LIST_EMPTY(&pm->pm_targets))
3177                         error = (pm->pm_flags & PMC_F_ATTACH_DONE) ? ESRCH :
3178                             pmc_attach_process(po->po_owner, pm);
3179
3180                 /*
3181                  * If the PMC is attached to its owner, then force a context
3182                  * switch to ensure that the MD state gets set correctly.
3183                  */
3184
3185                 if (error == 0) {
3186                         pm->pm_state = PMC_STATE_RUNNING;
3187                         if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER)
3188                                 pmc_force_context_switch();
3189                 }
3190
3191                 return (error);
3192         }
3193
3194
3195         /*
3196          * A system-wide PMC.
3197          *
3198          * Add the owner to the global list if this is a system-wide
3199          * sampling PMC.
3200          */
3201
3202         if (mode == PMC_MODE_SS) {
3203                 /*
3204                  * Log mapping information for all existing processes in the
3205                  * system.  Subsequent mappings are logged as they happen;
3206                  * see pmc_process_mmap().
3207                  */
3208                 if (po->po_logprocmaps == 0) {
3209                         pmc_log_all_process_mappings(po);
3210                         po->po_logprocmaps = 1;
3211                 }
3212                 po->po_sscount++;
3213                 if (po->po_sscount == 1) {
3214                         atomic_add_rel_int(&pmc_ss_count, 1);
3215                         CK_LIST_INSERT_HEAD(&pmc_ss_owners, po, po_ssnext);
3216                         PMCDBG1(PMC,OPS,1, "po=%p in global list", po);
3217                 }
3218         }
3219
3220         /*
3221          * Move to the CPU associated with this
3222          * PMC, and start the hardware.
3223          */
3224
3225         pmc_save_cpu_binding(&pb);
3226
3227         cpu = PMC_TO_CPU(pm);
3228
3229         if (!pmc_cpu_is_active(cpu))
3230                 return (ENXIO);
3231
3232         pmc_select_cpu(cpu);
3233
3234         /*
3235          * global PMCs are configured at allocation time
3236          * so write out the initial value and start the PMC.
3237          */
3238
3239         pm->pm_state = PMC_STATE_RUNNING;
3240
3241         critical_enter();
3242         if ((error = pcd->pcd_write_pmc(cpu, adjri,
3243                  PMC_IS_SAMPLING_MODE(mode) ?
3244                  pm->pm_sc.pm_reloadcount :
3245                  pm->pm_sc.pm_initial)) == 0) {
3246                 /* If a sampling mode PMC, reset stalled state. */
3247                 if (PMC_IS_SAMPLING_MODE(mode))
3248                         pm->pm_pcpu_state[cpu].pps_stalled = 0;
3249
3250                 /* Indicate that we desire this to run. Start it. */
3251                 pm->pm_pcpu_state[cpu].pps_cpustate = 1;
3252                 error = pcd->pcd_start_pmc(cpu, adjri);
3253         }
3254         critical_exit();
3255
3256         pmc_restore_cpu_binding(&pb);
3257
3258         return (error);
3259 }
3260
3261 /*
3262  * Stop a PMC.
3263  */
3264
3265 static int
3266 pmc_stop(struct pmc *pm)
3267 {
3268         struct pmc_owner *po;
3269         struct pmc_binding pb;
3270         struct pmc_classdep *pcd;
3271         int adjri, cpu, error, ri;
3272
3273         KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__));
3274
3275         PMCDBG3(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm,
3276             PMC_TO_MODE(pm), PMC_TO_ROWINDEX(pm));
3277
3278         pm->pm_state = PMC_STATE_STOPPED;
3279
3280         /*
3281          * If the PMC is a virtual mode one, changing the state to
3282          * non-RUNNING is enough to ensure that the PMC never gets
3283          * scheduled.
3284          *
3285          * If this PMC is current running on a CPU, then it will
3286          * handled correctly at the time its target process is context
3287          * switched out.
3288          */
3289
3290         if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
3291                 return 0;
3292
3293         /*
3294          * A system-mode PMC.  Move to the CPU associated with
3295          * this PMC, and stop the hardware.  We update the
3296          * 'initial count' so that a subsequent PMCSTART will
3297          * resume counting from the current hardware count.
3298          */
3299
3300         pmc_save_cpu_binding(&pb);
3301
3302         cpu = PMC_TO_CPU(pm);
3303
3304         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
3305             ("[pmc,%d] illegal cpu=%d", __LINE__, cpu));
3306
3307         if (!pmc_cpu_is_active(cpu))
3308                 return ENXIO;
3309
3310         pmc_select_cpu(cpu);
3311
3312         ri = PMC_TO_ROWINDEX(pm);
3313         pcd = pmc_ri_to_classdep(md, ri, &adjri);
3314
3315         pm->pm_pcpu_state[cpu].pps_cpustate = 0;
3316         critical_enter();
3317         if ((error = pcd->pcd_stop_pmc(cpu, adjri)) == 0)
3318                 error = pcd->pcd_read_pmc(cpu, adjri, &pm->pm_sc.pm_initial);
3319         critical_exit();
3320
3321         pmc_restore_cpu_binding(&pb);
3322
3323         po = pm->pm_owner;
3324
3325         /* remove this owner from the global list of SS PMC owners */
3326         if (PMC_TO_MODE(pm) == PMC_MODE_SS) {
3327                 po->po_sscount--;
3328                 if (po->po_sscount == 0) {
3329                         atomic_subtract_rel_int(&pmc_ss_count, 1);
3330                         CK_LIST_REMOVE(po, po_ssnext);
3331                         epoch_wait_preempt(global_epoch_preempt);
3332                         PMCDBG1(PMC,OPS,2,"po=%p removed from global list", po);
3333                 }
3334         }
3335
3336         return (error);
3337 }
3338
3339 static struct pmc_classdep *
3340 pmc_class_to_classdep(enum pmc_class class)
3341 {
3342         int n;
3343
3344         for (n = 0; n < md->pmd_nclass; n++)
3345                 if (md->pmd_classdep[n].pcd_class == class)
3346                         return (&md->pmd_classdep[n]);
3347         return (NULL);
3348 }
3349
3350 #if defined(HWPMC_DEBUG) && defined(KTR)
3351 static const char *pmc_op_to_name[] = {
3352 #undef  __PMC_OP
3353 #define __PMC_OP(N, D)  #N ,
3354         __PMC_OPS()
3355         NULL
3356 };
3357 #endif
3358
3359 /*
3360  * The syscall interface
3361  */
3362
3363 #define PMC_GET_SX_XLOCK(...) do {              \
3364         sx_xlock(&pmc_sx);                      \
3365         if (pmc_hook == NULL) {                 \
3366                 sx_xunlock(&pmc_sx);            \
3367                 return __VA_ARGS__;             \
3368         }                                       \
3369 } while (0)
3370
3371 #define PMC_DOWNGRADE_SX() do {                 \
3372         sx_downgrade(&pmc_sx);                  \
3373         is_sx_downgraded = 1;                   \
3374 } while (0)
3375
3376 static int
3377 pmc_syscall_handler(struct thread *td, void *syscall_args)
3378 {
3379         int error, is_sx_downgraded, op;
3380         struct pmc_syscall_args *c;
3381         void *pmclog_proc_handle;
3382         void *arg;
3383
3384         c = (struct pmc_syscall_args *)syscall_args;
3385         op = c->pmop_code;
3386         arg = c->pmop_data;
3387         /* PMC isn't set up yet */
3388         if (pmc_hook == NULL)
3389                 return (EINVAL);
3390         if (op == PMC_OP_CONFIGURELOG) {
3391                 /*
3392                  * We cannot create the logging process inside
3393                  * pmclog_configure_log() because there is a LOR
3394                  * between pmc_sx and process structure locks.
3395                  * Instead, pre-create the process and ignite the loop
3396                  * if everything is fine, otherwise direct the process
3397                  * to exit.
3398                  */
3399                 error = pmclog_proc_create(td, &pmclog_proc_handle);
3400                 if (error != 0)
3401                         goto done_syscall;
3402         }
3403
3404         PMC_GET_SX_XLOCK(ENOSYS);
3405         is_sx_downgraded = 0;
3406         PMCDBG3(MOD,PMS,1, "syscall op=%d \"%s\" arg=%p", op,
3407             pmc_op_to_name[op], arg);
3408
3409         error = 0;
3410         counter_u64_add(pmc_stats.pm_syscalls, 1);
3411
3412         switch (op) {
3413
3414
3415         /*
3416          * Configure a log file.
3417          *
3418          * XXX This OP will be reworked.
3419          */
3420
3421         case PMC_OP_CONFIGURELOG:
3422         {
3423                 struct proc *p;
3424                 struct pmc *pm;
3425                 struct pmc_owner *po;
3426                 struct pmc_op_configurelog cl;
3427
3428                 if ((error = copyin(arg, &cl, sizeof(cl))) != 0) {
3429                         pmclog_proc_ignite(pmclog_proc_handle, NULL);
3430                         break;
3431                 }
3432
3433                 /* mark this process as owning a log file */
3434                 p = td->td_proc;
3435                 if ((po = pmc_find_owner_descriptor(p)) == NULL)
3436                         if ((po = pmc_allocate_owner_descriptor(p)) == NULL) {
3437                                 pmclog_proc_ignite(pmclog_proc_handle, NULL);
3438                                 error = ENOMEM;
3439                                 break;
3440                         }
3441
3442                 /*
3443                  * If a valid fd was passed in, try to configure that,
3444                  * otherwise if 'fd' was less than zero and there was
3445                  * a log file configured, flush its buffers and
3446                  * de-configure it.
3447                  */
3448                 if (cl.pm_logfd >= 0) {
3449                         error = pmclog_configure_log(md, po, cl.pm_logfd);
3450                         pmclog_proc_ignite(pmclog_proc_handle, error == 0 ?
3451                             po : NULL);
3452                 } else if (po->po_flags & PMC_PO_OWNS_LOGFILE) {
3453                         pmclog_proc_ignite(pmclog_proc_handle, NULL);
3454                         error = pmclog_close(po);
3455                         if (error == 0) {
3456                                 LIST_FOREACH(pm, &po->po_pmcs, pm_next)
3457                                     if (pm->pm_flags & PMC_F_NEEDS_LOGFILE &&
3458                                         pm->pm_state == PMC_STATE_RUNNING)
3459                                             pmc_stop(pm);
3460                                 error = pmclog_deconfigure_log(po);
3461                         }
3462                 } else {
3463                         pmclog_proc_ignite(pmclog_proc_handle, NULL);
3464                         error = EINVAL;
3465                 }
3466         }
3467         break;
3468
3469         /*
3470          * Flush a log file.
3471          */
3472
3473         case PMC_OP_FLUSHLOG:
3474         {
3475                 struct pmc_owner *po;
3476
3477                 sx_assert(&pmc_sx, SX_XLOCKED);
3478
3479                 if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
3480                         error = EINVAL;
3481                         break;
3482                 }
3483
3484                 error = pmclog_flush(po, 0);
3485         }
3486         break;
3487
3488         /*
3489          * Close a log file.
3490          */
3491
3492         case PMC_OP_CLOSELOG:
3493         {
3494                 struct pmc_owner *po;
3495
3496                 sx_assert(&pmc_sx, SX_XLOCKED);
3497
3498                 if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
3499                         error = EINVAL;
3500                         break;
3501                 }
3502
3503                 error = pmclog_close(po);
3504         }
3505         break;
3506
3507         /*
3508          * Retrieve hardware configuration.
3509          */
3510
3511         case PMC_OP_GETCPUINFO: /* CPU information */
3512         {
3513                 struct pmc_op_getcpuinfo gci;
3514                 struct pmc_classinfo *pci;
3515                 struct pmc_classdep *pcd;
3516                 int cl;
3517
3518                 memset(&gci, 0, sizeof(gci));
3519                 gci.pm_cputype = md->pmd_cputype;
3520                 gci.pm_ncpu    = pmc_cpu_max();
3521                 gci.pm_npmc    = md->pmd_npmc;
3522                 gci.pm_nclass  = md->pmd_nclass;
3523                 pci = gci.pm_classes;
3524                 pcd = md->pmd_classdep;
3525                 for (cl = 0; cl < md->pmd_nclass; cl++, pci++, pcd++) {
3526                         pci->pm_caps  = pcd->pcd_caps;
3527                         pci->pm_class = pcd->pcd_class;
3528                         pci->pm_width = pcd->pcd_width;
3529                         pci->pm_num   = pcd->pcd_num;
3530                 }
3531                 error = copyout(&gci, arg, sizeof(gci));
3532         }
3533         break;
3534
3535         /*
3536          * Retrieve soft events list.
3537          */
3538         case PMC_OP_GETDYNEVENTINFO:
3539         {
3540                 enum pmc_class                  cl;
3541                 enum pmc_event                  ev;
3542                 struct pmc_op_getdyneventinfo   *gei;
3543                 struct pmc_dyn_event_descr      dev;
3544                 struct pmc_soft                 *ps;
3545                 uint32_t                        nevent;
3546
3547                 sx_assert(&pmc_sx, SX_LOCKED);
3548
3549                 gei = (struct pmc_op_getdyneventinfo *) arg;
3550
3551                 if ((error = copyin(&gei->pm_class, &cl, sizeof(cl))) != 0)
3552                         break;
3553
3554                 /* Only SOFT class is dynamic. */
3555                 if (cl != PMC_CLASS_SOFT) {
3556                         error = EINVAL;
3557                         break;
3558                 }
3559
3560                 nevent = 0;
3561                 for (ev = PMC_EV_SOFT_FIRST; (int)ev <= PMC_EV_SOFT_LAST; ev++) {
3562                         ps = pmc_soft_ev_acquire(ev);
3563                         if (ps == NULL)
3564                                 continue;
3565                         bcopy(&ps->ps_ev, &dev, sizeof(dev));
3566                         pmc_soft_ev_release(ps);
3567
3568                         error = copyout(&dev,
3569                             &gei->pm_events[nevent],
3570                             sizeof(struct pmc_dyn_event_descr));
3571                         if (error != 0)
3572                                 break;
3573                         nevent++;
3574                 }
3575                 if (error != 0)
3576                         break;
3577
3578                 error = copyout(&nevent, &gei->pm_nevent,
3579                     sizeof(nevent));
3580         }
3581         break;
3582
3583         /*
3584          * Get module statistics
3585          */
3586
3587         case PMC_OP_GETDRIVERSTATS:
3588         {
3589                 struct pmc_op_getdriverstats gms;
3590 #define CFETCH(a, b, field) a.field = counter_u64_fetch(b.field)
3591                 CFETCH(gms, pmc_stats, pm_intr_ignored);
3592                 CFETCH(gms, pmc_stats, pm_intr_processed);
3593                 CFETCH(gms, pmc_stats, pm_intr_bufferfull);
3594                 CFETCH(gms, pmc_stats, pm_syscalls);
3595                 CFETCH(gms, pmc_stats, pm_syscall_errors);
3596                 CFETCH(gms, pmc_stats, pm_buffer_requests);
3597                 CFETCH(gms, pmc_stats, pm_buffer_requests_failed);
3598                 CFETCH(gms, pmc_stats, pm_log_sweeps);
3599 #undef CFETCH
3600                 error = copyout(&gms, arg, sizeof(gms));
3601         }
3602         break;
3603
3604
3605         /*
3606          * Retrieve module version number
3607          */
3608
3609         case PMC_OP_GETMODULEVERSION:
3610         {
3611                 uint32_t cv, modv;
3612
3613                 /* retrieve the client's idea of the ABI version */
3614                 if ((error = copyin(arg, &cv, sizeof(uint32_t))) != 0)
3615                         break;
3616                 /* don't service clients newer than our driver */
3617                 modv = PMC_VERSION;
3618                 if ((cv & 0xFFFF0000) > (modv & 0xFFFF0000)) {
3619                         error = EPROGMISMATCH;
3620                         break;
3621                 }
3622                 error = copyout(&modv, arg, sizeof(int));
3623         }
3624         break;
3625
3626
3627         /*
3628          * Retrieve the state of all the PMCs on a given
3629          * CPU.
3630          */
3631
3632         case PMC_OP_GETPMCINFO:
3633         {
3634                 int ari;
3635                 struct pmc *pm;
3636                 size_t pmcinfo_size;
3637                 uint32_t cpu, n, npmc;
3638                 struct pmc_owner *po;
3639                 struct pmc_binding pb;
3640                 struct pmc_classdep *pcd;
3641                 struct pmc_info *p, *pmcinfo;
3642                 struct pmc_op_getpmcinfo *gpi;
3643
3644                 PMC_DOWNGRADE_SX();
3645
3646                 gpi = (struct pmc_op_getpmcinfo *) arg;
3647
3648                 if ((error = copyin(&gpi->pm_cpu, &cpu, sizeof(cpu))) != 0)
3649                         break;
3650
3651                 if (cpu >= pmc_cpu_max()) {
3652                         error = EINVAL;
3653                         break;
3654                 }
3655
3656                 if (!pmc_cpu_is_active(cpu)) {
3657                         error = ENXIO;
3658                         break;
3659                 }
3660
3661                 /* switch to CPU 'cpu' */
3662                 pmc_save_cpu_binding(&pb);
3663                 pmc_select_cpu(cpu);
3664
3665                 npmc = md->pmd_npmc;
3666
3667                 pmcinfo_size = npmc * sizeof(struct pmc_info);
3668                 pmcinfo = malloc(pmcinfo_size, M_PMC, M_WAITOK | M_ZERO);
3669
3670                 p = pmcinfo;
3671
3672                 for (n = 0; n < md->pmd_npmc; n++, p++) {
3673
3674                         pcd = pmc_ri_to_classdep(md, n, &ari);
3675
3676                         KASSERT(pcd != NULL,
3677                             ("[pmc,%d] null pcd ri=%d", __LINE__, n));
3678
3679                         if ((error = pcd->pcd_describe(cpu, ari, p, &pm)) != 0)
3680                                 break;
3681
3682                         if (PMC_ROW_DISP_IS_STANDALONE(n))
3683                                 p->pm_rowdisp = PMC_DISP_STANDALONE;
3684                         else if (PMC_ROW_DISP_IS_THREAD(n))
3685                                 p->pm_rowdisp = PMC_DISP_THREAD;
3686                         else
3687                                 p->pm_rowdisp = PMC_DISP_FREE;
3688
3689                         p->pm_ownerpid = -1;
3690
3691                         if (pm == NULL) /* no PMC associated */
3692                                 continue;
3693
3694                         po = pm->pm_owner;
3695
3696                         KASSERT(po->po_owner != NULL,
3697                             ("[pmc,%d] pmc_owner had a null proc pointer",
3698                                 __LINE__));
3699
3700                         p->pm_ownerpid = po->po_owner->p_pid;
3701                         p->pm_mode     = PMC_TO_MODE(pm);
3702                         p->pm_event    = pm->pm_event;
3703                         p->pm_flags    = pm->pm_flags;
3704
3705                         if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
3706                                 p->pm_reloadcount =
3707                                     pm->pm_sc.pm_reloadcount;
3708                 }
3709
3710                 pmc_restore_cpu_binding(&pb);
3711
3712                 /* now copy out the PMC info collected */
3713                 if (error == 0)
3714                         error = copyout(pmcinfo, &gpi->pm_pmcs, pmcinfo_size);
3715
3716                 free(pmcinfo, M_PMC);
3717         }
3718         break;
3719
3720
3721         /*
3722          * Set the administrative state of a PMC.  I.e. whether
3723          * the PMC is to be used or not.
3724          */
3725
3726         case PMC_OP_PMCADMIN:
3727         {
3728                 int cpu, ri;
3729                 enum pmc_state request;
3730                 struct pmc_cpu *pc;
3731                 struct pmc_hw *phw;
3732                 struct pmc_op_pmcadmin pma;
3733                 struct pmc_binding pb;
3734
3735                 sx_assert(&pmc_sx, SX_XLOCKED);
3736
3737                 KASSERT(td == curthread,
3738                     ("[pmc,%d] td != curthread", __LINE__));
3739
3740                 error = priv_check(td, PRIV_PMC_MANAGE);
3741                 if (error)
3742                         break;
3743
3744                 if ((error = copyin(arg, &pma, sizeof(pma))) != 0)
3745                         break;
3746
3747                 cpu = pma.pm_cpu;
3748
3749                 if (cpu < 0 || cpu >= (int) pmc_cpu_max()) {
3750                         error = EINVAL;
3751                         break;
3752                 }
3753
3754                 if (!pmc_cpu_is_active(cpu)) {
3755                         error = ENXIO;
3756                         break;
3757                 }
3758
3759                 request = pma.pm_state;
3760
3761                 if (request != PMC_STATE_DISABLED &&
3762                     request != PMC_STATE_FREE) {
3763                         error = EINVAL;
3764                         break;
3765                 }
3766
3767                 ri = pma.pm_pmc; /* pmc id == row index */
3768                 if (ri < 0 || ri >= (int) md->pmd_npmc) {
3769                         error = EINVAL;
3770                         break;
3771                 }
3772
3773                 /*
3774                  * We can't disable a PMC with a row-index allocated
3775                  * for process virtual PMCs.
3776                  */
3777
3778                 if (PMC_ROW_DISP_IS_THREAD(ri) &&
3779                     request == PMC_STATE_DISABLED) {
3780                         error = EBUSY;
3781                         break;
3782                 }
3783
3784                 /*
3785                  * otherwise, this PMC on this CPU is either free or
3786                  * in system-wide mode.
3787                  */
3788
3789                 pmc_save_cpu_binding(&pb);
3790                 pmc_select_cpu(cpu);
3791
3792                 pc  = pmc_pcpu[cpu];
3793                 phw = pc->pc_hwpmcs[ri];
3794
3795                 /*
3796                  * XXX do we need some kind of 'forced' disable?
3797                  */
3798
3799                 if (phw->phw_pmc == NULL) {
3800                         if (request == PMC_STATE_DISABLED &&
3801                             (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED)) {
3802                                 phw->phw_state &= ~PMC_PHW_FLAG_IS_ENABLED;
3803                                 PMC_MARK_ROW_STANDALONE(ri);
3804                         } else if (request == PMC_STATE_FREE &&
3805                             (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0) {
3806                                 phw->phw_state |=  PMC_PHW_FLAG_IS_ENABLED;
3807                                 PMC_UNMARK_ROW_STANDALONE(ri);
3808                         }
3809                         /* other cases are a no-op */
3810                 } else
3811                         error = EBUSY;
3812
3813                 pmc_restore_cpu_binding(&pb);
3814         }
3815         break;
3816
3817
3818         /*
3819          * Allocate a PMC.
3820          */
3821
3822         case PMC_OP_PMCALLOCATE:
3823         {
3824                 int adjri, n;
3825                 u_int cpu;
3826                 uint32_t caps;
3827                 struct pmc *pmc;
3828                 enum pmc_mode mode;
3829                 struct pmc_hw *phw;
3830                 struct pmc_binding pb;
3831                 struct pmc_classdep *pcd;
3832                 struct pmc_op_pmcallocate pa;
3833
3834                 if ((error = copyin(arg, &pa, sizeof(pa))) != 0)
3835                         break;
3836
3837                 caps = pa.pm_caps;
3838                 mode = pa.pm_mode;
3839                 cpu  = pa.pm_cpu;
3840
3841                 if ((mode != PMC_MODE_SS  &&  mode != PMC_MODE_SC  &&
3842                      mode != PMC_MODE_TS  &&  mode != PMC_MODE_TC) ||
3843                     (cpu != (u_int) PMC_CPU_ANY && cpu >= pmc_cpu_max())) {
3844                         error = EINVAL;
3845                         break;
3846                 }
3847
3848                 /*
3849                  * Virtual PMCs should only ask for a default CPU.
3850                  * System mode PMCs need to specify a non-default CPU.
3851                  */
3852
3853                 if ((PMC_IS_VIRTUAL_MODE(mode) && cpu != (u_int) PMC_CPU_ANY) ||
3854                     (PMC_IS_SYSTEM_MODE(mode) && cpu == (u_int) PMC_CPU_ANY)) {
3855                         error = EINVAL;
3856                         break;
3857                 }
3858
3859                 /*
3860                  * Check that an inactive CPU is not being asked for.
3861                  */
3862
3863                 if (PMC_IS_SYSTEM_MODE(mode) && !pmc_cpu_is_active(cpu)) {
3864                         error = ENXIO;
3865                         break;
3866                 }
3867
3868                 /*
3869                  * Refuse an allocation for a system-wide PMC if this
3870                  * process has been jailed, or if this process lacks
3871                  * super-user credentials and the sysctl tunable
3872                  * 'security.bsd.unprivileged_syspmcs' is zero.
3873                  */
3874
3875                 if (PMC_IS_SYSTEM_MODE(mode)) {
3876                         if (jailed(curthread->td_ucred)) {
3877                                 error = EPERM;
3878                                 break;
3879                         }
3880                         if (!pmc_unprivileged_syspmcs) {
3881                                 error = priv_check(curthread,
3882                                     PRIV_PMC_SYSTEM);
3883                                 if (error)
3884                                         break;
3885                         }
3886                 }
3887
3888                 /*
3889                  * Look for valid values for 'pm_flags'
3890                  */
3891
3892                 if ((pa.pm_flags & ~(PMC_F_DESCENDANTS | PMC_F_LOG_PROCCSW |
3893                     PMC_F_LOG_PROCEXIT | PMC_F_CALLCHAIN |
3894                     PMC_F_USERCALLCHAIN)) != 0) {
3895                         error = EINVAL;
3896                         break;
3897                 }
3898
3899                 /* PMC_F_USERCALLCHAIN is only valid with PMC_F_CALLCHAIN */
3900                 if ((pa.pm_flags & (PMC_F_CALLCHAIN | PMC_F_USERCALLCHAIN)) ==
3901                     PMC_F_USERCALLCHAIN) {
3902                         error = EINVAL;
3903                         break;
3904                 }
3905
3906                 /* PMC_F_USERCALLCHAIN is only valid for sampling mode */
3907                 if (pa.pm_flags & PMC_F_USERCALLCHAIN &&
3908                         mode != PMC_MODE_TS && mode != PMC_MODE_SS) {
3909                         error = EINVAL;
3910                         break;
3911                 }
3912
3913                 /* process logging options are not allowed for system PMCs */
3914                 if (PMC_IS_SYSTEM_MODE(mode) && (pa.pm_flags &
3915                     (PMC_F_LOG_PROCCSW | PMC_F_LOG_PROCEXIT))) {
3916                         error = EINVAL;
3917                         break;
3918                 }
3919
3920                 /*
3921                  * All sampling mode PMCs need to be able to interrupt the
3922                  * CPU.
3923                  */
3924                 if (PMC_IS_SAMPLING_MODE(mode))
3925                         caps |= PMC_CAP_INTERRUPT;
3926
3927                 /* A valid class specifier should have been passed in. */
3928                 pcd = pmc_class_to_classdep(pa.pm_class);
3929                 if (pcd == NULL) {
3930                         error = EINVAL;
3931                         break;
3932                 }
3933
3934                 /* The requested PMC capabilities should be feasible. */
3935                 if ((pcd->pcd_caps & caps) != caps) {
3936                         error = EOPNOTSUPP;
3937                         break;
3938                 }
3939
3940                 PMCDBG4(PMC,ALL,2, "event=%d caps=0x%x mode=%d cpu=%d",
3941                     pa.pm_ev, caps, mode, cpu);
3942
3943                 pmc = pmc_allocate_pmc_descriptor();
3944                 pmc->pm_id    = PMC_ID_MAKE_ID(cpu,pa.pm_mode,pa.pm_class,
3945                     PMC_ID_INVALID);
3946                 pmc->pm_event = pa.pm_ev;
3947                 pmc->pm_state = PMC_STATE_FREE;
3948                 pmc->pm_caps  = caps;
3949                 pmc->pm_flags = pa.pm_flags;
3950
3951                 /* XXX set lower bound on sampling for process counters */
3952                 if (PMC_IS_SAMPLING_MODE(mode)) {
3953                         /*
3954                          * Don't permit requested sample rate to be less than 1000
3955                          */
3956                         if (pa.pm_count < 1000)
3957                                 log(LOG_WARNING,
3958                                         "pmcallocate: passed sample rate %ju - setting to 1000\n",
3959                                         (uintmax_t)pa.pm_count);
3960                         pmc->pm_sc.pm_reloadcount = MAX(1000, pa.pm_count);
3961                 } else
3962                         pmc->pm_sc.pm_initial = pa.pm_count;
3963
3964                 /* switch thread to CPU 'cpu' */
3965                 pmc_save_cpu_binding(&pb);
3966
3967 #define PMC_IS_SHAREABLE_PMC(cpu, n)                            \
3968         (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_state &           \
3969          PMC_PHW_FLAG_IS_SHAREABLE)
3970 #define PMC_IS_UNALLOCATED(cpu, n)                              \
3971         (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_pmc == NULL)
3972
3973                 if (PMC_IS_SYSTEM_MODE(mode)) {
3974                         pmc_select_cpu(cpu);
3975                         for (n = pcd->pcd_ri; n < (int) md->pmd_npmc; n++) {
3976                                 pcd = pmc_ri_to_classdep(md, n, &adjri);
3977                                 if (pmc_can_allocate_row(n, mode) == 0 &&
3978                                     pmc_can_allocate_rowindex(
3979                                             curthread->td_proc, n, cpu) == 0 &&
3980                                     (PMC_IS_UNALLOCATED(cpu, n) ||
3981                                      PMC_IS_SHAREABLE_PMC(cpu, n)) &&
3982                                     pcd->pcd_allocate_pmc(cpu, adjri, pmc,
3983                                         &pa) == 0)
3984                                         break;
3985                         }
3986                 } else {
3987                         /* Process virtual mode */
3988                         for (n = pcd->pcd_ri; n < (int) md->pmd_npmc; n++) {
3989                                 pcd = pmc_ri_to_classdep(md, n, &adjri);
3990                                 if (pmc_can_allocate_row(n, mode) == 0 &&
3991                                     pmc_can_allocate_rowindex(
3992                                             curthread->td_proc, n,
3993                                             PMC_CPU_ANY) == 0 &&
3994                                     pcd->pcd_allocate_pmc(curthread->td_oncpu,
3995                                         adjri, pmc, &pa) == 0)
3996                                         break;
3997                         }
3998                 }
3999
4000 #undef  PMC_IS_UNALLOCATED
4001 #undef  PMC_IS_SHAREABLE_PMC
4002
4003                 pmc_restore_cpu_binding(&pb);
4004
4005                 if (n == (int) md->pmd_npmc) {
4006                         pmc_destroy_pmc_descriptor(pmc);
4007                         pmc = NULL;
4008                         error = EINVAL;
4009                         break;
4010                 }
4011
4012                 /* Fill in the correct value in the ID field */
4013                 pmc->pm_id = PMC_ID_MAKE_ID(cpu,mode,pa.pm_class,n);
4014
4015                 PMCDBG5(PMC,ALL,2, "ev=%d class=%d mode=%d n=%d -> pmcid=%x",
4016                     pmc->pm_event, pa.pm_class, mode, n, pmc->pm_id);
4017
4018                 /* Process mode PMCs with logging enabled need log files */
4019                 if (pmc->pm_flags & (PMC_F_LOG_PROCEXIT | PMC_F_LOG_PROCCSW))
4020                         pmc->pm_flags |= PMC_F_NEEDS_LOGFILE;
4021
4022                 /* All system mode sampling PMCs require a log file */
4023                 if (PMC_IS_SAMPLING_MODE(mode) && PMC_IS_SYSTEM_MODE(mode))
4024                         pmc->pm_flags |= PMC_F_NEEDS_LOGFILE;
4025
4026                 /*
4027                  * Configure global pmc's immediately
4028                  */
4029
4030                 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pmc))) {
4031
4032                         pmc_save_cpu_binding(&pb);
4033                         pmc_select_cpu(cpu);
4034
4035                         phw = pmc_pcpu[cpu]->pc_hwpmcs[n];
4036                         pcd = pmc_ri_to_classdep(md, n, &adjri);
4037
4038                         if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0 ||
4039                             (error = pcd->pcd_config_pmc(cpu, adjri, pmc)) != 0) {
4040                                 (void) pcd->pcd_release_pmc(cpu, adjri, pmc);
4041                                 pmc_destroy_pmc_descriptor(pmc);
4042                                 pmc = NULL;
4043                                 pmc_restore_cpu_binding(&pb);
4044                                 error = EPERM;
4045                                 break;
4046                         }
4047
4048                         pmc_restore_cpu_binding(&pb);
4049                 }
4050
4051                 pmc->pm_state    = PMC_STATE_ALLOCATED;
4052                 pmc->pm_class   = pa.pm_class;
4053
4054                 /*
4055                  * mark row disposition
4056                  */
4057
4058                 if (PMC_IS_SYSTEM_MODE(mode))
4059                         PMC_MARK_ROW_STANDALONE(n);
4060                 else
4061                         PMC_MARK_ROW_THREAD(n);
4062
4063                 /*
4064                  * Register this PMC with the current thread as its owner.
4065                  */
4066
4067                 if ((error =
4068                     pmc_register_owner(curthread->td_proc, pmc)) != 0) {
4069                         pmc_release_pmc_descriptor(pmc);
4070                         pmc_destroy_pmc_descriptor(pmc);
4071                         pmc = NULL;
4072                         break;
4073                 }
4074
4075
4076                 /*
4077                  * Return the allocated index.
4078                  */
4079
4080                 pa.pm_pmcid = pmc->pm_id;
4081
4082                 error = copyout(&pa, arg, sizeof(pa));
4083         }
4084         break;
4085
4086
4087         /*
4088          * Attach a PMC to a process.
4089          */
4090
4091         case PMC_OP_PMCATTACH:
4092         {
4093                 struct pmc *pm;
4094                 struct proc *p;
4095                 struct pmc_op_pmcattach a;
4096
4097                 sx_assert(&pmc_sx, SX_XLOCKED);
4098
4099                 if ((error = copyin(arg, &a, sizeof(a))) != 0)
4100                         break;
4101
4102                 if (a.pm_pid < 0) {
4103                         error = EINVAL;
4104                         break;
4105                 } else if (a.pm_pid == 0)
4106                         a.pm_pid = td->td_proc->p_pid;
4107
4108                 if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0)
4109                         break;
4110
4111                 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
4112                         error = EINVAL;
4113                         break;
4114                 }
4115
4116                 /* PMCs may be (re)attached only when allocated or stopped */
4117                 if (pm->pm_state == PMC_STATE_RUNNING) {
4118                         error = EBUSY;
4119                         break;
4120                 } else if (pm->pm_state != PMC_STATE_ALLOCATED &&
4121                     pm->pm_state != PMC_STATE_STOPPED) {
4122                         error = EINVAL;
4123                         break;
4124                 }
4125
4126                 /* lookup pid */
4127                 if ((p = pfind(a.pm_pid)) == NULL) {
4128                         error = ESRCH;
4129                         break;
4130                 }
4131
4132                 /*
4133                  * Ignore processes that are working on exiting.
4134                  */
4135                 if (p->p_flag & P_WEXIT) {
4136                         error = ESRCH;
4137                         PROC_UNLOCK(p); /* pfind() returns a locked process */
4138                         break;
4139                 }
4140
4141                 /*
4142                  * we are allowed to attach a PMC to a process if
4143                  * we can debug it.
4144                  */
4145                 error = p_candebug(curthread, p);
4146
4147                 PROC_UNLOCK(p);
4148
4149                 if (error == 0)
4150                         error = pmc_attach_process(p, pm);
4151         }
4152         break;
4153
4154
4155         /*
4156          * Detach an attached PMC from a process.
4157          */
4158
4159         case PMC_OP_PMCDETACH:
4160         {
4161                 struct pmc *pm;
4162                 struct proc *p;
4163                 struct pmc_op_pmcattach a;
4164
4165                 if ((error = copyin(arg, &a, sizeof(a))) != 0)
4166                         break;
4167
4168                 if (a.pm_pid < 0) {
4169                         error = EINVAL;
4170                         break;
4171                 } else if (a.pm_pid == 0)
4172                         a.pm_pid = td->td_proc->p_pid;
4173
4174                 if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0)
4175                         break;
4176
4177                 if ((p = pfind(a.pm_pid)) == NULL) {
4178                         error = ESRCH;
4179                         break;
4180                 }
4181
4182                 /*
4183                  * Treat processes that are in the process of exiting
4184                  * as if they were not present.
4185                  */
4186
4187                 if (p->p_flag & P_WEXIT)
4188                         error = ESRCH;
4189
4190                 PROC_UNLOCK(p); /* pfind() returns a locked process */
4191
4192                 if (error == 0)
4193                         error = pmc_detach_process(p, pm);
4194         }
4195         break;
4196
4197
4198         /*
4199          * Retrieve the MSR number associated with the counter
4200          * 'pmc_id'.  This allows processes to directly use RDPMC
4201          * instructions to read their PMCs, without the overhead of a
4202          * system call.
4203          */
4204
4205         case PMC_OP_PMCGETMSR:
4206         {
4207                 int adjri, ri;
4208                 struct pmc *pm;
4209                 struct pmc_target *pt;
4210                 struct pmc_op_getmsr gm;
4211                 struct pmc_classdep *pcd;
4212
4213                 PMC_DOWNGRADE_SX();
4214
4215                 if ((error = copyin(arg, &gm, sizeof(gm))) != 0)
4216                         break;
4217
4218                 if ((error = pmc_find_pmc(gm.pm_pmcid, &pm)) != 0)
4219                         break;
4220
4221                 /*
4222                  * The allocated PMC has to be a process virtual PMC,
4223                  * i.e., of type MODE_T[CS].  Global PMCs can only be
4224                  * read using the PMCREAD operation since they may be
4225                  * allocated on a different CPU than the one we could
4226                  * be running on at the time of the RDPMC instruction.
4227                  *
4228                  * The GETMSR operation is not allowed for PMCs that
4229                  * are inherited across processes.
4230                  */
4231
4232                 if (!PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) ||
4233                     (pm->pm_flags & PMC_F_DESCENDANTS)) {
4234                         error = EINVAL;
4235                         break;
4236                 }
4237
4238                 /*
4239                  * It only makes sense to use a RDPMC (or its
4240                  * equivalent instruction on non-x86 architectures) on
4241                  * a process that has allocated and attached a PMC to
4242                  * itself.  Conversely the PMC is only allowed to have
4243                  * one process attached to it -- its owner.
4244                  */
4245
4246                 if ((pt = LIST_FIRST(&pm->pm_targets)) == NULL ||
4247                     LIST_NEXT(pt, pt_next) != NULL ||
4248                     pt->pt_process->pp_proc != pm->pm_owner->po_owner) {
4249                         error = EINVAL;
4250                         break;
4251                 }
4252
4253                 ri = PMC_TO_ROWINDEX(pm);
4254                 pcd = pmc_ri_to_classdep(md, ri, &adjri);
4255
4256                 /* PMC class has no 'GETMSR' support */
4257                 if (pcd->pcd_get_msr == NULL) {
4258                         error = ENOSYS;
4259                         break;
4260                 }
4261
4262                 if ((error = (*pcd->pcd_get_msr)(adjri, &gm.pm_msr)) < 0)
4263                         break;
4264
4265                 if ((error = copyout(&gm, arg, sizeof(gm))) < 0)
4266                         break;
4267
4268                 /*
4269                  * Mark our process as using MSRs.  Update machine
4270                  * state using a forced context switch.
4271                  */
4272
4273                 pt->pt_process->pp_flags |= PMC_PP_ENABLE_MSR_ACCESS;
4274                 pmc_force_context_switch();
4275
4276         }
4277         break;
4278
4279         /*
4280          * Release an allocated PMC
4281          */
4282
4283         case PMC_OP_PMCRELEASE:
4284         {
4285                 pmc_id_t pmcid;
4286                 struct pmc *pm;
4287                 struct pmc_owner *po;
4288                 struct pmc_op_simple sp;
4289
4290                 /*
4291                  * Find PMC pointer for the named PMC.
4292                  *
4293                  * Use pmc_release_pmc_descriptor() to switch off the
4294                  * PMC, remove all its target threads, and remove the
4295                  * PMC from its owner's list.
4296                  *
4297                  * Remove the owner record if this is the last PMC
4298                  * owned.
4299                  *
4300                  * Free up space.
4301                  */
4302
4303                 if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
4304                         break;
4305
4306                 pmcid = sp.pm_pmcid;
4307
4308                 if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
4309                         break;
4310
4311                 po = pm->pm_owner;
4312                 pmc_release_pmc_descriptor(pm);
4313                 pmc_maybe_remove_owner(po);
4314                 pmc_destroy_pmc_descriptor(pm);
4315         }
4316         break;
4317
4318
4319         /*
4320          * Read and/or write a PMC.
4321          */
4322
4323         case PMC_OP_PMCRW:
4324         {
4325                 int adjri;
4326                 struct pmc *pm;
4327                 uint32_t cpu, ri;
4328                 pmc_value_t oldvalue;
4329                 struct pmc_binding pb;
4330                 struct pmc_op_pmcrw prw;
4331                 struct pmc_classdep *pcd;
4332                 struct pmc_op_pmcrw *pprw;
4333
4334                 PMC_DOWNGRADE_SX();
4335
4336                 if ((error = copyin(arg, &prw, sizeof(prw))) != 0)
4337                         break;
4338
4339                 ri = 0;
4340                 PMCDBG2(PMC,OPS,1, "rw id=%d flags=0x%x", prw.pm_pmcid,
4341                     prw.pm_flags);
4342
4343                 /* must have at least one flag set */
4344                 if ((prw.pm_flags & (PMC_F_OLDVALUE|PMC_F_NEWVALUE)) == 0) {
4345                         error = EINVAL;
4346                         break;
4347                 }
4348
4349                 /* locate pmc descriptor */
4350                 if ((error = pmc_find_pmc(prw.pm_pmcid, &pm)) != 0)
4351                         break;
4352
4353                 /* Can't read a PMC that hasn't been started. */
4354                 if (pm->pm_state != PMC_STATE_ALLOCATED &&
4355                     pm->pm_state != PMC_STATE_STOPPED &&
4356                     pm->pm_state != PMC_STATE_RUNNING) {
4357                         error = EINVAL;
4358                         break;
4359                 }
4360
4361                 /* writing a new value is allowed only for 'STOPPED' pmcs */
4362                 if (pm->pm_state == PMC_STATE_RUNNING &&
4363                     (prw.pm_flags & PMC_F_NEWVALUE)) {
4364                         error = EBUSY;
4365                         break;
4366                 }
4367
4368                 if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) {
4369
4370                         /*
4371                          * If this PMC is attached to its owner (i.e.,
4372                          * the process requesting this operation) and
4373                          * is running, then attempt to get an
4374                          * upto-date reading from hardware for a READ.
4375                          * Writes are only allowed when the PMC is
4376                          * stopped, so only update the saved value
4377                          * field.
4378                          *
4379                          * If the PMC is not running, or is not
4380                          * attached to its owner, read/write to the
4381                          * savedvalue field.
4382                          */
4383
4384                         ri = PMC_TO_ROWINDEX(pm);
4385                         pcd = pmc_ri_to_classdep(md, ri, &adjri);
4386
4387                         mtx_pool_lock_spin(pmc_mtxpool, pm);
4388                         cpu = curthread->td_oncpu;
4389
4390                         if (prw.pm_flags & PMC_F_OLDVALUE) {
4391                                 if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) &&
4392                                     (pm->pm_state == PMC_STATE_RUNNING))
4393                                         error = (*pcd->pcd_read_pmc)(cpu, adjri,
4394                                             &oldvalue);
4395                                 else
4396                                         oldvalue = pm->pm_gv.pm_savedvalue;
4397                         }
4398                         if (prw.pm_flags & PMC_F_NEWVALUE)
4399                                 pm->pm_gv.pm_savedvalue = prw.pm_value;
4400
4401                         mtx_pool_unlock_spin(pmc_mtxpool, pm);
4402
4403                 } else { /* System mode PMCs */
4404                         cpu = PMC_TO_CPU(pm);
4405                         ri  = PMC_TO_ROWINDEX(pm);
4406                         pcd = pmc_ri_to_classdep(md, ri, &adjri);
4407
4408                         if (!pmc_cpu_is_active(cpu)) {
4409                                 error = ENXIO;
4410                                 break;
4411                         }
4412
4413                         /* move this thread to CPU 'cpu' */
4414                         pmc_save_cpu_binding(&pb);
4415                         pmc_select_cpu(cpu);
4416
4417                         critical_enter();
4418                         /* save old value */
4419                         if (prw.pm_flags & PMC_F_OLDVALUE)
4420                                 if ((error = (*pcd->pcd_read_pmc)(cpu, adjri,
4421                                          &oldvalue)))
4422                                         goto error;
4423                         /* write out new value */
4424                         if (prw.pm_flags & PMC_F_NEWVALUE)
4425                                 error = (*pcd->pcd_write_pmc)(cpu, adjri,
4426                                     prw.pm_value);
4427                 error:
4428                         critical_exit();
4429                         pmc_restore_cpu_binding(&pb);
4430                         if (error)
4431                                 break;
4432                 }
4433
4434                 pprw = (struct pmc_op_pmcrw *) arg;
4435
4436 #ifdef  HWPMC_DEBUG
4437                 if (prw.pm_flags & PMC_F_NEWVALUE)
4438                         PMCDBG3(PMC,OPS,2, "rw id=%d new %jx -> old %jx",
4439                             ri, prw.pm_value, oldvalue);
4440                 else if (prw.pm_flags & PMC_F_OLDVALUE)
4441                         PMCDBG2(PMC,OPS,2, "rw id=%d -> old %jx", ri, oldvalue);
4442 #endif
4443
4444                 /* return old value if requested */
4445                 if (prw.pm_flags & PMC_F_OLDVALUE)
4446                         if ((error = copyout(&oldvalue, &pprw->pm_value,
4447                                  sizeof(prw.pm_value))))
4448                                 break;
4449
4450         }
4451         break;
4452
4453
4454         /*
4455          * Set the sampling rate for a sampling mode PMC and the
4456          * initial count for a counting mode PMC.
4457          */
4458
4459         case PMC_OP_PMCSETCOUNT:
4460         {
4461                 struct pmc *pm;
4462                 struct pmc_op_pmcsetcount sc;
4463
4464                 PMC_DOWNGRADE_SX();
4465
4466                 if ((error = copyin(arg, &sc, sizeof(sc))) != 0)
4467                         break;
4468
4469                 if ((error = pmc_find_pmc(sc.pm_pmcid, &pm)) != 0)
4470                         break;
4471
4472                 if (pm->pm_state == PMC_STATE_RUNNING) {
4473                         error = EBUSY;
4474                         break;
4475                 }
4476
4477                 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
4478                         /*
4479                          * Don't permit requested sample rate to be less than 1000
4480                          */
4481                         if (sc.pm_count < 1000)
4482                                 log(LOG_WARNING,
4483                                         "pmcsetcount: passed sample rate %ju - setting to 1000\n",
4484                                         (uintmax_t)sc.pm_count);
4485                         pm->pm_sc.pm_reloadcount = MAX(1000, sc.pm_count);
4486                 } else
4487                         pm->pm_sc.pm_initial = sc.pm_count;
4488         }
4489         break;
4490
4491
4492         /*
4493          * Start a PMC.
4494          */
4495
4496         case PMC_OP_PMCSTART:
4497         {
4498                 pmc_id_t pmcid;
4499                 struct pmc *pm;
4500                 struct pmc_op_simple sp;
4501
4502                 sx_assert(&pmc_sx, SX_XLOCKED);
4503
4504                 if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
4505                         break;
4506
4507                 pmcid = sp.pm_pmcid;
4508
4509                 if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
4510                         break;
4511
4512                 KASSERT(pmcid == pm->pm_id,
4513                     ("[pmc,%d] pmcid %x != id %x", __LINE__,
4514                         pm->pm_id, pmcid));
4515
4516                 if (pm->pm_state == PMC_STATE_RUNNING) /* already running */
4517                         break;
4518                 else if (pm->pm_state != PMC_STATE_STOPPED &&
4519                     pm->pm_state != PMC_STATE_ALLOCATED) {
4520                         error = EINVAL;
4521                         break;
4522                 }
4523
4524                 error = pmc_start(pm);
4525         }
4526         break;
4527
4528
4529         /*
4530          * Stop a PMC.
4531          */
4532
4533         case PMC_OP_PMCSTOP:
4534         {
4535                 pmc_id_t pmcid;
4536                 struct pmc *pm;
4537                 struct pmc_op_simple sp;
4538
4539                 PMC_DOWNGRADE_SX();
4540
4541                 if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
4542                         break;
4543
4544                 pmcid = sp.pm_pmcid;
4545
4546                 /*
4547                  * Mark the PMC as inactive and invoke the MD stop
4548                  * routines if needed.
4549                  */
4550
4551                 if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
4552                         break;
4553
4554                 KASSERT(pmcid == pm->pm_id,
4555                     ("[pmc,%d] pmc id %x != pmcid %x", __LINE__,
4556                         pm->pm_id, pmcid));
4557
4558                 if (pm->pm_state == PMC_STATE_STOPPED) /* already stopped */
4559                         break;
4560                 else if (pm->pm_state != PMC_STATE_RUNNING) {
4561                         error = EINVAL;
4562                         break;
4563                 }
4564
4565                 error = pmc_stop(pm);
4566         }
4567         break;
4568
4569
4570         /*
4571          * Write a user supplied value to the log file.
4572          */
4573
4574         case PMC_OP_WRITELOG:
4575         {
4576                 struct pmc_op_writelog wl;
4577                 struct pmc_owner *po;
4578
4579                 PMC_DOWNGRADE_SX();
4580
4581                 if ((error = copyin(arg, &wl, sizeof(wl))) != 0)
4582                         break;
4583
4584                 if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
4585                         error = EINVAL;
4586                         break;
4587                 }
4588
4589                 if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) {
4590                         error = EINVAL;
4591                         break;
4592                 }
4593
4594                 error = pmclog_process_userlog(po, &wl);
4595         }
4596         break;
4597
4598
4599         default:
4600                 error = EINVAL;
4601                 break;
4602         }
4603
4604         if (is_sx_downgraded)
4605                 sx_sunlock(&pmc_sx);
4606         else
4607                 sx_xunlock(&pmc_sx);
4608 done_syscall:
4609         if (error)
4610                 counter_u64_add(pmc_stats.pm_syscall_errors, 1);
4611
4612         return (error);
4613 }
4614
4615 /*
4616  * Helper functions
4617  */
4618
4619
4620 /*
4621  * Mark the thread as needing callchain capture and post an AST.  The
4622  * actual callchain capture will be done in a context where it is safe
4623  * to take page faults.
4624  */
4625
4626 static void
4627 pmc_post_callchain_callback(void)
4628 {
4629         struct thread *td;
4630
4631         td = curthread;
4632
4633         /*
4634          * If there is multiple PMCs for the same interrupt ignore new post
4635          */
4636         if (td->td_pflags & TDP_CALLCHAIN)
4637                 return;
4638
4639         /*
4640          * Mark this thread as needing callchain capture.
4641          * `td->td_pflags' will be safe to touch because this thread
4642          * was in user space when it was interrupted.
4643          */
4644         td->td_pflags |= TDP_CALLCHAIN;
4645
4646         /*
4647          * Don't let this thread migrate between CPUs until callchain
4648          * capture completes.
4649          */
4650         sched_pin();
4651
4652         return;
4653 }
4654
4655 /*
4656  * Find a free slot in the per-cpu array of samples and capture the
4657  * current callchain there.  If a sample was successfully added, a bit
4658  * is set in mask 'pmc_cpumask' denoting that the DO_SAMPLES hook
4659  * needs to be invoked from the clock handler.
4660  *
4661  * This function is meant to be called from an NMI handler.  It cannot
4662  * use any of the locking primitives supplied by the OS.
4663  */
4664
4665 static int
4666 pmc_add_sample(ring_type_t ring, struct pmc *pm, struct trapframe *tf)
4667 {
4668         int error, cpu, callchaindepth, inuserspace;
4669         struct thread *td;
4670         struct pmc_sample *ps;
4671         struct pmc_samplebuffer *psb;
4672
4673         error = 0;
4674
4675         /*
4676          * Allocate space for a sample buffer.
4677          */
4678         cpu = curcpu;
4679         psb = pmc_pcpu[cpu]->pc_sb[ring];
4680         inuserspace = TRAPF_USERMODE(tf);
4681         ps = PMC_PROD_SAMPLE(psb);
4682         if (psb->ps_considx != psb->ps_prodidx &&
4683                 ps->ps_nsamples) {      /* in use, reader hasn't caught up */
4684                 pm->pm_pcpu_state[cpu].pps_stalled = 1;
4685                 counter_u64_add(pmc_stats.pm_intr_bufferfull, 1);
4686                 PMCDBG6(SAM,INT,1,"(spc) cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d",
4687                     cpu, pm, (void *) tf, inuserspace,
4688                     (int) (psb->ps_prodidx & pmc_sample_mask),
4689                     (int) (psb->ps_considx & pmc_sample_mask));
4690                 callchaindepth = 1;
4691                 error = ENOMEM;
4692                 goto done;
4693         }
4694
4695         /* Fill in entry. */
4696         PMCDBG6(SAM,INT,1,"cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d", cpu, pm,
4697             (void *) tf, inuserspace,
4698             (int) (psb->ps_prodidx & pmc_sample_mask),
4699             (int) (psb->ps_considx & pmc_sample_mask));
4700
4701         td = curthread;
4702         ps->ps_pmc = pm;
4703         ps->ps_td = td;
4704         ps->ps_pid = td->td_proc->p_pid;
4705         ps->ps_tid = td->td_tid;
4706         ps->ps_tsc = pmc_rdtsc();
4707         ps->ps_ticks = ticks;
4708         ps->ps_cpu = cpu;
4709         ps->ps_flags = inuserspace ? PMC_CC_F_USERSPACE : 0;
4710
4711         callchaindepth = (pm->pm_flags & PMC_F_CALLCHAIN) ?
4712             pmc_callchaindepth : 1;
4713
4714         MPASS(ps->ps_pc != NULL);
4715         if (callchaindepth == 1)
4716                 ps->ps_pc[0] = PMC_TRAPFRAME_TO_PC(tf);
4717         else {
4718                 /*
4719                  * Kernel stack traversals can be done immediately,
4720                  * while we defer to an AST for user space traversals.
4721                  */
4722                 if (!inuserspace) {
4723                         callchaindepth =
4724                             pmc_save_kernel_callchain(ps->ps_pc,
4725                                 callchaindepth, tf);
4726                 } else {
4727                         pmc_post_callchain_callback();
4728                         callchaindepth = PMC_USER_CALLCHAIN_PENDING;
4729                 }
4730         }
4731
4732         ps->ps_nsamples = callchaindepth;       /* mark entry as in use */
4733         if (ring == PMC_UR) {
4734                 ps->ps_nsamples_actual = callchaindepth;        /* mark entry as in use */
4735                 ps->ps_nsamples = PMC_USER_CALLCHAIN_PENDING;
4736         } else
4737                 ps->ps_nsamples = callchaindepth;       /* mark entry as in use */
4738
4739         KASSERT(counter_u64_fetch(pm->pm_runcount) >= 0,
4740             ("[pmc,%d] pm=%p runcount %ld", __LINE__, (void *) pm,
4741                  (unsigned long)counter_u64_fetch(pm->pm_runcount)));
4742
4743         counter_u64_add(pm->pm_runcount, 1);    /* hold onto PMC */
4744         /* increment write pointer */
4745         psb->ps_prodidx++;
4746  done:
4747         /* mark CPU as needing processing */
4748         if (callchaindepth != PMC_USER_CALLCHAIN_PENDING)
4749                 DPCPU_SET(pmc_sampled, 1);
4750
4751         return (error);
4752 }
4753
4754 /*
4755  * Interrupt processing.
4756  *
4757  * This function is meant to be called from an NMI handler.  It cannot
4758  * use any of the locking primitives supplied by the OS.
4759  */
4760
4761 int
4762 pmc_process_interrupt(int ring, struct pmc *pm, struct trapframe *tf)
4763 {
4764         struct thread *td;
4765
4766         td = curthread;
4767         if ((pm->pm_flags & PMC_F_USERCALLCHAIN) &&
4768             (td->td_proc->p_flag & P_KPROC) == 0 &&
4769             !TRAPF_USERMODE(tf)) {
4770                 atomic_add_int(&td->td_pmcpend, 1);
4771                 return (pmc_add_sample(PMC_UR, pm, tf));
4772         }
4773         return (pmc_add_sample(ring, pm, tf));
4774 }
4775
4776 /*
4777  * Capture a user call chain.  This function will be called from ast()
4778  * before control returns to userland and before the process gets
4779  * rescheduled.
4780  */
4781
4782 static void
4783 pmc_capture_user_callchain(int cpu, int ring, struct trapframe *tf)
4784 {
4785         struct pmc *pm;
4786         struct thread *td;
4787         struct pmc_sample *ps;
4788         struct pmc_samplebuffer *psb;
4789         uint64_t considx, prodidx;
4790         int nsamples, nrecords, pass, iter;
4791 #ifdef  INVARIANTS
4792         int ncallchains;
4793         int nfree;
4794         int start_ticks = ticks;
4795 #endif
4796         psb = pmc_pcpu[cpu]->pc_sb[ring];
4797         td = curthread;
4798
4799         KASSERT(td->td_pflags & TDP_CALLCHAIN,
4800             ("[pmc,%d] Retrieving callchain for thread that doesn't want it",
4801                 __LINE__));
4802
4803 #ifdef  INVARIANTS
4804         ncallchains = 0;
4805         nfree = 0;
4806 #endif
4807         nrecords = INT_MAX;
4808         pass = 0;
4809  restart:
4810         if (ring == PMC_UR)
4811                 nrecords = atomic_readandclear_32(&td->td_pmcpend);
4812
4813         for (iter = 0, considx = psb->ps_considx, prodidx = psb->ps_prodidx;
4814             considx < prodidx && iter < pmc_nsamples; considx++, iter++) {
4815                 ps = PMC_CONS_SAMPLE_OFF(psb, considx);
4816
4817         /*
4818          * Iterate through all deferred callchain requests.
4819          * Walk from the current read pointer to the current
4820          * write pointer.
4821          */
4822
4823 #ifdef  INVARIANTS
4824                 if (ps->ps_nsamples == PMC_SAMPLE_FREE) {
4825                         nfree++;
4826                         continue;
4827                 }
4828
4829                 if ((ps->ps_pmc == NULL) ||
4830                     (ps->ps_pmc->pm_state != PMC_STATE_RUNNING))
4831                         nfree++;
4832 #endif
4833                 if (ps->ps_td != td ||
4834                    ps->ps_nsamples != PMC_USER_CALLCHAIN_PENDING ||
4835                    ps->ps_pmc->pm_state != PMC_STATE_RUNNING)
4836                         continue;
4837
4838                 KASSERT(ps->ps_cpu == cpu,
4839                     ("[pmc,%d] cpu mismatch ps_cpu=%d pcpu=%d", __LINE__,
4840                         ps->ps_cpu, PCPU_GET(cpuid)));
4841
4842                 pm = ps->ps_pmc;
4843
4844                 KASSERT(pm->pm_flags & PMC_F_CALLCHAIN,
4845                     ("[pmc,%d] Retrieving callchain for PMC that doesn't "
4846                         "want it", __LINE__));
4847
4848                 KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
4849                     ("[pmc,%d] runcount %ld", __LINE__, (unsigned long)counter_u64_fetch(pm->pm_runcount)));
4850
4851                 if (ring == PMC_UR) {
4852                         nsamples = ps->ps_nsamples_actual;
4853                         counter_u64_add(pmc_stats.pm_merges, 1);
4854                 } else
4855                         nsamples = 0;
4856
4857                 /*
4858                  * Retrieve the callchain and mark the sample buffer
4859                  * as 'processable' by the timer tick sweep code.
4860                  */
4861
4862 #ifdef INVARIANTS
4863                 ncallchains++;
4864 #endif
4865
4866                 if (__predict_true(nsamples < pmc_callchaindepth - 1))
4867                         nsamples += pmc_save_user_callchain(ps->ps_pc + nsamples,
4868                        pmc_callchaindepth - nsamples - 1, tf);
4869
4870                 /*
4871                  * We have to prevent hardclock from potentially overwriting
4872                  * this sample between when we read the value and when we set
4873                  * it
4874                  */
4875                 spinlock_enter();
4876                 /*
4877                  * Verify that the sample hasn't been dropped in the meantime
4878                  */
4879                 if (ps->ps_nsamples == PMC_USER_CALLCHAIN_PENDING) {
4880                         ps->ps_nsamples = nsamples;
4881                         /*
4882                          * If we couldn't get a sample, simply drop the reference
4883                          */
4884                         if (nsamples == 0)
4885                                 counter_u64_add(pm->pm_runcount, -1);
4886                 }
4887                 spinlock_exit();
4888                 if (nrecords-- == 1)
4889                         break;
4890         }
4891         if (__predict_false(ring == PMC_UR && td->td_pmcpend)) {
4892                 if (pass == 0) {
4893                         pass = 1;
4894                         goto restart;
4895                 }
4896                 /* only collect samples for this part once */
4897                 td->td_pmcpend = 0;
4898         }
4899
4900 #ifdef INVARIANTS
4901         if ((ticks - start_ticks) > hz)
4902                 log(LOG_ERR, "%s took %d ticks\n", __func__, (ticks - start_ticks));
4903 #endif
4904
4905         /* mark CPU as needing processing */
4906         DPCPU_SET(pmc_sampled, 1);
4907 }
4908
4909 /*
4910  * Process saved PC samples.
4911  */
4912
4913 static void
4914 pmc_process_samples(int cpu, ring_type_t ring)
4915 {
4916         struct pmc *pm;
4917         int adjri, n;
4918         struct thread *td;
4919         struct pmc_owner *po;
4920         struct pmc_sample *ps;
4921         struct pmc_classdep *pcd;
4922         struct pmc_samplebuffer *psb;
4923         uint64_t delta;
4924
4925         KASSERT(PCPU_GET(cpuid) == cpu,
4926             ("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__,
4927                 PCPU_GET(cpuid), cpu));
4928
4929         psb = pmc_pcpu[cpu]->pc_sb[ring];
4930         delta = psb->ps_prodidx - psb->ps_considx;
4931         MPASS(delta <= pmc_nsamples);
4932         MPASS(psb->ps_considx <= psb->ps_prodidx);
4933         for (n = 0; psb->ps_considx < psb->ps_prodidx; psb->ps_considx++, n++) {
4934                 ps = PMC_CONS_SAMPLE(psb);
4935
4936                 if (__predict_false(ps->ps_nsamples == PMC_SAMPLE_FREE))
4937                         continue;
4938                 pm = ps->ps_pmc;
4939                 /* skip non-running samples */
4940                 if (pm->pm_state != PMC_STATE_RUNNING)
4941                         goto entrydone;
4942
4943                 KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
4944                     ("[pmc,%d] pm=%p runcount %ld", __LINE__, (void *) pm,
4945                          (unsigned long)counter_u64_fetch(pm->pm_runcount)));
4946
4947                 po = pm->pm_owner;
4948
4949                 KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
4950                     ("[pmc,%d] pmc=%p non-sampling mode=%d", __LINE__,
4951                         pm, PMC_TO_MODE(pm)));
4952
4953
4954                 /* If there is a pending AST wait for completion */
4955                 if (ps->ps_nsamples == PMC_USER_CALLCHAIN_PENDING) {
4956                         /* if we've been waiting more than 1 tick to
4957                          * collect a callchain for this record then
4958                          * drop it and move on.
4959                          */
4960                         if (ticks - ps->ps_ticks > 1) {
4961                                 /*
4962                                  * track how often we hit this as it will
4963                                  * preferentially lose user samples
4964                                  * for long running system calls
4965                                  */
4966                                 counter_u64_add(pmc_stats.pm_overwrites, 1);
4967                                 goto entrydone;
4968                         }
4969                         /* Need a rescan at a later time. */
4970                         DPCPU_SET(pmc_sampled, 1);
4971                         break;
4972                 }
4973
4974                 PMCDBG6(SAM,OPS,1,"cpu=%d pm=%p n=%d fl=%x wr=%d rd=%d", cpu,
4975                     pm, ps->ps_nsamples, ps->ps_flags,
4976                     (int) (psb->ps_prodidx & pmc_sample_mask),
4977                     (int) (psb->ps_considx & pmc_sample_mask));
4978
4979                 /*
4980                  * If this is a process-mode PMC that is attached to
4981                  * its owner, and if the PC is in user mode, update
4982                  * profiling statistics like timer-based profiling
4983                  * would have done.
4984                  *
4985                  * Otherwise, this is either a sampling-mode PMC that
4986                  * is attached to a different process than its owner,
4987                  * or a system-wide sampling PMC. Dispatch a log
4988                  * entry to the PMC's owner process.
4989                  */
4990                 if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) {
4991                         if (ps->ps_flags & PMC_CC_F_USERSPACE) {
4992                                 td = FIRST_THREAD_IN_PROC(po->po_owner);
4993                                 addupc_intr(td, ps->ps_pc[0], 1);
4994                         }
4995                 } else
4996                         pmclog_process_callchain(pm, ps);
4997
4998         entrydone:
4999                 ps->ps_nsamples = 0; /* mark entry as free */
5000                 KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
5001                                 ("[pmc,%d] pm=%p runcount %ld", __LINE__, (void *) pm,
5002                                  (unsigned long)counter_u64_fetch(pm->pm_runcount)));
5003
5004                 counter_u64_add(pm->pm_runcount, -1);
5005         }
5006
5007         counter_u64_add(pmc_stats.pm_log_sweeps, 1);
5008
5009         /* Do not re-enable stalled PMCs if we failed to process any samples */
5010         if (n == 0)
5011                 return;
5012
5013         /*
5014          * Restart any stalled sampling PMCs on this CPU.
5015          *
5016          * If the NMI handler sets the pm_stalled field of a PMC after
5017          * the check below, we'll end up processing the stalled PMC at
5018          * the next hardclock tick.
5019          */
5020         for (n = 0; n < md->pmd_npmc; n++) {
5021                 pcd = pmc_ri_to_classdep(md, n, &adjri);
5022                 KASSERT(pcd != NULL,
5023                     ("[pmc,%d] null pcd ri=%d", __LINE__, n));
5024                 (void) (*pcd->pcd_get_config)(cpu,adjri,&pm);
5025
5026                 if (pm == NULL ||                        /* !cfg'ed */
5027                     pm->pm_state != PMC_STATE_RUNNING || /* !active */
5028                     !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || /* !sampling */
5029                         !pm->pm_pcpu_state[cpu].pps_cpustate  || /* !desired */
5030                     !pm->pm_pcpu_state[cpu].pps_stalled) /* !stalled */
5031                         continue;
5032
5033                 pm->pm_pcpu_state[cpu].pps_stalled = 0;
5034                 (*pcd->pcd_start_pmc)(cpu, adjri);
5035         }
5036 }
5037
5038 /*
5039  * Event handlers.
5040  */
5041
5042 /*
5043  * Handle a process exit.
5044  *
5045  * Remove this process from all hash tables.  If this process
5046  * owned any PMCs, turn off those PMCs and deallocate them,
5047  * removing any associations with target processes.
5048  *
5049  * This function will be called by the last 'thread' of a
5050  * process.
5051  *
5052  * XXX This eventhandler gets called early in the exit process.
5053  * Consider using a 'hook' invocation from thread_exit() or equivalent
5054  * spot.  Another negative is that kse_exit doesn't seem to call
5055  * exit1() [??].
5056  *
5057  */
5058
5059 static void
5060 pmc_process_exit(void *arg __unused, struct proc *p)
5061 {
5062         struct pmc *pm;
5063         int adjri, cpu;
5064         unsigned int ri;
5065         int is_using_hwpmcs;
5066         struct pmc_owner *po;
5067         struct pmc_process *pp;
5068         struct pmc_classdep *pcd;
5069         pmc_value_t newvalue, tmp;
5070
5071         PROC_LOCK(p);
5072         is_using_hwpmcs = p->p_flag & P_HWPMC;
5073         PROC_UNLOCK(p);
5074
5075         /*
5076          * Log a sysexit event to all SS PMC owners.
5077          */
5078         PMC_EPOCH_ENTER();
5079         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5080             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
5081                     pmclog_process_sysexit(po, p->p_pid);
5082         PMC_EPOCH_EXIT();
5083
5084         if (!is_using_hwpmcs)
5085                 return;
5086
5087         PMC_GET_SX_XLOCK();
5088         PMCDBG3(PRC,EXT,1,"process-exit proc=%p (%d, %s)", p, p->p_pid,
5089             p->p_comm);
5090
5091         /*
5092          * Since this code is invoked by the last thread in an exiting
5093          * process, we would have context switched IN at some prior
5094          * point.  However, with PREEMPTION, kernel mode context
5095          * switches may happen any time, so we want to disable a
5096          * context switch OUT till we get any PMCs targeting this
5097          * process off the hardware.
5098          *
5099          * We also need to atomically remove this process'
5100          * entry from our target process hash table, using
5101          * PMC_FLAG_REMOVE.
5102          */
5103         PMCDBG3(PRC,EXT,1, "process-exit proc=%p (%d, %s)", p, p->p_pid,
5104             p->p_comm);
5105
5106         critical_enter(); /* no preemption */
5107
5108         cpu = curthread->td_oncpu;
5109
5110         if ((pp = pmc_find_process_descriptor(p,
5111                  PMC_FLAG_REMOVE)) != NULL) {
5112
5113                 PMCDBG2(PRC,EXT,2,
5114                     "process-exit proc=%p pmc-process=%p", p, pp);
5115
5116                 /*
5117                  * The exiting process could the target of
5118                  * some PMCs which will be running on
5119                  * currently executing CPU.
5120                  *
5121                  * We need to turn these PMCs off like we
5122                  * would do at context switch OUT time.
5123                  */
5124                 for (ri = 0; ri < md->pmd_npmc; ri++) {
5125
5126                         /*
5127                          * Pick up the pmc pointer from hardware
5128                          * state similar to the CSW_OUT code.
5129                          */
5130                         pm = NULL;
5131
5132                         pcd = pmc_ri_to_classdep(md, ri, &adjri);
5133
5134                         (void) (*pcd->pcd_get_config)(cpu, adjri, &pm);
5135
5136                         PMCDBG2(PRC,EXT,2, "ri=%d pm=%p", ri, pm);
5137
5138                         if (pm == NULL ||
5139                             !PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
5140                                 continue;
5141
5142                         PMCDBG4(PRC,EXT,2, "ppmcs[%d]=%p pm=%p "
5143                             "state=%d", ri, pp->pp_pmcs[ri].pp_pmc,
5144                             pm, pm->pm_state);
5145
5146                         KASSERT(PMC_TO_ROWINDEX(pm) == ri,
5147                             ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
5148                                 __LINE__, PMC_TO_ROWINDEX(pm), ri));
5149
5150                         KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
5151                             ("[pmc,%d] pm %p != pp_pmcs[%d] %p",
5152                                 __LINE__, pm, ri, pp->pp_pmcs[ri].pp_pmc));
5153
5154                         KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
5155                             ("[pmc,%d] bad runcount ri %d rc %ld",
5156                                  __LINE__, ri, (unsigned long)counter_u64_fetch(pm->pm_runcount)));
5157
5158                         /*
5159                          * Change desired state, and then stop if not
5160                          * stalled. This two-step dance should avoid
5161                          * race conditions where an interrupt re-enables
5162                          * the PMC after this code has already checked
5163                          * the pm_stalled flag.
5164                          */
5165                         if (pm->pm_pcpu_state[cpu].pps_cpustate) {
5166                                 pm->pm_pcpu_state[cpu].pps_cpustate = 0;
5167                                 if (!pm->pm_pcpu_state[cpu].pps_stalled) {
5168                                         (void) pcd->pcd_stop_pmc(cpu, adjri);
5169
5170                                         if (PMC_TO_MODE(pm) == PMC_MODE_TC) {
5171                                                 pcd->pcd_read_pmc(cpu, adjri,
5172                                                     &newvalue);
5173                                                 tmp = newvalue -
5174                                                     PMC_PCPU_SAVED(cpu,ri);
5175
5176                                                 mtx_pool_lock_spin(pmc_mtxpool,
5177                                                     pm);
5178                                                 pm->pm_gv.pm_savedvalue += tmp;
5179                                                 pp->pp_pmcs[ri].pp_pmcval +=
5180                                                     tmp;
5181                                                 mtx_pool_unlock_spin(
5182                                                     pmc_mtxpool, pm);
5183                                         }
5184                                 }
5185                         }
5186
5187                         KASSERT((int64_t) counter_u64_fetch(pm->pm_runcount) > 0,
5188                             ("[pmc,%d] runcount is %d", __LINE__, ri));
5189
5190                         counter_u64_add(pm->pm_runcount, -1);
5191
5192                         (void) pcd->pcd_config_pmc(cpu, adjri, NULL);
5193                 }
5194
5195                 /*
5196                  * Inform the MD layer of this pseudo "context switch
5197                  * out"
5198                  */
5199                 (void) md->pmd_switch_out(pmc_pcpu[cpu], pp);
5200
5201                 critical_exit(); /* ok to be pre-empted now */
5202
5203                 /*
5204                  * Unlink this process from the PMCs that are
5205                  * targeting it.  This will send a signal to
5206                  * all PMC owner's whose PMCs are orphaned.
5207                  *
5208                  * Log PMC value at exit time if requested.
5209                  */
5210                 for (ri = 0; ri < md->pmd_npmc; ri++)
5211                         if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) {
5212                                 if (pm->pm_flags & PMC_F_NEEDS_LOGFILE &&
5213                                     PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm)))
5214                                         pmclog_process_procexit(pm, pp);
5215                                 pmc_unlink_target_process(pm, pp);
5216                         }
5217                 free(pp, M_PMC);
5218
5219         } else
5220                 critical_exit(); /* pp == NULL */
5221
5222
5223         /*
5224          * If the process owned PMCs, free them up and free up
5225          * memory.
5226          */
5227         if ((po = pmc_find_owner_descriptor(p)) != NULL) {
5228                 pmc_remove_owner(po);
5229                 pmc_destroy_owner_descriptor(po);
5230         }
5231
5232         sx_xunlock(&pmc_sx);
5233 }
5234
5235 /*
5236  * Handle a process fork.
5237  *
5238  * If the parent process 'p1' is under HWPMC monitoring, then copy
5239  * over any attached PMCs that have 'do_descendants' semantics.
5240  */
5241
5242 static void
5243 pmc_process_fork(void *arg __unused, struct proc *p1, struct proc *newproc,
5244     int flags)
5245 {
5246         int is_using_hwpmcs;
5247         unsigned int ri;
5248         uint32_t do_descendants;
5249         struct pmc *pm;
5250         struct pmc_owner *po;
5251         struct pmc_process *ppnew, *ppold;
5252
5253         (void) flags;           /* unused parameter */
5254
5255         PROC_LOCK(p1);
5256         is_using_hwpmcs = p1->p_flag & P_HWPMC;
5257         PROC_UNLOCK(p1);
5258
5259         /*
5260          * If there are system-wide sampling PMCs active, we need to
5261          * log all fork events to their owner's logs.
5262          */
5263         PMC_EPOCH_ENTER();
5264         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5265             if (po->po_flags & PMC_PO_OWNS_LOGFILE) {
5266                     pmclog_process_procfork(po, p1->p_pid, newproc->p_pid);
5267                         pmclog_process_proccreate(po, newproc, 1);
5268                 }
5269         PMC_EPOCH_EXIT();
5270
5271         if (!is_using_hwpmcs)
5272                 return;
5273
5274         PMC_GET_SX_XLOCK();
5275         PMCDBG4(PMC,FRK,1, "process-fork proc=%p (%d, %s) -> %p", p1,
5276             p1->p_pid, p1->p_comm, newproc);
5277
5278         /*
5279          * If the parent process (curthread->td_proc) is a
5280          * target of any PMCs, look for PMCs that are to be
5281          * inherited, and link these into the new process
5282          * descriptor.
5283          */
5284         if ((ppold = pmc_find_process_descriptor(curthread->td_proc,
5285                  PMC_FLAG_NONE)) == NULL)
5286                 goto done;              /* nothing to do */
5287
5288         do_descendants = 0;
5289         for (ri = 0; ri < md->pmd_npmc; ri++)
5290                 if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL)
5291                         do_descendants |= pm->pm_flags & PMC_F_DESCENDANTS;
5292         if (do_descendants == 0) /* nothing to do */
5293                 goto done;
5294
5295         /*
5296          * Now mark the new process as being tracked by this driver.
5297          */
5298         PROC_LOCK(newproc);
5299         newproc->p_flag |= P_HWPMC;
5300         PROC_UNLOCK(newproc);
5301
5302         /* allocate a descriptor for the new process  */
5303         if ((ppnew = pmc_find_process_descriptor(newproc,
5304                  PMC_FLAG_ALLOCATE)) == NULL)
5305                 goto done;
5306
5307         /*
5308          * Run through all PMCs that were targeting the old process
5309          * and which specified F_DESCENDANTS and attach them to the
5310          * new process.
5311          *
5312          * Log the fork event to all owners of PMCs attached to this
5313          * process, if not already logged.
5314          */
5315         for (ri = 0; ri < md->pmd_npmc; ri++)
5316                 if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL &&
5317                     (pm->pm_flags & PMC_F_DESCENDANTS)) {
5318                         pmc_link_target_process(pm, ppnew);
5319                         po = pm->pm_owner;
5320                         if (po->po_sscount == 0 &&
5321                             po->po_flags & PMC_PO_OWNS_LOGFILE)
5322                                 pmclog_process_procfork(po, p1->p_pid,
5323                                     newproc->p_pid);
5324                 }
5325
5326  done:
5327         sx_xunlock(&pmc_sx);
5328 }
5329
5330 static void
5331 pmc_process_threadcreate(struct thread *td)
5332 {
5333         struct pmc_owner *po;
5334
5335         PMC_EPOCH_ENTER();
5336         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5337             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
5338                         pmclog_process_threadcreate(po, td, 1);
5339         PMC_EPOCH_EXIT();
5340 }
5341
5342 static void
5343 pmc_process_threadexit(struct thread *td)
5344 {
5345         struct pmc_owner *po;
5346
5347         PMC_EPOCH_ENTER();
5348         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5349             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
5350                         pmclog_process_threadexit(po, td);
5351         PMC_EPOCH_EXIT();
5352 }
5353
5354 static void
5355 pmc_process_proccreate(struct proc *p)
5356 {
5357         struct pmc_owner *po;
5358
5359         PMC_EPOCH_ENTER();
5360         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5361             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
5362                         pmclog_process_proccreate(po, p, 1 /* sync */);
5363         PMC_EPOCH_EXIT();
5364 }
5365
5366 static void
5367 pmc_process_allproc(struct pmc *pm)
5368 {
5369         struct pmc_owner *po;
5370         struct thread *td;
5371         struct proc *p;
5372
5373         po = pm->pm_owner;
5374         if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
5375                 return;
5376         sx_slock(&allproc_lock);
5377         FOREACH_PROC_IN_SYSTEM(p) {
5378                 pmclog_process_proccreate(po, p, 0 /* sync */);
5379                 PROC_LOCK(p);
5380                 FOREACH_THREAD_IN_PROC(p, td)
5381                         pmclog_process_threadcreate(po, td, 0 /* sync */);
5382                 PROC_UNLOCK(p);
5383         }
5384         sx_sunlock(&allproc_lock);
5385         pmclog_flush(po, 0);
5386 }
5387
5388 static void
5389 pmc_kld_load(void *arg __unused, linker_file_t lf)
5390 {
5391         struct pmc_owner *po;
5392
5393         /*
5394          * Notify owners of system sampling PMCs about KLD operations.
5395          */
5396         PMC_EPOCH_ENTER();
5397         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5398                 if (po->po_flags & PMC_PO_OWNS_LOGFILE)
5399                         pmclog_process_map_in(po, (pid_t) -1,
5400                             (uintfptr_t) lf->address, lf->filename);
5401         PMC_EPOCH_EXIT();
5402
5403         /*
5404          * TODO: Notify owners of (all) process-sampling PMCs too.
5405          */
5406 }
5407
5408 static void
5409 pmc_kld_unload(void *arg __unused, const char *filename __unused,
5410     caddr_t address, size_t size)
5411 {
5412         struct pmc_owner *po;
5413
5414         PMC_EPOCH_ENTER();
5415         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5416                 if (po->po_flags & PMC_PO_OWNS_LOGFILE)
5417                         pmclog_process_map_out(po, (pid_t) -1,
5418                             (uintfptr_t) address, (uintfptr_t) address + size);
5419         PMC_EPOCH_EXIT();
5420
5421         /*
5422          * TODO: Notify owners of process-sampling PMCs.
5423          */
5424 }
5425
5426 /*
5427  * initialization
5428  */
5429 static const char *
5430 pmc_name_of_pmcclass(enum pmc_class class)
5431 {
5432
5433         switch (class) {
5434 #undef  __PMC_CLASS
5435 #define __PMC_CLASS(S,V,D)                                              \
5436         case PMC_CLASS_##S:                                             \
5437                 return #S;
5438         __PMC_CLASSES();
5439         default:
5440                 return ("<unknown>");
5441         }
5442 }
5443
5444 /*
5445  * Base class initializer: allocate structure and set default classes.
5446  */
5447 struct pmc_mdep *
5448 pmc_mdep_alloc(int nclasses)
5449 {
5450         struct pmc_mdep *md;
5451         int     n;
5452
5453         /* SOFT + md classes */
5454         n = 1 + nclasses;
5455         md = malloc(sizeof(struct pmc_mdep) + n *
5456             sizeof(struct pmc_classdep), M_PMC, M_WAITOK|M_ZERO);
5457         md->pmd_nclass = n;
5458
5459         /* Add base class. */
5460         pmc_soft_initialize(md);
5461         return md;
5462 }
5463
5464 void
5465 pmc_mdep_free(struct pmc_mdep *md)
5466 {
5467         pmc_soft_finalize(md);
5468         free(md, M_PMC);
5469 }
5470
5471 static int
5472 generic_switch_in(struct pmc_cpu *pc, struct pmc_process *pp)
5473 {
5474         (void) pc; (void) pp;
5475
5476         return (0);
5477 }
5478
5479 static int
5480 generic_switch_out(struct pmc_cpu *pc, struct pmc_process *pp)
5481 {
5482         (void) pc; (void) pp;
5483
5484         return (0);
5485 }
5486
5487 static struct pmc_mdep *
5488 pmc_generic_cpu_initialize(void)
5489 {
5490         struct pmc_mdep *md;
5491
5492         md = pmc_mdep_alloc(0);
5493
5494         md->pmd_cputype    = PMC_CPU_GENERIC;
5495
5496         md->pmd_pcpu_init  = NULL;
5497         md->pmd_pcpu_fini  = NULL;
5498         md->pmd_switch_in  = generic_switch_in;
5499         md->pmd_switch_out = generic_switch_out;
5500
5501         return (md);
5502 }
5503
5504 static void
5505 pmc_generic_cpu_finalize(struct pmc_mdep *md)
5506 {
5507         (void) md;
5508 }
5509
5510
5511 static int
5512 pmc_initialize(void)
5513 {
5514         int c, cpu, error, n, ri;
5515         unsigned int maxcpu, domain;
5516         struct pcpu *pc;
5517         struct pmc_binding pb;
5518         struct pmc_sample *ps;
5519         struct pmc_classdep *pcd;
5520         struct pmc_samplebuffer *sb;
5521
5522         md = NULL;
5523         error = 0;
5524
5525         pmc_stats.pm_intr_ignored = counter_u64_alloc(M_WAITOK);
5526         pmc_stats.pm_intr_processed = counter_u64_alloc(M_WAITOK);
5527         pmc_stats.pm_intr_bufferfull = counter_u64_alloc(M_WAITOK);
5528         pmc_stats.pm_syscalls = counter_u64_alloc(M_WAITOK);
5529         pmc_stats.pm_syscall_errors = counter_u64_alloc(M_WAITOK);
5530         pmc_stats.pm_buffer_requests = counter_u64_alloc(M_WAITOK);
5531         pmc_stats.pm_buffer_requests_failed = counter_u64_alloc(M_WAITOK);
5532         pmc_stats.pm_log_sweeps = counter_u64_alloc(M_WAITOK);
5533         pmc_stats.pm_merges = counter_u64_alloc(M_WAITOK);
5534         pmc_stats.pm_overwrites = counter_u64_alloc(M_WAITOK);
5535
5536 #ifdef  HWPMC_DEBUG
5537         /* parse debug flags first */
5538         if (TUNABLE_STR_FETCH(PMC_SYSCTL_NAME_PREFIX "debugflags",
5539                 pmc_debugstr, sizeof(pmc_debugstr)))
5540                 pmc_debugflags_parse(pmc_debugstr,
5541                     pmc_debugstr+strlen(pmc_debugstr));
5542 #endif
5543
5544         PMCDBG1(MOD,INI,0, "PMC Initialize (version %x)", PMC_VERSION);
5545
5546         /* check kernel version */
5547         if (pmc_kernel_version != PMC_VERSION) {
5548                 if (pmc_kernel_version == 0)
5549                         printf("hwpmc: this kernel has not been compiled with "
5550                             "'options HWPMC_HOOKS'.\n");
5551                 else
5552                         printf("hwpmc: kernel version (0x%x) does not match "
5553                             "module version (0x%x).\n", pmc_kernel_version,
5554                             PMC_VERSION);
5555                 return EPROGMISMATCH;
5556         }
5557
5558         /*
5559          * check sysctl parameters
5560          */
5561
5562         if (pmc_hashsize <= 0) {
5563                 (void) printf("hwpmc: tunable \"hashsize\"=%d must be "
5564                     "greater than zero.\n", pmc_hashsize);
5565                 pmc_hashsize = PMC_HASH_SIZE;
5566         }
5567
5568         if (pmc_nsamples <= 0 || pmc_nsamples > 65535) {
5569                 (void) printf("hwpmc: tunable \"nsamples\"=%d out of "
5570                     "range.\n", pmc_nsamples);
5571                 pmc_nsamples = PMC_NSAMPLES;
5572         }
5573         pmc_sample_mask = pmc_nsamples-1;
5574
5575         if (pmc_callchaindepth <= 0 ||
5576             pmc_callchaindepth > PMC_CALLCHAIN_DEPTH_MAX) {
5577                 (void) printf("hwpmc: tunable \"callchaindepth\"=%d out of "
5578                     "range - using %d.\n", pmc_callchaindepth,
5579                     PMC_CALLCHAIN_DEPTH_MAX);
5580                 pmc_callchaindepth = PMC_CALLCHAIN_DEPTH_MAX;
5581         }
5582
5583         md = pmc_md_initialize();
5584         if (md == NULL) {
5585                 /* Default to generic CPU. */
5586                 md = pmc_generic_cpu_initialize();
5587                 if (md == NULL)
5588                         return (ENOSYS);
5589         }
5590
5591         KASSERT(md->pmd_nclass >= 1 && md->pmd_npmc >= 1,
5592             ("[pmc,%d] no classes or pmcs", __LINE__));
5593
5594         /* Compute the map from row-indices to classdep pointers. */
5595         pmc_rowindex_to_classdep = malloc(sizeof(struct pmc_classdep *) *
5596             md->pmd_npmc, M_PMC, M_WAITOK|M_ZERO);
5597
5598         for (n = 0; n < md->pmd_npmc; n++)
5599                 pmc_rowindex_to_classdep[n] = NULL;
5600         for (ri = c = 0; c < md->pmd_nclass; c++) {
5601                 pcd = &md->pmd_classdep[c];
5602                 for (n = 0; n < pcd->pcd_num; n++, ri++)
5603                         pmc_rowindex_to_classdep[ri] = pcd;
5604         }
5605
5606         KASSERT(ri == md->pmd_npmc,
5607             ("[pmc,%d] npmc miscomputed: ri=%d, md->npmc=%d", __LINE__,
5608             ri, md->pmd_npmc));
5609
5610         maxcpu = pmc_cpu_max();
5611
5612         /* allocate space for the per-cpu array */
5613         pmc_pcpu = malloc(maxcpu * sizeof(struct pmc_cpu *), M_PMC,
5614             M_WAITOK|M_ZERO);
5615
5616         /* per-cpu 'saved values' for managing process-mode PMCs */
5617         pmc_pcpu_saved = malloc(sizeof(pmc_value_t) * maxcpu * md->pmd_npmc,
5618             M_PMC, M_WAITOK);
5619
5620         /* Perform CPU-dependent initialization. */
5621         pmc_save_cpu_binding(&pb);
5622         error = 0;
5623         for (cpu = 0; error == 0 && cpu < maxcpu; cpu++) {
5624                 if (!pmc_cpu_is_active(cpu))
5625                         continue;
5626                 pmc_select_cpu(cpu);
5627                 pmc_pcpu[cpu] = malloc(sizeof(struct pmc_cpu) +
5628                     md->pmd_npmc * sizeof(struct pmc_hw *), M_PMC,
5629                     M_WAITOK|M_ZERO);
5630                 if (md->pmd_pcpu_init)
5631                         error = md->pmd_pcpu_init(md, cpu);
5632                 for (n = 0; error == 0 && n < md->pmd_nclass; n++)
5633                         error = md->pmd_classdep[n].pcd_pcpu_init(md, cpu);
5634         }
5635         pmc_restore_cpu_binding(&pb);
5636
5637         if (error)
5638                 return (error);
5639
5640         /* allocate space for the sample array */
5641         for (cpu = 0; cpu < maxcpu; cpu++) {
5642                 if (!pmc_cpu_is_active(cpu))
5643                         continue;
5644                 pc = pcpu_find(cpu);
5645                 domain = pc->pc_domain;
5646                 sb = malloc_domainset(sizeof(struct pmc_samplebuffer) +
5647                     pmc_nsamples * sizeof(struct pmc_sample), M_PMC,
5648                     DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
5649
5650                 KASSERT(pmc_pcpu[cpu] != NULL,
5651                     ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
5652
5653                 sb->ps_callchains = malloc_domainset(pmc_callchaindepth *
5654                     pmc_nsamples * sizeof(uintptr_t), M_PMC,
5655                     DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
5656
5657                 for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
5658                         ps->ps_pc = sb->ps_callchains +
5659                             (n * pmc_callchaindepth);
5660
5661                 pmc_pcpu[cpu]->pc_sb[PMC_HR] = sb;
5662
5663                 sb = malloc_domainset(sizeof(struct pmc_samplebuffer) +
5664                     pmc_nsamples * sizeof(struct pmc_sample), M_PMC,
5665                     DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
5666
5667                 sb->ps_callchains = malloc_domainset(pmc_callchaindepth *
5668                     pmc_nsamples * sizeof(uintptr_t), M_PMC,
5669                     DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
5670                 for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
5671                         ps->ps_pc = sb->ps_callchains +
5672                             (n * pmc_callchaindepth);
5673
5674                 pmc_pcpu[cpu]->pc_sb[PMC_SR] = sb;
5675
5676                 sb = malloc_domainset(sizeof(struct pmc_samplebuffer) +
5677                     pmc_nsamples * sizeof(struct pmc_sample), M_PMC,
5678                     DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
5679                 sb->ps_callchains = malloc_domainset(pmc_callchaindepth *
5680                     pmc_nsamples * sizeof(uintptr_t), M_PMC,
5681                     DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
5682                 for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
5683                         ps->ps_pc = sb->ps_callchains + n * pmc_callchaindepth;
5684
5685                 pmc_pcpu[cpu]->pc_sb[PMC_UR] = sb;
5686         }
5687
5688         /* allocate space for the row disposition array */
5689         pmc_pmcdisp = malloc(sizeof(enum pmc_mode) * md->pmd_npmc,
5690             M_PMC, M_WAITOK|M_ZERO);
5691
5692         /* mark all PMCs as available */
5693         for (n = 0; n < (int) md->pmd_npmc; n++)
5694                 PMC_MARK_ROW_FREE(n);
5695
5696         /* allocate thread hash tables */
5697         pmc_ownerhash = hashinit(pmc_hashsize, M_PMC,
5698             &pmc_ownerhashmask);
5699
5700         pmc_processhash = hashinit(pmc_hashsize, M_PMC,
5701             &pmc_processhashmask);
5702         mtx_init(&pmc_processhash_mtx, "pmc-process-hash", "pmc-leaf",
5703             MTX_SPIN);
5704
5705         CK_LIST_INIT(&pmc_ss_owners);
5706         pmc_ss_count = 0;
5707
5708         /* allocate a pool of spin mutexes */
5709         pmc_mtxpool = mtx_pool_create("pmc-leaf", pmc_mtxpool_size,
5710             MTX_SPIN);
5711
5712         PMCDBG4(MOD,INI,1, "pmc_ownerhash=%p, mask=0x%lx "
5713             "targethash=%p mask=0x%lx", pmc_ownerhash, pmc_ownerhashmask,
5714             pmc_processhash, pmc_processhashmask);
5715
5716         /* Initialize a spin mutex for the thread free list. */
5717         mtx_init(&pmc_threadfreelist_mtx, "pmc-threadfreelist", "pmc-leaf",
5718             MTX_SPIN);
5719
5720         /* Initialize the task to prune the thread free list. */
5721         TASK_INIT(&free_task, 0, pmc_thread_descriptor_pool_free_task, NULL);
5722
5723         /* register process {exit,fork,exec} handlers */
5724         pmc_exit_tag = EVENTHANDLER_REGISTER(process_exit,
5725             pmc_process_exit, NULL, EVENTHANDLER_PRI_ANY);
5726         pmc_fork_tag = EVENTHANDLER_REGISTER(process_fork,
5727             pmc_process_fork, NULL, EVENTHANDLER_PRI_ANY);
5728
5729         /* register kld event handlers */
5730         pmc_kld_load_tag = EVENTHANDLER_REGISTER(kld_load, pmc_kld_load,
5731             NULL, EVENTHANDLER_PRI_ANY);
5732         pmc_kld_unload_tag = EVENTHANDLER_REGISTER(kld_unload, pmc_kld_unload,
5733             NULL, EVENTHANDLER_PRI_ANY);
5734
5735         /* initialize logging */
5736         pmclog_initialize();
5737
5738         /* set hook functions */
5739         pmc_intr = md->pmd_intr;
5740         wmb();
5741         pmc_hook = pmc_hook_handler;
5742
5743         if (error == 0) {
5744                 printf(PMC_MODULE_NAME ":");
5745                 for (n = 0; n < (int) md->pmd_nclass; n++) {
5746                         pcd = &md->pmd_classdep[n];
5747                         printf(" %s/%d/%d/0x%b",
5748                             pmc_name_of_pmcclass(pcd->pcd_class),
5749                             pcd->pcd_num,
5750                             pcd->pcd_width,
5751                             pcd->pcd_caps,
5752                             "\20"
5753                             "\1INT\2USR\3SYS\4EDG\5THR"
5754                             "\6REA\7WRI\10INV\11QUA\12PRC"
5755                             "\13TAG\14CSC");
5756                 }
5757                 printf("\n");
5758         }
5759
5760         return (error);
5761 }
5762
5763 /* prepare to be unloaded */
5764 static void
5765 pmc_cleanup(void)
5766 {
5767         int c, cpu;
5768         unsigned int maxcpu;
5769         struct pmc_ownerhash *ph;
5770         struct pmc_owner *po, *tmp;
5771         struct pmc_binding pb;
5772 #ifdef  HWPMC_DEBUG
5773         struct pmc_processhash *prh;
5774 #endif
5775
5776         PMCDBG0(MOD,INI,0, "cleanup");
5777
5778         /* switch off sampling */
5779         CPU_FOREACH(cpu)
5780                 DPCPU_ID_SET(cpu, pmc_sampled, 0);
5781         pmc_intr = NULL;
5782
5783         sx_xlock(&pmc_sx);
5784         if (pmc_hook == NULL) { /* being unloaded already */
5785                 sx_xunlock(&pmc_sx);
5786                 return;
5787         }
5788
5789         pmc_hook = NULL; /* prevent new threads from entering module */
5790
5791         /* deregister event handlers */
5792         EVENTHANDLER_DEREGISTER(process_fork, pmc_fork_tag);
5793         EVENTHANDLER_DEREGISTER(process_exit, pmc_exit_tag);
5794         EVENTHANDLER_DEREGISTER(kld_load, pmc_kld_load_tag);
5795         EVENTHANDLER_DEREGISTER(kld_unload, pmc_kld_unload_tag);
5796
5797         /* send SIGBUS to all owner threads, free up allocations */
5798         if (pmc_ownerhash)
5799                 for (ph = pmc_ownerhash;
5800                      ph <= &pmc_ownerhash[pmc_ownerhashmask];
5801                      ph++) {
5802                         LIST_FOREACH_SAFE(po, ph, po_next, tmp) {
5803                                 pmc_remove_owner(po);
5804
5805                                 /* send SIGBUS to owner processes */
5806                                 PMCDBG3(MOD,INI,2, "cleanup signal proc=%p "
5807                                     "(%d, %s)", po->po_owner,
5808                                     po->po_owner->p_pid,
5809                                     po->po_owner->p_comm);
5810
5811                                 PROC_LOCK(po->po_owner);
5812                                 kern_psignal(po->po_owner, SIGBUS);
5813                                 PROC_UNLOCK(po->po_owner);
5814
5815                                 pmc_destroy_owner_descriptor(po);
5816                         }
5817                 }
5818
5819         /* reclaim allocated data structures */
5820         taskqueue_drain(taskqueue_fast, &free_task);
5821         mtx_destroy(&pmc_threadfreelist_mtx);
5822         pmc_thread_descriptor_pool_drain();
5823
5824         if (pmc_mtxpool)
5825                 mtx_pool_destroy(&pmc_mtxpool);
5826
5827         mtx_destroy(&pmc_processhash_mtx);
5828         if (pmc_processhash) {
5829 #ifdef  HWPMC_DEBUG
5830                 struct pmc_process *pp;
5831
5832                 PMCDBG0(MOD,INI,3, "destroy process hash");
5833                 for (prh = pmc_processhash;
5834                      prh <= &pmc_processhash[pmc_processhashmask];
5835                      prh++)
5836                         LIST_FOREACH(pp, prh, pp_next)
5837                             PMCDBG1(MOD,INI,3, "pid=%d", pp->pp_proc->p_pid);
5838 #endif
5839
5840                 hashdestroy(pmc_processhash, M_PMC, pmc_processhashmask);
5841                 pmc_processhash = NULL;
5842         }
5843
5844         if (pmc_ownerhash) {
5845                 PMCDBG0(MOD,INI,3, "destroy owner hash");
5846                 hashdestroy(pmc_ownerhash, M_PMC, pmc_ownerhashmask);
5847                 pmc_ownerhash = NULL;
5848         }
5849
5850         KASSERT(CK_LIST_EMPTY(&pmc_ss_owners),
5851             ("[pmc,%d] Global SS owner list not empty", __LINE__));
5852         KASSERT(pmc_ss_count == 0,
5853             ("[pmc,%d] Global SS count not empty", __LINE__));
5854
5855         /* do processor and pmc-class dependent cleanup */
5856         maxcpu = pmc_cpu_max();
5857
5858         PMCDBG0(MOD,INI,3, "md cleanup");
5859         if (md) {
5860                 pmc_save_cpu_binding(&pb);
5861                 for (cpu = 0; cpu < maxcpu; cpu++) {
5862                         PMCDBG2(MOD,INI,1,"pmc-cleanup cpu=%d pcs=%p",
5863                             cpu, pmc_pcpu[cpu]);
5864                         if (!pmc_cpu_is_active(cpu) || pmc_pcpu[cpu] == NULL)
5865                                 continue;
5866                         pmc_select_cpu(cpu);
5867                         for (c = 0; c < md->pmd_nclass; c++)
5868                                 md->pmd_classdep[c].pcd_pcpu_fini(md, cpu);
5869                         if (md->pmd_pcpu_fini)
5870                                 md->pmd_pcpu_fini(md, cpu);
5871                 }
5872
5873                 if (md->pmd_cputype == PMC_CPU_GENERIC)
5874                         pmc_generic_cpu_finalize(md);
5875                 else
5876                         pmc_md_finalize(md);
5877
5878                 pmc_mdep_free(md);
5879                 md = NULL;
5880                 pmc_restore_cpu_binding(&pb);
5881         }
5882
5883         /* Free per-cpu descriptors. */
5884         for (cpu = 0; cpu < maxcpu; cpu++) {
5885                 if (!pmc_cpu_is_active(cpu))
5886                         continue;
5887                 KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_HR] != NULL,
5888                     ("[pmc,%d] Null hw cpu sample buffer cpu=%d", __LINE__,
5889                         cpu));
5890                 KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_SR] != NULL,
5891                     ("[pmc,%d] Null sw cpu sample buffer cpu=%d", __LINE__,
5892                         cpu));
5893                 KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_UR] != NULL,
5894                     ("[pmc,%d] Null userret cpu sample buffer cpu=%d", __LINE__,
5895                         cpu));
5896                 free(pmc_pcpu[cpu]->pc_sb[PMC_HR]->ps_callchains, M_PMC);
5897                 free(pmc_pcpu[cpu]->pc_sb[PMC_HR], M_PMC);
5898                 free(pmc_pcpu[cpu]->pc_sb[PMC_SR]->ps_callchains, M_PMC);
5899                 free(pmc_pcpu[cpu]->pc_sb[PMC_SR], M_PMC);
5900                 free(pmc_pcpu[cpu]->pc_sb[PMC_UR]->ps_callchains, M_PMC);
5901                 free(pmc_pcpu[cpu]->pc_sb[PMC_UR], M_PMC);
5902                 free(pmc_pcpu[cpu], M_PMC);
5903         }
5904
5905         free(pmc_pcpu, M_PMC);
5906         pmc_pcpu = NULL;
5907
5908         free(pmc_pcpu_saved, M_PMC);
5909         pmc_pcpu_saved = NULL;
5910
5911         if (pmc_pmcdisp) {
5912                 free(pmc_pmcdisp, M_PMC);
5913                 pmc_pmcdisp = NULL;
5914         }
5915
5916         if (pmc_rowindex_to_classdep) {
5917                 free(pmc_rowindex_to_classdep, M_PMC);
5918                 pmc_rowindex_to_classdep = NULL;
5919         }
5920
5921         pmclog_shutdown();
5922         counter_u64_free(pmc_stats.pm_intr_ignored);
5923         counter_u64_free(pmc_stats.pm_intr_processed);
5924         counter_u64_free(pmc_stats.pm_intr_bufferfull);
5925         counter_u64_free(pmc_stats.pm_syscalls);
5926         counter_u64_free(pmc_stats.pm_syscall_errors);
5927         counter_u64_free(pmc_stats.pm_buffer_requests);
5928         counter_u64_free(pmc_stats.pm_buffer_requests_failed);
5929         counter_u64_free(pmc_stats.pm_log_sweeps);
5930         counter_u64_free(pmc_stats.pm_merges);
5931         counter_u64_free(pmc_stats.pm_overwrites);
5932         sx_xunlock(&pmc_sx);    /* we are done */
5933 }
5934
5935 /*
5936  * The function called at load/unload.
5937  */
5938
5939 static int
5940 load (struct module *module __unused, int cmd, void *arg __unused)
5941 {
5942         int error;
5943
5944         error = 0;
5945
5946         switch (cmd) {
5947         case MOD_LOAD :
5948                 /* initialize the subsystem */
5949                 error = pmc_initialize();
5950                 if (error != 0)
5951                         break;
5952                 PMCDBG2(MOD,INI,1, "syscall=%d maxcpu=%d",
5953                     pmc_syscall_num, pmc_cpu_max());
5954                 break;
5955
5956
5957         case MOD_UNLOAD :
5958         case MOD_SHUTDOWN:
5959                 pmc_cleanup();
5960                 PMCDBG0(MOD,INI,1, "unloaded");
5961                 break;
5962
5963         default :
5964                 error = EINVAL; /* XXX should panic(9) */
5965                 break;
5966         }
5967
5968         return error;
5969 }