sys/dev/hwpmc/hwpmc_mod.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2003-2008 Joseph Koshy
   5  * Copyright (c) 2007 The FreeBSD Foundation
   6  * Copyright (c) 2018 Matthew Macy
   7  * All rights reserved.
   8  *
   9  * Portions of this software were developed by A. Joseph Koshy under
  10  * sponsorship from the FreeBSD Foundation and Google, Inc.
  11  *
  12  * Redistribution and use in source and binary forms, with or without
  13  * modification, are permitted provided that the following conditions
  14  * are met:
  15  * 1. Redistributions of source code must retain the above copyright
  16  *    notice, this list of conditions and the following disclaimer.
  17  * 2. Redistributions in binary form must reproduce the above copyright
  18  *    notice, this list of conditions and the following disclaimer in the
  19  *    documentation and/or other materials provided with the distribution.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  24  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  25  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  30  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  31  * SUCH DAMAGE.
  32  *
  33  */
  34
  35 #include <sys/cdefs.h>
  36 __FBSDID("$FreeBSD$");
  37
  38 #include <sys/param.h>
  39 #include <sys/eventhandler.h>
  40 #include <sys/gtaskqueue.h>
  41 #include <sys/jail.h>
  42 #include <sys/kernel.h>
  43 #include <sys/kthread.h>
  44 #include <sys/limits.h>
  45 #include <sys/lock.h>
  46 #include <sys/malloc.h>
  47 #include <sys/module.h>
  48 #include <sys/mount.h>
  49 #include <sys/mutex.h>
  50 #include <sys/pmc.h>
  51 #include <sys/pmckern.h>
  52 #include <sys/pmclog.h>
  53 #include <sys/priv.h>
  54 #include <sys/proc.h>
  55 #include <sys/queue.h>
  56 #include <sys/resourcevar.h>
  57 #include <sys/rwlock.h>
  58 #include <sys/sched.h>
  59 #include <sys/signalvar.h>
  60 #include <sys/smp.h>
  61 #include <sys/sx.h>
  62 #include <sys/sysctl.h>
  63 #include <sys/sysent.h>
  64 #include <sys/syslog.h>
  65 #include <sys/systm.h>
  66 #include <sys/vnode.h>
  67
  68 #include <sys/linker.h>         /* needs to be after <sys/malloc.h> */
  69
  70 #include <machine/atomic.h>
  71 #include <machine/md_var.h>
  72
  73 #include <vm/vm.h>
  74 #include <vm/vm_extern.h>
  75 #include <vm/pmap.h>
  76 #include <vm/vm_map.h>
  77 #include <vm/vm_object.h>
  78
  79 #include "hwpmc_soft.h"
  80
  81 #ifdef NUMA
  82 #define NDOMAINS vm_ndomains
  83 #else
  84 #define NDOMAINS 1
  85 #define malloc_domain(size, type, domain, flags) malloc((size), (type), (flags))
  86 #define free_domain(addr, type) free(addr, type)
  87 #endif
  88
  89 #define PMC_EPOCH_ENTER() struct epoch_tracker pmc_et; epoch_enter_preempt(global_epoch_preempt, &pmc_et)
  90 #define PMC_EPOCH_EXIT() epoch_exit_preempt(global_epoch_preempt, &pmc_et)
  91
  92 /*
  93  * Types
  94  */
  95
  96 enum pmc_flags {
  97         PMC_FLAG_NONE     = 0x00, /* do nothing */
  98         PMC_FLAG_REMOVE   = 0x01, /* atomically remove entry from hash */
  99         PMC_FLAG_ALLOCATE = 0x02, /* add entry to hash if not found */
 100         PMC_FLAG_NOWAIT   = 0x04, /* do not wait for mallocs */
 101 };
 102
 103 /*
 104  * The offset in sysent where the syscall is allocated.
 105  */
 106
 107 static int pmc_syscall_num = NO_SYSCALL;
 108 struct pmc_cpu          **pmc_pcpu;      /* per-cpu state */
 109 pmc_value_t             *pmc_pcpu_saved; /* saved PMC values: CSW handling */
 110
 111 #define PMC_PCPU_SAVED(C,R)     pmc_pcpu_saved[(R) + md->pmd_npmc*(C)]
 112
 113 struct mtx_pool         *pmc_mtxpool;
 114 static int              *pmc_pmcdisp;    /* PMC row dispositions */
 115
 116 #define PMC_ROW_DISP_IS_FREE(R)         (pmc_pmcdisp[(R)] == 0)
 117 #define PMC_ROW_DISP_IS_THREAD(R)       (pmc_pmcdisp[(R)] > 0)
 118 #define PMC_ROW_DISP_IS_STANDALONE(R)   (pmc_pmcdisp[(R)] < 0)
 119
 120 #define PMC_MARK_ROW_FREE(R) do {                                         \
 121         pmc_pmcdisp[(R)] = 0;                                             \
 122 } while (0)
 123
 124 #define PMC_MARK_ROW_STANDALONE(R) do {                                   \
 125         KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \
 126                     __LINE__));                                           \
 127         atomic_add_int(&pmc_pmcdisp[(R)], -1);                            \
 128         KASSERT(pmc_pmcdisp[(R)] >= (-pmc_cpu_max_active()),              \
 129                 ("[pmc,%d] row disposition error", __LINE__));            \
 130 } while (0)
 131
 132 #define PMC_UNMARK_ROW_STANDALONE(R) do {                                 \
 133         atomic_add_int(&pmc_pmcdisp[(R)], 1);                             \
 134         KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \
 135                     __LINE__));                                           \
 136 } while (0)
 137
 138 #define PMC_MARK_ROW_THREAD(R) do {                                       \
 139         KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \
 140                     __LINE__));                                           \
 141         atomic_add_int(&pmc_pmcdisp[(R)], 1);                             \
 142 } while (0)
 143
 144 #define PMC_UNMARK_ROW_THREAD(R) do {                                     \
 145         atomic_add_int(&pmc_pmcdisp[(R)], -1);                            \
 146         KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \
 147                     __LINE__));                                           \
 148 } while (0)
 149
 150
 151 /* various event handlers */
 152 static eventhandler_tag pmc_exit_tag, pmc_fork_tag, pmc_kld_load_tag,
 153     pmc_kld_unload_tag;
 154
 155 /* Module statistics */
 156 struct pmc_driverstats pmc_stats;
 157
 158
 159 /* Machine/processor dependent operations */
 160 static struct pmc_mdep  *md;
 161
 162 /*
 163  * Hash tables mapping owner processes and target threads to PMCs.
 164  */
 165
 166 struct mtx pmc_processhash_mtx;         /* spin mutex */
 167 static u_long pmc_processhashmask;
 168 static LIST_HEAD(pmc_processhash, pmc_process)  *pmc_processhash;
 169
 170 /*
 171  * Hash table of PMC owner descriptors.  This table is protected by
 172  * the shared PMC "sx" lock.
 173  */
 174
 175 static u_long pmc_ownerhashmask;
 176 static LIST_HEAD(pmc_ownerhash, pmc_owner)      *pmc_ownerhash;
 177
 178 /*
 179  * List of PMC owners with system-wide sampling PMCs.
 180  */
 181
 182 static CK_LIST_HEAD(, pmc_owner)                        pmc_ss_owners;
 183
 184 /*
 185  * List of free thread entries. This is protected by the spin
 186  * mutex.
 187  */
 188 static struct mtx pmc_threadfreelist_mtx;       /* spin mutex */
 189 static LIST_HEAD(, pmc_thread)                  pmc_threadfreelist;
 190 static int pmc_threadfreelist_entries=0;
 191 #define THREADENTRY_SIZE                                                \
 192 (sizeof(struct pmc_thread) + (md->pmd_npmc * sizeof(struct pmc_threadpmcstate)))
 193
 194 /*
 195  * Task to free thread descriptors
 196  */
 197 static struct grouptask free_gtask;
 198
 199 /*
 200  * A map of row indices to classdep structures.
 201  */
 202 static struct pmc_classdep **pmc_rowindex_to_classdep;
 203
 204 /*
 205  * Prototypes
 206  */
 207
 208 #ifdef  HWPMC_DEBUG
 209 static int      pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS);
 210 static int      pmc_debugflags_parse(char *newstr, char *fence);
 211 #endif
 212
 213 static int      load(struct module *module, int cmd, void *arg);
 214 static int      pmc_add_sample(ring_type_t ring, struct pmc *pm, struct trapframe *tf);
 215 static void     pmc_add_thread_descriptors_from_proc(struct proc *p,
 216     struct pmc_process *pp);
 217 static int      pmc_attach_process(struct proc *p, struct pmc *pm);
 218 static struct pmc *pmc_allocate_pmc_descriptor(void);
 219 static struct pmc_owner *pmc_allocate_owner_descriptor(struct proc *p);
 220 static int      pmc_attach_one_process(struct proc *p, struct pmc *pm);
 221 static int      pmc_can_allocate_rowindex(struct proc *p, unsigned int ri,
 222     int cpu);
 223 static int      pmc_can_attach(struct pmc *pm, struct proc *p);
 224 static void     pmc_capture_user_callchain(int cpu, int soft, struct trapframe *tf);
 225 static void     pmc_cleanup(void);
 226 static int      pmc_detach_process(struct proc *p, struct pmc *pm);
 227 static int      pmc_detach_one_process(struct proc *p, struct pmc *pm,
 228     int flags);
 229 static void     pmc_destroy_owner_descriptor(struct pmc_owner *po);
 230 static void     pmc_destroy_pmc_descriptor(struct pmc *pm);
 231 static void     pmc_destroy_process_descriptor(struct pmc_process *pp);
 232 static struct pmc_owner *pmc_find_owner_descriptor(struct proc *p);
 233 static int      pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm);
 234 static struct pmc *pmc_find_pmc_descriptor_in_process(struct pmc_owner *po,
 235     pmc_id_t pmc);
 236 static struct pmc_process *pmc_find_process_descriptor(struct proc *p,
 237     uint32_t mode);
 238 static struct pmc_thread *pmc_find_thread_descriptor(struct pmc_process *pp,
 239     struct thread *td, uint32_t mode);
 240 static void     pmc_force_context_switch(void);
 241 static void     pmc_link_target_process(struct pmc *pm,
 242     struct pmc_process *pp);
 243 static void     pmc_log_all_process_mappings(struct pmc_owner *po);
 244 static void     pmc_log_kernel_mappings(struct pmc *pm);
 245 static void     pmc_log_process_mappings(struct pmc_owner *po, struct proc *p);
 246 static void     pmc_maybe_remove_owner(struct pmc_owner *po);
 247 static void     pmc_process_csw_in(struct thread *td);
 248 static void     pmc_process_csw_out(struct thread *td);
 249 static void     pmc_process_exit(void *arg, struct proc *p);
 250 static void     pmc_process_fork(void *arg, struct proc *p1,
 251     struct proc *p2, int n);
 252 static void     pmc_process_samples(int cpu, ring_type_t soft);
 253 static void     pmc_release_pmc_descriptor(struct pmc *pmc);
 254 static void     pmc_process_thread_add(struct thread *td);
 255 static void     pmc_process_thread_delete(struct thread *td);
 256 static void     pmc_process_thread_userret(struct thread *td);
 257 static void     pmc_remove_owner(struct pmc_owner *po);
 258 static void     pmc_remove_process_descriptor(struct pmc_process *pp);
 259 static void     pmc_restore_cpu_binding(struct pmc_binding *pb);
 260 static void     pmc_save_cpu_binding(struct pmc_binding *pb);
 261 static void     pmc_select_cpu(int cpu);
 262 static int      pmc_start(struct pmc *pm);
 263 static int      pmc_stop(struct pmc *pm);
 264 static int      pmc_syscall_handler(struct thread *td, void *syscall_args);
 265 static struct pmc_thread *pmc_thread_descriptor_pool_alloc(void);
 266 static void     pmc_thread_descriptor_pool_drain(void);
 267 static void     pmc_thread_descriptor_pool_free(struct pmc_thread *pt);
 268 static void     pmc_unlink_target_process(struct pmc *pmc,
 269     struct pmc_process *pp);
 270 static int generic_switch_in(struct pmc_cpu *pc, struct pmc_process *pp);
 271 static int generic_switch_out(struct pmc_cpu *pc, struct pmc_process *pp);
 272 static struct pmc_mdep *pmc_generic_cpu_initialize(void);
 273 static void pmc_generic_cpu_finalize(struct pmc_mdep *md);
 274 static void pmc_post_callchain_callback(void);
 275 static void pmc_process_threadcreate(struct thread *td);
 276 static void pmc_process_threadexit(struct thread *td);
 277 static void pmc_process_proccreate(struct proc *p);
 278 static void pmc_process_allproc(struct pmc *pm);
 279
 280 /*
 281  * Kernel tunables and sysctl(8) interface.
 282  */
 283
 284 SYSCTL_DECL(_kern_hwpmc);
 285 SYSCTL_NODE(_kern_hwpmc, OID_AUTO, stats, CTLFLAG_RW, 0, "HWPMC stats");
 286
 287
 288 /* Stats. */
 289 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, intr_ignored, CTLFLAG_RW,
 290                                    &pmc_stats.pm_intr_ignored, "# of interrupts ignored");
 291 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, intr_processed, CTLFLAG_RW,
 292                                    &pmc_stats.pm_intr_processed, "# of interrupts processed");
 293 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, intr_bufferfull, CTLFLAG_RW,
 294                                    &pmc_stats.pm_intr_bufferfull, "# of interrupts where buffer was full");
 295 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, syscalls, CTLFLAG_RW,
 296                                    &pmc_stats.pm_syscalls, "# of syscalls");
 297 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, syscall_errors, CTLFLAG_RW,
 298                                    &pmc_stats.pm_syscall_errors, "# of syscall_errors");
 299 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, buffer_requests, CTLFLAG_RW,
 300                                    &pmc_stats.pm_buffer_requests, "# of buffer requests");
 301 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, buffer_requests_failed, CTLFLAG_RW,
 302                                    &pmc_stats.pm_buffer_requests_failed, "# of buffer requests which failed");
 303 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, log_sweeps, CTLFLAG_RW,
 304                                    &pmc_stats.pm_log_sweeps, "# of ?");
 305 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, merges, CTLFLAG_RW,
 306                                    &pmc_stats.pm_merges, "# of times kernel stack was found for user trace");
 307 SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, overwrites, CTLFLAG_RW,
 308                                    &pmc_stats.pm_overwrites, "# of times a sample was overwritten before being logged");
 309
 310 static int pmc_callchaindepth = PMC_CALLCHAIN_DEPTH;
 311 SYSCTL_INT(_kern_hwpmc, OID_AUTO, callchaindepth, CTLFLAG_RDTUN,
 312     &pmc_callchaindepth, 0, "depth of call chain records");
 313
 314 char pmc_cpuid[64];
 315 SYSCTL_STRING(_kern_hwpmc, OID_AUTO, cpuid, CTLFLAG_RD,
 316         pmc_cpuid, 0, "cpu version string");
 317 #ifdef  HWPMC_DEBUG
 318 struct pmc_debugflags pmc_debugflags = PMC_DEBUG_DEFAULT_FLAGS;
 319 char    pmc_debugstr[PMC_DEBUG_STRSIZE];
 320 TUNABLE_STR(PMC_SYSCTL_NAME_PREFIX "debugflags", pmc_debugstr,
 321     sizeof(pmc_debugstr));
 322 SYSCTL_PROC(_kern_hwpmc, OID_AUTO, debugflags,
 323     CTLTYPE_STRING | CTLFLAG_RWTUN | CTLFLAG_NOFETCH,
 324     0, 0, pmc_debugflags_sysctl_handler, "A", "debug flags");
 325 #endif
 326
 327
 328 /*
 329  * kern.hwpmc.hashrows -- determines the number of rows in the
 330  * of the hash table used to look up threads
 331  */
 332
 333 static int pmc_hashsize = PMC_HASH_SIZE;
 334 SYSCTL_INT(_kern_hwpmc, OID_AUTO, hashsize, CTLFLAG_RDTUN,
 335     &pmc_hashsize, 0, "rows in hash tables");
 336
 337 /*
 338  * kern.hwpmc.nsamples --- number of PC samples/callchain stacks per CPU
 339  */
 340
 341 static int pmc_nsamples = PMC_NSAMPLES;
 342 SYSCTL_INT(_kern_hwpmc, OID_AUTO, nsamples, CTLFLAG_RDTUN,
 343     &pmc_nsamples, 0, "number of PC samples per CPU");
 344
 345 static uint64_t pmc_sample_mask = PMC_NSAMPLES-1;
 346
 347 /*
 348  * kern.hwpmc.mtxpoolsize -- number of mutexes in the mutex pool.
 349  */
 350
 351 static int pmc_mtxpool_size = PMC_MTXPOOL_SIZE;
 352 SYSCTL_INT(_kern_hwpmc, OID_AUTO, mtxpoolsize, CTLFLAG_RDTUN,
 353     &pmc_mtxpool_size, 0, "size of spin mutex pool");
 354
 355
 356 /*
 357  * kern.hwpmc.threadfreelist_entries -- number of free entries
 358  */
 359
 360 SYSCTL_INT(_kern_hwpmc, OID_AUTO, threadfreelist_entries, CTLFLAG_RD,
 361     &pmc_threadfreelist_entries, 0, "number of avalable thread entries");
 362
 363
 364 /*
 365  * kern.hwpmc.threadfreelist_max -- maximum number of free entries
 366  */
 367
 368 static int pmc_threadfreelist_max = PMC_THREADLIST_MAX;
 369 SYSCTL_INT(_kern_hwpmc, OID_AUTO, threadfreelist_max, CTLFLAG_RW,
 370     &pmc_threadfreelist_max, 0,
 371     "maximum number of available thread entries before freeing some");
 372
 373
 374 /*
 375  * security.bsd.unprivileged_syspmcs -- allow non-root processes to
 376  * allocate system-wide PMCs.
 377  *
 378  * Allowing unprivileged processes to allocate system PMCs is convenient
 379  * if system-wide measurements need to be taken concurrently with other
 380  * per-process measurements.  This feature is turned off by default.
 381  */
 382
 383 static int pmc_unprivileged_syspmcs = 0;
 384 SYSCTL_INT(_security_bsd, OID_AUTO, unprivileged_syspmcs, CTLFLAG_RWTUN,
 385     &pmc_unprivileged_syspmcs, 0,
 386     "allow unprivileged process to allocate system PMCs");
 387
 388 /*
 389  * Hash function.  Discard the lower 2 bits of the pointer since
 390  * these are always zero for our uses.  The hash multiplier is
 391  * round((2^LONG_BIT) * ((sqrt(5)-1)/2)).
 392  */
 393
 394 #if     LONG_BIT == 64
 395 #define _PMC_HM         11400714819323198486u
 396 #elif   LONG_BIT == 32
 397 #define _PMC_HM         2654435769u
 398 #else
 399 #error  Must know the size of 'long' to compile
 400 #endif
 401
 402 #define PMC_HASH_PTR(P,M)       ((((unsigned long) (P) >> 2) * _PMC_HM) & (M))
 403
 404 /*
 405  * Syscall structures
 406  */
 407
 408 /* The `sysent' for the new syscall */
 409 static struct sysent pmc_sysent = {
 410         .sy_narg =      2,
 411         .sy_call =      pmc_syscall_handler,
 412 };
 413
 414 static struct syscall_module_data pmc_syscall_mod = {
 415         .chainevh =     load,
 416         .chainarg =     NULL,
 417         .offset =       &pmc_syscall_num,
 418         .new_sysent =   &pmc_sysent,
 419         .old_sysent =   { .sy_narg = 0, .sy_call = NULL },
 420         .flags =        SY_THR_STATIC_KLD,
 421 };
 422
 423 static moduledata_t pmc_mod = {
 424         .name =         PMC_MODULE_NAME,
 425         .evhand =       syscall_module_handler,
 426         .priv =         &pmc_syscall_mod,
 427 };
 428
 429 #ifdef EARLY_AP_STARTUP
 430 DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SYSCALLS, SI_ORDER_ANY);
 431 #else
 432 DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SMP, SI_ORDER_ANY);
 433 #endif
 434 MODULE_VERSION(pmc, PMC_VERSION);
 435
 436 #ifdef  HWPMC_DEBUG
 437 enum pmc_dbgparse_state {
 438         PMCDS_WS,               /* in whitespace */
 439         PMCDS_MAJOR,            /* seen a major keyword */
 440         PMCDS_MINOR
 441 };
 442
 443 static int
 444 pmc_debugflags_parse(char *newstr, char *fence)
 445 {
 446         char c, *p, *q;
 447         struct pmc_debugflags *tmpflags;
 448         int error, found, *newbits, tmp;
 449         size_t kwlen;
 450
 451         tmpflags = malloc(sizeof(*tmpflags), M_PMC, M_WAITOK|M_ZERO);
 452
 453         p = newstr;
 454         error = 0;
 455
 456         for (; p < fence && (c = *p); p++) {
 457
 458                 /* skip white space */
 459                 if (c == ' ' || c == '\t')
 460                         continue;
 461
 462                 /* look for a keyword followed by "=" */
 463                 for (q = p; p < fence && (c = *p) && c != '='; p++)
 464                         ;
 465                 if (c != '=') {
 466                         error = EINVAL;
 467                         goto done;
 468                 }
 469
 470                 kwlen = p - q;
 471                 newbits = NULL;
 472
 473                 /* lookup flag group name */
 474 #define DBG_SET_FLAG_MAJ(S,F)                                           \
 475                 if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0)  \
 476                         newbits = &tmpflags->pdb_ ## F;
 477
 478                 DBG_SET_FLAG_MAJ("cpu",         CPU);
 479                 DBG_SET_FLAG_MAJ("csw",         CSW);
 480                 DBG_SET_FLAG_MAJ("logging",     LOG);
 481                 DBG_SET_FLAG_MAJ("module",      MOD);
 482                 DBG_SET_FLAG_MAJ("md",          MDP);
 483                 DBG_SET_FLAG_MAJ("owner",       OWN);
 484                 DBG_SET_FLAG_MAJ("pmc",         PMC);
 485                 DBG_SET_FLAG_MAJ("process",     PRC);
 486                 DBG_SET_FLAG_MAJ("sampling",    SAM);
 487
 488                 if (newbits == NULL) {
 489                         error = EINVAL;
 490                         goto done;
 491                 }
 492
 493                 p++;            /* skip the '=' */
 494
 495                 /* Now parse the individual flags */
 496                 tmp = 0;
 497         newflag:
 498                 for (q = p; p < fence && (c = *p); p++)
 499                         if (c == ' ' || c == '\t' || c == ',')
 500                                 break;
 501
 502                 /* p == fence or c == ws or c == "," or c == 0 */
 503
 504                 if ((kwlen = p - q) == 0) {
 505                         *newbits = tmp;
 506                         continue;
 507                 }
 508
 509                 found = 0;
 510 #define DBG_SET_FLAG_MIN(S,F)                                           \
 511                 if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0)  \
 512                         tmp |= found = (1 << PMC_DEBUG_MIN_ ## F)
 513
 514                 /* a '*' denotes all possible flags in the group */
 515                 if (kwlen == 1 && *q == '*')
 516                         tmp = found = ~0;
 517                 /* look for individual flag names */
 518                 DBG_SET_FLAG_MIN("allocaterow", ALR);
 519                 DBG_SET_FLAG_MIN("allocate",    ALL);
 520                 DBG_SET_FLAG_MIN("attach",      ATT);
 521                 DBG_SET_FLAG_MIN("bind",        BND);
 522                 DBG_SET_FLAG_MIN("config",      CFG);
 523                 DBG_SET_FLAG_MIN("exec",        EXC);
 524                 DBG_SET_FLAG_MIN("exit",        EXT);
 525                 DBG_SET_FLAG_MIN("find",        FND);
 526                 DBG_SET_FLAG_MIN("flush",       FLS);
 527                 DBG_SET_FLAG_MIN("fork",        FRK);
 528                 DBG_SET_FLAG_MIN("getbuf",      GTB);
 529                 DBG_SET_FLAG_MIN("hook",        PMH);
 530                 DBG_SET_FLAG_MIN("init",        INI);
 531                 DBG_SET_FLAG_MIN("intr",        INT);
 532                 DBG_SET_FLAG_MIN("linktarget",  TLK);
 533                 DBG_SET_FLAG_MIN("mayberemove", OMR);
 534                 DBG_SET_FLAG_MIN("ops",         OPS);
 535                 DBG_SET_FLAG_MIN("read",        REA);
 536                 DBG_SET_FLAG_MIN("register",    REG);
 537                 DBG_SET_FLAG_MIN("release",     REL);
 538                 DBG_SET_FLAG_MIN("remove",      ORM);
 539                 DBG_SET_FLAG_MIN("sample",      SAM);
 540                 DBG_SET_FLAG_MIN("scheduleio",  SIO);
 541                 DBG_SET_FLAG_MIN("select",      SEL);
 542                 DBG_SET_FLAG_MIN("signal",      SIG);
 543                 DBG_SET_FLAG_MIN("swi",         SWI);
 544                 DBG_SET_FLAG_MIN("swo",         SWO);
 545                 DBG_SET_FLAG_MIN("start",       STA);
 546                 DBG_SET_FLAG_MIN("stop",        STO);
 547                 DBG_SET_FLAG_MIN("syscall",     PMS);
 548                 DBG_SET_FLAG_MIN("unlinktarget", TUL);
 549                 DBG_SET_FLAG_MIN("write",       WRI);
 550                 if (found == 0) {
 551                         /* unrecognized flag name */
 552                         error = EINVAL;
 553                         goto done;
 554                 }
 555
 556                 if (c == 0 || c == ' ' || c == '\t') {  /* end of flag group */
 557                         *newbits = tmp;
 558                         continue;
 559                 }
 560
 561                 p++;
 562                 goto newflag;
 563         }
 564
 565         /* save the new flag set */
 566         bcopy(tmpflags, &pmc_debugflags, sizeof(pmc_debugflags));
 567
 568  done:
 569         free(tmpflags, M_PMC);
 570         return error;
 571 }
 572
 573 static int
 574 pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS)
 575 {
 576         char *fence, *newstr;
 577         int error;
 578         unsigned int n;
 579
 580         (void) arg1; (void) arg2; /* unused parameters */
 581
 582         n = sizeof(pmc_debugstr);
 583         newstr = malloc(n, M_PMC, M_WAITOK|M_ZERO);
 584         (void) strlcpy(newstr, pmc_debugstr, n);
 585
 586         error = sysctl_handle_string(oidp, newstr, n, req);
 587
 588         /* if there is a new string, parse and copy it */
 589         if (error == 0 && req->newptr != NULL) {
 590                 fence = newstr + (n < req->newlen ? n : req->newlen + 1);
 591                 if ((error = pmc_debugflags_parse(newstr, fence)) == 0)
 592                         (void) strlcpy(pmc_debugstr, newstr,
 593                             sizeof(pmc_debugstr));
 594         }
 595
 596         free(newstr, M_PMC);
 597
 598         return error;
 599 }
 600 #endif
 601
 602 /*
 603  * Map a row index to a classdep structure and return the adjusted row
 604  * index for the PMC class index.
 605  */
 606 static struct pmc_classdep *
 607 pmc_ri_to_classdep(struct pmc_mdep *md, int ri, int *adjri)
 608 {
 609         struct pmc_classdep *pcd;
 610
 611         (void) md;
 612
 613         KASSERT(ri >= 0 && ri < md->pmd_npmc,
 614             ("[pmc,%d] illegal row-index %d", __LINE__, ri));
 615
 616         pcd = pmc_rowindex_to_classdep[ri];
 617
 618         KASSERT(pcd != NULL,
 619             ("[pmc,%d] ri %d null pcd", __LINE__, ri));
 620
 621         *adjri = ri - pcd->pcd_ri;
 622
 623         KASSERT(*adjri >= 0 && *adjri < pcd->pcd_num,
 624             ("[pmc,%d] adjusted row-index %d", __LINE__, *adjri));
 625
 626         return (pcd);
 627 }
 628
 629 /*
 630  * Concurrency Control
 631  *
 632  * The driver manages the following data structures:
 633  *
 634  *   - target process descriptors, one per target process
 635  *   - owner process descriptors (and attached lists), one per owner process
 636  *   - lookup hash tables for owner and target processes
 637  *   - PMC descriptors (and attached lists)
 638  *   - per-cpu hardware state
 639  *   - the 'hook' variable through which the kernel calls into
 640  *     this module
 641  *   - the machine hardware state (managed by the MD layer)
 642  *
 643  * These data structures are accessed from:
 644  *
 645  * - thread context-switch code
 646  * - interrupt handlers (possibly on multiple cpus)
 647  * - kernel threads on multiple cpus running on behalf of user
 648  *   processes doing system calls
 649  * - this driver's private kernel threads
 650  *
 651  * = Locks and Locking strategy =
 652  *
 653  * The driver uses four locking strategies for its operation:
 654  *
 655  * - The global SX lock "pmc_sx" is used to protect internal
 656  *   data structures.
 657  *
 658  *   Calls into the module by syscall() start with this lock being
 659  *   held in exclusive mode.  Depending on the requested operation,
 660  *   the lock may be downgraded to 'shared' mode to allow more
 661  *   concurrent readers into the module.  Calls into the module from
 662  *   other parts of the kernel acquire the lock in shared mode.
 663  *
 664  *   This SX lock is held in exclusive mode for any operations that
 665  *   modify the linkages between the driver's internal data structures.
 666  *
 667  *   The 'pmc_hook' function pointer is also protected by this lock.
 668  *   It is only examined with the sx lock held in exclusive mode.  The
 669  *   kernel module is allowed to be unloaded only with the sx lock held
 670  *   in exclusive mode.  In normal syscall handling, after acquiring the
 671  *   pmc_sx lock we first check that 'pmc_hook' is non-null before
 672  *   proceeding.  This prevents races between the thread unloading the module
 673  *   and other threads seeking to use the module.
 674  *
 675  * - Lookups of target process structures and owner process structures
 676  *   cannot use the global "pmc_sx" SX lock because these lookups need
 677  *   to happen during context switches and in other critical sections
 678  *   where sleeping is not allowed.  We protect these lookup tables
 679  *   with their own private spin-mutexes, "pmc_processhash_mtx" and
 680  *   "pmc_ownerhash_mtx".
 681  *
 682  * - Interrupt handlers work in a lock free manner.  At interrupt
 683  *   time, handlers look at the PMC pointer (phw->phw_pmc) configured
 684  *   when the PMC was started.  If this pointer is NULL, the interrupt
 685  *   is ignored after updating driver statistics.  We ensure that this
 686  *   pointer is set (using an atomic operation if necessary) before the
 687  *   PMC hardware is started.  Conversely, this pointer is unset atomically
 688  *   only after the PMC hardware is stopped.
 689  *
 690  *   We ensure that everything needed for the operation of an
 691  *   interrupt handler is available without it needing to acquire any
 692  *   locks.  We also ensure that a PMC's software state is destroyed only
 693  *   after the PMC is taken off hardware (on all CPUs).
 694  *
 695  * - Context-switch handling with process-private PMCs needs more
 696  *   care.
 697  *
 698  *   A given process may be the target of multiple PMCs.  For example,
 699  *   PMCATTACH and PMCDETACH may be requested by a process on one CPU
 700  *   while the target process is running on another.  A PMC could also
 701  *   be getting released because its owner is exiting.  We tackle
 702  *   these situations in the following manner:
 703  *
 704  *   - each target process structure 'pmc_process' has an array
 705  *     of 'struct pmc *' pointers, one for each hardware PMC.
 706  *
 707  *   - At context switch IN time, each "target" PMC in RUNNING state
 708  *     gets started on hardware and a pointer to each PMC is copied into
 709  *     the per-cpu phw array.  The 'runcount' for the PMC is
 710  *     incremented.
 711  *
 712  *   - At context switch OUT time, all process-virtual PMCs are stopped
 713  *     on hardware.  The saved value is added to the PMCs value field
 714  *     only if the PMC is in a non-deleted state (the PMCs state could
 715  *     have changed during the current time slice).
 716  *
 717  *     Note that since in-between a switch IN on a processor and a switch
 718  *     OUT, the PMC could have been released on another CPU.  Therefore
 719  *     context switch OUT always looks at the hardware state to turn
 720  *     OFF PMCs and will update a PMC's saved value only if reachable
 721  *     from the target process record.
 722  *
 723  *   - OP PMCRELEASE could be called on a PMC at any time (the PMC could
 724  *     be attached to many processes at the time of the call and could
 725  *     be active on multiple CPUs).
 726  *
 727  *     We prevent further scheduling of the PMC by marking it as in
 728  *     state 'DELETED'.  If the runcount of the PMC is non-zero then
 729  *     this PMC is currently running on a CPU somewhere.  The thread
 730  *     doing the PMCRELEASE operation waits by repeatedly doing a
 731  *     pause() till the runcount comes to zero.
 732  *
 733  * The contents of a PMC descriptor (struct pmc) are protected using
 734  * a spin-mutex.  In order to save space, we use a mutex pool.
 735  *
 736  * In terms of lock types used by witness(4), we use:
 737  * - Type "pmc-sx", used by the global SX lock.
 738  * - Type "pmc-sleep", for sleep mutexes used by logger threads.
 739  * - Type "pmc-per-proc", for protecting PMC owner descriptors.
 740  * - Type "pmc-leaf", used for all other spin mutexes.
 741  */
 742
 743 /*
 744  * save the cpu binding of the current kthread
 745  */
 746
 747 static void
 748 pmc_save_cpu_binding(struct pmc_binding *pb)
 749 {
 750         PMCDBG0(CPU,BND,2, "save-cpu");
 751         thread_lock(curthread);
 752         pb->pb_bound = sched_is_bound(curthread);
 753         pb->pb_cpu   = curthread->td_oncpu;
 754         thread_unlock(curthread);
 755         PMCDBG1(CPU,BND,2, "save-cpu cpu=%d", pb->pb_cpu);
 756 }
 757
 758 /*
 759  * restore the cpu binding of the current thread
 760  */
 761
 762 static void
 763 pmc_restore_cpu_binding(struct pmc_binding *pb)
 764 {
 765         PMCDBG2(CPU,BND,2, "restore-cpu curcpu=%d restore=%d",
 766             curthread->td_oncpu, pb->pb_cpu);
 767         thread_lock(curthread);
 768         if (pb->pb_bound)
 769                 sched_bind(curthread, pb->pb_cpu);
 770         else
 771                 sched_unbind(curthread);
 772         thread_unlock(curthread);
 773         PMCDBG0(CPU,BND,2, "restore-cpu done");
 774 }
 775
 776 /*
 777  * move execution over the specified cpu and bind it there.
 778  */
 779
 780 static void
 781 pmc_select_cpu(int cpu)
 782 {
 783         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 784             ("[pmc,%d] bad cpu number %d", __LINE__, cpu));
 785
 786         /* Never move to an inactive CPU. */
 787         KASSERT(pmc_cpu_is_active(cpu), ("[pmc,%d] selecting inactive "
 788             "CPU %d", __LINE__, cpu));
 789
 790         PMCDBG1(CPU,SEL,2, "select-cpu cpu=%d", cpu);
 791         thread_lock(curthread);
 792         sched_bind(curthread, cpu);
 793         thread_unlock(curthread);
 794
 795         KASSERT(curthread->td_oncpu == cpu,
 796             ("[pmc,%d] CPU not bound [cpu=%d, curr=%d]", __LINE__,
 797                 cpu, curthread->td_oncpu));
 798
 799         PMCDBG1(CPU,SEL,2, "select-cpu cpu=%d ok", cpu);
 800 }
 801
 802 /*
 803  * Force a context switch.
 804  *
 805  * We do this by pause'ing for 1 tick -- invoking mi_switch() is not
 806  * guaranteed to force a context switch.
 807  */
 808
 809 static void
 810 pmc_force_context_switch(void)
 811 {
 812
 813         pause("pmcctx", 1);
 814 }
 815
 816 uint64_t
 817 pmc_rdtsc(void)
 818 {
 819 #if defined(__i386__) || defined(__amd64__)
 820         if (__predict_true(amd_feature & AMDID_RDTSCP))
 821                 return rdtscp();
 822         else
 823                 return rdtsc();
 824 #else
 825         return get_cyclecount();
 826 #endif
 827 }
 828
 829 /*
 830  * Get the file name for an executable.  This is a simple wrapper
 831  * around vn_fullpath(9).
 832  */
 833
 834 static void
 835 pmc_getfilename(struct vnode *v, char **fullpath, char **freepath)
 836 {
 837
 838         *fullpath = "unknown";
 839         *freepath = NULL;
 840         vn_fullpath(curthread, v, fullpath, freepath);
 841 }
 842
 843 /*
 844  * remove an process owning PMCs
 845  */
 846
 847 void
 848 pmc_remove_owner(struct pmc_owner *po)
 849 {
 850         struct pmc *pm, *tmp;
 851
 852         sx_assert(&pmc_sx, SX_XLOCKED);
 853
 854         PMCDBG1(OWN,ORM,1, "remove-owner po=%p", po);
 855
 856         /* Remove descriptor from the owner hash table */
 857         LIST_REMOVE(po, po_next);
 858
 859         /* release all owned PMC descriptors */
 860         LIST_FOREACH_SAFE(pm, &po->po_pmcs, pm_next, tmp) {
 861                 PMCDBG1(OWN,ORM,2, "pmc=%p", pm);
 862                 KASSERT(pm->pm_owner == po,
 863                     ("[pmc,%d] owner %p != po %p", __LINE__, pm->pm_owner, po));
 864
 865                 pmc_release_pmc_descriptor(pm); /* will unlink from the list */
 866                 pmc_destroy_pmc_descriptor(pm);
 867         }
 868
 869         KASSERT(po->po_sscount == 0,
 870             ("[pmc,%d] SS count not zero", __LINE__));
 871         KASSERT(LIST_EMPTY(&po->po_pmcs),
 872             ("[pmc,%d] PMC list not empty", __LINE__));
 873
 874         /* de-configure the log file if present */
 875         if (po->po_flags & PMC_PO_OWNS_LOGFILE)
 876                 pmclog_deconfigure_log(po);
 877 }
 878
 879 /*
 880  * remove an owner process record if all conditions are met.
 881  */
 882
 883 static void
 884 pmc_maybe_remove_owner(struct pmc_owner *po)
 885 {
 886
 887         PMCDBG1(OWN,OMR,1, "maybe-remove-owner po=%p", po);
 888
 889         /*
 890          * Remove owner record if
 891          * - this process does not own any PMCs
 892          * - this process has not allocated a system-wide sampling buffer
 893          */
 894
 895         if (LIST_EMPTY(&po->po_pmcs) &&
 896             ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)) {
 897                 pmc_remove_owner(po);
 898                 pmc_destroy_owner_descriptor(po);
 899         }
 900 }
 901
 902 /*
 903  * Add an association between a target process and a PMC.
 904  */
 905
 906 static void
 907 pmc_link_target_process(struct pmc *pm, struct pmc_process *pp)
 908 {
 909         int ri;
 910         struct pmc_target *pt;
 911 #ifdef INVARIANTS
 912         struct pmc_thread *pt_td;
 913 #endif
 914
 915         sx_assert(&pmc_sx, SX_XLOCKED);
 916
 917         KASSERT(pm != NULL && pp != NULL,
 918             ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
 919         KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)),
 920             ("[pmc,%d] Attaching a non-process-virtual pmc=%p to pid=%d",
 921                 __LINE__, pm, pp->pp_proc->p_pid));
 922         KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= ((int) md->pmd_npmc - 1),
 923             ("[pmc,%d] Illegal reference count %d for process record %p",
 924                 __LINE__, pp->pp_refcnt, (void *) pp));
 925
 926         ri = PMC_TO_ROWINDEX(pm);
 927
 928         PMCDBG3(PRC,TLK,1, "link-target pmc=%p ri=%d pmc-process=%p",
 929             pm, ri, pp);
 930
 931 #ifdef  HWPMC_DEBUG
 932         LIST_FOREACH(pt, &pm->pm_targets, pt_next)
 933             if (pt->pt_process == pp)
 934                     KASSERT(0, ("[pmc,%d] pp %p already in pmc %p targets",
 935                                 __LINE__, pp, pm));
 936 #endif
 937
 938         pt = malloc(sizeof(struct pmc_target), M_PMC, M_WAITOK|M_ZERO);
 939         pt->pt_process = pp;
 940
 941         LIST_INSERT_HEAD(&pm->pm_targets, pt, pt_next);
 942
 943         atomic_store_rel_ptr((uintptr_t *)&pp->pp_pmcs[ri].pp_pmc,
 944             (uintptr_t)pm);
 945
 946         if (pm->pm_owner->po_owner == pp->pp_proc)
 947                 pm->pm_flags |= PMC_F_ATTACHED_TO_OWNER;
 948
 949         /*
 950          * Initialize the per-process values at this row index.
 951          */
 952         pp->pp_pmcs[ri].pp_pmcval = PMC_TO_MODE(pm) == PMC_MODE_TS ?
 953             pm->pm_sc.pm_reloadcount : 0;
 954
 955         pp->pp_refcnt++;
 956
 957 #ifdef INVARIANTS
 958         /* Confirm that the per-thread values at this row index are cleared. */
 959         if (PMC_TO_MODE(pm) == PMC_MODE_TS) {
 960                 mtx_lock_spin(pp->pp_tdslock);
 961                 LIST_FOREACH(pt_td, &pp->pp_tds, pt_next) {
 962                         KASSERT(pt_td->pt_pmcs[ri].pt_pmcval == (pmc_value_t) 0,
 963                             ("[pmc,%d] pt_pmcval not cleared for pid=%d at "
 964                             "ri=%d", __LINE__, pp->pp_proc->p_pid, ri));
 965                 }
 966                 mtx_unlock_spin(pp->pp_tdslock);
 967         }
 968 #endif
 969 }
 970
 971 /*
 972  * Removes the association between a target process and a PMC.
 973  */
 974
 975 static void
 976 pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp)
 977 {
 978         int ri;
 979         struct proc *p;
 980         struct pmc_target *ptgt;
 981         struct pmc_thread *pt;
 982
 983         sx_assert(&pmc_sx, SX_XLOCKED);
 984
 985         KASSERT(pm != NULL && pp != NULL,
 986             ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
 987
 988         KASSERT(pp->pp_refcnt >= 1 && pp->pp_refcnt <= (int) md->pmd_npmc,
 989             ("[pmc,%d] Illegal ref count %d on process record %p",
 990                 __LINE__, pp->pp_refcnt, (void *) pp));
 991
 992         ri = PMC_TO_ROWINDEX(pm);
 993
 994         PMCDBG3(PRC,TUL,1, "unlink-target pmc=%p ri=%d pmc-process=%p",
 995             pm, ri, pp);
 996
 997         KASSERT(pp->pp_pmcs[ri].pp_pmc == pm,
 998             ("[pmc,%d] PMC ri %d mismatch pmc %p pp->[ri] %p", __LINE__,
 999                 ri, pm, pp->pp_pmcs[ri].pp_pmc));
1000
1001         pp->pp_pmcs[ri].pp_pmc = NULL;
1002         pp->pp_pmcs[ri].pp_pmcval = (pmc_value_t) 0;
1003
1004         /* Clear the per-thread values at this row index. */
1005         if (PMC_TO_MODE(pm) == PMC_MODE_TS) {
1006                 mtx_lock_spin(pp->pp_tdslock);
1007                 LIST_FOREACH(pt, &pp->pp_tds, pt_next)
1008                         pt->pt_pmcs[ri].pt_pmcval = (pmc_value_t) 0;
1009                 mtx_unlock_spin(pp->pp_tdslock);
1010         }
1011
1012         /* Remove owner-specific flags */
1013         if (pm->pm_owner->po_owner == pp->pp_proc) {
1014                 pp->pp_flags &= ~PMC_PP_ENABLE_MSR_ACCESS;
1015                 pm->pm_flags &= ~PMC_F_ATTACHED_TO_OWNER;
1016         }
1017
1018         pp->pp_refcnt--;
1019
1020         /* Remove the target process from the PMC structure */
1021         LIST_FOREACH(ptgt, &pm->pm_targets, pt_next)
1022                 if (ptgt->pt_process == pp)
1023                         break;
1024
1025         KASSERT(ptgt != NULL, ("[pmc,%d] process %p (pp: %p) not found "
1026                     "in pmc %p", __LINE__, pp->pp_proc, pp, pm));
1027
1028         LIST_REMOVE(ptgt, pt_next);
1029         free(ptgt, M_PMC);
1030
1031         /* if the PMC now lacks targets, send the owner a SIGIO */
1032         if (LIST_EMPTY(&pm->pm_targets)) {
1033                 p = pm->pm_owner->po_owner;
1034                 PROC_LOCK(p);
1035                 kern_psignal(p, SIGIO);
1036                 PROC_UNLOCK(p);
1037
1038                 PMCDBG2(PRC,SIG,2, "signalling proc=%p signal=%d", p,
1039                     SIGIO);
1040         }
1041 }
1042
1043 /*
1044  * Check if PMC 'pm' may be attached to target process 't'.
1045  */
1046
1047 static int
1048 pmc_can_attach(struct pmc *pm, struct proc *t)
1049 {
1050         struct proc *o;         /* pmc owner */
1051         struct ucred *oc, *tc;  /* owner, target credentials */
1052         int decline_attach, i;
1053
1054         /*
1055          * A PMC's owner can always attach that PMC to itself.
1056          */
1057
1058         if ((o = pm->pm_owner->po_owner) == t)
1059                 return 0;
1060
1061         PROC_LOCK(o);
1062         oc = o->p_ucred;
1063         crhold(oc);
1064         PROC_UNLOCK(o);
1065
1066         PROC_LOCK(t);
1067         tc = t->p_ucred;
1068         crhold(tc);
1069         PROC_UNLOCK(t);
1070
1071         /*
1072          * The effective uid of the PMC owner should match at least one
1073          * of the {effective,real,saved} uids of the target process.
1074          */
1075
1076         decline_attach = oc->cr_uid != tc->cr_uid &&
1077             oc->cr_uid != tc->cr_svuid &&
1078             oc->cr_uid != tc->cr_ruid;
1079
1080         /*
1081          * Every one of the target's group ids, must be in the owner's
1082          * group list.
1083          */
1084         for (i = 0; !decline_attach && i < tc->cr_ngroups; i++)
1085                 decline_attach = !groupmember(tc->cr_groups[i], oc);
1086
1087         /* check the read and saved gids too */
1088         if (decline_attach == 0)
1089                 decline_attach = !groupmember(tc->cr_rgid, oc) ||
1090                     !groupmember(tc->cr_svgid, oc);
1091
1092         crfree(tc);
1093         crfree(oc);
1094
1095         return !decline_attach;
1096 }
1097
1098 /*
1099  * Attach a process to a PMC.
1100  */
1101
1102 static int
1103 pmc_attach_one_process(struct proc *p, struct pmc *pm)
1104 {
1105         int ri, error;
1106         char *fullpath, *freepath;
1107         struct pmc_process      *pp;
1108
1109         sx_assert(&pmc_sx, SX_XLOCKED);
1110
1111         PMCDBG5(PRC,ATT,2, "attach-one pm=%p ri=%d proc=%p (%d, %s)", pm,
1112             PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
1113
1114         /*
1115          * Locate the process descriptor corresponding to process 'p',
1116          * allocating space as needed.
1117          *
1118          * Verify that rowindex 'pm_rowindex' is free in the process
1119          * descriptor.
1120          *
1121          * If not, allocate space for a descriptor and link the
1122          * process descriptor and PMC.
1123          */
1124         ri = PMC_TO_ROWINDEX(pm);
1125
1126         /* mark process as using HWPMCs */
1127         PROC_LOCK(p);
1128         p->p_flag |= P_HWPMC;
1129         PROC_UNLOCK(p);
1130
1131         if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_ALLOCATE)) == NULL) {
1132                 error = ENOMEM;
1133                 goto fail;
1134         }
1135
1136         if (pp->pp_pmcs[ri].pp_pmc == pm) {/* already present at slot [ri] */
1137                 error = EEXIST;
1138                 goto fail;
1139         }
1140
1141         if (pp->pp_pmcs[ri].pp_pmc != NULL) {
1142                 error = EBUSY;
1143                 goto fail;
1144         }
1145
1146         pmc_link_target_process(pm, pp);
1147
1148         if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) &&
1149             (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) == 0)
1150                 pm->pm_flags |= PMC_F_NEEDS_LOGFILE;
1151
1152         pm->pm_flags |= PMC_F_ATTACH_DONE; /* mark as attached */
1153
1154         /* issue an attach event to a configured log file */
1155         if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE) {
1156                 if (p->p_flag & P_KPROC) {
1157                         fullpath = kernelname;
1158                         freepath = NULL;
1159                 } else {
1160                         pmc_getfilename(p->p_textvp, &fullpath, &freepath);
1161                         pmclog_process_pmcattach(pm, p->p_pid, fullpath);
1162                 }
1163                 free(freepath, M_TEMP);
1164                 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1165                         pmc_log_process_mappings(pm->pm_owner, p);
1166         }
1167
1168         return (0);
1169  fail:
1170         PROC_LOCK(p);
1171         p->p_flag &= ~P_HWPMC;
1172         PROC_UNLOCK(p);
1173         return (error);
1174 }
1175
1176 /*
1177  * Attach a process and optionally its children
1178  */
1179
1180 static int
1181 pmc_attach_process(struct proc *p, struct pmc *pm)
1182 {
1183         int error;
1184         struct proc *top;
1185
1186         sx_assert(&pmc_sx, SX_XLOCKED);
1187
1188         PMCDBG5(PRC,ATT,1, "attach pm=%p ri=%d proc=%p (%d, %s)", pm,
1189             PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
1190
1191
1192         /*
1193          * If this PMC successfully allowed a GETMSR operation
1194          * in the past, disallow further ATTACHes.
1195          */
1196
1197         if ((pm->pm_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0)
1198                 return EPERM;
1199
1200         if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
1201                 return pmc_attach_one_process(p, pm);
1202
1203         /*
1204          * Traverse all child processes, attaching them to
1205          * this PMC.
1206          */
1207
1208         sx_slock(&proctree_lock);
1209
1210         top = p;
1211
1212         for (;;) {
1213                 if ((error = pmc_attach_one_process(p, pm)) != 0)
1214                         break;
1215                 if (!LIST_EMPTY(&p->p_children))
1216                         p = LIST_FIRST(&p->p_children);
1217                 else for (;;) {
1218                         if (p == top)
1219                                 goto done;
1220                         if (LIST_NEXT(p, p_sibling)) {
1221                                 p = LIST_NEXT(p, p_sibling);
1222                                 break;
1223                         }
1224                         p = p->p_pptr;
1225                 }
1226         }
1227
1228         if (error)
1229                 (void) pmc_detach_process(top, pm);
1230
1231  done:
1232         sx_sunlock(&proctree_lock);
1233         return error;
1234 }
1235
1236 /*
1237  * Detach a process from a PMC.  If there are no other PMCs tracking
1238  * this process, remove the process structure from its hash table.  If
1239  * 'flags' contains PMC_FLAG_REMOVE, then free the process structure.
1240  */
1241
1242 static int
1243 pmc_detach_one_process(struct proc *p, struct pmc *pm, int flags)
1244 {
1245         int ri;
1246         struct pmc_process *pp;
1247
1248         sx_assert(&pmc_sx, SX_XLOCKED);
1249
1250         KASSERT(pm != NULL,
1251             ("[pmc,%d] null pm pointer", __LINE__));
1252
1253         ri = PMC_TO_ROWINDEX(pm);
1254
1255         PMCDBG6(PRC,ATT,2, "detach-one pm=%p ri=%d proc=%p (%d, %s) flags=0x%x",
1256             pm, ri, p, p->p_pid, p->p_comm, flags);
1257
1258         if ((pp = pmc_find_process_descriptor(p, 0)) == NULL)
1259                 return ESRCH;
1260
1261         if (pp->pp_pmcs[ri].pp_pmc != pm)
1262                 return EINVAL;
1263
1264         pmc_unlink_target_process(pm, pp);
1265
1266         /* Issue a detach entry if a log file is configured */
1267         if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE)
1268                 pmclog_process_pmcdetach(pm, p->p_pid);
1269
1270         /*
1271          * If there are no PMCs targeting this process, we remove its
1272          * descriptor from the target hash table and unset the P_HWPMC
1273          * flag in the struct proc.
1274          */
1275         KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= (int) md->pmd_npmc,
1276             ("[pmc,%d] Illegal refcnt %d for process struct %p",
1277                 __LINE__, pp->pp_refcnt, pp));
1278
1279         if (pp->pp_refcnt != 0) /* still a target of some PMC */
1280                 return 0;
1281
1282         pmc_remove_process_descriptor(pp);
1283
1284         if (flags & PMC_FLAG_REMOVE)
1285                 pmc_destroy_process_descriptor(pp);
1286
1287         PROC_LOCK(p);
1288         p->p_flag &= ~P_HWPMC;
1289         PROC_UNLOCK(p);
1290
1291         return 0;
1292 }
1293
1294 /*
1295  * Detach a process and optionally its descendants from a PMC.
1296  */
1297
1298 static int
1299 pmc_detach_process(struct proc *p, struct pmc *pm)
1300 {
1301         struct proc *top;
1302
1303         sx_assert(&pmc_sx, SX_XLOCKED);
1304
1305         PMCDBG5(PRC,ATT,1, "detach pm=%p ri=%d proc=%p (%d, %s)", pm,
1306             PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
1307
1308         if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
1309                 return pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE);
1310
1311         /*
1312          * Traverse all children, detaching them from this PMC.  We
1313          * ignore errors since we could be detaching a PMC from a
1314          * partially attached proc tree.
1315          */
1316
1317         sx_slock(&proctree_lock);
1318
1319         top = p;
1320
1321         for (;;) {
1322                 (void) pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE);
1323
1324                 if (!LIST_EMPTY(&p->p_children))
1325                         p = LIST_FIRST(&p->p_children);
1326                 else for (;;) {
1327                         if (p == top)
1328                                 goto done;
1329                         if (LIST_NEXT(p, p_sibling)) {
1330                                 p = LIST_NEXT(p, p_sibling);
1331                                 break;
1332                         }
1333                         p = p->p_pptr;
1334                 }
1335         }
1336
1337  done:
1338         sx_sunlock(&proctree_lock);
1339
1340         if (LIST_EMPTY(&pm->pm_targets))
1341                 pm->pm_flags &= ~PMC_F_ATTACH_DONE;
1342
1343         return 0;
1344 }
1345
1346
1347 /*
1348  * Thread context switch IN
1349  */
1350
1351 static void
1352 pmc_process_csw_in(struct thread *td)
1353 {
1354         int cpu;
1355         unsigned int adjri, ri;
1356         struct pmc *pm;
1357         struct proc *p;
1358         struct pmc_cpu *pc;
1359         struct pmc_hw *phw;
1360         pmc_value_t newvalue;
1361         struct pmc_process *pp;
1362         struct pmc_thread *pt;
1363         struct pmc_classdep *pcd;
1364
1365         p = td->td_proc;
1366         pt = NULL;
1367         if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE)) == NULL)
1368                 return;
1369
1370         KASSERT(pp->pp_proc == td->td_proc,
1371             ("[pmc,%d] not my thread state", __LINE__));
1372
1373         critical_enter(); /* no preemption from this point */
1374
1375         cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */
1376
1377         PMCDBG5(CSW,SWI,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p,
1378             p->p_pid, p->p_comm, pp);
1379
1380         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
1381             ("[pmc,%d] weird CPU id %d", __LINE__, cpu));
1382
1383         pc = pmc_pcpu[cpu];
1384
1385         for (ri = 0; ri < md->pmd_npmc; ri++) {
1386
1387                 if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL)
1388                         continue;
1389
1390                 KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)),
1391                     ("[pmc,%d] Target PMC in non-virtual mode (%d)",
1392                         __LINE__, PMC_TO_MODE(pm)));
1393
1394                 KASSERT(PMC_TO_ROWINDEX(pm) == ri,
1395                     ("[pmc,%d] Row index mismatch pmc %d != ri %d",
1396                         __LINE__, PMC_TO_ROWINDEX(pm), ri));
1397
1398                 /*
1399                  * Only PMCs that are marked as 'RUNNING' need
1400                  * be placed on hardware.
1401                  */
1402
1403                 if (pm->pm_state != PMC_STATE_RUNNING)
1404                         continue;
1405
1406                 KASSERT(counter_u64_fetch(pm->pm_runcount) >= 0,
1407             ("[pmc,%d] pm=%p runcount %ld", __LINE__, (void *) pm,
1408                  (unsigned long)counter_u64_fetch(pm->pm_runcount)));
1409
1410                 /* increment PMC runcount */
1411                 counter_u64_add(pm->pm_runcount, 1);
1412
1413                 /* configure the HWPMC we are going to use. */
1414                 pcd = pmc_ri_to_classdep(md, ri, &adjri);
1415                 pcd->pcd_config_pmc(cpu, adjri, pm);
1416
1417                 phw = pc->pc_hwpmcs[ri];
1418
1419                 KASSERT(phw != NULL,
1420                     ("[pmc,%d] null hw pointer", __LINE__));
1421
1422                 KASSERT(phw->phw_pmc == pm,
1423                     ("[pmc,%d] hw->pmc %p != pmc %p", __LINE__,
1424                         phw->phw_pmc, pm));
1425
1426                 /*
1427                  * Write out saved value and start the PMC.
1428                  *
1429                  * Sampling PMCs use a per-thread value, while
1430                  * counting mode PMCs use a per-pmc value that is
1431                  * inherited across descendants.
1432                  */
1433                 if (PMC_TO_MODE(pm) == PMC_MODE_TS) {
1434                         if (pt == NULL)
1435                                 pt = pmc_find_thread_descriptor(pp, td,
1436                                     PMC_FLAG_NONE);
1437
1438                         KASSERT(pt != NULL,
1439                             ("[pmc,%d] No thread found for td=%p", __LINE__,
1440                             td));
1441
1442                         mtx_pool_lock_spin(pmc_mtxpool, pm);
1443
1444                         /*
1445                          * If we have a thread descriptor, use the per-thread
1446                          * counter in the descriptor. If not, we will use
1447                          * a per-process counter.
1448                          *
1449                          * TODO: Remove the per-process "safety net" once
1450                          * we have thoroughly tested that we don't hit the
1451                          * above assert.
1452                          */
1453                         if (pt != NULL) {
1454                                 if (pt->pt_pmcs[ri].pt_pmcval > 0)
1455                                         newvalue = pt->pt_pmcs[ri].pt_pmcval;
1456                                 else
1457                                         newvalue = pm->pm_sc.pm_reloadcount;
1458                         } else {
1459                                 /*
1460                                  * Use the saved value calculated after the most
1461                                  * recent time a thread using the shared counter
1462                                  * switched out. Reset the saved count in case
1463                                  * another thread from this process switches in
1464                                  * before any threads switch out.
1465                                  */
1466
1467                                 newvalue = pp->pp_pmcs[ri].pp_pmcval;
1468                                 pp->pp_pmcs[ri].pp_pmcval =
1469                                     pm->pm_sc.pm_reloadcount;
1470                         }
1471                         mtx_pool_unlock_spin(pmc_mtxpool, pm);
1472                         KASSERT(newvalue > 0 && newvalue <=
1473                             pm->pm_sc.pm_reloadcount,
1474                             ("[pmc,%d] pmcval outside of expected range cpu=%d "
1475                             "ri=%d pmcval=%jx pm_reloadcount=%jx", __LINE__,
1476                             cpu, ri, newvalue, pm->pm_sc.pm_reloadcount));
1477                 } else {
1478                         KASSERT(PMC_TO_MODE(pm) == PMC_MODE_TC,
1479                             ("[pmc,%d] illegal mode=%d", __LINE__,
1480                             PMC_TO_MODE(pm)));
1481                         mtx_pool_lock_spin(pmc_mtxpool, pm);
1482                         newvalue = PMC_PCPU_SAVED(cpu, ri) =
1483                             pm->pm_gv.pm_savedvalue;
1484                         mtx_pool_unlock_spin(pmc_mtxpool, pm);
1485                 }
1486
1487                 PMCDBG3(CSW,SWI,1,"cpu=%d ri=%d new=%jd", cpu, ri, newvalue);
1488
1489                 pcd->pcd_write_pmc(cpu, adjri, newvalue);
1490
1491                 /* If a sampling mode PMC, reset stalled state. */
1492                 if (PMC_TO_MODE(pm) == PMC_MODE_TS)
1493                         pm->pm_pcpu_state[cpu].pps_stalled = 0;
1494
1495                 /* Indicate that we desire this to run. */
1496                 pm->pm_pcpu_state[cpu].pps_cpustate = 1;
1497
1498                 /* Start the PMC. */
1499                 pcd->pcd_start_pmc(cpu, adjri);
1500         }
1501
1502         /*
1503          * perform any other architecture/cpu dependent thread
1504          * switch-in actions.
1505          */
1506
1507         (void) (*md->pmd_switch_in)(pc, pp);
1508
1509         critical_exit();
1510
1511 }
1512
1513 /*
1514  * Thread context switch OUT.
1515  */
1516
1517 static void
1518 pmc_process_csw_out(struct thread *td)
1519 {
1520         int cpu;
1521         int64_t tmp;
1522         struct pmc *pm;
1523         struct proc *p;
1524         enum pmc_mode mode;
1525         struct pmc_cpu *pc;
1526         pmc_value_t newvalue;
1527         unsigned int adjri, ri;
1528         struct pmc_process *pp;
1529         struct pmc_thread *pt = NULL;
1530         struct pmc_classdep *pcd;
1531
1532
1533         /*
1534          * Locate our process descriptor; this may be NULL if
1535          * this process is exiting and we have already removed
1536          * the process from the target process table.
1537          *
1538          * Note that due to kernel preemption, multiple
1539          * context switches may happen while the process is
1540          * exiting.
1541          *
1542          * Note also that if the target process cannot be
1543          * found we still need to deconfigure any PMCs that
1544          * are currently running on hardware.
1545          */
1546
1547         p = td->td_proc;
1548         pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE);
1549
1550         /*
1551          * save PMCs
1552          */
1553
1554         critical_enter();
1555
1556         cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */
1557
1558         PMCDBG5(CSW,SWO,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p,
1559             p->p_pid, p->p_comm, pp);
1560
1561         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
1562             ("[pmc,%d weird CPU id %d", __LINE__, cpu));
1563
1564         pc = pmc_pcpu[cpu];
1565
1566         /*
1567          * When a PMC gets unlinked from a target PMC, it will
1568          * be removed from the target's pp_pmc[] array.
1569          *
1570          * However, on a MP system, the target could have been
1571          * executing on another CPU at the time of the unlink.
1572          * So, at context switch OUT time, we need to look at
1573          * the hardware to determine if a PMC is scheduled on
1574          * it.
1575          */
1576
1577         for (ri = 0; ri < md->pmd_npmc; ri++) {
1578
1579                 pcd = pmc_ri_to_classdep(md, ri, &adjri);
1580                 pm  = NULL;
1581                 (void) (*pcd->pcd_get_config)(cpu, adjri, &pm);
1582
1583                 if (pm == NULL) /* nothing at this row index */
1584                         continue;
1585
1586                 mode = PMC_TO_MODE(pm);
1587                 if (!PMC_IS_VIRTUAL_MODE(mode))
1588                         continue; /* not a process virtual PMC */
1589
1590                 KASSERT(PMC_TO_ROWINDEX(pm) == ri,
1591                     ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
1592                         __LINE__, PMC_TO_ROWINDEX(pm), ri));
1593
1594                 /*
1595                  * Change desired state, and then stop if not stalled.
1596                  * This two-step dance should avoid race conditions where
1597                  * an interrupt re-enables the PMC after this code has
1598                  * already checked the pm_stalled flag.
1599                  */
1600                 pm->pm_pcpu_state[cpu].pps_cpustate = 0;
1601                 if (pm->pm_pcpu_state[cpu].pps_stalled == 0)
1602                         pcd->pcd_stop_pmc(cpu, adjri);
1603
1604                 KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
1605                         ("[pmc,%d] pm=%p runcount %ld", __LINE__, (void *) pm,
1606                          (unsigned long)counter_u64_fetch(pm->pm_runcount)));
1607
1608                 /* reduce this PMC's runcount */
1609                 counter_u64_add(pm->pm_runcount, -1);
1610
1611                 /*
1612                  * If this PMC is associated with this process,
1613                  * save the reading.
1614                  */
1615
1616                 if (pm->pm_state != PMC_STATE_DELETED && pp != NULL &&
1617                     pp->pp_pmcs[ri].pp_pmc != NULL) {
1618                         KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
1619                             ("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__,
1620                                 pm, ri, pp->pp_pmcs[ri].pp_pmc));
1621
1622                         KASSERT(pp->pp_refcnt > 0,
1623                             ("[pmc,%d] pp refcnt = %d", __LINE__,
1624                                 pp->pp_refcnt));
1625
1626                         pcd->pcd_read_pmc(cpu, adjri, &newvalue);
1627
1628                         if (mode == PMC_MODE_TS) {
1629                                 PMCDBG3(CSW,SWO,1,"cpu=%d ri=%d val=%jd (samp)",
1630                                     cpu, ri, newvalue);
1631
1632                                 if (pt == NULL)
1633                                         pt = pmc_find_thread_descriptor(pp, td,
1634                                             PMC_FLAG_NONE);
1635
1636                                 KASSERT(pt != NULL,
1637                                     ("[pmc,%d] No thread found for td=%p",
1638                                     __LINE__, td));
1639
1640                                 mtx_pool_lock_spin(pmc_mtxpool, pm);
1641
1642                                 /*
1643                                  * If we have a thread descriptor, save the
1644                                  * per-thread counter in the descriptor. If not,
1645                                  * we will update the per-process counter.
1646                                  *
1647                                  * TODO: Remove the per-process "safety net"
1648                                  * once we have thoroughly tested that we
1649                                  * don't hit the above assert.
1650                                  */
1651                                 if (pt != NULL)
1652                                         pt->pt_pmcs[ri].pt_pmcval = newvalue;
1653                                 else {
1654                                         /*
1655                                          * For sampling process-virtual PMCs,
1656                                          * newvalue is the number of events to
1657                                          * be seen until the next sampling
1658                                          * interrupt. We can just add the events
1659                                          * left from this invocation to the
1660                                          * counter, then adjust in case we
1661                                          * overflow our range.
1662                                          *
1663                                          * (Recall that we reload the counter
1664                                          * every time we use it.)
1665                                          */
1666                                         pp->pp_pmcs[ri].pp_pmcval += newvalue;
1667                                         if (pp->pp_pmcs[ri].pp_pmcval >
1668                                             pm->pm_sc.pm_reloadcount)
1669                                                 pp->pp_pmcs[ri].pp_pmcval -=
1670                                                     pm->pm_sc.pm_reloadcount;
1671                                 }
1672                                 mtx_pool_unlock_spin(pmc_mtxpool, pm);
1673                         } else {
1674                                 tmp = newvalue - PMC_PCPU_SAVED(cpu,ri);
1675
1676                                 PMCDBG3(CSW,SWO,1,"cpu=%d ri=%d tmp=%jd (count)",
1677                                     cpu, ri, tmp);
1678
1679                                 /*
1680                                  * For counting process-virtual PMCs,
1681                                  * we expect the count to be
1682                                  * increasing monotonically, modulo a 64
1683                                  * bit wraparound.
1684                                  */
1685                                 KASSERT(tmp >= 0,
1686                                     ("[pmc,%d] negative increment cpu=%d "
1687                                      "ri=%d newvalue=%jx saved=%jx "
1688                                      "incr=%jx", __LINE__, cpu, ri,
1689                                      newvalue, PMC_PCPU_SAVED(cpu,ri), tmp));
1690
1691                                 mtx_pool_lock_spin(pmc_mtxpool, pm);
1692                                 pm->pm_gv.pm_savedvalue += tmp;
1693                                 pp->pp_pmcs[ri].pp_pmcval += tmp;
1694                                 mtx_pool_unlock_spin(pmc_mtxpool, pm);
1695
1696                                 if (pm->pm_flags & PMC_F_LOG_PROCCSW)
1697                                         pmclog_process_proccsw(pm, pp, tmp, td);
1698                         }
1699                 }
1700
1701                 /* mark hardware as free */
1702                 pcd->pcd_config_pmc(cpu, adjri, NULL);
1703         }
1704
1705         /*
1706          * perform any other architecture/cpu dependent thread
1707          * switch out functions.
1708          */
1709
1710         (void) (*md->pmd_switch_out)(pc, pp);
1711
1712         critical_exit();
1713 }
1714
1715 /*
1716  * A new thread for a process.
1717  */
1718 static void
1719 pmc_process_thread_add(struct thread *td)
1720 {
1721         struct pmc_process *pmc;
1722
1723         pmc = pmc_find_process_descriptor(td->td_proc, PMC_FLAG_NONE);
1724         if (pmc != NULL)
1725                 pmc_find_thread_descriptor(pmc, td, PMC_FLAG_ALLOCATE);
1726 }
1727
1728 /*
1729  * A thread delete for a process.
1730  */
1731 static void
1732 pmc_process_thread_delete(struct thread *td)
1733 {
1734         struct pmc_process *pmc;
1735
1736         pmc = pmc_find_process_descriptor(td->td_proc, PMC_FLAG_NONE);
1737         if (pmc != NULL)
1738                 pmc_thread_descriptor_pool_free(pmc_find_thread_descriptor(pmc,
1739                     td, PMC_FLAG_REMOVE));
1740 }
1741
1742 /*
1743  * A userret() call for a thread.
1744  */
1745 static void
1746 pmc_process_thread_userret(struct thread *td)
1747 {
1748         sched_pin();
1749         pmc_capture_user_callchain(curcpu, PMC_UR, td->td_frame);
1750         sched_unpin();
1751 }
1752
1753 /*
1754  * A mapping change for a process.
1755  */
1756
1757 static void
1758 pmc_process_mmap(struct thread *td, struct pmckern_map_in *pkm)
1759 {
1760         int ri;
1761         pid_t pid;
1762         char *fullpath, *freepath;
1763         const struct pmc *pm;
1764         struct pmc_owner *po;
1765         const struct pmc_process *pp;
1766
1767         freepath = fullpath = NULL;
1768         MPASS(!in_epoch(global_epoch_preempt));
1769         pmc_getfilename((struct vnode *) pkm->pm_file, &fullpath, &freepath);
1770
1771         pid = td->td_proc->p_pid;
1772
1773         PMC_EPOCH_ENTER();
1774         /* Inform owners of all system-wide sampling PMCs. */
1775         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
1776             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
1777                         pmclog_process_map_in(po, pid, pkm->pm_address, fullpath);
1778
1779         if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL)
1780                 goto done;
1781
1782         /*
1783          * Inform sampling PMC owners tracking this process.
1784          */
1785         for (ri = 0; ri < md->pmd_npmc; ri++)
1786                 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL &&
1787                     PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1788                         pmclog_process_map_in(pm->pm_owner,
1789                             pid, pkm->pm_address, fullpath);
1790
1791   done:
1792         if (freepath)
1793                 free(freepath, M_TEMP);
1794         PMC_EPOCH_EXIT();
1795 }
1796
1797
1798 /*
1799  * Log an munmap request.
1800  */
1801
1802 static void
1803 pmc_process_munmap(struct thread *td, struct pmckern_map_out *pkm)
1804 {
1805         int ri;
1806         pid_t pid;
1807         struct pmc_owner *po;
1808         const struct pmc *pm;
1809         const struct pmc_process *pp;
1810
1811         pid = td->td_proc->p_pid;
1812
1813         PMC_EPOCH_ENTER();
1814         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
1815             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
1816                 pmclog_process_map_out(po, pid, pkm->pm_address,
1817                     pkm->pm_address + pkm->pm_size);
1818         PMC_EPOCH_EXIT();
1819
1820         if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL)
1821                 return;
1822
1823         for (ri = 0; ri < md->pmd_npmc; ri++)
1824                 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL &&
1825                     PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1826                         pmclog_process_map_out(pm->pm_owner, pid,
1827                             pkm->pm_address, pkm->pm_address + pkm->pm_size);
1828 }
1829
1830 /*
1831  * Log mapping information about the kernel.
1832  */
1833
1834 static void
1835 pmc_log_kernel_mappings(struct pmc *pm)
1836 {
1837         struct pmc_owner *po;
1838         struct pmckern_map_in *km, *kmbase;
1839
1840         MPASS(in_epoch(global_epoch_preempt) || sx_xlocked(&pmc_sx));
1841         KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
1842             ("[pmc,%d] non-sampling PMC (%p) desires mapping information",
1843                 __LINE__, (void *) pm));
1844
1845         po = pm->pm_owner;
1846
1847         if (po->po_flags & PMC_PO_INITIAL_MAPPINGS_DONE)
1848                 return;
1849         if (PMC_TO_MODE(pm) == PMC_MODE_SS)
1850                 pmc_process_allproc(pm);
1851         /*
1852          * Log the current set of kernel modules.
1853          */
1854         kmbase = linker_hwpmc_list_objects();
1855         for (km = kmbase; km->pm_file != NULL; km++) {
1856                 PMCDBG2(LOG,REG,1,"%s %p", (char *) km->pm_file,
1857                     (void *) km->pm_address);
1858                 pmclog_process_map_in(po, (pid_t) -1, km->pm_address,
1859                     km->pm_file);
1860         }
1861         free(kmbase, M_LINKER);
1862
1863         po->po_flags |= PMC_PO_INITIAL_MAPPINGS_DONE;
1864 }
1865
1866 /*
1867  * Log the mappings for a single process.
1868  */
1869
1870 static void
1871 pmc_log_process_mappings(struct pmc_owner *po, struct proc *p)
1872 {
1873         vm_map_t map;
1874         struct vnode *vp;
1875         struct vmspace *vm;
1876         vm_map_entry_t entry;
1877         vm_offset_t last_end;
1878         u_int last_timestamp;
1879         struct vnode *last_vp;
1880         vm_offset_t start_addr;
1881         vm_object_t obj, lobj, tobj;
1882         char *fullpath, *freepath;
1883
1884         last_vp = NULL;
1885         last_end = (vm_offset_t) 0;
1886         fullpath = freepath = NULL;
1887
1888         if ((vm = vmspace_acquire_ref(p)) == NULL)
1889                 return;
1890
1891         map = &vm->vm_map;
1892         vm_map_lock_read(map);
1893
1894         for (entry = map->header.next; entry != &map->header; entry = entry->next) {
1895
1896                 if (entry == NULL) {
1897                         PMCDBG2(LOG,OPS,2, "hwpmc: vm_map entry unexpectedly "
1898                             "NULL! pid=%d vm_map=%p\n", p->p_pid, map);
1899                         break;
1900                 }
1901
1902                 /*
1903                  * We only care about executable map entries.
1904                  */
1905                 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) ||
1906                     !(entry->protection & VM_PROT_EXECUTE) ||
1907                     (entry->object.vm_object == NULL)) {
1908                         continue;
1909                 }
1910
1911                 obj = entry->object.vm_object;
1912                 VM_OBJECT_RLOCK(obj);
1913
1914                 /*
1915                  * Walk the backing_object list to find the base
1916                  * (non-shadowed) vm_object.
1917                  */
1918                 for (lobj = tobj = obj; tobj != NULL; tobj = tobj->backing_object) {
1919                         if (tobj != obj)
1920                                 VM_OBJECT_RLOCK(tobj);
1921                         if (lobj != obj)
1922                                 VM_OBJECT_RUNLOCK(lobj);
1923                         lobj = tobj;
1924                 }
1925
1926                 /*
1927                  * At this point lobj is the base vm_object and it is locked.
1928                  */
1929                 if (lobj == NULL) {
1930                         PMCDBG3(LOG,OPS,2, "hwpmc: lobj unexpectedly NULL! pid=%d "
1931                             "vm_map=%p vm_obj=%p\n", p->p_pid, map, obj);
1932                         VM_OBJECT_RUNLOCK(obj);
1933                         continue;
1934                 }
1935
1936                 vp = vm_object_vnode(lobj);
1937                 if (vp == NULL) {
1938                         if (lobj != obj)
1939                                 VM_OBJECT_RUNLOCK(lobj);
1940                         VM_OBJECT_RUNLOCK(obj);
1941                         continue;
1942                 }
1943
1944                 /*
1945                  * Skip contiguous regions that point to the same
1946                  * vnode, so we don't emit redundant MAP-IN
1947                  * directives.
1948                  */
1949                 if (entry->start == last_end && vp == last_vp) {
1950                         last_end = entry->end;
1951                         if (lobj != obj)
1952                                 VM_OBJECT_RUNLOCK(lobj);
1953                         VM_OBJECT_RUNLOCK(obj);
1954                         continue;
1955                 }
1956
1957                 /*
1958                  * We don't want to keep the proc's vm_map or this
1959                  * vm_object locked while we walk the pathname, since
1960                  * vn_fullpath() can sleep.  However, if we drop the
1961                  * lock, it's possible for concurrent activity to
1962                  * modify the vm_map list.  To protect against this,
1963                  * we save the vm_map timestamp before we release the
1964                  * lock, and check it after we reacquire the lock
1965                  * below.
1966                  */
1967                 start_addr = entry->start;
1968                 last_end = entry->end;
1969                 last_timestamp = map->timestamp;
1970                 vm_map_unlock_read(map);
1971
1972                 vref(vp);
1973                 if (lobj != obj)
1974                         VM_OBJECT_RUNLOCK(lobj);
1975
1976                 VM_OBJECT_RUNLOCK(obj);
1977
1978                 freepath = NULL;
1979                 pmc_getfilename(vp, &fullpath, &freepath);
1980                 last_vp = vp;
1981
1982                 vrele(vp);
1983
1984                 vp = NULL;
1985                 pmclog_process_map_in(po, p->p_pid, start_addr, fullpath);
1986                 if (freepath)
1987                         free(freepath, M_TEMP);
1988
1989                 vm_map_lock_read(map);
1990
1991                 /*
1992                  * If our saved timestamp doesn't match, this means
1993                  * that the vm_map was modified out from under us and
1994                  * we can't trust our current "entry" pointer.  Do a
1995                  * new lookup for this entry.  If there is no entry
1996                  * for this address range, vm_map_lookup_entry() will
1997                  * return the previous one, so we always want to go to
1998                  * entry->next on the next loop iteration.
1999                  *
2000                  * There is an edge condition here that can occur if
2001                  * there is no entry at or before this address.  In
2002                  * this situation, vm_map_lookup_entry returns
2003                  * &map->header, which would cause our loop to abort
2004                  * without processing the rest of the map.  However,
2005                  * in practice this will never happen for process
2006                  * vm_map.  This is because the executable's text
2007                  * segment is the first mapping in the proc's address
2008                  * space, and this mapping is never removed until the
2009                  * process exits, so there will always be a non-header
2010                  * entry at or before the requested address for
2011                  * vm_map_lookup_entry to return.
2012                  */
2013                 if (map->timestamp != last_timestamp)
2014                         vm_map_lookup_entry(map, last_end - 1, &entry);
2015         }
2016
2017         vm_map_unlock_read(map);
2018         vmspace_free(vm);
2019         return;
2020 }
2021
2022 /*
2023  * Log mappings for all processes in the system.
2024  */
2025
2026 static void
2027 pmc_log_all_process_mappings(struct pmc_owner *po)
2028 {
2029         struct proc *p, *top;
2030
2031         sx_assert(&pmc_sx, SX_XLOCKED);
2032
2033         if ((p = pfind(1)) == NULL)
2034                 panic("[pmc,%d] Cannot find init", __LINE__);
2035
2036         PROC_UNLOCK(p);
2037
2038         sx_slock(&proctree_lock);
2039
2040         top = p;
2041
2042         for (;;) {
2043                 pmc_log_process_mappings(po, p);
2044                 if (!LIST_EMPTY(&p->p_children))
2045                         p = LIST_FIRST(&p->p_children);
2046                 else for (;;) {
2047                         if (p == top)
2048                                 goto done;
2049                         if (LIST_NEXT(p, p_sibling)) {
2050                                 p = LIST_NEXT(p, p_sibling);
2051                                 break;
2052                         }
2053                         p = p->p_pptr;
2054                 }
2055         }
2056  done:
2057         sx_sunlock(&proctree_lock);
2058 }
2059
2060 /*
2061  * The 'hook' invoked from the kernel proper
2062  */
2063
2064
2065 #ifdef  HWPMC_DEBUG
2066 const char *pmc_hooknames[] = {
2067         /* these strings correspond to PMC_FN_* in <sys/pmckern.h> */
2068         "",
2069         "EXEC",
2070         "CSW-IN",
2071         "CSW-OUT",
2072         "SAMPLE",
2073         "UNUSED1",
2074         "UNUSED2",
2075         "MMAP",
2076         "MUNMAP",
2077         "CALLCHAIN-NMI",
2078         "CALLCHAIN-SOFT",
2079         "SOFTSAMPLING",
2080         "THR-CREATE",
2081         "THR-EXIT",
2082         "THR-USERRET",
2083         "THR-CREATE-LOG",
2084         "THR-EXIT-LOG",
2085         "PROC-CREATE-LOG"
2086 };
2087 #endif
2088
2089 static int
2090 pmc_hook_handler(struct thread *td, int function, void *arg)
2091 {
2092         int cpu;
2093
2094         PMCDBG4(MOD,PMH,1, "hook td=%p func=%d \"%s\" arg=%p", td, function,
2095             pmc_hooknames[function], arg);
2096
2097         switch (function)
2098         {
2099
2100         /*
2101          * Process exec()
2102          */
2103
2104         case PMC_FN_PROCESS_EXEC:
2105         {
2106                 char *fullpath, *freepath;
2107                 unsigned int ri;
2108                 int is_using_hwpmcs;
2109                 struct pmc *pm;
2110                 struct proc *p;
2111                 struct pmc_owner *po;
2112                 struct pmc_process *pp;
2113                 struct pmckern_procexec *pk;
2114
2115                 sx_assert(&pmc_sx, SX_XLOCKED);
2116
2117                 p = td->td_proc;
2118                 pmc_getfilename(p->p_textvp, &fullpath, &freepath);
2119
2120                 pk = (struct pmckern_procexec *) arg;
2121
2122                 PMC_EPOCH_ENTER();
2123                 /* Inform owners of SS mode PMCs of the exec event. */
2124                 CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
2125                     if (po->po_flags & PMC_PO_OWNS_LOGFILE)
2126                             pmclog_process_procexec(po, PMC_ID_INVALID,
2127                                 p->p_pid, pk->pm_entryaddr, fullpath);
2128                 PMC_EPOCH_EXIT();
2129
2130                 PROC_LOCK(p);
2131                 is_using_hwpmcs = p->p_flag & P_HWPMC;
2132                 PROC_UNLOCK(p);
2133
2134                 if (!is_using_hwpmcs) {
2135                         if (freepath)
2136                                 free(freepath, M_TEMP);
2137                         break;
2138                 }
2139
2140                 /*
2141                  * PMCs are not inherited across an exec():  remove any
2142                  * PMCs that this process is the owner of.
2143                  */
2144
2145                 if ((po = pmc_find_owner_descriptor(p)) != NULL) {
2146                         pmc_remove_owner(po);
2147                         pmc_destroy_owner_descriptor(po);
2148                 }
2149
2150                 /*
2151                  * If the process being exec'ed is not the target of any
2152                  * PMC, we are done.
2153                  */
2154                 if ((pp = pmc_find_process_descriptor(p, 0)) == NULL) {
2155                         if (freepath)
2156                                 free(freepath, M_TEMP);
2157                         break;
2158                 }
2159
2160                 /*
2161                  * Log the exec event to all monitoring owners.  Skip
2162                  * owners who have already received the event because
2163                  * they had system sampling PMCs active.
2164                  */
2165                 for (ri = 0; ri < md->pmd_npmc; ri++)
2166                         if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) {
2167                                 po = pm->pm_owner;
2168                                 if (po->po_sscount == 0 &&
2169                                     po->po_flags & PMC_PO_OWNS_LOGFILE)
2170                                         pmclog_process_procexec(po, pm->pm_id,
2171                                             p->p_pid, pk->pm_entryaddr,
2172                                             fullpath);
2173                         }
2174
2175                 if (freepath)
2176                         free(freepath, M_TEMP);
2177
2178
2179                 PMCDBG4(PRC,EXC,1, "exec proc=%p (%d, %s) cred-changed=%d",
2180                     p, p->p_pid, p->p_comm, pk->pm_credentialschanged);
2181
2182                 if (pk->pm_credentialschanged == 0) /* no change */
2183                         break;
2184
2185                 /*
2186                  * If the newly exec()'ed process has a different credential
2187                  * than before, allow it to be the target of a PMC only if
2188                  * the PMC's owner has sufficient privilege.
2189                  */
2190
2191                 for (ri = 0; ri < md->pmd_npmc; ri++)
2192                         if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL)
2193                                 if (pmc_can_attach(pm, td->td_proc) != 0)
2194                                         pmc_detach_one_process(td->td_proc,
2195                                             pm, PMC_FLAG_NONE);
2196
2197                 KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= (int) md->pmd_npmc,
2198                     ("[pmc,%d] Illegal ref count %d on pp %p", __LINE__,
2199                         pp->pp_refcnt, pp));
2200
2201                 /*
2202                  * If this process is no longer the target of any
2203                  * PMCs, we can remove the process entry and free
2204                  * up space.
2205                  */
2206
2207                 if (pp->pp_refcnt == 0) {
2208                         pmc_remove_process_descriptor(pp);
2209                         pmc_destroy_process_descriptor(pp);
2210                         break;
2211                 }
2212
2213         }
2214         break;
2215
2216         case PMC_FN_CSW_IN:
2217                 pmc_process_csw_in(td);
2218                 break;
2219
2220         case PMC_FN_CSW_OUT:
2221                 pmc_process_csw_out(td);
2222                 break;
2223
2224         /*
2225          * Process accumulated PC samples.
2226          *
2227          * This function is expected to be called by hardclock() for
2228          * each CPU that has accumulated PC samples.
2229          *
2230          * This function is to be executed on the CPU whose samples
2231          * are being processed.
2232          */
2233         case PMC_FN_DO_SAMPLES:
2234
2235                 /*
2236                  * Clear the cpu specific bit in the CPU mask before
2237                  * do the rest of the processing.  If the NMI handler
2238                  * gets invoked after the "atomic_clear_int()" call
2239                  * below but before "pmc_process_samples()" gets
2240                  * around to processing the interrupt, then we will
2241                  * come back here at the next hardclock() tick (and
2242                  * may find nothing to do if "pmc_process_samples()"
2243                  * had already processed the interrupt).  We don't
2244                  * lose the interrupt sample.
2245                  */
2246                 DPCPU_SET(pmc_sampled, 0);
2247                 cpu = PCPU_GET(cpuid);
2248                 pmc_process_samples(cpu, PMC_HR);
2249                 pmc_process_samples(cpu, PMC_SR);
2250                 pmc_process_samples(cpu, PMC_UR);
2251                 break;
2252
2253         case PMC_FN_MMAP:
2254                 pmc_process_mmap(td, (struct pmckern_map_in *) arg);
2255                 break;
2256
2257         case PMC_FN_MUNMAP:
2258                 MPASS(in_epoch(global_epoch_preempt) || sx_xlocked(&pmc_sx));
2259                 pmc_process_munmap(td, (struct pmckern_map_out *) arg);
2260                 break;
2261
2262         case PMC_FN_PROC_CREATE_LOG:
2263                 pmc_process_proccreate((struct proc *)arg);
2264                 break;
2265
2266         case PMC_FN_USER_CALLCHAIN:
2267                 /*
2268                  * Record a call chain.
2269                  */
2270                 KASSERT(td == curthread, ("[pmc,%d] td != curthread",
2271                     __LINE__));
2272
2273                 pmc_capture_user_callchain(PCPU_GET(cpuid), PMC_HR,
2274                     (struct trapframe *) arg);
2275
2276                 KASSERT(td->td_pinned == 1,
2277                         ("[pmc,%d] invalid td_pinned value", __LINE__));
2278                 sched_unpin();  /* Can migrate safely now. */
2279
2280                 td->td_pflags &= ~TDP_CALLCHAIN;
2281                 break;
2282
2283         case PMC_FN_USER_CALLCHAIN_SOFT:
2284                 /*
2285                  * Record a call chain.
2286                  */
2287                 KASSERT(td == curthread, ("[pmc,%d] td != curthread",
2288                     __LINE__));
2289
2290                 cpu = PCPU_GET(cpuid);
2291                 pmc_capture_user_callchain(cpu, PMC_SR,
2292                     (struct trapframe *) arg);
2293
2294                 KASSERT(td->td_pinned == 1,
2295                     ("[pmc,%d] invalid td_pinned value", __LINE__));
2296
2297                 sched_unpin();  /* Can migrate safely now. */
2298
2299                 td->td_pflags &= ~TDP_CALLCHAIN;
2300                 break;
2301
2302         case PMC_FN_SOFT_SAMPLING:
2303                 /*
2304                  * Call soft PMC sampling intr.
2305                  */
2306                 pmc_soft_intr((struct pmckern_soft *) arg);
2307                 break;
2308
2309         case PMC_FN_THR_CREATE:
2310                 pmc_process_thread_add(td);
2311                 pmc_process_threadcreate(td);
2312                 break;
2313
2314         case PMC_FN_THR_CREATE_LOG:
2315                 pmc_process_threadcreate(td);
2316                 break;
2317
2318         case PMC_FN_THR_EXIT:
2319                 KASSERT(td == curthread, ("[pmc,%d] td != curthread",
2320                     __LINE__));
2321                 pmc_process_thread_delete(td);
2322                 pmc_process_threadexit(td);
2323                 break;
2324         case PMC_FN_THR_EXIT_LOG:
2325                 pmc_process_threadexit(td);
2326                 break;
2327         case PMC_FN_THR_USERRET:
2328                 KASSERT(td == curthread, ("[pmc,%d] td != curthread",
2329                     __LINE__));
2330                 pmc_process_thread_userret(td);
2331                 break;
2332
2333         default:
2334 #ifdef  HWPMC_DEBUG
2335                 KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function));
2336 #endif
2337                 break;
2338
2339         }
2340
2341         return 0;
2342 }
2343
2344 /*
2345  * allocate a 'struct pmc_owner' descriptor in the owner hash table.
2346  */
2347
2348 static struct pmc_owner *
2349 pmc_allocate_owner_descriptor(struct proc *p)
2350 {
2351         uint32_t hindex;
2352         struct pmc_owner *po;
2353         struct pmc_ownerhash *poh;
2354
2355         hindex = PMC_HASH_PTR(p, pmc_ownerhashmask);
2356         poh = &pmc_ownerhash[hindex];
2357
2358         /* allocate space for N pointers and one descriptor struct */
2359         po = malloc(sizeof(struct pmc_owner), M_PMC, M_WAITOK|M_ZERO);
2360         po->po_owner = p;
2361         LIST_INSERT_HEAD(poh, po, po_next); /* insert into hash table */
2362
2363         TAILQ_INIT(&po->po_logbuffers);
2364         mtx_init(&po->po_mtx, "pmc-owner-mtx", "pmc-per-proc", MTX_SPIN);
2365
2366         PMCDBG4(OWN,ALL,1, "allocate-owner proc=%p (%d, %s) pmc-owner=%p",
2367             p, p->p_pid, p->p_comm, po);
2368
2369         return po;
2370 }
2371
2372 static void
2373 pmc_destroy_owner_descriptor(struct pmc_owner *po)
2374 {
2375
2376         PMCDBG4(OWN,REL,1, "destroy-owner po=%p proc=%p (%d, %s)",
2377             po, po->po_owner, po->po_owner->p_pid, po->po_owner->p_comm);
2378
2379         mtx_destroy(&po->po_mtx);
2380         free(po, M_PMC);
2381 }
2382
2383 /*
2384  * Allocate a thread descriptor from the free pool.
2385  *
2386  * NOTE: This *can* return NULL.
2387  */
2388 static struct pmc_thread *
2389 pmc_thread_descriptor_pool_alloc(void)
2390 {
2391         struct pmc_thread *pt;
2392
2393         mtx_lock_spin(&pmc_threadfreelist_mtx);
2394         if ((pt = LIST_FIRST(&pmc_threadfreelist)) != NULL) {
2395                 LIST_REMOVE(pt, pt_next);
2396                 pmc_threadfreelist_entries--;
2397         }
2398         mtx_unlock_spin(&pmc_threadfreelist_mtx);
2399
2400         return (pt);
2401 }
2402
2403 /*
2404  * Add a thread descriptor to the free pool. We use this instead of free()
2405  * to maintain a cache of free entries. Additionally, we can safely call
2406  * this function when we cannot call free(), such as in a critical section.
2407  *
2408  */
2409 static void
2410 pmc_thread_descriptor_pool_free(struct pmc_thread *pt)
2411 {
2412
2413         if (pt == NULL)
2414                 return;
2415
2416         memset(pt, 0, THREADENTRY_SIZE);
2417         mtx_lock_spin(&pmc_threadfreelist_mtx);
2418         LIST_INSERT_HEAD(&pmc_threadfreelist, pt, pt_next);
2419         pmc_threadfreelist_entries++;
2420         if (pmc_threadfreelist_entries > pmc_threadfreelist_max)
2421                 GROUPTASK_ENQUEUE(&free_gtask);
2422         mtx_unlock_spin(&pmc_threadfreelist_mtx);
2423 }
2424
2425 /*
2426  * A callout to manage the free list.
2427  */
2428 static void
2429 pmc_thread_descriptor_pool_free_task(void *arg __unused)
2430 {
2431         struct pmc_thread *pt;
2432         LIST_HEAD(, pmc_thread) tmplist;
2433         int delta;
2434
2435         LIST_INIT(&tmplist);
2436         /* Determine what changes, if any, we need to make. */
2437         mtx_lock_spin(&pmc_threadfreelist_mtx);
2438         delta = pmc_threadfreelist_entries - pmc_threadfreelist_max;
2439         while (delta > 0 &&
2440                    (pt = LIST_FIRST(&pmc_threadfreelist)) != NULL) {
2441                 delta--;
2442                 LIST_REMOVE(pt, pt_next);
2443                 LIST_INSERT_HEAD(&tmplist, pt, pt_next);
2444         }
2445         mtx_unlock_spin(&pmc_threadfreelist_mtx);
2446
2447         /* If there are entries to free, free them. */
2448         while (!LIST_EMPTY(&tmplist)) {
2449                 pt = LIST_FIRST(&tmplist);
2450                 LIST_REMOVE(pt, pt_next);
2451                 free(pt, M_PMC);
2452         }
2453 }
2454
2455 /*
2456  * Drain the thread free pool, freeing all allocations.
2457  */
2458 static void
2459 pmc_thread_descriptor_pool_drain()
2460 {
2461         struct pmc_thread *pt, *next;
2462
2463         LIST_FOREACH_SAFE(pt, &pmc_threadfreelist, pt_next, next) {
2464                 LIST_REMOVE(pt, pt_next);
2465                 free(pt, M_PMC);
2466         }
2467 }
2468
2469 /*
2470  * find the descriptor corresponding to thread 'td', adding or removing it
2471  * as specified by 'mode'.
2472  *
2473  * Note that this supports additional mode flags in addition to those
2474  * supported by pmc_find_process_descriptor():
2475  * PMC_FLAG_NOWAIT: Causes the function to not wait for mallocs.
2476  *     This makes it safe to call while holding certain other locks.
2477  */
2478
2479 static struct pmc_thread *
2480 pmc_find_thread_descriptor(struct pmc_process *pp, struct thread *td,
2481     uint32_t mode)
2482 {
2483         struct pmc_thread *pt = NULL, *ptnew = NULL;
2484         int wait_flag;
2485
2486         KASSERT(td != NULL, ("[pmc,%d] called to add NULL td", __LINE__));
2487
2488         /*
2489          * Pre-allocate memory in the PMC_FLAG_ALLOCATE case prior to
2490          * acquiring the lock.
2491          */
2492         if (mode & PMC_FLAG_ALLOCATE) {
2493                 if ((ptnew = pmc_thread_descriptor_pool_alloc()) == NULL) {
2494                         wait_flag = M_WAITOK;
2495                         if ((mode & PMC_FLAG_NOWAIT) || in_epoch(global_epoch_preempt))
2496                                 wait_flag = M_NOWAIT;
2497
2498                         ptnew = malloc(THREADENTRY_SIZE, M_PMC,
2499                             wait_flag|M_ZERO);
2500                 }
2501         }
2502
2503         mtx_lock_spin(pp->pp_tdslock);
2504
2505         LIST_FOREACH(pt, &pp->pp_tds, pt_next)
2506                 if (pt->pt_td == td)
2507                         break;
2508
2509         if ((mode & PMC_FLAG_REMOVE) && pt != NULL)
2510                 LIST_REMOVE(pt, pt_next);
2511
2512         if ((mode & PMC_FLAG_ALLOCATE) && pt == NULL && ptnew != NULL) {
2513                 pt = ptnew;
2514                 ptnew = NULL;
2515                 pt->pt_td = td;
2516                 LIST_INSERT_HEAD(&pp->pp_tds, pt, pt_next);
2517         }
2518
2519         mtx_unlock_spin(pp->pp_tdslock);
2520
2521         if (ptnew != NULL) {
2522                 free(ptnew, M_PMC);
2523         }
2524
2525         return pt;
2526 }
2527
2528 /*
2529  * Try to add thread descriptors for each thread in a process.
2530  */
2531
2532 static void
2533 pmc_add_thread_descriptors_from_proc(struct proc *p, struct pmc_process *pp)
2534 {
2535         struct thread *curtd;
2536         struct pmc_thread **tdlist;
2537         int i, tdcnt, tdlistsz;
2538
2539         KASSERT(!PROC_LOCKED(p), ("[pmc,%d] proc unexpectedly locked",
2540             __LINE__));
2541         tdcnt = 32;
2542  restart:
2543         tdlistsz = roundup2(tdcnt, 32);
2544
2545         tdcnt = 0;
2546         tdlist = malloc(sizeof(struct pmc_thread*) * tdlistsz, M_TEMP, M_WAITOK);
2547
2548         PROC_LOCK(p);
2549         FOREACH_THREAD_IN_PROC(p, curtd)
2550                 tdcnt++;
2551         if (tdcnt >= tdlistsz) {
2552                 PROC_UNLOCK(p);
2553                 free(tdlist, M_TEMP);
2554                 goto restart;
2555         }
2556         /*
2557          * Try to add each thread to the list without sleeping. If unable,
2558          * add to a queue to retry after dropping the process lock.
2559          */
2560         tdcnt = 0;
2561         FOREACH_THREAD_IN_PROC(p, curtd) {
2562                 tdlist[tdcnt] = pmc_find_thread_descriptor(pp, curtd,
2563                                                    PMC_FLAG_ALLOCATE|PMC_FLAG_NOWAIT);
2564                 if (tdlist[tdcnt] == NULL) {
2565                         PROC_UNLOCK(p);
2566                         for (i = 0; i <= tdcnt; i++)
2567                                 pmc_thread_descriptor_pool_free(tdlist[i]);
2568                         free(tdlist, M_TEMP);
2569                         goto restart;
2570                 }
2571                 tdcnt++;
2572         }
2573         PROC_UNLOCK(p);
2574         free(tdlist, M_TEMP);
2575 }
2576
2577 /*
2578  * find the descriptor corresponding to process 'p', adding or removing it
2579  * as specified by 'mode'.
2580  */
2581
2582 static struct pmc_process *
2583 pmc_find_process_descriptor(struct proc *p, uint32_t mode)
2584 {
2585         uint32_t hindex;
2586         struct pmc_process *pp, *ppnew;
2587         struct pmc_processhash *pph;
2588
2589         hindex = PMC_HASH_PTR(p, pmc_processhashmask);
2590         pph = &pmc_processhash[hindex];
2591
2592         ppnew = NULL;
2593
2594         /*
2595          * Pre-allocate memory in the PMC_FLAG_ALLOCATE case since we
2596          * cannot call malloc(9) once we hold a spin lock.
2597          */
2598         if (mode & PMC_FLAG_ALLOCATE)
2599                 ppnew = malloc(sizeof(struct pmc_process) + md->pmd_npmc *
2600                     sizeof(struct pmc_targetstate), M_PMC, M_WAITOK|M_ZERO);
2601
2602         mtx_lock_spin(&pmc_processhash_mtx);
2603         LIST_FOREACH(pp, pph, pp_next)
2604             if (pp->pp_proc == p)
2605                     break;
2606
2607         if ((mode & PMC_FLAG_REMOVE) && pp != NULL)
2608                 LIST_REMOVE(pp, pp_next);
2609
2610         if ((mode & PMC_FLAG_ALLOCATE) && pp == NULL &&
2611             ppnew != NULL) {
2612                 ppnew->pp_proc = p;
2613                 LIST_INIT(&ppnew->pp_tds);
2614                 ppnew->pp_tdslock = mtx_pool_find(pmc_mtxpool, ppnew);
2615                 LIST_INSERT_HEAD(pph, ppnew, pp_next);
2616                 mtx_unlock_spin(&pmc_processhash_mtx);
2617                 pp = ppnew;
2618                 ppnew = NULL;
2619
2620                 /* Add thread descriptors for this process' current threads. */
2621                 pmc_add_thread_descriptors_from_proc(p, pp);
2622         }
2623         else
2624                 mtx_unlock_spin(&pmc_processhash_mtx);
2625
2626         if (ppnew != NULL)
2627                 free(ppnew, M_PMC);
2628
2629         return pp;
2630 }
2631
2632 /*
2633  * remove a process descriptor from the process hash table.
2634  */
2635
2636 static void
2637 pmc_remove_process_descriptor(struct pmc_process *pp)
2638 {
2639         KASSERT(pp->pp_refcnt == 0,
2640             ("[pmc,%d] Removing process descriptor %p with count %d",
2641                 __LINE__, pp, pp->pp_refcnt));
2642
2643         mtx_lock_spin(&pmc_processhash_mtx);
2644         LIST_REMOVE(pp, pp_next);
2645         mtx_unlock_spin(&pmc_processhash_mtx);
2646 }
2647
2648 /*
2649  * destroy a process descriptor.
2650  */
2651
2652 static void
2653 pmc_destroy_process_descriptor(struct pmc_process *pp)
2654 {
2655         struct pmc_thread *pmc_td;
2656
2657         while ((pmc_td = LIST_FIRST(&pp->pp_tds)) != NULL) {
2658                 LIST_REMOVE(pmc_td, pt_next);
2659                 pmc_thread_descriptor_pool_free(pmc_td);
2660         }
2661         free(pp, M_PMC);
2662 }
2663
2664
2665 /*
2666  * find an owner descriptor corresponding to proc 'p'
2667  */
2668
2669 static struct pmc_owner *
2670 pmc_find_owner_descriptor(struct proc *p)
2671 {
2672         uint32_t hindex;
2673         struct pmc_owner *po;
2674         struct pmc_ownerhash *poh;
2675
2676         hindex = PMC_HASH_PTR(p, pmc_ownerhashmask);
2677         poh = &pmc_ownerhash[hindex];
2678
2679         po = NULL;
2680         LIST_FOREACH(po, poh, po_next)
2681             if (po->po_owner == p)
2682                     break;
2683
2684         PMCDBG5(OWN,FND,1, "find-owner proc=%p (%d, %s) hindex=0x%x -> "
2685             "pmc-owner=%p", p, p->p_pid, p->p_comm, hindex, po);
2686
2687         return po;
2688 }
2689
2690 /*
2691  * pmc_allocate_pmc_descriptor
2692  *
2693  * Allocate a pmc descriptor and initialize its
2694  * fields.
2695  */
2696
2697 static struct pmc *
2698 pmc_allocate_pmc_descriptor(void)
2699 {
2700         struct pmc *pmc;
2701
2702         pmc = malloc(sizeof(struct pmc), M_PMC, M_WAITOK|M_ZERO);
2703         pmc->pm_runcount = counter_u64_alloc(M_WAITOK);
2704         pmc->pm_pcpu_state = malloc(sizeof(struct pmc_pcpu_state)*mp_ncpus, M_PMC, M_WAITOK|M_ZERO);
2705         PMCDBG1(PMC,ALL,1, "allocate-pmc -> pmc=%p", pmc);
2706
2707         return pmc;
2708 }
2709
2710 /*
2711  * Destroy a pmc descriptor.
2712  */
2713
2714 static void
2715 pmc_destroy_pmc_descriptor(struct pmc *pm)
2716 {
2717
2718         KASSERT(pm->pm_state == PMC_STATE_DELETED ||
2719             pm->pm_state == PMC_STATE_FREE,
2720             ("[pmc,%d] destroying non-deleted PMC", __LINE__));
2721         KASSERT(LIST_EMPTY(&pm->pm_targets),
2722             ("[pmc,%d] destroying pmc with targets", __LINE__));
2723         KASSERT(pm->pm_owner == NULL,
2724             ("[pmc,%d] destroying pmc attached to an owner", __LINE__));
2725         KASSERT(counter_u64_fetch(pm->pm_runcount) == 0,
2726             ("[pmc,%d] pmc has non-zero run count %ld", __LINE__,
2727                  (unsigned long)counter_u64_fetch(pm->pm_runcount)));
2728
2729         counter_u64_free(pm->pm_runcount);
2730         free(pm->pm_pcpu_state, M_PMC);
2731         free(pm, M_PMC);
2732 }
2733
2734 static void
2735 pmc_wait_for_pmc_idle(struct pmc *pm)
2736 {
2737 #ifdef INVARIANTS
2738         volatile int maxloop;
2739
2740         maxloop = 100 * pmc_cpu_max();
2741 #endif
2742         /*
2743          * Loop (with a forced context switch) till the PMC's runcount
2744          * comes down to zero.
2745          */
2746         pmclog_flush(pm->pm_owner, 1);
2747         while (counter_u64_fetch(pm->pm_runcount) > 0) {
2748                 pmclog_flush(pm->pm_owner, 1);
2749 #ifdef INVARIANTS
2750                 maxloop--;
2751                 KASSERT(maxloop > 0,
2752                     ("[pmc,%d] (ri%d, rc%ld) waiting too long for "
2753                         "pmc to be free", __LINE__,
2754                          PMC_TO_ROWINDEX(pm), (unsigned long)counter_u64_fetch(pm->pm_runcount)));
2755 #endif
2756                 pmc_force_context_switch();
2757         }
2758 }
2759
2760 /*
2761  * This function does the following things:
2762  *
2763  *  - detaches the PMC from hardware
2764  *  - unlinks all target threads that were attached to it
2765  *  - removes the PMC from its owner's list
2766  *  - destroys the PMC private mutex
2767  *
2768  * Once this function completes, the given pmc pointer can be freed by
2769  * calling pmc_destroy_pmc_descriptor().
2770  */
2771
2772 static void
2773 pmc_release_pmc_descriptor(struct pmc *pm)
2774 {
2775         enum pmc_mode mode;
2776         struct pmc_hw *phw;
2777         u_int adjri, ri, cpu;
2778         struct pmc_owner *po;
2779         struct pmc_binding pb;
2780         struct pmc_process *pp;
2781         struct pmc_classdep *pcd;
2782         struct pmc_target *ptgt, *tmp;
2783
2784         sx_assert(&pmc_sx, SX_XLOCKED);
2785
2786         KASSERT(pm, ("[pmc,%d] null pmc", __LINE__));
2787
2788         ri   = PMC_TO_ROWINDEX(pm);
2789         pcd  = pmc_ri_to_classdep(md, ri, &adjri);
2790         mode = PMC_TO_MODE(pm);
2791
2792         PMCDBG3(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri,
2793             mode);
2794
2795         /*
2796          * First, we take the PMC off hardware.
2797          */
2798         cpu = 0;
2799         if (PMC_IS_SYSTEM_MODE(mode)) {
2800
2801                 /*
2802                  * A system mode PMC runs on a specific CPU.  Switch
2803                  * to this CPU and turn hardware off.
2804                  */
2805                 pmc_save_cpu_binding(&pb);
2806
2807                 cpu = PMC_TO_CPU(pm);
2808
2809                 pmc_select_cpu(cpu);
2810
2811                 /* switch off non-stalled CPUs */
2812                 pm->pm_pcpu_state[cpu].pps_cpustate = 0;
2813                 if (pm->pm_state == PMC_STATE_RUNNING &&
2814                         pm->pm_pcpu_state[cpu].pps_stalled == 0) {
2815
2816                         phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
2817
2818                         KASSERT(phw->phw_pmc == pm,
2819                             ("[pmc, %d] pmc ptr ri(%d) hw(%p) pm(%p)",
2820                                 __LINE__, ri, phw->phw_pmc, pm));
2821                         PMCDBG2(PMC,REL,2, "stopping cpu=%d ri=%d", cpu, ri);
2822
2823                         critical_enter();
2824                         pcd->pcd_stop_pmc(cpu, adjri);
2825                         critical_exit();
2826                 }
2827
2828                 PMCDBG2(PMC,REL,2, "decfg cpu=%d ri=%d", cpu, ri);
2829
2830                 critical_enter();
2831                 pcd->pcd_config_pmc(cpu, adjri, NULL);
2832                 critical_exit();
2833
2834                 /* adjust the global and process count of SS mode PMCs */
2835                 if (mode == PMC_MODE_SS && pm->pm_state == PMC_STATE_RUNNING) {
2836                         po = pm->pm_owner;
2837                         po->po_sscount--;
2838                         if (po->po_sscount == 0) {
2839                                 atomic_subtract_rel_int(&pmc_ss_count, 1);
2840                                 CK_LIST_REMOVE(po, po_ssnext);
2841                                 epoch_wait_preempt(global_epoch_preempt);
2842                         }
2843                 }
2844
2845                 pm->pm_state = PMC_STATE_DELETED;
2846
2847                 pmc_restore_cpu_binding(&pb);
2848
2849                 /*
2850                  * We could have references to this PMC structure in
2851                  * the per-cpu sample queues.  Wait for the queue to
2852                  * drain.
2853                  */
2854                 pmc_wait_for_pmc_idle(pm);
2855
2856         } else if (PMC_IS_VIRTUAL_MODE(mode)) {
2857
2858                 /*
2859                  * A virtual PMC could be running on multiple CPUs at
2860                  * a given instant.
2861                  *
2862                  * By marking its state as DELETED, we ensure that
2863                  * this PMC is never further scheduled on hardware.
2864                  *
2865                  * Then we wait till all CPUs are done with this PMC.
2866                  */
2867                 pm->pm_state = PMC_STATE_DELETED;
2868
2869
2870                 /* Wait for the PMCs runcount to come to zero. */
2871                 pmc_wait_for_pmc_idle(pm);
2872
2873                 /*
2874                  * At this point the PMC is off all CPUs and cannot be
2875                  * freshly scheduled onto a CPU.  It is now safe to
2876                  * unlink all targets from this PMC.  If a
2877                  * process-record's refcount falls to zero, we remove
2878                  * it from the hash table.  The module-wide SX lock
2879                  * protects us from races.
2880                  */
2881                 LIST_FOREACH_SAFE(ptgt, &pm->pm_targets, pt_next, tmp) {
2882                         pp = ptgt->pt_process;
2883                         pmc_unlink_target_process(pm, pp); /* frees 'ptgt' */
2884
2885                         PMCDBG1(PMC,REL,3, "pp->refcnt=%d", pp->pp_refcnt);
2886
2887                         /*
2888                          * If the target process record shows that no
2889                          * PMCs are attached to it, reclaim its space.
2890                          */
2891
2892                         if (pp->pp_refcnt == 0) {
2893                                 pmc_remove_process_descriptor(pp);
2894                                 pmc_destroy_process_descriptor(pp);
2895                         }
2896                 }
2897
2898                 cpu = curthread->td_oncpu; /* setup cpu for pmd_release() */
2899
2900         }
2901
2902         /*
2903          * Release any MD resources
2904          */
2905         (void) pcd->pcd_release_pmc(cpu, adjri, pm);
2906
2907         /*
2908          * Update row disposition
2909          */
2910
2911         if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm)))
2912                 PMC_UNMARK_ROW_STANDALONE(ri);
2913         else
2914                 PMC_UNMARK_ROW_THREAD(ri);
2915
2916         /* unlink from the owner's list */
2917         if (pm->pm_owner) {
2918                 LIST_REMOVE(pm, pm_next);
2919                 pm->pm_owner = NULL;
2920         }
2921 }
2922
2923 /*
2924  * Register an owner and a pmc.
2925  */
2926
2927 static int
2928 pmc_register_owner(struct proc *p, struct pmc *pmc)
2929 {
2930         struct pmc_owner *po;
2931
2932         sx_assert(&pmc_sx, SX_XLOCKED);
2933
2934         if ((po = pmc_find_owner_descriptor(p)) == NULL)
2935                 if ((po = pmc_allocate_owner_descriptor(p)) == NULL)
2936                         return ENOMEM;
2937
2938         KASSERT(pmc->pm_owner == NULL,
2939             ("[pmc,%d] attempting to own an initialized PMC", __LINE__));
2940         pmc->pm_owner  = po;
2941
2942         LIST_INSERT_HEAD(&po->po_pmcs, pmc, pm_next);
2943
2944         PROC_LOCK(p);
2945         p->p_flag |= P_HWPMC;
2946         PROC_UNLOCK(p);
2947
2948         if (po->po_flags & PMC_PO_OWNS_LOGFILE)
2949                 pmclog_process_pmcallocate(pmc);
2950
2951         PMCDBG2(PMC,REG,1, "register-owner pmc-owner=%p pmc=%p",
2952             po, pmc);
2953
2954         return 0;
2955 }
2956
2957 /*
2958  * Return the current row disposition:
2959  * == 0 => FREE
2960  *  > 0 => PROCESS MODE
2961  *  < 0 => SYSTEM MODE
2962  */
2963
2964 int
2965 pmc_getrowdisp(int ri)
2966 {
2967         return pmc_pmcdisp[ri];
2968 }
2969
2970 /*
2971  * Check if a PMC at row index 'ri' can be allocated to the current
2972  * process.
2973  *
2974  * Allocation can fail if:
2975  *   - the current process is already being profiled by a PMC at index 'ri',
2976  *     attached to it via OP_PMCATTACH.
2977  *   - the current process has already allocated a PMC at index 'ri'
2978  *     via OP_ALLOCATE.
2979  */
2980
2981 static int
2982 pmc_can_allocate_rowindex(struct proc *p, unsigned int ri, int cpu)
2983 {
2984         enum pmc_mode mode;
2985         struct pmc *pm;
2986         struct pmc_owner *po;
2987         struct pmc_process *pp;
2988
2989         PMCDBG5(PMC,ALR,1, "can-allocate-rowindex proc=%p (%d, %s) ri=%d "
2990             "cpu=%d", p, p->p_pid, p->p_comm, ri, cpu);
2991
2992         /*
2993          * We shouldn't have already allocated a process-mode PMC at
2994          * row index 'ri'.
2995          *
2996          * We shouldn't have allocated a system-wide PMC on the same
2997          * CPU and same RI.
2998          */
2999         if ((po = pmc_find_owner_descriptor(p)) != NULL)
3000                 LIST_FOREACH(pm, &po->po_pmcs, pm_next) {
3001                     if (PMC_TO_ROWINDEX(pm) == ri) {
3002                             mode = PMC_TO_MODE(pm);
3003                             if (PMC_IS_VIRTUAL_MODE(mode))
3004                                     return EEXIST;
3005                             if (PMC_IS_SYSTEM_MODE(mode) &&
3006                                 (int) PMC_TO_CPU(pm) == cpu)
3007                                     return EEXIST;
3008                     }
3009                 }
3010
3011         /*
3012          * We also shouldn't be the target of any PMC at this index
3013          * since otherwise a PMC_ATTACH to ourselves will fail.
3014          */
3015         if ((pp = pmc_find_process_descriptor(p, 0)) != NULL)
3016                 if (pp->pp_pmcs[ri].pp_pmc)
3017                         return EEXIST;
3018
3019         PMCDBG4(PMC,ALR,2, "can-allocate-rowindex proc=%p (%d, %s) ri=%d ok",
3020             p, p->p_pid, p->p_comm, ri);
3021
3022         return 0;
3023 }
3024
3025 /*
3026  * Check if a given PMC at row index 'ri' can be currently used in
3027  * mode 'mode'.
3028  */
3029
3030 static int
3031 pmc_can_allocate_row(int ri, enum pmc_mode mode)
3032 {
3033         enum pmc_disp   disp;
3034
3035         sx_assert(&pmc_sx, SX_XLOCKED);
3036
3037         PMCDBG2(PMC,ALR,1, "can-allocate-row ri=%d mode=%d", ri, mode);
3038
3039         if (PMC_IS_SYSTEM_MODE(mode))
3040                 disp = PMC_DISP_STANDALONE;
3041         else
3042                 disp = PMC_DISP_THREAD;
3043
3044         /*
3045          * check disposition for PMC row 'ri':
3046          *
3047          * Expected disposition         Row-disposition         Result
3048          *
3049          * STANDALONE                   STANDALONE or FREE      proceed
3050          * STANDALONE                   THREAD                  fail
3051          * THREAD                       THREAD or FREE          proceed
3052          * THREAD                       STANDALONE              fail
3053          */
3054
3055         if (!PMC_ROW_DISP_IS_FREE(ri) &&
3056             !(disp == PMC_DISP_THREAD && PMC_ROW_DISP_IS_THREAD(ri)) &&
3057             !(disp == PMC_DISP_STANDALONE && PMC_ROW_DISP_IS_STANDALONE(ri)))
3058                 return EBUSY;
3059
3060         /*
3061          * All OK
3062          */
3063
3064         PMCDBG2(PMC,ALR,2, "can-allocate-row ri=%d mode=%d ok", ri, mode);
3065
3066         return 0;
3067
3068 }
3069
3070 /*
3071  * Find a PMC descriptor with user handle 'pmcid' for thread 'td'.
3072  */
3073
3074 static struct pmc *
3075 pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, pmc_id_t pmcid)
3076 {
3077         struct pmc *pm;
3078
3079         KASSERT(PMC_ID_TO_ROWINDEX(pmcid) < md->pmd_npmc,
3080             ("[pmc,%d] Illegal pmc index %d (max %d)", __LINE__,
3081                 PMC_ID_TO_ROWINDEX(pmcid), md->pmd_npmc));
3082
3083         LIST_FOREACH(pm, &po->po_pmcs, pm_next)
3084             if (pm->pm_id == pmcid)
3085                     return pm;
3086
3087         return NULL;
3088 }
3089
3090 static int
3091 pmc_find_pmc(pmc_id_t pmcid, struct pmc **pmc)
3092 {
3093
3094         struct pmc *pm, *opm;
3095         struct pmc_owner *po;
3096         struct pmc_process *pp;
3097
3098         PMCDBG1(PMC,FND,1, "find-pmc id=%d", pmcid);
3099         if (PMC_ID_TO_ROWINDEX(pmcid) >= md->pmd_npmc)
3100                 return (EINVAL);
3101
3102         if ((po = pmc_find_owner_descriptor(curthread->td_proc)) == NULL) {
3103                 /*
3104                  * In case of PMC_F_DESCENDANTS child processes we will not find
3105                  * the current process in the owners hash list.  Find the owner
3106                  * process first and from there lookup the po.
3107                  */
3108                 if ((pp = pmc_find_process_descriptor(curthread->td_proc,
3109                     PMC_FLAG_NONE)) == NULL) {
3110                         return ESRCH;
3111                 } else {
3112                         opm = pp->pp_pmcs[PMC_ID_TO_ROWINDEX(pmcid)].pp_pmc;
3113                         if (opm == NULL)
3114                                 return ESRCH;
3115                         if ((opm->pm_flags & (PMC_F_ATTACHED_TO_OWNER|
3116                             PMC_F_DESCENDANTS)) != (PMC_F_ATTACHED_TO_OWNER|
3117                             PMC_F_DESCENDANTS))
3118                                 return ESRCH;
3119                         po = opm->pm_owner;
3120                 }
3121         }
3122
3123         if ((pm = pmc_find_pmc_descriptor_in_process(po, pmcid)) == NULL)
3124                 return EINVAL;
3125
3126         PMCDBG2(PMC,FND,2, "find-pmc id=%d -> pmc=%p", pmcid, pm);
3127
3128         *pmc = pm;
3129         return 0;
3130 }
3131
3132 /*
3133  * Start a PMC.
3134  */
3135
3136 static int
3137 pmc_start(struct pmc *pm)
3138 {
3139         enum pmc_mode mode;
3140         struct pmc_owner *po;
3141         struct pmc_binding pb;
3142         struct pmc_classdep *pcd;
3143         int adjri, error, cpu, ri;
3144
3145         KASSERT(pm != NULL,
3146             ("[pmc,%d] null pm", __LINE__));
3147
3148         mode = PMC_TO_MODE(pm);
3149         ri   = PMC_TO_ROWINDEX(pm);
3150         pcd  = pmc_ri_to_classdep(md, ri, &adjri);
3151
3152         error = 0;
3153
3154         PMCDBG3(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri);
3155
3156         po = pm->pm_owner;
3157
3158         /*
3159          * Disallow PMCSTART if a logfile is required but has not been
3160          * configured yet.
3161          */
3162         if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) &&
3163             (po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
3164                 return (EDOOFUS);       /* programming error */
3165
3166         /*
3167          * If this is a sampling mode PMC, log mapping information for
3168          * the kernel modules that are currently loaded.
3169          */
3170         if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
3171             pmc_log_kernel_mappings(pm);
3172
3173         if (PMC_IS_VIRTUAL_MODE(mode)) {
3174
3175                 /*
3176                  * If a PMCATTACH has never been done on this PMC,
3177                  * attach it to its owner process.
3178                  */
3179
3180                 if (LIST_EMPTY(&pm->pm_targets))
3181                         error = (pm->pm_flags & PMC_F_ATTACH_DONE) ? ESRCH :
3182                             pmc_attach_process(po->po_owner, pm);
3183
3184                 /*
3185                  * If the PMC is attached to its owner, then force a context
3186                  * switch to ensure that the MD state gets set correctly.
3187                  */
3188
3189                 if (error == 0) {
3190                         pm->pm_state = PMC_STATE_RUNNING;
3191                         if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER)
3192                                 pmc_force_context_switch();
3193                 }
3194
3195                 return (error);
3196         }
3197
3198
3199         /*
3200          * A system-wide PMC.
3201          *
3202          * Add the owner to the global list if this is a system-wide
3203          * sampling PMC.
3204          */
3205
3206         if (mode == PMC_MODE_SS) {
3207                 /*
3208                  * Log mapping information for all existing processes in the
3209                  * system.  Subsequent mappings are logged as they happen;
3210                  * see pmc_process_mmap().
3211                  */
3212                 if (po->po_logprocmaps == 0) {
3213                         pmc_log_all_process_mappings(po);
3214                         po->po_logprocmaps = 1;
3215                 }
3216                 po->po_sscount++;
3217                 if (po->po_sscount == 1) {
3218                         atomic_add_rel_int(&pmc_ss_count, 1);
3219                         CK_LIST_INSERT_HEAD(&pmc_ss_owners, po, po_ssnext);
3220                         PMCDBG1(PMC,OPS,1, "po=%p in global list", po);
3221                 }
3222         }
3223
3224         /*
3225          * Move to the CPU associated with this
3226          * PMC, and start the hardware.
3227          */
3228
3229         pmc_save_cpu_binding(&pb);
3230
3231         cpu = PMC_TO_CPU(pm);
3232
3233         if (!pmc_cpu_is_active(cpu))
3234                 return (ENXIO);
3235
3236         pmc_select_cpu(cpu);
3237
3238         /*
3239          * global PMCs are configured at allocation time
3240          * so write out the initial value and start the PMC.
3241          */
3242
3243         pm->pm_state = PMC_STATE_RUNNING;
3244
3245         critical_enter();
3246         if ((error = pcd->pcd_write_pmc(cpu, adjri,
3247                  PMC_IS_SAMPLING_MODE(mode) ?
3248                  pm->pm_sc.pm_reloadcount :
3249                  pm->pm_sc.pm_initial)) == 0) {
3250                 /* If a sampling mode PMC, reset stalled state. */
3251                 if (PMC_IS_SAMPLING_MODE(mode))
3252                         pm->pm_pcpu_state[cpu].pps_stalled = 0;
3253
3254                 /* Indicate that we desire this to run. Start it. */
3255                 pm->pm_pcpu_state[cpu].pps_cpustate = 1;
3256                 error = pcd->pcd_start_pmc(cpu, adjri);
3257         }
3258         critical_exit();
3259
3260         pmc_restore_cpu_binding(&pb);
3261
3262         return (error);
3263 }
3264
3265 /*
3266  * Stop a PMC.
3267  */
3268
3269 static int
3270 pmc_stop(struct pmc *pm)
3271 {
3272         struct pmc_owner *po;
3273         struct pmc_binding pb;
3274         struct pmc_classdep *pcd;
3275         int adjri, cpu, error, ri;
3276
3277         KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__));
3278
3279         PMCDBG3(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm,
3280             PMC_TO_MODE(pm), PMC_TO_ROWINDEX(pm));
3281
3282         pm->pm_state = PMC_STATE_STOPPED;
3283
3284         /*
3285          * If the PMC is a virtual mode one, changing the state to
3286          * non-RUNNING is enough to ensure that the PMC never gets
3287          * scheduled.
3288          *
3289          * If this PMC is current running on a CPU, then it will
3290          * handled correctly at the time its target process is context
3291          * switched out.
3292          */
3293
3294         if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
3295                 return 0;
3296
3297         /*
3298          * A system-mode PMC.  Move to the CPU associated with
3299          * this PMC, and stop the hardware.  We update the
3300          * 'initial count' so that a subsequent PMCSTART will
3301          * resume counting from the current hardware count.
3302          */
3303
3304         pmc_save_cpu_binding(&pb);
3305
3306         cpu = PMC_TO_CPU(pm);
3307
3308         KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
3309             ("[pmc,%d] illegal cpu=%d", __LINE__, cpu));
3310
3311         if (!pmc_cpu_is_active(cpu))
3312                 return ENXIO;
3313
3314         pmc_select_cpu(cpu);
3315
3316         ri = PMC_TO_ROWINDEX(pm);
3317         pcd = pmc_ri_to_classdep(md, ri, &adjri);
3318
3319         pm->pm_pcpu_state[cpu].pps_cpustate = 0;
3320         critical_enter();
3321         if ((error = pcd->pcd_stop_pmc(cpu, adjri)) == 0)
3322                 error = pcd->pcd_read_pmc(cpu, adjri, &pm->pm_sc.pm_initial);
3323         critical_exit();
3324
3325         pmc_restore_cpu_binding(&pb);
3326
3327         po = pm->pm_owner;
3328
3329         /* remove this owner from the global list of SS PMC owners */
3330         if (PMC_TO_MODE(pm) == PMC_MODE_SS) {
3331                 po->po_sscount--;
3332                 if (po->po_sscount == 0) {
3333                         atomic_subtract_rel_int(&pmc_ss_count, 1);
3334                         CK_LIST_REMOVE(po, po_ssnext);
3335                         epoch_wait_preempt(global_epoch_preempt);
3336                         PMCDBG1(PMC,OPS,2,"po=%p removed from global list", po);
3337                 }
3338         }
3339
3340         return (error);
3341 }
3342
3343 static struct pmc_classdep *
3344 pmc_class_to_classdep(enum pmc_class class)
3345 {
3346         int n;
3347
3348         for (n = 0; n < md->pmd_nclass; n++)
3349                 if (md->pmd_classdep[n].pcd_class == class)
3350                         return (&md->pmd_classdep[n]);
3351         return (NULL);
3352 }
3353
3354 #if defined(HWPMC_DEBUG) && defined(KTR)
3355 static const char *pmc_op_to_name[] = {
3356 #undef  __PMC_OP
3357 #define __PMC_OP(N, D)  #N ,
3358         __PMC_OPS()
3359         NULL
3360 };
3361 #endif
3362
3363 /*
3364  * The syscall interface
3365  */
3366
3367 #define PMC_GET_SX_XLOCK(...) do {              \
3368         sx_xlock(&pmc_sx);                      \
3369         if (pmc_hook == NULL) {                 \
3370                 sx_xunlock(&pmc_sx);            \
3371                 return __VA_ARGS__;             \
3372         }                                       \
3373 } while (0)
3374
3375 #define PMC_DOWNGRADE_SX() do {                 \
3376         sx_downgrade(&pmc_sx);                  \
3377         is_sx_downgraded = 1;                   \
3378 } while (0)
3379
3380 static int
3381 pmc_syscall_handler(struct thread *td, void *syscall_args)
3382 {
3383         int error, is_sx_downgraded, op;
3384         struct pmc_syscall_args *c;
3385         void *pmclog_proc_handle;
3386         void *arg;
3387
3388         c = (struct pmc_syscall_args *)syscall_args;
3389         op = c->pmop_code;
3390         arg = c->pmop_data;
3391         /* PMC isn't set up yet */
3392         if (pmc_hook == NULL)
3393                 return (EINVAL);
3394         if (op == PMC_OP_CONFIGURELOG) {
3395                 /*
3396                  * We cannot create the logging process inside
3397                  * pmclog_configure_log() because there is a LOR
3398                  * between pmc_sx and process structure locks.
3399                  * Instead, pre-create the process and ignite the loop
3400                  * if everything is fine, otherwise direct the process
3401                  * to exit.
3402                  */
3403                 error = pmclog_proc_create(td, &pmclog_proc_handle);
3404                 if (error != 0)
3405                         goto done_syscall;
3406         }
3407
3408         PMC_GET_SX_XLOCK(ENOSYS);
3409         is_sx_downgraded = 0;
3410         PMCDBG3(MOD,PMS,1, "syscall op=%d \"%s\" arg=%p", op,
3411             pmc_op_to_name[op], arg);
3412
3413         error = 0;
3414         counter_u64_add(pmc_stats.pm_syscalls, 1);
3415
3416         switch (op) {
3417
3418
3419         /*
3420          * Configure a log file.
3421          *
3422          * XXX This OP will be reworked.
3423          */
3424
3425         case PMC_OP_CONFIGURELOG:
3426         {
3427                 struct proc *p;
3428                 struct pmc *pm;
3429                 struct pmc_owner *po;
3430                 struct pmc_op_configurelog cl;
3431
3432                 if ((error = copyin(arg, &cl, sizeof(cl))) != 0) {
3433                         pmclog_proc_ignite(pmclog_proc_handle, NULL);
3434                         break;
3435                 }
3436
3437                 /* mark this process as owning a log file */
3438                 p = td->td_proc;
3439                 if ((po = pmc_find_owner_descriptor(p)) == NULL)
3440                         if ((po = pmc_allocate_owner_descriptor(p)) == NULL) {
3441                                 pmclog_proc_ignite(pmclog_proc_handle, NULL);
3442                                 error = ENOMEM;
3443                                 break;
3444                         }
3445
3446                 /*
3447                  * If a valid fd was passed in, try to configure that,
3448                  * otherwise if 'fd' was less than zero and there was
3449                  * a log file configured, flush its buffers and
3450                  * de-configure it.
3451                  */
3452                 if (cl.pm_logfd >= 0) {
3453                         error = pmclog_configure_log(md, po, cl.pm_logfd);
3454                         pmclog_proc_ignite(pmclog_proc_handle, error == 0 ?
3455                             po : NULL);
3456                 } else if (po->po_flags & PMC_PO_OWNS_LOGFILE) {
3457                         pmclog_proc_ignite(pmclog_proc_handle, NULL);
3458                         error = pmclog_close(po);
3459                         if (error == 0) {
3460                                 LIST_FOREACH(pm, &po->po_pmcs, pm_next)
3461                                     if (pm->pm_flags & PMC_F_NEEDS_LOGFILE &&
3462                                         pm->pm_state == PMC_STATE_RUNNING)
3463                                             pmc_stop(pm);
3464                                 error = pmclog_deconfigure_log(po);
3465                         }
3466                 } else {
3467                         pmclog_proc_ignite(pmclog_proc_handle, NULL);
3468                         error = EINVAL;
3469                 }
3470         }
3471         break;
3472
3473         /*
3474          * Flush a log file.
3475          */
3476
3477         case PMC_OP_FLUSHLOG:
3478         {
3479                 struct pmc_owner *po;
3480
3481                 sx_assert(&pmc_sx, SX_XLOCKED);
3482
3483                 if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
3484                         error = EINVAL;
3485                         break;
3486                 }
3487
3488                 error = pmclog_flush(po, 0);
3489         }
3490         break;
3491
3492         /*
3493          * Close a log file.
3494          */
3495
3496         case PMC_OP_CLOSELOG:
3497         {
3498                 struct pmc_owner *po;
3499
3500                 sx_assert(&pmc_sx, SX_XLOCKED);
3501
3502                 if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
3503                         error = EINVAL;
3504                         break;
3505                 }
3506
3507                 error = pmclog_close(po);
3508         }
3509         break;
3510
3511         /*
3512          * Retrieve hardware configuration.
3513          */
3514
3515         case PMC_OP_GETCPUINFO: /* CPU information */
3516         {
3517                 struct pmc_op_getcpuinfo gci;
3518                 struct pmc_classinfo *pci;
3519                 struct pmc_classdep *pcd;
3520                 int cl;
3521
3522                 gci.pm_cputype = md->pmd_cputype;
3523                 gci.pm_ncpu    = pmc_cpu_max();
3524                 gci.pm_npmc    = md->pmd_npmc;
3525                 gci.pm_nclass  = md->pmd_nclass;
3526                 pci = gci.pm_classes;
3527                 pcd = md->pmd_classdep;
3528                 for (cl = 0; cl < md->pmd_nclass; cl++, pci++, pcd++) {
3529                         pci->pm_caps  = pcd->pcd_caps;
3530                         pci->pm_class = pcd->pcd_class;
3531                         pci->pm_width = pcd->pcd_width;
3532                         pci->pm_num   = pcd->pcd_num;
3533                 }
3534                 error = copyout(&gci, arg, sizeof(gci));
3535         }
3536         break;
3537
3538         /*
3539          * Retrieve soft events list.
3540          */
3541         case PMC_OP_GETDYNEVENTINFO:
3542         {
3543                 enum pmc_class                  cl;
3544                 enum pmc_event                  ev;
3545                 struct pmc_op_getdyneventinfo   *gei;
3546                 struct pmc_dyn_event_descr      dev;
3547                 struct pmc_soft                 *ps;
3548                 uint32_t                        nevent;
3549
3550                 sx_assert(&pmc_sx, SX_LOCKED);
3551
3552                 gei = (struct pmc_op_getdyneventinfo *) arg;
3553
3554                 if ((error = copyin(&gei->pm_class, &cl, sizeof(cl))) != 0)
3555                         break;
3556
3557                 /* Only SOFT class is dynamic. */
3558                 if (cl != PMC_CLASS_SOFT) {
3559                         error = EINVAL;
3560                         break;
3561                 }
3562
3563                 nevent = 0;
3564                 for (ev = PMC_EV_SOFT_FIRST; (int)ev <= PMC_EV_SOFT_LAST; ev++) {
3565                         ps = pmc_soft_ev_acquire(ev);
3566                         if (ps == NULL)
3567                                 continue;
3568                         bcopy(&ps->ps_ev, &dev, sizeof(dev));
3569                         pmc_soft_ev_release(ps);
3570
3571                         error = copyout(&dev,
3572                             &gei->pm_events[nevent],
3573                             sizeof(struct pmc_dyn_event_descr));
3574                         if (error != 0)
3575                                 break;
3576                         nevent++;
3577                 }
3578                 if (error != 0)
3579                         break;
3580
3581                 error = copyout(&nevent, &gei->pm_nevent,
3582                     sizeof(nevent));
3583         }
3584         break;
3585
3586         /*
3587          * Get module statistics
3588          */
3589
3590         case PMC_OP_GETDRIVERSTATS:
3591         {
3592                 struct pmc_op_getdriverstats gms;
3593 #define CFETCH(a, b, field) a.field = counter_u64_fetch(b.field)
3594                 CFETCH(gms, pmc_stats, pm_intr_ignored);
3595                 CFETCH(gms, pmc_stats, pm_intr_processed);
3596                 CFETCH(gms, pmc_stats, pm_intr_bufferfull);
3597                 CFETCH(gms, pmc_stats, pm_syscalls);
3598                 CFETCH(gms, pmc_stats, pm_syscall_errors);
3599                 CFETCH(gms, pmc_stats, pm_buffer_requests);
3600                 CFETCH(gms, pmc_stats, pm_buffer_requests_failed);
3601                 CFETCH(gms, pmc_stats, pm_log_sweeps);
3602 #undef CFETCH
3603                 error = copyout(&gms, arg, sizeof(gms));
3604         }
3605         break;
3606
3607
3608         /*
3609          * Retrieve module version number
3610          */
3611
3612         case PMC_OP_GETMODULEVERSION:
3613         {
3614                 uint32_t cv, modv;
3615
3616                 /* retrieve the client's idea of the ABI version */
3617                 if ((error = copyin(arg, &cv, sizeof(uint32_t))) != 0)
3618                         break;
3619                 /* don't service clients newer than our driver */
3620                 modv = PMC_VERSION;
3621                 if ((cv & 0xFFFF0000) > (modv & 0xFFFF0000)) {
3622                         error = EPROGMISMATCH;
3623                         break;
3624                 }
3625                 error = copyout(&modv, arg, sizeof(int));
3626         }
3627         break;
3628
3629
3630         /*
3631          * Retrieve the state of all the PMCs on a given
3632          * CPU.
3633          */
3634
3635         case PMC_OP_GETPMCINFO:
3636         {
3637                 int ari;
3638                 struct pmc *pm;
3639                 size_t pmcinfo_size;
3640                 uint32_t cpu, n, npmc;
3641                 struct pmc_owner *po;
3642                 struct pmc_binding pb;
3643                 struct pmc_classdep *pcd;
3644                 struct pmc_info *p, *pmcinfo;
3645                 struct pmc_op_getpmcinfo *gpi;
3646
3647                 PMC_DOWNGRADE_SX();
3648
3649                 gpi = (struct pmc_op_getpmcinfo *) arg;
3650
3651                 if ((error = copyin(&gpi->pm_cpu, &cpu, sizeof(cpu))) != 0)
3652                         break;
3653
3654                 if (cpu >= pmc_cpu_max()) {
3655                         error = EINVAL;
3656                         break;
3657                 }
3658
3659                 if (!pmc_cpu_is_active(cpu)) {
3660                         error = ENXIO;
3661                         break;
3662                 }
3663
3664                 /* switch to CPU 'cpu' */
3665                 pmc_save_cpu_binding(&pb);
3666                 pmc_select_cpu(cpu);
3667
3668                 npmc = md->pmd_npmc;
3669
3670                 pmcinfo_size = npmc * sizeof(struct pmc_info);
3671                 pmcinfo = malloc(pmcinfo_size, M_PMC, M_WAITOK);
3672
3673                 p = pmcinfo;
3674
3675                 for (n = 0; n < md->pmd_npmc; n++, p++) {
3676
3677                         pcd = pmc_ri_to_classdep(md, n, &ari);
3678
3679                         KASSERT(pcd != NULL,
3680                             ("[pmc,%d] null pcd ri=%d", __LINE__, n));
3681
3682                         if ((error = pcd->pcd_describe(cpu, ari, p, &pm)) != 0)
3683                                 break;
3684
3685                         if (PMC_ROW_DISP_IS_STANDALONE(n))
3686                                 p->pm_rowdisp = PMC_DISP_STANDALONE;
3687                         else if (PMC_ROW_DISP_IS_THREAD(n))
3688                                 p->pm_rowdisp = PMC_DISP_THREAD;
3689                         else
3690                                 p->pm_rowdisp = PMC_DISP_FREE;
3691
3692                         p->pm_ownerpid = -1;
3693
3694                         if (pm == NULL) /* no PMC associated */
3695                                 continue;
3696
3697                         po = pm->pm_owner;
3698
3699                         KASSERT(po->po_owner != NULL,
3700                             ("[pmc,%d] pmc_owner had a null proc pointer",
3701                                 __LINE__));
3702
3703                         p->pm_ownerpid = po->po_owner->p_pid;
3704                         p->pm_mode     = PMC_TO_MODE(pm);
3705                         p->pm_event    = pm->pm_event;
3706                         p->pm_flags    = pm->pm_flags;
3707
3708                         if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
3709                                 p->pm_reloadcount =
3710                                     pm->pm_sc.pm_reloadcount;
3711                 }
3712
3713                 pmc_restore_cpu_binding(&pb);
3714
3715                 /* now copy out the PMC info collected */
3716                 if (error == 0)
3717                         error = copyout(pmcinfo, &gpi->pm_pmcs, pmcinfo_size);
3718
3719                 free(pmcinfo, M_PMC);
3720         }
3721         break;
3722
3723
3724         /*
3725          * Set the administrative state of a PMC.  I.e. whether
3726          * the PMC is to be used or not.
3727          */
3728
3729         case PMC_OP_PMCADMIN:
3730         {
3731                 int cpu, ri;
3732                 enum pmc_state request;
3733                 struct pmc_cpu *pc;
3734                 struct pmc_hw *phw;
3735                 struct pmc_op_pmcadmin pma;
3736                 struct pmc_binding pb;
3737
3738                 sx_assert(&pmc_sx, SX_XLOCKED);
3739
3740                 KASSERT(td == curthread,
3741                     ("[pmc,%d] td != curthread", __LINE__));
3742
3743                 error = priv_check(td, PRIV_PMC_MANAGE);
3744                 if (error)
3745                         break;
3746
3747                 if ((error = copyin(arg, &pma, sizeof(pma))) != 0)
3748                         break;
3749
3750                 cpu = pma.pm_cpu;
3751
3752                 if (cpu < 0 || cpu >= (int) pmc_cpu_max()) {
3753                         error = EINVAL;
3754                         break;
3755                 }
3756
3757                 if (!pmc_cpu_is_active(cpu)) {
3758                         error = ENXIO;
3759                         break;
3760                 }
3761
3762                 request = pma.pm_state;
3763
3764                 if (request != PMC_STATE_DISABLED &&
3765                     request != PMC_STATE_FREE) {
3766                         error = EINVAL;
3767                         break;
3768                 }
3769
3770                 ri = pma.pm_pmc; /* pmc id == row index */
3771                 if (ri < 0 || ri >= (int) md->pmd_npmc) {
3772                         error = EINVAL;
3773                         break;
3774                 }
3775
3776                 /*
3777                  * We can't disable a PMC with a row-index allocated
3778                  * for process virtual PMCs.
3779                  */
3780
3781                 if (PMC_ROW_DISP_IS_THREAD(ri) &&
3782                     request == PMC_STATE_DISABLED) {
3783                         error = EBUSY;
3784                         break;
3785                 }
3786
3787                 /*
3788                  * otherwise, this PMC on this CPU is either free or
3789                  * in system-wide mode.
3790                  */
3791
3792                 pmc_save_cpu_binding(&pb);
3793                 pmc_select_cpu(cpu);
3794
3795                 pc  = pmc_pcpu[cpu];
3796                 phw = pc->pc_hwpmcs[ri];
3797
3798                 /*
3799                  * XXX do we need some kind of 'forced' disable?
3800                  */
3801
3802                 if (phw->phw_pmc == NULL) {
3803                         if (request == PMC_STATE_DISABLED &&
3804                             (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED)) {
3805                                 phw->phw_state &= ~PMC_PHW_FLAG_IS_ENABLED;
3806                                 PMC_MARK_ROW_STANDALONE(ri);
3807                         } else if (request == PMC_STATE_FREE &&
3808                             (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0) {
3809                                 phw->phw_state |=  PMC_PHW_FLAG_IS_ENABLED;
3810                                 PMC_UNMARK_ROW_STANDALONE(ri);
3811                         }
3812                         /* other cases are a no-op */
3813                 } else
3814                         error = EBUSY;
3815
3816                 pmc_restore_cpu_binding(&pb);
3817         }
3818         break;
3819
3820
3821         /*
3822          * Allocate a PMC.
3823          */
3824
3825         case PMC_OP_PMCALLOCATE:
3826         {
3827                 int adjri, n;
3828                 u_int cpu;
3829                 uint32_t caps;
3830                 struct pmc *pmc;
3831                 enum pmc_mode mode;
3832                 struct pmc_hw *phw;
3833                 struct pmc_binding pb;
3834                 struct pmc_classdep *pcd;
3835                 struct pmc_op_pmcallocate pa;
3836
3837                 if ((error = copyin(arg, &pa, sizeof(pa))) != 0)
3838                         break;
3839
3840                 caps = pa.pm_caps;
3841                 mode = pa.pm_mode;
3842                 cpu  = pa.pm_cpu;
3843
3844                 if ((mode != PMC_MODE_SS  &&  mode != PMC_MODE_SC  &&
3845                      mode != PMC_MODE_TS  &&  mode != PMC_MODE_TC) ||
3846                     (cpu != (u_int) PMC_CPU_ANY && cpu >= pmc_cpu_max())) {
3847                         error = EINVAL;
3848                         break;
3849                 }
3850
3851                 /*
3852                  * Virtual PMCs should only ask for a default CPU.
3853                  * System mode PMCs need to specify a non-default CPU.
3854                  */
3855
3856                 if ((PMC_IS_VIRTUAL_MODE(mode) && cpu != (u_int) PMC_CPU_ANY) ||
3857                     (PMC_IS_SYSTEM_MODE(mode) && cpu == (u_int) PMC_CPU_ANY)) {
3858                         error = EINVAL;
3859                         break;
3860                 }
3861
3862                 /*
3863                  * Check that an inactive CPU is not being asked for.
3864                  */
3865
3866                 if (PMC_IS_SYSTEM_MODE(mode) && !pmc_cpu_is_active(cpu)) {
3867                         error = ENXIO;
3868                         break;
3869                 }
3870
3871                 /*
3872                  * Refuse an allocation for a system-wide PMC if this
3873                  * process has been jailed, or if this process lacks
3874                  * super-user credentials and the sysctl tunable
3875                  * 'security.bsd.unprivileged_syspmcs' is zero.
3876                  */
3877
3878                 if (PMC_IS_SYSTEM_MODE(mode)) {
3879                         if (jailed(curthread->td_ucred)) {
3880                                 error = EPERM;
3881                                 break;
3882                         }
3883                         if (!pmc_unprivileged_syspmcs) {
3884                                 error = priv_check(curthread,
3885                                     PRIV_PMC_SYSTEM);
3886                                 if (error)
3887                                         break;
3888                         }
3889                 }
3890
3891                 /*
3892                  * Look for valid values for 'pm_flags'
3893                  */
3894
3895                 if ((pa.pm_flags & ~(PMC_F_DESCENDANTS | PMC_F_LOG_PROCCSW |
3896                     PMC_F_LOG_PROCEXIT | PMC_F_CALLCHAIN |
3897                     PMC_F_USERCALLCHAIN)) != 0) {
3898                         error = EINVAL;
3899                         break;
3900                 }
3901
3902                 /* PMC_F_USERCALLCHAIN is only valid with PMC_F_CALLCHAIN */
3903                 if ((pa.pm_flags & (PMC_F_CALLCHAIN | PMC_F_USERCALLCHAIN)) ==
3904                     PMC_F_USERCALLCHAIN) {
3905                         error = EINVAL;
3906                         break;
3907                 }
3908
3909                 /* PMC_F_USERCALLCHAIN is only valid for sampling mode */
3910                 if (pa.pm_flags & PMC_F_USERCALLCHAIN &&
3911                         mode != PMC_MODE_TS && mode != PMC_MODE_SS) {
3912                         error = EINVAL;
3913                         break;
3914                 }
3915
3916                 /* process logging options are not allowed for system PMCs */
3917                 if (PMC_IS_SYSTEM_MODE(mode) && (pa.pm_flags &
3918                     (PMC_F_LOG_PROCCSW | PMC_F_LOG_PROCEXIT))) {
3919                         error = EINVAL;
3920                         break;
3921                 }
3922
3923                 /*
3924                  * All sampling mode PMCs need to be able to interrupt the
3925                  * CPU.
3926                  */
3927                 if (PMC_IS_SAMPLING_MODE(mode))
3928                         caps |= PMC_CAP_INTERRUPT;
3929
3930                 /* A valid class specifier should have been passed in. */
3931                 pcd = pmc_class_to_classdep(pa.pm_class);
3932                 if (pcd == NULL) {
3933                         error = EINVAL;
3934                         break;
3935                 }
3936
3937                 /* The requested PMC capabilities should be feasible. */
3938                 if ((pcd->pcd_caps & caps) != caps) {
3939                         error = EOPNOTSUPP;
3940                         break;
3941                 }
3942
3943                 PMCDBG4(PMC,ALL,2, "event=%d caps=0x%x mode=%d cpu=%d",
3944                     pa.pm_ev, caps, mode, cpu);
3945
3946                 pmc = pmc_allocate_pmc_descriptor();
3947                 pmc->pm_id    = PMC_ID_MAKE_ID(cpu,pa.pm_mode,pa.pm_class,
3948                     PMC_ID_INVALID);
3949                 pmc->pm_event = pa.pm_ev;
3950                 pmc->pm_state = PMC_STATE_FREE;
3951                 pmc->pm_caps  = caps;
3952                 pmc->pm_flags = pa.pm_flags;
3953
3954                 /* XXX set lower bound on sampling for process counters */
3955                 if (PMC_IS_SAMPLING_MODE(mode)) {
3956                         /*
3957                          * Don't permit requested sample rate to be less than 1000
3958                          */
3959                         if (pa.pm_count < 1000)
3960                                 log(LOG_WARNING,
3961                                         "pmcallocate: passed sample rate %ju - setting to 1000\n",
3962                                         (uintmax_t)pa.pm_count);
3963                         pmc->pm_sc.pm_reloadcount = MAX(1000, pa.pm_count);
3964                 } else
3965                         pmc->pm_sc.pm_initial = pa.pm_count;
3966
3967                 /* switch thread to CPU 'cpu' */
3968                 pmc_save_cpu_binding(&pb);
3969
3970 #define PMC_IS_SHAREABLE_PMC(cpu, n)                            \
3971         (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_state &           \
3972          PMC_PHW_FLAG_IS_SHAREABLE)
3973 #define PMC_IS_UNALLOCATED(cpu, n)                              \
3974         (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_pmc == NULL)
3975
3976                 if (PMC_IS_SYSTEM_MODE(mode)) {
3977                         pmc_select_cpu(cpu);
3978                         for (n = pcd->pcd_ri; n < (int) md->pmd_npmc; n++) {
3979                                 pcd = pmc_ri_to_classdep(md, n, &adjri);
3980                                 if (pmc_can_allocate_row(n, mode) == 0 &&
3981                                     pmc_can_allocate_rowindex(
3982                                             curthread->td_proc, n, cpu) == 0 &&
3983                                     (PMC_IS_UNALLOCATED(cpu, n) ||
3984                                      PMC_IS_SHAREABLE_PMC(cpu, n)) &&
3985                                     pcd->pcd_allocate_pmc(cpu, adjri, pmc,
3986                                         &pa) == 0)
3987                                         break;
3988                         }
3989                 } else {
3990                         /* Process virtual mode */
3991                         for (n = pcd->pcd_ri; n < (int) md->pmd_npmc; n++) {
3992                                 pcd = pmc_ri_to_classdep(md, n, &adjri);
3993                                 if (pmc_can_allocate_row(n, mode) == 0 &&
3994                                     pmc_can_allocate_rowindex(
3995                                             curthread->td_proc, n,
3996                                             PMC_CPU_ANY) == 0 &&
3997                                     pcd->pcd_allocate_pmc(curthread->td_oncpu,
3998                                         adjri, pmc, &pa) == 0)
3999                                         break;
4000                         }
4001                 }
4002
4003 #undef  PMC_IS_UNALLOCATED
4004 #undef  PMC_IS_SHAREABLE_PMC
4005
4006                 pmc_restore_cpu_binding(&pb);
4007
4008                 if (n == (int) md->pmd_npmc) {
4009                         pmc_destroy_pmc_descriptor(pmc);
4010                         pmc = NULL;
4011                         error = EINVAL;
4012                         break;
4013                 }
4014
4015                 /* Fill in the correct value in the ID field */
4016                 pmc->pm_id = PMC_ID_MAKE_ID(cpu,mode,pa.pm_class,n);
4017
4018                 PMCDBG5(PMC,ALL,2, "ev=%d class=%d mode=%d n=%d -> pmcid=%x",
4019                     pmc->pm_event, pa.pm_class, mode, n, pmc->pm_id);
4020
4021                 /* Process mode PMCs with logging enabled need log files */
4022                 if (pmc->pm_flags & (PMC_F_LOG_PROCEXIT | PMC_F_LOG_PROCCSW))
4023                         pmc->pm_flags |= PMC_F_NEEDS_LOGFILE;
4024
4025                 /* All system mode sampling PMCs require a log file */
4026                 if (PMC_IS_SAMPLING_MODE(mode) && PMC_IS_SYSTEM_MODE(mode))
4027                         pmc->pm_flags |= PMC_F_NEEDS_LOGFILE;
4028
4029                 /*
4030                  * Configure global pmc's immediately
4031                  */
4032
4033                 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pmc))) {
4034
4035                         pmc_save_cpu_binding(&pb);
4036                         pmc_select_cpu(cpu);
4037
4038                         phw = pmc_pcpu[cpu]->pc_hwpmcs[n];
4039                         pcd = pmc_ri_to_classdep(md, n, &adjri);
4040
4041                         if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0 ||
4042                             (error = pcd->pcd_config_pmc(cpu, adjri, pmc)) != 0) {
4043                                 (void) pcd->pcd_release_pmc(cpu, adjri, pmc);
4044                                 pmc_destroy_pmc_descriptor(pmc);
4045                                 pmc = NULL;
4046                                 pmc_restore_cpu_binding(&pb);
4047                                 error = EPERM;
4048                                 break;
4049                         }
4050
4051                         pmc_restore_cpu_binding(&pb);
4052                 }
4053
4054                 pmc->pm_state    = PMC_STATE_ALLOCATED;
4055                 pmc->pm_class   = pa.pm_class;
4056
4057                 /*
4058                  * mark row disposition
4059                  */
4060
4061                 if (PMC_IS_SYSTEM_MODE(mode))
4062                         PMC_MARK_ROW_STANDALONE(n);
4063                 else
4064                         PMC_MARK_ROW_THREAD(n);
4065
4066                 /*
4067                  * Register this PMC with the current thread as its owner.
4068                  */
4069
4070                 if ((error =
4071                     pmc_register_owner(curthread->td_proc, pmc)) != 0) {
4072                         pmc_release_pmc_descriptor(pmc);
4073                         pmc_destroy_pmc_descriptor(pmc);
4074                         pmc = NULL;
4075                         break;
4076                 }
4077
4078
4079                 /*
4080                  * Return the allocated index.
4081                  */
4082
4083                 pa.pm_pmcid = pmc->pm_id;
4084
4085                 error = copyout(&pa, arg, sizeof(pa));
4086         }
4087         break;
4088
4089
4090         /*
4091          * Attach a PMC to a process.
4092          */
4093
4094         case PMC_OP_PMCATTACH:
4095         {
4096                 struct pmc *pm;
4097                 struct proc *p;
4098                 struct pmc_op_pmcattach a;
4099
4100                 sx_assert(&pmc_sx, SX_XLOCKED);
4101
4102                 if ((error = copyin(arg, &a, sizeof(a))) != 0)
4103                         break;
4104
4105                 if (a.pm_pid < 0) {
4106                         error = EINVAL;
4107                         break;
4108                 } else if (a.pm_pid == 0)
4109                         a.pm_pid = td->td_proc->p_pid;
4110
4111                 if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0)
4112                         break;
4113
4114                 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
4115                         error = EINVAL;
4116                         break;
4117                 }
4118
4119                 /* PMCs may be (re)attached only when allocated or stopped */
4120                 if (pm->pm_state == PMC_STATE_RUNNING) {
4121                         error = EBUSY;
4122                         break;
4123                 } else if (pm->pm_state != PMC_STATE_ALLOCATED &&
4124                     pm->pm_state != PMC_STATE_STOPPED) {
4125                         error = EINVAL;
4126                         break;
4127                 }
4128
4129                 /* lookup pid */
4130                 if ((p = pfind(a.pm_pid)) == NULL) {
4131                         error = ESRCH;
4132                         break;
4133                 }
4134
4135                 /*
4136                  * Ignore processes that are working on exiting.
4137                  */
4138                 if (p->p_flag & P_WEXIT) {
4139                         error = ESRCH;
4140                         PROC_UNLOCK(p); /* pfind() returns a locked process */
4141                         break;
4142                 }
4143
4144                 /*
4145                  * we are allowed to attach a PMC to a process if
4146                  * we can debug it.
4147                  */
4148                 error = p_candebug(curthread, p);
4149
4150                 PROC_UNLOCK(p);
4151
4152                 if (error == 0)
4153                         error = pmc_attach_process(p, pm);
4154         }
4155         break;
4156
4157
4158         /*
4159          * Detach an attached PMC from a process.
4160          */
4161
4162         case PMC_OP_PMCDETACH:
4163         {
4164                 struct pmc *pm;
4165                 struct proc *p;
4166                 struct pmc_op_pmcattach a;
4167
4168                 if ((error = copyin(arg, &a, sizeof(a))) != 0)
4169                         break;
4170
4171                 if (a.pm_pid < 0) {
4172                         error = EINVAL;
4173                         break;
4174                 } else if (a.pm_pid == 0)
4175                         a.pm_pid = td->td_proc->p_pid;
4176
4177                 if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0)
4178                         break;
4179
4180                 if ((p = pfind(a.pm_pid)) == NULL) {
4181                         error = ESRCH;
4182                         break;
4183                 }
4184
4185                 /*
4186                  * Treat processes that are in the process of exiting
4187                  * as if they were not present.
4188                  */
4189
4190                 if (p->p_flag & P_WEXIT)
4191                         error = ESRCH;
4192
4193                 PROC_UNLOCK(p); /* pfind() returns a locked process */
4194
4195                 if (error == 0)
4196                         error = pmc_detach_process(p, pm);
4197         }
4198         break;
4199
4200
4201         /*
4202          * Retrieve the MSR number associated with the counter
4203          * 'pmc_id'.  This allows processes to directly use RDPMC
4204          * instructions to read their PMCs, without the overhead of a
4205          * system call.
4206          */
4207
4208         case PMC_OP_PMCGETMSR:
4209         {
4210                 int adjri, ri;
4211                 struct pmc *pm;
4212                 struct pmc_target *pt;
4213                 struct pmc_op_getmsr gm;
4214                 struct pmc_classdep *pcd;
4215
4216                 PMC_DOWNGRADE_SX();
4217
4218                 if ((error = copyin(arg, &gm, sizeof(gm))) != 0)
4219                         break;
4220
4221                 if ((error = pmc_find_pmc(gm.pm_pmcid, &pm)) != 0)
4222                         break;
4223
4224                 /*
4225                  * The allocated PMC has to be a process virtual PMC,
4226                  * i.e., of type MODE_T[CS].  Global PMCs can only be
4227                  * read using the PMCREAD operation since they may be
4228                  * allocated on a different CPU than the one we could
4229                  * be running on at the time of the RDPMC instruction.
4230                  *
4231                  * The GETMSR operation is not allowed for PMCs that
4232                  * are inherited across processes.
4233                  */
4234
4235                 if (!PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) ||
4236                     (pm->pm_flags & PMC_F_DESCENDANTS)) {
4237                         error = EINVAL;
4238                         break;
4239                 }
4240
4241                 /*
4242                  * It only makes sense to use a RDPMC (or its
4243                  * equivalent instruction on non-x86 architectures) on
4244                  * a process that has allocated and attached a PMC to
4245                  * itself.  Conversely the PMC is only allowed to have
4246                  * one process attached to it -- its owner.
4247                  */
4248
4249                 if ((pt = LIST_FIRST(&pm->pm_targets)) == NULL ||
4250                     LIST_NEXT(pt, pt_next) != NULL ||
4251                     pt->pt_process->pp_proc != pm->pm_owner->po_owner) {
4252                         error = EINVAL;
4253                         break;
4254                 }
4255
4256                 ri = PMC_TO_ROWINDEX(pm);
4257                 pcd = pmc_ri_to_classdep(md, ri, &adjri);
4258
4259                 /* PMC class has no 'GETMSR' support */
4260                 if (pcd->pcd_get_msr == NULL) {
4261                         error = ENOSYS;
4262                         break;
4263                 }
4264
4265                 if ((error = (*pcd->pcd_get_msr)(adjri, &gm.pm_msr)) < 0)
4266                         break;
4267
4268                 if ((error = copyout(&gm, arg, sizeof(gm))) < 0)
4269                         break;
4270
4271                 /*
4272                  * Mark our process as using MSRs.  Update machine
4273                  * state using a forced context switch.
4274                  */
4275
4276                 pt->pt_process->pp_flags |= PMC_PP_ENABLE_MSR_ACCESS;
4277                 pmc_force_context_switch();
4278
4279         }
4280         break;
4281
4282         /*
4283          * Release an allocated PMC
4284          */
4285
4286         case PMC_OP_PMCRELEASE:
4287         {
4288                 pmc_id_t pmcid;
4289                 struct pmc *pm;
4290                 struct pmc_owner *po;
4291                 struct pmc_op_simple sp;
4292
4293                 /*
4294                  * Find PMC pointer for the named PMC.
4295                  *
4296                  * Use pmc_release_pmc_descriptor() to switch off the
4297                  * PMC, remove all its target threads, and remove the
4298                  * PMC from its owner's list.
4299                  *
4300                  * Remove the owner record if this is the last PMC
4301                  * owned.
4302                  *
4303                  * Free up space.
4304                  */
4305
4306                 if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
4307                         break;
4308
4309                 pmcid = sp.pm_pmcid;
4310
4311                 if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
4312                         break;
4313
4314                 po = pm->pm_owner;
4315                 pmc_release_pmc_descriptor(pm);
4316                 pmc_maybe_remove_owner(po);
4317                 pmc_destroy_pmc_descriptor(pm);
4318         }
4319         break;
4320
4321
4322         /*
4323          * Read and/or write a PMC.
4324          */
4325
4326         case PMC_OP_PMCRW:
4327         {
4328                 int adjri;
4329                 struct pmc *pm;
4330                 uint32_t cpu, ri;
4331                 pmc_value_t oldvalue;
4332                 struct pmc_binding pb;
4333                 struct pmc_op_pmcrw prw;
4334                 struct pmc_classdep *pcd;
4335                 struct pmc_op_pmcrw *pprw;
4336
4337                 PMC_DOWNGRADE_SX();
4338
4339                 if ((error = copyin(arg, &prw, sizeof(prw))) != 0)
4340                         break;
4341
4342                 ri = 0;
4343                 PMCDBG2(PMC,OPS,1, "rw id=%d flags=0x%x", prw.pm_pmcid,
4344                     prw.pm_flags);
4345
4346                 /* must have at least one flag set */
4347                 if ((prw.pm_flags & (PMC_F_OLDVALUE|PMC_F_NEWVALUE)) == 0) {
4348                         error = EINVAL;
4349                         break;
4350                 }
4351
4352                 /* locate pmc descriptor */
4353                 if ((error = pmc_find_pmc(prw.pm_pmcid, &pm)) != 0)
4354                         break;
4355
4356                 /* Can't read a PMC that hasn't been started. */
4357                 if (pm->pm_state != PMC_STATE_ALLOCATED &&
4358                     pm->pm_state != PMC_STATE_STOPPED &&
4359                     pm->pm_state != PMC_STATE_RUNNING) {
4360                         error = EINVAL;
4361                         break;
4362                 }
4363
4364                 /* writing a new value is allowed only for 'STOPPED' pmcs */
4365                 if (pm->pm_state == PMC_STATE_RUNNING &&
4366                     (prw.pm_flags & PMC_F_NEWVALUE)) {
4367                         error = EBUSY;
4368                         break;
4369                 }
4370
4371                 if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) {
4372
4373                         /*
4374                          * If this PMC is attached to its owner (i.e.,
4375                          * the process requesting this operation) and
4376                          * is running, then attempt to get an
4377                          * upto-date reading from hardware for a READ.
4378                          * Writes are only allowed when the PMC is
4379                          * stopped, so only update the saved value
4380                          * field.
4381                          *
4382                          * If the PMC is not running, or is not
4383                          * attached to its owner, read/write to the
4384                          * savedvalue field.
4385                          */
4386
4387                         ri = PMC_TO_ROWINDEX(pm);
4388                         pcd = pmc_ri_to_classdep(md, ri, &adjri);
4389
4390                         mtx_pool_lock_spin(pmc_mtxpool, pm);
4391                         cpu = curthread->td_oncpu;
4392
4393                         if (prw.pm_flags & PMC_F_OLDVALUE) {
4394                                 if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) &&
4395                                     (pm->pm_state == PMC_STATE_RUNNING))
4396                                         error = (*pcd->pcd_read_pmc)(cpu, adjri,
4397                                             &oldvalue);
4398                                 else
4399                                         oldvalue = pm->pm_gv.pm_savedvalue;
4400                         }
4401                         if (prw.pm_flags & PMC_F_NEWVALUE)
4402                                 pm->pm_gv.pm_savedvalue = prw.pm_value;
4403
4404                         mtx_pool_unlock_spin(pmc_mtxpool, pm);
4405
4406                 } else { /* System mode PMCs */
4407                         cpu = PMC_TO_CPU(pm);
4408                         ri  = PMC_TO_ROWINDEX(pm);
4409                         pcd = pmc_ri_to_classdep(md, ri, &adjri);
4410
4411                         if (!pmc_cpu_is_active(cpu)) {
4412                                 error = ENXIO;
4413                                 break;
4414                         }
4415
4416                         /* move this thread to CPU 'cpu' */
4417                         pmc_save_cpu_binding(&pb);
4418                         pmc_select_cpu(cpu);
4419
4420                         critical_enter();
4421                         /* save old value */
4422                         if (prw.pm_flags & PMC_F_OLDVALUE)
4423                                 if ((error = (*pcd->pcd_read_pmc)(cpu, adjri,
4424                                          &oldvalue)))
4425                                         goto error;
4426                         /* write out new value */
4427                         if (prw.pm_flags & PMC_F_NEWVALUE)
4428                                 error = (*pcd->pcd_write_pmc)(cpu, adjri,
4429                                     prw.pm_value);
4430                 error:
4431                         critical_exit();
4432                         pmc_restore_cpu_binding(&pb);
4433                         if (error)
4434                                 break;
4435                 }
4436
4437                 pprw = (struct pmc_op_pmcrw *) arg;
4438
4439 #ifdef  HWPMC_DEBUG
4440                 if (prw.pm_flags & PMC_F_NEWVALUE)
4441                         PMCDBG3(PMC,OPS,2, "rw id=%d new %jx -> old %jx",
4442                             ri, prw.pm_value, oldvalue);
4443                 else if (prw.pm_flags & PMC_F_OLDVALUE)
4444                         PMCDBG2(PMC,OPS,2, "rw id=%d -> old %jx", ri, oldvalue);
4445 #endif
4446
4447                 /* return old value if requested */
4448                 if (prw.pm_flags & PMC_F_OLDVALUE)
4449                         if ((error = copyout(&oldvalue, &pprw->pm_value,
4450                                  sizeof(prw.pm_value))))
4451                                 break;
4452
4453         }
4454         break;
4455
4456
4457         /*
4458          * Set the sampling rate for a sampling mode PMC and the
4459          * initial count for a counting mode PMC.
4460          */
4461
4462         case PMC_OP_PMCSETCOUNT:
4463         {
4464                 struct pmc *pm;
4465                 struct pmc_op_pmcsetcount sc;
4466
4467                 PMC_DOWNGRADE_SX();
4468
4469                 if ((error = copyin(arg, &sc, sizeof(sc))) != 0)
4470                         break;
4471
4472                 if ((error = pmc_find_pmc(sc.pm_pmcid, &pm)) != 0)
4473                         break;
4474
4475                 if (pm->pm_state == PMC_STATE_RUNNING) {
4476                         error = EBUSY;
4477                         break;
4478                 }
4479
4480                 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
4481                         /*
4482                          * Don't permit requested sample rate to be less than 1000
4483                          */
4484                         if (sc.pm_count < 1000)
4485                                 log(LOG_WARNING,
4486                                         "pmcsetcount: passed sample rate %ju - setting to 1000\n",
4487                                         (uintmax_t)sc.pm_count);
4488                         pm->pm_sc.pm_reloadcount = MAX(1000, sc.pm_count);
4489                 } else
4490                         pm->pm_sc.pm_initial = sc.pm_count;
4491         }
4492         break;
4493
4494
4495         /*
4496          * Start a PMC.
4497          */
4498
4499         case PMC_OP_PMCSTART:
4500         {
4501                 pmc_id_t pmcid;
4502                 struct pmc *pm;
4503                 struct pmc_op_simple sp;
4504
4505                 sx_assert(&pmc_sx, SX_XLOCKED);
4506
4507                 if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
4508                         break;
4509
4510                 pmcid = sp.pm_pmcid;
4511
4512                 if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
4513                         break;
4514
4515                 KASSERT(pmcid == pm->pm_id,
4516                     ("[pmc,%d] pmcid %x != id %x", __LINE__,
4517                         pm->pm_id, pmcid));
4518
4519                 if (pm->pm_state == PMC_STATE_RUNNING) /* already running */
4520                         break;
4521                 else if (pm->pm_state != PMC_STATE_STOPPED &&
4522                     pm->pm_state != PMC_STATE_ALLOCATED) {
4523                         error = EINVAL;
4524                         break;
4525                 }
4526
4527                 error = pmc_start(pm);
4528         }
4529         break;
4530
4531
4532         /*
4533          * Stop a PMC.
4534          */
4535
4536         case PMC_OP_PMCSTOP:
4537         {
4538                 pmc_id_t pmcid;
4539                 struct pmc *pm;
4540                 struct pmc_op_simple sp;
4541
4542                 PMC_DOWNGRADE_SX();
4543
4544                 if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
4545                         break;
4546
4547                 pmcid = sp.pm_pmcid;
4548
4549                 /*
4550                  * Mark the PMC as inactive and invoke the MD stop
4551                  * routines if needed.
4552                  */
4553
4554                 if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
4555                         break;
4556
4557                 KASSERT(pmcid == pm->pm_id,
4558                     ("[pmc,%d] pmc id %x != pmcid %x", __LINE__,
4559                         pm->pm_id, pmcid));
4560
4561                 if (pm->pm_state == PMC_STATE_STOPPED) /* already stopped */
4562                         break;
4563                 else if (pm->pm_state != PMC_STATE_RUNNING) {
4564                         error = EINVAL;
4565                         break;
4566                 }
4567
4568                 error = pmc_stop(pm);
4569         }
4570         break;
4571
4572
4573         /*
4574          * Write a user supplied value to the log file.
4575          */
4576
4577         case PMC_OP_WRITELOG:
4578         {
4579                 struct pmc_op_writelog wl;
4580                 struct pmc_owner *po;
4581
4582                 PMC_DOWNGRADE_SX();
4583
4584                 if ((error = copyin(arg, &wl, sizeof(wl))) != 0)
4585                         break;
4586
4587                 if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
4588                         error = EINVAL;
4589                         break;
4590                 }
4591
4592                 if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) {
4593                         error = EINVAL;
4594                         break;
4595                 }
4596
4597                 error = pmclog_process_userlog(po, &wl);
4598         }
4599         break;
4600
4601
4602         default:
4603                 error = EINVAL;
4604                 break;
4605         }
4606
4607         if (is_sx_downgraded)
4608                 sx_sunlock(&pmc_sx);
4609         else
4610                 sx_xunlock(&pmc_sx);
4611 done_syscall:
4612         if (error)
4613                 counter_u64_add(pmc_stats.pm_syscall_errors, 1);
4614
4615         return (error);
4616 }
4617
4618 /*
4619  * Helper functions
4620  */
4621
4622
4623 /*
4624  * Mark the thread as needing callchain capture and post an AST.  The
4625  * actual callchain capture will be done in a context where it is safe
4626  * to take page faults.
4627  */
4628
4629 static void
4630 pmc_post_callchain_callback(void)
4631 {
4632         struct thread *td;
4633
4634         td = curthread;
4635
4636         /*
4637          * If there is multiple PMCs for the same interrupt ignore new post
4638          */
4639         if (td->td_pflags & TDP_CALLCHAIN)
4640                 return;
4641
4642         /*
4643          * Mark this thread as needing callchain capture.
4644          * `td->td_pflags' will be safe to touch because this thread
4645          * was in user space when it was interrupted.
4646          */
4647         td->td_pflags |= TDP_CALLCHAIN;
4648
4649         /*
4650          * Don't let this thread migrate between CPUs until callchain
4651          * capture completes.
4652          */
4653         sched_pin();
4654
4655         return;
4656 }
4657
4658 /*
4659  * Find a free slot in the per-cpu array of samples and capture the
4660  * current callchain there.  If a sample was successfully added, a bit
4661  * is set in mask 'pmc_cpumask' denoting that the DO_SAMPLES hook
4662  * needs to be invoked from the clock handler.
4663  *
4664  * This function is meant to be called from an NMI handler.  It cannot
4665  * use any of the locking primitives supplied by the OS.
4666  */
4667
4668 static int
4669 pmc_add_sample(ring_type_t ring, struct pmc *pm, struct trapframe *tf)
4670 {
4671         int error, cpu, callchaindepth, inuserspace;
4672         struct thread *td;
4673         struct pmc_sample *ps;
4674         struct pmc_samplebuffer *psb;
4675
4676         error = 0;
4677
4678         /*
4679          * Allocate space for a sample buffer.
4680          */
4681         cpu = curcpu;
4682         psb = pmc_pcpu[cpu]->pc_sb[ring];
4683         inuserspace = TRAPF_USERMODE(tf);
4684         ps = PMC_PROD_SAMPLE(psb);
4685         if (psb->ps_considx != psb->ps_prodidx &&
4686                 ps->ps_nsamples) {      /* in use, reader hasn't caught up */
4687                 pm->pm_pcpu_state[cpu].pps_stalled = 1;
4688                 counter_u64_add(pmc_stats.pm_intr_bufferfull, 1);
4689                 PMCDBG6(SAM,INT,1,"(spc) cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d",
4690                     cpu, pm, (void *) tf, inuserspace,
4691                     (int) (psb->ps_prodidx & pmc_sample_mask),
4692                     (int) (psb->ps_considx & pmc_sample_mask));
4693                 callchaindepth = 1;
4694                 error = ENOMEM;
4695                 goto done;
4696         }
4697
4698         /* Fill in entry. */
4699         PMCDBG6(SAM,INT,1,"cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d", cpu, pm,
4700             (void *) tf, inuserspace,
4701             (int) (psb->ps_prodidx & pmc_sample_mask),
4702             (int) (psb->ps_considx & pmc_sample_mask));
4703
4704         td = curthread;
4705         ps->ps_pmc = pm;
4706         ps->ps_td = td;
4707         ps->ps_pid = td->td_proc->p_pid;
4708         ps->ps_tid = td->td_tid;
4709         ps->ps_tsc = pmc_rdtsc();
4710         ps->ps_ticks = ticks;
4711         ps->ps_cpu = cpu;
4712         ps->ps_flags = inuserspace ? PMC_CC_F_USERSPACE : 0;
4713
4714         callchaindepth = (pm->pm_flags & PMC_F_CALLCHAIN) ?
4715             pmc_callchaindepth : 1;
4716
4717         MPASS(ps->ps_pc != NULL);
4718         if (callchaindepth == 1)
4719                 ps->ps_pc[0] = PMC_TRAPFRAME_TO_PC(tf);
4720         else {
4721                 /*
4722                  * Kernel stack traversals can be done immediately,
4723                  * while we defer to an AST for user space traversals.
4724                  */
4725                 if (!inuserspace) {
4726                         callchaindepth =
4727                             pmc_save_kernel_callchain(ps->ps_pc,
4728                                 callchaindepth, tf);
4729                 } else {
4730                         pmc_post_callchain_callback();
4731                         callchaindepth = PMC_USER_CALLCHAIN_PENDING;
4732                 }
4733         }
4734
4735         ps->ps_nsamples = callchaindepth;       /* mark entry as in use */
4736         if (ring == PMC_UR) {
4737                 ps->ps_nsamples_actual = callchaindepth;        /* mark entry as in use */
4738                 ps->ps_nsamples = PMC_USER_CALLCHAIN_PENDING;
4739         } else
4740                 ps->ps_nsamples = callchaindepth;       /* mark entry as in use */
4741
4742         KASSERT(counter_u64_fetch(pm->pm_runcount) >= 0,
4743             ("[pmc,%d] pm=%p runcount %ld", __LINE__, (void *) pm,
4744                  (unsigned long)counter_u64_fetch(pm->pm_runcount)));
4745
4746         counter_u64_add(pm->pm_runcount, 1);    /* hold onto PMC */
4747         /* increment write pointer */
4748         psb->ps_prodidx++;
4749  done:
4750         /* mark CPU as needing processing */
4751         if (callchaindepth != PMC_USER_CALLCHAIN_PENDING)
4752                 DPCPU_SET(pmc_sampled, 1);
4753
4754         return (error);
4755 }
4756
4757 /*
4758  * Interrupt processing.
4759  *
4760  * This function is meant to be called from an NMI handler.  It cannot
4761  * use any of the locking primitives supplied by the OS.
4762  */
4763
4764 int
4765 pmc_process_interrupt(int ring, struct pmc *pm, struct trapframe *tf)
4766 {
4767         struct thread *td;
4768
4769         td = curthread;
4770         if ((pm->pm_flags & PMC_F_USERCALLCHAIN) &&
4771             (td->td_proc->p_flag & P_KPROC) == 0 &&
4772             !TRAPF_USERMODE(tf)) {
4773                 atomic_add_int(&td->td_pmcpend, 1);
4774                 return (pmc_add_sample(PMC_UR, pm, tf));
4775         }
4776         return (pmc_add_sample(ring, pm, tf));
4777 }
4778
4779 /*
4780  * Capture a user call chain.  This function will be called from ast()
4781  * before control returns to userland and before the process gets
4782  * rescheduled.
4783  */
4784
4785 static void
4786 pmc_capture_user_callchain(int cpu, int ring, struct trapframe *tf)
4787 {
4788         struct pmc *pm;
4789         struct thread *td;
4790         struct pmc_sample *ps;
4791         struct pmc_samplebuffer *psb;
4792         uint64_t considx, prodidx;
4793         int nsamples, nrecords, pass, iter;
4794 #ifdef  INVARIANTS
4795         int ncallchains;
4796         int nfree;
4797         int start_ticks = ticks;
4798 #endif
4799         psb = pmc_pcpu[cpu]->pc_sb[ring];
4800         td = curthread;
4801
4802         KASSERT(td->td_pflags & TDP_CALLCHAIN,
4803             ("[pmc,%d] Retrieving callchain for thread that doesn't want it",
4804                 __LINE__));
4805
4806 #ifdef  INVARIANTS
4807         ncallchains = 0;
4808         nfree = 0;
4809 #endif
4810         nrecords = INT_MAX;
4811         pass = 0;
4812  restart:
4813         if (ring == PMC_UR)
4814                 nrecords = atomic_readandclear_32(&td->td_pmcpend);
4815
4816         for (iter = 0, considx = psb->ps_considx, prodidx = psb->ps_prodidx;
4817             considx < prodidx && iter < pmc_nsamples; considx++, iter++) {
4818                 ps = PMC_CONS_SAMPLE_OFF(psb, considx);
4819
4820         /*
4821          * Iterate through all deferred callchain requests.
4822          * Walk from the current read pointer to the current
4823          * write pointer.
4824          */
4825
4826 #ifdef  INVARIANTS
4827                 if (ps->ps_nsamples == PMC_SAMPLE_FREE) {
4828                         nfree++;
4829                         continue;
4830                 }
4831
4832                 if ((ps->ps_pmc == NULL) ||
4833                     (ps->ps_pmc->pm_state != PMC_STATE_RUNNING))
4834                         nfree++;
4835 #endif
4836                 if (ps->ps_td != td ||
4837                    ps->ps_nsamples == PMC_USER_CALLCHAIN_PENDING ||
4838                    ps->ps_pmc->pm_state != PMC_STATE_RUNNING)
4839                         continue;
4840
4841                 KASSERT(ps->ps_cpu == cpu,
4842                     ("[pmc,%d] cpu mismatch ps_cpu=%d pcpu=%d", __LINE__,
4843                         ps->ps_cpu, PCPU_GET(cpuid)));
4844
4845                 pm = ps->ps_pmc;
4846
4847                 KASSERT(pm->pm_flags & PMC_F_CALLCHAIN,
4848                     ("[pmc,%d] Retrieving callchain for PMC that doesn't "
4849                         "want it", __LINE__));
4850
4851                 KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
4852                     ("[pmc,%d] runcount %ld", __LINE__, (unsigned long)counter_u64_fetch(pm->pm_runcount)));
4853
4854                 if (ring == PMC_UR) {
4855                         nsamples = ps->ps_nsamples_actual;
4856                         counter_u64_add(pmc_stats.pm_merges, 1);
4857                 } else
4858                         nsamples = 0;
4859
4860                 /*
4861                  * Retrieve the callchain and mark the sample buffer
4862                  * as 'processable' by the timer tick sweep code.
4863                  */
4864
4865 #ifdef INVARIANTS
4866                 ncallchains++;
4867 #endif
4868
4869                 if (__predict_true(nsamples < pmc_callchaindepth - 1))
4870                         nsamples += pmc_save_user_callchain(ps->ps_pc + nsamples,
4871                        pmc_callchaindepth - nsamples - 1, tf);
4872
4873                 /*
4874                  * We have to prevent hardclock from potentially overwriting
4875                  * this sample between when we read the value and when we set
4876                  * it
4877                  */
4878                 spinlock_enter();
4879                 /*
4880                  * Verify that the sample hasn't been dropped in the meantime
4881                  */
4882                 if (ps->ps_nsamples == PMC_USER_CALLCHAIN_PENDING) {
4883                         ps->ps_nsamples = nsamples;
4884                         /*
4885                          * If we couldn't get a sample, simply drop the reference
4886                          */
4887                         if (nsamples == 0)
4888                                 counter_u64_add(pm->pm_runcount, -1);
4889                 }
4890                 spinlock_exit();
4891                 if (nrecords-- == 1)
4892                         break;
4893         }
4894         if (__predict_false(ring == PMC_UR && td->td_pmcpend)) {
4895                 if (pass == 0) {
4896                         pass = 1;
4897                         goto restart;
4898                 }
4899                 /* only collect samples for this part once */
4900                 td->td_pmcpend = 0;
4901         }
4902
4903 #ifdef INVARIANTS
4904         if ((ticks - start_ticks) > hz)
4905                 log(LOG_ERR, "%s took %d ticks\n", __func__, (ticks - start_ticks));
4906 #endif
4907
4908         /* mark CPU as needing processing */
4909         DPCPU_SET(pmc_sampled, 1);
4910 }
4911
4912 /*
4913  * Process saved PC samples.
4914  */
4915
4916 static void
4917 pmc_process_samples(int cpu, ring_type_t ring)
4918 {
4919         struct pmc *pm;
4920         int adjri, n;
4921         struct thread *td;
4922         struct pmc_owner *po;
4923         struct pmc_sample *ps;
4924         struct pmc_classdep *pcd;
4925         struct pmc_samplebuffer *psb;
4926         uint64_t delta;
4927
4928         KASSERT(PCPU_GET(cpuid) == cpu,
4929             ("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__,
4930                 PCPU_GET(cpuid), cpu));
4931
4932         psb = pmc_pcpu[cpu]->pc_sb[ring];
4933         delta = psb->ps_prodidx - psb->ps_considx;
4934         MPASS(delta <= pmc_nsamples);
4935         MPASS(psb->ps_considx <= psb->ps_prodidx);
4936         for (n = 0; psb->ps_considx < psb->ps_prodidx; psb->ps_considx++, n++) {
4937                 ps = PMC_CONS_SAMPLE(psb);
4938
4939                 if (__predict_false(ps->ps_nsamples == PMC_SAMPLE_FREE))
4940                         continue;
4941                 pm = ps->ps_pmc;
4942                 /* skip non-running samples */
4943                 if (pm->pm_state != PMC_STATE_RUNNING)
4944                         goto entrydone;
4945
4946                 KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
4947                     ("[pmc,%d] pm=%p runcount %ld", __LINE__, (void *) pm,
4948                          (unsigned long)counter_u64_fetch(pm->pm_runcount)));
4949
4950                 po = pm->pm_owner;
4951
4952                 KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
4953                     ("[pmc,%d] pmc=%p non-sampling mode=%d", __LINE__,
4954                         pm, PMC_TO_MODE(pm)));
4955
4956
4957                 /* If there is a pending AST wait for completion */
4958                 if (ps->ps_nsamples == PMC_USER_CALLCHAIN_PENDING) {
4959                         /* if sample is more than 65 ms old, drop it */
4960                         if (ticks - ps->ps_ticks > (hz >> 4)) {
4961                                 /*
4962                                  * track how often we hit this as it will
4963                                  * preferentially lose user samples
4964                                  * for long running system calls
4965                                  */
4966                                 counter_u64_add(pmc_stats.pm_overwrites, 1);
4967                                 goto entrydone;
4968                         }
4969                         /* Need a rescan at a later time. */
4970                         DPCPU_SET(pmc_sampled, 1);
4971                         break;
4972                 }
4973
4974                 PMCDBG6(SAM,OPS,1,"cpu=%d pm=%p n=%d fl=%x wr=%d rd=%d", cpu,
4975                     pm, ps->ps_nsamples, ps->ps_flags,
4976                     (int) (psb->ps_prodidx & pmc_sample_mask),
4977                     (int) (psb->ps_considx & pmc_sample_mask));
4978
4979                 /*
4980                  * If this is a process-mode PMC that is attached to
4981                  * its owner, and if the PC is in user mode, update
4982                  * profiling statistics like timer-based profiling
4983                  * would have done.
4984                  *
4985                  * Otherwise, this is either a sampling-mode PMC that
4986                  * is attached to a different process than its owner,
4987                  * or a system-wide sampling PMC. Dispatch a log
4988                  * entry to the PMC's owner process.
4989                  */
4990                 if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) {
4991                         if (ps->ps_flags & PMC_CC_F_USERSPACE) {
4992                                 td = FIRST_THREAD_IN_PROC(po->po_owner);
4993                                 addupc_intr(td, ps->ps_pc[0], 1);
4994                         }
4995                 } else
4996                         pmclog_process_callchain(pm, ps);
4997
4998         entrydone:
4999                 ps->ps_nsamples = 0; /* mark entry as free */
5000                 KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
5001                                 ("[pmc,%d] pm=%p runcount %ld", __LINE__, (void *) pm,
5002                                  (unsigned long)counter_u64_fetch(pm->pm_runcount)));
5003
5004                 counter_u64_add(pm->pm_runcount, -1);
5005         }
5006
5007         counter_u64_add(pmc_stats.pm_log_sweeps, 1);
5008
5009         /* Do not re-enable stalled PMCs if we failed to process any samples */
5010         if (n == 0)
5011                 return;
5012
5013         /*
5014          * Restart any stalled sampling PMCs on this CPU.
5015          *
5016          * If the NMI handler sets the pm_stalled field of a PMC after
5017          * the check below, we'll end up processing the stalled PMC at
5018          * the next hardclock tick.
5019          */
5020         for (n = 0; n < md->pmd_npmc; n++) {
5021                 pcd = pmc_ri_to_classdep(md, n, &adjri);
5022                 KASSERT(pcd != NULL,
5023                     ("[pmc,%d] null pcd ri=%d", __LINE__, n));
5024                 (void) (*pcd->pcd_get_config)(cpu,adjri,&pm);
5025
5026                 if (pm == NULL ||                        /* !cfg'ed */
5027                     pm->pm_state != PMC_STATE_RUNNING || /* !active */
5028                     !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || /* !sampling */
5029                         !pm->pm_pcpu_state[cpu].pps_cpustate  || /* !desired */
5030                     !pm->pm_pcpu_state[cpu].pps_stalled) /* !stalled */
5031                         continue;
5032
5033                 pm->pm_pcpu_state[cpu].pps_stalled = 0;
5034                 (*pcd->pcd_start_pmc)(cpu, adjri);
5035         }
5036 }
5037
5038 /*
5039  * Event handlers.
5040  */
5041
5042 /*
5043  * Handle a process exit.
5044  *
5045  * Remove this process from all hash tables.  If this process
5046  * owned any PMCs, turn off those PMCs and deallocate them,
5047  * removing any associations with target processes.
5048  *
5049  * This function will be called by the last 'thread' of a
5050  * process.
5051  *
5052  * XXX This eventhandler gets called early in the exit process.
5053  * Consider using a 'hook' invocation from thread_exit() or equivalent
5054  * spot.  Another negative is that kse_exit doesn't seem to call
5055  * exit1() [??].
5056  *
5057  */
5058
5059 static void
5060 pmc_process_exit(void *arg __unused, struct proc *p)
5061 {
5062         struct pmc *pm;
5063         int adjri, cpu;
5064         unsigned int ri;
5065         int is_using_hwpmcs;
5066         struct pmc_owner *po;
5067         struct pmc_process *pp;
5068         struct pmc_classdep *pcd;
5069         pmc_value_t newvalue, tmp;
5070
5071         PROC_LOCK(p);
5072         is_using_hwpmcs = p->p_flag & P_HWPMC;
5073         PROC_UNLOCK(p);
5074
5075         /*
5076          * Log a sysexit event to all SS PMC owners.
5077          */
5078         PMC_EPOCH_ENTER();
5079         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5080             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
5081                     pmclog_process_sysexit(po, p->p_pid);
5082         PMC_EPOCH_EXIT();
5083
5084         if (!is_using_hwpmcs)
5085                 return;
5086
5087         PMC_GET_SX_XLOCK();
5088         PMCDBG3(PRC,EXT,1,"process-exit proc=%p (%d, %s)", p, p->p_pid,
5089             p->p_comm);
5090
5091         /*
5092          * Since this code is invoked by the last thread in an exiting
5093          * process, we would have context switched IN at some prior
5094          * point.  However, with PREEMPTION, kernel mode context
5095          * switches may happen any time, so we want to disable a
5096          * context switch OUT till we get any PMCs targeting this
5097          * process off the hardware.
5098          *
5099          * We also need to atomically remove this process'
5100          * entry from our target process hash table, using
5101          * PMC_FLAG_REMOVE.
5102          */
5103         PMCDBG3(PRC,EXT,1, "process-exit proc=%p (%d, %s)", p, p->p_pid,
5104             p->p_comm);
5105
5106         critical_enter(); /* no preemption */
5107
5108         cpu = curthread->td_oncpu;
5109
5110         if ((pp = pmc_find_process_descriptor(p,
5111                  PMC_FLAG_REMOVE)) != NULL) {
5112
5113                 PMCDBG2(PRC,EXT,2,
5114                     "process-exit proc=%p pmc-process=%p", p, pp);
5115
5116                 /*
5117                  * The exiting process could the target of
5118                  * some PMCs which will be running on
5119                  * currently executing CPU.
5120                  *
5121                  * We need to turn these PMCs off like we
5122                  * would do at context switch OUT time.
5123                  */
5124                 for (ri = 0; ri < md->pmd_npmc; ri++) {
5125
5126                         /*
5127                          * Pick up the pmc pointer from hardware
5128                          * state similar to the CSW_OUT code.
5129                          */
5130                         pm = NULL;
5131
5132                         pcd = pmc_ri_to_classdep(md, ri, &adjri);
5133
5134                         (void) (*pcd->pcd_get_config)(cpu, adjri, &pm);
5135
5136                         PMCDBG2(PRC,EXT,2, "ri=%d pm=%p", ri, pm);
5137
5138                         if (pm == NULL ||
5139                             !PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
5140                                 continue;
5141
5142                         PMCDBG4(PRC,EXT,2, "ppmcs[%d]=%p pm=%p "
5143                             "state=%d", ri, pp->pp_pmcs[ri].pp_pmc,
5144                             pm, pm->pm_state);
5145
5146                         KASSERT(PMC_TO_ROWINDEX(pm) == ri,
5147                             ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
5148                                 __LINE__, PMC_TO_ROWINDEX(pm), ri));
5149
5150                         KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
5151                             ("[pmc,%d] pm %p != pp_pmcs[%d] %p",
5152                                 __LINE__, pm, ri, pp->pp_pmcs[ri].pp_pmc));
5153
5154                         KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
5155                             ("[pmc,%d] bad runcount ri %d rc %ld",
5156                                  __LINE__, ri, (unsigned long)counter_u64_fetch(pm->pm_runcount)));
5157
5158                         /*
5159                          * Change desired state, and then stop if not
5160                          * stalled. This two-step dance should avoid
5161                          * race conditions where an interrupt re-enables
5162                          * the PMC after this code has already checked
5163                          * the pm_stalled flag.
5164                          */
5165                         if (pm->pm_pcpu_state[cpu].pps_cpustate) {
5166                                 pm->pm_pcpu_state[cpu].pps_cpustate = 0;
5167                                 if (!pm->pm_pcpu_state[cpu].pps_stalled) {
5168                                         (void) pcd->pcd_stop_pmc(cpu, adjri);
5169
5170                                         if (PMC_TO_MODE(pm) == PMC_MODE_TC) {
5171                                                 pcd->pcd_read_pmc(cpu, adjri,
5172                                                     &newvalue);
5173                                                 tmp = newvalue -
5174                                                     PMC_PCPU_SAVED(cpu,ri);
5175
5176                                                 mtx_pool_lock_spin(pmc_mtxpool,
5177                                                     pm);
5178                                                 pm->pm_gv.pm_savedvalue += tmp;
5179                                                 pp->pp_pmcs[ri].pp_pmcval +=
5180                                                     tmp;
5181                                                 mtx_pool_unlock_spin(
5182                                                     pmc_mtxpool, pm);
5183                                         }
5184                                 }
5185                         }
5186
5187                         KASSERT((int64_t) counter_u64_fetch(pm->pm_runcount) > 0,
5188                             ("[pmc,%d] runcount is %d", __LINE__, ri));
5189
5190                         counter_u64_add(pm->pm_runcount, -1);
5191
5192                         (void) pcd->pcd_config_pmc(cpu, adjri, NULL);
5193                 }
5194
5195                 /*
5196                  * Inform the MD layer of this pseudo "context switch
5197                  * out"
5198                  */
5199                 (void) md->pmd_switch_out(pmc_pcpu[cpu], pp);
5200
5201                 critical_exit(); /* ok to be pre-empted now */
5202
5203                 /*
5204                  * Unlink this process from the PMCs that are
5205                  * targeting it.  This will send a signal to
5206                  * all PMC owner's whose PMCs are orphaned.
5207                  *
5208                  * Log PMC value at exit time if requested.
5209                  */
5210                 for (ri = 0; ri < md->pmd_npmc; ri++)
5211                         if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) {
5212                                 if (pm->pm_flags & PMC_F_NEEDS_LOGFILE &&
5213                                     PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm)))
5214                                         pmclog_process_procexit(pm, pp);
5215                                 pmc_unlink_target_process(pm, pp);
5216                         }
5217                 free(pp, M_PMC);
5218
5219         } else
5220                 critical_exit(); /* pp == NULL */
5221
5222
5223         /*
5224          * If the process owned PMCs, free them up and free up
5225          * memory.
5226          */
5227         if ((po = pmc_find_owner_descriptor(p)) != NULL) {
5228                 pmc_remove_owner(po);
5229                 pmc_destroy_owner_descriptor(po);
5230         }
5231
5232         sx_xunlock(&pmc_sx);
5233 }
5234
5235 /*
5236  * Handle a process fork.
5237  *
5238  * If the parent process 'p1' is under HWPMC monitoring, then copy
5239  * over any attached PMCs that have 'do_descendants' semantics.
5240  */
5241
5242 static void
5243 pmc_process_fork(void *arg __unused, struct proc *p1, struct proc *newproc,
5244     int flags)
5245 {
5246         int is_using_hwpmcs;
5247         unsigned int ri;
5248         uint32_t do_descendants;
5249         struct pmc *pm;
5250         struct pmc_owner *po;
5251         struct pmc_process *ppnew, *ppold;
5252
5253         (void) flags;           /* unused parameter */
5254
5255         PROC_LOCK(p1);
5256         is_using_hwpmcs = p1->p_flag & P_HWPMC;
5257         PROC_UNLOCK(p1);
5258
5259         /*
5260          * If there are system-wide sampling PMCs active, we need to
5261          * log all fork events to their owner's logs.
5262          */
5263         PMC_EPOCH_ENTER();
5264         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5265             if (po->po_flags & PMC_PO_OWNS_LOGFILE) {
5266                     pmclog_process_procfork(po, p1->p_pid, newproc->p_pid);
5267                         pmclog_process_proccreate(po, newproc, 1);
5268                 }
5269         PMC_EPOCH_EXIT();
5270
5271         if (!is_using_hwpmcs)
5272                 return;
5273
5274         PMC_GET_SX_XLOCK();
5275         PMCDBG4(PMC,FRK,1, "process-fork proc=%p (%d, %s) -> %p", p1,
5276             p1->p_pid, p1->p_comm, newproc);
5277
5278         /*
5279          * If the parent process (curthread->td_proc) is a
5280          * target of any PMCs, look for PMCs that are to be
5281          * inherited, and link these into the new process
5282          * descriptor.
5283          */
5284         if ((ppold = pmc_find_process_descriptor(curthread->td_proc,
5285                  PMC_FLAG_NONE)) == NULL)
5286                 goto done;              /* nothing to do */
5287
5288         do_descendants = 0;
5289         for (ri = 0; ri < md->pmd_npmc; ri++)
5290                 if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL)
5291                         do_descendants |= pm->pm_flags & PMC_F_DESCENDANTS;
5292         if (do_descendants == 0) /* nothing to do */
5293                 goto done;
5294
5295         /*
5296          * Now mark the new process as being tracked by this driver.
5297          */
5298         PROC_LOCK(newproc);
5299         newproc->p_flag |= P_HWPMC;
5300         PROC_UNLOCK(newproc);
5301
5302         /* allocate a descriptor for the new process  */
5303         if ((ppnew = pmc_find_process_descriptor(newproc,
5304                  PMC_FLAG_ALLOCATE)) == NULL)
5305                 goto done;
5306
5307         /*
5308          * Run through all PMCs that were targeting the old process
5309          * and which specified F_DESCENDANTS and attach them to the
5310          * new process.
5311          *
5312          * Log the fork event to all owners of PMCs attached to this
5313          * process, if not already logged.
5314          */
5315         for (ri = 0; ri < md->pmd_npmc; ri++)
5316                 if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL &&
5317                     (pm->pm_flags & PMC_F_DESCENDANTS)) {
5318                         pmc_link_target_process(pm, ppnew);
5319                         po = pm->pm_owner;
5320                         if (po->po_sscount == 0 &&
5321                             po->po_flags & PMC_PO_OWNS_LOGFILE)
5322                                 pmclog_process_procfork(po, p1->p_pid,
5323                                     newproc->p_pid);
5324                 }
5325
5326  done:
5327         sx_xunlock(&pmc_sx);
5328 }
5329
5330 static void
5331 pmc_process_threadcreate(struct thread *td)
5332 {
5333         struct pmc_owner *po;
5334
5335         PMC_EPOCH_ENTER();
5336         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5337             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
5338                         pmclog_process_threadcreate(po, td, 1);
5339         PMC_EPOCH_EXIT();
5340 }
5341
5342 static void
5343 pmc_process_threadexit(struct thread *td)
5344 {
5345         struct pmc_owner *po;
5346
5347         PMC_EPOCH_ENTER();
5348         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5349             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
5350                         pmclog_process_threadexit(po, td);
5351         PMC_EPOCH_EXIT();
5352 }
5353
5354 static void
5355 pmc_process_proccreate(struct proc *p)
5356 {
5357         struct pmc_owner *po;
5358
5359         PMC_EPOCH_ENTER();
5360         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5361             if (po->po_flags & PMC_PO_OWNS_LOGFILE)
5362                         pmclog_process_proccreate(po, p, 1 /* sync */);
5363         PMC_EPOCH_EXIT();
5364 }
5365
5366 static void
5367 pmc_process_allproc(struct pmc *pm)
5368 {
5369         struct pmc_owner *po;
5370         struct thread *td;
5371         struct proc *p;
5372
5373         po = pm->pm_owner;
5374         if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
5375                 return;
5376         sx_slock(&allproc_lock);
5377         FOREACH_PROC_IN_SYSTEM(p) {
5378                 pmclog_process_proccreate(po, p, 0 /* sync */);
5379                 PROC_LOCK(p);
5380                 FOREACH_THREAD_IN_PROC(p, td)
5381                         pmclog_process_threadcreate(po, td, 0 /* sync */);
5382                 PROC_UNLOCK(p);
5383         }
5384         sx_sunlock(&allproc_lock);
5385         pmclog_flush(po, 0);
5386 }
5387
5388 static void
5389 pmc_kld_load(void *arg __unused, linker_file_t lf)
5390 {
5391         struct pmc_owner *po;
5392
5393         /*
5394          * Notify owners of system sampling PMCs about KLD operations.
5395          */
5396         PMC_EPOCH_ENTER();
5397         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5398                 if (po->po_flags & PMC_PO_OWNS_LOGFILE)
5399                         pmclog_process_map_in(po, (pid_t) -1,
5400                             (uintfptr_t) lf->address, lf->filename);
5401         PMC_EPOCH_EXIT();
5402
5403         /*
5404          * TODO: Notify owners of (all) process-sampling PMCs too.
5405          */
5406 }
5407
5408 static void
5409 pmc_kld_unload(void *arg __unused, const char *filename __unused,
5410     caddr_t address, size_t size)
5411 {
5412         struct pmc_owner *po;
5413
5414         PMC_EPOCH_ENTER();
5415         CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
5416                 if (po->po_flags & PMC_PO_OWNS_LOGFILE)
5417                         pmclog_process_map_out(po, (pid_t) -1,
5418                             (uintfptr_t) address, (uintfptr_t) address + size);
5419         PMC_EPOCH_EXIT();
5420
5421         /*
5422          * TODO: Notify owners of process-sampling PMCs.
5423          */
5424 }
5425
5426 /*
5427  * initialization
5428  */
5429 static const char *
5430 pmc_name_of_pmcclass(enum pmc_class class)
5431 {
5432
5433         switch (class) {
5434 #undef  __PMC_CLASS
5435 #define __PMC_CLASS(S,V,D)                                              \
5436         case PMC_CLASS_##S:                                             \
5437                 return #S;
5438         __PMC_CLASSES();
5439         default:
5440                 return ("<unknown>");
5441         }
5442 }
5443
5444 /*
5445  * Base class initializer: allocate structure and set default classes.
5446  */
5447 struct pmc_mdep *
5448 pmc_mdep_alloc(int nclasses)
5449 {
5450         struct pmc_mdep *md;
5451         int     n;
5452
5453         /* SOFT + md classes */
5454         n = 1 + nclasses;
5455         md = malloc(sizeof(struct pmc_mdep) + n *
5456             sizeof(struct pmc_classdep), M_PMC, M_WAITOK|M_ZERO);
5457         md->pmd_nclass = n;
5458
5459         /* Add base class. */
5460         pmc_soft_initialize(md);
5461         return md;
5462 }
5463
5464 void
5465 pmc_mdep_free(struct pmc_mdep *md)
5466 {
5467         pmc_soft_finalize(md);
5468         free(md, M_PMC);
5469 }
5470
5471 static int
5472 generic_switch_in(struct pmc_cpu *pc, struct pmc_process *pp)
5473 {
5474         (void) pc; (void) pp;
5475
5476         return (0);
5477 }
5478
5479 static int
5480 generic_switch_out(struct pmc_cpu *pc, struct pmc_process *pp)
5481 {
5482         (void) pc; (void) pp;
5483
5484         return (0);
5485 }
5486
5487 static struct pmc_mdep *
5488 pmc_generic_cpu_initialize(void)
5489 {
5490         struct pmc_mdep *md;
5491
5492         md = pmc_mdep_alloc(0);
5493
5494         md->pmd_cputype    = PMC_CPU_GENERIC;
5495
5496         md->pmd_pcpu_init  = NULL;
5497         md->pmd_pcpu_fini  = NULL;
5498         md->pmd_switch_in  = generic_switch_in;
5499         md->pmd_switch_out = generic_switch_out;
5500
5501         return (md);
5502 }
5503
5504 static void
5505 pmc_generic_cpu_finalize(struct pmc_mdep *md)
5506 {
5507         (void) md;
5508 }
5509
5510
5511 static int
5512 pmc_initialize(void)
5513 {
5514         int c, cpu, error, n, ri;
5515         unsigned int maxcpu, domain;
5516         struct pcpu *pc;
5517         struct pmc_binding pb;
5518         struct pmc_sample *ps;
5519         struct pmc_classdep *pcd;
5520         struct pmc_samplebuffer *sb;
5521
5522         md = NULL;
5523         error = 0;
5524
5525         pmc_stats.pm_intr_ignored = counter_u64_alloc(M_WAITOK);
5526         pmc_stats.pm_intr_processed = counter_u64_alloc(M_WAITOK);
5527         pmc_stats.pm_intr_bufferfull = counter_u64_alloc(M_WAITOK);
5528         pmc_stats.pm_syscalls = counter_u64_alloc(M_WAITOK);
5529         pmc_stats.pm_syscall_errors = counter_u64_alloc(M_WAITOK);
5530         pmc_stats.pm_buffer_requests = counter_u64_alloc(M_WAITOK);
5531         pmc_stats.pm_buffer_requests_failed = counter_u64_alloc(M_WAITOK);
5532         pmc_stats.pm_log_sweeps = counter_u64_alloc(M_WAITOK);
5533         pmc_stats.pm_merges = counter_u64_alloc(M_WAITOK);
5534         pmc_stats.pm_overwrites = counter_u64_alloc(M_WAITOK);
5535
5536 #ifdef  HWPMC_DEBUG
5537         /* parse debug flags first */
5538         if (TUNABLE_STR_FETCH(PMC_SYSCTL_NAME_PREFIX "debugflags",
5539                 pmc_debugstr, sizeof(pmc_debugstr)))
5540                 pmc_debugflags_parse(pmc_debugstr,
5541                     pmc_debugstr+strlen(pmc_debugstr));
5542 #endif
5543
5544         PMCDBG1(MOD,INI,0, "PMC Initialize (version %x)", PMC_VERSION);
5545
5546         /* check kernel version */
5547         if (pmc_kernel_version != PMC_VERSION) {
5548                 if (pmc_kernel_version == 0)
5549                         printf("hwpmc: this kernel has not been compiled with "
5550                             "'options HWPMC_HOOKS'.\n");
5551                 else
5552                         printf("hwpmc: kernel version (0x%x) does not match "
5553                             "module version (0x%x).\n", pmc_kernel_version,
5554                             PMC_VERSION);
5555                 return EPROGMISMATCH;
5556         }
5557
5558         /*
5559          * check sysctl parameters
5560          */
5561
5562         if (pmc_hashsize <= 0) {
5563                 (void) printf("hwpmc: tunable \"hashsize\"=%d must be "
5564                     "greater than zero.\n", pmc_hashsize);
5565                 pmc_hashsize = PMC_HASH_SIZE;
5566         }
5567
5568         if (pmc_nsamples <= 0 || pmc_nsamples > 65535) {
5569                 (void) printf("hwpmc: tunable \"nsamples\"=%d out of "
5570                     "range.\n", pmc_nsamples);
5571                 pmc_nsamples = PMC_NSAMPLES;
5572         }
5573         pmc_sample_mask = pmc_nsamples-1;
5574
5575         if (pmc_callchaindepth <= 0 ||
5576             pmc_callchaindepth > PMC_CALLCHAIN_DEPTH_MAX) {
5577                 (void) printf("hwpmc: tunable \"callchaindepth\"=%d out of "
5578                     "range - using %d.\n", pmc_callchaindepth,
5579                     PMC_CALLCHAIN_DEPTH_MAX);
5580                 pmc_callchaindepth = PMC_CALLCHAIN_DEPTH_MAX;
5581         }
5582
5583         md = pmc_md_initialize();
5584         if (md == NULL) {
5585                 /* Default to generic CPU. */
5586                 md = pmc_generic_cpu_initialize();
5587                 if (md == NULL)
5588                         return (ENOSYS);
5589         }
5590
5591         KASSERT(md->pmd_nclass >= 1 && md->pmd_npmc >= 1,
5592             ("[pmc,%d] no classes or pmcs", __LINE__));
5593
5594         /* Compute the map from row-indices to classdep pointers. */
5595         pmc_rowindex_to_classdep = malloc(sizeof(struct pmc_classdep *) *
5596             md->pmd_npmc, M_PMC, M_WAITOK|M_ZERO);
5597
5598         for (n = 0; n < md->pmd_npmc; n++)
5599                 pmc_rowindex_to_classdep[n] = NULL;
5600         for (ri = c = 0; c < md->pmd_nclass; c++) {
5601                 pcd = &md->pmd_classdep[c];
5602                 for (n = 0; n < pcd->pcd_num; n++, ri++)
5603                         pmc_rowindex_to_classdep[ri] = pcd;
5604         }
5605
5606         KASSERT(ri == md->pmd_npmc,
5607             ("[pmc,%d] npmc miscomputed: ri=%d, md->npmc=%d", __LINE__,
5608             ri, md->pmd_npmc));
5609
5610         maxcpu = pmc_cpu_max();
5611
5612         /* allocate space for the per-cpu array */
5613         pmc_pcpu = malloc(maxcpu * sizeof(struct pmc_cpu *), M_PMC,
5614             M_WAITOK|M_ZERO);
5615
5616         /* per-cpu 'saved values' for managing process-mode PMCs */
5617         pmc_pcpu_saved = malloc(sizeof(pmc_value_t) * maxcpu * md->pmd_npmc,
5618             M_PMC, M_WAITOK);
5619
5620         /* Perform CPU-dependent initialization. */
5621         pmc_save_cpu_binding(&pb);
5622         error = 0;
5623         for (cpu = 0; error == 0 && cpu < maxcpu; cpu++) {
5624                 if (!pmc_cpu_is_active(cpu))
5625                         continue;
5626                 pmc_select_cpu(cpu);
5627                 pmc_pcpu[cpu] = malloc(sizeof(struct pmc_cpu) +
5628                     md->pmd_npmc * sizeof(struct pmc_hw *), M_PMC,
5629                     M_WAITOK|M_ZERO);
5630                 if (md->pmd_pcpu_init)
5631                         error = md->pmd_pcpu_init(md, cpu);
5632                 for (n = 0; error == 0 && n < md->pmd_nclass; n++)
5633                         error = md->pmd_classdep[n].pcd_pcpu_init(md, cpu);
5634         }
5635         pmc_restore_cpu_binding(&pb);
5636
5637         if (error)
5638                 return (error);
5639
5640         /* allocate space for the sample array */
5641         for (cpu = 0; cpu < maxcpu; cpu++) {
5642                 if (!pmc_cpu_is_active(cpu))
5643                         continue;
5644                 pc = pcpu_find(cpu);
5645                 domain = pc->pc_domain;
5646                 sb = malloc_domain(sizeof(struct pmc_samplebuffer) +
5647                         pmc_nsamples * sizeof(struct pmc_sample), M_PMC, domain,
5648                     M_WAITOK|M_ZERO);
5649
5650                 KASSERT(pmc_pcpu[cpu] != NULL,
5651                     ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
5652
5653                 sb->ps_callchains = malloc_domain(pmc_callchaindepth * pmc_nsamples *
5654                         sizeof(uintptr_t), M_PMC, domain, M_WAITOK|M_ZERO);
5655
5656                 for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
5657                         ps->ps_pc = sb->ps_callchains +
5658                             (n * pmc_callchaindepth);
5659
5660                 pmc_pcpu[cpu]->pc_sb[PMC_HR] = sb;
5661
5662                 sb = malloc_domain(sizeof(struct pmc_samplebuffer) +
5663                         pmc_nsamples * sizeof(struct pmc_sample), M_PMC, domain,
5664                     M_WAITOK|M_ZERO);
5665
5666                 KASSERT(pmc_pcpu[cpu] != NULL,
5667                     ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
5668
5669                 sb->ps_callchains = malloc_domain(pmc_callchaindepth * pmc_nsamples *
5670                         sizeof(uintptr_t), M_PMC, domain, M_WAITOK|M_ZERO);
5671
5672                 for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
5673                         ps->ps_pc = sb->ps_callchains +
5674                             (n * pmc_callchaindepth);
5675
5676                 pmc_pcpu[cpu]->pc_sb[PMC_SR] = sb;
5677
5678                 sb = malloc_domain(sizeof(struct pmc_samplebuffer) +
5679                         pmc_nsamples * sizeof(struct pmc_sample), M_PMC, domain,
5680                     M_WAITOK|M_ZERO);
5681
5682                 KASSERT(pmc_pcpu[cpu] != NULL,
5683                     ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
5684
5685                 sb->ps_callchains = malloc_domain(pmc_callchaindepth * pmc_nsamples *
5686                     sizeof(uintptr_t), M_PMC, domain, M_WAITOK|M_ZERO);
5687
5688                 for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
5689                         ps->ps_pc = sb->ps_callchains +
5690                             (n * pmc_callchaindepth);
5691
5692                 pmc_pcpu[cpu]->pc_sb[PMC_UR] = sb;
5693         }
5694
5695         /* allocate space for the row disposition array */
5696         pmc_pmcdisp = malloc(sizeof(enum pmc_mode) * md->pmd_npmc,
5697             M_PMC, M_WAITOK|M_ZERO);
5698
5699         /* mark all PMCs as available */
5700         for (n = 0; n < (int) md->pmd_npmc; n++)
5701                 PMC_MARK_ROW_FREE(n);
5702
5703         /* allocate thread hash tables */
5704         pmc_ownerhash = hashinit(pmc_hashsize, M_PMC,
5705             &pmc_ownerhashmask);
5706
5707         pmc_processhash = hashinit(pmc_hashsize, M_PMC,
5708             &pmc_processhashmask);
5709         mtx_init(&pmc_processhash_mtx, "pmc-process-hash", "pmc-leaf",
5710             MTX_SPIN);
5711
5712         CK_LIST_INIT(&pmc_ss_owners);
5713         pmc_ss_count = 0;
5714
5715         /* allocate a pool of spin mutexes */
5716         pmc_mtxpool = mtx_pool_create("pmc-leaf", pmc_mtxpool_size,
5717             MTX_SPIN);
5718
5719         PMCDBG4(MOD,INI,1, "pmc_ownerhash=%p, mask=0x%lx "
5720             "targethash=%p mask=0x%lx", pmc_ownerhash, pmc_ownerhashmask,
5721             pmc_processhash, pmc_processhashmask);
5722
5723         /* Initialize a spin mutex for the thread free list. */
5724         mtx_init(&pmc_threadfreelist_mtx, "pmc-threadfreelist", "pmc-leaf",
5725             MTX_SPIN);
5726
5727         /*
5728          * Initialize the callout to monitor the thread free list.
5729          * This callout will also handle the initial population of the list.
5730          */
5731         taskqgroup_config_gtask_init(NULL, &free_gtask, pmc_thread_descriptor_pool_free_task, "thread descriptor pool free task");
5732
5733         /* register process {exit,fork,exec} handlers */
5734         pmc_exit_tag = EVENTHANDLER_REGISTER(process_exit,
5735             pmc_process_exit, NULL, EVENTHANDLER_PRI_ANY);
5736         pmc_fork_tag = EVENTHANDLER_REGISTER(process_fork,
5737             pmc_process_fork, NULL, EVENTHANDLER_PRI_ANY);
5738
5739         /* register kld event handlers */
5740         pmc_kld_load_tag = EVENTHANDLER_REGISTER(kld_load, pmc_kld_load,
5741             NULL, EVENTHANDLER_PRI_ANY);
5742         pmc_kld_unload_tag = EVENTHANDLER_REGISTER(kld_unload, pmc_kld_unload,
5743             NULL, EVENTHANDLER_PRI_ANY);
5744
5745         /* initialize logging */
5746         pmclog_initialize();
5747
5748         /* set hook functions */
5749         pmc_intr = md->pmd_intr;
5750         wmb();
5751         pmc_hook = pmc_hook_handler;
5752
5753         if (error == 0) {
5754                 printf(PMC_MODULE_NAME ":");
5755                 for (n = 0; n < (int) md->pmd_nclass; n++) {
5756                         pcd = &md->pmd_classdep[n];
5757                         printf(" %s/%d/%d/0x%b",
5758                             pmc_name_of_pmcclass(pcd->pcd_class),
5759                             pcd->pcd_num,
5760                             pcd->pcd_width,
5761                             pcd->pcd_caps,
5762                             "\20"
5763                             "\1INT\2USR\3SYS\4EDG\5THR"
5764                             "\6REA\7WRI\10INV\11QUA\12PRC"
5765                             "\13TAG\14CSC");
5766                 }
5767                 printf("\n");
5768         }
5769
5770         return (error);
5771 }
5772
5773 /* prepare to be unloaded */
5774 static void
5775 pmc_cleanup(void)
5776 {
5777         int c, cpu;
5778         unsigned int maxcpu;
5779         struct pmc_ownerhash *ph;
5780         struct pmc_owner *po, *tmp;
5781         struct pmc_binding pb;
5782 #ifdef  HWPMC_DEBUG
5783         struct pmc_processhash *prh;
5784 #endif
5785
5786         PMCDBG0(MOD,INI,0, "cleanup");
5787
5788         /* switch off sampling */
5789         CPU_FOREACH(cpu)
5790                 DPCPU_ID_SET(cpu, pmc_sampled, 0);
5791         pmc_intr = NULL;
5792
5793         sx_xlock(&pmc_sx);
5794         if (pmc_hook == NULL) { /* being unloaded already */
5795                 sx_xunlock(&pmc_sx);
5796                 return;
5797         }
5798
5799         pmc_hook = NULL; /* prevent new threads from entering module */
5800
5801         /* deregister event handlers */
5802         EVENTHANDLER_DEREGISTER(process_fork, pmc_fork_tag);
5803         EVENTHANDLER_DEREGISTER(process_exit, pmc_exit_tag);
5804         EVENTHANDLER_DEREGISTER(kld_load, pmc_kld_load_tag);
5805         EVENTHANDLER_DEREGISTER(kld_unload, pmc_kld_unload_tag);
5806
5807         /* send SIGBUS to all owner threads, free up allocations */
5808         if (pmc_ownerhash)
5809                 for (ph = pmc_ownerhash;
5810                      ph <= &pmc_ownerhash[pmc_ownerhashmask];
5811                      ph++) {
5812                         LIST_FOREACH_SAFE(po, ph, po_next, tmp) {
5813                                 pmc_remove_owner(po);
5814
5815                                 /* send SIGBUS to owner processes */
5816                                 PMCDBG3(MOD,INI,2, "cleanup signal proc=%p "
5817                                     "(%d, %s)", po->po_owner,
5818                                     po->po_owner->p_pid,
5819                                     po->po_owner->p_comm);
5820
5821                                 PROC_LOCK(po->po_owner);
5822                                 kern_psignal(po->po_owner, SIGBUS);
5823                                 PROC_UNLOCK(po->po_owner);
5824
5825                                 pmc_destroy_owner_descriptor(po);
5826                         }
5827                 }
5828
5829         /* reclaim allocated data structures */
5830         mtx_destroy(&pmc_threadfreelist_mtx);
5831         pmc_thread_descriptor_pool_drain();
5832
5833         if (pmc_mtxpool)
5834                 mtx_pool_destroy(&pmc_mtxpool);
5835
5836         mtx_destroy(&pmc_processhash_mtx);
5837         taskqgroup_config_gtask_deinit(&free_gtask);
5838         if (pmc_processhash) {
5839 #ifdef  HWPMC_DEBUG
5840                 struct pmc_process *pp;
5841
5842                 PMCDBG0(MOD,INI,3, "destroy process hash");
5843                 for (prh = pmc_processhash;
5844                      prh <= &pmc_processhash[pmc_processhashmask];
5845                      prh++)
5846                         LIST_FOREACH(pp, prh, pp_next)
5847                             PMCDBG1(MOD,INI,3, "pid=%d", pp->pp_proc->p_pid);
5848 #endif
5849
5850                 hashdestroy(pmc_processhash, M_PMC, pmc_processhashmask);
5851                 pmc_processhash = NULL;
5852         }
5853
5854         if (pmc_ownerhash) {
5855                 PMCDBG0(MOD,INI,3, "destroy owner hash");
5856                 hashdestroy(pmc_ownerhash, M_PMC, pmc_ownerhashmask);
5857                 pmc_ownerhash = NULL;
5858         }
5859
5860         KASSERT(CK_LIST_EMPTY(&pmc_ss_owners),
5861             ("[pmc,%d] Global SS owner list not empty", __LINE__));
5862         KASSERT(pmc_ss_count == 0,
5863             ("[pmc,%d] Global SS count not empty", __LINE__));
5864
5865         /* do processor and pmc-class dependent cleanup */
5866         maxcpu = pmc_cpu_max();
5867
5868         PMCDBG0(MOD,INI,3, "md cleanup");
5869         if (md) {
5870                 pmc_save_cpu_binding(&pb);
5871                 for (cpu = 0; cpu < maxcpu; cpu++) {
5872                         PMCDBG2(MOD,INI,1,"pmc-cleanup cpu=%d pcs=%p",
5873                             cpu, pmc_pcpu[cpu]);
5874                         if (!pmc_cpu_is_active(cpu) || pmc_pcpu[cpu] == NULL)
5875                                 continue;
5876                         pmc_select_cpu(cpu);
5877                         for (c = 0; c < md->pmd_nclass; c++)
5878                                 md->pmd_classdep[c].pcd_pcpu_fini(md, cpu);
5879                         if (md->pmd_pcpu_fini)
5880                                 md->pmd_pcpu_fini(md, cpu);
5881                 }
5882
5883                 if (md->pmd_cputype == PMC_CPU_GENERIC)
5884                         pmc_generic_cpu_finalize(md);
5885                 else
5886                         pmc_md_finalize(md);
5887
5888                 pmc_mdep_free(md);
5889                 md = NULL;
5890                 pmc_restore_cpu_binding(&pb);
5891         }
5892
5893         /* Free per-cpu descriptors. */
5894         for (cpu = 0; cpu < maxcpu; cpu++) {
5895                 if (!pmc_cpu_is_active(cpu))
5896                         continue;
5897                 KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_HR] != NULL,
5898                     ("[pmc,%d] Null hw cpu sample buffer cpu=%d", __LINE__,
5899                         cpu));
5900                 KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_SR] != NULL,
5901                     ("[pmc,%d] Null sw cpu sample buffer cpu=%d", __LINE__,
5902                         cpu));
5903                 KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_UR] != NULL,
5904                     ("[pmc,%d] Null userret cpu sample buffer cpu=%d", __LINE__,
5905                         cpu));
5906                 free_domain(pmc_pcpu[cpu]->pc_sb[PMC_HR]->ps_callchains, M_PMC);
5907                 free_domain(pmc_pcpu[cpu]->pc_sb[PMC_HR], M_PMC);
5908                 free_domain(pmc_pcpu[cpu]->pc_sb[PMC_SR]->ps_callchains, M_PMC);
5909                 free_domain(pmc_pcpu[cpu]->pc_sb[PMC_SR], M_PMC);
5910                 free_domain(pmc_pcpu[cpu]->pc_sb[PMC_UR]->ps_callchains, M_PMC);
5911                 free_domain(pmc_pcpu[cpu]->pc_sb[PMC_UR], M_PMC);
5912                 free_domain(pmc_pcpu[cpu], M_PMC);
5913         }
5914
5915         free(pmc_pcpu, M_PMC);
5916         pmc_pcpu = NULL;
5917
5918         free(pmc_pcpu_saved, M_PMC);
5919         pmc_pcpu_saved = NULL;
5920
5921         if (pmc_pmcdisp) {
5922                 free(pmc_pmcdisp, M_PMC);
5923                 pmc_pmcdisp = NULL;
5924         }
5925
5926         if (pmc_rowindex_to_classdep) {
5927                 free(pmc_rowindex_to_classdep, M_PMC);
5928                 pmc_rowindex_to_classdep = NULL;
5929         }
5930
5931         pmclog_shutdown();
5932         counter_u64_free(pmc_stats.pm_intr_ignored);
5933         counter_u64_free(pmc_stats.pm_intr_processed);
5934         counter_u64_free(pmc_stats.pm_intr_bufferfull);
5935         counter_u64_free(pmc_stats.pm_syscalls);
5936         counter_u64_free(pmc_stats.pm_syscall_errors);
5937         counter_u64_free(pmc_stats.pm_buffer_requests);
5938         counter_u64_free(pmc_stats.pm_buffer_requests_failed);
5939         counter_u64_free(pmc_stats.pm_log_sweeps);
5940         counter_u64_free(pmc_stats.pm_merges);
5941         counter_u64_free(pmc_stats.pm_overwrites);
5942         sx_xunlock(&pmc_sx);    /* we are done */
5943 }
5944
5945 /*
5946  * The function called at load/unload.
5947  */
5948
5949 static int
5950 load (struct module *module __unused, int cmd, void *arg __unused)
5951 {
5952         int error;
5953
5954         error = 0;
5955
5956         switch (cmd) {
5957         case MOD_LOAD :
5958                 /* initialize the subsystem */
5959                 error = pmc_initialize();
5960                 if (error != 0)
5961                         break;
5962                 PMCDBG2(MOD,INI,1, "syscall=%d maxcpu=%d",
5963                     pmc_syscall_num, pmc_cpu_max());
5964                 break;
5965
5966
5967         case MOD_UNLOAD :
5968         case MOD_SHUTDOWN:
5969                 pmc_cleanup();
5970                 PMCDBG0(MOD,INI,1, "unloaded");
5971                 break;
5972
5973         default :
5974                 error = EINVAL; /* XXX should panic(9) */
5975                 break;
5976         }
5977
5978         return error;
5979 }