sys/sys/pmckern.h

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2003-2007, Joseph Koshy
   5  * Copyright (c) 2007 The FreeBSD Foundation
   6  * All rights reserved.
   7  *
   8  * Portions of this software were developed by A. Joseph Koshy under
   9  * sponsorship from the FreeBSD Foundation and Google, Inc.
  10  *
  11  * Redistribution and use in source and binary forms, with or without
  12  * modification, are permitted provided that the following conditions
  13  * are met:
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in the
  18  *    documentation and/or other materials provided with the distribution.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  30  * SUCH DAMAGE.
  31  *
  32  * $FreeBSD$
  33  */
  34
  35 /*
  36  * PMC interface used by the base kernel.
  37  */
  38
  39 #ifndef _SYS_PMCKERN_H_
  40 #define _SYS_PMCKERN_H_
  41
  42 #include <sys/param.h>
  43 #include <sys/systm.h>
  44 #include <sys/kernel.h>
  45 #include <sys/lock.h>
  46 #include <sys/proc.h>
  47 #include <sys/sx.h>
  48 #include <sys/pmc.h>
  49
  50 #include <machine/cpufunc.h>
  51
  52 #define PMC_FN_PROCESS_EXEC             1
  53 #define PMC_FN_CSW_IN                   2
  54 #define PMC_FN_CSW_OUT                  3
  55 #define PMC_FN_DO_SAMPLES               4
  56 #define PMC_FN_UNUSED1                  5
  57 #define PMC_FN_UNUSED2                  6
  58 #define PMC_FN_MMAP                     7
  59 #define PMC_FN_MUNMAP                   8
  60 #define PMC_FN_USER_CALLCHAIN           9
  61 #define PMC_FN_USER_CALLCHAIN_SOFT      10
  62 #define PMC_FN_SOFT_SAMPLING            11
  63 #define PMC_FN_THR_CREATE               12
  64 #define PMC_FN_THR_EXIT                 13
  65 #define PMC_FN_THR_USERRET              14
  66 #define PMC_FN_THR_CREATE_LOG           15
  67 #define PMC_FN_THR_EXIT_LOG             16
  68 #define PMC_FN_PROC_CREATE_LOG          17
  69
  70 typedef enum ring_type {
  71         PMC_HR = 0,     /* Hardware ring buffer */
  72                 PMC_SR = 1,     /* Software ring buffer */
  73         PMC_UR = 2,     /* userret ring buffer */
  74                 PMC_NUM_SR = PMC_UR+1
  75 } ring_type_t;
  76
  77 struct pmckern_procexec {
  78         int             pm_credentialschanged;
  79         uintfptr_t      pm_entryaddr;
  80 };
  81
  82 struct pmckern_map_in {
  83         void            *pm_file;       /* filename or vnode pointer */
  84         uintfptr_t      pm_address;     /* address object is loaded at */
  85 };
  86
  87 struct pmckern_map_out {
  88         uintfptr_t      pm_address;     /* start address of region */
  89         size_t          pm_size;        /* size of unmapped region */
  90 };
  91
  92 struct pmckern_soft {
  93         enum pmc_event          pm_ev;
  94         int                     pm_cpu;
  95         struct trapframe        *pm_tf;
  96 };
  97
  98 /*
  99  * Soft PMC.
 100  */
 101
 102 #define PMC_SOFT_DEFINE_EX(prov, mod, func, name, alloc, release)               \
 103         struct pmc_soft pmc_##prov##_##mod##_##func##_##name =                  \
 104             { 0, alloc, release, { #prov "_" #mod "_" #func "." #name, 0 } };   \
 105         SYSINIT(pmc_##prov##_##mod##_##func##_##name##_init, SI_SUB_KDTRACE,    \
 106             SI_ORDER_SECOND + 1, pmc_soft_ev_register,                          \
 107             &pmc_##prov##_##mod##_##func##_##name );                            \
 108         SYSUNINIT(pmc_##prov##_##mod##_##func##_##name##_uninit,                \
 109             SI_SUB_KDTRACE, SI_ORDER_SECOND + 1, pmc_soft_ev_deregister,        \
 110             &pmc_##prov##_##mod##_##func##_##name )
 111
 112 #define PMC_SOFT_DEFINE(prov, mod, func, name)                                  \
 113         PMC_SOFT_DEFINE_EX(prov, mod, func, name, NULL, NULL)
 114
 115 #define PMC_SOFT_DECLARE(prov, mod, func, name)                                 \
 116         extern struct pmc_soft pmc_##prov##_##mod##_##func##_##name
 117
 118 /*
 119  * PMC_SOFT_CALL can be used anywhere in the kernel.
 120  * Require md defined PMC_FAKE_TRAPFRAME.
 121  */
 122 #ifdef PMC_FAKE_TRAPFRAME
 123 #define PMC_SOFT_CALL(pr, mo, fu, na)                                           \
 124 do {                                                                            \
 125         if (__predict_false(pmc_##pr##_##mo##_##fu##_##na.ps_running)) {        \
 126                 struct pmckern_soft ks;                                         \
 127                 register_t intr;                                                \
 128                 intr = intr_disable();                                          \
 129                 PMC_FAKE_TRAPFRAME(&pmc_tf[curcpu]);                            \
 130                 ks.pm_ev = pmc_##pr##_##mo##_##fu##_##na.ps_ev.pm_ev_code;      \
 131                 ks.pm_cpu = PCPU_GET(cpuid);                                    \
 132                 ks.pm_tf = &pmc_tf[curcpu];                                     \
 133                 PMC_CALL_HOOK_UNLOCKED(curthread,                               \
 134                     PMC_FN_SOFT_SAMPLING, (void *) &ks);                        \
 135                 intr_restore(intr);                                             \
 136         }                                                                       \
 137 } while (0)
 138 #else
 139 #define PMC_SOFT_CALL(pr, mo, fu, na)                                           \
 140 do {                                                                            \
 141 } while (0)
 142 #endif
 143
 144 /*
 145  * PMC_SOFT_CALL_TF need to be used carefully.
 146  * Userland capture will be done during AST processing.
 147  */
 148 #define PMC_SOFT_CALL_TF(pr, mo, fu, na, tf)                                    \
 149 do {                                                                            \
 150         if (__predict_false(pmc_##pr##_##mo##_##fu##_##na.ps_running)) {        \
 151                 struct pmckern_soft ks;                                         \
 152                 register_t intr;                                                \
 153                 intr = intr_disable();                                          \
 154                 ks.pm_ev = pmc_##pr##_##mo##_##fu##_##na.ps_ev.pm_ev_code;      \
 155                 ks.pm_cpu = PCPU_GET(cpuid);                                    \
 156                 ks.pm_tf = tf;                                                  \
 157                 PMC_CALL_HOOK_UNLOCKED(curthread,                               \
 158                     PMC_FN_SOFT_SAMPLING, (void *) &ks);                        \
 159                 intr_restore(intr);                                             \
 160         }                                                                       \
 161 } while (0)
 162
 163 struct pmc_soft {
 164         int                             ps_running;
 165         void                            (*ps_alloc)(void);
 166         void                            (*ps_release)(void);
 167         struct pmc_dyn_event_descr      ps_ev;
 168 };
 169
 170 struct pmclog_buffer;
 171
 172 struct pmc_domain_buffer_header {
 173         struct mtx pdbh_mtx;
 174         TAILQ_HEAD(, pmclog_buffer) pdbh_head;
 175         struct pmclog_buffer *pdbh_plbs;
 176         int pdbh_ncpus;
 177 } __aligned(CACHE_LINE_SIZE);
 178
 179 /* hook */
 180 extern int (*pmc_hook)(struct thread *_td, int _function, void *_arg);
 181 extern int (*pmc_intr)(struct trapframe *_frame);
 182
 183 /* SX lock protecting the hook */
 184 extern struct sx pmc_sx;
 185
 186 /* Per-cpu flags indicating availability of sampling data */
 187 DPCPU_DECLARE(uint8_t, pmc_sampled);
 188
 189 /* Count of system-wide sampling PMCs in existence */
 190 extern volatile int pmc_ss_count;
 191
 192 /* kernel version number */
 193 extern const int pmc_kernel_version;
 194
 195 /* PMC soft per cpu trapframe */
 196 extern struct trapframe pmc_tf[MAXCPU];
 197
 198 /* per domain buffer header list */
 199 extern struct pmc_domain_buffer_header *pmc_dom_hdrs[MAXMEMDOM];
 200
 201 /* Quick check if preparatory work is necessary */
 202 #define PMC_HOOK_INSTALLED(cmd) __predict_false(pmc_hook != NULL)
 203
 204 /* Hook invocation; for use within the kernel */
 205 #define PMC_CALL_HOOK(t, cmd, arg)              \
 206 do {                                                            \
 207     struct epoch_tracker et;                                            \
 208         epoch_enter_preempt(global_epoch_preempt, &et);         \
 209         if (pmc_hook != NULL)                   \
 210                 (pmc_hook)((t), (cmd), (arg));  \
 211         epoch_exit_preempt(global_epoch_preempt, &et);  \
 212 } while (0)
 213
 214 /* Hook invocation that needs an exclusive lock */
 215 #define PMC_CALL_HOOK_X(t, cmd, arg)            \
 216 do {                                            \
 217         sx_xlock(&pmc_sx);                      \
 218         if (pmc_hook != NULL)                   \
 219                 (pmc_hook)((t), (cmd), (arg));  \
 220         sx_xunlock(&pmc_sx);                    \
 221 } while (0)
 222
 223 /*
 224  * Some hook invocations (e.g., from context switch and clock handling
 225  * code) need to be lock-free.
 226  */
 227 #define PMC_CALL_HOOK_UNLOCKED(t, cmd, arg)     \
 228 do {                                            \
 229         if (pmc_hook != NULL)                   \
 230                 (pmc_hook)((t), (cmd), (arg));  \
 231 } while (0)
 232
 233 #define PMC_SWITCH_CONTEXT(t,cmd)       PMC_CALL_HOOK_UNLOCKED(t,cmd,NULL)
 234
 235 /* Check if a process is using HWPMCs.*/
 236 #define PMC_PROC_IS_USING_PMCS(p)                               \
 237         (__predict_false(p->p_flag & P_HWPMC))
 238
 239 #define PMC_THREAD_HAS_SAMPLES(td)                              \
 240         (__predict_false((td)->td_pmcpend))
 241
 242 /* Check if a thread have pending user capture. */
 243 #define PMC_IS_PENDING_CALLCHAIN(p)                             \
 244         (__predict_false((p)->td_pflags & TDP_CALLCHAIN))
 245
 246 #define PMC_SYSTEM_SAMPLING_ACTIVE()            (pmc_ss_count > 0)
 247
 248 /* Check if a CPU has recorded samples. */
 249 #define PMC_CPU_HAS_SAMPLES(C)  (__predict_false(DPCPU_ID_GET((C), pmc_sampled)))
 250
 251 /*
 252  * Helper functions.
 253  */
 254 int             pmc_cpu_is_disabled(int _cpu);  /* deprecated */
 255 int             pmc_cpu_is_active(int _cpu);
 256 int             pmc_cpu_is_present(int _cpu);
 257 int             pmc_cpu_is_primary(int _cpu);
 258 unsigned int    pmc_cpu_max(void);
 259
 260 #ifdef  INVARIANTS
 261 int             pmc_cpu_max_active(void);
 262 #endif
 263
 264 /*
 265  * Soft events functions.
 266  */
 267 void pmc_soft_ev_register(struct pmc_soft *ps);
 268 void pmc_soft_ev_deregister(struct pmc_soft *ps);
 269 struct pmc_soft *pmc_soft_ev_acquire(enum pmc_event ev);
 270 void pmc_soft_ev_release(struct pmc_soft *ps);
 271
 272 #endif /* _SYS_PMCKERN_H_ */