sys/amd64/amd64/prof_machdep.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 1996 Bruce D. Evans.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #ifdef GUPROF
  33
  34 #include <sys/param.h>
  35 #include <sys/systm.h>
  36 #include <sys/bus.h>
  37 #include <sys/cpu.h>
  38 #include <sys/eventhandler.h>
  39 #include <sys/gmon.h>
  40 #include <sys/kernel.h>
  41 #include <sys/smp.h>
  42 #include <sys/sysctl.h>
  43
  44 #include <machine/clock.h>
  45 #include <machine/timerreg.h>
  46
  47 #define CPUTIME_CLOCK_UNINITIALIZED     0
  48 #define CPUTIME_CLOCK_I8254             1
  49 #define CPUTIME_CLOCK_TSC               2
  50 #define CPUTIME_CLOCK_I8254_SHIFT       7
  51
  52 int     cputime_bias = 1;       /* initialize for locality of reference */
  53
  54 static int      cputime_clock = CPUTIME_CLOCK_UNINITIALIZED;
  55 static int      cputime_prof_active;
  56 #endif /* GUPROF */
  57
  58 #ifdef __GNUCLIKE_ASM
  59 #if defined(SMP) && defined(GUPROF)
  60 #define MPLOCK "                                                \n\
  61         movl    $1,%edx                                         \n\
  62 9:                                                              \n\
  63         xorl    %eax,%eax                                       \n\
  64         lock                                                    \n\
  65         cmpxchgl %edx,mcount_lock                               \n\
  66         jne     9b                                              \n"
  67 #define MPUNLOCK "movl  $0,mcount_lock                          \n"
  68 #else /* !(SMP && GUPROF) */
  69 #define MPLOCK
  70 #define MPUNLOCK
  71 #endif /* SMP && GUPROF */
  72
  73 __asm("                                                         \n\
  74 GM_STATE        =       0                                       \n\
  75 GMON_PROF_OFF   =       3                                       \n\
  76                                                                 \n\
  77         .text                                                   \n\
  78         .p2align 4,0x90                                         \n\
  79         .globl  __mcount                                        \n\
  80         .type   __mcount,@function                              \n\
  81 __mcount:                                                       \n\
  82         #                                                       \n\
  83         # Check that we are profiling.  Do it early for speed.  \n\
  84         #                                                       \n\
  85         cmpl    $GMON_PROF_OFF,_gmonparam+GM_STATE              \n\
  86         je      .mcount_exit                                    \n\
  87         #                                                       \n\
  88         # __mcount is the same as [.]mcount except the caller   \n\
  89         # hasn't changed the stack except to call here, so the  \n\
  90         # caller's raddr is above our raddr.                    \n\
  91         #                                                       \n\
  92         pushq   %rax                                            \n\
  93         pushq   %rdx                                            \n\
  94         pushq   %rcx                                            \n\
  95         pushq   %rsi                                            \n\
  96         pushq   %rdi                                            \n\
  97         pushq   %r8                                             \n\
  98         pushq   %r9                                             \n\
  99         movq    7*8+8(%rsp),%rdi                                \n\
 100         jmp     .got_frompc                                     \n\
 101                                                                 \n\
 102         .p2align 4,0x90                                         \n\
 103         .globl  .mcount                                         \n\
 104 .mcount:                                                        \n\
 105         cmpl    $GMON_PROF_OFF,_gmonparam+GM_STATE              \n\
 106         je      .mcount_exit                                    \n\
 107         #                                                       \n\
 108         # The caller's stack frame has already been built, so   \n\
 109         # %rbp is the caller's frame pointer.  The caller's     \n\
 110         # raddr is in the caller's frame following the caller's \n\
 111         # caller's frame pointer.                               \n\
 112         #                                                       \n\
 113         pushq   %rax                                            \n\
 114         pushq   %rdx                                            \n\
 115         pushq   %rcx                                            \n\
 116         pushq   %rsi                                            \n\
 117         pushq   %rdi                                            \n\
 118         pushq   %r8                                             \n\
 119         pushq   %r9                                             \n\
 120         movq    8(%rbp),%rdi                                    \n\
 121 .got_frompc:                                                    \n\
 122         #                                                       \n\
 123         # Our raddr is the caller's pc.                         \n\
 124         #                                                       \n\
 125         movq    7*8(%rsp),%rsi                                  \n\
 126                                                                 \n\
 127         pushfq                                                  \n\
 128         cli                                                     \n"
 129         MPLOCK "                                                \n\
 130         call    mcount                                          \n"
 131         MPUNLOCK "                                              \n\
 132         popfq                                                   \n\
 133         popq    %r9                                             \n\
 134         popq    %r8                                             \n\
 135         popq    %rdi                                            \n\
 136         popq    %rsi                                            \n\
 137         popq    %rcx                                            \n\
 138         popq    %rdx                                            \n\
 139         popq    %rax                                            \n\
 140 .mcount_exit:                                                   \n\
 141         ret     $0                                              \n\
 142 ");
 143 #else /* !__GNUCLIKE_ASM */
 144 #error "this file needs to be ported to your compiler"
 145 #endif /* __GNUCLIKE_ASM */
 146
 147 #ifdef GUPROF
 148 /*
 149  * [.]mexitcount saves the return register(s), loads selfpc and calls
 150  * mexitcount(selfpc) to do the work.  Someday it should be in a machine
 151  * dependent file together with cputime(), __mcount and [.]mcount.  cputime()
 152  * can't just be put in machdep.c because it has to be compiled without -pg.
 153  */
 154 #ifdef __GNUCLIKE_ASM
 155 __asm("                                                         \n\
 156         .text                                                   \n\
 157 #                                                               \n\
 158 # Dummy label to be seen when gprof -u hides [.]mexitcount.     \n\
 159 #                                                               \n\
 160         .p2align 4,0x90                                         \n\
 161         .globl  __mexitcount                                    \n\
 162         .type   __mexitcount,@function                          \n\
 163 __mexitcount:                                                   \n\
 164         nop                                                     \n\
 165                                                                 \n\
 166 GMON_PROF_HIRES =       4                                       \n\
 167                                                                 \n\
 168         .p2align 4,0x90                                         \n\
 169         .globl  .mexitcount                                     \n\
 170 .mexitcount:                                                    \n\
 171         cmpl    $GMON_PROF_HIRES,_gmonparam+GM_STATE            \n\
 172         jne     .mexitcount_exit                                \n\
 173         pushq   %rax                                            \n\
 174         pushq   %rdx                                            \n\
 175         pushq   %rcx                                            \n\
 176         pushq   %rsi                                            \n\
 177         pushq   %rdi                                            \n\
 178         pushq   %r8                                             \n\
 179         pushq   %r9                                             \n\
 180         movq    7*8(%rsp),%rdi                                  \n\
 181         pushfq                                                  \n\
 182         cli                                                     \n"
 183         MPLOCK "                                                \n\
 184         call    mexitcount                                      \n"
 185         MPUNLOCK "                                              \n\
 186         popfq                                                   \n\
 187         popq    %r9                                             \n\
 188         popq    %r8                                             \n\
 189         popq    %rdi                                            \n\
 190         popq    %rsi                                            \n\
 191         popq    %rcx                                            \n\
 192         popq    %rdx                                            \n\
 193         popq    %rax                                            \n\
 194 .mexitcount_exit:                                               \n\
 195         ret     $0                                              \n\
 196 ");
 197 #endif /* __GNUCLIKE_ASM */
 198
 199 /*
 200  * Return the time elapsed since the last call.  The units are machine-
 201  * dependent.
 202  */
 203 int
 204 cputime()
 205 {
 206         u_int count;
 207         int delta;
 208         u_char high, low;
 209         static u_int prev_count;
 210
 211         if (cputime_clock == CPUTIME_CLOCK_TSC) {
 212                 /*
 213                  * Scale the TSC a little to make cputime()'s frequency
 214                  * fit in an int, assuming that the TSC frequency fits
 215                  * in a u_int.  Use a fixed scale since dynamic scaling
 216                  * would be slower and we can't really use the low bit
 217                  * of precision.
 218                  */
 219                 count = (u_int)rdtsc() & ~1u;
 220                 delta = (int)(count - prev_count) >> 1;
 221                 prev_count = count;
 222                 return (delta);
 223         }
 224
 225         /*
 226          * Read the current value of the 8254 timer counter 0.
 227          */
 228         outb(TIMER_MODE, TIMER_SEL0 | TIMER_LATCH);
 229         low = inb(TIMER_CNTR0);
 230         high = inb(TIMER_CNTR0);
 231         count = ((high << 8) | low) << CPUTIME_CLOCK_I8254_SHIFT;
 232
 233         /*
 234          * The timer counts down from TIMER_CNTR0_MAX to 0 and then resets.
 235          * While profiling is enabled, this routine is called at least twice
 236          * per timer reset (for mcounting and mexitcounting hardclock()),
 237          * so at most one reset has occurred since the last call, and one
 238          * has occurred iff the current count is larger than the previous
 239          * count.  This allows counter underflow to be detected faster
 240          * than in microtime().
 241          */
 242         delta = prev_count - count;
 243         prev_count = count;
 244         if ((int) delta <= 0)
 245                 return (delta + (i8254_max_count << CPUTIME_CLOCK_I8254_SHIFT));
 246         return (delta);
 247 }
 248
 249 static int
 250 sysctl_machdep_cputime_clock(SYSCTL_HANDLER_ARGS)
 251 {
 252         int clock;
 253         int error;
 254
 255         clock = cputime_clock;
 256         error = sysctl_handle_opaque(oidp, &clock, sizeof clock, req);
 257         if (error == 0 && req->newptr != NULL) {
 258                 if (clock < 0 || clock > CPUTIME_CLOCK_TSC)
 259                         return (EINVAL);
 260                 cputime_clock = clock;
 261         }
 262         return (error);
 263 }
 264
 265 SYSCTL_PROC(_machdep, OID_AUTO, cputime_clock,
 266     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 0, sizeof(u_int),
 267     sysctl_machdep_cputime_clock, "I",
 268     "");
 269
 270 /*
 271  * The start and stop routines need not be here since we turn off profiling
 272  * before calling them.  They are here for convenience.
 273  */
 274
 275 void
 276 startguprof(gp)
 277         struct gmonparam *gp;
 278 {
 279         uint64_t freq;
 280
 281         freq = atomic_load_acq_64(&tsc_freq);
 282         if (cputime_clock == CPUTIME_CLOCK_UNINITIALIZED) {
 283                 if (freq != 0 && mp_ncpus == 1)
 284                         cputime_clock = CPUTIME_CLOCK_TSC;
 285                 else
 286                         cputime_clock = CPUTIME_CLOCK_I8254;
 287         }
 288         if (cputime_clock == CPUTIME_CLOCK_TSC) {
 289                 gp->profrate = freq >> 1;
 290                 cputime_prof_active = 1;
 291         } else
 292                 gp->profrate = i8254_freq << CPUTIME_CLOCK_I8254_SHIFT;
 293         cputime_bias = 0;
 294         cputime();
 295 }
 296
 297 void
 298 stopguprof(gp)
 299         struct gmonparam *gp;
 300 {
 301         if (cputime_clock == CPUTIME_CLOCK_TSC)
 302                 cputime_prof_active = 0;
 303 }
 304
 305 /* If the cpu frequency changed while profiling, report a warning. */
 306 static void
 307 tsc_freq_changed(void *arg, const struct cf_level *level, int status)
 308 {
 309
 310         /*
 311          * If there was an error during the transition or
 312          * TSC is P-state invariant, don't do anything.
 313          */
 314         if (status != 0 || tsc_is_invariant)
 315                 return;
 316         if (cputime_prof_active && cputime_clock == CPUTIME_CLOCK_TSC)
 317                 printf("warning: cpu freq changed while profiling active\n");
 318 }
 319
 320 EVENTHANDLER_DEFINE(cpufreq_post_change, tsc_freq_changed, NULL,
 321     EVENTHANDLER_PRI_ANY);
 322
 323 #endif /* GUPROF */