sys/x86/x86/tsc.c

   1 /*-
   2  * Copyright (c) 1998-2003 Poul-Henning Kamp
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #include <sys/cdefs.h>
  28 __FBSDID("$FreeBSD$");
  29
  30 #include "opt_compat.h"
  31 #include "opt_clock.h"
  32
  33 #include <sys/param.h>
  34 #include <sys/bus.h>
  35 #include <sys/cpu.h>
  36 #include <sys/limits.h>
  37 #include <sys/malloc.h>
  38 #include <sys/systm.h>
  39 #include <sys/sysctl.h>
  40 #include <sys/time.h>
  41 #include <sys/timetc.h>
  42 #include <sys/kernel.h>
  43 #include <sys/power.h>
  44 #include <sys/smp.h>
  45 #include <sys/vdso.h>
  46 #include <machine/clock.h>
  47 #include <machine/cputypes.h>
  48 #include <machine/md_var.h>
  49 #include <machine/specialreg.h>
  50 #include <x86/vmware.h>
  51
  52 #include "cpufreq_if.h"
  53
  54 uint64_t        tsc_freq;
  55 int             tsc_is_invariant;
  56 int             tsc_perf_stat;
  57
  58 static eventhandler_tag tsc_levels_tag, tsc_pre_tag, tsc_post_tag;
  59
  60 SYSCTL_INT(_kern_timecounter, OID_AUTO, invariant_tsc, CTLFLAG_RDTUN,
  61     &tsc_is_invariant, 0, "Indicates whether the TSC is P-state invariant");
  62 TUNABLE_INT("kern.timecounter.invariant_tsc", &tsc_is_invariant);
  63
  64 #ifdef SMP
  65 int     smp_tsc;
  66 SYSCTL_INT(_kern_timecounter, OID_AUTO, smp_tsc, CTLFLAG_RDTUN, &smp_tsc, 0,
  67     "Indicates whether the TSC is safe to use in SMP mode");
  68 TUNABLE_INT("kern.timecounter.smp_tsc", &smp_tsc);
  69
  70 int     smp_tsc_adjust = 0;
  71 SYSCTL_INT(_kern_timecounter, OID_AUTO, smp_tsc_adjust, CTLFLAG_RDTUN,
  72     &smp_tsc_adjust, 0, "Try to adjust TSC on APs to match BSP");
  73 TUNABLE_INT("kern.timecounter.smp_tsc_adjust", &smp_tsc_adjust);
  74 #endif
  75
  76 static int      tsc_shift = 1;
  77 SYSCTL_INT(_kern_timecounter, OID_AUTO, tsc_shift, CTLFLAG_RDTUN,
  78     &tsc_shift, 0, "Shift to pre-apply for the maximum TSC frequency");
  79 TUNABLE_INT("kern.timecounter.tsc_shift", &tsc_shift);
  80
  81 static int      tsc_disabled;
  82 SYSCTL_INT(_machdep, OID_AUTO, disable_tsc, CTLFLAG_RDTUN, &tsc_disabled, 0,
  83     "Disable x86 Time Stamp Counter");
  84 TUNABLE_INT("machdep.disable_tsc", &tsc_disabled);
  85
  86 static int      tsc_skip_calibration;
  87 SYSCTL_INT(_machdep, OID_AUTO, disable_tsc_calibration, CTLFLAG_RDTUN,
  88     &tsc_skip_calibration, 0, "Disable TSC frequency calibration");
  89 TUNABLE_INT("machdep.disable_tsc_calibration", &tsc_skip_calibration);
  90
  91 static void tsc_freq_changed(void *arg, const struct cf_level *level,
  92     int status);
  93 static void tsc_freq_changing(void *arg, const struct cf_level *level,
  94     int *status);
  95 static unsigned tsc_get_timecount(struct timecounter *tc);
  96 static inline unsigned tsc_get_timecount_low(struct timecounter *tc);
  97 static unsigned tsc_get_timecount_lfence(struct timecounter *tc);
  98 static unsigned tsc_get_timecount_low_lfence(struct timecounter *tc);
  99 static unsigned tsc_get_timecount_mfence(struct timecounter *tc);
 100 static unsigned tsc_get_timecount_low_mfence(struct timecounter *tc);
 101 static void tsc_levels_changed(void *arg, int unit);
 102
 103 static struct timecounter tsc_timecounter = {
 104         tsc_get_timecount,      /* get_timecount */
 105         0,                      /* no poll_pps */
 106         ~0u,                    /* counter_mask */
 107         0,                      /* frequency */
 108         "TSC",                  /* name */
 109         800,                    /* quality (adjusted in code) */
 110 };
 111
 112 static void
 113 tsc_freq_vmware(void)
 114 {
 115         u_int regs[4];
 116
 117         if (hv_high >= 0x40000010) {
 118                 do_cpuid(0x40000010, regs);
 119                 tsc_freq = regs[0] * 1000;
 120         } else {
 121                 vmware_hvcall(VMW_HVCMD_GETHZ, regs);
 122                 if (regs[1] != UINT_MAX)
 123                         tsc_freq = regs[0] | ((uint64_t)regs[1] << 32);
 124         }
 125         tsc_is_invariant = 1;
 126 }
 127
 128 static void
 129 tsc_freq_intel(void)
 130 {
 131         char brand[48];
 132         u_int regs[4];
 133         uint64_t freq;
 134         char *p;
 135         u_int i;
 136
 137         /*
 138          * Intel Processor Identification and the CPUID Instruction
 139          * Application Note 485.
 140          * http://www.intel.com/assets/pdf/appnote/241618.pdf
 141          */
 142         if (cpu_exthigh >= 0x80000004) {
 143                 p = brand;
 144                 for (i = 0x80000002; i < 0x80000005; i++) {
 145                         do_cpuid(i, regs);
 146                         memcpy(p, regs, sizeof(regs));
 147                         p += sizeof(regs);
 148                 }
 149                 p = NULL;
 150                 for (i = 0; i < sizeof(brand) - 1; i++)
 151                         if (brand[i] == 'H' && brand[i + 1] == 'z')
 152                                 p = brand + i;
 153                 if (p != NULL) {
 154                         p -= 5;
 155                         switch (p[4]) {
 156                         case 'M':
 157                                 i = 1;
 158                                 break;
 159                         case 'G':
 160                                 i = 1000;
 161                                 break;
 162                         case 'T':
 163                                 i = 1000000;
 164                                 break;
 165                         default:
 166                                 return;
 167                         }
 168 #define C2D(c)  ((c) - '0')
 169                         if (p[1] == '.') {
 170                                 freq = C2D(p[0]) * 1000;
 171                                 freq += C2D(p[2]) * 100;
 172                                 freq += C2D(p[3]) * 10;
 173                                 freq *= i * 1000;
 174                         } else {
 175                                 freq = C2D(p[0]) * 1000;
 176                                 freq += C2D(p[1]) * 100;
 177                                 freq += C2D(p[2]) * 10;
 178                                 freq += C2D(p[3]);
 179                                 freq *= i * 1000000;
 180                         }
 181 #undef C2D
 182                         tsc_freq = freq;
 183                 }
 184         }
 185 }
 186
 187 static void
 188 probe_tsc_freq(void)
 189 {
 190         u_int regs[4];
 191         uint64_t tsc1, tsc2;
 192
 193         if (cpu_high >= 6) {
 194                 do_cpuid(6, regs);
 195                 if ((regs[2] & CPUID_PERF_STAT) != 0) {
 196                         /*
 197                          * XXX Some emulators expose host CPUID without actual
 198                          * support for these MSRs.  We must test whether they
 199                          * really work.
 200                          */
 201                         wrmsr(MSR_MPERF, 0);
 202                         wrmsr(MSR_APERF, 0);
 203                         DELAY(10);
 204                         if (rdmsr(MSR_MPERF) > 0 && rdmsr(MSR_APERF) > 0)
 205                                 tsc_perf_stat = 1;
 206                 }
 207         }
 208
 209         if (vm_guest == VM_GUEST_VMWARE) {
 210                 tsc_freq_vmware();
 211                 return;
 212         }
 213
 214         switch (cpu_vendor_id) {
 215         case CPU_VENDOR_AMD:
 216                 if ((amd_pminfo & AMDPM_TSC_INVARIANT) != 0 ||
 217                     (vm_guest == VM_GUEST_NO &&
 218                     CPUID_TO_FAMILY(cpu_id) >= 0x10))
 219                         tsc_is_invariant = 1;
 220                 if (cpu_feature & CPUID_SSE2) {
 221                         tsc_timecounter.tc_get_timecount =
 222                             tsc_get_timecount_mfence;
 223                 }
 224                 break;
 225         case CPU_VENDOR_INTEL:
 226                 if ((amd_pminfo & AMDPM_TSC_INVARIANT) != 0 ||
 227                     (vm_guest == VM_GUEST_NO &&
 228                     ((CPUID_TO_FAMILY(cpu_id) == 0x6 &&
 229                     CPUID_TO_MODEL(cpu_id) >= 0xe) ||
 230                     (CPUID_TO_FAMILY(cpu_id) == 0xf &&
 231                     CPUID_TO_MODEL(cpu_id) >= 0x3))))
 232                         tsc_is_invariant = 1;
 233                 if (cpu_feature & CPUID_SSE2) {
 234                         tsc_timecounter.tc_get_timecount =
 235                             tsc_get_timecount_lfence;
 236                 }
 237                 break;
 238         case CPU_VENDOR_CENTAUR:
 239                 if (vm_guest == VM_GUEST_NO &&
 240                     CPUID_TO_FAMILY(cpu_id) == 0x6 &&
 241                     CPUID_TO_MODEL(cpu_id) >= 0xf &&
 242                     (rdmsr(0x1203) & 0x100000000ULL) == 0)
 243                         tsc_is_invariant = 1;
 244                 if (cpu_feature & CPUID_SSE2) {
 245                         tsc_timecounter.tc_get_timecount =
 246                             tsc_get_timecount_lfence;
 247                 }
 248                 break;
 249         }
 250
 251         if (tsc_skip_calibration) {
 252                 if (cpu_vendor_id == CPU_VENDOR_INTEL)
 253                         tsc_freq_intel();
 254                 return;
 255         }
 256
 257         if (bootverbose)
 258                 printf("Calibrating TSC clock ... ");
 259         tsc1 = rdtsc();
 260         DELAY(1000000);
 261         tsc2 = rdtsc();
 262         tsc_freq = tsc2 - tsc1;
 263         if (bootverbose)
 264                 printf("TSC clock: %ju Hz\n", (intmax_t)tsc_freq);
 265 }
 266
 267 void
 268 init_TSC(void)
 269 {
 270
 271         if ((cpu_feature & CPUID_TSC) == 0 || tsc_disabled)
 272                 return;
 273
 274 #ifdef __i386__
 275         /* The TSC is known to be broken on certain CPUs. */
 276         switch (cpu_vendor_id) {
 277         case CPU_VENDOR_AMD:
 278                 switch (cpu_id & 0xFF0) {
 279                 case 0x500:
 280                         /* K5 Model 0 */
 281                         return;
 282                 }
 283                 break;
 284         case CPU_VENDOR_CENTAUR:
 285                 switch (cpu_id & 0xff0) {
 286                 case 0x540:
 287                         /*
 288                          * http://www.centtech.com/c6_data_sheet.pdf
 289                          *
 290                          * I-12 RDTSC may return incoherent values in EDX:EAX
 291                          * I-13 RDTSC hangs when certain event counters are used
 292                          */
 293                         return;
 294                 }
 295                 break;
 296         case CPU_VENDOR_NSC:
 297                 switch (cpu_id & 0xff0) {
 298                 case 0x540:
 299                         if ((cpu_id & CPUID_STEPPING) == 0)
 300                                 return;
 301                         break;
 302                 }
 303                 break;
 304         }
 305 #endif
 306
 307         probe_tsc_freq();
 308
 309         /*
 310          * Inform CPU accounting about our boot-time clock rate.  This will
 311          * be updated if someone loads a cpufreq driver after boot that
 312          * discovers a new max frequency.
 313          */
 314         if (tsc_freq != 0)
 315                 set_cputicker(rdtsc, tsc_freq, !tsc_is_invariant);
 316
 317         if (tsc_is_invariant)
 318                 return;
 319
 320         /* Register to find out about changes in CPU frequency. */
 321         tsc_pre_tag = EVENTHANDLER_REGISTER(cpufreq_pre_change,
 322             tsc_freq_changing, NULL, EVENTHANDLER_PRI_FIRST);
 323         tsc_post_tag = EVENTHANDLER_REGISTER(cpufreq_post_change,
 324             tsc_freq_changed, NULL, EVENTHANDLER_PRI_FIRST);
 325         tsc_levels_tag = EVENTHANDLER_REGISTER(cpufreq_levels_changed,
 326             tsc_levels_changed, NULL, EVENTHANDLER_PRI_ANY);
 327 }
 328
 329 #ifdef SMP
 330
 331 /*
 332  * RDTSC is not a serializing instruction, and does not drain
 333  * instruction stream, so we need to drain the stream before executing
 334  * it.  It could be fixed by use of RDTSCP, except the instruction is
 335  * not available everywhere.
 336  *
 337  * Use CPUID for draining in the boot-time SMP constistency test.  The
 338  * timecounters use MFENCE for AMD CPUs, and LFENCE for others (Intel
 339  * and VIA) when SSE2 is present, and nothing on older machines which
 340  * also do not issue RDTSC prematurely.  There, testing for SSE2 and
 341  * vendor is too cumbersome, and we learn about TSC presence from CPUID.
 342  *
 343  * Do not use do_cpuid(), since we do not need CPUID results, which
 344  * have to be written into memory with do_cpuid().
 345  */
 346 #define TSC_READ(x)                                                     \
 347 static void                                                             \
 348 tsc_read_##x(void *arg)                                                 \
 349 {                                                                       \
 350         uint64_t *tsc = arg;                                            \
 351         u_int cpu = PCPU_GET(cpuid);                                    \
 352                                                                         \
 353         __asm __volatile("cpuid" : : : "eax", "ebx", "ecx", "edx");     \
 354         tsc[cpu * 3 + x] = rdtsc();                                     \
 355 }
 356 TSC_READ(0)
 357 TSC_READ(1)
 358 TSC_READ(2)
 359 #undef TSC_READ
 360
 361 #define N       1000
 362
 363 static void
 364 comp_smp_tsc(void *arg)
 365 {
 366         uint64_t *tsc;
 367         int64_t d1, d2;
 368         u_int cpu = PCPU_GET(cpuid);
 369         u_int i, j, size;
 370
 371         size = (mp_maxid + 1) * 3;
 372         for (i = 0, tsc = arg; i < N; i++, tsc += size)
 373                 CPU_FOREACH(j) {
 374                         if (j == cpu)
 375                                 continue;
 376                         d1 = tsc[cpu * 3 + 1] - tsc[j * 3];
 377                         d2 = tsc[cpu * 3 + 2] - tsc[j * 3 + 1];
 378                         if (d1 <= 0 || d2 <= 0) {
 379                                 smp_tsc = 0;
 380                                 return;
 381                         }
 382                 }
 383 }
 384
 385 static void
 386 adj_smp_tsc(void *arg)
 387 {
 388         uint64_t *tsc;
 389         int64_t d, min, max;
 390         u_int cpu = PCPU_GET(cpuid);
 391         u_int first, i, size;
 392
 393         first = CPU_FIRST();
 394         if (cpu == first)
 395                 return;
 396         min = INT64_MIN;
 397         max = INT64_MAX;
 398         size = (mp_maxid + 1) * 3;
 399         for (i = 0, tsc = arg; i < N; i++, tsc += size) {
 400                 d = tsc[first * 3] - tsc[cpu * 3 + 1];
 401                 if (d > min)
 402                         min = d;
 403                 d = tsc[first * 3 + 1] - tsc[cpu * 3 + 2];
 404                 if (d > min)
 405                         min = d;
 406                 d = tsc[first * 3 + 1] - tsc[cpu * 3];
 407                 if (d < max)
 408                         max = d;
 409                 d = tsc[first * 3 + 2] - tsc[cpu * 3 + 1];
 410                 if (d < max)
 411                         max = d;
 412         }
 413         if (min > max)
 414                 return;
 415         d = min / 2 + max / 2;
 416         __asm __volatile (
 417                 "movl $0x10, %%ecx\n\t"
 418                 "rdmsr\n\t"
 419                 "addl %%edi, %%eax\n\t"
 420                 "adcl %%esi, %%edx\n\t"
 421                 "wrmsr\n"
 422                 : /* No output */
 423                 : "D" ((uint32_t)d), "S" ((uint32_t)(d >> 32))
 424                 : "ax", "cx", "dx", "cc"
 425         );
 426 }
 427
 428 static int
 429 test_tsc(void)
 430 {
 431         uint64_t *data, *tsc;
 432         u_int i, size, adj;
 433
 434         if ((!smp_tsc && !tsc_is_invariant) || vm_guest)
 435                 return (-100);
 436         size = (mp_maxid + 1) * 3;
 437         data = malloc(sizeof(*data) * size * N, M_TEMP, M_WAITOK);
 438         adj = 0;
 439 retry:
 440         for (i = 0, tsc = data; i < N; i++, tsc += size)
 441                 smp_rendezvous(tsc_read_0, tsc_read_1, tsc_read_2, tsc);
 442         smp_tsc = 1;    /* XXX */
 443         smp_rendezvous(smp_no_rendevous_barrier, comp_smp_tsc,
 444             smp_no_rendevous_barrier, data);
 445         if (!smp_tsc && adj < smp_tsc_adjust) {
 446                 adj++;
 447                 smp_rendezvous(smp_no_rendevous_barrier, adj_smp_tsc,
 448                     smp_no_rendevous_barrier, data);
 449                 goto retry;
 450         }
 451         free(data, M_TEMP);
 452         if (bootverbose)
 453                 printf("SMP: %sed TSC synchronization test%s\n",
 454                     smp_tsc ? "pass" : "fail",
 455                     adj > 0 ? " after adjustment" : "");
 456         if (smp_tsc && tsc_is_invariant) {
 457                 switch (cpu_vendor_id) {
 458                 case CPU_VENDOR_AMD:
 459                         /*
 460                          * Starting with Family 15h processors, TSC clock
 461                          * source is in the north bridge.  Check whether
 462                          * we have a single-socket/multi-core platform.
 463                          * XXX Need more work for complex cases.
 464                          */
 465                         if (CPUID_TO_FAMILY(cpu_id) < 0x15 ||
 466                             (amd_feature2 & AMDID2_CMP) == 0 ||
 467                             smp_cpus > (cpu_procinfo2 & AMDID_CMP_CORES) + 1)
 468                                 break;
 469                         return (1000);
 470                 case CPU_VENDOR_INTEL:
 471                         /*
 472                          * XXX Assume Intel platforms have synchronized TSCs.
 473                          */
 474                         return (1000);
 475                 }
 476                 return (800);
 477         }
 478         return (-100);
 479 }
 480
 481 #undef N
 482
 483 #else
 484
 485 /*
 486  * The function is not called, it is provided to avoid linking failure
 487  * on uniprocessor kernel.
 488  */
 489 static int
 490 test_tsc(void)
 491 {
 492
 493         return (0);
 494 }
 495
 496 #endif /* SMP */
 497
 498 static void
 499 init_TSC_tc(void)
 500 {
 501         uint64_t max_freq;
 502         int shift;
 503
 504         if ((cpu_feature & CPUID_TSC) == 0 || tsc_disabled)
 505                 return;
 506
 507         /*
 508          * Limit timecounter frequency to fit in an int and prevent it from
 509          * overflowing too fast.
 510          */
 511         max_freq = UINT_MAX;
 512
 513         /*
 514          * We can not use the TSC if we support APM.  Precise timekeeping
 515          * on an APM'ed machine is at best a fools pursuit, since
 516          * any and all of the time spent in various SMM code can't
 517          * be reliably accounted for.  Reading the RTC is your only
 518          * source of reliable time info.  The i8254 loses too, of course,
 519          * but we need to have some kind of time...
 520          * We don't know at this point whether APM is going to be used
 521          * or not, nor when it might be activated.  Play it safe.
 522          */
 523         if (power_pm_get_type() == POWER_PM_TYPE_APM) {
 524                 tsc_timecounter.tc_quality = -1000;
 525                 if (bootverbose)
 526                         printf("TSC timecounter disabled: APM enabled.\n");
 527                 goto init;
 528         }
 529
 530         /*
 531          * We cannot use the TSC if it stops incrementing while idle.
 532          * Intel CPUs without a C-state invariant TSC can stop the TSC
 533          * in either C2 or C3.
 534          */
 535         if (cpu_deepest_sleep >= 2 && cpu_vendor_id == CPU_VENDOR_INTEL &&
 536             (amd_pminfo & AMDPM_TSC_INVARIANT) == 0) {
 537                 tsc_timecounter.tc_quality = -1000;
 538                 tsc_timecounter.tc_flags |= TC_FLAGS_C2STOP;
 539                 if (bootverbose)
 540                         printf("TSC timecounter disabled: C2/C3 may halt it.\n");
 541                 goto init;
 542         }
 543
 544         /*
 545          * We can not use the TSC in SMP mode unless the TSCs on all CPUs
 546          * are synchronized.  If the user is sure that the system has
 547          * synchronized TSCs, set kern.timecounter.smp_tsc tunable to a
 548          * non-zero value.  The TSC seems unreliable in virtualized SMP
 549          * environments, so it is set to a negative quality in those cases.
 550          */
 551         if (mp_ncpus > 1)
 552                 tsc_timecounter.tc_quality = test_tsc();
 553         else if (tsc_is_invariant)
 554                 tsc_timecounter.tc_quality = 1000;
 555         max_freq >>= tsc_shift;
 556
 557 init:
 558         for (shift = 0; shift <= 31 && (tsc_freq >> shift) > max_freq; shift++)
 559                 ;
 560         if ((cpu_feature & CPUID_SSE2) != 0 && mp_ncpus > 1) {
 561                 if (cpu_vendor_id == CPU_VENDOR_AMD) {
 562                         tsc_timecounter.tc_get_timecount = shift > 0 ?
 563                             tsc_get_timecount_low_mfence :
 564                             tsc_get_timecount_mfence;
 565                 } else {
 566                         tsc_timecounter.tc_get_timecount = shift > 0 ?
 567                             tsc_get_timecount_low_lfence :
 568                             tsc_get_timecount_lfence;
 569                 }
 570         } else {
 571                 tsc_timecounter.tc_get_timecount = shift > 0 ?
 572                     tsc_get_timecount_low : tsc_get_timecount;
 573         }
 574         if (shift > 0) {
 575                 tsc_timecounter.tc_name = "TSC-low";
 576                 if (bootverbose)
 577                         printf("TSC timecounter discards lower %d bit(s)\n",
 578                             shift);
 579         }
 580         if (tsc_freq != 0) {
 581                 tsc_timecounter.tc_frequency = tsc_freq >> shift;
 582                 tsc_timecounter.tc_priv = (void *)(intptr_t)shift;
 583                 tc_init(&tsc_timecounter);
 584         }
 585 }
 586 SYSINIT(tsc_tc, SI_SUB_SMP, SI_ORDER_ANY, init_TSC_tc, NULL);
 587
 588 /*
 589  * When cpufreq levels change, find out about the (new) max frequency.  We
 590  * use this to update CPU accounting in case it got a lower estimate at boot.
 591  */
 592 static void
 593 tsc_levels_changed(void *arg, int unit)
 594 {
 595         device_t cf_dev;
 596         struct cf_level *levels;
 597         int count, error;
 598         uint64_t max_freq;
 599
 600         /* Only use values from the first CPU, assuming all are equal. */
 601         if (unit != 0)
 602                 return;
 603
 604         /* Find the appropriate cpufreq device instance. */
 605         cf_dev = devclass_get_device(devclass_find("cpufreq"), unit);
 606         if (cf_dev == NULL) {
 607                 printf("tsc_levels_changed() called but no cpufreq device?\n");
 608                 return;
 609         }
 610
 611         /* Get settings from the device and find the max frequency. */
 612         count = 64;
 613         levels = malloc(count * sizeof(*levels), M_TEMP, M_NOWAIT);
 614         if (levels == NULL)
 615                 return;
 616         error = CPUFREQ_LEVELS(cf_dev, levels, &count);
 617         if (error == 0 && count != 0) {
 618                 max_freq = (uint64_t)levels[0].total_set.freq * 1000000;
 619                 set_cputicker(rdtsc, max_freq, 1);
 620         } else
 621                 printf("tsc_levels_changed: no max freq found\n");
 622         free(levels, M_TEMP);
 623 }
 624
 625 /*
 626  * If the TSC timecounter is in use, veto the pending change.  It may be
 627  * possible in the future to handle a dynamically-changing timecounter rate.
 628  */
 629 static void
 630 tsc_freq_changing(void *arg, const struct cf_level *level, int *status)
 631 {
 632
 633         if (*status != 0 || timecounter != &tsc_timecounter)
 634                 return;
 635
 636         printf("timecounter TSC must not be in use when "
 637             "changing frequencies; change denied\n");
 638         *status = EBUSY;
 639 }
 640
 641 /* Update TSC freq with the value indicated by the caller. */
 642 static void
 643 tsc_freq_changed(void *arg, const struct cf_level *level, int status)
 644 {
 645         uint64_t freq;
 646
 647         /* If there was an error during the transition, don't do anything. */
 648         if (tsc_disabled || status != 0)
 649                 return;
 650
 651         /* Total setting for this level gives the new frequency in MHz. */
 652         freq = (uint64_t)level->total_set.freq * 1000000;
 653         atomic_store_rel_64(&tsc_freq, freq);
 654         tsc_timecounter.tc_frequency =
 655             freq >> (int)(intptr_t)tsc_timecounter.tc_priv;
 656 }
 657
 658 static int
 659 sysctl_machdep_tsc_freq(SYSCTL_HANDLER_ARGS)
 660 {
 661         int error;
 662         uint64_t freq;
 663
 664         freq = atomic_load_acq_64(&tsc_freq);
 665         if (freq == 0)
 666                 return (EOPNOTSUPP);
 667         error = sysctl_handle_64(oidp, &freq, 0, req);
 668         if (error == 0 && req->newptr != NULL) {
 669                 atomic_store_rel_64(&tsc_freq, freq);
 670                 atomic_store_rel_64(&tsc_timecounter.tc_frequency,
 671                     freq >> (int)(intptr_t)tsc_timecounter.tc_priv);
 672         }
 673         return (error);
 674 }
 675
 676 SYSCTL_PROC(_machdep, OID_AUTO, tsc_freq, CTLTYPE_U64 | CTLFLAG_RW,
 677     0, 0, sysctl_machdep_tsc_freq, "QU", "Time Stamp Counter frequency");
 678
 679 static u_int
 680 tsc_get_timecount(struct timecounter *tc __unused)
 681 {
 682
 683         return (rdtsc32());
 684 }
 685
 686 static inline u_int
 687 tsc_get_timecount_low(struct timecounter *tc)
 688 {
 689         uint32_t rv;
 690
 691         __asm __volatile("rdtsc; shrd %%cl, %%edx, %0"
 692             : "=a" (rv) : "c" ((int)(intptr_t)tc->tc_priv) : "edx");
 693         return (rv);
 694 }
 695
 696 static u_int
 697 tsc_get_timecount_lfence(struct timecounter *tc __unused)
 698 {
 699
 700         lfence();
 701         return (rdtsc32());
 702 }
 703
 704 static u_int
 705 tsc_get_timecount_low_lfence(struct timecounter *tc)
 706 {
 707
 708         lfence();
 709         return (tsc_get_timecount_low(tc));
 710 }
 711
 712 static u_int
 713 tsc_get_timecount_mfence(struct timecounter *tc __unused)
 714 {
 715
 716         mfence();
 717         return (rdtsc32());
 718 }
 719
 720 static u_int
 721 tsc_get_timecount_low_mfence(struct timecounter *tc)
 722 {
 723
 724         mfence();
 725         return (tsc_get_timecount_low(tc));
 726 }
 727
 728 uint32_t
 729 cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th)
 730 {
 731
 732         vdso_th->th_x86_shift = (int)(intptr_t)timecounter->tc_priv;
 733         bzero(vdso_th->th_res, sizeof(vdso_th->th_res));
 734         return (timecounter == &tsc_timecounter);
 735 }
 736
 737 #ifdef COMPAT_FREEBSD32
 738 uint32_t
 739 cpu_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32)
 740 {
 741
 742         vdso_th32->th_x86_shift = (int)(intptr_t)timecounter->tc_priv;
 743         bzero(vdso_th32->th_res, sizeof(vdso_th32->th_res));
 744         return (timecounter == &tsc_timecounter);
 745 }
 746 #endif