sys/kern/kern_clocksource.c

   1 /*-
   2  * Copyright (c) 2010-2013 Alexander Motin <mav@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer,
  10  *    without modification, immediately at the beginning of the file.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 #include <sys/cdefs.h>
  28 __FBSDID("$FreeBSD$");
  29
  30 /*
  31  * Common routines to manage event timers hardware.
  32  */
  33
  34 #include "opt_device_polling.h"
  35 #include "opt_kdtrace.h"
  36
  37 #include <sys/param.h>
  38 #include <sys/systm.h>
  39 #include <sys/bus.h>
  40 #include <sys/limits.h>
  41 #include <sys/lock.h>
  42 #include <sys/kdb.h>
  43 #include <sys/ktr.h>
  44 #include <sys/mutex.h>
  45 #include <sys/proc.h>
  46 #include <sys/kernel.h>
  47 #include <sys/sched.h>
  48 #include <sys/smp.h>
  49 #include <sys/sysctl.h>
  50 #include <sys/timeet.h>
  51 #include <sys/timetc.h>
  52
  53 #include <machine/atomic.h>
  54 #include <machine/clock.h>
  55 #include <machine/cpu.h>
  56 #include <machine/smp.h>
  57
  58 int                     cpu_deepest_sleep = 0;  /* Deepest Cx state available. */
  59 int                     cpu_disable_c2_sleep = 0; /* Timer dies in C2. */
  60 int                     cpu_disable_c3_sleep = 0; /* Timer dies in C3. */
  61
  62 static void             setuptimer(void);
  63 static void             loadtimer(sbintime_t now, int first);
  64 static int              doconfigtimer(void);
  65 static void             configtimer(int start);
  66 static int              round_freq(struct eventtimer *et, int freq);
  67
  68 static sbintime_t       getnextcpuevent(int idle);
  69 static sbintime_t       getnextevent(void);
  70 static int              handleevents(sbintime_t now, int fake);
  71
  72 static struct mtx       et_hw_mtx;
  73
  74 #define ET_HW_LOCK(state)                                               \
  75         {                                                               \
  76                 if (timer->et_flags & ET_FLAGS_PERCPU)                  \
  77                         mtx_lock_spin(&(state)->et_hw_mtx);             \
  78                 else                                                    \
  79                         mtx_lock_spin(&et_hw_mtx);                      \
  80         }
  81
  82 #define ET_HW_UNLOCK(state)                                             \
  83         {                                                               \
  84                 if (timer->et_flags & ET_FLAGS_PERCPU)                  \
  85                         mtx_unlock_spin(&(state)->et_hw_mtx);           \
  86                 else                                                    \
  87                         mtx_unlock_spin(&et_hw_mtx);                    \
  88         }
  89
  90 static struct eventtimer *timer = NULL;
  91 static sbintime_t       timerperiod;    /* Timer period for periodic mode. */
  92 static sbintime_t       statperiod;     /* statclock() events period. */
  93 static sbintime_t       profperiod;     /* profclock() events period. */
  94 static sbintime_t       nexttick;       /* Next global timer tick time. */
  95 static u_int            busy = 1;       /* Reconfiguration is in progress. */
  96 static int              profiling = 0;  /* Profiling events enabled. */
  97
  98 static char             timername[32];  /* Wanted timer. */
  99 TUNABLE_STR("kern.eventtimer.timer", timername, sizeof(timername));
 100
 101 static int              singlemul = 0;  /* Multiplier for periodic mode. */
 102 TUNABLE_INT("kern.eventtimer.singlemul", &singlemul);
 103 SYSCTL_INT(_kern_eventtimer, OID_AUTO, singlemul, CTLFLAG_RW, &singlemul,
 104     0, "Multiplier for periodic mode");
 105
 106 static u_int            idletick = 0;   /* Run periodic events when idle. */
 107 TUNABLE_INT("kern.eventtimer.idletick", &idletick);
 108 SYSCTL_UINT(_kern_eventtimer, OID_AUTO, idletick, CTLFLAG_RW, &idletick,
 109     0, "Run periodic events when idle");
 110
 111 static int              periodic = 0;   /* Periodic or one-shot mode. */
 112 static int              want_periodic = 0; /* What mode to prefer. */
 113 TUNABLE_INT("kern.eventtimer.periodic", &want_periodic);
 114
 115 struct pcpu_state {
 116         struct mtx      et_hw_mtx;      /* Per-CPU timer mutex. */
 117         u_int           action;         /* Reconfiguration requests. */
 118         u_int           handle;         /* Immediate handle resuests. */
 119         sbintime_t      now;            /* Last tick time. */
 120         sbintime_t      nextevent;      /* Next scheduled event on this CPU. */
 121         sbintime_t      nexttick;       /* Next timer tick time. */
 122         sbintime_t      nexthard;       /* Next hardlock() event. */
 123         sbintime_t      nextstat;       /* Next statclock() event. */
 124         sbintime_t      nextprof;       /* Next profclock() event. */
 125         sbintime_t      nextcall;       /* Next callout event. */
 126         sbintime_t      nextcallopt;    /* Next optional callout event. */
 127         int             ipi;            /* This CPU needs IPI. */
 128         int             idle;           /* This CPU is in idle mode. */
 129 };
 130
 131 static DPCPU_DEFINE(struct pcpu_state, timerstate);
 132 DPCPU_DEFINE(sbintime_t, hardclocktime);
 133
 134 /*
 135  * Timer broadcast IPI handler.
 136  */
 137 int
 138 hardclockintr(void)
 139 {
 140         sbintime_t now;
 141         struct pcpu_state *state;
 142         int done;
 143
 144         if (doconfigtimer() || busy)
 145                 return (FILTER_HANDLED);
 146         state = DPCPU_PTR(timerstate);
 147         now = state->now;
 148         CTR3(KTR_SPARE2, "ipi  at %d:    now  %d.%08x",
 149             curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff));
 150         done = handleevents(now, 0);
 151         return (done ? FILTER_HANDLED : FILTER_STRAY);
 152 }
 153
 154 /*
 155  * Handle all events for specified time on this CPU
 156  */
 157 static int
 158 handleevents(sbintime_t now, int fake)
 159 {
 160         sbintime_t t, *hct;
 161         struct trapframe *frame;
 162         struct pcpu_state *state;
 163         int usermode;
 164         int done, runs;
 165
 166         CTR3(KTR_SPARE2, "handle at %d:  now  %d.%08x",
 167             curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff));
 168         done = 0;
 169         if (fake) {
 170                 frame = NULL;
 171                 usermode = 0;
 172         } else {
 173                 frame = curthread->td_intr_frame;
 174                 usermode = TRAPF_USERMODE(frame);
 175         }
 176
 177         state = DPCPU_PTR(timerstate);
 178
 179         runs = 0;
 180         while (now >= state->nexthard) {
 181                 state->nexthard += tick_sbt;
 182                 runs++;
 183         }
 184         if (runs) {
 185                 hct = DPCPU_PTR(hardclocktime);
 186                 *hct = state->nexthard - tick_sbt;
 187                 if (fake < 2) {
 188                         hardclock_cnt(runs, usermode);
 189                         done = 1;
 190                 }
 191         }
 192         runs = 0;
 193         while (now >= state->nextstat) {
 194                 state->nextstat += statperiod;
 195                 runs++;
 196         }
 197         if (runs && fake < 2) {
 198                 statclock_cnt(runs, usermode);
 199                 done = 1;
 200         }
 201         if (profiling) {
 202                 runs = 0;
 203                 while (now >= state->nextprof) {
 204                         state->nextprof += profperiod;
 205                         runs++;
 206                 }
 207                 if (runs && !fake) {
 208                         profclock_cnt(runs, usermode, TRAPF_PC(frame));
 209                         done = 1;
 210                 }
 211         } else
 212                 state->nextprof = state->nextstat;
 213         if (now >= state->nextcallopt) {
 214                 state->nextcall = state->nextcallopt = INT64_MAX;
 215                 callout_process(now);
 216         }
 217
 218         t = getnextcpuevent(0);
 219         ET_HW_LOCK(state);
 220         if (!busy) {
 221                 state->idle = 0;
 222                 state->nextevent = t;
 223                 loadtimer(now, (fake == 2) &&
 224                     (timer->et_flags & ET_FLAGS_PERCPU));
 225         }
 226         ET_HW_UNLOCK(state);
 227         return (done);
 228 }
 229
 230 /*
 231  * Schedule binuptime of the next event on current CPU.
 232  */
 233 static sbintime_t
 234 getnextcpuevent(int idle)
 235 {
 236         sbintime_t event;
 237         struct pcpu_state *state;
 238         u_int hardfreq;
 239
 240         state = DPCPU_PTR(timerstate);
 241         /* Handle hardclock() events, skipping some if CPU is idle. */
 242         event = state->nexthard;
 243         if (idle) {
 244                 hardfreq = (u_int)hz / 2;
 245                 if (tc_min_ticktock_freq > 2
 246 #ifdef SMP
 247                     && curcpu == CPU_FIRST()
 248 #endif
 249                     )
 250                         hardfreq = hz / tc_min_ticktock_freq;
 251                 if (hardfreq > 1)
 252                         event += tick_sbt * (hardfreq - 1);
 253         }
 254         /* Handle callout events. */
 255         if (event > state->nextcall)
 256                 event = state->nextcall;
 257         if (!idle) { /* If CPU is active - handle other types of events. */
 258                 if (event > state->nextstat)
 259                         event = state->nextstat;
 260                 if (profiling && event > state->nextprof)
 261                         event = state->nextprof;
 262         }
 263         return (event);
 264 }
 265
 266 /*
 267  * Schedule binuptime of the next event on all CPUs.
 268  */
 269 static sbintime_t
 270 getnextevent(void)
 271 {
 272         struct pcpu_state *state;
 273         sbintime_t event;
 274 #ifdef SMP
 275         int     cpu;
 276 #endif
 277         int     c;
 278
 279         state = DPCPU_PTR(timerstate);
 280         event = state->nextevent;
 281         c = -1;
 282 #ifdef SMP
 283         if ((timer->et_flags & ET_FLAGS_PERCPU) == 0) {
 284                 CPU_FOREACH(cpu) {
 285                         state = DPCPU_ID_PTR(cpu, timerstate);
 286                         if (event > state->nextevent) {
 287                                 event = state->nextevent;
 288                                 c = cpu;
 289                         }
 290                 }
 291         }
 292 #endif
 293         CTR4(KTR_SPARE2, "next at %d:    next %d.%08x by %d",
 294             curcpu, (int)(event >> 32), (u_int)(event & 0xffffffff), c);
 295         return (event);
 296 }
 297
 298 /* Hardware timer callback function. */
 299 static void
 300 timercb(struct eventtimer *et, void *arg)
 301 {
 302         sbintime_t now;
 303         sbintime_t *next;
 304         struct pcpu_state *state;
 305 #ifdef SMP
 306         int cpu, bcast;
 307 #endif
 308
 309         /* Do not touch anything if somebody reconfiguring timers. */
 310         if (busy)
 311                 return;
 312         /* Update present and next tick times. */
 313         state = DPCPU_PTR(timerstate);
 314         if (et->et_flags & ET_FLAGS_PERCPU) {
 315                 next = &state->nexttick;
 316         } else
 317                 next = &nexttick;
 318         now = sbinuptime();
 319         if (periodic)
 320                 *next = now + timerperiod;
 321         else
 322                 *next = -1;     /* Next tick is not scheduled yet. */
 323         state->now = now;
 324         CTR3(KTR_SPARE2, "intr at %d:    now  %d.%08x",
 325             curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff));
 326
 327 #ifdef SMP
 328         /* Prepare broadcasting to other CPUs for non-per-CPU timers. */
 329         bcast = 0;
 330         if ((et->et_flags & ET_FLAGS_PERCPU) == 0 && smp_started) {
 331                 CPU_FOREACH(cpu) {
 332                         state = DPCPU_ID_PTR(cpu, timerstate);
 333                         ET_HW_LOCK(state);
 334                         state->now = now;
 335                         if (now >= state->nextevent) {
 336                                 state->nextevent += SBT_1S;
 337                                 if (curcpu != cpu) {
 338                                         state->ipi = 1;
 339                                         bcast = 1;
 340                                 }
 341                         }
 342                         ET_HW_UNLOCK(state);
 343                 }
 344         }
 345 #endif
 346
 347         /* Handle events for this time on this CPU. */
 348         handleevents(now, 0);
 349
 350 #ifdef SMP
 351         /* Broadcast interrupt to other CPUs for non-per-CPU timers. */
 352         if (bcast) {
 353                 CPU_FOREACH(cpu) {
 354                         if (curcpu == cpu)
 355                                 continue;
 356                         state = DPCPU_ID_PTR(cpu, timerstate);
 357                         if (state->ipi) {
 358                                 state->ipi = 0;
 359                                 ipi_cpu(cpu, IPI_HARDCLOCK);
 360                         }
 361                 }
 362         }
 363 #endif
 364 }
 365
 366 /*
 367  * Load new value into hardware timer.
 368  */
 369 static void
 370 loadtimer(sbintime_t now, int start)
 371 {
 372         struct pcpu_state *state;
 373         sbintime_t new;
 374         sbintime_t *next;
 375         uint64_t tmp;
 376         int eq;
 377
 378         if (timer->et_flags & ET_FLAGS_PERCPU) {
 379                 state = DPCPU_PTR(timerstate);
 380                 next = &state->nexttick;
 381         } else
 382                 next = &nexttick;
 383         if (periodic) {
 384                 if (start) {
 385                         /*
 386                          * Try to start all periodic timers aligned
 387                          * to period to make events synchronous.
 388                          */
 389                         tmp = now % timerperiod;
 390                         new = timerperiod - tmp;
 391                         if (new < tmp)          /* Left less then passed. */
 392                                 new += timerperiod;
 393                         CTR5(KTR_SPARE2, "load p at %d:   now %d.%08x first in %d.%08x",
 394                             curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff),
 395                             (int)(new >> 32), (u_int)(new & 0xffffffff));
 396                         *next = new + now;
 397                         et_start(timer, new, timerperiod);
 398                 }
 399         } else {
 400                 new = getnextevent();
 401                 eq = (new == *next);
 402                 CTR4(KTR_SPARE2, "load at %d:    next %d.%08x eq %d",
 403                     curcpu, (int)(new >> 32), (u_int)(new & 0xffffffff), eq);
 404                 if (!eq) {
 405                         *next = new;
 406                         et_start(timer, new - now, 0);
 407                 }
 408         }
 409 }
 410
 411 /*
 412  * Prepare event timer parameters after configuration changes.
 413  */
 414 static void
 415 setuptimer(void)
 416 {
 417         int freq;
 418
 419         if (periodic && (timer->et_flags & ET_FLAGS_PERIODIC) == 0)
 420                 periodic = 0;
 421         else if (!periodic && (timer->et_flags & ET_FLAGS_ONESHOT) == 0)
 422                 periodic = 1;
 423         singlemul = MIN(MAX(singlemul, 1), 20);
 424         freq = hz * singlemul;
 425         while (freq < (profiling ? profhz : stathz))
 426                 freq += hz;
 427         freq = round_freq(timer, freq);
 428         timerperiod = SBT_1S / freq;
 429 }
 430
 431 /*
 432  * Reconfigure specified per-CPU timer on other CPU. Called from IPI handler.
 433  */
 434 static int
 435 doconfigtimer(void)
 436 {
 437         sbintime_t now;
 438         struct pcpu_state *state;
 439
 440         state = DPCPU_PTR(timerstate);
 441         switch (atomic_load_acq_int(&state->action)) {
 442         case 1:
 443                 now = sbinuptime();
 444                 ET_HW_LOCK(state);
 445                 loadtimer(now, 1);
 446                 ET_HW_UNLOCK(state);
 447                 state->handle = 0;
 448                 atomic_store_rel_int(&state->action, 0);
 449                 return (1);
 450         case 2:
 451                 ET_HW_LOCK(state);
 452                 et_stop(timer);
 453                 ET_HW_UNLOCK(state);
 454                 state->handle = 0;
 455                 atomic_store_rel_int(&state->action, 0);
 456                 return (1);
 457         }
 458         if (atomic_readandclear_int(&state->handle) && !busy) {
 459                 now = sbinuptime();
 460                 handleevents(now, 0);
 461                 return (1);
 462         }
 463         return (0);
 464 }
 465
 466 /*
 467  * Reconfigure specified timer.
 468  * For per-CPU timers use IPI to make other CPUs to reconfigure.
 469  */
 470 static void
 471 configtimer(int start)
 472 {
 473         sbintime_t now, next;
 474         struct pcpu_state *state;
 475         int cpu;
 476
 477         if (start) {
 478                 setuptimer();
 479                 now = sbinuptime();
 480         } else
 481                 now = 0;
 482         critical_enter();
 483         ET_HW_LOCK(DPCPU_PTR(timerstate));
 484         if (start) {
 485                 /* Initialize time machine parameters. */
 486                 next = now + timerperiod;
 487                 if (periodic)
 488                         nexttick = next;
 489                 else
 490                         nexttick = -1;
 491                 CPU_FOREACH(cpu) {
 492                         state = DPCPU_ID_PTR(cpu, timerstate);
 493                         state->now = now;
 494                         if (!smp_started && cpu != CPU_FIRST())
 495                                 state->nextevent = INT64_MAX;
 496                         else
 497                                 state->nextevent = next;
 498                         if (periodic)
 499                                 state->nexttick = next;
 500                         else
 501                                 state->nexttick = -1;
 502                         state->nexthard = next;
 503                         state->nextstat = next;
 504                         state->nextprof = next;
 505                         state->nextcall = next;
 506                         state->nextcallopt = next;
 507                         hardclock_sync(cpu);
 508                 }
 509                 busy = 0;
 510                 /* Start global timer or per-CPU timer of this CPU. */
 511                 loadtimer(now, 1);
 512         } else {
 513                 busy = 1;
 514                 /* Stop global timer or per-CPU timer of this CPU. */
 515                 et_stop(timer);
 516         }
 517         ET_HW_UNLOCK(DPCPU_PTR(timerstate));
 518 #ifdef SMP
 519         /* If timer is global or there is no other CPUs yet - we are done. */
 520         if ((timer->et_flags & ET_FLAGS_PERCPU) == 0 || !smp_started) {
 521                 critical_exit();
 522                 return;
 523         }
 524         /* Set reconfigure flags for other CPUs. */
 525         CPU_FOREACH(cpu) {
 526                 state = DPCPU_ID_PTR(cpu, timerstate);
 527                 atomic_store_rel_int(&state->action,
 528                     (cpu == curcpu) ? 0 : ( start ? 1 : 2));
 529         }
 530         /* Broadcast reconfigure IPI. */
 531         ipi_all_but_self(IPI_HARDCLOCK);
 532         /* Wait for reconfiguration completed. */
 533 restart:
 534         cpu_spinwait();
 535         CPU_FOREACH(cpu) {
 536                 if (cpu == curcpu)
 537                         continue;
 538                 state = DPCPU_ID_PTR(cpu, timerstate);
 539                 if (atomic_load_acq_int(&state->action))
 540                         goto restart;
 541         }
 542 #endif
 543         critical_exit();
 544 }
 545
 546 /*
 547  * Calculate nearest frequency supported by hardware timer.
 548  */
 549 static int
 550 round_freq(struct eventtimer *et, int freq)
 551 {
 552         uint64_t div;
 553
 554         if (et->et_frequency != 0) {
 555                 div = lmax((et->et_frequency + freq / 2) / freq, 1);
 556                 if (et->et_flags & ET_FLAGS_POW2DIV)
 557                         div = 1 << (flsl(div + div / 2) - 1);
 558                 freq = (et->et_frequency + div / 2) / div;
 559         }
 560         if (et->et_min_period > SBT_1S)
 561                 panic("Event timer \"%s\" doesn't support sub-second periods!",
 562                     et->et_name);
 563         else if (et->et_min_period != 0)
 564                 freq = min(freq, SBT2FREQ(et->et_min_period));
 565         if (et->et_max_period < SBT_1S && et->et_max_period != 0)
 566                 freq = max(freq, SBT2FREQ(et->et_max_period));
 567         return (freq);
 568 }
 569
 570 /*
 571  * Configure and start event timers (BSP part).
 572  */
 573 void
 574 cpu_initclocks_bsp(void)
 575 {
 576         struct pcpu_state *state;
 577         int base, div, cpu;
 578
 579         mtx_init(&et_hw_mtx, "et_hw_mtx", NULL, MTX_SPIN);
 580         CPU_FOREACH(cpu) {
 581                 state = DPCPU_ID_PTR(cpu, timerstate);
 582                 mtx_init(&state->et_hw_mtx, "et_hw_mtx", NULL, MTX_SPIN);
 583                 state->nextcall = INT64_MAX;
 584                 state->nextcallopt = INT64_MAX;
 585         }
 586         periodic = want_periodic;
 587         /* Grab requested timer or the best of present. */
 588         if (timername[0])
 589                 timer = et_find(timername, 0, 0);
 590         if (timer == NULL && periodic) {
 591                 timer = et_find(NULL,
 592                     ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
 593         }
 594         if (timer == NULL) {
 595                 timer = et_find(NULL,
 596                     ET_FLAGS_ONESHOT, ET_FLAGS_ONESHOT);
 597         }
 598         if (timer == NULL && !periodic) {
 599                 timer = et_find(NULL,
 600                     ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
 601         }
 602         if (timer == NULL)
 603                 panic("No usable event timer found!");
 604         et_init(timer, timercb, NULL, NULL);
 605
 606         /* Adapt to timer capabilities. */
 607         if (periodic && (timer->et_flags & ET_FLAGS_PERIODIC) == 0)
 608                 periodic = 0;
 609         else if (!periodic && (timer->et_flags & ET_FLAGS_ONESHOT) == 0)
 610                 periodic = 1;
 611         if (timer->et_flags & ET_FLAGS_C3STOP)
 612                 cpu_disable_c3_sleep++;
 613
 614         /*
 615          * We honor the requested 'hz' value.
 616          * We want to run stathz in the neighborhood of 128hz.
 617          * We would like profhz to run as often as possible.
 618          */
 619         if (singlemul <= 0 || singlemul > 20) {
 620                 if (hz >= 1500 || (hz % 128) == 0)
 621                         singlemul = 1;
 622                 else if (hz >= 750)
 623                         singlemul = 2;
 624                 else
 625                         singlemul = 4;
 626         }
 627         if (periodic) {
 628                 base = round_freq(timer, hz * singlemul);
 629                 singlemul = max((base + hz / 2) / hz, 1);
 630                 hz = (base + singlemul / 2) / singlemul;
 631                 if (base <= 128)
 632                         stathz = base;
 633                 else {
 634                         div = base / 128;
 635                         if (div >= singlemul && (div % singlemul) == 0)
 636                                 div++;
 637                         stathz = base / div;
 638                 }
 639                 profhz = stathz;
 640                 while ((profhz + stathz) <= 128 * 64)
 641                         profhz += stathz;
 642                 profhz = round_freq(timer, profhz);
 643         } else {
 644                 hz = round_freq(timer, hz);
 645                 stathz = round_freq(timer, 127);
 646                 profhz = round_freq(timer, stathz * 64);
 647         }
 648         tick = 1000000 / hz;
 649         tick_sbt = SBT_1S / hz;
 650         tick_bt = sbttobt(tick_sbt);
 651         statperiod = SBT_1S / stathz;
 652         profperiod = SBT_1S / profhz;
 653         ET_LOCK();
 654         configtimer(1);
 655         ET_UNLOCK();
 656 }
 657
 658 /*
 659  * Start per-CPU event timers on APs.
 660  */
 661 void
 662 cpu_initclocks_ap(void)
 663 {
 664         sbintime_t now;
 665         struct pcpu_state *state;
 666         struct thread *td;
 667
 668         state = DPCPU_PTR(timerstate);
 669         now = sbinuptime();
 670         ET_HW_LOCK(state);
 671         state->now = now;
 672         hardclock_sync(curcpu);
 673         spinlock_enter();
 674         ET_HW_UNLOCK(state);
 675         td = curthread;
 676         td->td_intr_nesting_level++;
 677         handleevents(state->now, 2);
 678         td->td_intr_nesting_level--;
 679         spinlock_exit();
 680 }
 681
 682 /*
 683  * Switch to profiling clock rates.
 684  */
 685 void
 686 cpu_startprofclock(void)
 687 {
 688
 689         ET_LOCK();
 690         if (profiling == 0) {
 691                 if (periodic) {
 692                         configtimer(0);
 693                         profiling = 1;
 694                         configtimer(1);
 695                 } else
 696                         profiling = 1;
 697         } else
 698                 profiling++;
 699         ET_UNLOCK();
 700 }
 701
 702 /*
 703  * Switch to regular clock rates.
 704  */
 705 void
 706 cpu_stopprofclock(void)
 707 {
 708
 709         ET_LOCK();
 710         if (profiling == 1) {
 711                 if (periodic) {
 712                         configtimer(0);
 713                         profiling = 0;
 714                         configtimer(1);
 715                 } else
 716                 profiling = 0;
 717         } else
 718                 profiling--;
 719         ET_UNLOCK();
 720 }
 721
 722 /*
 723  * Switch to idle mode (all ticks handled).
 724  */
 725 sbintime_t
 726 cpu_idleclock(void)
 727 {
 728         sbintime_t now, t;
 729         struct pcpu_state *state;
 730
 731         if (idletick || busy ||
 732             (periodic && (timer->et_flags & ET_FLAGS_PERCPU))
 733 #ifdef DEVICE_POLLING
 734             || curcpu == CPU_FIRST()
 735 #endif
 736             )
 737                 return (-1);
 738         state = DPCPU_PTR(timerstate);
 739         if (periodic)
 740                 now = state->now;
 741         else
 742                 now = sbinuptime();
 743         CTR3(KTR_SPARE2, "idle at %d:    now  %d.%08x",
 744             curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff));
 745         t = getnextcpuevent(1);
 746         ET_HW_LOCK(state);
 747         state->idle = 1;
 748         state->nextevent = t;
 749         if (!periodic)
 750                 loadtimer(now, 0);
 751         ET_HW_UNLOCK(state);
 752         return (MAX(t - now, 0));
 753 }
 754
 755 /*
 756  * Switch to active mode (skip empty ticks).
 757  */
 758 void
 759 cpu_activeclock(void)
 760 {
 761         sbintime_t now;
 762         struct pcpu_state *state;
 763         struct thread *td;
 764
 765         state = DPCPU_PTR(timerstate);
 766         if (state->idle == 0 || busy)
 767                 return;
 768         if (periodic)
 769                 now = state->now;
 770         else
 771                 now = sbinuptime();
 772         CTR3(KTR_SPARE2, "active at %d:  now  %d.%08x",
 773             curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff));
 774         spinlock_enter();
 775         td = curthread;
 776         td->td_intr_nesting_level++;
 777         handleevents(now, 1);
 778         td->td_intr_nesting_level--;
 779         spinlock_exit();
 780 }
 781
 782 /*
 783  * Change the frequency of the given timer.  This changes et->et_frequency and
 784  * if et is the active timer it reconfigures the timer on all CPUs.  This is
 785  * intended to be a private interface for the use of et_change_frequency() only.
 786  */
 787 void
 788 cpu_et_frequency(struct eventtimer *et, uint64_t newfreq)
 789 {
 790
 791         ET_LOCK();
 792         if (et == timer) {
 793                 configtimer(0);
 794                 et->et_frequency = newfreq;
 795                 configtimer(1);
 796         } else
 797                 et->et_frequency = newfreq;
 798         ET_UNLOCK();
 799 }
 800
 801 void
 802 cpu_new_callout(int cpu, sbintime_t bt, sbintime_t bt_opt)
 803 {
 804         struct pcpu_state *state;
 805
 806         /* Do not touch anything if somebody reconfiguring timers. */
 807         if (busy)
 808                 return;
 809         CTR6(KTR_SPARE2, "new co at %d:    on %d at %d.%08x - %d.%08x",
 810             curcpu, cpu, (int)(bt_opt >> 32), (u_int)(bt_opt & 0xffffffff),
 811             (int)(bt >> 32), (u_int)(bt & 0xffffffff));
 812         state = DPCPU_ID_PTR(cpu, timerstate);
 813         ET_HW_LOCK(state);
 814
 815         /*
 816          * If there is callout time already set earlier -- do nothing.
 817          * This check may appear redundant because we check already in
 818          * callout_process() but this double check guarantees we're safe
 819          * with respect to race conditions between interrupts execution
 820          * and scheduling.
 821          */
 822         state->nextcallopt = bt_opt;
 823         if (bt >= state->nextcall)
 824                 goto done;
 825         state->nextcall = bt;
 826         /* If there is some other event set earlier -- do nothing. */
 827         if (bt >= state->nextevent)
 828                 goto done;
 829         state->nextevent = bt;
 830         /* If timer is periodic -- there is nothing to reprogram. */
 831         if (periodic)
 832                 goto done;
 833         /* If timer is global or of the current CPU -- reprogram it. */
 834         if ((timer->et_flags & ET_FLAGS_PERCPU) == 0 || cpu == curcpu) {
 835                 loadtimer(sbinuptime(), 0);
 836 done:
 837                 ET_HW_UNLOCK(state);
 838                 return;
 839         }
 840         /* Otherwise make other CPU to reprogram it. */
 841         state->handle = 1;
 842         ET_HW_UNLOCK(state);
 843 #ifdef SMP
 844         ipi_cpu(cpu, IPI_HARDCLOCK);
 845 #endif
 846 }
 847
 848 /*
 849  * Report or change the active event timers hardware.
 850  */
 851 static int
 852 sysctl_kern_eventtimer_timer(SYSCTL_HANDLER_ARGS)
 853 {
 854         char buf[32];
 855         struct eventtimer *et;
 856         int error;
 857
 858         ET_LOCK();
 859         et = timer;
 860         snprintf(buf, sizeof(buf), "%s", et->et_name);
 861         ET_UNLOCK();
 862         error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
 863         ET_LOCK();
 864         et = timer;
 865         if (error != 0 || req->newptr == NULL ||
 866             strcasecmp(buf, et->et_name) == 0) {
 867                 ET_UNLOCK();
 868                 return (error);
 869         }
 870         et = et_find(buf, 0, 0);
 871         if (et == NULL) {
 872                 ET_UNLOCK();
 873                 return (ENOENT);
 874         }
 875         configtimer(0);
 876         et_free(timer);
 877         if (et->et_flags & ET_FLAGS_C3STOP)
 878                 cpu_disable_c3_sleep++;
 879         if (timer->et_flags & ET_FLAGS_C3STOP)
 880                 cpu_disable_c3_sleep--;
 881         periodic = want_periodic;
 882         timer = et;
 883         et_init(timer, timercb, NULL, NULL);
 884         configtimer(1);
 885         ET_UNLOCK();
 886         return (error);
 887 }
 888 SYSCTL_PROC(_kern_eventtimer, OID_AUTO, timer,
 889     CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE,
 890     0, 0, sysctl_kern_eventtimer_timer, "A", "Chosen event timer");
 891
 892 /*
 893  * Report or change the active event timer periodicity.
 894  */
 895 static int
 896 sysctl_kern_eventtimer_periodic(SYSCTL_HANDLER_ARGS)
 897 {
 898         int error, val;
 899
 900         val = periodic;
 901         error = sysctl_handle_int(oidp, &val, 0, req);
 902         if (error != 0 || req->newptr == NULL)
 903                 return (error);
 904         ET_LOCK();
 905         configtimer(0);
 906         periodic = want_periodic = val;
 907         configtimer(1);
 908         ET_UNLOCK();
 909         return (error);
 910 }
 911 SYSCTL_PROC(_kern_eventtimer, OID_AUTO, periodic,
 912     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
 913     0, 0, sysctl_kern_eventtimer_periodic, "I", "Enable event timer periodic mode");
 914
 915 #include "opt_ddb.h"
 916
 917 #ifdef DDB
 918 #include <ddb/ddb.h>
 919
 920 DB_SHOW_COMMAND(clocksource, db_show_clocksource)
 921 {
 922         struct pcpu_state *st;
 923         int c;
 924
 925         CPU_FOREACH(c) {
 926                 st = DPCPU_ID_PTR(c, timerstate);
 927                 db_printf(
 928                     "CPU %2d: action %d handle %d  ipi %d idle %d\n"
 929                     "        now %#jx nevent %#jx (%jd)\n"
 930                     "        ntick %#jx (%jd) nhard %#jx (%jd)\n"
 931                     "        nstat %#jx (%jd) nprof %#jx (%jd)\n"
 932                     "        ncall %#jx (%jd) ncallopt %#jx (%jd)\n",
 933                     c, st->action, st->handle, st->ipi, st->idle,
 934                     (uintmax_t)st->now,
 935                     (uintmax_t)st->nextevent,
 936                     (uintmax_t)(st->nextevent - st->now) / tick_sbt,
 937                     (uintmax_t)st->nexttick,
 938                     (uintmax_t)(st->nexttick - st->now) / tick_sbt,
 939                     (uintmax_t)st->nexthard,
 940                     (uintmax_t)(st->nexthard - st->now) / tick_sbt,
 941                     (uintmax_t)st->nextstat,
 942                     (uintmax_t)(st->nextstat - st->now) / tick_sbt,
 943                     (uintmax_t)st->nextprof,
 944                     (uintmax_t)(st->nextprof - st->now) / tick_sbt,
 945                     (uintmax_t)st->nextcall,
 946                     (uintmax_t)(st->nextcall - st->now) / tick_sbt,
 947                     (uintmax_t)st->nextcallopt,
 948                     (uintmax_t)(st->nextcallopt - st->now) / tick_sbt);
 949         }
 950 }
 951
 952 #endif