sys/kern/kern_clocksource.c

   1 /*-
   2  * Copyright (c) 2010-2012 Alexander Motin <mav@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer,
  10  *    without modification, immediately at the beginning of the file.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 #include <sys/cdefs.h>
  28 __FBSDID("$FreeBSD$");
  29
  30 /*
  31  * Common routines to manage event timers hardware.
  32  */
  33
  34 #include "opt_device_polling.h"
  35 #include "opt_kdtrace.h"
  36
  37 #include <sys/param.h>
  38 #include <sys/systm.h>
  39 #include <sys/bus.h>
  40 #include <sys/lock.h>
  41 #include <sys/kdb.h>
  42 #include <sys/ktr.h>
  43 #include <sys/mutex.h>
  44 #include <sys/proc.h>
  45 #include <sys/kernel.h>
  46 #include <sys/sched.h>
  47 #include <sys/smp.h>
  48 #include <sys/sysctl.h>
  49 #include <sys/timeet.h>
  50 #include <sys/timetc.h>
  51
  52 #include <machine/atomic.h>
  53 #include <machine/clock.h>
  54 #include <machine/cpu.h>
  55 #include <machine/smp.h>
  56
  57 #ifdef KDTRACE_HOOKS
  58 #include <sys/dtrace_bsd.h>
  59 cyclic_clock_func_t     cyclic_clock_func = NULL;
  60 #endif
  61
  62 int                     cpu_deepest_sleep = 0;  /* Deepest Cx state available. */
  63 int                     cpu_disable_c2_sleep = 0; /* Timer dies in C2. */
  64 int                     cpu_disable_c3_sleep = 0; /* Timer dies in C3. */
  65
  66 static void             setuptimer(void);
  67 static void             loadtimer(struct bintime *now, int first);
  68 static int              doconfigtimer(void);
  69 static void             configtimer(int start);
  70 static int              round_freq(struct eventtimer *et, int freq);
  71
  72 static void             getnextcpuevent(struct bintime *event, int idle);
  73 static void             getnextevent(struct bintime *event);
  74 static int              handleevents(struct bintime *now, int fake);
  75 #ifdef SMP
  76 static void             cpu_new_callout(int cpu, int ticks);
  77 #endif
  78
  79 static struct mtx       et_hw_mtx;
  80
  81 #define ET_HW_LOCK(state)                                               \
  82         {                                                               \
  83                 if (timer->et_flags & ET_FLAGS_PERCPU)                  \
  84                         mtx_lock_spin(&(state)->et_hw_mtx);             \
  85                 else                                                    \
  86                         mtx_lock_spin(&et_hw_mtx);                      \
  87         }
  88
  89 #define ET_HW_UNLOCK(state)                                             \
  90         {                                                               \
  91                 if (timer->et_flags & ET_FLAGS_PERCPU)                  \
  92                         mtx_unlock_spin(&(state)->et_hw_mtx);           \
  93                 else                                                    \
  94                         mtx_unlock_spin(&et_hw_mtx);                    \
  95         }
  96
  97 static struct eventtimer *timer = NULL;
  98 static struct bintime   timerperiod;    /* Timer period for periodic mode. */
  99 static struct bintime   hardperiod;     /* hardclock() events period. */
 100 static struct bintime   statperiod;     /* statclock() events period. */
 101 static struct bintime   profperiod;     /* profclock() events period. */
 102 static struct bintime   nexttick;       /* Next global timer tick time. */
 103 static struct bintime   nexthard;       /* Next global hardlock() event. */
 104 static u_int            busy = 0;       /* Reconfiguration is in progress. */
 105 static int              profiling = 0;  /* Profiling events enabled. */
 106
 107 static char             timername[32];  /* Wanted timer. */
 108 TUNABLE_STR("kern.eventtimer.timer", timername, sizeof(timername));
 109
 110 static int              singlemul = 0;  /* Multiplier for periodic mode. */
 111 TUNABLE_INT("kern.eventtimer.singlemul", &singlemul);
 112 SYSCTL_INT(_kern_eventtimer, OID_AUTO, singlemul, CTLFLAG_RW, &singlemul,
 113     0, "Multiplier for periodic mode");
 114
 115 static u_int            idletick = 0;   /* Run periodic events when idle. */
 116 TUNABLE_INT("kern.eventtimer.idletick", &idletick);
 117 SYSCTL_UINT(_kern_eventtimer, OID_AUTO, idletick, CTLFLAG_RW, &idletick,
 118     0, "Run periodic events when idle");
 119
 120 static u_int            activetick = 1; /* Run all periodic events when active. */
 121 TUNABLE_INT("kern.eventtimer.activetick", &activetick);
 122 SYSCTL_UINT(_kern_eventtimer, OID_AUTO, activetick, CTLFLAG_RW, &activetick,
 123     0, "Run all periodic events when active");
 124
 125 static int              periodic = 0;   /* Periodic or one-shot mode. */
 126 static int              want_periodic = 0; /* What mode to prefer. */
 127 TUNABLE_INT("kern.eventtimer.periodic", &want_periodic);
 128
 129 struct pcpu_state {
 130         struct mtx      et_hw_mtx;      /* Per-CPU timer mutex. */
 131         u_int           action;         /* Reconfiguration requests. */
 132         u_int           handle;         /* Immediate handle resuests. */
 133         struct bintime  now;            /* Last tick time. */
 134         struct bintime  nextevent;      /* Next scheduled event on this CPU. */
 135         struct bintime  nexttick;       /* Next timer tick time. */
 136         struct bintime  nexthard;       /* Next hardlock() event. */
 137         struct bintime  nextstat;       /* Next statclock() event. */
 138         struct bintime  nextprof;       /* Next profclock() event. */
 139 #ifdef KDTRACE_HOOKS
 140         struct bintime  nextcyc;        /* Next OpenSolaris cyclics event. */
 141 #endif
 142         int             ipi;            /* This CPU needs IPI. */
 143         int             idle;           /* This CPU is in idle mode. */
 144 };
 145
 146 static DPCPU_DEFINE(struct pcpu_state, timerstate);
 147
 148 #define FREQ2BT(freq, bt)                                               \
 149 {                                                                       \
 150         (bt)->sec = 0;                                                  \
 151         (bt)->frac = ((uint64_t)0x8000000000000000  / (freq)) << 1;     \
 152 }
 153 #define BT2FREQ(bt)                                                     \
 154         (((uint64_t)0x8000000000000000 + ((bt)->frac >> 2)) /           \
 155             ((bt)->frac >> 1))
 156
 157 /*
 158  * Timer broadcast IPI handler.
 159  */
 160 int
 161 hardclockintr(void)
 162 {
 163         struct bintime now;
 164         struct pcpu_state *state;
 165         int done;
 166
 167         if (doconfigtimer() || busy)
 168                 return (FILTER_HANDLED);
 169         state = DPCPU_PTR(timerstate);
 170         now = state->now;
 171         CTR4(KTR_SPARE2, "ipi  at %d:    now  %d.%08x%08x",
 172             curcpu, now.sec, (unsigned int)(now.frac >> 32),
 173                              (unsigned int)(now.frac & 0xffffffff));
 174         done = handleevents(&now, 0);
 175         return (done ? FILTER_HANDLED : FILTER_STRAY);
 176 }
 177
 178 /*
 179  * Handle all events for specified time on this CPU
 180  */
 181 static int
 182 handleevents(struct bintime *now, int fake)
 183 {
 184         struct bintime t;
 185         struct trapframe *frame;
 186         struct pcpu_state *state;
 187         uintfptr_t pc;
 188         int usermode;
 189         int done, runs;
 190
 191         CTR4(KTR_SPARE2, "handle at %d:  now  %d.%08x%08x",
 192             curcpu, now->sec, (unsigned int)(now->frac >> 32),
 193                      (unsigned int)(now->frac & 0xffffffff));
 194         done = 0;
 195         if (fake) {
 196                 frame = NULL;
 197                 usermode = 0;
 198                 pc = 0;
 199         } else {
 200                 frame = curthread->td_intr_frame;
 201                 usermode = TRAPF_USERMODE(frame);
 202                 pc = TRAPF_PC(frame);
 203         }
 204
 205         state = DPCPU_PTR(timerstate);
 206
 207         runs = 0;
 208         while (bintime_cmp(now, &state->nexthard, >=)) {
 209                 bintime_add(&state->nexthard, &hardperiod);
 210                 runs++;
 211         }
 212         if ((timer->et_flags & ET_FLAGS_PERCPU) == 0 &&
 213             bintime_cmp(&state->nexthard, &nexthard, >))
 214                 nexthard = state->nexthard;
 215         if (runs && fake < 2) {
 216                 hardclock_cnt(runs, usermode);
 217                 done = 1;
 218         }
 219         runs = 0;
 220         while (bintime_cmp(now, &state->nextstat, >=)) {
 221                 bintime_add(&state->nextstat, &statperiod);
 222                 runs++;
 223         }
 224         if (runs && fake < 2) {
 225                 statclock_cnt(runs, usermode);
 226                 done = 1;
 227         }
 228         if (profiling) {
 229                 runs = 0;
 230                 while (bintime_cmp(now, &state->nextprof, >=)) {
 231                         bintime_add(&state->nextprof, &profperiod);
 232                         runs++;
 233                 }
 234                 if (runs && !fake) {
 235                         profclock_cnt(runs, usermode, pc);
 236                         done = 1;
 237                 }
 238         } else
 239                 state->nextprof = state->nextstat;
 240
 241 #ifdef KDTRACE_HOOKS
 242         if (fake == 0 && cyclic_clock_func != NULL &&
 243             state->nextcyc.sec != -1 &&
 244             bintime_cmp(now, &state->nextcyc, >=)) {
 245                 state->nextcyc.sec = -1;
 246                 (*cyclic_clock_func)(frame);
 247         }
 248 #endif
 249
 250         getnextcpuevent(&t, 0);
 251         if (fake == 2) {
 252                 state->nextevent = t;
 253                 return (done);
 254         }
 255         ET_HW_LOCK(state);
 256         if (!busy) {
 257                 state->idle = 0;
 258                 state->nextevent = t;
 259                 loadtimer(now, 0);
 260         }
 261         ET_HW_UNLOCK(state);
 262         return (done);
 263 }
 264
 265 /*
 266  * Schedule binuptime of the next event on current CPU.
 267  */
 268 static void
 269 getnextcpuevent(struct bintime *event, int idle)
 270 {
 271         struct bintime tmp;
 272         struct pcpu_state *state;
 273         int skip;
 274
 275         state = DPCPU_PTR(timerstate);
 276         /* Handle hardclock() events. */
 277         *event = state->nexthard;
 278         if (idle || (!activetick && !profiling &&
 279             (timer->et_flags & ET_FLAGS_PERCPU) == 0)) {
 280                 skip = idle ? 4 : (stathz / 2);
 281                 if (curcpu == CPU_FIRST() && tc_min_ticktock_freq > skip)
 282                         skip = tc_min_ticktock_freq;
 283                 skip = callout_tickstofirst(hz / skip) - 1;
 284                 CTR2(KTR_SPARE2, "skip   at %d: %d", curcpu, skip);
 285                 tmp = hardperiod;
 286                 bintime_mul(&tmp, skip);
 287                 bintime_add(event, &tmp);
 288         }
 289         if (!idle) { /* If CPU is active - handle other types of events. */
 290                 if (bintime_cmp(event, &state->nextstat, >))
 291                         *event = state->nextstat;
 292                 if (profiling && bintime_cmp(event, &state->nextprof, >))
 293                         *event = state->nextprof;
 294         }
 295 #ifdef KDTRACE_HOOKS
 296         if (state->nextcyc.sec != -1 && bintime_cmp(event, &state->nextcyc, >))
 297                 *event = state->nextcyc;
 298 #endif
 299 }
 300
 301 /*
 302  * Schedule binuptime of the next event on all CPUs.
 303  */
 304 static void
 305 getnextevent(struct bintime *event)
 306 {
 307         struct pcpu_state *state;
 308 #ifdef SMP
 309         int     cpu;
 310 #endif
 311         int     c, nonidle;
 312
 313         state = DPCPU_PTR(timerstate);
 314         *event = state->nextevent;
 315         c = curcpu;
 316         nonidle = !state->idle;
 317         if ((timer->et_flags & ET_FLAGS_PERCPU) == 0) {
 318 #ifdef SMP
 319                 CPU_FOREACH(cpu) {
 320                         if (curcpu == cpu)
 321                                 continue;
 322                         state = DPCPU_ID_PTR(cpu, timerstate);
 323                         nonidle += !state->idle;
 324                         if (bintime_cmp(event, &state->nextevent, >)) {
 325                                 *event = state->nextevent;
 326                                 c = cpu;
 327                         }
 328                 }
 329 #endif
 330                 if (nonidle != 0 && bintime_cmp(event, &nexthard, >))
 331                         *event = nexthard;
 332         }
 333         CTR5(KTR_SPARE2, "next at %d:    next %d.%08x%08x by %d",
 334             curcpu, event->sec, (unsigned int)(event->frac >> 32),
 335                              (unsigned int)(event->frac & 0xffffffff), c);
 336 }
 337
 338 /* Hardware timer callback function. */
 339 static void
 340 timercb(struct eventtimer *et, void *arg)
 341 {
 342         struct bintime now;
 343         struct bintime *next;
 344         struct pcpu_state *state;
 345 #ifdef SMP
 346         int cpu, bcast;
 347 #endif
 348
 349         /* Do not touch anything if somebody reconfiguring timers. */
 350         if (busy)
 351                 return;
 352         /* Update present and next tick times. */
 353         state = DPCPU_PTR(timerstate);
 354         if (et->et_flags & ET_FLAGS_PERCPU) {
 355                 next = &state->nexttick;
 356         } else
 357                 next = &nexttick;
 358         if (periodic) {
 359                 now = *next;    /* Ex-next tick time becomes present time. */
 360                 bintime_add(next, &timerperiod); /* Next tick in 1 period. */
 361         } else {
 362                 binuptime(&now);        /* Get present time from hardware. */
 363                 next->sec = -1;         /* Next tick is not scheduled yet. */
 364         }
 365         state->now = now;
 366         CTR4(KTR_SPARE2, "intr at %d:    now  %d.%08x%08x",
 367             curcpu, now.sec, (unsigned int)(now.frac >> 32),
 368                              (unsigned int)(now.frac & 0xffffffff));
 369
 370 #ifdef SMP
 371         /* Prepare broadcasting to other CPUs for non-per-CPU timers. */
 372         bcast = 0;
 373         if ((et->et_flags & ET_FLAGS_PERCPU) == 0 && smp_started) {
 374                 CPU_FOREACH(cpu) {
 375                         state = DPCPU_ID_PTR(cpu, timerstate);
 376                         ET_HW_LOCK(state);
 377                         state->now = now;
 378                         if (bintime_cmp(&now, &state->nextevent, >=)) {
 379                                 state->nextevent.sec++;
 380                                 if (curcpu != cpu) {
 381                                         state->ipi = 1;
 382                                         bcast = 1;
 383                                 }
 384                         }
 385                         ET_HW_UNLOCK(state);
 386                 }
 387         }
 388 #endif
 389
 390         /* Handle events for this time on this CPU. */
 391         handleevents(&now, 0);
 392
 393 #ifdef SMP
 394         /* Broadcast interrupt to other CPUs for non-per-CPU timers. */
 395         if (bcast) {
 396                 CPU_FOREACH(cpu) {
 397                         if (curcpu == cpu)
 398                                 continue;
 399                         state = DPCPU_ID_PTR(cpu, timerstate);
 400                         if (state->ipi) {
 401                                 state->ipi = 0;
 402                                 ipi_cpu(cpu, IPI_HARDCLOCK);
 403                         }
 404                 }
 405         }
 406 #endif
 407 }
 408
 409 /*
 410  * Load new value into hardware timer.
 411  */
 412 static void
 413 loadtimer(struct bintime *now, int start)
 414 {
 415         struct pcpu_state *state;
 416         struct bintime new;
 417         struct bintime *next;
 418         uint64_t tmp;
 419         int eq;
 420
 421         if (timer->et_flags & ET_FLAGS_PERCPU) {
 422                 state = DPCPU_PTR(timerstate);
 423                 next = &state->nexttick;
 424         } else
 425                 next = &nexttick;
 426         if (periodic) {
 427                 if (start) {
 428                         /*
 429                          * Try to start all periodic timers aligned
 430                          * to period to make events synchronous.
 431                          */
 432                         tmp = ((uint64_t)now->sec << 36) + (now->frac >> 28);
 433                         tmp = (tmp % (timerperiod.frac >> 28)) << 28;
 434                         new.sec = 0;
 435                         new.frac = timerperiod.frac - tmp;
 436                         if (new.frac < tmp)     /* Left less then passed. */
 437                                 bintime_add(&new, &timerperiod);
 438                         CTR5(KTR_SPARE2, "load p at %d:   now %d.%08x first in %d.%08x",
 439                             curcpu, now->sec, (unsigned int)(now->frac >> 32),
 440                             new.sec, (unsigned int)(new.frac >> 32));
 441                         *next = new;
 442                         bintime_add(next, now);
 443                         et_start(timer, &new, &timerperiod);
 444                 }
 445         } else {
 446                 getnextevent(&new);
 447                 eq = bintime_cmp(&new, next, ==);
 448                 CTR5(KTR_SPARE2, "load at %d:    next %d.%08x%08x eq %d",
 449                     curcpu, new.sec, (unsigned int)(new.frac >> 32),
 450                              (unsigned int)(new.frac & 0xffffffff),
 451                              eq);
 452                 if (!eq) {
 453                         *next = new;
 454                         bintime_sub(&new, now);
 455                         et_start(timer, &new, NULL);
 456                 }
 457         }
 458 }
 459
 460 /*
 461  * Prepare event timer parameters after configuration changes.
 462  */
 463 static void
 464 setuptimer(void)
 465 {
 466         int freq;
 467
 468         if (periodic && (timer->et_flags & ET_FLAGS_PERIODIC) == 0)
 469                 periodic = 0;
 470         else if (!periodic && (timer->et_flags & ET_FLAGS_ONESHOT) == 0)
 471                 periodic = 1;
 472         singlemul = MIN(MAX(singlemul, 1), 20);
 473         freq = hz * singlemul;
 474         while (freq < (profiling ? profhz : stathz))
 475                 freq += hz;
 476         freq = round_freq(timer, freq);
 477         FREQ2BT(freq, &timerperiod);
 478 }
 479
 480 /*
 481  * Reconfigure specified per-CPU timer on other CPU. Called from IPI handler.
 482  */
 483 static int
 484 doconfigtimer(void)
 485 {
 486         struct bintime now;
 487         struct pcpu_state *state;
 488
 489         state = DPCPU_PTR(timerstate);
 490         switch (atomic_load_acq_int(&state->action)) {
 491         case 1:
 492                 binuptime(&now);
 493                 ET_HW_LOCK(state);
 494                 loadtimer(&now, 1);
 495                 ET_HW_UNLOCK(state);
 496                 state->handle = 0;
 497                 atomic_store_rel_int(&state->action, 0);
 498                 return (1);
 499         case 2:
 500                 ET_HW_LOCK(state);
 501                 et_stop(timer);
 502                 ET_HW_UNLOCK(state);
 503                 state->handle = 0;
 504                 atomic_store_rel_int(&state->action, 0);
 505                 return (1);
 506         }
 507         if (atomic_readandclear_int(&state->handle) && !busy) {
 508                 binuptime(&now);
 509                 handleevents(&now, 0);
 510                 return (1);
 511         }
 512         return (0);
 513 }
 514
 515 /*
 516  * Reconfigure specified timer.
 517  * For per-CPU timers use IPI to make other CPUs to reconfigure.
 518  */
 519 static void
 520 configtimer(int start)
 521 {
 522         struct bintime now, next;
 523         struct pcpu_state *state;
 524         int cpu;
 525
 526         if (start) {
 527                 setuptimer();
 528                 binuptime(&now);
 529         }
 530         critical_enter();
 531         ET_HW_LOCK(DPCPU_PTR(timerstate));
 532         if (start) {
 533                 /* Initialize time machine parameters. */
 534                 next = now;
 535                 bintime_add(&next, &timerperiod);
 536                 if (periodic)
 537                         nexttick = next;
 538                 else
 539                         nexttick.sec = -1;
 540                 CPU_FOREACH(cpu) {
 541                         state = DPCPU_ID_PTR(cpu, timerstate);
 542                         state->now = now;
 543                         state->nextevent = next;
 544                         if (periodic)
 545                                 state->nexttick = next;
 546                         else
 547                                 state->nexttick.sec = -1;
 548                         state->nexthard = next;
 549                         state->nextstat = next;
 550                         state->nextprof = next;
 551                         hardclock_sync(cpu);
 552                 }
 553                 busy = 0;
 554                 /* Start global timer or per-CPU timer of this CPU. */
 555                 loadtimer(&now, 1);
 556         } else {
 557                 busy = 1;
 558                 /* Stop global timer or per-CPU timer of this CPU. */
 559                 et_stop(timer);
 560         }
 561         ET_HW_UNLOCK(DPCPU_PTR(timerstate));
 562 #ifdef SMP
 563         /* If timer is global or there is no other CPUs yet - we are done. */
 564         if ((timer->et_flags & ET_FLAGS_PERCPU) == 0 || !smp_started) {
 565                 critical_exit();
 566                 return;
 567         }
 568         /* Set reconfigure flags for other CPUs. */
 569         CPU_FOREACH(cpu) {
 570                 state = DPCPU_ID_PTR(cpu, timerstate);
 571                 atomic_store_rel_int(&state->action,
 572                     (cpu == curcpu) ? 0 : ( start ? 1 : 2));
 573         }
 574         /* Broadcast reconfigure IPI. */
 575         ipi_all_but_self(IPI_HARDCLOCK);
 576         /* Wait for reconfiguration completed. */
 577 restart:
 578         cpu_spinwait();
 579         CPU_FOREACH(cpu) {
 580                 if (cpu == curcpu)
 581                         continue;
 582                 state = DPCPU_ID_PTR(cpu, timerstate);
 583                 if (atomic_load_acq_int(&state->action))
 584                         goto restart;
 585         }
 586 #endif
 587         critical_exit();
 588 }
 589
 590 /*
 591  * Calculate nearest frequency supported by hardware timer.
 592  */
 593 static int
 594 round_freq(struct eventtimer *et, int freq)
 595 {
 596         uint64_t div;
 597
 598         if (et->et_frequency != 0) {
 599                 div = lmax((et->et_frequency + freq / 2) / freq, 1);
 600                 if (et->et_flags & ET_FLAGS_POW2DIV)
 601                         div = 1 << (flsl(div + div / 2) - 1);
 602                 freq = (et->et_frequency + div / 2) / div;
 603         }
 604         if (et->et_min_period.sec > 0)
 605                 freq = 0;
 606         else if (et->et_min_period.frac != 0)
 607                 freq = min(freq, BT2FREQ(&et->et_min_period));
 608         if (et->et_max_period.sec == 0 && et->et_max_period.frac != 0)
 609                 freq = max(freq, BT2FREQ(&et->et_max_period));
 610         return (freq);
 611 }
 612
 613 /*
 614  * Configure and start event timers (BSP part).
 615  */
 616 void
 617 cpu_initclocks_bsp(void)
 618 {
 619         struct pcpu_state *state;
 620         int base, div, cpu;
 621
 622         mtx_init(&et_hw_mtx, "et_hw_mtx", NULL, MTX_SPIN);
 623         CPU_FOREACH(cpu) {
 624                 state = DPCPU_ID_PTR(cpu, timerstate);
 625                 mtx_init(&state->et_hw_mtx, "et_hw_mtx", NULL, MTX_SPIN);
 626 #ifdef KDTRACE_HOOKS
 627                 state->nextcyc.sec = -1;
 628 #endif
 629         }
 630 #ifdef SMP
 631         callout_new_inserted = cpu_new_callout;
 632 #endif
 633         periodic = want_periodic;
 634         /* Grab requested timer or the best of present. */
 635         if (timername[0])
 636                 timer = et_find(timername, 0, 0);
 637         if (timer == NULL && periodic) {
 638                 timer = et_find(NULL,
 639                     ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
 640         }
 641         if (timer == NULL) {
 642                 timer = et_find(NULL,
 643                     ET_FLAGS_ONESHOT, ET_FLAGS_ONESHOT);
 644         }
 645         if (timer == NULL && !periodic) {
 646                 timer = et_find(NULL,
 647                     ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
 648         }
 649         if (timer == NULL)
 650                 panic("No usable event timer found!");
 651         et_init(timer, timercb, NULL, NULL);
 652
 653         /* Adapt to timer capabilities. */
 654         if (periodic && (timer->et_flags & ET_FLAGS_PERIODIC) == 0)
 655                 periodic = 0;
 656         else if (!periodic && (timer->et_flags & ET_FLAGS_ONESHOT) == 0)
 657                 periodic = 1;
 658         if (timer->et_flags & ET_FLAGS_C3STOP)
 659                 cpu_disable_c3_sleep++;
 660
 661         /*
 662          * We honor the requested 'hz' value.
 663          * We want to run stathz in the neighborhood of 128hz.
 664          * We would like profhz to run as often as possible.
 665          */
 666         if (singlemul <= 0 || singlemul > 20) {
 667                 if (hz >= 1500 || (hz % 128) == 0)
 668                         singlemul = 1;
 669                 else if (hz >= 750)
 670                         singlemul = 2;
 671                 else
 672                         singlemul = 4;
 673         }
 674         if (periodic) {
 675                 base = round_freq(timer, hz * singlemul);
 676                 singlemul = max((base + hz / 2) / hz, 1);
 677                 hz = (base + singlemul / 2) / singlemul;
 678                 if (base <= 128)
 679                         stathz = base;
 680                 else {
 681                         div = base / 128;
 682                         if (div >= singlemul && (div % singlemul) == 0)
 683                                 div++;
 684                         stathz = base / div;
 685                 }
 686                 profhz = stathz;
 687                 while ((profhz + stathz) <= 128 * 64)
 688                         profhz += stathz;
 689                 profhz = round_freq(timer, profhz);
 690         } else {
 691                 hz = round_freq(timer, hz);
 692                 stathz = round_freq(timer, 127);
 693                 profhz = round_freq(timer, stathz * 64);
 694         }
 695         tick = 1000000 / hz;
 696         FREQ2BT(hz, &hardperiod);
 697         FREQ2BT(stathz, &statperiod);
 698         FREQ2BT(profhz, &profperiod);
 699         ET_LOCK();
 700         configtimer(1);
 701         ET_UNLOCK();
 702 }
 703
 704 /*
 705  * Start per-CPU event timers on APs.
 706  */
 707 void
 708 cpu_initclocks_ap(void)
 709 {
 710         struct bintime now;
 711         struct pcpu_state *state;
 712
 713         state = DPCPU_PTR(timerstate);
 714         binuptime(&now);
 715         ET_HW_LOCK(state);
 716         if ((timer->et_flags & ET_FLAGS_PERCPU) == 0 && periodic) {
 717                 state->now = nexttick;
 718                 bintime_sub(&state->now, &timerperiod);
 719         } else
 720                 state->now = now;
 721         hardclock_sync(curcpu);
 722         handleevents(&state->now, 2);
 723         if (timer->et_flags & ET_FLAGS_PERCPU)
 724                 loadtimer(&now, 1);
 725         ET_HW_UNLOCK(state);
 726 }
 727
 728 /*
 729  * Switch to profiling clock rates.
 730  */
 731 void
 732 cpu_startprofclock(void)
 733 {
 734
 735         ET_LOCK();
 736         if (profiling == 0) {
 737                 if (periodic) {
 738                         configtimer(0);
 739                         profiling = 1;
 740                         configtimer(1);
 741                 } else
 742                         profiling = 1;
 743         } else
 744                 profiling++;
 745         ET_UNLOCK();
 746 }
 747
 748 /*
 749  * Switch to regular clock rates.
 750  */
 751 void
 752 cpu_stopprofclock(void)
 753 {
 754
 755         ET_LOCK();
 756         if (profiling == 1) {
 757                 if (periodic) {
 758                         configtimer(0);
 759                         profiling = 0;
 760                         configtimer(1);
 761                 } else
 762                 profiling = 0;
 763         } else
 764                 profiling--;
 765         ET_UNLOCK();
 766 }
 767
 768 /*
 769  * Switch to idle mode (all ticks handled).
 770  */
 771 void
 772 cpu_idleclock(void)
 773 {
 774         struct bintime now, t;
 775         struct pcpu_state *state;
 776
 777         if (idletick || busy ||
 778             (periodic && (timer->et_flags & ET_FLAGS_PERCPU))
 779 #ifdef DEVICE_POLLING
 780             || curcpu == CPU_FIRST()
 781 #endif
 782             )
 783                 return;
 784         state = DPCPU_PTR(timerstate);
 785         if (periodic)
 786                 now = state->now;
 787         else
 788                 binuptime(&now);
 789         CTR4(KTR_SPARE2, "idle at %d:    now  %d.%08x%08x",
 790             curcpu, now.sec, (unsigned int)(now.frac >> 32),
 791                              (unsigned int)(now.frac & 0xffffffff));
 792         getnextcpuevent(&t, 1);
 793         ET_HW_LOCK(state);
 794         state->idle = 1;
 795         state->nextevent = t;
 796         if (!periodic)
 797                 loadtimer(&now, 0);
 798         ET_HW_UNLOCK(state);
 799 }
 800
 801 /*
 802  * Switch to active mode (skip empty ticks).
 803  */
 804 void
 805 cpu_activeclock(void)
 806 {
 807         struct bintime now;
 808         struct pcpu_state *state;
 809         struct thread *td;
 810
 811         state = DPCPU_PTR(timerstate);
 812         if (state->idle == 0 || busy)
 813                 return;
 814         if (periodic)
 815                 now = state->now;
 816         else
 817                 binuptime(&now);
 818         CTR4(KTR_SPARE2, "active at %d:  now  %d.%08x%08x",
 819             curcpu, now.sec, (unsigned int)(now.frac >> 32),
 820                              (unsigned int)(now.frac & 0xffffffff));
 821         spinlock_enter();
 822         td = curthread;
 823         td->td_intr_nesting_level++;
 824         handleevents(&now, 1);
 825         td->td_intr_nesting_level--;
 826         spinlock_exit();
 827 }
 828
 829 #ifdef KDTRACE_HOOKS
 830 void
 831 clocksource_cyc_set(const struct bintime *t)
 832 {
 833         struct bintime now;
 834         struct pcpu_state *state;
 835
 836         state = DPCPU_PTR(timerstate);
 837         if (periodic)
 838                 now = state->now;
 839         else
 840                 binuptime(&now);
 841
 842         CTR4(KTR_SPARE2, "set_cyc at %d:  now  %d.%08x%08x",
 843             curcpu, now.sec, (unsigned int)(now.frac >> 32),
 844                              (unsigned int)(now.frac & 0xffffffff));
 845         CTR4(KTR_SPARE2, "set_cyc at %d:  t  %d.%08x%08x",
 846             curcpu, t->sec, (unsigned int)(t->frac >> 32),
 847                              (unsigned int)(t->frac & 0xffffffff));
 848
 849         ET_HW_LOCK(state);
 850         if (bintime_cmp(t, &state->nextcyc, ==)) {
 851                 ET_HW_UNLOCK(state);
 852                 return;
 853         }
 854         state->nextcyc = *t;
 855         if (bintime_cmp(&state->nextcyc, &state->nextevent, >=)) {
 856                 ET_HW_UNLOCK(state);
 857                 return;
 858         }
 859         state->nextevent = state->nextcyc;
 860         if (!periodic)
 861                 loadtimer(&now, 0);
 862         ET_HW_UNLOCK(state);
 863 }
 864 #endif
 865
 866 #ifdef SMP
 867 static void
 868 cpu_new_callout(int cpu, int ticks)
 869 {
 870         struct bintime tmp;
 871         struct pcpu_state *state;
 872
 873         CTR3(KTR_SPARE2, "new co at %d:    on %d in %d",
 874             curcpu, cpu, ticks);
 875         state = DPCPU_ID_PTR(cpu, timerstate);
 876         ET_HW_LOCK(state);
 877         if (state->idle == 0 || busy) {
 878                 ET_HW_UNLOCK(state);
 879                 return;
 880         }
 881         /*
 882          * If timer is periodic - just update next event time for target CPU.
 883          * If timer is global - there is chance it is already programmed.
 884          */
 885         if (periodic || (timer->et_flags & ET_FLAGS_PERCPU) == 0) {
 886                 tmp = hardperiod;
 887                 bintime_mul(&tmp, ticks - 1);
 888                 bintime_add(&tmp, &state->nexthard);
 889                 if (bintime_cmp(&tmp, &state->nextevent, <))
 890                         state->nextevent = tmp;
 891                 if (periodic ||
 892                     bintime_cmp(&state->nextevent, &nexttick, >=)) {
 893                         ET_HW_UNLOCK(state);
 894                         return;
 895                 }
 896         }
 897         /*
 898          * Otherwise we have to wake that CPU up, as we can't get present
 899          * bintime to reprogram global timer from here. If timer is per-CPU,
 900          * we by definition can't do it from here.
 901          */
 902         ET_HW_UNLOCK(state);
 903         if (timer->et_flags & ET_FLAGS_PERCPU) {
 904                 state->handle = 1;
 905                 ipi_cpu(cpu, IPI_HARDCLOCK);
 906         } else {
 907                 if (!cpu_idle_wakeup(cpu))
 908                         ipi_cpu(cpu, IPI_AST);
 909         }
 910 }
 911 #endif
 912
 913 /*
 914  * Report or change the active event timers hardware.
 915  */
 916 static int
 917 sysctl_kern_eventtimer_timer(SYSCTL_HANDLER_ARGS)
 918 {
 919         char buf[32];
 920         struct eventtimer *et;
 921         int error;
 922
 923         ET_LOCK();
 924         et = timer;
 925         snprintf(buf, sizeof(buf), "%s", et->et_name);
 926         ET_UNLOCK();
 927         error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
 928         ET_LOCK();
 929         et = timer;
 930         if (error != 0 || req->newptr == NULL ||
 931             strcasecmp(buf, et->et_name) == 0) {
 932                 ET_UNLOCK();
 933                 return (error);
 934         }
 935         et = et_find(buf, 0, 0);
 936         if (et == NULL) {
 937                 ET_UNLOCK();
 938                 return (ENOENT);
 939         }
 940         configtimer(0);
 941         et_free(timer);
 942         if (et->et_flags & ET_FLAGS_C3STOP)
 943                 cpu_disable_c3_sleep++;
 944         if (timer->et_flags & ET_FLAGS_C3STOP)
 945                 cpu_disable_c3_sleep--;
 946         periodic = want_periodic;
 947         timer = et;
 948         et_init(timer, timercb, NULL, NULL);
 949         configtimer(1);
 950         ET_UNLOCK();
 951         return (error);
 952 }
 953 SYSCTL_PROC(_kern_eventtimer, OID_AUTO, timer,
 954     CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE,
 955     0, 0, sysctl_kern_eventtimer_timer, "A", "Chosen event timer");
 956
 957 /*
 958  * Report or change the active event timer periodicity.
 959  */
 960 static int
 961 sysctl_kern_eventtimer_periodic(SYSCTL_HANDLER_ARGS)
 962 {
 963         int error, val;
 964
 965         val = periodic;
 966         error = sysctl_handle_int(oidp, &val, 0, req);
 967         if (error != 0 || req->newptr == NULL)
 968                 return (error);
 969         ET_LOCK();
 970         configtimer(0);
 971         periodic = want_periodic = val;
 972         configtimer(1);
 973         ET_UNLOCK();
 974         return (error);
 975 }
 976 SYSCTL_PROC(_kern_eventtimer, OID_AUTO, periodic,
 977     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
 978     0, 0, sysctl_kern_eventtimer_periodic, "I", "Enable event timer periodic mode");