sys/x86/x86/local_apic.c

   1 /*-
   2  * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org>
   3  * Copyright (c) 1996, by Steve Passe
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. The name of the developer may NOT be used to endorse or promote products
  12  *    derived from this software without specific prior written permission.
  13  * 3. Neither the name of the author nor the names of any co-contributors
  14  *    may be used to endorse or promote products derived from this software
  15  *    without specific prior written permission.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 /*
  31  * Local APIC support on Pentium and later processors.
  32  */
  33
  34 #include <sys/cdefs.h>
  35 __FBSDID("$FreeBSD$");
  36
  37 #include "opt_atpic.h"
  38 #include "opt_hwpmc_hooks.h"
  39
  40 #include "opt_ddb.h"
  41
  42 #include <sys/param.h>
  43 #include <sys/systm.h>
  44 #include <sys/bus.h>
  45 #include <sys/kernel.h>
  46 #include <sys/lock.h>
  47 #include <sys/mutex.h>
  48 #include <sys/pcpu.h>
  49 #include <sys/proc.h>
  50 #include <sys/sched.h>
  51 #include <sys/smp.h>
  52 #include <sys/sysctl.h>
  53 #include <sys/timeet.h>
  54
  55 #include <vm/vm.h>
  56 #include <vm/pmap.h>
  57
  58 #include <x86/apicreg.h>
  59 #include <machine/clock.h>
  60 #include <machine/cpufunc.h>
  61 #include <machine/cputypes.h>
  62 #include <machine/frame.h>
  63 #include <machine/intr_machdep.h>
  64 #include <x86/apicvar.h>
  65 #include <x86/mca.h>
  66 #include <machine/md_var.h>
  67 #include <machine/smp.h>
  68 #include <machine/specialreg.h>
  69 #include <x86/init.h>
  70
  71 #ifdef DDB
  72 #include <sys/interrupt.h>
  73 #include <ddb/ddb.h>
  74 #endif
  75
  76 #ifdef __amd64__
  77 #define SDT_APIC        SDT_SYSIGT
  78 #define SDT_APICT       SDT_SYSIGT
  79 #define GSEL_APIC       0
  80 #else
  81 #define SDT_APIC        SDT_SYS386IGT
  82 #define SDT_APICT       SDT_SYS386TGT
  83 #define GSEL_APIC       GSEL(GCODE_SEL, SEL_KPL)
  84 #endif
  85
  86 /* Sanity checks on IDT vectors. */
  87 CTASSERT(APIC_IO_INTS + APIC_NUM_IOINTS == APIC_TIMER_INT);
  88 CTASSERT(APIC_TIMER_INT < APIC_LOCAL_INTS);
  89 CTASSERT(APIC_LOCAL_INTS == 240);
  90 CTASSERT(IPI_STOP < APIC_SPURIOUS_INT);
  91
  92 /* Magic IRQ values for the timer and syscalls. */
  93 #define IRQ_TIMER       (NUM_IO_INTS + 1)
  94 #define IRQ_SYSCALL     (NUM_IO_INTS + 2)
  95 #define IRQ_DTRACE_RET  (NUM_IO_INTS + 3)
  96 #define IRQ_EVTCHN      (NUM_IO_INTS + 4)
  97
  98 enum lat_timer_mode {
  99         LAT_MODE_UNDEF =        0,
 100         LAT_MODE_PERIODIC =     1,
 101         LAT_MODE_ONESHOT =      2,
 102         LAT_MODE_DEADLINE =     3,
 103 };
 104
 105 /*
 106  * Support for local APICs.  Local APICs manage interrupts on each
 107  * individual processor as opposed to I/O APICs which receive interrupts
 108  * from I/O devices and then forward them on to the local APICs.
 109  *
 110  * Local APICs can also send interrupts to each other thus providing the
 111  * mechanism for IPIs.
 112  */
 113
 114 struct lvt {
 115         u_int lvt_edgetrigger:1;
 116         u_int lvt_activehi:1;
 117         u_int lvt_masked:1;
 118         u_int lvt_active:1;
 119         u_int lvt_mode:16;
 120         u_int lvt_vector:8;
 121 };
 122
 123 struct lapic {
 124         struct lvt la_lvts[APIC_LVT_MAX + 1];
 125         u_int la_id:8;
 126         u_int la_cluster:4;
 127         u_int la_cluster_id:2;
 128         u_int la_present:1;
 129         u_long *la_timer_count;
 130         uint64_t la_timer_period;
 131         enum lat_timer_mode la_timer_mode;
 132         uint32_t lvt_timer_base;
 133         uint32_t lvt_timer_last;
 134         /* Include IDT_SYSCALL to make indexing easier. */
 135         int la_ioint_irqs[APIC_NUM_IOINTS + 1];
 136 } static lapics[MAX_APIC_ID + 1];
 137
 138 /* Global defaults for local APIC LVT entries. */
 139 static struct lvt lvts[APIC_LVT_MAX + 1] = {
 140         { 1, 1, 1, 1, APIC_LVT_DM_EXTINT, 0 },  /* LINT0: masked ExtINT */
 141         { 1, 1, 0, 1, APIC_LVT_DM_NMI, 0 },     /* LINT1: NMI */
 142         { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_TIMER_INT },      /* Timer */
 143         { 1, 1, 0, 1, APIC_LVT_DM_FIXED, APIC_ERROR_INT },      /* Error */
 144         { 1, 1, 1, 1, APIC_LVT_DM_NMI, 0 },     /* PMC */
 145         { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_THERMAL_INT },    /* Thermal */
 146         { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_CMC_INT },        /* CMCI */
 147 };
 148
 149 static inthand_t *ioint_handlers[] = {
 150         NULL,                   /* 0 - 31 */
 151         IDTVEC(apic_isr1),      /* 32 - 63 */
 152         IDTVEC(apic_isr2),      /* 64 - 95 */
 153         IDTVEC(apic_isr3),      /* 96 - 127 */
 154         IDTVEC(apic_isr4),      /* 128 - 159 */
 155         IDTVEC(apic_isr5),      /* 160 - 191 */
 156         IDTVEC(apic_isr6),      /* 192 - 223 */
 157         IDTVEC(apic_isr7),      /* 224 - 255 */
 158 };
 159
 160
 161 static u_int32_t lapic_timer_divisors[] = {
 162         APIC_TDCR_1, APIC_TDCR_2, APIC_TDCR_4, APIC_TDCR_8, APIC_TDCR_16,
 163         APIC_TDCR_32, APIC_TDCR_64, APIC_TDCR_128
 164 };
 165
 166 extern inthand_t IDTVEC(rsvd);
 167
 168 volatile char *lapic_map;
 169 vm_paddr_t lapic_paddr;
 170 int x2apic_mode;
 171 int lapic_eoi_suppression;
 172 static int lapic_timer_tsc_deadline;
 173 static u_long lapic_timer_divisor;
 174 static struct eventtimer lapic_et;
 175 #ifdef SMP
 176 static uint64_t lapic_ipi_wait_mult;
 177 #endif
 178
 179 SYSCTL_NODE(_hw, OID_AUTO, apic, CTLFLAG_RD, 0, "APIC options");
 180 SYSCTL_INT(_hw_apic, OID_AUTO, x2apic_mode, CTLFLAG_RD, &x2apic_mode, 0, "");
 181 SYSCTL_INT(_hw_apic, OID_AUTO, eoi_suppression, CTLFLAG_RD,
 182     &lapic_eoi_suppression, 0, "");
 183 SYSCTL_INT(_hw_apic, OID_AUTO, timer_tsc_deadline, CTLFLAG_RD,
 184     &lapic_timer_tsc_deadline, 0, "");
 185
 186 static uint32_t
 187 lapic_read32(enum LAPIC_REGISTERS reg)
 188 {
 189         uint32_t res;
 190
 191         if (x2apic_mode) {
 192                 res = rdmsr32(MSR_APIC_000 + reg);
 193         } else {
 194                 res = *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL);
 195         }
 196         return (res);
 197 }
 198
 199 static void
 200 lapic_write32(enum LAPIC_REGISTERS reg, uint32_t val)
 201 {
 202
 203         if (x2apic_mode) {
 204                 mfence();
 205                 wrmsr(MSR_APIC_000 + reg, val);
 206         } else {
 207                 *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL) = val;
 208         }
 209 }
 210
 211 static void
 212 lapic_write32_nofence(enum LAPIC_REGISTERS reg, uint32_t val)
 213 {
 214
 215         if (x2apic_mode) {
 216                 wrmsr(MSR_APIC_000 + reg, val);
 217         } else {
 218                 *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL) = val;
 219         }
 220 }
 221
 222 #ifdef SMP
 223 static uint64_t
 224 lapic_read_icr(void)
 225 {
 226         uint64_t v;
 227         uint32_t vhi, vlo;
 228
 229         if (x2apic_mode) {
 230                 v = rdmsr(MSR_APIC_000 + LAPIC_ICR_LO);
 231         } else {
 232                 vhi = lapic_read32(LAPIC_ICR_HI);
 233                 vlo = lapic_read32(LAPIC_ICR_LO);
 234                 v = ((uint64_t)vhi << 32) | vlo;
 235         }
 236         return (v);
 237 }
 238
 239 static uint64_t
 240 lapic_read_icr_lo(void)
 241 {
 242
 243         return (lapic_read32(LAPIC_ICR_LO));
 244 }
 245
 246 static void
 247 lapic_write_icr(uint32_t vhi, uint32_t vlo)
 248 {
 249         uint64_t v;
 250
 251         if (x2apic_mode) {
 252                 v = ((uint64_t)vhi << 32) | vlo;
 253                 mfence();
 254                 wrmsr(MSR_APIC_000 + LAPIC_ICR_LO, v);
 255         } else {
 256                 lapic_write32(LAPIC_ICR_HI, vhi);
 257                 lapic_write32(LAPIC_ICR_LO, vlo);
 258         }
 259 }
 260 #endif /* SMP */
 261
 262 static void
 263 native_lapic_enable_x2apic(void)
 264 {
 265         uint64_t apic_base;
 266
 267         apic_base = rdmsr(MSR_APICBASE);
 268         apic_base |= APICBASE_X2APIC | APICBASE_ENABLED;
 269         wrmsr(MSR_APICBASE, apic_base);
 270 }
 271
 272 static void     lapic_enable(void);
 273 static void     lapic_resume(struct pic *pic, bool suspend_cancelled);
 274 static void     lapic_timer_oneshot(struct lapic *);
 275 static void     lapic_timer_oneshot_nointr(struct lapic *, uint32_t);
 276 static void     lapic_timer_periodic(struct lapic *);
 277 static void     lapic_timer_deadline(struct lapic *);
 278 static void     lapic_timer_stop(struct lapic *);
 279 static void     lapic_timer_set_divisor(u_int divisor);
 280 static uint32_t lvt_mode(struct lapic *la, u_int pin, uint32_t value);
 281 static int      lapic_et_start(struct eventtimer *et,
 282                     sbintime_t first, sbintime_t period);
 283 static int      lapic_et_stop(struct eventtimer *et);
 284 static u_int    apic_idt_to_irq(u_int apic_id, u_int vector);
 285 static void     lapic_set_tpr(u_int vector);
 286
 287 struct pic lapic_pic = { .pic_resume = lapic_resume };
 288
 289 /* Forward declarations for apic_ops */
 290 static void     native_lapic_create(u_int apic_id, int boot_cpu);
 291 static void     native_lapic_init(vm_paddr_t addr);
 292 static void     native_lapic_xapic_mode(void);
 293 static void     native_lapic_setup(int boot);
 294 static void     native_lapic_dump(const char *str);
 295 static void     native_lapic_disable(void);
 296 static void     native_lapic_eoi(void);
 297 static int      native_lapic_id(void);
 298 static int      native_lapic_intr_pending(u_int vector);
 299 static u_int    native_apic_cpuid(u_int apic_id);
 300 static u_int    native_apic_alloc_vector(u_int apic_id, u_int irq);
 301 static u_int    native_apic_alloc_vectors(u_int apic_id, u_int *irqs,
 302                     u_int count, u_int align);
 303 static void     native_apic_disable_vector(u_int apic_id, u_int vector);
 304 static void     native_apic_enable_vector(u_int apic_id, u_int vector);
 305 static void     native_apic_free_vector(u_int apic_id, u_int vector, u_int irq);
 306 static void     native_lapic_set_logical_id(u_int apic_id, u_int cluster,
 307                     u_int cluster_id);
 308 static int      native_lapic_enable_pmc(void);
 309 static void     native_lapic_disable_pmc(void);
 310 static void     native_lapic_reenable_pmc(void);
 311 static void     native_lapic_enable_cmc(void);
 312 static int      native_lapic_set_lvt_mask(u_int apic_id, u_int lvt,
 313                     u_char masked);
 314 static int      native_lapic_set_lvt_mode(u_int apic_id, u_int lvt,
 315                     uint32_t mode);
 316 static int      native_lapic_set_lvt_polarity(u_int apic_id, u_int lvt,
 317                     enum intr_polarity pol);
 318 static int      native_lapic_set_lvt_triggermode(u_int apic_id, u_int lvt,
 319                     enum intr_trigger trigger);
 320 #ifdef SMP
 321 static void     native_lapic_ipi_raw(register_t icrlo, u_int dest);
 322 static void     native_lapic_ipi_vectored(u_int vector, int dest);
 323 static int      native_lapic_ipi_wait(int delay);
 324 #endif /* SMP */
 325 static int      native_lapic_ipi_alloc(inthand_t *ipifunc);
 326 static void     native_lapic_ipi_free(int vector);
 327
 328 struct apic_ops apic_ops = {
 329         .create                 = native_lapic_create,
 330         .init                   = native_lapic_init,
 331         .xapic_mode             = native_lapic_xapic_mode,
 332         .setup                  = native_lapic_setup,
 333         .dump                   = native_lapic_dump,
 334         .disable                = native_lapic_disable,
 335         .eoi                    = native_lapic_eoi,
 336         .id                     = native_lapic_id,
 337         .intr_pending           = native_lapic_intr_pending,
 338         .set_logical_id         = native_lapic_set_logical_id,
 339         .cpuid                  = native_apic_cpuid,
 340         .alloc_vector           = native_apic_alloc_vector,
 341         .alloc_vectors          = native_apic_alloc_vectors,
 342         .enable_vector          = native_apic_enable_vector,
 343         .disable_vector         = native_apic_disable_vector,
 344         .free_vector            = native_apic_free_vector,
 345         .enable_pmc             = native_lapic_enable_pmc,
 346         .disable_pmc            = native_lapic_disable_pmc,
 347         .reenable_pmc           = native_lapic_reenable_pmc,
 348         .enable_cmc             = native_lapic_enable_cmc,
 349 #ifdef SMP
 350         .ipi_raw                = native_lapic_ipi_raw,
 351         .ipi_vectored           = native_lapic_ipi_vectored,
 352         .ipi_wait               = native_lapic_ipi_wait,
 353 #endif
 354         .ipi_alloc              = native_lapic_ipi_alloc,
 355         .ipi_free               = native_lapic_ipi_free,
 356         .set_lvt_mask           = native_lapic_set_lvt_mask,
 357         .set_lvt_mode           = native_lapic_set_lvt_mode,
 358         .set_lvt_polarity       = native_lapic_set_lvt_polarity,
 359         .set_lvt_triggermode    = native_lapic_set_lvt_triggermode,
 360 };
 361
 362 static uint32_t
 363 lvt_mode(struct lapic *la, u_int pin, uint32_t value)
 364 {
 365         struct lvt *lvt;
 366
 367         KASSERT(pin <= APIC_LVT_MAX, ("%s: pin %u out of range", __func__, pin));
 368         if (la->la_lvts[pin].lvt_active)
 369                 lvt = &la->la_lvts[pin];
 370         else
 371                 lvt = &lvts[pin];
 372
 373         value &= ~(APIC_LVT_M | APIC_LVT_TM | APIC_LVT_IIPP | APIC_LVT_DM |
 374             APIC_LVT_VECTOR);
 375         if (lvt->lvt_edgetrigger == 0)
 376                 value |= APIC_LVT_TM;
 377         if (lvt->lvt_activehi == 0)
 378                 value |= APIC_LVT_IIPP_INTALO;
 379         if (lvt->lvt_masked)
 380                 value |= APIC_LVT_M;
 381         value |= lvt->lvt_mode;
 382         switch (lvt->lvt_mode) {
 383         case APIC_LVT_DM_NMI:
 384         case APIC_LVT_DM_SMI:
 385         case APIC_LVT_DM_INIT:
 386         case APIC_LVT_DM_EXTINT:
 387                 if (!lvt->lvt_edgetrigger && bootverbose) {
 388                         printf("lapic%u: Forcing LINT%u to edge trigger\n",
 389                             la->la_id, pin);
 390                         value |= APIC_LVT_TM;
 391                 }
 392                 /* Use a vector of 0. */
 393                 break;
 394         case APIC_LVT_DM_FIXED:
 395                 value |= lvt->lvt_vector;
 396                 break;
 397         default:
 398                 panic("bad APIC LVT delivery mode: %#x\n", value);
 399         }
 400         return (value);
 401 }
 402
 403 /*
 404  * Map the local APIC and setup necessary interrupt vectors.
 405  */
 406 static void
 407 native_lapic_init(vm_paddr_t addr)
 408 {
 409 #ifdef SMP
 410         uint64_t r, r1, r2, rx;
 411 #endif
 412         uint32_t ver;
 413         u_int regs[4];
 414         int i, arat;
 415
 416         /*
 417          * Enable x2APIC mode if possible. Map the local APIC
 418          * registers page.
 419          *
 420          * Keep the LAPIC registers page mapped uncached for x2APIC
 421          * mode too, to have direct map page attribute set to
 422          * uncached.  This is needed to work around CPU errata present
 423          * on all Intel processors.
 424          */
 425         KASSERT(trunc_page(addr) == addr,
 426             ("local APIC not aligned on a page boundary"));
 427         lapic_paddr = addr;
 428         lapic_map = pmap_mapdev(addr, PAGE_SIZE);
 429         if (x2apic_mode) {
 430                 native_lapic_enable_x2apic();
 431                 lapic_map = NULL;
 432         }
 433
 434         /* Setup the spurious interrupt handler. */
 435         setidt(APIC_SPURIOUS_INT, IDTVEC(spuriousint), SDT_APIC, SEL_KPL,
 436             GSEL_APIC);
 437
 438         /* Perform basic initialization of the BSP's local APIC. */
 439         lapic_enable();
 440
 441         /* Set BSP's per-CPU local APIC ID. */
 442         PCPU_SET(apic_id, lapic_id());
 443
 444         /* Local APIC timer interrupt. */
 445         setidt(APIC_TIMER_INT, IDTVEC(timerint), SDT_APIC, SEL_KPL, GSEL_APIC);
 446
 447         /* Local APIC error interrupt. */
 448         setidt(APIC_ERROR_INT, IDTVEC(errorint), SDT_APIC, SEL_KPL, GSEL_APIC);
 449
 450         /* XXX: Thermal interrupt */
 451
 452         /* Local APIC CMCI. */
 453         setidt(APIC_CMC_INT, IDTVEC(cmcint), SDT_APICT, SEL_KPL, GSEL_APIC);
 454
 455         if ((resource_int_value("apic", 0, "clock", &i) != 0 || i != 0)) {
 456                 arat = 0;
 457                 /* Intel CPUID 0x06 EAX[2] set if APIC timer runs in C3. */
 458                 if (cpu_vendor_id == CPU_VENDOR_INTEL && cpu_high >= 6) {
 459                         do_cpuid(0x06, regs);
 460                         if ((regs[0] & CPUTPM1_ARAT) != 0)
 461                                 arat = 1;
 462                 }
 463                 bzero(&lapic_et, sizeof(lapic_et));
 464                 lapic_et.et_name = "LAPIC";
 465                 lapic_et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT |
 466                     ET_FLAGS_PERCPU;
 467                 lapic_et.et_quality = 600;
 468                 if (!arat) {
 469                         lapic_et.et_flags |= ET_FLAGS_C3STOP;
 470                         lapic_et.et_quality -= 200;
 471                 } else if ((cpu_feature & CPUID_TSC) != 0 &&
 472                     (cpu_feature2 & CPUID2_TSCDLT) != 0 &&
 473                     tsc_is_invariant && tsc_freq != 0) {
 474                         lapic_timer_tsc_deadline = 1;
 475                         TUNABLE_INT_FETCH("hw.lapic_tsc_deadline",
 476                             &lapic_timer_tsc_deadline);
 477                 }
 478
 479                 lapic_et.et_frequency = 0;
 480                 /* We don't know frequency yet, so trying to guess. */
 481                 lapic_et.et_min_period = 0x00001000LL;
 482                 lapic_et.et_max_period = SBT_1S;
 483                 lapic_et.et_start = lapic_et_start;
 484                 lapic_et.et_stop = lapic_et_stop;
 485                 lapic_et.et_priv = NULL;
 486                 et_register(&lapic_et);
 487         }
 488
 489         /*
 490          * Set lapic_eoi_suppression after lapic_enable(), to not
 491          * enable suppression in the hardware prematurely.  Note that
 492          * we by default enable suppression even when system only has
 493          * one IO-APIC, since EOI is broadcasted to all APIC agents,
 494          * including CPUs, otherwise.
 495          *
 496          * It seems that at least some KVM versions report
 497          * EOI_SUPPRESSION bit, but auto-EOI does not work.
 498          */
 499         ver = lapic_read32(LAPIC_VERSION);
 500         if ((ver & APIC_VER_EOI_SUPPRESSION) != 0) {
 501                 lapic_eoi_suppression = 1;
 502                 if (vm_guest == VM_GUEST_VM &&
 503                     !strcmp(hv_vendor, "KVMKVMKVM")) {
 504                         if (bootverbose)
 505                                 printf(
 506                        "KVM -- disabling lapic eoi suppression\n");
 507                         lapic_eoi_suppression = 0;
 508                 }
 509                 TUNABLE_INT_FETCH("hw.lapic_eoi_suppression",
 510                     &lapic_eoi_suppression);
 511         }
 512
 513 #ifdef SMP
 514 #define LOOPS   1000000
 515         /*
 516          * Calibrate the busy loop waiting for IPI ack in xAPIC mode.
 517          * lapic_ipi_wait_mult contains the number of iterations which
 518          * approximately delay execution for 1 microsecond (the
 519          * argument to native_lapic_ipi_wait() is in microseconds).
 520          *
 521          * We assume that TSC is present and already measured.
 522          * Possible TSC frequency jumps are irrelevant to the
 523          * calibration loop below, the CPU clock management code is
 524          * not yet started, and we do not enter sleep states.
 525          */
 526         KASSERT((cpu_feature & CPUID_TSC) != 0 && tsc_freq != 0,
 527             ("TSC not initialized"));
 528         r = rdtsc();
 529         for (rx = 0; rx < LOOPS; rx++) {
 530                 (void)lapic_read_icr_lo();
 531                 ia32_pause();
 532         }
 533         r = rdtsc() - r;
 534         r1 = tsc_freq * LOOPS;
 535         r2 = r * 1000000;
 536         lapic_ipi_wait_mult = r1 >= r2 ? r1 / r2 : 1;
 537         if (bootverbose) {
 538                 printf("LAPIC: ipi_wait() us multiplier %ju (r %ju tsc %ju)\n",
 539                     (uintmax_t)lapic_ipi_wait_mult, (uintmax_t)r,
 540                     (uintmax_t)tsc_freq);
 541         }
 542 #undef LOOPS
 543 #endif /* SMP */
 544 }
 545
 546 /*
 547  * Create a local APIC instance.
 548  */
 549 static void
 550 native_lapic_create(u_int apic_id, int boot_cpu)
 551 {
 552         int i;
 553
 554         if (apic_id > MAX_APIC_ID) {
 555                 printf("APIC: Ignoring local APIC with ID %d\n", apic_id);
 556                 if (boot_cpu)
 557                         panic("Can't ignore BSP");
 558                 return;
 559         }
 560         KASSERT(!lapics[apic_id].la_present, ("duplicate local APIC %u",
 561             apic_id));
 562
 563         /*
 564          * Assume no local LVT overrides and a cluster of 0 and
 565          * intra-cluster ID of 0.
 566          */
 567         lapics[apic_id].la_present = 1;
 568         lapics[apic_id].la_id = apic_id;
 569         for (i = 0; i <= APIC_LVT_MAX; i++) {
 570                 lapics[apic_id].la_lvts[i] = lvts[i];
 571                 lapics[apic_id].la_lvts[i].lvt_active = 0;
 572         }
 573         for (i = 0; i <= APIC_NUM_IOINTS; i++)
 574             lapics[apic_id].la_ioint_irqs[i] = -1;
 575         lapics[apic_id].la_ioint_irqs[IDT_SYSCALL - APIC_IO_INTS] = IRQ_SYSCALL;
 576         lapics[apic_id].la_ioint_irqs[APIC_TIMER_INT - APIC_IO_INTS] =
 577             IRQ_TIMER;
 578 #ifdef KDTRACE_HOOKS
 579         lapics[apic_id].la_ioint_irqs[IDT_DTRACE_RET - APIC_IO_INTS] =
 580             IRQ_DTRACE_RET;
 581 #endif
 582 #ifdef XENHVM
 583         lapics[apic_id].la_ioint_irqs[IDT_EVTCHN - APIC_IO_INTS] = IRQ_EVTCHN;
 584 #endif
 585
 586
 587 #ifdef SMP
 588         cpu_add(apic_id, boot_cpu);
 589 #endif
 590 }
 591
 592 /*
 593  * Dump contents of local APIC registers
 594  */
 595 static void
 596 native_lapic_dump(const char* str)
 597 {
 598         uint32_t maxlvt;
 599
 600         maxlvt = (lapic_read32(LAPIC_VERSION) & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
 601         printf("cpu%d %s:\n", PCPU_GET(cpuid), str);
 602         printf("     ID: 0x%08x   VER: 0x%08x LDR: 0x%08x DFR: 0x%08x",
 603             lapic_read32(LAPIC_ID), lapic_read32(LAPIC_VERSION),
 604             lapic_read32(LAPIC_LDR), x2apic_mode ? 0 : lapic_read32(LAPIC_DFR));
 605         if ((cpu_feature2 & CPUID2_X2APIC) != 0)
 606                 printf(" x2APIC: %d", x2apic_mode);
 607         printf("\n  lint0: 0x%08x lint1: 0x%08x TPR: 0x%08x SVR: 0x%08x\n",
 608             lapic_read32(LAPIC_LVT_LINT0), lapic_read32(LAPIC_LVT_LINT1),
 609             lapic_read32(LAPIC_TPR), lapic_read32(LAPIC_SVR));
 610         printf("  timer: 0x%08x therm: 0x%08x err: 0x%08x",
 611             lapic_read32(LAPIC_LVT_TIMER), lapic_read32(LAPIC_LVT_THERMAL),
 612             lapic_read32(LAPIC_LVT_ERROR));
 613         if (maxlvt >= APIC_LVT_PMC)
 614                 printf(" pmc: 0x%08x", lapic_read32(LAPIC_LVT_PCINT));
 615         printf("\n");
 616         if (maxlvt >= APIC_LVT_CMCI)
 617                 printf("   cmci: 0x%08x\n", lapic_read32(LAPIC_LVT_CMCI));
 618 }
 619
 620 static void
 621 native_lapic_xapic_mode(void)
 622 {
 623         register_t saveintr;
 624
 625         saveintr = intr_disable();
 626         if (x2apic_mode)
 627                 native_lapic_enable_x2apic();
 628         intr_restore(saveintr);
 629 }
 630
 631 static void
 632 native_lapic_setup(int boot)
 633 {
 634         struct lapic *la;
 635         uint32_t maxlvt;
 636         register_t saveintr;
 637         char buf[MAXCOMLEN + 1];
 638
 639         saveintr = intr_disable();
 640
 641         la = &lapics[lapic_id()];
 642         KASSERT(la->la_present, ("missing APIC structure"));
 643         maxlvt = (lapic_read32(LAPIC_VERSION) & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
 644
 645         /* Initialize the TPR to allow all interrupts. */
 646         lapic_set_tpr(0);
 647
 648         /* Setup spurious vector and enable the local APIC. */
 649         lapic_enable();
 650
 651         /* Program LINT[01] LVT entries. */
 652         lapic_write32(LAPIC_LVT_LINT0, lvt_mode(la, APIC_LVT_LINT0,
 653             lapic_read32(LAPIC_LVT_LINT0)));
 654         lapic_write32(LAPIC_LVT_LINT1, lvt_mode(la, APIC_LVT_LINT1,
 655             lapic_read32(LAPIC_LVT_LINT1)));
 656
 657         /* Program the PMC LVT entry if present. */
 658         if (maxlvt >= APIC_LVT_PMC) {
 659                 lapic_write32(LAPIC_LVT_PCINT, lvt_mode(la, APIC_LVT_PMC,
 660                     LAPIC_LVT_PCINT));
 661         }
 662
 663         /* Program timer LVT and setup handler. */
 664         la->lvt_timer_base = lvt_mode(la, APIC_LVT_TIMER,
 665             lapic_read32(LAPIC_LVT_TIMER));
 666         la->lvt_timer_last = la->lvt_timer_base;
 667         lapic_write32(LAPIC_LVT_TIMER, la->lvt_timer_base);
 668         if (boot) {
 669                 snprintf(buf, sizeof(buf), "cpu%d:timer", PCPU_GET(cpuid));
 670                 intrcnt_add(buf, &la->la_timer_count);
 671         }
 672
 673         /* Setup the timer if configured. */
 674         if (la->la_timer_mode != LAT_MODE_UNDEF) {
 675                 KASSERT(la->la_timer_period != 0, ("lapic%u: zero divisor",
 676                     lapic_id()));
 677                 switch (la->la_timer_mode) {
 678                 case LAT_MODE_PERIODIC:
 679                         lapic_timer_set_divisor(lapic_timer_divisor);
 680                         lapic_timer_periodic(la);
 681                         break;
 682                 case LAT_MODE_ONESHOT:
 683                         lapic_timer_set_divisor(lapic_timer_divisor);
 684                         lapic_timer_oneshot(la);
 685                         break;
 686                 case LAT_MODE_DEADLINE:
 687                         lapic_timer_deadline(la);
 688                         break;
 689                 default:
 690                         panic("corrupted la_timer_mode %p %d", la,
 691                             la->la_timer_mode);
 692                 }
 693         }
 694
 695         /* Program error LVT and clear any existing errors. */
 696         lapic_write32(LAPIC_LVT_ERROR, lvt_mode(la, APIC_LVT_ERROR,
 697             lapic_read32(LAPIC_LVT_ERROR)));
 698         lapic_write32(LAPIC_ESR, 0);
 699
 700         /* XXX: Thermal LVT */
 701
 702         /* Program the CMCI LVT entry if present. */
 703         if (maxlvt >= APIC_LVT_CMCI) {
 704                 lapic_write32(LAPIC_LVT_CMCI, lvt_mode(la, APIC_LVT_CMCI,
 705                     lapic_read32(LAPIC_LVT_CMCI)));
 706         }
 707
 708         intr_restore(saveintr);
 709 }
 710
 711 static void
 712 native_lapic_reenable_pmc(void)
 713 {
 714 #ifdef HWPMC_HOOKS
 715         uint32_t value;
 716
 717         value = lapic_read32(LAPIC_LVT_PCINT);
 718         value &= ~APIC_LVT_M;
 719         lapic_write32(LAPIC_LVT_PCINT, value);
 720 #endif
 721 }
 722
 723 #ifdef HWPMC_HOOKS
 724 static void
 725 lapic_update_pmc(void *dummy)
 726 {
 727         struct lapic *la;
 728
 729         la = &lapics[lapic_id()];
 730         lapic_write32(LAPIC_LVT_PCINT, lvt_mode(la, APIC_LVT_PMC,
 731             lapic_read32(LAPIC_LVT_PCINT)));
 732 }
 733 #endif
 734
 735 static int
 736 native_lapic_enable_pmc(void)
 737 {
 738 #ifdef HWPMC_HOOKS
 739         u_int32_t maxlvt;
 740
 741         /* Fail if the local APIC is not present. */
 742         if (!x2apic_mode && lapic_map == NULL)
 743                 return (0);
 744
 745         /* Fail if the PMC LVT is not present. */
 746         maxlvt = (lapic_read32(LAPIC_VERSION) & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
 747         if (maxlvt < APIC_LVT_PMC)
 748                 return (0);
 749
 750         lvts[APIC_LVT_PMC].lvt_masked = 0;
 751
 752 #ifdef SMP
 753         /*
 754          * If hwpmc was loaded at boot time then the APs may not be
 755          * started yet.  In that case, don't forward the request to
 756          * them as they will program the lvt when they start.
 757          */
 758         if (smp_started)
 759                 smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL);
 760         else
 761 #endif
 762                 lapic_update_pmc(NULL);
 763         return (1);
 764 #else
 765         return (0);
 766 #endif
 767 }
 768
 769 static void
 770 native_lapic_disable_pmc(void)
 771 {
 772 #ifdef HWPMC_HOOKS
 773         u_int32_t maxlvt;
 774
 775         /* Fail if the local APIC is not present. */
 776         if (!x2apic_mode && lapic_map == NULL)
 777                 return;
 778
 779         /* Fail if the PMC LVT is not present. */
 780         maxlvt = (lapic_read32(LAPIC_VERSION) & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
 781         if (maxlvt < APIC_LVT_PMC)
 782                 return;
 783
 784         lvts[APIC_LVT_PMC].lvt_masked = 1;
 785
 786 #ifdef SMP
 787         /* The APs should always be started when hwpmc is unloaded. */
 788         KASSERT(mp_ncpus == 1 || smp_started, ("hwpmc unloaded too early"));
 789 #endif
 790         smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL);
 791 #endif
 792 }
 793
 794 static void
 795 lapic_calibrate_initcount(struct eventtimer *et, struct lapic *la)
 796 {
 797         u_long value;
 798
 799         /* Start off with a divisor of 2 (power on reset default). */
 800         lapic_timer_divisor = 2;
 801         /* Try to calibrate the local APIC timer. */
 802         do {
 803                 lapic_timer_set_divisor(lapic_timer_divisor);
 804                 lapic_timer_oneshot_nointr(la, APIC_TIMER_MAX_COUNT);
 805                 DELAY(1000000);
 806                 value = APIC_TIMER_MAX_COUNT - lapic_read32(LAPIC_CCR_TIMER);
 807                 if (value != APIC_TIMER_MAX_COUNT)
 808                         break;
 809                 lapic_timer_divisor <<= 1;
 810         } while (lapic_timer_divisor <= 128);
 811         if (lapic_timer_divisor > 128)
 812                 panic("lapic: Divisor too big");
 813         if (bootverbose) {
 814                 printf("lapic: Divisor %lu, Frequency %lu Hz\n",
 815                     lapic_timer_divisor, value);
 816         }
 817         et->et_frequency = value;
 818 }
 819
 820 static void
 821 lapic_calibrate_deadline(struct eventtimer *et, struct lapic *la __unused)
 822 {
 823
 824         et->et_frequency = tsc_freq;
 825         if (bootverbose) {
 826                 printf("lapic: deadline tsc mode, Frequency %ju Hz\n",
 827                     (uintmax_t)et->et_frequency);
 828         }
 829 }
 830
 831 static int
 832 lapic_et_start(struct eventtimer *et, sbintime_t first, sbintime_t period)
 833 {
 834         struct lapic *la;
 835
 836         la = &lapics[PCPU_GET(apic_id)];
 837         if (et->et_frequency == 0) {
 838                 if (lapic_timer_tsc_deadline)
 839                         lapic_calibrate_deadline(et, la);
 840                 else
 841                         lapic_calibrate_initcount(et, la);
 842                 et->et_min_period = (0x00000002LLU << 32) / et->et_frequency;
 843                 et->et_max_period = (0xfffffffeLLU << 32) / et->et_frequency;
 844         }
 845         if (period != 0) {
 846                 if (la->la_timer_mode == LAT_MODE_UNDEF)
 847                         lapic_timer_set_divisor(lapic_timer_divisor);
 848                 la->la_timer_mode = LAT_MODE_PERIODIC;
 849                 la->la_timer_period = ((uint32_t)et->et_frequency * period) >>
 850                     32;
 851                 lapic_timer_periodic(la);
 852         } else if (lapic_timer_tsc_deadline) {
 853                 la->la_timer_mode = LAT_MODE_DEADLINE;
 854                 la->la_timer_period = (et->et_frequency * first) >> 32;
 855                 lapic_timer_deadline(la);
 856         } else {
 857                 if (la->la_timer_mode == LAT_MODE_UNDEF)
 858                         lapic_timer_set_divisor(lapic_timer_divisor);
 859                 la->la_timer_mode = LAT_MODE_ONESHOT;
 860                 la->la_timer_period = ((uint32_t)et->et_frequency * first) >>
 861                     32;
 862                 lapic_timer_oneshot(la);
 863         }
 864         return (0);
 865 }
 866
 867 static int
 868 lapic_et_stop(struct eventtimer *et)
 869 {
 870         struct lapic *la;
 871
 872         la = &lapics[PCPU_GET(apic_id)];
 873         lapic_timer_stop(la);
 874         la->la_timer_mode = LAT_MODE_UNDEF;
 875         return (0);
 876 }
 877
 878 static void
 879 native_lapic_disable(void)
 880 {
 881         uint32_t value;
 882
 883         /* Software disable the local APIC. */
 884         value = lapic_read32(LAPIC_SVR);
 885         value &= ~APIC_SVR_SWEN;
 886         lapic_write32(LAPIC_SVR, value);
 887 }
 888
 889 static void
 890 lapic_enable(void)
 891 {
 892         uint32_t value;
 893
 894         /* Program the spurious vector to enable the local APIC. */
 895         value = lapic_read32(LAPIC_SVR);
 896         value &= ~(APIC_SVR_VECTOR | APIC_SVR_FOCUS);
 897         value |= APIC_SVR_FEN | APIC_SVR_SWEN | APIC_SPURIOUS_INT;
 898         if (lapic_eoi_suppression)
 899                 value |= APIC_SVR_EOI_SUPPRESSION;
 900         lapic_write32(LAPIC_SVR, value);
 901 }
 902
 903 /* Reset the local APIC on the BSP during resume. */
 904 static void
 905 lapic_resume(struct pic *pic, bool suspend_cancelled)
 906 {
 907
 908         lapic_setup(0);
 909 }
 910
 911 static int
 912 native_lapic_id(void)
 913 {
 914         uint32_t v;
 915
 916         KASSERT(x2apic_mode || lapic_map != NULL, ("local APIC is not mapped"));
 917         v = lapic_read32(LAPIC_ID);
 918         if (!x2apic_mode)
 919                 v >>= APIC_ID_SHIFT;
 920         return (v);
 921 }
 922
 923 static int
 924 native_lapic_intr_pending(u_int vector)
 925 {
 926         uint32_t irr;
 927
 928         /*
 929          * The IRR registers are an array of registers each of which
 930          * only describes 32 interrupts in the low 32 bits.  Thus, we
 931          * divide the vector by 32 to get the register index.
 932          * Finally, we modulus the vector by 32 to determine the
 933          * individual bit to test.
 934          */
 935         irr = lapic_read32(LAPIC_IRR0 + vector / 32);
 936         return (irr & 1 << (vector % 32));
 937 }
 938
 939 static void
 940 native_lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id)
 941 {
 942         struct lapic *la;
 943
 944         KASSERT(lapics[apic_id].la_present, ("%s: APIC %u doesn't exist",
 945             __func__, apic_id));
 946         KASSERT(cluster <= APIC_MAX_CLUSTER, ("%s: cluster %u too big",
 947             __func__, cluster));
 948         KASSERT(cluster_id <= APIC_MAX_INTRACLUSTER_ID,
 949             ("%s: intra cluster id %u too big", __func__, cluster_id));
 950         la = &lapics[apic_id];
 951         la->la_cluster = cluster;
 952         la->la_cluster_id = cluster_id;
 953 }
 954
 955 static int
 956 native_lapic_set_lvt_mask(u_int apic_id, u_int pin, u_char masked)
 957 {
 958
 959         if (pin > APIC_LVT_MAX)
 960                 return (EINVAL);
 961         if (apic_id == APIC_ID_ALL) {
 962                 lvts[pin].lvt_masked = masked;
 963                 if (bootverbose)
 964                         printf("lapic:");
 965         } else {
 966                 KASSERT(lapics[apic_id].la_present,
 967                     ("%s: missing APIC %u", __func__, apic_id));
 968                 lapics[apic_id].la_lvts[pin].lvt_masked = masked;
 969                 lapics[apic_id].la_lvts[pin].lvt_active = 1;
 970                 if (bootverbose)
 971                         printf("lapic%u:", apic_id);
 972         }
 973         if (bootverbose)
 974                 printf(" LINT%u %s\n", pin, masked ? "masked" : "unmasked");
 975         return (0);
 976 }
 977
 978 static int
 979 native_lapic_set_lvt_mode(u_int apic_id, u_int pin, u_int32_t mode)
 980 {
 981         struct lvt *lvt;
 982
 983         if (pin > APIC_LVT_MAX)
 984                 return (EINVAL);
 985         if (apic_id == APIC_ID_ALL) {
 986                 lvt = &lvts[pin];
 987                 if (bootverbose)
 988                         printf("lapic:");
 989         } else {
 990                 KASSERT(lapics[apic_id].la_present,
 991                     ("%s: missing APIC %u", __func__, apic_id));
 992                 lvt = &lapics[apic_id].la_lvts[pin];
 993                 lvt->lvt_active = 1;
 994                 if (bootverbose)
 995                         printf("lapic%u:", apic_id);
 996         }
 997         lvt->lvt_mode = mode;
 998         switch (mode) {
 999         case APIC_LVT_DM_NMI:
1000         case APIC_LVT_DM_SMI:
1001         case APIC_LVT_DM_INIT:
1002         case APIC_LVT_DM_EXTINT:
1003                 lvt->lvt_edgetrigger = 1;
1004                 lvt->lvt_activehi = 1;
1005                 if (mode == APIC_LVT_DM_EXTINT)
1006                         lvt->lvt_masked = 1;
1007                 else
1008                         lvt->lvt_masked = 0;
1009                 break;
1010         default:
1011                 panic("Unsupported delivery mode: 0x%x\n", mode);
1012         }
1013         if (bootverbose) {
1014                 printf(" Routing ");
1015                 switch (mode) {
1016                 case APIC_LVT_DM_NMI:
1017                         printf("NMI");
1018                         break;
1019                 case APIC_LVT_DM_SMI:
1020                         printf("SMI");
1021                         break;
1022                 case APIC_LVT_DM_INIT:
1023                         printf("INIT");
1024                         break;
1025                 case APIC_LVT_DM_EXTINT:
1026                         printf("ExtINT");
1027                         break;
1028                 }
1029                 printf(" -> LINT%u\n", pin);
1030         }
1031         return (0);
1032 }
1033
1034 static int
1035 native_lapic_set_lvt_polarity(u_int apic_id, u_int pin, enum intr_polarity pol)
1036 {
1037
1038         if (pin > APIC_LVT_MAX || pol == INTR_POLARITY_CONFORM)
1039                 return (EINVAL);
1040         if (apic_id == APIC_ID_ALL) {
1041                 lvts[pin].lvt_activehi = (pol == INTR_POLARITY_HIGH);
1042                 if (bootverbose)
1043                         printf("lapic:");
1044         } else {
1045                 KASSERT(lapics[apic_id].la_present,
1046                     ("%s: missing APIC %u", __func__, apic_id));
1047                 lapics[apic_id].la_lvts[pin].lvt_active = 1;
1048                 lapics[apic_id].la_lvts[pin].lvt_activehi =
1049                     (pol == INTR_POLARITY_HIGH);
1050                 if (bootverbose)
1051                         printf("lapic%u:", apic_id);
1052         }
1053         if (bootverbose)
1054                 printf(" LINT%u polarity: %s\n", pin,
1055                     pol == INTR_POLARITY_HIGH ? "high" : "low");
1056         return (0);
1057 }
1058
1059 static int
1060 native_lapic_set_lvt_triggermode(u_int apic_id, u_int pin,
1061      enum intr_trigger trigger)
1062 {
1063
1064         if (pin > APIC_LVT_MAX || trigger == INTR_TRIGGER_CONFORM)
1065                 return (EINVAL);
1066         if (apic_id == APIC_ID_ALL) {
1067                 lvts[pin].lvt_edgetrigger = (trigger == INTR_TRIGGER_EDGE);
1068                 if (bootverbose)
1069                         printf("lapic:");
1070         } else {
1071                 KASSERT(lapics[apic_id].la_present,
1072                     ("%s: missing APIC %u", __func__, apic_id));
1073                 lapics[apic_id].la_lvts[pin].lvt_edgetrigger =
1074                     (trigger == INTR_TRIGGER_EDGE);
1075                 lapics[apic_id].la_lvts[pin].lvt_active = 1;
1076                 if (bootverbose)
1077                         printf("lapic%u:", apic_id);
1078         }
1079         if (bootverbose)
1080                 printf(" LINT%u trigger: %s\n", pin,
1081                     trigger == INTR_TRIGGER_EDGE ? "edge" : "level");
1082         return (0);
1083 }
1084
1085 /*
1086  * Adjust the TPR of the current CPU so that it blocks all interrupts below
1087  * the passed in vector.
1088  */
1089 static void
1090 lapic_set_tpr(u_int vector)
1091 {
1092 #ifdef CHEAP_TPR
1093         lapic_write32(LAPIC_TPR, vector);
1094 #else
1095         uint32_t tpr;
1096
1097         tpr = lapic_read32(LAPIC_TPR) & ~APIC_TPR_PRIO;
1098         tpr |= vector;
1099         lapic_write32(LAPIC_TPR, tpr);
1100 #endif
1101 }
1102
1103 static void
1104 native_lapic_eoi(void)
1105 {
1106
1107         lapic_write32_nofence(LAPIC_EOI, 0);
1108 }
1109
1110 void
1111 lapic_handle_intr(int vector, struct trapframe *frame)
1112 {
1113         struct intsrc *isrc;
1114
1115         isrc = intr_lookup_source(apic_idt_to_irq(PCPU_GET(apic_id),
1116             vector));
1117         intr_execute_handlers(isrc, frame);
1118 }
1119
1120 void
1121 lapic_handle_timer(struct trapframe *frame)
1122 {
1123         struct lapic *la;
1124         struct trapframe *oldframe;
1125         struct thread *td;
1126
1127         /* Send EOI first thing. */
1128         lapic_eoi();
1129
1130 #if defined(SMP) && !defined(SCHED_ULE)
1131         /*
1132          * Don't do any accounting for the disabled HTT cores, since it
1133          * will provide misleading numbers for the userland.
1134          *
1135          * No locking is necessary here, since even if we lose the race
1136          * when hlt_cpus_mask changes it is not a big deal, really.
1137          *
1138          * Don't do that for ULE, since ULE doesn't consider hlt_cpus_mask
1139          * and unlike other schedulers it actually schedules threads to
1140          * those CPUs.
1141          */
1142         if (CPU_ISSET(PCPU_GET(cpuid), &hlt_cpus_mask))
1143                 return;
1144 #endif
1145
1146         /* Look up our local APIC structure for the tick counters. */
1147         la = &lapics[PCPU_GET(apic_id)];
1148         (*la->la_timer_count)++;
1149         critical_enter();
1150         if (lapic_et.et_active) {
1151                 td = curthread;
1152                 td->td_intr_nesting_level++;
1153                 oldframe = td->td_intr_frame;
1154                 td->td_intr_frame = frame;
1155                 lapic_et.et_event_cb(&lapic_et, lapic_et.et_arg);
1156                 td->td_intr_frame = oldframe;
1157                 td->td_intr_nesting_level--;
1158         }
1159         critical_exit();
1160 }
1161
1162 static void
1163 lapic_timer_set_divisor(u_int divisor)
1164 {
1165
1166         KASSERT(powerof2(divisor), ("lapic: invalid divisor %u", divisor));
1167         KASSERT(ffs(divisor) <= sizeof(lapic_timer_divisors) /
1168             sizeof(u_int32_t), ("lapic: invalid divisor %u", divisor));
1169         lapic_write32(LAPIC_DCR_TIMER, lapic_timer_divisors[ffs(divisor) - 1]);
1170 }
1171
1172 static void
1173 lapic_timer_oneshot(struct lapic *la)
1174 {
1175         uint32_t value;
1176
1177         value = la->lvt_timer_base;
1178         value &= ~(APIC_LVTT_TM | APIC_LVT_M);
1179         value |= APIC_LVTT_TM_ONE_SHOT;
1180         la->lvt_timer_last = value;
1181         lapic_write32(LAPIC_LVT_TIMER, value);
1182         lapic_write32(LAPIC_ICR_TIMER, la->la_timer_period);
1183 }
1184
1185 static void
1186 lapic_timer_oneshot_nointr(struct lapic *la, uint32_t count)
1187 {
1188         uint32_t value;
1189
1190         value = la->lvt_timer_base;
1191         value &= ~APIC_LVTT_TM;
1192         value |= APIC_LVTT_TM_ONE_SHOT | APIC_LVT_M;
1193         la->lvt_timer_last = value;
1194         lapic_write32(LAPIC_LVT_TIMER, value);
1195         lapic_write32(LAPIC_ICR_TIMER, count);
1196 }
1197
1198 static void
1199 lapic_timer_periodic(struct lapic *la)
1200 {
1201         uint32_t value;
1202
1203         value = la->lvt_timer_base;
1204         value &= ~(APIC_LVTT_TM | APIC_LVT_M);
1205         value |= APIC_LVTT_TM_PERIODIC;
1206         la->lvt_timer_last = value;
1207         lapic_write32(LAPIC_LVT_TIMER, value);
1208         lapic_write32(LAPIC_ICR_TIMER, la->la_timer_period);
1209 }
1210
1211 static void
1212 lapic_timer_deadline(struct lapic *la)
1213 {
1214         uint32_t value;
1215
1216         value = la->lvt_timer_base;
1217         value &= ~(APIC_LVTT_TM | APIC_LVT_M);
1218         value |= APIC_LVTT_TM_TSCDLT;
1219         if (value != la->lvt_timer_last) {
1220                 la->lvt_timer_last = value;
1221                 lapic_write32_nofence(LAPIC_LVT_TIMER, value);
1222                 if (!x2apic_mode)
1223                         mfence();
1224         }
1225         wrmsr(MSR_TSC_DEADLINE, la->la_timer_period + rdtsc());
1226 }
1227
1228 static void
1229 lapic_timer_stop(struct lapic *la)
1230 {
1231         uint32_t value;
1232
1233         if (la->la_timer_mode == LAT_MODE_DEADLINE) {
1234                 wrmsr(MSR_TSC_DEADLINE, 0);
1235                 mfence();
1236         } else {
1237                 value = la->lvt_timer_base;
1238                 value &= ~APIC_LVTT_TM;
1239                 value |= APIC_LVT_M;
1240                 la->lvt_timer_last = value;
1241                 lapic_write32(LAPIC_LVT_TIMER, value);
1242         }
1243 }
1244
1245 void
1246 lapic_handle_cmc(void)
1247 {
1248
1249         lapic_eoi();
1250         cmc_intr();
1251 }
1252
1253 /*
1254  * Called from the mca_init() to activate the CMC interrupt if this CPU is
1255  * responsible for monitoring any MC banks for CMC events.  Since mca_init()
1256  * is called prior to lapic_setup() during boot, this just needs to unmask
1257  * this CPU's LVT_CMCI entry.
1258  */
1259 static void
1260 native_lapic_enable_cmc(void)
1261 {
1262         u_int apic_id;
1263
1264 #ifdef DEV_ATPIC
1265         if (!x2apic_mode && lapic_map == NULL)
1266                 return;
1267 #endif
1268         apic_id = PCPU_GET(apic_id);
1269         KASSERT(lapics[apic_id].la_present,
1270             ("%s: missing APIC %u", __func__, apic_id));
1271         lapics[apic_id].la_lvts[APIC_LVT_CMCI].lvt_masked = 0;
1272         lapics[apic_id].la_lvts[APIC_LVT_CMCI].lvt_active = 1;
1273         if (bootverbose)
1274                 printf("lapic%u: CMCI unmasked\n", apic_id);
1275 }
1276
1277 void
1278 lapic_handle_error(void)
1279 {
1280         uint32_t esr;
1281
1282         /*
1283          * Read the contents of the error status register.  Write to
1284          * the register first before reading from it to force the APIC
1285          * to update its value to indicate any errors that have
1286          * occurred since the previous write to the register.
1287          */
1288         lapic_write32(LAPIC_ESR, 0);
1289         esr = lapic_read32(LAPIC_ESR);
1290
1291         printf("CPU%d: local APIC error 0x%x\n", PCPU_GET(cpuid), esr);
1292         lapic_eoi();
1293 }
1294
1295 static u_int
1296 native_apic_cpuid(u_int apic_id)
1297 {
1298 #ifdef SMP
1299         return apic_cpuids[apic_id];
1300 #else
1301         return 0;
1302 #endif
1303 }
1304
1305 /* Request a free IDT vector to be used by the specified IRQ. */
1306 static u_int
1307 native_apic_alloc_vector(u_int apic_id, u_int irq)
1308 {
1309         u_int vector;
1310
1311         KASSERT(irq < NUM_IO_INTS, ("Invalid IRQ %u", irq));
1312
1313         /*
1314          * Search for a free vector.  Currently we just use a very simple
1315          * algorithm to find the first free vector.
1316          */
1317         mtx_lock_spin(&icu_lock);
1318         for (vector = 0; vector < APIC_NUM_IOINTS; vector++) {
1319                 if (lapics[apic_id].la_ioint_irqs[vector] != -1)
1320                         continue;
1321                 lapics[apic_id].la_ioint_irqs[vector] = irq;
1322                 mtx_unlock_spin(&icu_lock);
1323                 return (vector + APIC_IO_INTS);
1324         }
1325         mtx_unlock_spin(&icu_lock);
1326         return (0);
1327 }
1328
1329 /*
1330  * Request 'count' free contiguous IDT vectors to be used by 'count'
1331  * IRQs.  'count' must be a power of two and the vectors will be
1332  * aligned on a boundary of 'align'.  If the request cannot be
1333  * satisfied, 0 is returned.
1334  */
1335 static u_int
1336 native_apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align)
1337 {
1338         u_int first, run, vector;
1339
1340         KASSERT(powerof2(count), ("bad count"));
1341         KASSERT(powerof2(align), ("bad align"));
1342         KASSERT(align >= count, ("align < count"));
1343 #ifdef INVARIANTS
1344         for (run = 0; run < count; run++)
1345                 KASSERT(irqs[run] < NUM_IO_INTS, ("Invalid IRQ %u at index %u",
1346                     irqs[run], run));
1347 #endif
1348
1349         /*
1350          * Search for 'count' free vectors.  As with apic_alloc_vector(),
1351          * this just uses a simple first fit algorithm.
1352          */
1353         run = 0;
1354         first = 0;
1355         mtx_lock_spin(&icu_lock);
1356         for (vector = 0; vector < APIC_NUM_IOINTS; vector++) {
1357
1358                 /* Vector is in use, end run. */
1359                 if (lapics[apic_id].la_ioint_irqs[vector] != -1) {
1360                         run = 0;
1361                         first = 0;
1362                         continue;
1363                 }
1364
1365                 /* Start a new run if run == 0 and vector is aligned. */
1366                 if (run == 0) {
1367                         if ((vector & (align - 1)) != 0)
1368                                 continue;
1369                         first = vector;
1370                 }
1371                 run++;
1372
1373                 /* Keep looping if the run isn't long enough yet. */
1374                 if (run < count)
1375                         continue;
1376
1377                 /* Found a run, assign IRQs and return the first vector. */
1378                 for (vector = 0; vector < count; vector++)
1379                         lapics[apic_id].la_ioint_irqs[first + vector] =
1380                             irqs[vector];
1381                 mtx_unlock_spin(&icu_lock);
1382                 return (first + APIC_IO_INTS);
1383         }
1384         mtx_unlock_spin(&icu_lock);
1385         printf("APIC: Couldn't find APIC vectors for %u IRQs\n", count);
1386         return (0);
1387 }
1388
1389 /*
1390  * Enable a vector for a particular apic_id.  Since all lapics share idt
1391  * entries and ioint_handlers this enables the vector on all lapics.  lapics
1392  * which do not have the vector configured would report spurious interrupts
1393  * should it fire.
1394  */
1395 static void
1396 native_apic_enable_vector(u_int apic_id, u_int vector)
1397 {
1398
1399         KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry"));
1400         KASSERT(ioint_handlers[vector / 32] != NULL,
1401             ("No ISR handler for vector %u", vector));
1402 #ifdef KDTRACE_HOOKS
1403         KASSERT(vector != IDT_DTRACE_RET,
1404             ("Attempt to overwrite DTrace entry"));
1405 #endif
1406         setidt(vector, ioint_handlers[vector / 32], SDT_APIC, SEL_KPL,
1407             GSEL_APIC);
1408 }
1409
1410 static void
1411 native_apic_disable_vector(u_int apic_id, u_int vector)
1412 {
1413
1414         KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry"));
1415 #ifdef KDTRACE_HOOKS
1416         KASSERT(vector != IDT_DTRACE_RET,
1417             ("Attempt to overwrite DTrace entry"));
1418 #endif
1419         KASSERT(ioint_handlers[vector / 32] != NULL,
1420             ("No ISR handler for vector %u", vector));
1421 #ifdef notyet
1422         /*
1423          * We can not currently clear the idt entry because other cpus
1424          * may have a valid vector at this offset.
1425          */
1426         setidt(vector, &IDTVEC(rsvd), SDT_APICT, SEL_KPL, GSEL_APIC);
1427 #endif
1428 }
1429
1430 /* Release an APIC vector when it's no longer in use. */
1431 static void
1432 native_apic_free_vector(u_int apic_id, u_int vector, u_int irq)
1433 {
1434         struct thread *td;
1435
1436         KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL &&
1437             vector <= APIC_IO_INTS + APIC_NUM_IOINTS,
1438             ("Vector %u does not map to an IRQ line", vector));
1439         KASSERT(irq < NUM_IO_INTS, ("Invalid IRQ %u", irq));
1440         KASSERT(lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] ==
1441             irq, ("IRQ mismatch"));
1442 #ifdef KDTRACE_HOOKS
1443         KASSERT(vector != IDT_DTRACE_RET,
1444             ("Attempt to overwrite DTrace entry"));
1445 #endif
1446
1447         /*
1448          * Bind us to the cpu that owned the vector before freeing it so
1449          * we don't lose an interrupt delivery race.
1450          */
1451         td = curthread;
1452         if (!rebooting) {
1453                 thread_lock(td);
1454                 if (sched_is_bound(td))
1455                         panic("apic_free_vector: Thread already bound.\n");
1456                 sched_bind(td, apic_cpuid(apic_id));
1457                 thread_unlock(td);
1458         }
1459         mtx_lock_spin(&icu_lock);
1460         lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] = -1;
1461         mtx_unlock_spin(&icu_lock);
1462         if (!rebooting) {
1463                 thread_lock(td);
1464                 sched_unbind(td);
1465                 thread_unlock(td);
1466         }
1467 }
1468
1469 /* Map an IDT vector (APIC) to an IRQ (interrupt source). */
1470 static u_int
1471 apic_idt_to_irq(u_int apic_id, u_int vector)
1472 {
1473         int irq;
1474
1475         KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL &&
1476             vector <= APIC_IO_INTS + APIC_NUM_IOINTS,
1477             ("Vector %u does not map to an IRQ line", vector));
1478 #ifdef KDTRACE_HOOKS
1479         KASSERT(vector != IDT_DTRACE_RET,
1480             ("Attempt to overwrite DTrace entry"));
1481 #endif
1482         irq = lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS];
1483         if (irq < 0)
1484                 irq = 0;
1485         return (irq);
1486 }
1487
1488 #ifdef DDB
1489 /*
1490  * Dump data about APIC IDT vector mappings.
1491  */
1492 DB_SHOW_COMMAND(apic, db_show_apic)
1493 {
1494         struct intsrc *isrc;
1495         int i, verbose;
1496         u_int apic_id;
1497         u_int irq;
1498
1499         if (strcmp(modif, "vv") == 0)
1500                 verbose = 2;
1501         else if (strcmp(modif, "v") == 0)
1502                 verbose = 1;
1503         else
1504                 verbose = 0;
1505         for (apic_id = 0; apic_id <= MAX_APIC_ID; apic_id++) {
1506                 if (lapics[apic_id].la_present == 0)
1507                         continue;
1508                 db_printf("Interrupts bound to lapic %u\n", apic_id);
1509                 for (i = 0; i < APIC_NUM_IOINTS + 1 && !db_pager_quit; i++) {
1510                         irq = lapics[apic_id].la_ioint_irqs[i];
1511                         if (irq == -1 || irq == IRQ_SYSCALL)
1512                                 continue;
1513 #ifdef KDTRACE_HOOKS
1514                         if (irq == IRQ_DTRACE_RET)
1515                                 continue;
1516 #endif
1517 #ifdef XENHVM
1518                         if (irq == IRQ_EVTCHN)
1519                                 continue;
1520 #endif
1521                         db_printf("vec 0x%2x -> ", i + APIC_IO_INTS);
1522                         if (irq == IRQ_TIMER)
1523                                 db_printf("lapic timer\n");
1524                         else if (irq < NUM_IO_INTS) {
1525                                 isrc = intr_lookup_source(irq);
1526                                 if (isrc == NULL || verbose == 0)
1527                                         db_printf("IRQ %u\n", irq);
1528                                 else
1529                                         db_dump_intr_event(isrc->is_event,
1530                                             verbose == 2);
1531                         } else
1532                                 db_printf("IRQ %u ???\n", irq);
1533                 }
1534         }
1535 }
1536
1537 static void
1538 dump_mask(const char *prefix, uint32_t v, int base)
1539 {
1540         int i, first;
1541
1542         first = 1;
1543         for (i = 0; i < 32; i++)
1544                 if (v & (1 << i)) {
1545                         if (first) {
1546                                 db_printf("%s:", prefix);
1547                                 first = 0;
1548                         }
1549                         db_printf(" %02x", base + i);
1550                 }
1551         if (!first)
1552                 db_printf("\n");
1553 }
1554
1555 /* Show info from the lapic regs for this CPU. */
1556 DB_SHOW_COMMAND(lapic, db_show_lapic)
1557 {
1558         uint32_t v;
1559
1560         db_printf("lapic ID = %d\n", lapic_id());
1561         v = lapic_read32(LAPIC_VERSION);
1562         db_printf("version  = %d.%d\n", (v & APIC_VER_VERSION) >> 4,
1563             v & 0xf);
1564         db_printf("max LVT  = %d\n", (v & APIC_VER_MAXLVT) >> MAXLVTSHIFT);
1565         v = lapic_read32(LAPIC_SVR);
1566         db_printf("SVR      = %02x (%s)\n", v & APIC_SVR_VECTOR,
1567             v & APIC_SVR_ENABLE ? "enabled" : "disabled");
1568         db_printf("TPR      = %02x\n", lapic_read32(LAPIC_TPR));
1569
1570 #define dump_field(prefix, regn, index)                                 \
1571         dump_mask(__XSTRING(prefix ## index),                           \
1572             lapic_read32(LAPIC_ ## regn ## index),                      \
1573             index * 32)
1574
1575         db_printf("In-service Interrupts:\n");
1576         dump_field(isr, ISR, 0);
1577         dump_field(isr, ISR, 1);
1578         dump_field(isr, ISR, 2);
1579         dump_field(isr, ISR, 3);
1580         dump_field(isr, ISR, 4);
1581         dump_field(isr, ISR, 5);
1582         dump_field(isr, ISR, 6);
1583         dump_field(isr, ISR, 7);
1584
1585         db_printf("TMR Interrupts:\n");
1586         dump_field(tmr, TMR, 0);
1587         dump_field(tmr, TMR, 1);
1588         dump_field(tmr, TMR, 2);
1589         dump_field(tmr, TMR, 3);
1590         dump_field(tmr, TMR, 4);
1591         dump_field(tmr, TMR, 5);
1592         dump_field(tmr, TMR, 6);
1593         dump_field(tmr, TMR, 7);
1594
1595         db_printf("IRR Interrupts:\n");
1596         dump_field(irr, IRR, 0);
1597         dump_field(irr, IRR, 1);
1598         dump_field(irr, IRR, 2);
1599         dump_field(irr, IRR, 3);
1600         dump_field(irr, IRR, 4);
1601         dump_field(irr, IRR, 5);
1602         dump_field(irr, IRR, 6);
1603         dump_field(irr, IRR, 7);
1604
1605 #undef dump_field
1606 }
1607 #endif
1608
1609 /*
1610  * APIC probing support code.  This includes code to manage enumerators.
1611  */
1612
1613 static SLIST_HEAD(, apic_enumerator) enumerators =
1614         SLIST_HEAD_INITIALIZER(enumerators);
1615 static struct apic_enumerator *best_enum;
1616
1617 void
1618 apic_register_enumerator(struct apic_enumerator *enumerator)
1619 {
1620 #ifdef INVARIANTS
1621         struct apic_enumerator *apic_enum;
1622
1623         SLIST_FOREACH(apic_enum, &enumerators, apic_next) {
1624                 if (apic_enum == enumerator)
1625                         panic("%s: Duplicate register of %s", __func__,
1626                             enumerator->apic_name);
1627         }
1628 #endif
1629         SLIST_INSERT_HEAD(&enumerators, enumerator, apic_next);
1630 }
1631
1632 /*
1633  * We have to look for CPU's very, very early because certain subsystems
1634  * want to know how many CPU's we have extremely early on in the boot
1635  * process.
1636  */
1637 static void
1638 apic_init(void *dummy __unused)
1639 {
1640         struct apic_enumerator *enumerator;
1641         int retval, best;
1642
1643         /* We only support built in local APICs. */
1644         if (!(cpu_feature & CPUID_APIC))
1645                 return;
1646
1647         /* Don't probe if APIC mode is disabled. */
1648         if (resource_disabled("apic", 0))
1649                 return;
1650
1651         /* Probe all the enumerators to find the best match. */
1652         best_enum = NULL;
1653         best = 0;
1654         SLIST_FOREACH(enumerator, &enumerators, apic_next) {
1655                 retval = enumerator->apic_probe();
1656                 if (retval > 0)
1657                         continue;
1658                 if (best_enum == NULL || best < retval) {
1659                         best_enum = enumerator;
1660                         best = retval;
1661                 }
1662         }
1663         if (best_enum == NULL) {
1664                 if (bootverbose)
1665                         printf("APIC: Could not find any APICs.\n");
1666 #ifndef DEV_ATPIC
1667                 panic("running without device atpic requires a local APIC");
1668 #endif
1669                 return;
1670         }
1671
1672         if (bootverbose)
1673                 printf("APIC: Using the %s enumerator.\n",
1674                     best_enum->apic_name);
1675
1676 #ifdef I686_CPU
1677         /*
1678          * To work around an errata, we disable the local APIC on some
1679          * CPUs during early startup.  We need to turn the local APIC back
1680          * on on such CPUs now.
1681          */
1682         ppro_reenable_apic();
1683 #endif
1684
1685         /* Probe the CPU's in the system. */
1686         retval = best_enum->apic_probe_cpus();
1687         if (retval != 0)
1688                 printf("%s: Failed to probe CPUs: returned %d\n",
1689                     best_enum->apic_name, retval);
1690
1691 }
1692 SYSINIT(apic_init, SI_SUB_TUNABLES - 1, SI_ORDER_SECOND, apic_init, NULL);
1693
1694 /*
1695  * Setup the local APIC.  We have to do this prior to starting up the APs
1696  * in the SMP case.
1697  */
1698 static void
1699 apic_setup_local(void *dummy __unused)
1700 {
1701         int retval;
1702
1703         if (best_enum == NULL)
1704                 return;
1705
1706         /* Initialize the local APIC. */
1707         retval = best_enum->apic_setup_local();
1708         if (retval != 0)
1709                 printf("%s: Failed to setup the local APIC: returned %d\n",
1710                     best_enum->apic_name, retval);
1711 }
1712 SYSINIT(apic_setup_local, SI_SUB_CPU, SI_ORDER_SECOND, apic_setup_local, NULL);
1713
1714 /*
1715  * Setup the I/O APICs.
1716  */
1717 static void
1718 apic_setup_io(void *dummy __unused)
1719 {
1720         int retval;
1721
1722         if (best_enum == NULL)
1723                 return;
1724
1725         /*
1726          * Local APIC must be registered before other PICs and pseudo PICs
1727          * for proper suspend/resume order.
1728          */
1729         intr_register_pic(&lapic_pic);
1730
1731         retval = best_enum->apic_setup_io();
1732         if (retval != 0)
1733                 printf("%s: Failed to setup I/O APICs: returned %d\n",
1734                     best_enum->apic_name, retval);
1735
1736         /*
1737          * Finish setting up the local APIC on the BSP once we know
1738          * how to properly program the LINT pins.  In particular, this
1739          * enables the EOI suppression mode, if LAPIC support it and
1740          * user did not disabled the mode.
1741          */
1742         lapic_setup(1);
1743         if (bootverbose)
1744                 lapic_dump("BSP");
1745
1746         /* Enable the MSI "pic". */
1747         init_ops.msi_init();
1748 }
1749 SYSINIT(apic_setup_io, SI_SUB_INTR, SI_ORDER_THIRD, apic_setup_io, NULL);
1750
1751 #ifdef SMP
1752 /*
1753  * Inter Processor Interrupt functions.  The lapic_ipi_*() functions are
1754  * private to the MD code.  The public interface for the rest of the
1755  * kernel is defined in mp_machdep.c.
1756  */
1757
1758 /*
1759  * Wait delay microseconds for IPI to be sent.  If delay is -1, we
1760  * wait forever.
1761  */
1762 static int
1763 native_lapic_ipi_wait(int delay)
1764 {
1765         uint64_t rx;
1766
1767         /* LAPIC_ICR.APIC_DELSTAT_MASK is undefined in x2APIC mode */
1768         if (x2apic_mode)
1769                 return (1);
1770
1771         for (rx = 0; delay == -1 || rx < lapic_ipi_wait_mult * delay; rx++) {
1772                 if ((lapic_read_icr_lo() & APIC_DELSTAT_MASK) ==
1773                     APIC_DELSTAT_IDLE)
1774                         return (1);
1775                 ia32_pause();
1776         }
1777         return (0);
1778 }
1779
1780 static void
1781 native_lapic_ipi_raw(register_t icrlo, u_int dest)
1782 {
1783         uint64_t icr;
1784         uint32_t vhi, vlo;
1785         register_t saveintr;
1786
1787         /* XXX: Need more sanity checking of icrlo? */
1788         KASSERT(x2apic_mode || lapic_map != NULL,
1789             ("%s called too early", __func__));
1790         KASSERT(x2apic_mode ||
1791             (dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0,
1792             ("%s: invalid dest field", __func__));
1793         KASSERT((icrlo & APIC_ICRLO_RESV_MASK) == 0,
1794             ("%s: reserved bits set in ICR LO register", __func__));
1795
1796         /* Set destination in ICR HI register if it is being used. */
1797         if (!x2apic_mode) {
1798                 saveintr = intr_disable();
1799                 icr = lapic_read_icr();
1800         }
1801
1802         if ((icrlo & APIC_DEST_MASK) == APIC_DEST_DESTFLD) {
1803                 if (x2apic_mode) {
1804                         vhi = dest;
1805                 } else {
1806                         vhi = icr >> 32;
1807                         vhi &= ~APIC_ID_MASK;
1808                         vhi |= dest << APIC_ID_SHIFT;
1809                 }
1810         } else {
1811                 vhi = 0;
1812         }
1813
1814         /* Program the contents of the IPI and dispatch it. */
1815         if (x2apic_mode) {
1816                 vlo = icrlo;
1817         } else {
1818                 vlo = icr;
1819                 vlo &= APIC_ICRLO_RESV_MASK;
1820                 vlo |= icrlo;
1821         }
1822         lapic_write_icr(vhi, vlo);
1823         if (!x2apic_mode)
1824                 intr_restore(saveintr);
1825 }
1826
1827 #define BEFORE_SPIN     50000
1828 #ifdef DETECT_DEADLOCK
1829 #define AFTER_SPIN      50
1830 #endif
1831
1832 static void
1833 native_lapic_ipi_vectored(u_int vector, int dest)
1834 {
1835         register_t icrlo, destfield;
1836
1837         KASSERT((vector & ~APIC_VECTOR_MASK) == 0,
1838             ("%s: invalid vector %d", __func__, vector));
1839
1840         icrlo = APIC_DESTMODE_PHY | APIC_TRIGMOD_EDGE | APIC_LEVEL_ASSERT;
1841
1842         /*
1843          * NMI IPIs are just fake vectors used to send a NMI.  Use special rules
1844          * regarding NMIs if passed, otherwise specify the vector.
1845          */
1846         if (vector >= IPI_NMI_FIRST)
1847                 icrlo |= APIC_DELMODE_NMI;
1848         else
1849                 icrlo |= vector | APIC_DELMODE_FIXED;
1850         destfield = 0;
1851         switch (dest) {
1852         case APIC_IPI_DEST_SELF:
1853                 icrlo |= APIC_DEST_SELF;
1854                 break;
1855         case APIC_IPI_DEST_ALL:
1856                 icrlo |= APIC_DEST_ALLISELF;
1857                 break;
1858         case APIC_IPI_DEST_OTHERS:
1859                 icrlo |= APIC_DEST_ALLESELF;
1860                 break;
1861         default:
1862                 KASSERT(x2apic_mode ||
1863                     (dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0,
1864                     ("%s: invalid destination 0x%x", __func__, dest));
1865                 destfield = dest;
1866         }
1867
1868         /* Wait for an earlier IPI to finish. */
1869         if (!lapic_ipi_wait(BEFORE_SPIN)) {
1870                 if (panicstr != NULL)
1871                         return;
1872                 else
1873                         panic("APIC: Previous IPI is stuck");
1874         }
1875
1876         lapic_ipi_raw(icrlo, destfield);
1877
1878 #ifdef DETECT_DEADLOCK
1879         /* Wait for IPI to be delivered. */
1880         if (!lapic_ipi_wait(AFTER_SPIN)) {
1881 #ifdef needsattention
1882                 /*
1883                  * XXX FIXME:
1884                  *
1885                  * The above function waits for the message to actually be
1886                  * delivered.  It breaks out after an arbitrary timeout
1887                  * since the message should eventually be delivered (at
1888                  * least in theory) and that if it wasn't we would catch
1889                  * the failure with the check above when the next IPI is
1890                  * sent.
1891                  *
1892                  * We could skip this wait entirely, EXCEPT it probably
1893                  * protects us from other routines that assume that the
1894                  * message was delivered and acted upon when this function
1895                  * returns.
1896                  */
1897                 printf("APIC: IPI might be stuck\n");
1898 #else /* !needsattention */
1899                 /* Wait until mesage is sent without a timeout. */
1900                 while (lapic_read_icr_lo() & APIC_DELSTAT_PEND)
1901                         ia32_pause();
1902 #endif /* needsattention */
1903         }
1904 #endif /* DETECT_DEADLOCK */
1905 }
1906
1907 #endif /* SMP */
1908
1909 /*
1910  * Since the IDT is shared by all CPUs the IPI slot update needs to be globally
1911  * visible.
1912  *
1913  * Consider the case where an IPI is generated immediately after allocation:
1914  *     vector = lapic_ipi_alloc(ipifunc);
1915  *     ipi_selected(other_cpus, vector);
1916  *
1917  * In xAPIC mode a write to ICR_LO has serializing semantics because the
1918  * APIC page is mapped as an uncached region. In x2APIC mode there is an
1919  * explicit 'mfence' before the ICR MSR is written. Therefore in both cases
1920  * the IDT slot update is globally visible before the IPI is delivered.
1921  */
1922 static int
1923 native_lapic_ipi_alloc(inthand_t *ipifunc)
1924 {
1925         struct gate_descriptor *ip;
1926         long func;
1927         int idx, vector;
1928
1929         KASSERT(ipifunc != &IDTVEC(rsvd), ("invalid ipifunc %p", ipifunc));
1930
1931         vector = -1;
1932         mtx_lock_spin(&icu_lock);
1933         for (idx = IPI_DYN_FIRST; idx <= IPI_DYN_LAST; idx++) {
1934                 ip = &idt[idx];
1935                 func = (ip->gd_hioffset << 16) | ip->gd_looffset;
1936                 if (func == (uintptr_t)&IDTVEC(rsvd)) {
1937                         vector = idx;
1938                         setidt(vector, ipifunc, SDT_APIC, SEL_KPL, GSEL_APIC);
1939                         break;
1940                 }
1941         }
1942         mtx_unlock_spin(&icu_lock);
1943         return (vector);
1944 }
1945
1946 static void
1947 native_lapic_ipi_free(int vector)
1948 {
1949         struct gate_descriptor *ip;
1950         long func;
1951
1952         KASSERT(vector >= IPI_DYN_FIRST && vector <= IPI_DYN_LAST,
1953             ("%s: invalid vector %d", __func__, vector));
1954
1955         mtx_lock_spin(&icu_lock);
1956         ip = &idt[vector];
1957         func = (ip->gd_hioffset << 16) | ip->gd_looffset;
1958         KASSERT(func != (uintptr_t)&IDTVEC(rsvd),
1959             ("invalid idtfunc %#lx", func));
1960         setidt(vector, &IDTVEC(rsvd), SDT_APICT, SEL_KPL, GSEL_APIC);
1961         mtx_unlock_spin(&icu_lock);
1962 }