sys/x86/x86/local_apic.c

   1 /*-
   2  * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org>
   3  * Copyright (c) 1996, by Steve Passe
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. The name of the developer may NOT be used to endorse or promote products
  12  *    derived from this software without specific prior written permission.
  13  * 3. Neither the name of the author nor the names of any co-contributors
  14  *    may be used to endorse or promote products derived from this software
  15  *    without specific prior written permission.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 /*
  31  * Local APIC support on Pentium and later processors.
  32  */
  33
  34 #include <sys/cdefs.h>
  35 __FBSDID("$FreeBSD$");
  36
  37 #include "opt_atpic.h"
  38 #include "opt_hwpmc_hooks.h"
  39
  40 #include "opt_ddb.h"
  41
  42 #include <sys/param.h>
  43 #include <sys/systm.h>
  44 #include <sys/bus.h>
  45 #include <sys/kernel.h>
  46 #include <sys/lock.h>
  47 #include <sys/mutex.h>
  48 #include <sys/pcpu.h>
  49 #include <sys/proc.h>
  50 #include <sys/sched.h>
  51 #include <sys/smp.h>
  52 #include <sys/sysctl.h>
  53 #include <sys/timeet.h>
  54
  55 #include <vm/vm.h>
  56 #include <vm/pmap.h>
  57
  58 #include <x86/apicreg.h>
  59 #include <machine/clock.h>
  60 #include <machine/cpufunc.h>
  61 #include <machine/cputypes.h>
  62 #include <machine/frame.h>
  63 #include <machine/intr_machdep.h>
  64 #include <x86/apicvar.h>
  65 #include <x86/mca.h>
  66 #include <machine/md_var.h>
  67 #include <machine/smp.h>
  68 #include <machine/specialreg.h>
  69 #include <x86/init.h>
  70
  71 #ifdef DDB
  72 #include <sys/interrupt.h>
  73 #include <ddb/ddb.h>
  74 #endif
  75
  76 #ifdef __amd64__
  77 #define SDT_APIC        SDT_SYSIGT
  78 #define SDT_APICT       SDT_SYSIGT
  79 #define GSEL_APIC       0
  80 #else
  81 #define SDT_APIC        SDT_SYS386IGT
  82 #define SDT_APICT       SDT_SYS386TGT
  83 #define GSEL_APIC       GSEL(GCODE_SEL, SEL_KPL)
  84 #endif
  85
  86 /* Sanity checks on IDT vectors. */
  87 CTASSERT(APIC_IO_INTS + APIC_NUM_IOINTS == APIC_TIMER_INT);
  88 CTASSERT(APIC_TIMER_INT < APIC_LOCAL_INTS);
  89 CTASSERT(APIC_LOCAL_INTS == 240);
  90 CTASSERT(IPI_STOP < APIC_SPURIOUS_INT);
  91
  92 /* Magic IRQ values for the timer and syscalls. */
  93 #define IRQ_TIMER       (NUM_IO_INTS + 1)
  94 #define IRQ_SYSCALL     (NUM_IO_INTS + 2)
  95 #define IRQ_DTRACE_RET  (NUM_IO_INTS + 3)
  96 #define IRQ_EVTCHN      (NUM_IO_INTS + 4)
  97
  98 enum lat_timer_mode {
  99         LAT_MODE_UNDEF =        0,
 100         LAT_MODE_PERIODIC =     1,
 101         LAT_MODE_ONESHOT =      2,
 102         LAT_MODE_DEADLINE =     3,
 103 };
 104
 105 /*
 106  * Support for local APICs.  Local APICs manage interrupts on each
 107  * individual processor as opposed to I/O APICs which receive interrupts
 108  * from I/O devices and then forward them on to the local APICs.
 109  *
 110  * Local APICs can also send interrupts to each other thus providing the
 111  * mechanism for IPIs.
 112  */
 113
 114 struct lvt {
 115         u_int lvt_edgetrigger:1;
 116         u_int lvt_activehi:1;
 117         u_int lvt_masked:1;
 118         u_int lvt_active:1;
 119         u_int lvt_mode:16;
 120         u_int lvt_vector:8;
 121 };
 122
 123 struct lapic {
 124         struct lvt la_lvts[APIC_LVT_MAX + 1];
 125         u_int la_id:8;
 126         u_int la_cluster:4;
 127         u_int la_cluster_id:2;
 128         u_int la_present:1;
 129         u_long *la_timer_count;
 130         uint64_t la_timer_period;
 131         enum lat_timer_mode la_timer_mode;
 132         uint32_t lvt_timer_base;
 133         uint32_t lvt_timer_last;
 134         /* Include IDT_SYSCALL to make indexing easier. */
 135         int la_ioint_irqs[APIC_NUM_IOINTS + 1];
 136 } static lapics[MAX_APIC_ID + 1];
 137
 138 /* Global defaults for local APIC LVT entries. */
 139 static struct lvt lvts[APIC_LVT_MAX + 1] = {
 140         { 1, 1, 1, 1, APIC_LVT_DM_EXTINT, 0 },  /* LINT0: masked ExtINT */
 141         { 1, 1, 0, 1, APIC_LVT_DM_NMI, 0 },     /* LINT1: NMI */
 142         { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_TIMER_INT },      /* Timer */
 143         { 1, 1, 0, 1, APIC_LVT_DM_FIXED, APIC_ERROR_INT },      /* Error */
 144         { 1, 1, 1, 1, APIC_LVT_DM_NMI, 0 },     /* PMC */
 145         { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_THERMAL_INT },    /* Thermal */
 146         { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_CMC_INT },        /* CMCI */
 147 };
 148
 149 static inthand_t *ioint_handlers[] = {
 150         NULL,                   /* 0 - 31 */
 151         IDTVEC(apic_isr1),      /* 32 - 63 */
 152         IDTVEC(apic_isr2),      /* 64 - 95 */
 153         IDTVEC(apic_isr3),      /* 96 - 127 */
 154         IDTVEC(apic_isr4),      /* 128 - 159 */
 155         IDTVEC(apic_isr5),      /* 160 - 191 */
 156         IDTVEC(apic_isr6),      /* 192 - 223 */
 157         IDTVEC(apic_isr7),      /* 224 - 255 */
 158 };
 159
 160
 161 static u_int32_t lapic_timer_divisors[] = {
 162         APIC_TDCR_1, APIC_TDCR_2, APIC_TDCR_4, APIC_TDCR_8, APIC_TDCR_16,
 163         APIC_TDCR_32, APIC_TDCR_64, APIC_TDCR_128
 164 };
 165
 166 extern inthand_t IDTVEC(rsvd);
 167
 168 volatile char *lapic_map;
 169 vm_paddr_t lapic_paddr;
 170 int x2apic_mode;
 171 int lapic_eoi_suppression;
 172 static int lapic_timer_tsc_deadline;
 173 static u_long lapic_timer_divisor, count_freq;
 174 static struct eventtimer lapic_et;
 175 #ifdef SMP
 176 static uint64_t lapic_ipi_wait_mult;
 177 #endif
 178
 179 SYSCTL_NODE(_hw, OID_AUTO, apic, CTLFLAG_RD, 0, "APIC options");
 180 SYSCTL_INT(_hw_apic, OID_AUTO, x2apic_mode, CTLFLAG_RD, &x2apic_mode, 0, "");
 181 SYSCTL_INT(_hw_apic, OID_AUTO, eoi_suppression, CTLFLAG_RD,
 182     &lapic_eoi_suppression, 0, "");
 183 SYSCTL_INT(_hw_apic, OID_AUTO, timer_tsc_deadline, CTLFLAG_RD,
 184     &lapic_timer_tsc_deadline, 0, "");
 185
 186 static uint32_t
 187 lapic_read32(enum LAPIC_REGISTERS reg)
 188 {
 189         uint32_t res;
 190
 191         if (x2apic_mode) {
 192                 res = rdmsr32(MSR_APIC_000 + reg);
 193         } else {
 194                 res = *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL);
 195         }
 196         return (res);
 197 }
 198
 199 static void
 200 lapic_write32(enum LAPIC_REGISTERS reg, uint32_t val)
 201 {
 202
 203         if (x2apic_mode) {
 204                 mfence();
 205                 wrmsr(MSR_APIC_000 + reg, val);
 206         } else {
 207                 *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL) = val;
 208         }
 209 }
 210
 211 static void
 212 lapic_write32_nofence(enum LAPIC_REGISTERS reg, uint32_t val)
 213 {
 214
 215         if (x2apic_mode) {
 216                 wrmsr(MSR_APIC_000 + reg, val);
 217         } else {
 218                 *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL) = val;
 219         }
 220 }
 221
 222 #ifdef SMP
 223 static uint64_t
 224 lapic_read_icr(void)
 225 {
 226         uint64_t v;
 227         uint32_t vhi, vlo;
 228
 229         if (x2apic_mode) {
 230                 v = rdmsr(MSR_APIC_000 + LAPIC_ICR_LO);
 231         } else {
 232                 vhi = lapic_read32(LAPIC_ICR_HI);
 233                 vlo = lapic_read32(LAPIC_ICR_LO);
 234                 v = ((uint64_t)vhi << 32) | vlo;
 235         }
 236         return (v);
 237 }
 238
 239 static uint64_t
 240 lapic_read_icr_lo(void)
 241 {
 242
 243         return (lapic_read32(LAPIC_ICR_LO));
 244 }
 245
 246 static void
 247 lapic_write_icr(uint32_t vhi, uint32_t vlo)
 248 {
 249         uint64_t v;
 250
 251         if (x2apic_mode) {
 252                 v = ((uint64_t)vhi << 32) | vlo;
 253                 mfence();
 254                 wrmsr(MSR_APIC_000 + LAPIC_ICR_LO, v);
 255         } else {
 256                 lapic_write32(LAPIC_ICR_HI, vhi);
 257                 lapic_write32(LAPIC_ICR_LO, vlo);
 258         }
 259 }
 260 #endif /* SMP */
 261
 262 static void
 263 native_lapic_enable_x2apic(void)
 264 {
 265         uint64_t apic_base;
 266
 267         apic_base = rdmsr(MSR_APICBASE);
 268         apic_base |= APICBASE_X2APIC | APICBASE_ENABLED;
 269         wrmsr(MSR_APICBASE, apic_base);
 270 }
 271
 272 static bool
 273 native_lapic_is_x2apic(void)
 274 {
 275         uint64_t apic_base;
 276
 277         apic_base = rdmsr(MSR_APICBASE);
 278         return ((apic_base & (APICBASE_X2APIC | APICBASE_ENABLED)) ==
 279             (APICBASE_X2APIC | APICBASE_ENABLED));
 280 }
 281
 282 static void     lapic_enable(void);
 283 static void     lapic_resume(struct pic *pic, bool suspend_cancelled);
 284 static void     lapic_timer_oneshot(struct lapic *);
 285 static void     lapic_timer_oneshot_nointr(struct lapic *, uint32_t);
 286 static void     lapic_timer_periodic(struct lapic *);
 287 static void     lapic_timer_deadline(struct lapic *);
 288 static void     lapic_timer_stop(struct lapic *);
 289 static void     lapic_timer_set_divisor(u_int divisor);
 290 static uint32_t lvt_mode(struct lapic *la, u_int pin, uint32_t value);
 291 static int      lapic_et_start(struct eventtimer *et,
 292                     sbintime_t first, sbintime_t period);
 293 static int      lapic_et_stop(struct eventtimer *et);
 294 static u_int    apic_idt_to_irq(u_int apic_id, u_int vector);
 295 static void     lapic_set_tpr(u_int vector);
 296
 297 struct pic lapic_pic = { .pic_resume = lapic_resume };
 298
 299 /* Forward declarations for apic_ops */
 300 static void     native_lapic_create(u_int apic_id, int boot_cpu);
 301 static void     native_lapic_init(vm_paddr_t addr);
 302 static void     native_lapic_xapic_mode(void);
 303 static void     native_lapic_setup(int boot);
 304 static void     native_lapic_dump(const char *str);
 305 static void     native_lapic_disable(void);
 306 static void     native_lapic_eoi(void);
 307 static int      native_lapic_id(void);
 308 static int      native_lapic_intr_pending(u_int vector);
 309 static u_int    native_apic_cpuid(u_int apic_id);
 310 static u_int    native_apic_alloc_vector(u_int apic_id, u_int irq);
 311 static u_int    native_apic_alloc_vectors(u_int apic_id, u_int *irqs,
 312                     u_int count, u_int align);
 313 static void     native_apic_disable_vector(u_int apic_id, u_int vector);
 314 static void     native_apic_enable_vector(u_int apic_id, u_int vector);
 315 static void     native_apic_free_vector(u_int apic_id, u_int vector, u_int irq);
 316 static void     native_lapic_set_logical_id(u_int apic_id, u_int cluster,
 317                     u_int cluster_id);
 318 static int      native_lapic_enable_pmc(void);
 319 static void     native_lapic_disable_pmc(void);
 320 static void     native_lapic_reenable_pmc(void);
 321 static void     native_lapic_enable_cmc(void);
 322 static int      native_lapic_set_lvt_mask(u_int apic_id, u_int lvt,
 323                     u_char masked);
 324 static int      native_lapic_set_lvt_mode(u_int apic_id, u_int lvt,
 325                     uint32_t mode);
 326 static int      native_lapic_set_lvt_polarity(u_int apic_id, u_int lvt,
 327                     enum intr_polarity pol);
 328 static int      native_lapic_set_lvt_triggermode(u_int apic_id, u_int lvt,
 329                     enum intr_trigger trigger);
 330 #ifdef SMP
 331 static void     native_lapic_ipi_raw(register_t icrlo, u_int dest);
 332 static void     native_lapic_ipi_vectored(u_int vector, int dest);
 333 static int      native_lapic_ipi_wait(int delay);
 334 #endif /* SMP */
 335 static int      native_lapic_ipi_alloc(inthand_t *ipifunc);
 336 static void     native_lapic_ipi_free(int vector);
 337
 338 struct apic_ops apic_ops = {
 339         .create                 = native_lapic_create,
 340         .init                   = native_lapic_init,
 341         .xapic_mode             = native_lapic_xapic_mode,
 342         .is_x2apic              = native_lapic_is_x2apic,
 343         .setup                  = native_lapic_setup,
 344         .dump                   = native_lapic_dump,
 345         .disable                = native_lapic_disable,
 346         .eoi                    = native_lapic_eoi,
 347         .id                     = native_lapic_id,
 348         .intr_pending           = native_lapic_intr_pending,
 349         .set_logical_id         = native_lapic_set_logical_id,
 350         .cpuid                  = native_apic_cpuid,
 351         .alloc_vector           = native_apic_alloc_vector,
 352         .alloc_vectors          = native_apic_alloc_vectors,
 353         .enable_vector          = native_apic_enable_vector,
 354         .disable_vector         = native_apic_disable_vector,
 355         .free_vector            = native_apic_free_vector,
 356         .enable_pmc             = native_lapic_enable_pmc,
 357         .disable_pmc            = native_lapic_disable_pmc,
 358         .reenable_pmc           = native_lapic_reenable_pmc,
 359         .enable_cmc             = native_lapic_enable_cmc,
 360 #ifdef SMP
 361         .ipi_raw                = native_lapic_ipi_raw,
 362         .ipi_vectored           = native_lapic_ipi_vectored,
 363         .ipi_wait               = native_lapic_ipi_wait,
 364 #endif
 365         .ipi_alloc              = native_lapic_ipi_alloc,
 366         .ipi_free               = native_lapic_ipi_free,
 367         .set_lvt_mask           = native_lapic_set_lvt_mask,
 368         .set_lvt_mode           = native_lapic_set_lvt_mode,
 369         .set_lvt_polarity       = native_lapic_set_lvt_polarity,
 370         .set_lvt_triggermode    = native_lapic_set_lvt_triggermode,
 371 };
 372
 373 static uint32_t
 374 lvt_mode(struct lapic *la, u_int pin, uint32_t value)
 375 {
 376         struct lvt *lvt;
 377
 378         KASSERT(pin <= APIC_LVT_MAX, ("%s: pin %u out of range", __func__, pin));
 379         if (la->la_lvts[pin].lvt_active)
 380                 lvt = &la->la_lvts[pin];
 381         else
 382                 lvt = &lvts[pin];
 383
 384         value &= ~(APIC_LVT_M | APIC_LVT_TM | APIC_LVT_IIPP | APIC_LVT_DM |
 385             APIC_LVT_VECTOR);
 386         if (lvt->lvt_edgetrigger == 0)
 387                 value |= APIC_LVT_TM;
 388         if (lvt->lvt_activehi == 0)
 389                 value |= APIC_LVT_IIPP_INTALO;
 390         if (lvt->lvt_masked)
 391                 value |= APIC_LVT_M;
 392         value |= lvt->lvt_mode;
 393         switch (lvt->lvt_mode) {
 394         case APIC_LVT_DM_NMI:
 395         case APIC_LVT_DM_SMI:
 396         case APIC_LVT_DM_INIT:
 397         case APIC_LVT_DM_EXTINT:
 398                 if (!lvt->lvt_edgetrigger && bootverbose) {
 399                         printf("lapic%u: Forcing LINT%u to edge trigger\n",
 400                             la->la_id, pin);
 401                         value |= APIC_LVT_TM;
 402                 }
 403                 /* Use a vector of 0. */
 404                 break;
 405         case APIC_LVT_DM_FIXED:
 406                 value |= lvt->lvt_vector;
 407                 break;
 408         default:
 409                 panic("bad APIC LVT delivery mode: %#x\n", value);
 410         }
 411         return (value);
 412 }
 413
 414 /*
 415  * Map the local APIC and setup necessary interrupt vectors.
 416  */
 417 static void
 418 native_lapic_init(vm_paddr_t addr)
 419 {
 420 #ifdef SMP
 421         uint64_t r, r1, r2, rx;
 422 #endif
 423         uint32_t ver;
 424         u_int regs[4];
 425         int i, arat;
 426
 427         /*
 428          * Enable x2APIC mode if possible. Map the local APIC
 429          * registers page.
 430          *
 431          * Keep the LAPIC registers page mapped uncached for x2APIC
 432          * mode too, to have direct map page attribute set to
 433          * uncached.  This is needed to work around CPU errata present
 434          * on all Intel processors.
 435          */
 436         KASSERT(trunc_page(addr) == addr,
 437             ("local APIC not aligned on a page boundary"));
 438         lapic_paddr = addr;
 439         lapic_map = pmap_mapdev(addr, PAGE_SIZE);
 440         if (x2apic_mode) {
 441                 native_lapic_enable_x2apic();
 442                 lapic_map = NULL;
 443         }
 444
 445         /* Setup the spurious interrupt handler. */
 446         setidt(APIC_SPURIOUS_INT, IDTVEC(spuriousint), SDT_APIC, SEL_KPL,
 447             GSEL_APIC);
 448
 449         /* Perform basic initialization of the BSP's local APIC. */
 450         lapic_enable();
 451
 452         /* Set BSP's per-CPU local APIC ID. */
 453         PCPU_SET(apic_id, lapic_id());
 454
 455         /* Local APIC timer interrupt. */
 456         setidt(APIC_TIMER_INT, IDTVEC(timerint), SDT_APIC, SEL_KPL, GSEL_APIC);
 457
 458         /* Local APIC error interrupt. */
 459         setidt(APIC_ERROR_INT, IDTVEC(errorint), SDT_APIC, SEL_KPL, GSEL_APIC);
 460
 461         /* XXX: Thermal interrupt */
 462
 463         /* Local APIC CMCI. */
 464         setidt(APIC_CMC_INT, IDTVEC(cmcint), SDT_APICT, SEL_KPL, GSEL_APIC);
 465
 466         if ((resource_int_value("apic", 0, "clock", &i) != 0 || i != 0)) {
 467                 arat = 0;
 468                 /* Intel CPUID 0x06 EAX[2] set if APIC timer runs in C3. */
 469                 if (cpu_vendor_id == CPU_VENDOR_INTEL && cpu_high >= 6) {
 470                         do_cpuid(0x06, regs);
 471                         if ((regs[0] & CPUTPM1_ARAT) != 0)
 472                                 arat = 1;
 473                 }
 474                 bzero(&lapic_et, sizeof(lapic_et));
 475                 lapic_et.et_name = "LAPIC";
 476                 lapic_et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT |
 477                     ET_FLAGS_PERCPU;
 478                 lapic_et.et_quality = 600;
 479                 if (!arat) {
 480                         lapic_et.et_flags |= ET_FLAGS_C3STOP;
 481                         lapic_et.et_quality -= 200;
 482                 } else if ((cpu_feature & CPUID_TSC) != 0 &&
 483                     (cpu_feature2 & CPUID2_TSCDLT) != 0 &&
 484                     tsc_is_invariant && tsc_freq != 0) {
 485                         lapic_timer_tsc_deadline = 1;
 486                         TUNABLE_INT_FETCH("hw.lapic_tsc_deadline",
 487                             &lapic_timer_tsc_deadline);
 488                 }
 489
 490                 lapic_et.et_frequency = 0;
 491                 /* We don't know frequency yet, so trying to guess. */
 492                 lapic_et.et_min_period = 0x00001000LL;
 493                 lapic_et.et_max_period = SBT_1S;
 494                 lapic_et.et_start = lapic_et_start;
 495                 lapic_et.et_stop = lapic_et_stop;
 496                 lapic_et.et_priv = NULL;
 497                 et_register(&lapic_et);
 498         }
 499
 500         /*
 501          * Set lapic_eoi_suppression after lapic_enable(), to not
 502          * enable suppression in the hardware prematurely.  Note that
 503          * we by default enable suppression even when system only has
 504          * one IO-APIC, since EOI is broadcasted to all APIC agents,
 505          * including CPUs, otherwise.
 506          *
 507          * It seems that at least some KVM versions report
 508          * EOI_SUPPRESSION bit, but auto-EOI does not work.
 509          */
 510         ver = lapic_read32(LAPIC_VERSION);
 511         if ((ver & APIC_VER_EOI_SUPPRESSION) != 0) {
 512                 lapic_eoi_suppression = 1;
 513                 if (vm_guest == VM_GUEST_KVM) {
 514                         if (bootverbose)
 515                                 printf(
 516                        "KVM -- disabling lapic eoi suppression\n");
 517                         lapic_eoi_suppression = 0;
 518                 }
 519                 TUNABLE_INT_FETCH("hw.lapic_eoi_suppression",
 520                     &lapic_eoi_suppression);
 521         }
 522
 523 #ifdef SMP
 524 #define LOOPS   100000
 525         /*
 526          * Calibrate the busy loop waiting for IPI ack in xAPIC mode.
 527          * lapic_ipi_wait_mult contains the number of iterations which
 528          * approximately delay execution for 1 microsecond (the
 529          * argument to native_lapic_ipi_wait() is in microseconds).
 530          *
 531          * We assume that TSC is present and already measured.
 532          * Possible TSC frequency jumps are irrelevant to the
 533          * calibration loop below, the CPU clock management code is
 534          * not yet started, and we do not enter sleep states.
 535          */
 536         KASSERT((cpu_feature & CPUID_TSC) != 0 && tsc_freq != 0,
 537             ("TSC not initialized"));
 538         if (!x2apic_mode) {
 539                 r = rdtsc();
 540                 for (rx = 0; rx < LOOPS; rx++) {
 541                         (void)lapic_read_icr_lo();
 542                         ia32_pause();
 543                 }
 544                 r = rdtsc() - r;
 545                 r1 = tsc_freq * LOOPS;
 546                 r2 = r * 1000000;
 547                 lapic_ipi_wait_mult = r1 >= r2 ? r1 / r2 : 1;
 548                 if (bootverbose) {
 549                         printf("LAPIC: ipi_wait() us multiplier %ju (r %ju "
 550                             "tsc %ju)\n", (uintmax_t)lapic_ipi_wait_mult,
 551                             (uintmax_t)r, (uintmax_t)tsc_freq);
 552                 }
 553         }
 554 #undef LOOPS
 555 #endif /* SMP */
 556 }
 557
 558 /*
 559  * Create a local APIC instance.
 560  */
 561 static void
 562 native_lapic_create(u_int apic_id, int boot_cpu)
 563 {
 564         int i;
 565
 566         if (apic_id > MAX_APIC_ID) {
 567                 printf("APIC: Ignoring local APIC with ID %d\n", apic_id);
 568                 if (boot_cpu)
 569                         panic("Can't ignore BSP");
 570                 return;
 571         }
 572         KASSERT(!lapics[apic_id].la_present, ("duplicate local APIC %u",
 573             apic_id));
 574
 575         /*
 576          * Assume no local LVT overrides and a cluster of 0 and
 577          * intra-cluster ID of 0.
 578          */
 579         lapics[apic_id].la_present = 1;
 580         lapics[apic_id].la_id = apic_id;
 581         for (i = 0; i <= APIC_LVT_MAX; i++) {
 582                 lapics[apic_id].la_lvts[i] = lvts[i];
 583                 lapics[apic_id].la_lvts[i].lvt_active = 0;
 584         }
 585         for (i = 0; i <= APIC_NUM_IOINTS; i++)
 586             lapics[apic_id].la_ioint_irqs[i] = -1;
 587         lapics[apic_id].la_ioint_irqs[IDT_SYSCALL - APIC_IO_INTS] = IRQ_SYSCALL;
 588         lapics[apic_id].la_ioint_irqs[APIC_TIMER_INT - APIC_IO_INTS] =
 589             IRQ_TIMER;
 590 #ifdef KDTRACE_HOOKS
 591         lapics[apic_id].la_ioint_irqs[IDT_DTRACE_RET - APIC_IO_INTS] =
 592             IRQ_DTRACE_RET;
 593 #endif
 594 #ifdef XENHVM
 595         lapics[apic_id].la_ioint_irqs[IDT_EVTCHN - APIC_IO_INTS] = IRQ_EVTCHN;
 596 #endif
 597
 598
 599 #ifdef SMP
 600         cpu_add(apic_id, boot_cpu);
 601 #endif
 602 }
 603
 604 /*
 605  * Dump contents of local APIC registers
 606  */
 607 static void
 608 native_lapic_dump(const char* str)
 609 {
 610         uint32_t maxlvt;
 611
 612         maxlvt = (lapic_read32(LAPIC_VERSION) & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
 613         printf("cpu%d %s:\n", PCPU_GET(cpuid), str);
 614         printf("     ID: 0x%08x   VER: 0x%08x LDR: 0x%08x DFR: 0x%08x",
 615             lapic_read32(LAPIC_ID), lapic_read32(LAPIC_VERSION),
 616             lapic_read32(LAPIC_LDR), x2apic_mode ? 0 : lapic_read32(LAPIC_DFR));
 617         if ((cpu_feature2 & CPUID2_X2APIC) != 0)
 618                 printf(" x2APIC: %d", x2apic_mode);
 619         printf("\n  lint0: 0x%08x lint1: 0x%08x TPR: 0x%08x SVR: 0x%08x\n",
 620             lapic_read32(LAPIC_LVT_LINT0), lapic_read32(LAPIC_LVT_LINT1),
 621             lapic_read32(LAPIC_TPR), lapic_read32(LAPIC_SVR));
 622         printf("  timer: 0x%08x therm: 0x%08x err: 0x%08x",
 623             lapic_read32(LAPIC_LVT_TIMER), lapic_read32(LAPIC_LVT_THERMAL),
 624             lapic_read32(LAPIC_LVT_ERROR));
 625         if (maxlvt >= APIC_LVT_PMC)
 626                 printf(" pmc: 0x%08x", lapic_read32(LAPIC_LVT_PCINT));
 627         printf("\n");
 628         if (maxlvt >= APIC_LVT_CMCI)
 629                 printf("   cmci: 0x%08x\n", lapic_read32(LAPIC_LVT_CMCI));
 630 }
 631
 632 static void
 633 native_lapic_xapic_mode(void)
 634 {
 635         register_t saveintr;
 636
 637         saveintr = intr_disable();
 638         if (x2apic_mode)
 639                 native_lapic_enable_x2apic();
 640         intr_restore(saveintr);
 641 }
 642
 643 static void
 644 native_lapic_setup(int boot)
 645 {
 646         struct lapic *la;
 647         uint32_t maxlvt;
 648         register_t saveintr;
 649         char buf[MAXCOMLEN + 1];
 650
 651         saveintr = intr_disable();
 652
 653         la = &lapics[lapic_id()];
 654         KASSERT(la->la_present, ("missing APIC structure"));
 655         maxlvt = (lapic_read32(LAPIC_VERSION) & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
 656
 657         /* Initialize the TPR to allow all interrupts. */
 658         lapic_set_tpr(0);
 659
 660         /* Setup spurious vector and enable the local APIC. */
 661         lapic_enable();
 662
 663         /* Program LINT[01] LVT entries. */
 664         lapic_write32(LAPIC_LVT_LINT0, lvt_mode(la, APIC_LVT_LINT0,
 665             lapic_read32(LAPIC_LVT_LINT0)));
 666         lapic_write32(LAPIC_LVT_LINT1, lvt_mode(la, APIC_LVT_LINT1,
 667             lapic_read32(LAPIC_LVT_LINT1)));
 668
 669         /* Program the PMC LVT entry if present. */
 670         if (maxlvt >= APIC_LVT_PMC) {
 671                 lapic_write32(LAPIC_LVT_PCINT, lvt_mode(la, APIC_LVT_PMC,
 672                     LAPIC_LVT_PCINT));
 673         }
 674
 675         /* Program timer LVT and setup handler. */
 676         la->lvt_timer_base = lvt_mode(la, APIC_LVT_TIMER,
 677             lapic_read32(LAPIC_LVT_TIMER));
 678         la->lvt_timer_last = la->lvt_timer_base;
 679         lapic_write32(LAPIC_LVT_TIMER, la->lvt_timer_base);
 680         if (boot) {
 681                 snprintf(buf, sizeof(buf), "cpu%d:timer", PCPU_GET(cpuid));
 682                 intrcnt_add(buf, &la->la_timer_count);
 683         }
 684
 685         /* Setup the timer if configured. */
 686         if (la->la_timer_mode != LAT_MODE_UNDEF) {
 687                 KASSERT(la->la_timer_period != 0, ("lapic%u: zero divisor",
 688                     lapic_id()));
 689                 switch (la->la_timer_mode) {
 690                 case LAT_MODE_PERIODIC:
 691                         lapic_timer_set_divisor(lapic_timer_divisor);
 692                         lapic_timer_periodic(la);
 693                         break;
 694                 case LAT_MODE_ONESHOT:
 695                         lapic_timer_set_divisor(lapic_timer_divisor);
 696                         lapic_timer_oneshot(la);
 697                         break;
 698                 case LAT_MODE_DEADLINE:
 699                         lapic_timer_deadline(la);
 700                         break;
 701                 default:
 702                         panic("corrupted la_timer_mode %p %d", la,
 703                             la->la_timer_mode);
 704                 }
 705         }
 706
 707         /* Program error LVT and clear any existing errors. */
 708         lapic_write32(LAPIC_LVT_ERROR, lvt_mode(la, APIC_LVT_ERROR,
 709             lapic_read32(LAPIC_LVT_ERROR)));
 710         lapic_write32(LAPIC_ESR, 0);
 711
 712         /* XXX: Thermal LVT */
 713
 714         /* Program the CMCI LVT entry if present. */
 715         if (maxlvt >= APIC_LVT_CMCI) {
 716                 lapic_write32(LAPIC_LVT_CMCI, lvt_mode(la, APIC_LVT_CMCI,
 717                     lapic_read32(LAPIC_LVT_CMCI)));
 718         }
 719
 720         intr_restore(saveintr);
 721 }
 722
 723 static void
 724 native_lapic_reenable_pmc(void)
 725 {
 726 #ifdef HWPMC_HOOKS
 727         uint32_t value;
 728
 729         value = lapic_read32(LAPIC_LVT_PCINT);
 730         value &= ~APIC_LVT_M;
 731         lapic_write32(LAPIC_LVT_PCINT, value);
 732 #endif
 733 }
 734
 735 #ifdef HWPMC_HOOKS
 736 static void
 737 lapic_update_pmc(void *dummy)
 738 {
 739         struct lapic *la;
 740
 741         la = &lapics[lapic_id()];
 742         lapic_write32(LAPIC_LVT_PCINT, lvt_mode(la, APIC_LVT_PMC,
 743             lapic_read32(LAPIC_LVT_PCINT)));
 744 }
 745 #endif
 746
 747 static int
 748 native_lapic_enable_pmc(void)
 749 {
 750 #ifdef HWPMC_HOOKS
 751         u_int32_t maxlvt;
 752
 753         /* Fail if the local APIC is not present. */
 754         if (!x2apic_mode && lapic_map == NULL)
 755                 return (0);
 756
 757         /* Fail if the PMC LVT is not present. */
 758         maxlvt = (lapic_read32(LAPIC_VERSION) & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
 759         if (maxlvt < APIC_LVT_PMC)
 760                 return (0);
 761
 762         lvts[APIC_LVT_PMC].lvt_masked = 0;
 763
 764 #ifdef EARLY_AP_STARTUP
 765         MPASS(mp_ncpus == 1 || smp_started);
 766         smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL);
 767 #else
 768 #ifdef SMP
 769         /*
 770          * If hwpmc was loaded at boot time then the APs may not be
 771          * started yet.  In that case, don't forward the request to
 772          * them as they will program the lvt when they start.
 773          */
 774         if (smp_started)
 775                 smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL);
 776         else
 777 #endif
 778                 lapic_update_pmc(NULL);
 779 #endif
 780         return (1);
 781 #else
 782         return (0);
 783 #endif
 784 }
 785
 786 static void
 787 native_lapic_disable_pmc(void)
 788 {
 789 #ifdef HWPMC_HOOKS
 790         u_int32_t maxlvt;
 791
 792         /* Fail if the local APIC is not present. */
 793         if (!x2apic_mode && lapic_map == NULL)
 794                 return;
 795
 796         /* Fail if the PMC LVT is not present. */
 797         maxlvt = (lapic_read32(LAPIC_VERSION) & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
 798         if (maxlvt < APIC_LVT_PMC)
 799                 return;
 800
 801         lvts[APIC_LVT_PMC].lvt_masked = 1;
 802
 803 #ifdef SMP
 804         /* The APs should always be started when hwpmc is unloaded. */
 805         KASSERT(mp_ncpus == 1 || smp_started, ("hwpmc unloaded too early"));
 806 #endif
 807         smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL);
 808 #endif
 809 }
 810
 811 static void
 812 lapic_calibrate_initcount(struct eventtimer *et, struct lapic *la)
 813 {
 814         u_long value;
 815
 816         /* Start off with a divisor of 2 (power on reset default). */
 817         lapic_timer_divisor = 2;
 818         /* Try to calibrate the local APIC timer. */
 819         do {
 820                 lapic_timer_set_divisor(lapic_timer_divisor);
 821                 lapic_timer_oneshot_nointr(la, APIC_TIMER_MAX_COUNT);
 822                 DELAY(1000000);
 823                 value = APIC_TIMER_MAX_COUNT - lapic_read32(LAPIC_CCR_TIMER);
 824                 if (value != APIC_TIMER_MAX_COUNT)
 825                         break;
 826                 lapic_timer_divisor <<= 1;
 827         } while (lapic_timer_divisor <= 128);
 828         if (lapic_timer_divisor > 128)
 829                 panic("lapic: Divisor too big");
 830         if (bootverbose) {
 831                 printf("lapic: Divisor %lu, Frequency %lu Hz\n",
 832                     lapic_timer_divisor, value);
 833         }
 834         count_freq = value;
 835 }
 836
 837 static void
 838 lapic_calibrate_deadline(struct eventtimer *et, struct lapic *la __unused)
 839 {
 840
 841         if (bootverbose) {
 842                 printf("lapic: deadline tsc mode, Frequency %ju Hz\n",
 843                     (uintmax_t)tsc_freq);
 844         }
 845 }
 846
 847 static void
 848 lapic_change_mode(struct eventtimer *et, struct lapic *la,
 849     enum lat_timer_mode newmode)
 850 {
 851
 852         if (la->la_timer_mode == newmode)
 853                 return;
 854         switch (newmode) {
 855         case LAT_MODE_PERIODIC:
 856                 lapic_timer_set_divisor(lapic_timer_divisor);
 857                 et->et_frequency = count_freq;
 858                 break;
 859         case LAT_MODE_DEADLINE:
 860                 et->et_frequency = tsc_freq;
 861                 break;
 862         case LAT_MODE_ONESHOT:
 863                 lapic_timer_set_divisor(lapic_timer_divisor);
 864                 et->et_frequency = count_freq;
 865                 break;
 866         default:
 867                 panic("lapic_change_mode %d", newmode);
 868         }
 869         la->la_timer_mode = newmode;
 870         et->et_min_period = (0x00000002LLU << 32) / et->et_frequency;
 871         et->et_max_period = (0xfffffffeLLU << 32) / et->et_frequency;
 872 }
 873
 874 static int
 875 lapic_et_start(struct eventtimer *et, sbintime_t first, sbintime_t period)
 876 {
 877         struct lapic *la;
 878
 879         la = &lapics[PCPU_GET(apic_id)];
 880         if (et->et_frequency == 0) {
 881                 lapic_calibrate_initcount(et, la);
 882                 if (lapic_timer_tsc_deadline)
 883                         lapic_calibrate_deadline(et, la);
 884         }
 885         if (period != 0) {
 886                 lapic_change_mode(et, la, LAT_MODE_PERIODIC);
 887                 la->la_timer_period = ((uint32_t)et->et_frequency * period) >>
 888                     32;
 889                 lapic_timer_periodic(la);
 890         } else if (lapic_timer_tsc_deadline) {
 891                 lapic_change_mode(et, la, LAT_MODE_DEADLINE);
 892                 la->la_timer_period = (et->et_frequency * first) >> 32;
 893                 lapic_timer_deadline(la);
 894         } else {
 895                 lapic_change_mode(et, la, LAT_MODE_ONESHOT);
 896                 la->la_timer_period = ((uint32_t)et->et_frequency * first) >>
 897                     32;
 898                 lapic_timer_oneshot(la);
 899         }
 900         return (0);
 901 }
 902
 903 static int
 904 lapic_et_stop(struct eventtimer *et)
 905 {
 906         struct lapic *la;
 907
 908         la = &lapics[PCPU_GET(apic_id)];
 909         lapic_timer_stop(la);
 910         la->la_timer_mode = LAT_MODE_UNDEF;
 911         return (0);
 912 }
 913
 914 static void
 915 native_lapic_disable(void)
 916 {
 917         uint32_t value;
 918
 919         /* Software disable the local APIC. */
 920         value = lapic_read32(LAPIC_SVR);
 921         value &= ~APIC_SVR_SWEN;
 922         lapic_write32(LAPIC_SVR, value);
 923 }
 924
 925 static void
 926 lapic_enable(void)
 927 {
 928         uint32_t value;
 929
 930         /* Program the spurious vector to enable the local APIC. */
 931         value = lapic_read32(LAPIC_SVR);
 932         value &= ~(APIC_SVR_VECTOR | APIC_SVR_FOCUS);
 933         value |= APIC_SVR_FEN | APIC_SVR_SWEN | APIC_SPURIOUS_INT;
 934         if (lapic_eoi_suppression)
 935                 value |= APIC_SVR_EOI_SUPPRESSION;
 936         lapic_write32(LAPIC_SVR, value);
 937 }
 938
 939 /* Reset the local APIC on the BSP during resume. */
 940 static void
 941 lapic_resume(struct pic *pic, bool suspend_cancelled)
 942 {
 943
 944         lapic_setup(0);
 945 }
 946
 947 static int
 948 native_lapic_id(void)
 949 {
 950         uint32_t v;
 951
 952         KASSERT(x2apic_mode || lapic_map != NULL, ("local APIC is not mapped"));
 953         v = lapic_read32(LAPIC_ID);
 954         if (!x2apic_mode)
 955                 v >>= APIC_ID_SHIFT;
 956         return (v);
 957 }
 958
 959 static int
 960 native_lapic_intr_pending(u_int vector)
 961 {
 962         uint32_t irr;
 963
 964         /*
 965          * The IRR registers are an array of registers each of which
 966          * only describes 32 interrupts in the low 32 bits.  Thus, we
 967          * divide the vector by 32 to get the register index.
 968          * Finally, we modulus the vector by 32 to determine the
 969          * individual bit to test.
 970          */
 971         irr = lapic_read32(LAPIC_IRR0 + vector / 32);
 972         return (irr & 1 << (vector % 32));
 973 }
 974
 975 static void
 976 native_lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id)
 977 {
 978         struct lapic *la;
 979
 980         KASSERT(lapics[apic_id].la_present, ("%s: APIC %u doesn't exist",
 981             __func__, apic_id));
 982         KASSERT(cluster <= APIC_MAX_CLUSTER, ("%s: cluster %u too big",
 983             __func__, cluster));
 984         KASSERT(cluster_id <= APIC_MAX_INTRACLUSTER_ID,
 985             ("%s: intra cluster id %u too big", __func__, cluster_id));
 986         la = &lapics[apic_id];
 987         la->la_cluster = cluster;
 988         la->la_cluster_id = cluster_id;
 989 }
 990
 991 static int
 992 native_lapic_set_lvt_mask(u_int apic_id, u_int pin, u_char masked)
 993 {
 994
 995         if (pin > APIC_LVT_MAX)
 996                 return (EINVAL);
 997         if (apic_id == APIC_ID_ALL) {
 998                 lvts[pin].lvt_masked = masked;
 999                 if (bootverbose)
1000                         printf("lapic:");
1001         } else {
1002                 KASSERT(lapics[apic_id].la_present,
1003                     ("%s: missing APIC %u", __func__, apic_id));
1004                 lapics[apic_id].la_lvts[pin].lvt_masked = masked;
1005                 lapics[apic_id].la_lvts[pin].lvt_active = 1;
1006                 if (bootverbose)
1007                         printf("lapic%u:", apic_id);
1008         }
1009         if (bootverbose)
1010                 printf(" LINT%u %s\n", pin, masked ? "masked" : "unmasked");
1011         return (0);
1012 }
1013
1014 static int
1015 native_lapic_set_lvt_mode(u_int apic_id, u_int pin, u_int32_t mode)
1016 {
1017         struct lvt *lvt;
1018
1019         if (pin > APIC_LVT_MAX)
1020                 return (EINVAL);
1021         if (apic_id == APIC_ID_ALL) {
1022                 lvt = &lvts[pin];
1023                 if (bootverbose)
1024                         printf("lapic:");
1025         } else {
1026                 KASSERT(lapics[apic_id].la_present,
1027                     ("%s: missing APIC %u", __func__, apic_id));
1028                 lvt = &lapics[apic_id].la_lvts[pin];
1029                 lvt->lvt_active = 1;
1030                 if (bootverbose)
1031                         printf("lapic%u:", apic_id);
1032         }
1033         lvt->lvt_mode = mode;
1034         switch (mode) {
1035         case APIC_LVT_DM_NMI:
1036         case APIC_LVT_DM_SMI:
1037         case APIC_LVT_DM_INIT:
1038         case APIC_LVT_DM_EXTINT:
1039                 lvt->lvt_edgetrigger = 1;
1040                 lvt->lvt_activehi = 1;
1041                 if (mode == APIC_LVT_DM_EXTINT)
1042                         lvt->lvt_masked = 1;
1043                 else
1044                         lvt->lvt_masked = 0;
1045                 break;
1046         default:
1047                 panic("Unsupported delivery mode: 0x%x\n", mode);
1048         }
1049         if (bootverbose) {
1050                 printf(" Routing ");
1051                 switch (mode) {
1052                 case APIC_LVT_DM_NMI:
1053                         printf("NMI");
1054                         break;
1055                 case APIC_LVT_DM_SMI:
1056                         printf("SMI");
1057                         break;
1058                 case APIC_LVT_DM_INIT:
1059                         printf("INIT");
1060                         break;
1061                 case APIC_LVT_DM_EXTINT:
1062                         printf("ExtINT");
1063                         break;
1064                 }
1065                 printf(" -> LINT%u\n", pin);
1066         }
1067         return (0);
1068 }
1069
1070 static int
1071 native_lapic_set_lvt_polarity(u_int apic_id, u_int pin, enum intr_polarity pol)
1072 {
1073
1074         if (pin > APIC_LVT_MAX || pol == INTR_POLARITY_CONFORM)
1075                 return (EINVAL);
1076         if (apic_id == APIC_ID_ALL) {
1077                 lvts[pin].lvt_activehi = (pol == INTR_POLARITY_HIGH);
1078                 if (bootverbose)
1079                         printf("lapic:");
1080         } else {
1081                 KASSERT(lapics[apic_id].la_present,
1082                     ("%s: missing APIC %u", __func__, apic_id));
1083                 lapics[apic_id].la_lvts[pin].lvt_active = 1;
1084                 lapics[apic_id].la_lvts[pin].lvt_activehi =
1085                     (pol == INTR_POLARITY_HIGH);
1086                 if (bootverbose)
1087                         printf("lapic%u:", apic_id);
1088         }
1089         if (bootverbose)
1090                 printf(" LINT%u polarity: %s\n", pin,
1091                     pol == INTR_POLARITY_HIGH ? "high" : "low");
1092         return (0);
1093 }
1094
1095 static int
1096 native_lapic_set_lvt_triggermode(u_int apic_id, u_int pin,
1097      enum intr_trigger trigger)
1098 {
1099
1100         if (pin > APIC_LVT_MAX || trigger == INTR_TRIGGER_CONFORM)
1101                 return (EINVAL);
1102         if (apic_id == APIC_ID_ALL) {
1103                 lvts[pin].lvt_edgetrigger = (trigger == INTR_TRIGGER_EDGE);
1104                 if (bootverbose)
1105                         printf("lapic:");
1106         } else {
1107                 KASSERT(lapics[apic_id].la_present,
1108                     ("%s: missing APIC %u", __func__, apic_id));
1109                 lapics[apic_id].la_lvts[pin].lvt_edgetrigger =
1110                     (trigger == INTR_TRIGGER_EDGE);
1111                 lapics[apic_id].la_lvts[pin].lvt_active = 1;
1112                 if (bootverbose)
1113                         printf("lapic%u:", apic_id);
1114         }
1115         if (bootverbose)
1116                 printf(" LINT%u trigger: %s\n", pin,
1117                     trigger == INTR_TRIGGER_EDGE ? "edge" : "level");
1118         return (0);
1119 }
1120
1121 /*
1122  * Adjust the TPR of the current CPU so that it blocks all interrupts below
1123  * the passed in vector.
1124  */
1125 static void
1126 lapic_set_tpr(u_int vector)
1127 {
1128 #ifdef CHEAP_TPR
1129         lapic_write32(LAPIC_TPR, vector);
1130 #else
1131         uint32_t tpr;
1132
1133         tpr = lapic_read32(LAPIC_TPR) & ~APIC_TPR_PRIO;
1134         tpr |= vector;
1135         lapic_write32(LAPIC_TPR, tpr);
1136 #endif
1137 }
1138
1139 static void
1140 native_lapic_eoi(void)
1141 {
1142
1143         lapic_write32_nofence(LAPIC_EOI, 0);
1144 }
1145
1146 void
1147 lapic_handle_intr(int vector, struct trapframe *frame)
1148 {
1149         struct intsrc *isrc;
1150
1151         isrc = intr_lookup_source(apic_idt_to_irq(PCPU_GET(apic_id),
1152             vector));
1153         intr_execute_handlers(isrc, frame);
1154 }
1155
1156 void
1157 lapic_handle_timer(struct trapframe *frame)
1158 {
1159         struct lapic *la;
1160         struct trapframe *oldframe;
1161         struct thread *td;
1162
1163         /* Send EOI first thing. */
1164         lapic_eoi();
1165
1166 #if defined(SMP) && !defined(SCHED_ULE)
1167         /*
1168          * Don't do any accounting for the disabled HTT cores, since it
1169          * will provide misleading numbers for the userland.
1170          *
1171          * No locking is necessary here, since even if we lose the race
1172          * when hlt_cpus_mask changes it is not a big deal, really.
1173          *
1174          * Don't do that for ULE, since ULE doesn't consider hlt_cpus_mask
1175          * and unlike other schedulers it actually schedules threads to
1176          * those CPUs.
1177          */
1178         if (CPU_ISSET(PCPU_GET(cpuid), &hlt_cpus_mask))
1179                 return;
1180 #endif
1181
1182         /* Look up our local APIC structure for the tick counters. */
1183         la = &lapics[PCPU_GET(apic_id)];
1184         (*la->la_timer_count)++;
1185         critical_enter();
1186         if (lapic_et.et_active) {
1187                 td = curthread;
1188                 td->td_intr_nesting_level++;
1189                 oldframe = td->td_intr_frame;
1190                 td->td_intr_frame = frame;
1191                 lapic_et.et_event_cb(&lapic_et, lapic_et.et_arg);
1192                 td->td_intr_frame = oldframe;
1193                 td->td_intr_nesting_level--;
1194         }
1195         critical_exit();
1196 }
1197
1198 static void
1199 lapic_timer_set_divisor(u_int divisor)
1200 {
1201
1202         KASSERT(powerof2(divisor), ("lapic: invalid divisor %u", divisor));
1203         KASSERT(ffs(divisor) <= nitems(lapic_timer_divisors),
1204                 ("lapic: invalid divisor %u", divisor));
1205         lapic_write32(LAPIC_DCR_TIMER, lapic_timer_divisors[ffs(divisor) - 1]);
1206 }
1207
1208 static void
1209 lapic_timer_oneshot(struct lapic *la)
1210 {
1211         uint32_t value;
1212
1213         value = la->lvt_timer_base;
1214         value &= ~(APIC_LVTT_TM | APIC_LVT_M);
1215         value |= APIC_LVTT_TM_ONE_SHOT;
1216         la->lvt_timer_last = value;
1217         lapic_write32(LAPIC_LVT_TIMER, value);
1218         lapic_write32(LAPIC_ICR_TIMER, la->la_timer_period);
1219 }
1220
1221 static void
1222 lapic_timer_oneshot_nointr(struct lapic *la, uint32_t count)
1223 {
1224         uint32_t value;
1225
1226         value = la->lvt_timer_base;
1227         value &= ~APIC_LVTT_TM;
1228         value |= APIC_LVTT_TM_ONE_SHOT | APIC_LVT_M;
1229         la->lvt_timer_last = value;
1230         lapic_write32(LAPIC_LVT_TIMER, value);
1231         lapic_write32(LAPIC_ICR_TIMER, count);
1232 }
1233
1234 static void
1235 lapic_timer_periodic(struct lapic *la)
1236 {
1237         uint32_t value;
1238
1239         value = la->lvt_timer_base;
1240         value &= ~(APIC_LVTT_TM | APIC_LVT_M);
1241         value |= APIC_LVTT_TM_PERIODIC;
1242         la->lvt_timer_last = value;
1243         lapic_write32(LAPIC_LVT_TIMER, value);
1244         lapic_write32(LAPIC_ICR_TIMER, la->la_timer_period);
1245 }
1246
1247 static void
1248 lapic_timer_deadline(struct lapic *la)
1249 {
1250         uint32_t value;
1251
1252         value = la->lvt_timer_base;
1253         value &= ~(APIC_LVTT_TM | APIC_LVT_M);
1254         value |= APIC_LVTT_TM_TSCDLT;
1255         if (value != la->lvt_timer_last) {
1256                 la->lvt_timer_last = value;
1257                 lapic_write32_nofence(LAPIC_LVT_TIMER, value);
1258                 if (!x2apic_mode)
1259                         mfence();
1260         }
1261         wrmsr(MSR_TSC_DEADLINE, la->la_timer_period + rdtsc());
1262 }
1263
1264 static void
1265 lapic_timer_stop(struct lapic *la)
1266 {
1267         uint32_t value;
1268
1269         if (la->la_timer_mode == LAT_MODE_DEADLINE) {
1270                 wrmsr(MSR_TSC_DEADLINE, 0);
1271                 mfence();
1272         } else {
1273                 value = la->lvt_timer_base;
1274                 value &= ~APIC_LVTT_TM;
1275                 value |= APIC_LVT_M;
1276                 la->lvt_timer_last = value;
1277                 lapic_write32(LAPIC_LVT_TIMER, value);
1278         }
1279 }
1280
1281 void
1282 lapic_handle_cmc(void)
1283 {
1284
1285         lapic_eoi();
1286         cmc_intr();
1287 }
1288
1289 /*
1290  * Called from the mca_init() to activate the CMC interrupt if this CPU is
1291  * responsible for monitoring any MC banks for CMC events.  Since mca_init()
1292  * is called prior to lapic_setup() during boot, this just needs to unmask
1293  * this CPU's LVT_CMCI entry.
1294  */
1295 static void
1296 native_lapic_enable_cmc(void)
1297 {
1298         u_int apic_id;
1299
1300 #ifdef DEV_ATPIC
1301         if (!x2apic_mode && lapic_map == NULL)
1302                 return;
1303 #endif
1304         apic_id = PCPU_GET(apic_id);
1305         KASSERT(lapics[apic_id].la_present,
1306             ("%s: missing APIC %u", __func__, apic_id));
1307         lapics[apic_id].la_lvts[APIC_LVT_CMCI].lvt_masked = 0;
1308         lapics[apic_id].la_lvts[APIC_LVT_CMCI].lvt_active = 1;
1309         if (bootverbose)
1310                 printf("lapic%u: CMCI unmasked\n", apic_id);
1311 }
1312
1313 void
1314 lapic_handle_error(void)
1315 {
1316         uint32_t esr;
1317
1318         /*
1319          * Read the contents of the error status register.  Write to
1320          * the register first before reading from it to force the APIC
1321          * to update its value to indicate any errors that have
1322          * occurred since the previous write to the register.
1323          */
1324         lapic_write32(LAPIC_ESR, 0);
1325         esr = lapic_read32(LAPIC_ESR);
1326
1327         printf("CPU%d: local APIC error 0x%x\n", PCPU_GET(cpuid), esr);
1328         lapic_eoi();
1329 }
1330
1331 static u_int
1332 native_apic_cpuid(u_int apic_id)
1333 {
1334 #ifdef SMP
1335         return apic_cpuids[apic_id];
1336 #else
1337         return 0;
1338 #endif
1339 }
1340
1341 /* Request a free IDT vector to be used by the specified IRQ. */
1342 static u_int
1343 native_apic_alloc_vector(u_int apic_id, u_int irq)
1344 {
1345         u_int vector;
1346
1347         KASSERT(irq < NUM_IO_INTS, ("Invalid IRQ %u", irq));
1348
1349         /*
1350          * Search for a free vector.  Currently we just use a very simple
1351          * algorithm to find the first free vector.
1352          */
1353         mtx_lock_spin(&icu_lock);
1354         for (vector = 0; vector < APIC_NUM_IOINTS; vector++) {
1355                 if (lapics[apic_id].la_ioint_irqs[vector] != -1)
1356                         continue;
1357                 lapics[apic_id].la_ioint_irqs[vector] = irq;
1358                 mtx_unlock_spin(&icu_lock);
1359                 return (vector + APIC_IO_INTS);
1360         }
1361         mtx_unlock_spin(&icu_lock);
1362         return (0);
1363 }
1364
1365 /*
1366  * Request 'count' free contiguous IDT vectors to be used by 'count'
1367  * IRQs.  'count' must be a power of two and the vectors will be
1368  * aligned on a boundary of 'align'.  If the request cannot be
1369  * satisfied, 0 is returned.
1370  */
1371 static u_int
1372 native_apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align)
1373 {
1374         u_int first, run, vector;
1375
1376         KASSERT(powerof2(count), ("bad count"));
1377         KASSERT(powerof2(align), ("bad align"));
1378         KASSERT(align >= count, ("align < count"));
1379 #ifdef INVARIANTS
1380         for (run = 0; run < count; run++)
1381                 KASSERT(irqs[run] < NUM_IO_INTS, ("Invalid IRQ %u at index %u",
1382                     irqs[run], run));
1383 #endif
1384
1385         /*
1386          * Search for 'count' free vectors.  As with apic_alloc_vector(),
1387          * this just uses a simple first fit algorithm.
1388          */
1389         run = 0;
1390         first = 0;
1391         mtx_lock_spin(&icu_lock);
1392         for (vector = 0; vector < APIC_NUM_IOINTS; vector++) {
1393
1394                 /* Vector is in use, end run. */
1395                 if (lapics[apic_id].la_ioint_irqs[vector] != -1) {
1396                         run = 0;
1397                         first = 0;
1398                         continue;
1399                 }
1400
1401                 /* Start a new run if run == 0 and vector is aligned. */
1402                 if (run == 0) {
1403                         if ((vector & (align - 1)) != 0)
1404                                 continue;
1405                         first = vector;
1406                 }
1407                 run++;
1408
1409                 /* Keep looping if the run isn't long enough yet. */
1410                 if (run < count)
1411                         continue;
1412
1413                 /* Found a run, assign IRQs and return the first vector. */
1414                 for (vector = 0; vector < count; vector++)
1415                         lapics[apic_id].la_ioint_irqs[first + vector] =
1416                             irqs[vector];
1417                 mtx_unlock_spin(&icu_lock);
1418                 return (first + APIC_IO_INTS);
1419         }
1420         mtx_unlock_spin(&icu_lock);
1421         printf("APIC: Couldn't find APIC vectors for %u IRQs\n", count);
1422         return (0);
1423 }
1424
1425 /*
1426  * Enable a vector for a particular apic_id.  Since all lapics share idt
1427  * entries and ioint_handlers this enables the vector on all lapics.  lapics
1428  * which do not have the vector configured would report spurious interrupts
1429  * should it fire.
1430  */
1431 static void
1432 native_apic_enable_vector(u_int apic_id, u_int vector)
1433 {
1434
1435         KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry"));
1436         KASSERT(ioint_handlers[vector / 32] != NULL,
1437             ("No ISR handler for vector %u", vector));
1438 #ifdef KDTRACE_HOOKS
1439         KASSERT(vector != IDT_DTRACE_RET,
1440             ("Attempt to overwrite DTrace entry"));
1441 #endif
1442         setidt(vector, ioint_handlers[vector / 32], SDT_APIC, SEL_KPL,
1443             GSEL_APIC);
1444 }
1445
1446 static void
1447 native_apic_disable_vector(u_int apic_id, u_int vector)
1448 {
1449
1450         KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry"));
1451 #ifdef KDTRACE_HOOKS
1452         KASSERT(vector != IDT_DTRACE_RET,
1453             ("Attempt to overwrite DTrace entry"));
1454 #endif
1455         KASSERT(ioint_handlers[vector / 32] != NULL,
1456             ("No ISR handler for vector %u", vector));
1457 #ifdef notyet
1458         /*
1459          * We can not currently clear the idt entry because other cpus
1460          * may have a valid vector at this offset.
1461          */
1462         setidt(vector, &IDTVEC(rsvd), SDT_APICT, SEL_KPL, GSEL_APIC);
1463 #endif
1464 }
1465
1466 /* Release an APIC vector when it's no longer in use. */
1467 static void
1468 native_apic_free_vector(u_int apic_id, u_int vector, u_int irq)
1469 {
1470         struct thread *td;
1471
1472         KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL &&
1473             vector <= APIC_IO_INTS + APIC_NUM_IOINTS,
1474             ("Vector %u does not map to an IRQ line", vector));
1475         KASSERT(irq < NUM_IO_INTS, ("Invalid IRQ %u", irq));
1476         KASSERT(lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] ==
1477             irq, ("IRQ mismatch"));
1478 #ifdef KDTRACE_HOOKS
1479         KASSERT(vector != IDT_DTRACE_RET,
1480             ("Attempt to overwrite DTrace entry"));
1481 #endif
1482
1483         /*
1484          * Bind us to the cpu that owned the vector before freeing it so
1485          * we don't lose an interrupt delivery race.
1486          */
1487         td = curthread;
1488         if (!rebooting) {
1489                 thread_lock(td);
1490                 if (sched_is_bound(td))
1491                         panic("apic_free_vector: Thread already bound.\n");
1492                 sched_bind(td, apic_cpuid(apic_id));
1493                 thread_unlock(td);
1494         }
1495         mtx_lock_spin(&icu_lock);
1496         lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] = -1;
1497         mtx_unlock_spin(&icu_lock);
1498         if (!rebooting) {
1499                 thread_lock(td);
1500                 sched_unbind(td);
1501                 thread_unlock(td);
1502         }
1503 }
1504
1505 /* Map an IDT vector (APIC) to an IRQ (interrupt source). */
1506 static u_int
1507 apic_idt_to_irq(u_int apic_id, u_int vector)
1508 {
1509         int irq;
1510
1511         KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL &&
1512             vector <= APIC_IO_INTS + APIC_NUM_IOINTS,
1513             ("Vector %u does not map to an IRQ line", vector));
1514 #ifdef KDTRACE_HOOKS
1515         KASSERT(vector != IDT_DTRACE_RET,
1516             ("Attempt to overwrite DTrace entry"));
1517 #endif
1518         irq = lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS];
1519         if (irq < 0)
1520                 irq = 0;
1521         return (irq);
1522 }
1523
1524 #ifdef DDB
1525 /*
1526  * Dump data about APIC IDT vector mappings.
1527  */
1528 DB_SHOW_COMMAND(apic, db_show_apic)
1529 {
1530         struct intsrc *isrc;
1531         int i, verbose;
1532         u_int apic_id;
1533         u_int irq;
1534
1535         if (strcmp(modif, "vv") == 0)
1536                 verbose = 2;
1537         else if (strcmp(modif, "v") == 0)
1538                 verbose = 1;
1539         else
1540                 verbose = 0;
1541         for (apic_id = 0; apic_id <= MAX_APIC_ID; apic_id++) {
1542                 if (lapics[apic_id].la_present == 0)
1543                         continue;
1544                 db_printf("Interrupts bound to lapic %u\n", apic_id);
1545                 for (i = 0; i < APIC_NUM_IOINTS + 1 && !db_pager_quit; i++) {
1546                         irq = lapics[apic_id].la_ioint_irqs[i];
1547                         if (irq == -1 || irq == IRQ_SYSCALL)
1548                                 continue;
1549 #ifdef KDTRACE_HOOKS
1550                         if (irq == IRQ_DTRACE_RET)
1551                                 continue;
1552 #endif
1553 #ifdef XENHVM
1554                         if (irq == IRQ_EVTCHN)
1555                                 continue;
1556 #endif
1557                         db_printf("vec 0x%2x -> ", i + APIC_IO_INTS);
1558                         if (irq == IRQ_TIMER)
1559                                 db_printf("lapic timer\n");
1560                         else if (irq < NUM_IO_INTS) {
1561                                 isrc = intr_lookup_source(irq);
1562                                 if (isrc == NULL || verbose == 0)
1563                                         db_printf("IRQ %u\n", irq);
1564                                 else
1565                                         db_dump_intr_event(isrc->is_event,
1566                                             verbose == 2);
1567                         } else
1568                                 db_printf("IRQ %u ???\n", irq);
1569                 }
1570         }
1571 }
1572
1573 static void
1574 dump_mask(const char *prefix, uint32_t v, int base)
1575 {
1576         int i, first;
1577
1578         first = 1;
1579         for (i = 0; i < 32; i++)
1580                 if (v & (1 << i)) {
1581                         if (first) {
1582                                 db_printf("%s:", prefix);
1583                                 first = 0;
1584                         }
1585                         db_printf(" %02x", base + i);
1586                 }
1587         if (!first)
1588                 db_printf("\n");
1589 }
1590
1591 /* Show info from the lapic regs for this CPU. */
1592 DB_SHOW_COMMAND(lapic, db_show_lapic)
1593 {
1594         uint32_t v;
1595
1596         db_printf("lapic ID = %d\n", lapic_id());
1597         v = lapic_read32(LAPIC_VERSION);
1598         db_printf("version  = %d.%d\n", (v & APIC_VER_VERSION) >> 4,
1599             v & 0xf);
1600         db_printf("max LVT  = %d\n", (v & APIC_VER_MAXLVT) >> MAXLVTSHIFT);
1601         v = lapic_read32(LAPIC_SVR);
1602         db_printf("SVR      = %02x (%s)\n", v & APIC_SVR_VECTOR,
1603             v & APIC_SVR_ENABLE ? "enabled" : "disabled");
1604         db_printf("TPR      = %02x\n", lapic_read32(LAPIC_TPR));
1605
1606 #define dump_field(prefix, regn, index)                                 \
1607         dump_mask(__XSTRING(prefix ## index),                           \
1608             lapic_read32(LAPIC_ ## regn ## index),                      \
1609             index * 32)
1610
1611         db_printf("In-service Interrupts:\n");
1612         dump_field(isr, ISR, 0);
1613         dump_field(isr, ISR, 1);
1614         dump_field(isr, ISR, 2);
1615         dump_field(isr, ISR, 3);
1616         dump_field(isr, ISR, 4);
1617         dump_field(isr, ISR, 5);
1618         dump_field(isr, ISR, 6);
1619         dump_field(isr, ISR, 7);
1620
1621         db_printf("TMR Interrupts:\n");
1622         dump_field(tmr, TMR, 0);
1623         dump_field(tmr, TMR, 1);
1624         dump_field(tmr, TMR, 2);
1625         dump_field(tmr, TMR, 3);
1626         dump_field(tmr, TMR, 4);
1627         dump_field(tmr, TMR, 5);
1628         dump_field(tmr, TMR, 6);
1629         dump_field(tmr, TMR, 7);
1630
1631         db_printf("IRR Interrupts:\n");
1632         dump_field(irr, IRR, 0);
1633         dump_field(irr, IRR, 1);
1634         dump_field(irr, IRR, 2);
1635         dump_field(irr, IRR, 3);
1636         dump_field(irr, IRR, 4);
1637         dump_field(irr, IRR, 5);
1638         dump_field(irr, IRR, 6);
1639         dump_field(irr, IRR, 7);
1640
1641 #undef dump_field
1642 }
1643 #endif
1644
1645 /*
1646  * APIC probing support code.  This includes code to manage enumerators.
1647  */
1648
1649 static SLIST_HEAD(, apic_enumerator) enumerators =
1650         SLIST_HEAD_INITIALIZER(enumerators);
1651 static struct apic_enumerator *best_enum;
1652
1653 void
1654 apic_register_enumerator(struct apic_enumerator *enumerator)
1655 {
1656 #ifdef INVARIANTS
1657         struct apic_enumerator *apic_enum;
1658
1659         SLIST_FOREACH(apic_enum, &enumerators, apic_next) {
1660                 if (apic_enum == enumerator)
1661                         panic("%s: Duplicate register of %s", __func__,
1662                             enumerator->apic_name);
1663         }
1664 #endif
1665         SLIST_INSERT_HEAD(&enumerators, enumerator, apic_next);
1666 }
1667
1668 /*
1669  * We have to look for CPU's very, very early because certain subsystems
1670  * want to know how many CPU's we have extremely early on in the boot
1671  * process.
1672  */
1673 static void
1674 apic_init(void *dummy __unused)
1675 {
1676         struct apic_enumerator *enumerator;
1677         int retval, best;
1678
1679         /* We only support built in local APICs. */
1680         if (!(cpu_feature & CPUID_APIC))
1681                 return;
1682
1683         /* Don't probe if APIC mode is disabled. */
1684         if (resource_disabled("apic", 0))
1685                 return;
1686
1687         /* Probe all the enumerators to find the best match. */
1688         best_enum = NULL;
1689         best = 0;
1690         SLIST_FOREACH(enumerator, &enumerators, apic_next) {
1691                 retval = enumerator->apic_probe();
1692                 if (retval > 0)
1693                         continue;
1694                 if (best_enum == NULL || best < retval) {
1695                         best_enum = enumerator;
1696                         best = retval;
1697                 }
1698         }
1699         if (best_enum == NULL) {
1700                 if (bootverbose)
1701                         printf("APIC: Could not find any APICs.\n");
1702 #ifndef DEV_ATPIC
1703                 panic("running without device atpic requires a local APIC");
1704 #endif
1705                 return;
1706         }
1707
1708         if (bootverbose)
1709                 printf("APIC: Using the %s enumerator.\n",
1710                     best_enum->apic_name);
1711
1712 #ifdef I686_CPU
1713         /*
1714          * To work around an errata, we disable the local APIC on some
1715          * CPUs during early startup.  We need to turn the local APIC back
1716          * on on such CPUs now.
1717          */
1718         ppro_reenable_apic();
1719 #endif
1720
1721         /* Probe the CPU's in the system. */
1722         retval = best_enum->apic_probe_cpus();
1723         if (retval != 0)
1724                 printf("%s: Failed to probe CPUs: returned %d\n",
1725                     best_enum->apic_name, retval);
1726
1727 }
1728 SYSINIT(apic_init, SI_SUB_TUNABLES - 1, SI_ORDER_SECOND, apic_init, NULL);
1729
1730 /*
1731  * Setup the local APIC.  We have to do this prior to starting up the APs
1732  * in the SMP case.
1733  */
1734 static void
1735 apic_setup_local(void *dummy __unused)
1736 {
1737         int retval;
1738
1739         if (best_enum == NULL)
1740                 return;
1741
1742         /* Initialize the local APIC. */
1743         retval = best_enum->apic_setup_local();
1744         if (retval != 0)
1745                 printf("%s: Failed to setup the local APIC: returned %d\n",
1746                     best_enum->apic_name, retval);
1747 }
1748 SYSINIT(apic_setup_local, SI_SUB_CPU, SI_ORDER_SECOND, apic_setup_local, NULL);
1749
1750 /*
1751  * Setup the I/O APICs.
1752  */
1753 static void
1754 apic_setup_io(void *dummy __unused)
1755 {
1756         int retval;
1757
1758         if (best_enum == NULL)
1759                 return;
1760
1761         /*
1762          * Local APIC must be registered before other PICs and pseudo PICs
1763          * for proper suspend/resume order.
1764          */
1765         intr_register_pic(&lapic_pic);
1766
1767         retval = best_enum->apic_setup_io();
1768         if (retval != 0)
1769                 printf("%s: Failed to setup I/O APICs: returned %d\n",
1770                     best_enum->apic_name, retval);
1771
1772         /*
1773          * Finish setting up the local APIC on the BSP once we know
1774          * how to properly program the LINT pins.  In particular, this
1775          * enables the EOI suppression mode, if LAPIC support it and
1776          * user did not disabled the mode.
1777          */
1778         lapic_setup(1);
1779         if (bootverbose)
1780                 lapic_dump("BSP");
1781
1782         /* Enable the MSI "pic". */
1783         init_ops.msi_init();
1784 }
1785 SYSINIT(apic_setup_io, SI_SUB_INTR, SI_ORDER_THIRD, apic_setup_io, NULL);
1786
1787 #ifdef SMP
1788 /*
1789  * Inter Processor Interrupt functions.  The lapic_ipi_*() functions are
1790  * private to the MD code.  The public interface for the rest of the
1791  * kernel is defined in mp_machdep.c.
1792  */
1793
1794 /*
1795  * Wait delay microseconds for IPI to be sent.  If delay is -1, we
1796  * wait forever.
1797  */
1798 static int
1799 native_lapic_ipi_wait(int delay)
1800 {
1801         uint64_t rx;
1802
1803         /* LAPIC_ICR.APIC_DELSTAT_MASK is undefined in x2APIC mode */
1804         if (x2apic_mode)
1805                 return (1);
1806
1807         for (rx = 0; delay == -1 || rx < lapic_ipi_wait_mult * delay; rx++) {
1808                 if ((lapic_read_icr_lo() & APIC_DELSTAT_MASK) ==
1809                     APIC_DELSTAT_IDLE)
1810                         return (1);
1811                 ia32_pause();
1812         }
1813         return (0);
1814 }
1815
1816 static void
1817 native_lapic_ipi_raw(register_t icrlo, u_int dest)
1818 {
1819         uint64_t icr;
1820         uint32_t vhi, vlo;
1821         register_t saveintr;
1822
1823         /* XXX: Need more sanity checking of icrlo? */
1824         KASSERT(x2apic_mode || lapic_map != NULL,
1825             ("%s called too early", __func__));
1826         KASSERT(x2apic_mode ||
1827             (dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0,
1828             ("%s: invalid dest field", __func__));
1829         KASSERT((icrlo & APIC_ICRLO_RESV_MASK) == 0,
1830             ("%s: reserved bits set in ICR LO register", __func__));
1831
1832         /* Set destination in ICR HI register if it is being used. */
1833         if (!x2apic_mode) {
1834                 saveintr = intr_disable();
1835                 icr = lapic_read_icr();
1836         }
1837
1838         if ((icrlo & APIC_DEST_MASK) == APIC_DEST_DESTFLD) {
1839                 if (x2apic_mode) {
1840                         vhi = dest;
1841                 } else {
1842                         vhi = icr >> 32;
1843                         vhi &= ~APIC_ID_MASK;
1844                         vhi |= dest << APIC_ID_SHIFT;
1845                 }
1846         } else {
1847                 vhi = 0;
1848         }
1849
1850         /* Program the contents of the IPI and dispatch it. */
1851         if (x2apic_mode) {
1852                 vlo = icrlo;
1853         } else {
1854                 vlo = icr;
1855                 vlo &= APIC_ICRLO_RESV_MASK;
1856                 vlo |= icrlo;
1857         }
1858         lapic_write_icr(vhi, vlo);
1859         if (!x2apic_mode)
1860                 intr_restore(saveintr);
1861 }
1862
1863 #define BEFORE_SPIN     50000
1864 #ifdef DETECT_DEADLOCK
1865 #define AFTER_SPIN      50
1866 #endif
1867
1868 static void
1869 native_lapic_ipi_vectored(u_int vector, int dest)
1870 {
1871         register_t icrlo, destfield;
1872
1873         KASSERT((vector & ~APIC_VECTOR_MASK) == 0,
1874             ("%s: invalid vector %d", __func__, vector));
1875
1876         icrlo = APIC_DESTMODE_PHY | APIC_TRIGMOD_EDGE | APIC_LEVEL_ASSERT;
1877
1878         /*
1879          * NMI IPIs are just fake vectors used to send a NMI.  Use special rules
1880          * regarding NMIs if passed, otherwise specify the vector.
1881          */
1882         if (vector >= IPI_NMI_FIRST)
1883                 icrlo |= APIC_DELMODE_NMI;
1884         else
1885                 icrlo |= vector | APIC_DELMODE_FIXED;
1886         destfield = 0;
1887         switch (dest) {
1888         case APIC_IPI_DEST_SELF:
1889                 icrlo |= APIC_DEST_SELF;
1890                 break;
1891         case APIC_IPI_DEST_ALL:
1892                 icrlo |= APIC_DEST_ALLISELF;
1893                 break;
1894         case APIC_IPI_DEST_OTHERS:
1895                 icrlo |= APIC_DEST_ALLESELF;
1896                 break;
1897         default:
1898                 KASSERT(x2apic_mode ||
1899                     (dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0,
1900                     ("%s: invalid destination 0x%x", __func__, dest));
1901                 destfield = dest;
1902         }
1903
1904         /* Wait for an earlier IPI to finish. */
1905         if (!lapic_ipi_wait(BEFORE_SPIN)) {
1906                 if (panicstr != NULL)
1907                         return;
1908                 else
1909                         panic("APIC: Previous IPI is stuck");
1910         }
1911
1912         lapic_ipi_raw(icrlo, destfield);
1913
1914 #ifdef DETECT_DEADLOCK
1915         /* Wait for IPI to be delivered. */
1916         if (!lapic_ipi_wait(AFTER_SPIN)) {
1917 #ifdef needsattention
1918                 /*
1919                  * XXX FIXME:
1920                  *
1921                  * The above function waits for the message to actually be
1922                  * delivered.  It breaks out after an arbitrary timeout
1923                  * since the message should eventually be delivered (at
1924                  * least in theory) and that if it wasn't we would catch
1925                  * the failure with the check above when the next IPI is
1926                  * sent.
1927                  *
1928                  * We could skip this wait entirely, EXCEPT it probably
1929                  * protects us from other routines that assume that the
1930                  * message was delivered and acted upon when this function
1931                  * returns.
1932                  */
1933                 printf("APIC: IPI might be stuck\n");
1934 #else /* !needsattention */
1935                 /* Wait until mesage is sent without a timeout. */
1936                 while (lapic_read_icr_lo() & APIC_DELSTAT_PEND)
1937                         ia32_pause();
1938 #endif /* needsattention */
1939         }
1940 #endif /* DETECT_DEADLOCK */
1941 }
1942
1943 #endif /* SMP */
1944
1945 /*
1946  * Since the IDT is shared by all CPUs the IPI slot update needs to be globally
1947  * visible.
1948  *
1949  * Consider the case where an IPI is generated immediately after allocation:
1950  *     vector = lapic_ipi_alloc(ipifunc);
1951  *     ipi_selected(other_cpus, vector);
1952  *
1953  * In xAPIC mode a write to ICR_LO has serializing semantics because the
1954  * APIC page is mapped as an uncached region. In x2APIC mode there is an
1955  * explicit 'mfence' before the ICR MSR is written. Therefore in both cases
1956  * the IDT slot update is globally visible before the IPI is delivered.
1957  */
1958 static int
1959 native_lapic_ipi_alloc(inthand_t *ipifunc)
1960 {
1961         struct gate_descriptor *ip;
1962         long func;
1963         int idx, vector;
1964
1965         KASSERT(ipifunc != &IDTVEC(rsvd), ("invalid ipifunc %p", ipifunc));
1966
1967         vector = -1;
1968         mtx_lock_spin(&icu_lock);
1969         for (idx = IPI_DYN_FIRST; idx <= IPI_DYN_LAST; idx++) {
1970                 ip = &idt[idx];
1971                 func = (ip->gd_hioffset << 16) | ip->gd_looffset;
1972                 if (func == (uintptr_t)&IDTVEC(rsvd)) {
1973                         vector = idx;
1974                         setidt(vector, ipifunc, SDT_APIC, SEL_KPL, GSEL_APIC);
1975                         break;
1976                 }
1977         }
1978         mtx_unlock_spin(&icu_lock);
1979         return (vector);
1980 }
1981
1982 static void
1983 native_lapic_ipi_free(int vector)
1984 {
1985         struct gate_descriptor *ip;
1986         long func;
1987
1988         KASSERT(vector >= IPI_DYN_FIRST && vector <= IPI_DYN_LAST,
1989             ("%s: invalid vector %d", __func__, vector));
1990
1991         mtx_lock_spin(&icu_lock);
1992         ip = &idt[vector];
1993         func = (ip->gd_hioffset << 16) | ip->gd_looffset;
1994         KASSERT(func != (uintptr_t)&IDTVEC(rsvd),
1995             ("invalid idtfunc %#lx", func));
1996         setidt(vector, &IDTVEC(rsvd), SDT_APICT, SEL_KPL, GSEL_APIC);
1997         mtx_unlock_spin(&icu_lock);
1998 }