sys/x86/x86/local_apic.c

   1 /*-
   2  * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org>
   3  * Copyright (c) 1996, by Steve Passe
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. The name of the developer may NOT be used to endorse or promote products
  12  *    derived from this software without specific prior written permission.
  13  * 3. Neither the name of the author nor the names of any co-contributors
  14  *    may be used to endorse or promote products derived from this software
  15  *    without specific prior written permission.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 /*
  31  * Local APIC support on Pentium and later processors.
  32  */
  33
  34 #include <sys/cdefs.h>
  35 __FBSDID("$FreeBSD$");
  36
  37 #include "opt_atpic.h"
  38 #include "opt_hwpmc_hooks.h"
  39
  40 #include "opt_ddb.h"
  41
  42 #include <sys/param.h>
  43 #include <sys/systm.h>
  44 #include <sys/bus.h>
  45 #include <sys/kernel.h>
  46 #include <sys/lock.h>
  47 #include <sys/mutex.h>
  48 #include <sys/pcpu.h>
  49 #include <sys/proc.h>
  50 #include <sys/sched.h>
  51 #include <sys/smp.h>
  52 #include <sys/sysctl.h>
  53 #include <sys/timeet.h>
  54
  55 #include <vm/vm.h>
  56 #include <vm/pmap.h>
  57
  58 #include <x86/apicreg.h>
  59 #include <machine/clock.h>
  60 #include <machine/cpufunc.h>
  61 #include <machine/cputypes.h>
  62 #include <machine/frame.h>
  63 #include <machine/intr_machdep.h>
  64 #include <x86/apicvar.h>
  65 #include <x86/mca.h>
  66 #include <machine/md_var.h>
  67 #include <machine/smp.h>
  68 #include <machine/specialreg.h>
  69 #include <x86/init.h>
  70
  71 #ifdef DDB
  72 #include <sys/interrupt.h>
  73 #include <ddb/ddb.h>
  74 #endif
  75
  76 #ifdef __amd64__
  77 #define SDT_APIC        SDT_SYSIGT
  78 #define SDT_APICT       SDT_SYSIGT
  79 #define GSEL_APIC       0
  80 #else
  81 #define SDT_APIC        SDT_SYS386IGT
  82 #define SDT_APICT       SDT_SYS386TGT
  83 #define GSEL_APIC       GSEL(GCODE_SEL, SEL_KPL)
  84 #endif
  85
  86 /* Sanity checks on IDT vectors. */
  87 CTASSERT(APIC_IO_INTS + APIC_NUM_IOINTS == APIC_TIMER_INT);
  88 CTASSERT(APIC_TIMER_INT < APIC_LOCAL_INTS);
  89 CTASSERT(APIC_LOCAL_INTS == 240);
  90 CTASSERT(IPI_STOP < APIC_SPURIOUS_INT);
  91
  92 /* Magic IRQ values for the timer and syscalls. */
  93 #define IRQ_TIMER       (NUM_IO_INTS + 1)
  94 #define IRQ_SYSCALL     (NUM_IO_INTS + 2)
  95 #define IRQ_DTRACE_RET  (NUM_IO_INTS + 3)
  96 #define IRQ_EVTCHN      (NUM_IO_INTS + 4)
  97
  98 enum lat_timer_mode {
  99         LAT_MODE_UNDEF =        0,
 100         LAT_MODE_PERIODIC =     1,
 101         LAT_MODE_ONESHOT =      2,
 102         LAT_MODE_DEADLINE =     3,
 103 };
 104
 105 /*
 106  * Support for local APICs.  Local APICs manage interrupts on each
 107  * individual processor as opposed to I/O APICs which receive interrupts
 108  * from I/O devices and then forward them on to the local APICs.
 109  *
 110  * Local APICs can also send interrupts to each other thus providing the
 111  * mechanism for IPIs.
 112  */
 113
 114 struct lvt {
 115         u_int lvt_edgetrigger:1;
 116         u_int lvt_activehi:1;
 117         u_int lvt_masked:1;
 118         u_int lvt_active:1;
 119         u_int lvt_mode:16;
 120         u_int lvt_vector:8;
 121 };
 122
 123 struct lapic {
 124         struct lvt la_lvts[APIC_LVT_MAX + 1];
 125         u_int la_id:8;
 126         u_int la_cluster:4;
 127         u_int la_cluster_id:2;
 128         u_int la_present:1;
 129         u_long *la_timer_count;
 130         uint64_t la_timer_period;
 131         enum lat_timer_mode la_timer_mode;
 132         uint32_t lvt_timer_base;
 133         uint32_t lvt_timer_last;
 134         /* Include IDT_SYSCALL to make indexing easier. */
 135         int la_ioint_irqs[APIC_NUM_IOINTS + 1];
 136 } static lapics[MAX_APIC_ID + 1];
 137
 138 /* Global defaults for local APIC LVT entries. */
 139 static struct lvt lvts[APIC_LVT_MAX + 1] = {
 140         { 1, 1, 1, 1, APIC_LVT_DM_EXTINT, 0 },  /* LINT0: masked ExtINT */
 141         { 1, 1, 0, 1, APIC_LVT_DM_NMI, 0 },     /* LINT1: NMI */
 142         { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_TIMER_INT },      /* Timer */
 143         { 1, 1, 0, 1, APIC_LVT_DM_FIXED, APIC_ERROR_INT },      /* Error */
 144         { 1, 1, 1, 1, APIC_LVT_DM_NMI, 0 },     /* PMC */
 145         { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_THERMAL_INT },    /* Thermal */
 146         { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_CMC_INT },        /* CMCI */
 147 };
 148
 149 static inthand_t *ioint_handlers[] = {
 150         NULL,                   /* 0 - 31 */
 151         IDTVEC(apic_isr1),      /* 32 - 63 */
 152         IDTVEC(apic_isr2),      /* 64 - 95 */
 153         IDTVEC(apic_isr3),      /* 96 - 127 */
 154         IDTVEC(apic_isr4),      /* 128 - 159 */
 155         IDTVEC(apic_isr5),      /* 160 - 191 */
 156         IDTVEC(apic_isr6),      /* 192 - 223 */
 157         IDTVEC(apic_isr7),      /* 224 - 255 */
 158 };
 159
 160
 161 static u_int32_t lapic_timer_divisors[] = {
 162         APIC_TDCR_1, APIC_TDCR_2, APIC_TDCR_4, APIC_TDCR_8, APIC_TDCR_16,
 163         APIC_TDCR_32, APIC_TDCR_64, APIC_TDCR_128
 164 };
 165
 166 extern inthand_t IDTVEC(rsvd);
 167
 168 volatile char *lapic_map;
 169 vm_paddr_t lapic_paddr;
 170 int x2apic_mode;
 171 int lapic_eoi_suppression;
 172 static int lapic_timer_tsc_deadline;
 173 static u_long lapic_timer_divisor, count_freq;
 174 static struct eventtimer lapic_et;
 175 #ifdef SMP
 176 static uint64_t lapic_ipi_wait_mult;
 177 #endif
 178
 179 SYSCTL_NODE(_hw, OID_AUTO, apic, CTLFLAG_RD, 0, "APIC options");
 180 SYSCTL_INT(_hw_apic, OID_AUTO, x2apic_mode, CTLFLAG_RD, &x2apic_mode, 0, "");
 181 SYSCTL_INT(_hw_apic, OID_AUTO, eoi_suppression, CTLFLAG_RD,
 182     &lapic_eoi_suppression, 0, "");
 183 SYSCTL_INT(_hw_apic, OID_AUTO, timer_tsc_deadline, CTLFLAG_RD,
 184     &lapic_timer_tsc_deadline, 0, "");
 185
 186 static uint32_t
 187 lapic_read32(enum LAPIC_REGISTERS reg)
 188 {
 189         uint32_t res;
 190
 191         if (x2apic_mode) {
 192                 res = rdmsr32(MSR_APIC_000 + reg);
 193         } else {
 194                 res = *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL);
 195         }
 196         return (res);
 197 }
 198
 199 static void
 200 lapic_write32(enum LAPIC_REGISTERS reg, uint32_t val)
 201 {
 202
 203         if (x2apic_mode) {
 204                 mfence();
 205                 wrmsr(MSR_APIC_000 + reg, val);
 206         } else {
 207                 *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL) = val;
 208         }
 209 }
 210
 211 static void
 212 lapic_write32_nofence(enum LAPIC_REGISTERS reg, uint32_t val)
 213 {
 214
 215         if (x2apic_mode) {
 216                 wrmsr(MSR_APIC_000 + reg, val);
 217         } else {
 218                 *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL) = val;
 219         }
 220 }
 221
 222 #ifdef SMP
 223 static uint64_t
 224 lapic_read_icr(void)
 225 {
 226         uint64_t v;
 227         uint32_t vhi, vlo;
 228
 229         if (x2apic_mode) {
 230                 v = rdmsr(MSR_APIC_000 + LAPIC_ICR_LO);
 231         } else {
 232                 vhi = lapic_read32(LAPIC_ICR_HI);
 233                 vlo = lapic_read32(LAPIC_ICR_LO);
 234                 v = ((uint64_t)vhi << 32) | vlo;
 235         }
 236         return (v);
 237 }
 238
 239 static uint64_t
 240 lapic_read_icr_lo(void)
 241 {
 242
 243         return (lapic_read32(LAPIC_ICR_LO));
 244 }
 245
 246 static void
 247 lapic_write_icr(uint32_t vhi, uint32_t vlo)
 248 {
 249         uint64_t v;
 250
 251         if (x2apic_mode) {
 252                 v = ((uint64_t)vhi << 32) | vlo;
 253                 mfence();
 254                 wrmsr(MSR_APIC_000 + LAPIC_ICR_LO, v);
 255         } else {
 256                 lapic_write32(LAPIC_ICR_HI, vhi);
 257                 lapic_write32(LAPIC_ICR_LO, vlo);
 258         }
 259 }
 260 #endif /* SMP */
 261
 262 static void
 263 native_lapic_enable_x2apic(void)
 264 {
 265         uint64_t apic_base;
 266
 267         apic_base = rdmsr(MSR_APICBASE);
 268         apic_base |= APICBASE_X2APIC | APICBASE_ENABLED;
 269         wrmsr(MSR_APICBASE, apic_base);
 270 }
 271
 272 static bool
 273 native_lapic_is_x2apic(void)
 274 {
 275         uint64_t apic_base;
 276
 277         apic_base = rdmsr(MSR_APICBASE);
 278         return ((apic_base & (APICBASE_X2APIC | APICBASE_ENABLED)) ==
 279             (APICBASE_X2APIC | APICBASE_ENABLED));
 280 }
 281
 282 static void     lapic_enable(void);
 283 static void     lapic_resume(struct pic *pic, bool suspend_cancelled);
 284 static void     lapic_timer_oneshot(struct lapic *);
 285 static void     lapic_timer_oneshot_nointr(struct lapic *, uint32_t);
 286 static void     lapic_timer_periodic(struct lapic *);
 287 static void     lapic_timer_deadline(struct lapic *);
 288 static void     lapic_timer_stop(struct lapic *);
 289 static void     lapic_timer_set_divisor(u_int divisor);
 290 static uint32_t lvt_mode(struct lapic *la, u_int pin, uint32_t value);
 291 static int      lapic_et_start(struct eventtimer *et,
 292                     sbintime_t first, sbintime_t period);
 293 static int      lapic_et_stop(struct eventtimer *et);
 294 static u_int    apic_idt_to_irq(u_int apic_id, u_int vector);
 295 static void     lapic_set_tpr(u_int vector);
 296
 297 struct pic lapic_pic = { .pic_resume = lapic_resume };
 298
 299 /* Forward declarations for apic_ops */
 300 static void     native_lapic_create(u_int apic_id, int boot_cpu);
 301 static void     native_lapic_init(vm_paddr_t addr);
 302 static void     native_lapic_xapic_mode(void);
 303 static void     native_lapic_setup(int boot);
 304 static void     native_lapic_dump(const char *str);
 305 static void     native_lapic_disable(void);
 306 static void     native_lapic_eoi(void);
 307 static int      native_lapic_id(void);
 308 static int      native_lapic_intr_pending(u_int vector);
 309 static u_int    native_apic_cpuid(u_int apic_id);
 310 static u_int    native_apic_alloc_vector(u_int apic_id, u_int irq);
 311 static u_int    native_apic_alloc_vectors(u_int apic_id, u_int *irqs,
 312                     u_int count, u_int align);
 313 static void     native_apic_disable_vector(u_int apic_id, u_int vector);
 314 static void     native_apic_enable_vector(u_int apic_id, u_int vector);
 315 static void     native_apic_free_vector(u_int apic_id, u_int vector, u_int irq);
 316 static void     native_lapic_set_logical_id(u_int apic_id, u_int cluster,
 317                     u_int cluster_id);
 318 static int      native_lapic_enable_pmc(void);
 319 static void     native_lapic_disable_pmc(void);
 320 static void     native_lapic_reenable_pmc(void);
 321 static void     native_lapic_enable_cmc(void);
 322 static int      native_lapic_set_lvt_mask(u_int apic_id, u_int lvt,
 323                     u_char masked);
 324 static int      native_lapic_set_lvt_mode(u_int apic_id, u_int lvt,
 325                     uint32_t mode);
 326 static int      native_lapic_set_lvt_polarity(u_int apic_id, u_int lvt,
 327                     enum intr_polarity pol);
 328 static int      native_lapic_set_lvt_triggermode(u_int apic_id, u_int lvt,
 329                     enum intr_trigger trigger);
 330 #ifdef SMP
 331 static void     native_lapic_ipi_raw(register_t icrlo, u_int dest);
 332 static void     native_lapic_ipi_vectored(u_int vector, int dest);
 333 static int      native_lapic_ipi_wait(int delay);
 334 #endif /* SMP */
 335 static int      native_lapic_ipi_alloc(inthand_t *ipifunc);
 336 static void     native_lapic_ipi_free(int vector);
 337
 338 struct apic_ops apic_ops = {
 339         .create                 = native_lapic_create,
 340         .init                   = native_lapic_init,
 341         .xapic_mode             = native_lapic_xapic_mode,
 342         .is_x2apic              = native_lapic_is_x2apic,
 343         .setup                  = native_lapic_setup,
 344         .dump                   = native_lapic_dump,
 345         .disable                = native_lapic_disable,
 346         .eoi                    = native_lapic_eoi,
 347         .id                     = native_lapic_id,
 348         .intr_pending           = native_lapic_intr_pending,
 349         .set_logical_id         = native_lapic_set_logical_id,
 350         .cpuid                  = native_apic_cpuid,
 351         .alloc_vector           = native_apic_alloc_vector,
 352         .alloc_vectors          = native_apic_alloc_vectors,
 353         .enable_vector          = native_apic_enable_vector,
 354         .disable_vector         = native_apic_disable_vector,
 355         .free_vector            = native_apic_free_vector,
 356         .enable_pmc             = native_lapic_enable_pmc,
 357         .disable_pmc            = native_lapic_disable_pmc,
 358         .reenable_pmc           = native_lapic_reenable_pmc,
 359         .enable_cmc             = native_lapic_enable_cmc,
 360 #ifdef SMP
 361         .ipi_raw                = native_lapic_ipi_raw,
 362         .ipi_vectored           = native_lapic_ipi_vectored,
 363         .ipi_wait               = native_lapic_ipi_wait,
 364 #endif
 365         .ipi_alloc              = native_lapic_ipi_alloc,
 366         .ipi_free               = native_lapic_ipi_free,
 367         .set_lvt_mask           = native_lapic_set_lvt_mask,
 368         .set_lvt_mode           = native_lapic_set_lvt_mode,
 369         .set_lvt_polarity       = native_lapic_set_lvt_polarity,
 370         .set_lvt_triggermode    = native_lapic_set_lvt_triggermode,
 371 };
 372
 373 static uint32_t
 374 lvt_mode(struct lapic *la, u_int pin, uint32_t value)
 375 {
 376         struct lvt *lvt;
 377
 378         KASSERT(pin <= APIC_LVT_MAX, ("%s: pin %u out of range", __func__, pin));
 379         if (la->la_lvts[pin].lvt_active)
 380                 lvt = &la->la_lvts[pin];
 381         else
 382                 lvt = &lvts[pin];
 383
 384         value &= ~(APIC_LVT_M | APIC_LVT_TM | APIC_LVT_IIPP | APIC_LVT_DM |
 385             APIC_LVT_VECTOR);
 386         if (lvt->lvt_edgetrigger == 0)
 387                 value |= APIC_LVT_TM;
 388         if (lvt->lvt_activehi == 0)
 389                 value |= APIC_LVT_IIPP_INTALO;
 390         if (lvt->lvt_masked)
 391                 value |= APIC_LVT_M;
 392         value |= lvt->lvt_mode;
 393         switch (lvt->lvt_mode) {
 394         case APIC_LVT_DM_NMI:
 395         case APIC_LVT_DM_SMI:
 396         case APIC_LVT_DM_INIT:
 397         case APIC_LVT_DM_EXTINT:
 398                 if (!lvt->lvt_edgetrigger && bootverbose) {
 399                         printf("lapic%u: Forcing LINT%u to edge trigger\n",
 400                             la->la_id, pin);
 401                         value |= APIC_LVT_TM;
 402                 }
 403                 /* Use a vector of 0. */
 404                 break;
 405         case APIC_LVT_DM_FIXED:
 406                 value |= lvt->lvt_vector;
 407                 break;
 408         default:
 409                 panic("bad APIC LVT delivery mode: %#x\n", value);
 410         }
 411         return (value);
 412 }
 413
 414 /*
 415  * Map the local APIC and setup necessary interrupt vectors.
 416  */
 417 static void
 418 native_lapic_init(vm_paddr_t addr)
 419 {
 420 #ifdef SMP
 421         uint64_t r, r1, r2, rx;
 422 #endif
 423         uint32_t ver;
 424         u_int regs[4];
 425         int i, arat;
 426
 427         /*
 428          * Enable x2APIC mode if possible. Map the local APIC
 429          * registers page.
 430          *
 431          * Keep the LAPIC registers page mapped uncached for x2APIC
 432          * mode too, to have direct map page attribute set to
 433          * uncached.  This is needed to work around CPU errata present
 434          * on all Intel processors.
 435          */
 436         KASSERT(trunc_page(addr) == addr,
 437             ("local APIC not aligned on a page boundary"));
 438         lapic_paddr = addr;
 439         lapic_map = pmap_mapdev(addr, PAGE_SIZE);
 440         if (x2apic_mode) {
 441                 native_lapic_enable_x2apic();
 442                 lapic_map = NULL;
 443         }
 444
 445         /* Setup the spurious interrupt handler. */
 446         setidt(APIC_SPURIOUS_INT, IDTVEC(spuriousint), SDT_APIC, SEL_KPL,
 447             GSEL_APIC);
 448
 449         /* Perform basic initialization of the BSP's local APIC. */
 450         lapic_enable();
 451
 452         /* Set BSP's per-CPU local APIC ID. */
 453         PCPU_SET(apic_id, lapic_id());
 454
 455         /* Local APIC timer interrupt. */
 456         setidt(APIC_TIMER_INT, IDTVEC(timerint), SDT_APIC, SEL_KPL, GSEL_APIC);
 457
 458         /* Local APIC error interrupt. */
 459         setidt(APIC_ERROR_INT, IDTVEC(errorint), SDT_APIC, SEL_KPL, GSEL_APIC);
 460
 461         /* XXX: Thermal interrupt */
 462
 463         /* Local APIC CMCI. */
 464         setidt(APIC_CMC_INT, IDTVEC(cmcint), SDT_APICT, SEL_KPL, GSEL_APIC);
 465
 466         if ((resource_int_value("apic", 0, "clock", &i) != 0 || i != 0)) {
 467                 arat = 0;
 468                 /* Intel CPUID 0x06 EAX[2] set if APIC timer runs in C3. */
 469                 if (cpu_vendor_id == CPU_VENDOR_INTEL && cpu_high >= 6) {
 470                         do_cpuid(0x06, regs);
 471                         if ((regs[0] & CPUTPM1_ARAT) != 0)
 472                                 arat = 1;
 473                 }
 474                 bzero(&lapic_et, sizeof(lapic_et));
 475                 lapic_et.et_name = "LAPIC";
 476                 lapic_et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT |
 477                     ET_FLAGS_PERCPU;
 478                 lapic_et.et_quality = 600;
 479                 if (!arat) {
 480                         lapic_et.et_flags |= ET_FLAGS_C3STOP;
 481                         lapic_et.et_quality = 100;
 482                 }
 483                 if ((cpu_feature & CPUID_TSC) != 0 &&
 484                     (cpu_feature2 & CPUID2_TSCDLT) != 0 &&
 485                     tsc_is_invariant && tsc_freq != 0) {
 486                         lapic_timer_tsc_deadline = 1;
 487                         TUNABLE_INT_FETCH("hw.lapic_tsc_deadline",
 488                             &lapic_timer_tsc_deadline);
 489                 }
 490
 491                 lapic_et.et_frequency = 0;
 492                 /* We don't know frequency yet, so trying to guess. */
 493                 lapic_et.et_min_period = 0x00001000LL;
 494                 lapic_et.et_max_period = SBT_1S;
 495                 lapic_et.et_start = lapic_et_start;
 496                 lapic_et.et_stop = lapic_et_stop;
 497                 lapic_et.et_priv = NULL;
 498                 et_register(&lapic_et);
 499         }
 500
 501         /*
 502          * Set lapic_eoi_suppression after lapic_enable(), to not
 503          * enable suppression in the hardware prematurely.  Note that
 504          * we by default enable suppression even when system only has
 505          * one IO-APIC, since EOI is broadcasted to all APIC agents,
 506          * including CPUs, otherwise.
 507          *
 508          * It seems that at least some KVM versions report
 509          * EOI_SUPPRESSION bit, but auto-EOI does not work.
 510          */
 511         ver = lapic_read32(LAPIC_VERSION);
 512         if ((ver & APIC_VER_EOI_SUPPRESSION) != 0) {
 513                 lapic_eoi_suppression = 1;
 514                 if (vm_guest == VM_GUEST_KVM) {
 515                         if (bootverbose)
 516                                 printf(
 517                        "KVM -- disabling lapic eoi suppression\n");
 518                         lapic_eoi_suppression = 0;
 519                 }
 520                 TUNABLE_INT_FETCH("hw.lapic_eoi_suppression",
 521                     &lapic_eoi_suppression);
 522         }
 523
 524 #ifdef SMP
 525 #define LOOPS   100000
 526         /*
 527          * Calibrate the busy loop waiting for IPI ack in xAPIC mode.
 528          * lapic_ipi_wait_mult contains the number of iterations which
 529          * approximately delay execution for 1 microsecond (the
 530          * argument to native_lapic_ipi_wait() is in microseconds).
 531          *
 532          * We assume that TSC is present and already measured.
 533          * Possible TSC frequency jumps are irrelevant to the
 534          * calibration loop below, the CPU clock management code is
 535          * not yet started, and we do not enter sleep states.
 536          */
 537         KASSERT((cpu_feature & CPUID_TSC) != 0 && tsc_freq != 0,
 538             ("TSC not initialized"));
 539         if (!x2apic_mode) {
 540                 r = rdtsc();
 541                 for (rx = 0; rx < LOOPS; rx++) {
 542                         (void)lapic_read_icr_lo();
 543                         ia32_pause();
 544                 }
 545                 r = rdtsc() - r;
 546                 r1 = tsc_freq * LOOPS;
 547                 r2 = r * 1000000;
 548                 lapic_ipi_wait_mult = r1 >= r2 ? r1 / r2 : 1;
 549                 if (bootverbose) {
 550                         printf("LAPIC: ipi_wait() us multiplier %ju (r %ju "
 551                             "tsc %ju)\n", (uintmax_t)lapic_ipi_wait_mult,
 552                             (uintmax_t)r, (uintmax_t)tsc_freq);
 553                 }
 554         }
 555 #undef LOOPS
 556 #endif /* SMP */
 557 }
 558
 559 /*
 560  * Create a local APIC instance.
 561  */
 562 static void
 563 native_lapic_create(u_int apic_id, int boot_cpu)
 564 {
 565         int i;
 566
 567         if (apic_id > MAX_APIC_ID) {
 568                 printf("APIC: Ignoring local APIC with ID %d\n", apic_id);
 569                 if (boot_cpu)
 570                         panic("Can't ignore BSP");
 571                 return;
 572         }
 573         KASSERT(!lapics[apic_id].la_present, ("duplicate local APIC %u",
 574             apic_id));
 575
 576         /*
 577          * Assume no local LVT overrides and a cluster of 0 and
 578          * intra-cluster ID of 0.
 579          */
 580         lapics[apic_id].la_present = 1;
 581         lapics[apic_id].la_id = apic_id;
 582         for (i = 0; i <= APIC_LVT_MAX; i++) {
 583                 lapics[apic_id].la_lvts[i] = lvts[i];
 584                 lapics[apic_id].la_lvts[i].lvt_active = 0;
 585         }
 586         for (i = 0; i <= APIC_NUM_IOINTS; i++)
 587             lapics[apic_id].la_ioint_irqs[i] = -1;
 588         lapics[apic_id].la_ioint_irqs[IDT_SYSCALL - APIC_IO_INTS] = IRQ_SYSCALL;
 589         lapics[apic_id].la_ioint_irqs[APIC_TIMER_INT - APIC_IO_INTS] =
 590             IRQ_TIMER;
 591 #ifdef KDTRACE_HOOKS
 592         lapics[apic_id].la_ioint_irqs[IDT_DTRACE_RET - APIC_IO_INTS] =
 593             IRQ_DTRACE_RET;
 594 #endif
 595 #ifdef XENHVM
 596         lapics[apic_id].la_ioint_irqs[IDT_EVTCHN - APIC_IO_INTS] = IRQ_EVTCHN;
 597 #endif
 598
 599
 600 #ifdef SMP
 601         cpu_add(apic_id, boot_cpu);
 602 #endif
 603 }
 604
 605 /*
 606  * Dump contents of local APIC registers
 607  */
 608 static void
 609 native_lapic_dump(const char* str)
 610 {
 611         uint32_t maxlvt;
 612
 613         maxlvt = (lapic_read32(LAPIC_VERSION) & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
 614         printf("cpu%d %s:\n", PCPU_GET(cpuid), str);
 615         printf("     ID: 0x%08x   VER: 0x%08x LDR: 0x%08x DFR: 0x%08x",
 616             lapic_read32(LAPIC_ID), lapic_read32(LAPIC_VERSION),
 617             lapic_read32(LAPIC_LDR), x2apic_mode ? 0 : lapic_read32(LAPIC_DFR));
 618         if ((cpu_feature2 & CPUID2_X2APIC) != 0)
 619                 printf(" x2APIC: %d", x2apic_mode);
 620         printf("\n  lint0: 0x%08x lint1: 0x%08x TPR: 0x%08x SVR: 0x%08x\n",
 621             lapic_read32(LAPIC_LVT_LINT0), lapic_read32(LAPIC_LVT_LINT1),
 622             lapic_read32(LAPIC_TPR), lapic_read32(LAPIC_SVR));
 623         printf("  timer: 0x%08x therm: 0x%08x err: 0x%08x",
 624             lapic_read32(LAPIC_LVT_TIMER), lapic_read32(LAPIC_LVT_THERMAL),
 625             lapic_read32(LAPIC_LVT_ERROR));
 626         if (maxlvt >= APIC_LVT_PMC)
 627                 printf(" pmc: 0x%08x", lapic_read32(LAPIC_LVT_PCINT));
 628         printf("\n");
 629         if (maxlvt >= APIC_LVT_CMCI)
 630                 printf("   cmci: 0x%08x\n", lapic_read32(LAPIC_LVT_CMCI));
 631 }
 632
 633 static void
 634 native_lapic_xapic_mode(void)
 635 {
 636         register_t saveintr;
 637
 638         saveintr = intr_disable();
 639         if (x2apic_mode)
 640                 native_lapic_enable_x2apic();
 641         intr_restore(saveintr);
 642 }
 643
 644 static void
 645 native_lapic_setup(int boot)
 646 {
 647         struct lapic *la;
 648         uint32_t maxlvt;
 649         register_t saveintr;
 650         char buf[MAXCOMLEN + 1];
 651
 652         saveintr = intr_disable();
 653
 654         la = &lapics[lapic_id()];
 655         KASSERT(la->la_present, ("missing APIC structure"));
 656         maxlvt = (lapic_read32(LAPIC_VERSION) & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
 657
 658         /* Initialize the TPR to allow all interrupts. */
 659         lapic_set_tpr(0);
 660
 661         /* Setup spurious vector and enable the local APIC. */
 662         lapic_enable();
 663
 664         /* Program LINT[01] LVT entries. */
 665         lapic_write32(LAPIC_LVT_LINT0, lvt_mode(la, APIC_LVT_LINT0,
 666             lapic_read32(LAPIC_LVT_LINT0)));
 667         lapic_write32(LAPIC_LVT_LINT1, lvt_mode(la, APIC_LVT_LINT1,
 668             lapic_read32(LAPIC_LVT_LINT1)));
 669
 670         /* Program the PMC LVT entry if present. */
 671         if (maxlvt >= APIC_LVT_PMC) {
 672                 lapic_write32(LAPIC_LVT_PCINT, lvt_mode(la, APIC_LVT_PMC,
 673                     LAPIC_LVT_PCINT));
 674         }
 675
 676         /* Program timer LVT and setup handler. */
 677         la->lvt_timer_base = lvt_mode(la, APIC_LVT_TIMER,
 678             lapic_read32(LAPIC_LVT_TIMER));
 679         la->lvt_timer_last = la->lvt_timer_base;
 680         lapic_write32(LAPIC_LVT_TIMER, la->lvt_timer_base);
 681         if (boot) {
 682                 snprintf(buf, sizeof(buf), "cpu%d:timer", PCPU_GET(cpuid));
 683                 intrcnt_add(buf, &la->la_timer_count);
 684         }
 685
 686         /* Setup the timer if configured. */
 687         if (la->la_timer_mode != LAT_MODE_UNDEF) {
 688                 KASSERT(la->la_timer_period != 0, ("lapic%u: zero divisor",
 689                     lapic_id()));
 690                 switch (la->la_timer_mode) {
 691                 case LAT_MODE_PERIODIC:
 692                         lapic_timer_set_divisor(lapic_timer_divisor);
 693                         lapic_timer_periodic(la);
 694                         break;
 695                 case LAT_MODE_ONESHOT:
 696                         lapic_timer_set_divisor(lapic_timer_divisor);
 697                         lapic_timer_oneshot(la);
 698                         break;
 699                 case LAT_MODE_DEADLINE:
 700                         lapic_timer_deadline(la);
 701                         break;
 702                 default:
 703                         panic("corrupted la_timer_mode %p %d", la,
 704                             la->la_timer_mode);
 705                 }
 706         }
 707
 708         /* Program error LVT and clear any existing errors. */
 709         lapic_write32(LAPIC_LVT_ERROR, lvt_mode(la, APIC_LVT_ERROR,
 710             lapic_read32(LAPIC_LVT_ERROR)));
 711         lapic_write32(LAPIC_ESR, 0);
 712
 713         /* XXX: Thermal LVT */
 714
 715         /* Program the CMCI LVT entry if present. */
 716         if (maxlvt >= APIC_LVT_CMCI) {
 717                 lapic_write32(LAPIC_LVT_CMCI, lvt_mode(la, APIC_LVT_CMCI,
 718                     lapic_read32(LAPIC_LVT_CMCI)));
 719         }
 720
 721         intr_restore(saveintr);
 722 }
 723
 724 static void
 725 native_lapic_reenable_pmc(void)
 726 {
 727 #ifdef HWPMC_HOOKS
 728         uint32_t value;
 729
 730         value = lapic_read32(LAPIC_LVT_PCINT);
 731         value &= ~APIC_LVT_M;
 732         lapic_write32(LAPIC_LVT_PCINT, value);
 733 #endif
 734 }
 735
 736 #ifdef HWPMC_HOOKS
 737 static void
 738 lapic_update_pmc(void *dummy)
 739 {
 740         struct lapic *la;
 741
 742         la = &lapics[lapic_id()];
 743         lapic_write32(LAPIC_LVT_PCINT, lvt_mode(la, APIC_LVT_PMC,
 744             lapic_read32(LAPIC_LVT_PCINT)));
 745 }
 746 #endif
 747
 748 static int
 749 native_lapic_enable_pmc(void)
 750 {
 751 #ifdef HWPMC_HOOKS
 752         u_int32_t maxlvt;
 753
 754         /* Fail if the local APIC is not present. */
 755         if (!x2apic_mode && lapic_map == NULL)
 756                 return (0);
 757
 758         /* Fail if the PMC LVT is not present. */
 759         maxlvt = (lapic_read32(LAPIC_VERSION) & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
 760         if (maxlvt < APIC_LVT_PMC)
 761                 return (0);
 762
 763         lvts[APIC_LVT_PMC].lvt_masked = 0;
 764
 765 #ifdef EARLY_AP_STARTUP
 766         MPASS(mp_ncpus == 1 || smp_started);
 767         smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL);
 768 #else
 769 #ifdef SMP
 770         /*
 771          * If hwpmc was loaded at boot time then the APs may not be
 772          * started yet.  In that case, don't forward the request to
 773          * them as they will program the lvt when they start.
 774          */
 775         if (smp_started)
 776                 smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL);
 777         else
 778 #endif
 779                 lapic_update_pmc(NULL);
 780 #endif
 781         return (1);
 782 #else
 783         return (0);
 784 #endif
 785 }
 786
 787 static void
 788 native_lapic_disable_pmc(void)
 789 {
 790 #ifdef HWPMC_HOOKS
 791         u_int32_t maxlvt;
 792
 793         /* Fail if the local APIC is not present. */
 794         if (!x2apic_mode && lapic_map == NULL)
 795                 return;
 796
 797         /* Fail if the PMC LVT is not present. */
 798         maxlvt = (lapic_read32(LAPIC_VERSION) & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
 799         if (maxlvt < APIC_LVT_PMC)
 800                 return;
 801
 802         lvts[APIC_LVT_PMC].lvt_masked = 1;
 803
 804 #ifdef SMP
 805         /* The APs should always be started when hwpmc is unloaded. */
 806         KASSERT(mp_ncpus == 1 || smp_started, ("hwpmc unloaded too early"));
 807 #endif
 808         smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL);
 809 #endif
 810 }
 811
 812 static void
 813 lapic_calibrate_initcount(struct eventtimer *et, struct lapic *la)
 814 {
 815         u_long value;
 816
 817         /* Start off with a divisor of 2 (power on reset default). */
 818         lapic_timer_divisor = 2;
 819         /* Try to calibrate the local APIC timer. */
 820         do {
 821                 lapic_timer_set_divisor(lapic_timer_divisor);
 822                 lapic_timer_oneshot_nointr(la, APIC_TIMER_MAX_COUNT);
 823                 DELAY(1000000);
 824                 value = APIC_TIMER_MAX_COUNT - lapic_read32(LAPIC_CCR_TIMER);
 825                 if (value != APIC_TIMER_MAX_COUNT)
 826                         break;
 827                 lapic_timer_divisor <<= 1;
 828         } while (lapic_timer_divisor <= 128);
 829         if (lapic_timer_divisor > 128)
 830                 panic("lapic: Divisor too big");
 831         if (bootverbose) {
 832                 printf("lapic: Divisor %lu, Frequency %lu Hz\n",
 833                     lapic_timer_divisor, value);
 834         }
 835         count_freq = value;
 836 }
 837
 838 static void
 839 lapic_calibrate_deadline(struct eventtimer *et, struct lapic *la __unused)
 840 {
 841
 842         if (bootverbose) {
 843                 printf("lapic: deadline tsc mode, Frequency %ju Hz\n",
 844                     (uintmax_t)tsc_freq);
 845         }
 846 }
 847
 848 static void
 849 lapic_change_mode(struct eventtimer *et, struct lapic *la,
 850     enum lat_timer_mode newmode)
 851 {
 852
 853         if (la->la_timer_mode == newmode)
 854                 return;
 855         switch (newmode) {
 856         case LAT_MODE_PERIODIC:
 857                 lapic_timer_set_divisor(lapic_timer_divisor);
 858                 et->et_frequency = count_freq;
 859                 break;
 860         case LAT_MODE_DEADLINE:
 861                 et->et_frequency = tsc_freq;
 862                 break;
 863         case LAT_MODE_ONESHOT:
 864                 lapic_timer_set_divisor(lapic_timer_divisor);
 865                 et->et_frequency = count_freq;
 866                 break;
 867         default:
 868                 panic("lapic_change_mode %d", newmode);
 869         }
 870         la->la_timer_mode = newmode;
 871         et->et_min_period = (0x00000002LLU << 32) / et->et_frequency;
 872         et->et_max_period = (0xfffffffeLLU << 32) / et->et_frequency;
 873 }
 874
 875 static int
 876 lapic_et_start(struct eventtimer *et, sbintime_t first, sbintime_t period)
 877 {
 878         struct lapic *la;
 879
 880         la = &lapics[PCPU_GET(apic_id)];
 881         if (et->et_frequency == 0) {
 882                 lapic_calibrate_initcount(et, la);
 883                 if (lapic_timer_tsc_deadline)
 884                         lapic_calibrate_deadline(et, la);
 885         }
 886         if (period != 0) {
 887                 lapic_change_mode(et, la, LAT_MODE_PERIODIC);
 888                 la->la_timer_period = ((uint32_t)et->et_frequency * period) >>
 889                     32;
 890                 lapic_timer_periodic(la);
 891         } else if (lapic_timer_tsc_deadline) {
 892                 lapic_change_mode(et, la, LAT_MODE_DEADLINE);
 893                 la->la_timer_period = (et->et_frequency * first) >> 32;
 894                 lapic_timer_deadline(la);
 895         } else {
 896                 lapic_change_mode(et, la, LAT_MODE_ONESHOT);
 897                 la->la_timer_period = ((uint32_t)et->et_frequency * first) >>
 898                     32;
 899                 lapic_timer_oneshot(la);
 900         }
 901         return (0);
 902 }
 903
 904 static int
 905 lapic_et_stop(struct eventtimer *et)
 906 {
 907         struct lapic *la;
 908
 909         la = &lapics[PCPU_GET(apic_id)];
 910         lapic_timer_stop(la);
 911         la->la_timer_mode = LAT_MODE_UNDEF;
 912         return (0);
 913 }
 914
 915 static void
 916 native_lapic_disable(void)
 917 {
 918         uint32_t value;
 919
 920         /* Software disable the local APIC. */
 921         value = lapic_read32(LAPIC_SVR);
 922         value &= ~APIC_SVR_SWEN;
 923         lapic_write32(LAPIC_SVR, value);
 924 }
 925
 926 static void
 927 lapic_enable(void)
 928 {
 929         uint32_t value;
 930
 931         /* Program the spurious vector to enable the local APIC. */
 932         value = lapic_read32(LAPIC_SVR);
 933         value &= ~(APIC_SVR_VECTOR | APIC_SVR_FOCUS);
 934         value |= APIC_SVR_FEN | APIC_SVR_SWEN | APIC_SPURIOUS_INT;
 935         if (lapic_eoi_suppression)
 936                 value |= APIC_SVR_EOI_SUPPRESSION;
 937         lapic_write32(LAPIC_SVR, value);
 938 }
 939
 940 /* Reset the local APIC on the BSP during resume. */
 941 static void
 942 lapic_resume(struct pic *pic, bool suspend_cancelled)
 943 {
 944
 945         lapic_setup(0);
 946 }
 947
 948 static int
 949 native_lapic_id(void)
 950 {
 951         uint32_t v;
 952
 953         KASSERT(x2apic_mode || lapic_map != NULL, ("local APIC is not mapped"));
 954         v = lapic_read32(LAPIC_ID);
 955         if (!x2apic_mode)
 956                 v >>= APIC_ID_SHIFT;
 957         return (v);
 958 }
 959
 960 static int
 961 native_lapic_intr_pending(u_int vector)
 962 {
 963         uint32_t irr;
 964
 965         /*
 966          * The IRR registers are an array of registers each of which
 967          * only describes 32 interrupts in the low 32 bits.  Thus, we
 968          * divide the vector by 32 to get the register index.
 969          * Finally, we modulus the vector by 32 to determine the
 970          * individual bit to test.
 971          */
 972         irr = lapic_read32(LAPIC_IRR0 + vector / 32);
 973         return (irr & 1 << (vector % 32));
 974 }
 975
 976 static void
 977 native_lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id)
 978 {
 979         struct lapic *la;
 980
 981         KASSERT(lapics[apic_id].la_present, ("%s: APIC %u doesn't exist",
 982             __func__, apic_id));
 983         KASSERT(cluster <= APIC_MAX_CLUSTER, ("%s: cluster %u too big",
 984             __func__, cluster));
 985         KASSERT(cluster_id <= APIC_MAX_INTRACLUSTER_ID,
 986             ("%s: intra cluster id %u too big", __func__, cluster_id));
 987         la = &lapics[apic_id];
 988         la->la_cluster = cluster;
 989         la->la_cluster_id = cluster_id;
 990 }
 991
 992 static int
 993 native_lapic_set_lvt_mask(u_int apic_id, u_int pin, u_char masked)
 994 {
 995
 996         if (pin > APIC_LVT_MAX)
 997                 return (EINVAL);
 998         if (apic_id == APIC_ID_ALL) {
 999                 lvts[pin].lvt_masked = masked;
1000                 if (bootverbose)
1001                         printf("lapic:");
1002         } else {
1003                 KASSERT(lapics[apic_id].la_present,
1004                     ("%s: missing APIC %u", __func__, apic_id));
1005                 lapics[apic_id].la_lvts[pin].lvt_masked = masked;
1006                 lapics[apic_id].la_lvts[pin].lvt_active = 1;
1007                 if (bootverbose)
1008                         printf("lapic%u:", apic_id);
1009         }
1010         if (bootverbose)
1011                 printf(" LINT%u %s\n", pin, masked ? "masked" : "unmasked");
1012         return (0);
1013 }
1014
1015 static int
1016 native_lapic_set_lvt_mode(u_int apic_id, u_int pin, u_int32_t mode)
1017 {
1018         struct lvt *lvt;
1019
1020         if (pin > APIC_LVT_MAX)
1021                 return (EINVAL);
1022         if (apic_id == APIC_ID_ALL) {
1023                 lvt = &lvts[pin];
1024                 if (bootverbose)
1025                         printf("lapic:");
1026         } else {
1027                 KASSERT(lapics[apic_id].la_present,
1028                     ("%s: missing APIC %u", __func__, apic_id));
1029                 lvt = &lapics[apic_id].la_lvts[pin];
1030                 lvt->lvt_active = 1;
1031                 if (bootverbose)
1032                         printf("lapic%u:", apic_id);
1033         }
1034         lvt->lvt_mode = mode;
1035         switch (mode) {
1036         case APIC_LVT_DM_NMI:
1037         case APIC_LVT_DM_SMI:
1038         case APIC_LVT_DM_INIT:
1039         case APIC_LVT_DM_EXTINT:
1040                 lvt->lvt_edgetrigger = 1;
1041                 lvt->lvt_activehi = 1;
1042                 if (mode == APIC_LVT_DM_EXTINT)
1043                         lvt->lvt_masked = 1;
1044                 else
1045                         lvt->lvt_masked = 0;
1046                 break;
1047         default:
1048                 panic("Unsupported delivery mode: 0x%x\n", mode);
1049         }
1050         if (bootverbose) {
1051                 printf(" Routing ");
1052                 switch (mode) {
1053                 case APIC_LVT_DM_NMI:
1054                         printf("NMI");
1055                         break;
1056                 case APIC_LVT_DM_SMI:
1057                         printf("SMI");
1058                         break;
1059                 case APIC_LVT_DM_INIT:
1060                         printf("INIT");
1061                         break;
1062                 case APIC_LVT_DM_EXTINT:
1063                         printf("ExtINT");
1064                         break;
1065                 }
1066                 printf(" -> LINT%u\n", pin);
1067         }
1068         return (0);
1069 }
1070
1071 static int
1072 native_lapic_set_lvt_polarity(u_int apic_id, u_int pin, enum intr_polarity pol)
1073 {
1074
1075         if (pin > APIC_LVT_MAX || pol == INTR_POLARITY_CONFORM)
1076                 return (EINVAL);
1077         if (apic_id == APIC_ID_ALL) {
1078                 lvts[pin].lvt_activehi = (pol == INTR_POLARITY_HIGH);
1079                 if (bootverbose)
1080                         printf("lapic:");
1081         } else {
1082                 KASSERT(lapics[apic_id].la_present,
1083                     ("%s: missing APIC %u", __func__, apic_id));
1084                 lapics[apic_id].la_lvts[pin].lvt_active = 1;
1085                 lapics[apic_id].la_lvts[pin].lvt_activehi =
1086                     (pol == INTR_POLARITY_HIGH);
1087                 if (bootverbose)
1088                         printf("lapic%u:", apic_id);
1089         }
1090         if (bootverbose)
1091                 printf(" LINT%u polarity: %s\n", pin,
1092                     pol == INTR_POLARITY_HIGH ? "high" : "low");
1093         return (0);
1094 }
1095
1096 static int
1097 native_lapic_set_lvt_triggermode(u_int apic_id, u_int pin,
1098      enum intr_trigger trigger)
1099 {
1100
1101         if (pin > APIC_LVT_MAX || trigger == INTR_TRIGGER_CONFORM)
1102                 return (EINVAL);
1103         if (apic_id == APIC_ID_ALL) {
1104                 lvts[pin].lvt_edgetrigger = (trigger == INTR_TRIGGER_EDGE);
1105                 if (bootverbose)
1106                         printf("lapic:");
1107         } else {
1108                 KASSERT(lapics[apic_id].la_present,
1109                     ("%s: missing APIC %u", __func__, apic_id));
1110                 lapics[apic_id].la_lvts[pin].lvt_edgetrigger =
1111                     (trigger == INTR_TRIGGER_EDGE);
1112                 lapics[apic_id].la_lvts[pin].lvt_active = 1;
1113                 if (bootverbose)
1114                         printf("lapic%u:", apic_id);
1115         }
1116         if (bootverbose)
1117                 printf(" LINT%u trigger: %s\n", pin,
1118                     trigger == INTR_TRIGGER_EDGE ? "edge" : "level");
1119         return (0);
1120 }
1121
1122 /*
1123  * Adjust the TPR of the current CPU so that it blocks all interrupts below
1124  * the passed in vector.
1125  */
1126 static void
1127 lapic_set_tpr(u_int vector)
1128 {
1129 #ifdef CHEAP_TPR
1130         lapic_write32(LAPIC_TPR, vector);
1131 #else
1132         uint32_t tpr;
1133
1134         tpr = lapic_read32(LAPIC_TPR) & ~APIC_TPR_PRIO;
1135         tpr |= vector;
1136         lapic_write32(LAPIC_TPR, tpr);
1137 #endif
1138 }
1139
1140 static void
1141 native_lapic_eoi(void)
1142 {
1143
1144         lapic_write32_nofence(LAPIC_EOI, 0);
1145 }
1146
1147 void
1148 lapic_handle_intr(int vector, struct trapframe *frame)
1149 {
1150         struct intsrc *isrc;
1151
1152         isrc = intr_lookup_source(apic_idt_to_irq(PCPU_GET(apic_id),
1153             vector));
1154         intr_execute_handlers(isrc, frame);
1155 }
1156
1157 void
1158 lapic_handle_timer(struct trapframe *frame)
1159 {
1160         struct lapic *la;
1161         struct trapframe *oldframe;
1162         struct thread *td;
1163
1164         /* Send EOI first thing. */
1165         lapic_eoi();
1166
1167 #if defined(SMP) && !defined(SCHED_ULE)
1168         /*
1169          * Don't do any accounting for the disabled HTT cores, since it
1170          * will provide misleading numbers for the userland.
1171          *
1172          * No locking is necessary here, since even if we lose the race
1173          * when hlt_cpus_mask changes it is not a big deal, really.
1174          *
1175          * Don't do that for ULE, since ULE doesn't consider hlt_cpus_mask
1176          * and unlike other schedulers it actually schedules threads to
1177          * those CPUs.
1178          */
1179         if (CPU_ISSET(PCPU_GET(cpuid), &hlt_cpus_mask))
1180                 return;
1181 #endif
1182
1183         /* Look up our local APIC structure for the tick counters. */
1184         la = &lapics[PCPU_GET(apic_id)];
1185         (*la->la_timer_count)++;
1186         critical_enter();
1187         if (lapic_et.et_active) {
1188                 td = curthread;
1189                 td->td_intr_nesting_level++;
1190                 oldframe = td->td_intr_frame;
1191                 td->td_intr_frame = frame;
1192                 lapic_et.et_event_cb(&lapic_et, lapic_et.et_arg);
1193                 td->td_intr_frame = oldframe;
1194                 td->td_intr_nesting_level--;
1195         }
1196         critical_exit();
1197 }
1198
1199 static void
1200 lapic_timer_set_divisor(u_int divisor)
1201 {
1202
1203         KASSERT(powerof2(divisor), ("lapic: invalid divisor %u", divisor));
1204         KASSERT(ffs(divisor) <= nitems(lapic_timer_divisors),
1205                 ("lapic: invalid divisor %u", divisor));
1206         lapic_write32(LAPIC_DCR_TIMER, lapic_timer_divisors[ffs(divisor) - 1]);
1207 }
1208
1209 static void
1210 lapic_timer_oneshot(struct lapic *la)
1211 {
1212         uint32_t value;
1213
1214         value = la->lvt_timer_base;
1215         value &= ~(APIC_LVTT_TM | APIC_LVT_M);
1216         value |= APIC_LVTT_TM_ONE_SHOT;
1217         la->lvt_timer_last = value;
1218         lapic_write32(LAPIC_LVT_TIMER, value);
1219         lapic_write32(LAPIC_ICR_TIMER, la->la_timer_period);
1220 }
1221
1222 static void
1223 lapic_timer_oneshot_nointr(struct lapic *la, uint32_t count)
1224 {
1225         uint32_t value;
1226
1227         value = la->lvt_timer_base;
1228         value &= ~APIC_LVTT_TM;
1229         value |= APIC_LVTT_TM_ONE_SHOT | APIC_LVT_M;
1230         la->lvt_timer_last = value;
1231         lapic_write32(LAPIC_LVT_TIMER, value);
1232         lapic_write32(LAPIC_ICR_TIMER, count);
1233 }
1234
1235 static void
1236 lapic_timer_periodic(struct lapic *la)
1237 {
1238         uint32_t value;
1239
1240         value = la->lvt_timer_base;
1241         value &= ~(APIC_LVTT_TM | APIC_LVT_M);
1242         value |= APIC_LVTT_TM_PERIODIC;
1243         la->lvt_timer_last = value;
1244         lapic_write32(LAPIC_LVT_TIMER, value);
1245         lapic_write32(LAPIC_ICR_TIMER, la->la_timer_period);
1246 }
1247
1248 static void
1249 lapic_timer_deadline(struct lapic *la)
1250 {
1251         uint32_t value;
1252
1253         value = la->lvt_timer_base;
1254         value &= ~(APIC_LVTT_TM | APIC_LVT_M);
1255         value |= APIC_LVTT_TM_TSCDLT;
1256         if (value != la->lvt_timer_last) {
1257                 la->lvt_timer_last = value;
1258                 lapic_write32_nofence(LAPIC_LVT_TIMER, value);
1259                 if (!x2apic_mode)
1260                         mfence();
1261         }
1262         wrmsr(MSR_TSC_DEADLINE, la->la_timer_period + rdtsc());
1263 }
1264
1265 static void
1266 lapic_timer_stop(struct lapic *la)
1267 {
1268         uint32_t value;
1269
1270         if (la->la_timer_mode == LAT_MODE_DEADLINE) {
1271                 wrmsr(MSR_TSC_DEADLINE, 0);
1272                 mfence();
1273         } else {
1274                 value = la->lvt_timer_base;
1275                 value &= ~APIC_LVTT_TM;
1276                 value |= APIC_LVT_M;
1277                 la->lvt_timer_last = value;
1278                 lapic_write32(LAPIC_LVT_TIMER, value);
1279         }
1280 }
1281
1282 void
1283 lapic_handle_cmc(void)
1284 {
1285
1286         lapic_eoi();
1287         cmc_intr();
1288 }
1289
1290 /*
1291  * Called from the mca_init() to activate the CMC interrupt if this CPU is
1292  * responsible for monitoring any MC banks for CMC events.  Since mca_init()
1293  * is called prior to lapic_setup() during boot, this just needs to unmask
1294  * this CPU's LVT_CMCI entry.
1295  */
1296 static void
1297 native_lapic_enable_cmc(void)
1298 {
1299         u_int apic_id;
1300
1301 #ifdef DEV_ATPIC
1302         if (!x2apic_mode && lapic_map == NULL)
1303                 return;
1304 #endif
1305         apic_id = PCPU_GET(apic_id);
1306         KASSERT(lapics[apic_id].la_present,
1307             ("%s: missing APIC %u", __func__, apic_id));
1308         lapics[apic_id].la_lvts[APIC_LVT_CMCI].lvt_masked = 0;
1309         lapics[apic_id].la_lvts[APIC_LVT_CMCI].lvt_active = 1;
1310         if (bootverbose)
1311                 printf("lapic%u: CMCI unmasked\n", apic_id);
1312 }
1313
1314 void
1315 lapic_handle_error(void)
1316 {
1317         uint32_t esr;
1318
1319         /*
1320          * Read the contents of the error status register.  Write to
1321          * the register first before reading from it to force the APIC
1322          * to update its value to indicate any errors that have
1323          * occurred since the previous write to the register.
1324          */
1325         lapic_write32(LAPIC_ESR, 0);
1326         esr = lapic_read32(LAPIC_ESR);
1327
1328         printf("CPU%d: local APIC error 0x%x\n", PCPU_GET(cpuid), esr);
1329         lapic_eoi();
1330 }
1331
1332 static u_int
1333 native_apic_cpuid(u_int apic_id)
1334 {
1335 #ifdef SMP
1336         return apic_cpuids[apic_id];
1337 #else
1338         return 0;
1339 #endif
1340 }
1341
1342 /* Request a free IDT vector to be used by the specified IRQ. */
1343 static u_int
1344 native_apic_alloc_vector(u_int apic_id, u_int irq)
1345 {
1346         u_int vector;
1347
1348         KASSERT(irq < NUM_IO_INTS, ("Invalid IRQ %u", irq));
1349
1350         /*
1351          * Search for a free vector.  Currently we just use a very simple
1352          * algorithm to find the first free vector.
1353          */
1354         mtx_lock_spin(&icu_lock);
1355         for (vector = 0; vector < APIC_NUM_IOINTS; vector++) {
1356                 if (lapics[apic_id].la_ioint_irqs[vector] != -1)
1357                         continue;
1358                 lapics[apic_id].la_ioint_irqs[vector] = irq;
1359                 mtx_unlock_spin(&icu_lock);
1360                 return (vector + APIC_IO_INTS);
1361         }
1362         mtx_unlock_spin(&icu_lock);
1363         return (0);
1364 }
1365
1366 /*
1367  * Request 'count' free contiguous IDT vectors to be used by 'count'
1368  * IRQs.  'count' must be a power of two and the vectors will be
1369  * aligned on a boundary of 'align'.  If the request cannot be
1370  * satisfied, 0 is returned.
1371  */
1372 static u_int
1373 native_apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align)
1374 {
1375         u_int first, run, vector;
1376
1377         KASSERT(powerof2(count), ("bad count"));
1378         KASSERT(powerof2(align), ("bad align"));
1379         KASSERT(align >= count, ("align < count"));
1380 #ifdef INVARIANTS
1381         for (run = 0; run < count; run++)
1382                 KASSERT(irqs[run] < NUM_IO_INTS, ("Invalid IRQ %u at index %u",
1383                     irqs[run], run));
1384 #endif
1385
1386         /*
1387          * Search for 'count' free vectors.  As with apic_alloc_vector(),
1388          * this just uses a simple first fit algorithm.
1389          */
1390         run = 0;
1391         first = 0;
1392         mtx_lock_spin(&icu_lock);
1393         for (vector = 0; vector < APIC_NUM_IOINTS; vector++) {
1394
1395                 /* Vector is in use, end run. */
1396                 if (lapics[apic_id].la_ioint_irqs[vector] != -1) {
1397                         run = 0;
1398                         first = 0;
1399                         continue;
1400                 }
1401
1402                 /* Start a new run if run == 0 and vector is aligned. */
1403                 if (run == 0) {
1404                         if ((vector & (align - 1)) != 0)
1405                                 continue;
1406                         first = vector;
1407                 }
1408                 run++;
1409
1410                 /* Keep looping if the run isn't long enough yet. */
1411                 if (run < count)
1412                         continue;
1413
1414                 /* Found a run, assign IRQs and return the first vector. */
1415                 for (vector = 0; vector < count; vector++)
1416                         lapics[apic_id].la_ioint_irqs[first + vector] =
1417                             irqs[vector];
1418                 mtx_unlock_spin(&icu_lock);
1419                 return (first + APIC_IO_INTS);
1420         }
1421         mtx_unlock_spin(&icu_lock);
1422         printf("APIC: Couldn't find APIC vectors for %u IRQs\n", count);
1423         return (0);
1424 }
1425
1426 /*
1427  * Enable a vector for a particular apic_id.  Since all lapics share idt
1428  * entries and ioint_handlers this enables the vector on all lapics.  lapics
1429  * which do not have the vector configured would report spurious interrupts
1430  * should it fire.
1431  */
1432 static void
1433 native_apic_enable_vector(u_int apic_id, u_int vector)
1434 {
1435
1436         KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry"));
1437         KASSERT(ioint_handlers[vector / 32] != NULL,
1438             ("No ISR handler for vector %u", vector));
1439 #ifdef KDTRACE_HOOKS
1440         KASSERT(vector != IDT_DTRACE_RET,
1441             ("Attempt to overwrite DTrace entry"));
1442 #endif
1443         setidt(vector, ioint_handlers[vector / 32], SDT_APIC, SEL_KPL,
1444             GSEL_APIC);
1445 }
1446
1447 static void
1448 native_apic_disable_vector(u_int apic_id, u_int vector)
1449 {
1450
1451         KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry"));
1452 #ifdef KDTRACE_HOOKS
1453         KASSERT(vector != IDT_DTRACE_RET,
1454             ("Attempt to overwrite DTrace entry"));
1455 #endif
1456         KASSERT(ioint_handlers[vector / 32] != NULL,
1457             ("No ISR handler for vector %u", vector));
1458 #ifdef notyet
1459         /*
1460          * We can not currently clear the idt entry because other cpus
1461          * may have a valid vector at this offset.
1462          */
1463         setidt(vector, &IDTVEC(rsvd), SDT_APICT, SEL_KPL, GSEL_APIC);
1464 #endif
1465 }
1466
1467 /* Release an APIC vector when it's no longer in use. */
1468 static void
1469 native_apic_free_vector(u_int apic_id, u_int vector, u_int irq)
1470 {
1471         struct thread *td;
1472
1473         KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL &&
1474             vector <= APIC_IO_INTS + APIC_NUM_IOINTS,
1475             ("Vector %u does not map to an IRQ line", vector));
1476         KASSERT(irq < NUM_IO_INTS, ("Invalid IRQ %u", irq));
1477         KASSERT(lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] ==
1478             irq, ("IRQ mismatch"));
1479 #ifdef KDTRACE_HOOKS
1480         KASSERT(vector != IDT_DTRACE_RET,
1481             ("Attempt to overwrite DTrace entry"));
1482 #endif
1483
1484         /*
1485          * Bind us to the cpu that owned the vector before freeing it so
1486          * we don't lose an interrupt delivery race.
1487          */
1488         td = curthread;
1489         if (!rebooting) {
1490                 thread_lock(td);
1491                 if (sched_is_bound(td))
1492                         panic("apic_free_vector: Thread already bound.\n");
1493                 sched_bind(td, apic_cpuid(apic_id));
1494                 thread_unlock(td);
1495         }
1496         mtx_lock_spin(&icu_lock);
1497         lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] = -1;
1498         mtx_unlock_spin(&icu_lock);
1499         if (!rebooting) {
1500                 thread_lock(td);
1501                 sched_unbind(td);
1502                 thread_unlock(td);
1503         }
1504 }
1505
1506 /* Map an IDT vector (APIC) to an IRQ (interrupt source). */
1507 static u_int
1508 apic_idt_to_irq(u_int apic_id, u_int vector)
1509 {
1510         int irq;
1511
1512         KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL &&
1513             vector <= APIC_IO_INTS + APIC_NUM_IOINTS,
1514             ("Vector %u does not map to an IRQ line", vector));
1515 #ifdef KDTRACE_HOOKS
1516         KASSERT(vector != IDT_DTRACE_RET,
1517             ("Attempt to overwrite DTrace entry"));
1518 #endif
1519         irq = lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS];
1520         if (irq < 0)
1521                 irq = 0;
1522         return (irq);
1523 }
1524
1525 #ifdef DDB
1526 /*
1527  * Dump data about APIC IDT vector mappings.
1528  */
1529 DB_SHOW_COMMAND(apic, db_show_apic)
1530 {
1531         struct intsrc *isrc;
1532         int i, verbose;
1533         u_int apic_id;
1534         u_int irq;
1535
1536         if (strcmp(modif, "vv") == 0)
1537                 verbose = 2;
1538         else if (strcmp(modif, "v") == 0)
1539                 verbose = 1;
1540         else
1541                 verbose = 0;
1542         for (apic_id = 0; apic_id <= MAX_APIC_ID; apic_id++) {
1543                 if (lapics[apic_id].la_present == 0)
1544                         continue;
1545                 db_printf("Interrupts bound to lapic %u\n", apic_id);
1546                 for (i = 0; i < APIC_NUM_IOINTS + 1 && !db_pager_quit; i++) {
1547                         irq = lapics[apic_id].la_ioint_irqs[i];
1548                         if (irq == -1 || irq == IRQ_SYSCALL)
1549                                 continue;
1550 #ifdef KDTRACE_HOOKS
1551                         if (irq == IRQ_DTRACE_RET)
1552                                 continue;
1553 #endif
1554 #ifdef XENHVM
1555                         if (irq == IRQ_EVTCHN)
1556                                 continue;
1557 #endif
1558                         db_printf("vec 0x%2x -> ", i + APIC_IO_INTS);
1559                         if (irq == IRQ_TIMER)
1560                                 db_printf("lapic timer\n");
1561                         else if (irq < NUM_IO_INTS) {
1562                                 isrc = intr_lookup_source(irq);
1563                                 if (isrc == NULL || verbose == 0)
1564                                         db_printf("IRQ %u\n", irq);
1565                                 else
1566                                         db_dump_intr_event(isrc->is_event,
1567                                             verbose == 2);
1568                         } else
1569                                 db_printf("IRQ %u ???\n", irq);
1570                 }
1571         }
1572 }
1573
1574 static void
1575 dump_mask(const char *prefix, uint32_t v, int base)
1576 {
1577         int i, first;
1578
1579         first = 1;
1580         for (i = 0; i < 32; i++)
1581                 if (v & (1 << i)) {
1582                         if (first) {
1583                                 db_printf("%s:", prefix);
1584                                 first = 0;
1585                         }
1586                         db_printf(" %02x", base + i);
1587                 }
1588         if (!first)
1589                 db_printf("\n");
1590 }
1591
1592 /* Show info from the lapic regs for this CPU. */
1593 DB_SHOW_COMMAND(lapic, db_show_lapic)
1594 {
1595         uint32_t v;
1596
1597         db_printf("lapic ID = %d\n", lapic_id());
1598         v = lapic_read32(LAPIC_VERSION);
1599         db_printf("version  = %d.%d\n", (v & APIC_VER_VERSION) >> 4,
1600             v & 0xf);
1601         db_printf("max LVT  = %d\n", (v & APIC_VER_MAXLVT) >> MAXLVTSHIFT);
1602         v = lapic_read32(LAPIC_SVR);
1603         db_printf("SVR      = %02x (%s)\n", v & APIC_SVR_VECTOR,
1604             v & APIC_SVR_ENABLE ? "enabled" : "disabled");
1605         db_printf("TPR      = %02x\n", lapic_read32(LAPIC_TPR));
1606
1607 #define dump_field(prefix, regn, index)                                 \
1608         dump_mask(__XSTRING(prefix ## index),                           \
1609             lapic_read32(LAPIC_ ## regn ## index),                      \
1610             index * 32)
1611
1612         db_printf("In-service Interrupts:\n");
1613         dump_field(isr, ISR, 0);
1614         dump_field(isr, ISR, 1);
1615         dump_field(isr, ISR, 2);
1616         dump_field(isr, ISR, 3);
1617         dump_field(isr, ISR, 4);
1618         dump_field(isr, ISR, 5);
1619         dump_field(isr, ISR, 6);
1620         dump_field(isr, ISR, 7);
1621
1622         db_printf("TMR Interrupts:\n");
1623         dump_field(tmr, TMR, 0);
1624         dump_field(tmr, TMR, 1);
1625         dump_field(tmr, TMR, 2);
1626         dump_field(tmr, TMR, 3);
1627         dump_field(tmr, TMR, 4);
1628         dump_field(tmr, TMR, 5);
1629         dump_field(tmr, TMR, 6);
1630         dump_field(tmr, TMR, 7);
1631
1632         db_printf("IRR Interrupts:\n");
1633         dump_field(irr, IRR, 0);
1634         dump_field(irr, IRR, 1);
1635         dump_field(irr, IRR, 2);
1636         dump_field(irr, IRR, 3);
1637         dump_field(irr, IRR, 4);
1638         dump_field(irr, IRR, 5);
1639         dump_field(irr, IRR, 6);
1640         dump_field(irr, IRR, 7);
1641
1642 #undef dump_field
1643 }
1644 #endif
1645
1646 /*
1647  * APIC probing support code.  This includes code to manage enumerators.
1648  */
1649
1650 static SLIST_HEAD(, apic_enumerator) enumerators =
1651         SLIST_HEAD_INITIALIZER(enumerators);
1652 static struct apic_enumerator *best_enum;
1653
1654 void
1655 apic_register_enumerator(struct apic_enumerator *enumerator)
1656 {
1657 #ifdef INVARIANTS
1658         struct apic_enumerator *apic_enum;
1659
1660         SLIST_FOREACH(apic_enum, &enumerators, apic_next) {
1661                 if (apic_enum == enumerator)
1662                         panic("%s: Duplicate register of %s", __func__,
1663                             enumerator->apic_name);
1664         }
1665 #endif
1666         SLIST_INSERT_HEAD(&enumerators, enumerator, apic_next);
1667 }
1668
1669 /*
1670  * We have to look for CPU's very, very early because certain subsystems
1671  * want to know how many CPU's we have extremely early on in the boot
1672  * process.
1673  */
1674 static void
1675 apic_init(void *dummy __unused)
1676 {
1677         struct apic_enumerator *enumerator;
1678         int retval, best;
1679
1680         /* We only support built in local APICs. */
1681         if (!(cpu_feature & CPUID_APIC))
1682                 return;
1683
1684         /* Don't probe if APIC mode is disabled. */
1685         if (resource_disabled("apic", 0))
1686                 return;
1687
1688         /* Probe all the enumerators to find the best match. */
1689         best_enum = NULL;
1690         best = 0;
1691         SLIST_FOREACH(enumerator, &enumerators, apic_next) {
1692                 retval = enumerator->apic_probe();
1693                 if (retval > 0)
1694                         continue;
1695                 if (best_enum == NULL || best < retval) {
1696                         best_enum = enumerator;
1697                         best = retval;
1698                 }
1699         }
1700         if (best_enum == NULL) {
1701                 if (bootverbose)
1702                         printf("APIC: Could not find any APICs.\n");
1703 #ifndef DEV_ATPIC
1704                 panic("running without device atpic requires a local APIC");
1705 #endif
1706                 return;
1707         }
1708
1709         if (bootverbose)
1710                 printf("APIC: Using the %s enumerator.\n",
1711                     best_enum->apic_name);
1712
1713 #ifdef I686_CPU
1714         /*
1715          * To work around an errata, we disable the local APIC on some
1716          * CPUs during early startup.  We need to turn the local APIC back
1717          * on on such CPUs now.
1718          */
1719         ppro_reenable_apic();
1720 #endif
1721
1722         /* Probe the CPU's in the system. */
1723         retval = best_enum->apic_probe_cpus();
1724         if (retval != 0)
1725                 printf("%s: Failed to probe CPUs: returned %d\n",
1726                     best_enum->apic_name, retval);
1727
1728 }
1729 SYSINIT(apic_init, SI_SUB_TUNABLES - 1, SI_ORDER_SECOND, apic_init, NULL);
1730
1731 /*
1732  * Setup the local APIC.  We have to do this prior to starting up the APs
1733  * in the SMP case.
1734  */
1735 static void
1736 apic_setup_local(void *dummy __unused)
1737 {
1738         int retval;
1739
1740         if (best_enum == NULL)
1741                 return;
1742
1743         /* Initialize the local APIC. */
1744         retval = best_enum->apic_setup_local();
1745         if (retval != 0)
1746                 printf("%s: Failed to setup the local APIC: returned %d\n",
1747                     best_enum->apic_name, retval);
1748 }
1749 SYSINIT(apic_setup_local, SI_SUB_CPU, SI_ORDER_SECOND, apic_setup_local, NULL);
1750
1751 /*
1752  * Setup the I/O APICs.
1753  */
1754 static void
1755 apic_setup_io(void *dummy __unused)
1756 {
1757         int retval;
1758
1759         if (best_enum == NULL)
1760                 return;
1761
1762         /*
1763          * Local APIC must be registered before other PICs and pseudo PICs
1764          * for proper suspend/resume order.
1765          */
1766         intr_register_pic(&lapic_pic);
1767
1768         retval = best_enum->apic_setup_io();
1769         if (retval != 0)
1770                 printf("%s: Failed to setup I/O APICs: returned %d\n",
1771                     best_enum->apic_name, retval);
1772
1773         /*
1774          * Finish setting up the local APIC on the BSP once we know
1775          * how to properly program the LINT pins.  In particular, this
1776          * enables the EOI suppression mode, if LAPIC support it and
1777          * user did not disabled the mode.
1778          */
1779         lapic_setup(1);
1780         if (bootverbose)
1781                 lapic_dump("BSP");
1782
1783         /* Enable the MSI "pic". */
1784         init_ops.msi_init();
1785 }
1786 SYSINIT(apic_setup_io, SI_SUB_INTR, SI_ORDER_THIRD, apic_setup_io, NULL);
1787
1788 #ifdef SMP
1789 /*
1790  * Inter Processor Interrupt functions.  The lapic_ipi_*() functions are
1791  * private to the MD code.  The public interface for the rest of the
1792  * kernel is defined in mp_machdep.c.
1793  */
1794
1795 /*
1796  * Wait delay microseconds for IPI to be sent.  If delay is -1, we
1797  * wait forever.
1798  */
1799 static int
1800 native_lapic_ipi_wait(int delay)
1801 {
1802         uint64_t rx;
1803
1804         /* LAPIC_ICR.APIC_DELSTAT_MASK is undefined in x2APIC mode */
1805         if (x2apic_mode)
1806                 return (1);
1807
1808         for (rx = 0; delay == -1 || rx < lapic_ipi_wait_mult * delay; rx++) {
1809                 if ((lapic_read_icr_lo() & APIC_DELSTAT_MASK) ==
1810                     APIC_DELSTAT_IDLE)
1811                         return (1);
1812                 ia32_pause();
1813         }
1814         return (0);
1815 }
1816
1817 static void
1818 native_lapic_ipi_raw(register_t icrlo, u_int dest)
1819 {
1820         uint64_t icr;
1821         uint32_t vhi, vlo;
1822         register_t saveintr;
1823
1824         /* XXX: Need more sanity checking of icrlo? */
1825         KASSERT(x2apic_mode || lapic_map != NULL,
1826             ("%s called too early", __func__));
1827         KASSERT(x2apic_mode ||
1828             (dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0,
1829             ("%s: invalid dest field", __func__));
1830         KASSERT((icrlo & APIC_ICRLO_RESV_MASK) == 0,
1831             ("%s: reserved bits set in ICR LO register", __func__));
1832
1833         /* Set destination in ICR HI register if it is being used. */
1834         if (!x2apic_mode) {
1835                 saveintr = intr_disable();
1836                 icr = lapic_read_icr();
1837         }
1838
1839         if ((icrlo & APIC_DEST_MASK) == APIC_DEST_DESTFLD) {
1840                 if (x2apic_mode) {
1841                         vhi = dest;
1842                 } else {
1843                         vhi = icr >> 32;
1844                         vhi &= ~APIC_ID_MASK;
1845                         vhi |= dest << APIC_ID_SHIFT;
1846                 }
1847         } else {
1848                 vhi = 0;
1849         }
1850
1851         /* Program the contents of the IPI and dispatch it. */
1852         if (x2apic_mode) {
1853                 vlo = icrlo;
1854         } else {
1855                 vlo = icr;
1856                 vlo &= APIC_ICRLO_RESV_MASK;
1857                 vlo |= icrlo;
1858         }
1859         lapic_write_icr(vhi, vlo);
1860         if (!x2apic_mode)
1861                 intr_restore(saveintr);
1862 }
1863
1864 #define BEFORE_SPIN     50000
1865 #ifdef DETECT_DEADLOCK
1866 #define AFTER_SPIN      50
1867 #endif
1868
1869 static void
1870 native_lapic_ipi_vectored(u_int vector, int dest)
1871 {
1872         register_t icrlo, destfield;
1873
1874         KASSERT((vector & ~APIC_VECTOR_MASK) == 0,
1875             ("%s: invalid vector %d", __func__, vector));
1876
1877         icrlo = APIC_DESTMODE_PHY | APIC_TRIGMOD_EDGE | APIC_LEVEL_ASSERT;
1878
1879         /*
1880          * NMI IPIs are just fake vectors used to send a NMI.  Use special rules
1881          * regarding NMIs if passed, otherwise specify the vector.
1882          */
1883         if (vector >= IPI_NMI_FIRST)
1884                 icrlo |= APIC_DELMODE_NMI;
1885         else
1886                 icrlo |= vector | APIC_DELMODE_FIXED;
1887         destfield = 0;
1888         switch (dest) {
1889         case APIC_IPI_DEST_SELF:
1890                 icrlo |= APIC_DEST_SELF;
1891                 break;
1892         case APIC_IPI_DEST_ALL:
1893                 icrlo |= APIC_DEST_ALLISELF;
1894                 break;
1895         case APIC_IPI_DEST_OTHERS:
1896                 icrlo |= APIC_DEST_ALLESELF;
1897                 break;
1898         default:
1899                 KASSERT(x2apic_mode ||
1900                     (dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0,
1901                     ("%s: invalid destination 0x%x", __func__, dest));
1902                 destfield = dest;
1903         }
1904
1905         /* Wait for an earlier IPI to finish. */
1906         if (!lapic_ipi_wait(BEFORE_SPIN)) {
1907                 if (panicstr != NULL)
1908                         return;
1909                 else
1910                         panic("APIC: Previous IPI is stuck");
1911         }
1912
1913         lapic_ipi_raw(icrlo, destfield);
1914
1915 #ifdef DETECT_DEADLOCK
1916         /* Wait for IPI to be delivered. */
1917         if (!lapic_ipi_wait(AFTER_SPIN)) {
1918 #ifdef needsattention
1919                 /*
1920                  * XXX FIXME:
1921                  *
1922                  * The above function waits for the message to actually be
1923                  * delivered.  It breaks out after an arbitrary timeout
1924                  * since the message should eventually be delivered (at
1925                  * least in theory) and that if it wasn't we would catch
1926                  * the failure with the check above when the next IPI is
1927                  * sent.
1928                  *
1929                  * We could skip this wait entirely, EXCEPT it probably
1930                  * protects us from other routines that assume that the
1931                  * message was delivered and acted upon when this function
1932                  * returns.
1933                  */
1934                 printf("APIC: IPI might be stuck\n");
1935 #else /* !needsattention */
1936                 /* Wait until mesage is sent without a timeout. */
1937                 while (lapic_read_icr_lo() & APIC_DELSTAT_PEND)
1938                         ia32_pause();
1939 #endif /* needsattention */
1940         }
1941 #endif /* DETECT_DEADLOCK */
1942 }
1943
1944 #endif /* SMP */
1945
1946 /*
1947  * Since the IDT is shared by all CPUs the IPI slot update needs to be globally
1948  * visible.
1949  *
1950  * Consider the case where an IPI is generated immediately after allocation:
1951  *     vector = lapic_ipi_alloc(ipifunc);
1952  *     ipi_selected(other_cpus, vector);
1953  *
1954  * In xAPIC mode a write to ICR_LO has serializing semantics because the
1955  * APIC page is mapped as an uncached region. In x2APIC mode there is an
1956  * explicit 'mfence' before the ICR MSR is written. Therefore in both cases
1957  * the IDT slot update is globally visible before the IPI is delivered.
1958  */
1959 static int
1960 native_lapic_ipi_alloc(inthand_t *ipifunc)
1961 {
1962         struct gate_descriptor *ip;
1963         long func;
1964         int idx, vector;
1965
1966         KASSERT(ipifunc != &IDTVEC(rsvd), ("invalid ipifunc %p", ipifunc));
1967
1968         vector = -1;
1969         mtx_lock_spin(&icu_lock);
1970         for (idx = IPI_DYN_FIRST; idx <= IPI_DYN_LAST; idx++) {
1971                 ip = &idt[idx];
1972                 func = (ip->gd_hioffset << 16) | ip->gd_looffset;
1973                 if (func == (uintptr_t)&IDTVEC(rsvd)) {
1974                         vector = idx;
1975                         setidt(vector, ipifunc, SDT_APIC, SEL_KPL, GSEL_APIC);
1976                         break;
1977                 }
1978         }
1979         mtx_unlock_spin(&icu_lock);
1980         return (vector);
1981 }
1982
1983 static void
1984 native_lapic_ipi_free(int vector)
1985 {
1986         struct gate_descriptor *ip;
1987         long func;
1988
1989         KASSERT(vector >= IPI_DYN_FIRST && vector <= IPI_DYN_LAST,
1990             ("%s: invalid vector %d", __func__, vector));
1991
1992         mtx_lock_spin(&icu_lock);
1993         ip = &idt[vector];
1994         func = (ip->gd_hioffset << 16) | ip->gd_looffset;
1995         KASSERT(func != (uintptr_t)&IDTVEC(rsvd),
1996             ("invalid idtfunc %#lx", func));
1997         setidt(vector, &IDTVEC(rsvd), SDT_APICT, SEL_KPL, GSEL_APIC);
1998         mtx_unlock_spin(&icu_lock);
1999 }