sys/x86/x86/local_apic.c

   1 /*-
   2  * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org>
   3  * Copyright (c) 1996, by Steve Passe
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. The name of the developer may NOT be used to endorse or promote products
  12  *    derived from this software without specific prior written permission.
  13  * 3. Neither the name of the author nor the names of any co-contributors
  14  *    may be used to endorse or promote products derived from this software
  15  *    without specific prior written permission.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 /*
  31  * Local APIC support on Pentium and later processors.
  32  */
  33
  34 #include <sys/cdefs.h>
  35 __FBSDID("$FreeBSD$");
  36
  37 #include "opt_atpic.h"
  38 #include "opt_hwpmc_hooks.h"
  39
  40 #include "opt_ddb.h"
  41
  42 #include <sys/param.h>
  43 #include <sys/systm.h>
  44 #include <sys/bus.h>
  45 #include <sys/kernel.h>
  46 #include <sys/lock.h>
  47 #include <sys/malloc.h>
  48 #include <sys/mutex.h>
  49 #include <sys/pcpu.h>
  50 #include <sys/proc.h>
  51 #include <sys/sched.h>
  52 #include <sys/smp.h>
  53 #include <sys/sysctl.h>
  54 #include <sys/timeet.h>
  55
  56 #include <vm/vm.h>
  57 #include <vm/pmap.h>
  58
  59 #include <x86/apicreg.h>
  60 #include <machine/clock.h>
  61 #include <machine/cpufunc.h>
  62 #include <machine/cputypes.h>
  63 #include <machine/frame.h>
  64 #include <machine/intr_machdep.h>
  65 #include <x86/apicvar.h>
  66 #include <x86/mca.h>
  67 #include <machine/md_var.h>
  68 #include <machine/smp.h>
  69 #include <machine/specialreg.h>
  70 #include <x86/init.h>
  71
  72 #ifdef DDB
  73 #include <sys/interrupt.h>
  74 #include <ddb/ddb.h>
  75 #endif
  76
  77 #ifdef __amd64__
  78 #define SDT_APIC        SDT_SYSIGT
  79 #define SDT_APICT       SDT_SYSIGT
  80 #define GSEL_APIC       0
  81 #else
  82 #define SDT_APIC        SDT_SYS386IGT
  83 #define SDT_APICT       SDT_SYS386TGT
  84 #define GSEL_APIC       GSEL(GCODE_SEL, SEL_KPL)
  85 #endif
  86
  87 static MALLOC_DEFINE(M_LAPIC, "local_apic", "Local APIC items");
  88
  89 /* Sanity checks on IDT vectors. */
  90 CTASSERT(APIC_IO_INTS + APIC_NUM_IOINTS == APIC_TIMER_INT);
  91 CTASSERT(APIC_TIMER_INT < APIC_LOCAL_INTS);
  92 CTASSERT(APIC_LOCAL_INTS == 240);
  93 CTASSERT(IPI_STOP < APIC_SPURIOUS_INT);
  94
  95 /* Magic IRQ values for the timer and syscalls. */
  96 #define IRQ_TIMER       (NUM_IO_INTS + 1)
  97 #define IRQ_SYSCALL     (NUM_IO_INTS + 2)
  98 #define IRQ_DTRACE_RET  (NUM_IO_INTS + 3)
  99 #define IRQ_EVTCHN      (NUM_IO_INTS + 4)
 100
 101 enum lat_timer_mode {
 102         LAT_MODE_UNDEF =        0,
 103         LAT_MODE_PERIODIC =     1,
 104         LAT_MODE_ONESHOT =      2,
 105         LAT_MODE_DEADLINE =     3,
 106 };
 107
 108 /*
 109  * Support for local APICs.  Local APICs manage interrupts on each
 110  * individual processor as opposed to I/O APICs which receive interrupts
 111  * from I/O devices and then forward them on to the local APICs.
 112  *
 113  * Local APICs can also send interrupts to each other thus providing the
 114  * mechanism for IPIs.
 115  */
 116
 117 struct lvt {
 118         u_int lvt_edgetrigger:1;
 119         u_int lvt_activehi:1;
 120         u_int lvt_masked:1;
 121         u_int lvt_active:1;
 122         u_int lvt_mode:16;
 123         u_int lvt_vector:8;
 124 };
 125
 126 struct lapic {
 127         struct lvt la_lvts[APIC_LVT_MAX + 1];
 128         struct lvt la_elvts[APIC_ELVT_MAX + 1];;
 129         u_int la_id:8;
 130         u_int la_cluster:4;
 131         u_int la_cluster_id:2;
 132         u_int la_present:1;
 133         u_long *la_timer_count;
 134         uint64_t la_timer_period;
 135         enum lat_timer_mode la_timer_mode;
 136         uint32_t lvt_timer_base;
 137         uint32_t lvt_timer_last;
 138         /* Include IDT_SYSCALL to make indexing easier. */
 139         int la_ioint_irqs[APIC_NUM_IOINTS + 1];
 140 } static *lapics;
 141
 142 /* Global defaults for local APIC LVT entries. */
 143 static struct lvt lvts[APIC_LVT_MAX + 1] = {
 144         { 1, 1, 1, 1, APIC_LVT_DM_EXTINT, 0 },  /* LINT0: masked ExtINT */
 145         { 1, 1, 0, 1, APIC_LVT_DM_NMI, 0 },     /* LINT1: NMI */
 146         { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_TIMER_INT },      /* Timer */
 147         { 1, 1, 0, 1, APIC_LVT_DM_FIXED, APIC_ERROR_INT },      /* Error */
 148         { 1, 1, 1, 1, APIC_LVT_DM_NMI, 0 },     /* PMC */
 149         { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_THERMAL_INT },    /* Thermal */
 150         { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_CMC_INT },        /* CMCI */
 151 };
 152
 153 /* Global defaults for AMD local APIC ELVT entries. */
 154 static struct lvt elvts[APIC_ELVT_MAX + 1] = {
 155         { 1, 1, 1, 0, APIC_LVT_DM_FIXED, 0 },
 156         { 1, 1, 1, 0, APIC_LVT_DM_FIXED, APIC_CMC_INT },
 157         { 1, 1, 1, 0, APIC_LVT_DM_FIXED, 0 },
 158         { 1, 1, 1, 0, APIC_LVT_DM_FIXED, 0 },
 159 };
 160
 161 static inthand_t *ioint_handlers[] = {
 162         NULL,                   /* 0 - 31 */
 163         IDTVEC(apic_isr1),      /* 32 - 63 */
 164         IDTVEC(apic_isr2),      /* 64 - 95 */
 165         IDTVEC(apic_isr3),      /* 96 - 127 */
 166         IDTVEC(apic_isr4),      /* 128 - 159 */
 167         IDTVEC(apic_isr5),      /* 160 - 191 */
 168         IDTVEC(apic_isr6),      /* 192 - 223 */
 169         IDTVEC(apic_isr7),      /* 224 - 255 */
 170 };
 171
 172
 173 static u_int32_t lapic_timer_divisors[] = {
 174         APIC_TDCR_1, APIC_TDCR_2, APIC_TDCR_4, APIC_TDCR_8, APIC_TDCR_16,
 175         APIC_TDCR_32, APIC_TDCR_64, APIC_TDCR_128
 176 };
 177
 178 extern inthand_t IDTVEC(rsvd);
 179
 180 volatile char *lapic_map;
 181 vm_paddr_t lapic_paddr;
 182 int x2apic_mode;
 183 int lapic_eoi_suppression;
 184 static int lapic_timer_tsc_deadline;
 185 static u_long lapic_timer_divisor, count_freq;
 186 static struct eventtimer lapic_et;
 187 #ifdef SMP
 188 static uint64_t lapic_ipi_wait_mult;
 189 #endif
 190 unsigned int max_apic_id;
 191
 192 SYSCTL_NODE(_hw, OID_AUTO, apic, CTLFLAG_RD, 0, "APIC options");
 193 SYSCTL_INT(_hw_apic, OID_AUTO, x2apic_mode, CTLFLAG_RD, &x2apic_mode, 0, "");
 194 SYSCTL_INT(_hw_apic, OID_AUTO, eoi_suppression, CTLFLAG_RD,
 195     &lapic_eoi_suppression, 0, "");
 196 SYSCTL_INT(_hw_apic, OID_AUTO, timer_tsc_deadline, CTLFLAG_RD,
 197     &lapic_timer_tsc_deadline, 0, "");
 198
 199 static uint32_t
 200 lapic_read32(enum LAPIC_REGISTERS reg)
 201 {
 202         uint32_t res;
 203
 204         if (x2apic_mode) {
 205                 res = rdmsr32(MSR_APIC_000 + reg);
 206         } else {
 207                 res = *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL);
 208         }
 209         return (res);
 210 }
 211
 212 static void
 213 lapic_write32(enum LAPIC_REGISTERS reg, uint32_t val)
 214 {
 215
 216         if (x2apic_mode) {
 217                 mfence();
 218                 wrmsr(MSR_APIC_000 + reg, val);
 219         } else {
 220                 *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL) = val;
 221         }
 222 }
 223
 224 static void
 225 lapic_write32_nofence(enum LAPIC_REGISTERS reg, uint32_t val)
 226 {
 227
 228         if (x2apic_mode) {
 229                 wrmsr(MSR_APIC_000 + reg, val);
 230         } else {
 231                 *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL) = val;
 232         }
 233 }
 234
 235 #ifdef SMP
 236 static uint64_t
 237 lapic_read_icr(void)
 238 {
 239         uint64_t v;
 240         uint32_t vhi, vlo;
 241
 242         if (x2apic_mode) {
 243                 v = rdmsr(MSR_APIC_000 + LAPIC_ICR_LO);
 244         } else {
 245                 vhi = lapic_read32(LAPIC_ICR_HI);
 246                 vlo = lapic_read32(LAPIC_ICR_LO);
 247                 v = ((uint64_t)vhi << 32) | vlo;
 248         }
 249         return (v);
 250 }
 251
 252 static uint64_t
 253 lapic_read_icr_lo(void)
 254 {
 255
 256         return (lapic_read32(LAPIC_ICR_LO));
 257 }
 258
 259 static void
 260 lapic_write_icr(uint32_t vhi, uint32_t vlo)
 261 {
 262         uint64_t v;
 263
 264         if (x2apic_mode) {
 265                 v = ((uint64_t)vhi << 32) | vlo;
 266                 mfence();
 267                 wrmsr(MSR_APIC_000 + LAPIC_ICR_LO, v);
 268         } else {
 269                 lapic_write32(LAPIC_ICR_HI, vhi);
 270                 lapic_write32(LAPIC_ICR_LO, vlo);
 271         }
 272 }
 273 #endif /* SMP */
 274
 275 static void
 276 native_lapic_enable_x2apic(void)
 277 {
 278         uint64_t apic_base;
 279
 280         apic_base = rdmsr(MSR_APICBASE);
 281         apic_base |= APICBASE_X2APIC | APICBASE_ENABLED;
 282         wrmsr(MSR_APICBASE, apic_base);
 283 }
 284
 285 static bool
 286 native_lapic_is_x2apic(void)
 287 {
 288         uint64_t apic_base;
 289
 290         apic_base = rdmsr(MSR_APICBASE);
 291         return ((apic_base & (APICBASE_X2APIC | APICBASE_ENABLED)) ==
 292             (APICBASE_X2APIC | APICBASE_ENABLED));
 293 }
 294
 295 static void     lapic_enable(void);
 296 static void     lapic_resume(struct pic *pic, bool suspend_cancelled);
 297 static void     lapic_timer_oneshot(struct lapic *);
 298 static void     lapic_timer_oneshot_nointr(struct lapic *, uint32_t);
 299 static void     lapic_timer_periodic(struct lapic *);
 300 static void     lapic_timer_deadline(struct lapic *);
 301 static void     lapic_timer_stop(struct lapic *);
 302 static void     lapic_timer_set_divisor(u_int divisor);
 303 static uint32_t lvt_mode(struct lapic *la, u_int pin, uint32_t value);
 304 static int      lapic_et_start(struct eventtimer *et,
 305                     sbintime_t first, sbintime_t period);
 306 static int      lapic_et_stop(struct eventtimer *et);
 307 static u_int    apic_idt_to_irq(u_int apic_id, u_int vector);
 308 static void     lapic_set_tpr(u_int vector);
 309
 310 struct pic lapic_pic = { .pic_resume = lapic_resume };
 311
 312 /* Forward declarations for apic_ops */
 313 static void     native_lapic_create(u_int apic_id, int boot_cpu);
 314 static void     native_lapic_init(vm_paddr_t addr);
 315 static void     native_lapic_xapic_mode(void);
 316 static void     native_lapic_setup(int boot);
 317 static void     native_lapic_dump(const char *str);
 318 static void     native_lapic_disable(void);
 319 static void     native_lapic_eoi(void);
 320 static int      native_lapic_id(void);
 321 static int      native_lapic_intr_pending(u_int vector);
 322 static u_int    native_apic_cpuid(u_int apic_id);
 323 static u_int    native_apic_alloc_vector(u_int apic_id, u_int irq);
 324 static u_int    native_apic_alloc_vectors(u_int apic_id, u_int *irqs,
 325                     u_int count, u_int align);
 326 static void     native_apic_disable_vector(u_int apic_id, u_int vector);
 327 static void     native_apic_enable_vector(u_int apic_id, u_int vector);
 328 static void     native_apic_free_vector(u_int apic_id, u_int vector, u_int irq);
 329 static void     native_lapic_set_logical_id(u_int apic_id, u_int cluster,
 330                     u_int cluster_id);
 331 static int      native_lapic_enable_pmc(void);
 332 static void     native_lapic_disable_pmc(void);
 333 static void     native_lapic_reenable_pmc(void);
 334 static void     native_lapic_enable_cmc(void);
 335 static int      native_lapic_enable_mca_elvt(void);
 336 static int      native_lapic_set_lvt_mask(u_int apic_id, u_int lvt,
 337                     u_char masked);
 338 static int      native_lapic_set_lvt_mode(u_int apic_id, u_int lvt,
 339                     uint32_t mode);
 340 static int      native_lapic_set_lvt_polarity(u_int apic_id, u_int lvt,
 341                     enum intr_polarity pol);
 342 static int      native_lapic_set_lvt_triggermode(u_int apic_id, u_int lvt,
 343                     enum intr_trigger trigger);
 344 #ifdef SMP
 345 static void     native_lapic_ipi_raw(register_t icrlo, u_int dest);
 346 static void     native_lapic_ipi_vectored(u_int vector, int dest);
 347 static int      native_lapic_ipi_wait(int delay);
 348 #endif /* SMP */
 349 static int      native_lapic_ipi_alloc(inthand_t *ipifunc);
 350 static void     native_lapic_ipi_free(int vector);
 351
 352 struct apic_ops apic_ops = {
 353         .create                 = native_lapic_create,
 354         .init                   = native_lapic_init,
 355         .xapic_mode             = native_lapic_xapic_mode,
 356         .is_x2apic              = native_lapic_is_x2apic,
 357         .setup                  = native_lapic_setup,
 358         .dump                   = native_lapic_dump,
 359         .disable                = native_lapic_disable,
 360         .eoi                    = native_lapic_eoi,
 361         .id                     = native_lapic_id,
 362         .intr_pending           = native_lapic_intr_pending,
 363         .set_logical_id         = native_lapic_set_logical_id,
 364         .cpuid                  = native_apic_cpuid,
 365         .alloc_vector           = native_apic_alloc_vector,
 366         .alloc_vectors          = native_apic_alloc_vectors,
 367         .enable_vector          = native_apic_enable_vector,
 368         .disable_vector         = native_apic_disable_vector,
 369         .free_vector            = native_apic_free_vector,
 370         .enable_pmc             = native_lapic_enable_pmc,
 371         .disable_pmc            = native_lapic_disable_pmc,
 372         .reenable_pmc           = native_lapic_reenable_pmc,
 373         .enable_cmc             = native_lapic_enable_cmc,
 374         .enable_mca_elvt        = native_lapic_enable_mca_elvt,
 375 #ifdef SMP
 376         .ipi_raw                = native_lapic_ipi_raw,
 377         .ipi_vectored           = native_lapic_ipi_vectored,
 378         .ipi_wait               = native_lapic_ipi_wait,
 379 #endif
 380         .ipi_alloc              = native_lapic_ipi_alloc,
 381         .ipi_free               = native_lapic_ipi_free,
 382         .set_lvt_mask           = native_lapic_set_lvt_mask,
 383         .set_lvt_mode           = native_lapic_set_lvt_mode,
 384         .set_lvt_polarity       = native_lapic_set_lvt_polarity,
 385         .set_lvt_triggermode    = native_lapic_set_lvt_triggermode,
 386 };
 387
 388 static uint32_t
 389 lvt_mode_impl(struct lapic *la, struct lvt *lvt, u_int pin, uint32_t value)
 390 {
 391
 392         value &= ~(APIC_LVT_M | APIC_LVT_TM | APIC_LVT_IIPP | APIC_LVT_DM |
 393             APIC_LVT_VECTOR);
 394         if (lvt->lvt_edgetrigger == 0)
 395                 value |= APIC_LVT_TM;
 396         if (lvt->lvt_activehi == 0)
 397                 value |= APIC_LVT_IIPP_INTALO;
 398         if (lvt->lvt_masked)
 399                 value |= APIC_LVT_M;
 400         value |= lvt->lvt_mode;
 401         switch (lvt->lvt_mode) {
 402         case APIC_LVT_DM_NMI:
 403         case APIC_LVT_DM_SMI:
 404         case APIC_LVT_DM_INIT:
 405         case APIC_LVT_DM_EXTINT:
 406                 if (!lvt->lvt_edgetrigger && bootverbose) {
 407                         printf("lapic%u: Forcing LINT%u to edge trigger\n",
 408                             la->la_id, pin);
 409                         value &= ~APIC_LVT_TM;
 410                 }
 411                 /* Use a vector of 0. */
 412                 break;
 413         case APIC_LVT_DM_FIXED:
 414                 value |= lvt->lvt_vector;
 415                 break;
 416         default:
 417                 panic("bad APIC LVT delivery mode: %#x\n", value);
 418         }
 419         return (value);
 420 }
 421
 422 static uint32_t
 423 lvt_mode(struct lapic *la, u_int pin, uint32_t value)
 424 {
 425         struct lvt *lvt;
 426
 427         KASSERT(pin <= APIC_LVT_MAX,
 428             ("%s: pin %u out of range", __func__, pin));
 429         if (la->la_lvts[pin].lvt_active)
 430                 lvt = &la->la_lvts[pin];
 431         else
 432                 lvt = &lvts[pin];
 433
 434         return (lvt_mode_impl(la, lvt, pin, value));
 435 }
 436
 437 static uint32_t
 438 elvt_mode(struct lapic *la, u_int idx, uint32_t value)
 439 {
 440         struct lvt *elvt;
 441
 442         KASSERT(idx <= APIC_ELVT_MAX,
 443             ("%s: idx %u out of range", __func__, idx));
 444
 445         elvt = &la->la_elvts[idx];
 446         KASSERT(elvt->lvt_active, ("%s: ELVT%u is not active", __func__, idx));
 447         KASSERT(elvt->lvt_edgetrigger,
 448             ("%s: ELVT%u is not edge triggered", __func__, idx));
 449         KASSERT(elvt->lvt_activehi,
 450             ("%s: ELVT%u is not active high", __func__, idx));
 451         return (lvt_mode_impl(la, elvt, idx, value));
 452 }
 453
 454 /*
 455  * Map the local APIC and setup necessary interrupt vectors.
 456  */
 457 static void
 458 native_lapic_init(vm_paddr_t addr)
 459 {
 460 #ifdef SMP
 461         uint64_t r, r1, r2, rx;
 462 #endif
 463         uint32_t ver;
 464         u_int regs[4];
 465         int i, arat;
 466
 467         /*
 468          * Enable x2APIC mode if possible. Map the local APIC
 469          * registers page.
 470          *
 471          * Keep the LAPIC registers page mapped uncached for x2APIC
 472          * mode too, to have direct map page attribute set to
 473          * uncached.  This is needed to work around CPU errata present
 474          * on all Intel processors.
 475          */
 476         KASSERT(trunc_page(addr) == addr,
 477             ("local APIC not aligned on a page boundary"));
 478         lapic_paddr = addr;
 479         lapic_map = pmap_mapdev(addr, PAGE_SIZE);
 480         if (x2apic_mode) {
 481                 native_lapic_enable_x2apic();
 482                 lapic_map = NULL;
 483         }
 484
 485         /* Setup the spurious interrupt handler. */
 486         setidt(APIC_SPURIOUS_INT, IDTVEC(spuriousint), SDT_APIC, SEL_KPL,
 487             GSEL_APIC);
 488
 489         /* Perform basic initialization of the BSP's local APIC. */
 490         lapic_enable();
 491
 492         /* Set BSP's per-CPU local APIC ID. */
 493         PCPU_SET(apic_id, lapic_id());
 494
 495         /* Local APIC timer interrupt. */
 496         setidt(APIC_TIMER_INT, IDTVEC(timerint), SDT_APIC, SEL_KPL, GSEL_APIC);
 497
 498         /* Local APIC error interrupt. */
 499         setidt(APIC_ERROR_INT, IDTVEC(errorint), SDT_APIC, SEL_KPL, GSEL_APIC);
 500
 501         /* XXX: Thermal interrupt */
 502
 503         /* Local APIC CMCI. */
 504         setidt(APIC_CMC_INT, IDTVEC(cmcint), SDT_APICT, SEL_KPL, GSEL_APIC);
 505
 506         if ((resource_int_value("apic", 0, "clock", &i) != 0 || i != 0)) {
 507                 arat = 0;
 508                 /* Intel CPUID 0x06 EAX[2] set if APIC timer runs in C3. */
 509                 if (cpu_vendor_id == CPU_VENDOR_INTEL && cpu_high >= 6) {
 510                         do_cpuid(0x06, regs);
 511                         if ((regs[0] & CPUTPM1_ARAT) != 0)
 512                                 arat = 1;
 513                 }
 514                 bzero(&lapic_et, sizeof(lapic_et));
 515                 lapic_et.et_name = "LAPIC";
 516                 lapic_et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT |
 517                     ET_FLAGS_PERCPU;
 518                 lapic_et.et_quality = 600;
 519                 if (!arat) {
 520                         lapic_et.et_flags |= ET_FLAGS_C3STOP;
 521                         lapic_et.et_quality = 100;
 522                 }
 523                 if ((cpu_feature & CPUID_TSC) != 0 &&
 524                     (cpu_feature2 & CPUID2_TSCDLT) != 0 &&
 525                     tsc_is_invariant && tsc_freq != 0) {
 526                         lapic_timer_tsc_deadline = 1;
 527                         TUNABLE_INT_FETCH("hw.lapic_tsc_deadline",
 528                             &lapic_timer_tsc_deadline);
 529                 }
 530
 531                 lapic_et.et_frequency = 0;
 532                 /* We don't know frequency yet, so trying to guess. */
 533                 lapic_et.et_min_period = 0x00001000LL;
 534                 lapic_et.et_max_period = SBT_1S;
 535                 lapic_et.et_start = lapic_et_start;
 536                 lapic_et.et_stop = lapic_et_stop;
 537                 lapic_et.et_priv = NULL;
 538                 et_register(&lapic_et);
 539         }
 540
 541         /*
 542          * Set lapic_eoi_suppression after lapic_enable(), to not
 543          * enable suppression in the hardware prematurely.  Note that
 544          * we by default enable suppression even when system only has
 545          * one IO-APIC, since EOI is broadcasted to all APIC agents,
 546          * including CPUs, otherwise.
 547          *
 548          * It seems that at least some KVM versions report
 549          * EOI_SUPPRESSION bit, but auto-EOI does not work.
 550          */
 551         ver = lapic_read32(LAPIC_VERSION);
 552         if ((ver & APIC_VER_EOI_SUPPRESSION) != 0) {
 553                 lapic_eoi_suppression = 1;
 554                 if (vm_guest == VM_GUEST_KVM) {
 555                         if (bootverbose)
 556                                 printf(
 557                        "KVM -- disabling lapic eoi suppression\n");
 558                         lapic_eoi_suppression = 0;
 559                 }
 560                 TUNABLE_INT_FETCH("hw.lapic_eoi_suppression",
 561                     &lapic_eoi_suppression);
 562         }
 563
 564 #ifdef SMP
 565 #define LOOPS   100000
 566         /*
 567          * Calibrate the busy loop waiting for IPI ack in xAPIC mode.
 568          * lapic_ipi_wait_mult contains the number of iterations which
 569          * approximately delay execution for 1 microsecond (the
 570          * argument to native_lapic_ipi_wait() is in microseconds).
 571          *
 572          * We assume that TSC is present and already measured.
 573          * Possible TSC frequency jumps are irrelevant to the
 574          * calibration loop below, the CPU clock management code is
 575          * not yet started, and we do not enter sleep states.
 576          */
 577         KASSERT((cpu_feature & CPUID_TSC) != 0 && tsc_freq != 0,
 578             ("TSC not initialized"));
 579         if (!x2apic_mode) {
 580                 r = rdtsc();
 581                 for (rx = 0; rx < LOOPS; rx++) {
 582                         (void)lapic_read_icr_lo();
 583                         ia32_pause();
 584                 }
 585                 r = rdtsc() - r;
 586                 r1 = tsc_freq * LOOPS;
 587                 r2 = r * 1000000;
 588                 lapic_ipi_wait_mult = r1 >= r2 ? r1 / r2 : 1;
 589                 if (bootverbose) {
 590                         printf("LAPIC: ipi_wait() us multiplier %ju (r %ju "
 591                             "tsc %ju)\n", (uintmax_t)lapic_ipi_wait_mult,
 592                             (uintmax_t)r, (uintmax_t)tsc_freq);
 593                 }
 594         }
 595 #undef LOOPS
 596 #endif /* SMP */
 597 }
 598
 599 /*
 600  * Create a local APIC instance.
 601  */
 602 static void
 603 native_lapic_create(u_int apic_id, int boot_cpu)
 604 {
 605         int i;
 606
 607         if (apic_id > max_apic_id) {
 608                 printf("APIC: Ignoring local APIC with ID %d\n", apic_id);
 609                 if (boot_cpu)
 610                         panic("Can't ignore BSP");
 611                 return;
 612         }
 613         KASSERT(!lapics[apic_id].la_present, ("duplicate local APIC %u",
 614             apic_id));
 615
 616         /*
 617          * Assume no local LVT overrides and a cluster of 0 and
 618          * intra-cluster ID of 0.
 619          */
 620         lapics[apic_id].la_present = 1;
 621         lapics[apic_id].la_id = apic_id;
 622         for (i = 0; i <= APIC_LVT_MAX; i++) {
 623                 lapics[apic_id].la_lvts[i] = lvts[i];
 624                 lapics[apic_id].la_lvts[i].lvt_active = 0;
 625         }
 626         for (i = 0; i <= APIC_ELVT_MAX; i++) {
 627                 lapics[apic_id].la_elvts[i] = elvts[i];
 628                 lapics[apic_id].la_elvts[i].lvt_active = 0;
 629         }
 630         for (i = 0; i <= APIC_NUM_IOINTS; i++)
 631             lapics[apic_id].la_ioint_irqs[i] = -1;
 632         lapics[apic_id].la_ioint_irqs[IDT_SYSCALL - APIC_IO_INTS] = IRQ_SYSCALL;
 633         lapics[apic_id].la_ioint_irqs[APIC_TIMER_INT - APIC_IO_INTS] =
 634             IRQ_TIMER;
 635 #ifdef KDTRACE_HOOKS
 636         lapics[apic_id].la_ioint_irqs[IDT_DTRACE_RET - APIC_IO_INTS] =
 637             IRQ_DTRACE_RET;
 638 #endif
 639 #ifdef XENHVM
 640         lapics[apic_id].la_ioint_irqs[IDT_EVTCHN - APIC_IO_INTS] = IRQ_EVTCHN;
 641 #endif
 642
 643
 644 #ifdef SMP
 645         cpu_add(apic_id, boot_cpu);
 646 #endif
 647 }
 648
 649 static inline uint32_t
 650 amd_read_ext_features(void)
 651 {
 652         uint32_t version;
 653
 654         if (cpu_vendor_id != CPU_VENDOR_AMD)
 655                 return (0);
 656         version = lapic_read32(LAPIC_VERSION);
 657         if ((version & APIC_VER_AMD_EXT_SPACE) != 0)
 658                 return (lapic_read32(LAPIC_EXT_FEATURES));
 659         else
 660                 return (0);
 661 }
 662
 663 static inline uint32_t
 664 amd_read_elvt_count(void)
 665 {
 666         uint32_t extf;
 667         uint32_t count;
 668
 669         extf = amd_read_ext_features();
 670         count = (extf & APIC_EXTF_ELVT_MASK) >> APIC_EXTF_ELVT_SHIFT;
 671         count = min(count, APIC_ELVT_MAX + 1);
 672         return (count);
 673 }
 674
 675 /*
 676  * Dump contents of local APIC registers
 677  */
 678 static void
 679 native_lapic_dump(const char* str)
 680 {
 681         uint32_t version;
 682         uint32_t maxlvt;
 683         uint32_t extf;
 684         int elvt_count;
 685         int i;
 686
 687         version = lapic_read32(LAPIC_VERSION);
 688         maxlvt = (version & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
 689         printf("cpu%d %s:\n", PCPU_GET(cpuid), str);
 690         printf("     ID: 0x%08x   VER: 0x%08x LDR: 0x%08x DFR: 0x%08x",
 691             lapic_read32(LAPIC_ID), version,
 692             lapic_read32(LAPIC_LDR), x2apic_mode ? 0 : lapic_read32(LAPIC_DFR));
 693         if ((cpu_feature2 & CPUID2_X2APIC) != 0)
 694                 printf(" x2APIC: %d", x2apic_mode);
 695         printf("\n  lint0: 0x%08x lint1: 0x%08x TPR: 0x%08x SVR: 0x%08x\n",
 696             lapic_read32(LAPIC_LVT_LINT0), lapic_read32(LAPIC_LVT_LINT1),
 697             lapic_read32(LAPIC_TPR), lapic_read32(LAPIC_SVR));
 698         printf("  timer: 0x%08x therm: 0x%08x err: 0x%08x",
 699             lapic_read32(LAPIC_LVT_TIMER), lapic_read32(LAPIC_LVT_THERMAL),
 700             lapic_read32(LAPIC_LVT_ERROR));
 701         if (maxlvt >= APIC_LVT_PMC)
 702                 printf(" pmc: 0x%08x", lapic_read32(LAPIC_LVT_PCINT));
 703         printf("\n");
 704         if (maxlvt >= APIC_LVT_CMCI)
 705                 printf("   cmci: 0x%08x\n", lapic_read32(LAPIC_LVT_CMCI));
 706         extf = amd_read_ext_features();
 707         if (extf != 0) {
 708                 printf("   AMD ext features: 0x%08x\n", extf);
 709                 elvt_count = amd_read_elvt_count();
 710                 for (i = 0; i < elvt_count; i++)
 711                         printf("   AMD elvt%d: 0x%08x\n", i,
 712                             lapic_read32(LAPIC_EXT_LVT0 + i));
 713         }
 714 }
 715
 716 static void
 717 native_lapic_xapic_mode(void)
 718 {
 719         register_t saveintr;
 720
 721         saveintr = intr_disable();
 722         if (x2apic_mode)
 723                 native_lapic_enable_x2apic();
 724         intr_restore(saveintr);
 725 }
 726
 727 static void
 728 native_lapic_setup(int boot)
 729 {
 730         struct lapic *la;
 731         uint32_t version;
 732         uint32_t maxlvt;
 733         register_t saveintr;
 734         char buf[MAXCOMLEN + 1];
 735         int elvt_count;
 736         int i;
 737
 738         saveintr = intr_disable();
 739
 740         la = &lapics[lapic_id()];
 741         KASSERT(la->la_present, ("missing APIC structure"));
 742         version = lapic_read32(LAPIC_VERSION);
 743         maxlvt = (version & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
 744
 745         /* Initialize the TPR to allow all interrupts. */
 746         lapic_set_tpr(0);
 747
 748         /* Setup spurious vector and enable the local APIC. */
 749         lapic_enable();
 750
 751         /* Program LINT[01] LVT entries. */
 752         lapic_write32(LAPIC_LVT_LINT0, lvt_mode(la, APIC_LVT_LINT0,
 753             lapic_read32(LAPIC_LVT_LINT0)));
 754         lapic_write32(LAPIC_LVT_LINT1, lvt_mode(la, APIC_LVT_LINT1,
 755             lapic_read32(LAPIC_LVT_LINT1)));
 756
 757         /* Program the PMC LVT entry if present. */
 758         if (maxlvt >= APIC_LVT_PMC) {
 759                 lapic_write32(LAPIC_LVT_PCINT, lvt_mode(la, APIC_LVT_PMC,
 760                     LAPIC_LVT_PCINT));
 761         }
 762
 763         /* Program timer LVT and setup handler. */
 764         la->lvt_timer_base = lvt_mode(la, APIC_LVT_TIMER,
 765             lapic_read32(LAPIC_LVT_TIMER));
 766         la->lvt_timer_last = la->lvt_timer_base;
 767         lapic_write32(LAPIC_LVT_TIMER, la->lvt_timer_base);
 768         if (boot) {
 769                 snprintf(buf, sizeof(buf), "cpu%d:timer", PCPU_GET(cpuid));
 770                 intrcnt_add(buf, &la->la_timer_count);
 771         }
 772
 773         /* Setup the timer if configured. */
 774         if (la->la_timer_mode != LAT_MODE_UNDEF) {
 775                 KASSERT(la->la_timer_period != 0, ("lapic%u: zero divisor",
 776                     lapic_id()));
 777                 switch (la->la_timer_mode) {
 778                 case LAT_MODE_PERIODIC:
 779                         lapic_timer_set_divisor(lapic_timer_divisor);
 780                         lapic_timer_periodic(la);
 781                         break;
 782                 case LAT_MODE_ONESHOT:
 783                         lapic_timer_set_divisor(lapic_timer_divisor);
 784                         lapic_timer_oneshot(la);
 785                         break;
 786                 case LAT_MODE_DEADLINE:
 787                         lapic_timer_deadline(la);
 788                         break;
 789                 default:
 790                         panic("corrupted la_timer_mode %p %d", la,
 791                             la->la_timer_mode);
 792                 }
 793         }
 794
 795         /* Program error LVT and clear any existing errors. */
 796         lapic_write32(LAPIC_LVT_ERROR, lvt_mode(la, APIC_LVT_ERROR,
 797             lapic_read32(LAPIC_LVT_ERROR)));
 798         lapic_write32(LAPIC_ESR, 0);
 799
 800         /* XXX: Thermal LVT */
 801
 802         /* Program the CMCI LVT entry if present. */
 803         if (maxlvt >= APIC_LVT_CMCI) {
 804                 lapic_write32(LAPIC_LVT_CMCI, lvt_mode(la, APIC_LVT_CMCI,
 805                     lapic_read32(LAPIC_LVT_CMCI)));
 806         }
 807
 808         elvt_count = amd_read_elvt_count();
 809         for (i = 0; i < elvt_count; i++) {
 810                 if (la->la_elvts[i].lvt_active)
 811                         lapic_write32(LAPIC_EXT_LVT0 + i,
 812                             elvt_mode(la, i, lapic_read32(LAPIC_EXT_LVT0 + i)));
 813         }
 814
 815         intr_restore(saveintr);
 816 }
 817
 818 static void
 819 native_lapic_reenable_pmc(void)
 820 {
 821 #ifdef HWPMC_HOOKS
 822         uint32_t value;
 823
 824         value = lapic_read32(LAPIC_LVT_PCINT);
 825         value &= ~APIC_LVT_M;
 826         lapic_write32(LAPIC_LVT_PCINT, value);
 827 #endif
 828 }
 829
 830 #ifdef HWPMC_HOOKS
 831 static void
 832 lapic_update_pmc(void *dummy)
 833 {
 834         struct lapic *la;
 835
 836         la = &lapics[lapic_id()];
 837         lapic_write32(LAPIC_LVT_PCINT, lvt_mode(la, APIC_LVT_PMC,
 838             lapic_read32(LAPIC_LVT_PCINT)));
 839 }
 840 #endif
 841
 842 static int
 843 native_lapic_enable_pmc(void)
 844 {
 845 #ifdef HWPMC_HOOKS
 846         u_int32_t maxlvt;
 847
 848         /* Fail if the local APIC is not present. */
 849         if (!x2apic_mode && lapic_map == NULL)
 850                 return (0);
 851
 852         /* Fail if the PMC LVT is not present. */
 853         maxlvt = (lapic_read32(LAPIC_VERSION) & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
 854         if (maxlvt < APIC_LVT_PMC)
 855                 return (0);
 856
 857         lvts[APIC_LVT_PMC].lvt_masked = 0;
 858
 859 #ifdef EARLY_AP_STARTUP
 860         MPASS(mp_ncpus == 1 || smp_started);
 861         smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL);
 862 #else
 863 #ifdef SMP
 864         /*
 865          * If hwpmc was loaded at boot time then the APs may not be
 866          * started yet.  In that case, don't forward the request to
 867          * them as they will program the lvt when they start.
 868          */
 869         if (smp_started)
 870                 smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL);
 871         else
 872 #endif
 873                 lapic_update_pmc(NULL);
 874 #endif
 875         return (1);
 876 #else
 877         return (0);
 878 #endif
 879 }
 880
 881 static void
 882 native_lapic_disable_pmc(void)
 883 {
 884 #ifdef HWPMC_HOOKS
 885         u_int32_t maxlvt;
 886
 887         /* Fail if the local APIC is not present. */
 888         if (!x2apic_mode && lapic_map == NULL)
 889                 return;
 890
 891         /* Fail if the PMC LVT is not present. */
 892         maxlvt = (lapic_read32(LAPIC_VERSION) & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
 893         if (maxlvt < APIC_LVT_PMC)
 894                 return;
 895
 896         lvts[APIC_LVT_PMC].lvt_masked = 1;
 897
 898 #ifdef SMP
 899         /* The APs should always be started when hwpmc is unloaded. */
 900         KASSERT(mp_ncpus == 1 || smp_started, ("hwpmc unloaded too early"));
 901 #endif
 902         smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL);
 903 #endif
 904 }
 905
 906 static void
 907 lapic_calibrate_initcount(struct eventtimer *et, struct lapic *la)
 908 {
 909         u_long value;
 910
 911         /* Start off with a divisor of 2 (power on reset default). */
 912         lapic_timer_divisor = 2;
 913         /* Try to calibrate the local APIC timer. */
 914         do {
 915                 lapic_timer_set_divisor(lapic_timer_divisor);
 916                 lapic_timer_oneshot_nointr(la, APIC_TIMER_MAX_COUNT);
 917                 DELAY(1000000);
 918                 value = APIC_TIMER_MAX_COUNT - lapic_read32(LAPIC_CCR_TIMER);
 919                 if (value != APIC_TIMER_MAX_COUNT)
 920                         break;
 921                 lapic_timer_divisor <<= 1;
 922         } while (lapic_timer_divisor <= 128);
 923         if (lapic_timer_divisor > 128)
 924                 panic("lapic: Divisor too big");
 925         if (bootverbose) {
 926                 printf("lapic: Divisor %lu, Frequency %lu Hz\n",
 927                     lapic_timer_divisor, value);
 928         }
 929         count_freq = value;
 930 }
 931
 932 static void
 933 lapic_calibrate_deadline(struct eventtimer *et, struct lapic *la __unused)
 934 {
 935
 936         if (bootverbose) {
 937                 printf("lapic: deadline tsc mode, Frequency %ju Hz\n",
 938                     (uintmax_t)tsc_freq);
 939         }
 940 }
 941
 942 static void
 943 lapic_change_mode(struct eventtimer *et, struct lapic *la,
 944     enum lat_timer_mode newmode)
 945 {
 946
 947         if (la->la_timer_mode == newmode)
 948                 return;
 949         switch (newmode) {
 950         case LAT_MODE_PERIODIC:
 951                 lapic_timer_set_divisor(lapic_timer_divisor);
 952                 et->et_frequency = count_freq;
 953                 break;
 954         case LAT_MODE_DEADLINE:
 955                 et->et_frequency = tsc_freq;
 956                 break;
 957         case LAT_MODE_ONESHOT:
 958                 lapic_timer_set_divisor(lapic_timer_divisor);
 959                 et->et_frequency = count_freq;
 960                 break;
 961         default:
 962                 panic("lapic_change_mode %d", newmode);
 963         }
 964         la->la_timer_mode = newmode;
 965         et->et_min_period = (0x00000002LLU << 32) / et->et_frequency;
 966         et->et_max_period = (0xfffffffeLLU << 32) / et->et_frequency;
 967 }
 968
 969 static int
 970 lapic_et_start(struct eventtimer *et, sbintime_t first, sbintime_t period)
 971 {
 972         struct lapic *la;
 973
 974         la = &lapics[PCPU_GET(apic_id)];
 975         if (et->et_frequency == 0) {
 976                 lapic_calibrate_initcount(et, la);
 977                 if (lapic_timer_tsc_deadline)
 978                         lapic_calibrate_deadline(et, la);
 979         }
 980         if (period != 0) {
 981                 lapic_change_mode(et, la, LAT_MODE_PERIODIC);
 982                 la->la_timer_period = ((uint32_t)et->et_frequency * period) >>
 983                     32;
 984                 lapic_timer_periodic(la);
 985         } else if (lapic_timer_tsc_deadline) {
 986                 lapic_change_mode(et, la, LAT_MODE_DEADLINE);
 987                 la->la_timer_period = (et->et_frequency * first) >> 32;
 988                 lapic_timer_deadline(la);
 989         } else {
 990                 lapic_change_mode(et, la, LAT_MODE_ONESHOT);
 991                 la->la_timer_period = ((uint32_t)et->et_frequency * first) >>
 992                     32;
 993                 lapic_timer_oneshot(la);
 994         }
 995         return (0);
 996 }
 997
 998 static int
 999 lapic_et_stop(struct eventtimer *et)
1000 {
1001         struct lapic *la;
1002
1003         la = &lapics[PCPU_GET(apic_id)];
1004         lapic_timer_stop(la);
1005         la->la_timer_mode = LAT_MODE_UNDEF;
1006         return (0);
1007 }
1008
1009 static void
1010 native_lapic_disable(void)
1011 {
1012         uint32_t value;
1013
1014         /* Software disable the local APIC. */
1015         value = lapic_read32(LAPIC_SVR);
1016         value &= ~APIC_SVR_SWEN;
1017         lapic_write32(LAPIC_SVR, value);
1018 }
1019
1020 static void
1021 lapic_enable(void)
1022 {
1023         uint32_t value;
1024
1025         /* Program the spurious vector to enable the local APIC. */
1026         value = lapic_read32(LAPIC_SVR);
1027         value &= ~(APIC_SVR_VECTOR | APIC_SVR_FOCUS);
1028         value |= APIC_SVR_FEN | APIC_SVR_SWEN | APIC_SPURIOUS_INT;
1029         if (lapic_eoi_suppression)
1030                 value |= APIC_SVR_EOI_SUPPRESSION;
1031         lapic_write32(LAPIC_SVR, value);
1032 }
1033
1034 /* Reset the local APIC on the BSP during resume. */
1035 static void
1036 lapic_resume(struct pic *pic, bool suspend_cancelled)
1037 {
1038
1039         lapic_setup(0);
1040 }
1041
1042 static int
1043 native_lapic_id(void)
1044 {
1045         uint32_t v;
1046
1047         KASSERT(x2apic_mode || lapic_map != NULL, ("local APIC is not mapped"));
1048         v = lapic_read32(LAPIC_ID);
1049         if (!x2apic_mode)
1050                 v >>= APIC_ID_SHIFT;
1051         return (v);
1052 }
1053
1054 static int
1055 native_lapic_intr_pending(u_int vector)
1056 {
1057         uint32_t irr;
1058
1059         /*
1060          * The IRR registers are an array of registers each of which
1061          * only describes 32 interrupts in the low 32 bits.  Thus, we
1062          * divide the vector by 32 to get the register index.
1063          * Finally, we modulus the vector by 32 to determine the
1064          * individual bit to test.
1065          */
1066         irr = lapic_read32(LAPIC_IRR0 + vector / 32);
1067         return (irr & 1 << (vector % 32));
1068 }
1069
1070 static void
1071 native_lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id)
1072 {
1073         struct lapic *la;
1074
1075         KASSERT(lapics[apic_id].la_present, ("%s: APIC %u doesn't exist",
1076             __func__, apic_id));
1077         KASSERT(cluster <= APIC_MAX_CLUSTER, ("%s: cluster %u too big",
1078             __func__, cluster));
1079         KASSERT(cluster_id <= APIC_MAX_INTRACLUSTER_ID,
1080             ("%s: intra cluster id %u too big", __func__, cluster_id));
1081         la = &lapics[apic_id];
1082         la->la_cluster = cluster;
1083         la->la_cluster_id = cluster_id;
1084 }
1085
1086 static int
1087 native_lapic_set_lvt_mask(u_int apic_id, u_int pin, u_char masked)
1088 {
1089
1090         if (pin > APIC_LVT_MAX)
1091                 return (EINVAL);
1092         if (apic_id == APIC_ID_ALL) {
1093                 lvts[pin].lvt_masked = masked;
1094                 if (bootverbose)
1095                         printf("lapic:");
1096         } else {
1097                 KASSERT(lapics[apic_id].la_present,
1098                     ("%s: missing APIC %u", __func__, apic_id));
1099                 lapics[apic_id].la_lvts[pin].lvt_masked = masked;
1100                 lapics[apic_id].la_lvts[pin].lvt_active = 1;
1101                 if (bootverbose)
1102                         printf("lapic%u:", apic_id);
1103         }
1104         if (bootverbose)
1105                 printf(" LINT%u %s\n", pin, masked ? "masked" : "unmasked");
1106         return (0);
1107 }
1108
1109 static int
1110 native_lapic_set_lvt_mode(u_int apic_id, u_int pin, u_int32_t mode)
1111 {
1112         struct lvt *lvt;
1113
1114         if (pin > APIC_LVT_MAX)
1115                 return (EINVAL);
1116         if (apic_id == APIC_ID_ALL) {
1117                 lvt = &lvts[pin];
1118                 if (bootverbose)
1119                         printf("lapic:");
1120         } else {
1121                 KASSERT(lapics[apic_id].la_present,
1122                     ("%s: missing APIC %u", __func__, apic_id));
1123                 lvt = &lapics[apic_id].la_lvts[pin];
1124                 lvt->lvt_active = 1;
1125                 if (bootverbose)
1126                         printf("lapic%u:", apic_id);
1127         }
1128         lvt->lvt_mode = mode;
1129         switch (mode) {
1130         case APIC_LVT_DM_NMI:
1131         case APIC_LVT_DM_SMI:
1132         case APIC_LVT_DM_INIT:
1133         case APIC_LVT_DM_EXTINT:
1134                 lvt->lvt_edgetrigger = 1;
1135                 lvt->lvt_activehi = 1;
1136                 if (mode == APIC_LVT_DM_EXTINT)
1137                         lvt->lvt_masked = 1;
1138                 else
1139                         lvt->lvt_masked = 0;
1140                 break;
1141         default:
1142                 panic("Unsupported delivery mode: 0x%x\n", mode);
1143         }
1144         if (bootverbose) {
1145                 printf(" Routing ");
1146                 switch (mode) {
1147                 case APIC_LVT_DM_NMI:
1148                         printf("NMI");
1149                         break;
1150                 case APIC_LVT_DM_SMI:
1151                         printf("SMI");
1152                         break;
1153                 case APIC_LVT_DM_INIT:
1154                         printf("INIT");
1155                         break;
1156                 case APIC_LVT_DM_EXTINT:
1157                         printf("ExtINT");
1158                         break;
1159                 }
1160                 printf(" -> LINT%u\n", pin);
1161         }
1162         return (0);
1163 }
1164
1165 static int
1166 native_lapic_set_lvt_polarity(u_int apic_id, u_int pin, enum intr_polarity pol)
1167 {
1168
1169         if (pin > APIC_LVT_MAX || pol == INTR_POLARITY_CONFORM)
1170                 return (EINVAL);
1171         if (apic_id == APIC_ID_ALL) {
1172                 lvts[pin].lvt_activehi = (pol == INTR_POLARITY_HIGH);
1173                 if (bootverbose)
1174                         printf("lapic:");
1175         } else {
1176                 KASSERT(lapics[apic_id].la_present,
1177                     ("%s: missing APIC %u", __func__, apic_id));
1178                 lapics[apic_id].la_lvts[pin].lvt_active = 1;
1179                 lapics[apic_id].la_lvts[pin].lvt_activehi =
1180                     (pol == INTR_POLARITY_HIGH);
1181                 if (bootverbose)
1182                         printf("lapic%u:", apic_id);
1183         }
1184         if (bootverbose)
1185                 printf(" LINT%u polarity: %s\n", pin,
1186                     pol == INTR_POLARITY_HIGH ? "high" : "low");
1187         return (0);
1188 }
1189
1190 static int
1191 native_lapic_set_lvt_triggermode(u_int apic_id, u_int pin,
1192      enum intr_trigger trigger)
1193 {
1194
1195         if (pin > APIC_LVT_MAX || trigger == INTR_TRIGGER_CONFORM)
1196                 return (EINVAL);
1197         if (apic_id == APIC_ID_ALL) {
1198                 lvts[pin].lvt_edgetrigger = (trigger == INTR_TRIGGER_EDGE);
1199                 if (bootverbose)
1200                         printf("lapic:");
1201         } else {
1202                 KASSERT(lapics[apic_id].la_present,
1203                     ("%s: missing APIC %u", __func__, apic_id));
1204                 lapics[apic_id].la_lvts[pin].lvt_edgetrigger =
1205                     (trigger == INTR_TRIGGER_EDGE);
1206                 lapics[apic_id].la_lvts[pin].lvt_active = 1;
1207                 if (bootverbose)
1208                         printf("lapic%u:", apic_id);
1209         }
1210         if (bootverbose)
1211                 printf(" LINT%u trigger: %s\n", pin,
1212                     trigger == INTR_TRIGGER_EDGE ? "edge" : "level");
1213         return (0);
1214 }
1215
1216 /*
1217  * Adjust the TPR of the current CPU so that it blocks all interrupts below
1218  * the passed in vector.
1219  */
1220 static void
1221 lapic_set_tpr(u_int vector)
1222 {
1223 #ifdef CHEAP_TPR
1224         lapic_write32(LAPIC_TPR, vector);
1225 #else
1226         uint32_t tpr;
1227
1228         tpr = lapic_read32(LAPIC_TPR) & ~APIC_TPR_PRIO;
1229         tpr |= vector;
1230         lapic_write32(LAPIC_TPR, tpr);
1231 #endif
1232 }
1233
1234 static void
1235 native_lapic_eoi(void)
1236 {
1237
1238         lapic_write32_nofence(LAPIC_EOI, 0);
1239 }
1240
1241 void
1242 lapic_handle_intr(int vector, struct trapframe *frame)
1243 {
1244         struct intsrc *isrc;
1245
1246         isrc = intr_lookup_source(apic_idt_to_irq(PCPU_GET(apic_id),
1247             vector));
1248         intr_execute_handlers(isrc, frame);
1249 }
1250
1251 void
1252 lapic_handle_timer(struct trapframe *frame)
1253 {
1254         struct lapic *la;
1255         struct trapframe *oldframe;
1256         struct thread *td;
1257
1258         /* Send EOI first thing. */
1259         lapic_eoi();
1260
1261 #if defined(SMP) && !defined(SCHED_ULE)
1262         /*
1263          * Don't do any accounting for the disabled HTT cores, since it
1264          * will provide misleading numbers for the userland.
1265          *
1266          * No locking is necessary here, since even if we lose the race
1267          * when hlt_cpus_mask changes it is not a big deal, really.
1268          *
1269          * Don't do that for ULE, since ULE doesn't consider hlt_cpus_mask
1270          * and unlike other schedulers it actually schedules threads to
1271          * those CPUs.
1272          */
1273         if (CPU_ISSET(PCPU_GET(cpuid), &hlt_cpus_mask))
1274                 return;
1275 #endif
1276
1277         /* Look up our local APIC structure for the tick counters. */
1278         la = &lapics[PCPU_GET(apic_id)];
1279         (*la->la_timer_count)++;
1280         critical_enter();
1281         if (lapic_et.et_active) {
1282                 td = curthread;
1283                 td->td_intr_nesting_level++;
1284                 oldframe = td->td_intr_frame;
1285                 td->td_intr_frame = frame;
1286                 lapic_et.et_event_cb(&lapic_et, lapic_et.et_arg);
1287                 td->td_intr_frame = oldframe;
1288                 td->td_intr_nesting_level--;
1289         }
1290         critical_exit();
1291 }
1292
1293 static void
1294 lapic_timer_set_divisor(u_int divisor)
1295 {
1296
1297         KASSERT(powerof2(divisor), ("lapic: invalid divisor %u", divisor));
1298         KASSERT(ffs(divisor) <= nitems(lapic_timer_divisors),
1299                 ("lapic: invalid divisor %u", divisor));
1300         lapic_write32(LAPIC_DCR_TIMER, lapic_timer_divisors[ffs(divisor) - 1]);
1301 }
1302
1303 static void
1304 lapic_timer_oneshot(struct lapic *la)
1305 {
1306         uint32_t value;
1307
1308         value = la->lvt_timer_base;
1309         value &= ~(APIC_LVTT_TM | APIC_LVT_M);
1310         value |= APIC_LVTT_TM_ONE_SHOT;
1311         la->lvt_timer_last = value;
1312         lapic_write32(LAPIC_LVT_TIMER, value);
1313         lapic_write32(LAPIC_ICR_TIMER, la->la_timer_period);
1314 }
1315
1316 static void
1317 lapic_timer_oneshot_nointr(struct lapic *la, uint32_t count)
1318 {
1319         uint32_t value;
1320
1321         value = la->lvt_timer_base;
1322         value &= ~APIC_LVTT_TM;
1323         value |= APIC_LVTT_TM_ONE_SHOT | APIC_LVT_M;
1324         la->lvt_timer_last = value;
1325         lapic_write32(LAPIC_LVT_TIMER, value);
1326         lapic_write32(LAPIC_ICR_TIMER, count);
1327 }
1328
1329 static void
1330 lapic_timer_periodic(struct lapic *la)
1331 {
1332         uint32_t value;
1333
1334         value = la->lvt_timer_base;
1335         value &= ~(APIC_LVTT_TM | APIC_LVT_M);
1336         value |= APIC_LVTT_TM_PERIODIC;
1337         la->lvt_timer_last = value;
1338         lapic_write32(LAPIC_LVT_TIMER, value);
1339         lapic_write32(LAPIC_ICR_TIMER, la->la_timer_period);
1340 }
1341
1342 static void
1343 lapic_timer_deadline(struct lapic *la)
1344 {
1345         uint32_t value;
1346
1347         value = la->lvt_timer_base;
1348         value &= ~(APIC_LVTT_TM | APIC_LVT_M);
1349         value |= APIC_LVTT_TM_TSCDLT;
1350         if (value != la->lvt_timer_last) {
1351                 la->lvt_timer_last = value;
1352                 lapic_write32_nofence(LAPIC_LVT_TIMER, value);
1353                 if (!x2apic_mode)
1354                         mfence();
1355         }
1356         wrmsr(MSR_TSC_DEADLINE, la->la_timer_period + rdtsc());
1357 }
1358
1359 static void
1360 lapic_timer_stop(struct lapic *la)
1361 {
1362         uint32_t value;
1363
1364         if (la->la_timer_mode == LAT_MODE_DEADLINE) {
1365                 wrmsr(MSR_TSC_DEADLINE, 0);
1366                 mfence();
1367         } else {
1368                 value = la->lvt_timer_base;
1369                 value &= ~APIC_LVTT_TM;
1370                 value |= APIC_LVT_M;
1371                 la->lvt_timer_last = value;
1372                 lapic_write32(LAPIC_LVT_TIMER, value);
1373         }
1374 }
1375
1376 void
1377 lapic_handle_cmc(void)
1378 {
1379
1380         lapic_eoi();
1381         cmc_intr();
1382 }
1383
1384 /*
1385  * Called from the mca_init() to activate the CMC interrupt if this CPU is
1386  * responsible for monitoring any MC banks for CMC events.  Since mca_init()
1387  * is called prior to lapic_setup() during boot, this just needs to unmask
1388  * this CPU's LVT_CMCI entry.
1389  */
1390 static void
1391 native_lapic_enable_cmc(void)
1392 {
1393         u_int apic_id;
1394
1395 #ifdef DEV_ATPIC
1396         if (!x2apic_mode && lapic_map == NULL)
1397                 return;
1398 #endif
1399         apic_id = PCPU_GET(apic_id);
1400         KASSERT(lapics[apic_id].la_present,
1401             ("%s: missing APIC %u", __func__, apic_id));
1402         lapics[apic_id].la_lvts[APIC_LVT_CMCI].lvt_masked = 0;
1403         lapics[apic_id].la_lvts[APIC_LVT_CMCI].lvt_active = 1;
1404         if (bootverbose)
1405                 printf("lapic%u: CMCI unmasked\n", apic_id);
1406 }
1407
1408 static int
1409 native_lapic_enable_mca_elvt(void)
1410 {
1411         u_int apic_id;
1412         uint32_t value;
1413         int elvt_count;
1414
1415 #ifdef DEV_ATPIC
1416         if (lapic_map == NULL)
1417                 return (-1);
1418 #endif
1419
1420         apic_id = PCPU_GET(apic_id);
1421         KASSERT(lapics[apic_id].la_present,
1422             ("%s: missing APIC %u", __func__, apic_id));
1423         elvt_count = amd_read_elvt_count();
1424         if (elvt_count <= APIC_ELVT_MCA)
1425                 return (-1);
1426
1427         value = lapic_read32(LAPIC_EXT_LVT0 + APIC_ELVT_MCA);
1428         if ((value & APIC_LVT_M) == 0) {
1429                 if (bootverbose)
1430                         printf("AMD MCE Thresholding Extended LVT is already active\n");
1431                 return (APIC_ELVT_MCA);
1432         }
1433         lapics[apic_id].la_elvts[APIC_ELVT_MCA].lvt_masked = 0;
1434         lapics[apic_id].la_elvts[APIC_ELVT_MCA].lvt_active = 1;
1435         if (bootverbose)
1436                 printf("lapic%u: MCE Thresholding ELVT unmasked\n", apic_id);
1437         return (APIC_ELVT_MCA);
1438 }
1439
1440 void
1441 lapic_handle_error(void)
1442 {
1443         uint32_t esr;
1444
1445         /*
1446          * Read the contents of the error status register.  Write to
1447          * the register first before reading from it to force the APIC
1448          * to update its value to indicate any errors that have
1449          * occurred since the previous write to the register.
1450          */
1451         lapic_write32(LAPIC_ESR, 0);
1452         esr = lapic_read32(LAPIC_ESR);
1453
1454         printf("CPU%d: local APIC error 0x%x\n", PCPU_GET(cpuid), esr);
1455         lapic_eoi();
1456 }
1457
1458 static u_int
1459 native_apic_cpuid(u_int apic_id)
1460 {
1461 #ifdef SMP
1462         return apic_cpuids[apic_id];
1463 #else
1464         return 0;
1465 #endif
1466 }
1467
1468 /* Request a free IDT vector to be used by the specified IRQ. */
1469 static u_int
1470 native_apic_alloc_vector(u_int apic_id, u_int irq)
1471 {
1472         u_int vector;
1473
1474         KASSERT(irq < NUM_IO_INTS, ("Invalid IRQ %u", irq));
1475
1476         /*
1477          * Search for a free vector.  Currently we just use a very simple
1478          * algorithm to find the first free vector.
1479          */
1480         mtx_lock_spin(&icu_lock);
1481         for (vector = 0; vector < APIC_NUM_IOINTS; vector++) {
1482                 if (lapics[apic_id].la_ioint_irqs[vector] != -1)
1483                         continue;
1484                 lapics[apic_id].la_ioint_irqs[vector] = irq;
1485                 mtx_unlock_spin(&icu_lock);
1486                 return (vector + APIC_IO_INTS);
1487         }
1488         mtx_unlock_spin(&icu_lock);
1489         return (0);
1490 }
1491
1492 /*
1493  * Request 'count' free contiguous IDT vectors to be used by 'count'
1494  * IRQs.  'count' must be a power of two and the vectors will be
1495  * aligned on a boundary of 'align'.  If the request cannot be
1496  * satisfied, 0 is returned.
1497  */
1498 static u_int
1499 native_apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align)
1500 {
1501         u_int first, run, vector;
1502
1503         KASSERT(powerof2(count), ("bad count"));
1504         KASSERT(powerof2(align), ("bad align"));
1505         KASSERT(align >= count, ("align < count"));
1506 #ifdef INVARIANTS
1507         for (run = 0; run < count; run++)
1508                 KASSERT(irqs[run] < NUM_IO_INTS, ("Invalid IRQ %u at index %u",
1509                     irqs[run], run));
1510 #endif
1511
1512         /*
1513          * Search for 'count' free vectors.  As with apic_alloc_vector(),
1514          * this just uses a simple first fit algorithm.
1515          */
1516         run = 0;
1517         first = 0;
1518         mtx_lock_spin(&icu_lock);
1519         for (vector = 0; vector < APIC_NUM_IOINTS; vector++) {
1520
1521                 /* Vector is in use, end run. */
1522                 if (lapics[apic_id].la_ioint_irqs[vector] != -1) {
1523                         run = 0;
1524                         first = 0;
1525                         continue;
1526                 }
1527
1528                 /* Start a new run if run == 0 and vector is aligned. */
1529                 if (run == 0) {
1530                         if ((vector & (align - 1)) != 0)
1531                                 continue;
1532                         first = vector;
1533                 }
1534                 run++;
1535
1536                 /* Keep looping if the run isn't long enough yet. */
1537                 if (run < count)
1538                         continue;
1539
1540                 /* Found a run, assign IRQs and return the first vector. */
1541                 for (vector = 0; vector < count; vector++)
1542                         lapics[apic_id].la_ioint_irqs[first + vector] =
1543                             irqs[vector];
1544                 mtx_unlock_spin(&icu_lock);
1545                 return (first + APIC_IO_INTS);
1546         }
1547         mtx_unlock_spin(&icu_lock);
1548         printf("APIC: Couldn't find APIC vectors for %u IRQs\n", count);
1549         return (0);
1550 }
1551
1552 /*
1553  * Enable a vector for a particular apic_id.  Since all lapics share idt
1554  * entries and ioint_handlers this enables the vector on all lapics.  lapics
1555  * which do not have the vector configured would report spurious interrupts
1556  * should it fire.
1557  */
1558 static void
1559 native_apic_enable_vector(u_int apic_id, u_int vector)
1560 {
1561
1562         KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry"));
1563         KASSERT(ioint_handlers[vector / 32] != NULL,
1564             ("No ISR handler for vector %u", vector));
1565 #ifdef KDTRACE_HOOKS
1566         KASSERT(vector != IDT_DTRACE_RET,
1567             ("Attempt to overwrite DTrace entry"));
1568 #endif
1569         setidt(vector, ioint_handlers[vector / 32], SDT_APIC, SEL_KPL,
1570             GSEL_APIC);
1571 }
1572
1573 static void
1574 native_apic_disable_vector(u_int apic_id, u_int vector)
1575 {
1576
1577         KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry"));
1578 #ifdef KDTRACE_HOOKS
1579         KASSERT(vector != IDT_DTRACE_RET,
1580             ("Attempt to overwrite DTrace entry"));
1581 #endif
1582         KASSERT(ioint_handlers[vector / 32] != NULL,
1583             ("No ISR handler for vector %u", vector));
1584 #ifdef notyet
1585         /*
1586          * We can not currently clear the idt entry because other cpus
1587          * may have a valid vector at this offset.
1588          */
1589         setidt(vector, &IDTVEC(rsvd), SDT_APICT, SEL_KPL, GSEL_APIC);
1590 #endif
1591 }
1592
1593 /* Release an APIC vector when it's no longer in use. */
1594 static void
1595 native_apic_free_vector(u_int apic_id, u_int vector, u_int irq)
1596 {
1597         struct thread *td;
1598
1599         KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL &&
1600             vector <= APIC_IO_INTS + APIC_NUM_IOINTS,
1601             ("Vector %u does not map to an IRQ line", vector));
1602         KASSERT(irq < NUM_IO_INTS, ("Invalid IRQ %u", irq));
1603         KASSERT(lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] ==
1604             irq, ("IRQ mismatch"));
1605 #ifdef KDTRACE_HOOKS
1606         KASSERT(vector != IDT_DTRACE_RET,
1607             ("Attempt to overwrite DTrace entry"));
1608 #endif
1609
1610         /*
1611          * Bind us to the cpu that owned the vector before freeing it so
1612          * we don't lose an interrupt delivery race.
1613          */
1614         td = curthread;
1615         if (!rebooting) {
1616                 thread_lock(td);
1617                 if (sched_is_bound(td))
1618                         panic("apic_free_vector: Thread already bound.\n");
1619                 sched_bind(td, apic_cpuid(apic_id));
1620                 thread_unlock(td);
1621         }
1622         mtx_lock_spin(&icu_lock);
1623         lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] = -1;
1624         mtx_unlock_spin(&icu_lock);
1625         if (!rebooting) {
1626                 thread_lock(td);
1627                 sched_unbind(td);
1628                 thread_unlock(td);
1629         }
1630 }
1631
1632 /* Map an IDT vector (APIC) to an IRQ (interrupt source). */
1633 static u_int
1634 apic_idt_to_irq(u_int apic_id, u_int vector)
1635 {
1636         int irq;
1637
1638         KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL &&
1639             vector <= APIC_IO_INTS + APIC_NUM_IOINTS,
1640             ("Vector %u does not map to an IRQ line", vector));
1641 #ifdef KDTRACE_HOOKS
1642         KASSERT(vector != IDT_DTRACE_RET,
1643             ("Attempt to overwrite DTrace entry"));
1644 #endif
1645         irq = lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS];
1646         if (irq < 0)
1647                 irq = 0;
1648         return (irq);
1649 }
1650
1651 #ifdef DDB
1652 /*
1653  * Dump data about APIC IDT vector mappings.
1654  */
1655 DB_SHOW_COMMAND(apic, db_show_apic)
1656 {
1657         struct intsrc *isrc;
1658         int i, verbose;
1659         u_int apic_id;
1660         u_int irq;
1661
1662         if (strcmp(modif, "vv") == 0)
1663                 verbose = 2;
1664         else if (strcmp(modif, "v") == 0)
1665                 verbose = 1;
1666         else
1667                 verbose = 0;
1668         for (apic_id = 0; apic_id <= max_apic_id; apic_id++) {
1669                 if (lapics[apic_id].la_present == 0)
1670                         continue;
1671                 db_printf("Interrupts bound to lapic %u\n", apic_id);
1672                 for (i = 0; i < APIC_NUM_IOINTS + 1 && !db_pager_quit; i++) {
1673                         irq = lapics[apic_id].la_ioint_irqs[i];
1674                         if (irq == -1 || irq == IRQ_SYSCALL)
1675                                 continue;
1676 #ifdef KDTRACE_HOOKS
1677                         if (irq == IRQ_DTRACE_RET)
1678                                 continue;
1679 #endif
1680 #ifdef XENHVM
1681                         if (irq == IRQ_EVTCHN)
1682                                 continue;
1683 #endif
1684                         db_printf("vec 0x%2x -> ", i + APIC_IO_INTS);
1685                         if (irq == IRQ_TIMER)
1686                                 db_printf("lapic timer\n");
1687                         else if (irq < NUM_IO_INTS) {
1688                                 isrc = intr_lookup_source(irq);
1689                                 if (isrc == NULL || verbose == 0)
1690                                         db_printf("IRQ %u\n", irq);
1691                                 else
1692                                         db_dump_intr_event(isrc->is_event,
1693                                             verbose == 2);
1694                         } else
1695                                 db_printf("IRQ %u ???\n", irq);
1696                 }
1697         }
1698 }
1699
1700 static void
1701 dump_mask(const char *prefix, uint32_t v, int base)
1702 {
1703         int i, first;
1704
1705         first = 1;
1706         for (i = 0; i < 32; i++)
1707                 if (v & (1 << i)) {
1708                         if (first) {
1709                                 db_printf("%s:", prefix);
1710                                 first = 0;
1711                         }
1712                         db_printf(" %02x", base + i);
1713                 }
1714         if (!first)
1715                 db_printf("\n");
1716 }
1717
1718 /* Show info from the lapic regs for this CPU. */
1719 DB_SHOW_COMMAND(lapic, db_show_lapic)
1720 {
1721         uint32_t v;
1722
1723         db_printf("lapic ID = %d\n", lapic_id());
1724         v = lapic_read32(LAPIC_VERSION);
1725         db_printf("version  = %d.%d\n", (v & APIC_VER_VERSION) >> 4,
1726             v & 0xf);
1727         db_printf("max LVT  = %d\n", (v & APIC_VER_MAXLVT) >> MAXLVTSHIFT);
1728         v = lapic_read32(LAPIC_SVR);
1729         db_printf("SVR      = %02x (%s)\n", v & APIC_SVR_VECTOR,
1730             v & APIC_SVR_ENABLE ? "enabled" : "disabled");
1731         db_printf("TPR      = %02x\n", lapic_read32(LAPIC_TPR));
1732
1733 #define dump_field(prefix, regn, index)                                 \
1734         dump_mask(__XSTRING(prefix ## index),                           \
1735             lapic_read32(LAPIC_ ## regn ## index),                      \
1736             index * 32)
1737
1738         db_printf("In-service Interrupts:\n");
1739         dump_field(isr, ISR, 0);
1740         dump_field(isr, ISR, 1);
1741         dump_field(isr, ISR, 2);
1742         dump_field(isr, ISR, 3);
1743         dump_field(isr, ISR, 4);
1744         dump_field(isr, ISR, 5);
1745         dump_field(isr, ISR, 6);
1746         dump_field(isr, ISR, 7);
1747
1748         db_printf("TMR Interrupts:\n");
1749         dump_field(tmr, TMR, 0);
1750         dump_field(tmr, TMR, 1);
1751         dump_field(tmr, TMR, 2);
1752         dump_field(tmr, TMR, 3);
1753         dump_field(tmr, TMR, 4);
1754         dump_field(tmr, TMR, 5);
1755         dump_field(tmr, TMR, 6);
1756         dump_field(tmr, TMR, 7);
1757
1758         db_printf("IRR Interrupts:\n");
1759         dump_field(irr, IRR, 0);
1760         dump_field(irr, IRR, 1);
1761         dump_field(irr, IRR, 2);
1762         dump_field(irr, IRR, 3);
1763         dump_field(irr, IRR, 4);
1764         dump_field(irr, IRR, 5);
1765         dump_field(irr, IRR, 6);
1766         dump_field(irr, IRR, 7);
1767
1768 #undef dump_field
1769 }
1770 #endif
1771
1772 /*
1773  * APIC probing support code.  This includes code to manage enumerators.
1774  */
1775
1776 static SLIST_HEAD(, apic_enumerator) enumerators =
1777         SLIST_HEAD_INITIALIZER(enumerators);
1778 static struct apic_enumerator *best_enum;
1779
1780 void
1781 apic_register_enumerator(struct apic_enumerator *enumerator)
1782 {
1783 #ifdef INVARIANTS
1784         struct apic_enumerator *apic_enum;
1785
1786         SLIST_FOREACH(apic_enum, &enumerators, apic_next) {
1787                 if (apic_enum == enumerator)
1788                         panic("%s: Duplicate register of %s", __func__,
1789                             enumerator->apic_name);
1790         }
1791 #endif
1792         SLIST_INSERT_HEAD(&enumerators, enumerator, apic_next);
1793 }
1794
1795 /*
1796  * We have to look for CPU's very, very early because certain subsystems
1797  * want to know how many CPU's we have extremely early on in the boot
1798  * process.
1799  */
1800 static void
1801 apic_init(void *dummy __unused)
1802 {
1803         struct apic_enumerator *enumerator;
1804         int retval, best;
1805
1806         /* We only support built in local APICs. */
1807         if (!(cpu_feature & CPUID_APIC))
1808                 return;
1809
1810         /* Don't probe if APIC mode is disabled. */
1811         if (resource_disabled("apic", 0))
1812                 return;
1813
1814         /* Probe all the enumerators to find the best match. */
1815         best_enum = NULL;
1816         best = 0;
1817         SLIST_FOREACH(enumerator, &enumerators, apic_next) {
1818                 retval = enumerator->apic_probe();
1819                 if (retval > 0)
1820                         continue;
1821                 if (best_enum == NULL || best < retval) {
1822                         best_enum = enumerator;
1823                         best = retval;
1824                 }
1825         }
1826         if (best_enum == NULL) {
1827                 if (bootverbose)
1828                         printf("APIC: Could not find any APICs.\n");
1829 #ifndef DEV_ATPIC
1830                 panic("running without device atpic requires a local APIC");
1831 #endif
1832                 return;
1833         }
1834
1835         if (bootverbose)
1836                 printf("APIC: Using the %s enumerator.\n",
1837                     best_enum->apic_name);
1838
1839 #ifdef I686_CPU
1840         /*
1841          * To work around an errata, we disable the local APIC on some
1842          * CPUs during early startup.  We need to turn the local APIC back
1843          * on on such CPUs now.
1844          */
1845         ppro_reenable_apic();
1846 #endif
1847
1848         /* Probe the CPU's in the system. */
1849         retval = best_enum->apic_probe_cpus();
1850         if (retval != 0)
1851                 printf("%s: Failed to probe CPUs: returned %d\n",
1852                     best_enum->apic_name, retval);
1853
1854 }
1855 SYSINIT(apic_init, SI_SUB_TUNABLES - 1, SI_ORDER_SECOND, apic_init, NULL);
1856
1857 /*
1858  * Setup the local APIC.  We have to do this prior to starting up the APs
1859  * in the SMP case.
1860  */
1861 static void
1862 apic_setup_local(void *dummy __unused)
1863 {
1864         int retval;
1865
1866         if (best_enum == NULL)
1867                 return;
1868
1869         lapics = malloc(sizeof(*lapics) * (max_apic_id + 1), M_LAPIC,
1870             M_WAITOK | M_ZERO);
1871
1872         /* Initialize the local APIC. */
1873         retval = best_enum->apic_setup_local();
1874         if (retval != 0)
1875                 printf("%s: Failed to setup the local APIC: returned %d\n",
1876                     best_enum->apic_name, retval);
1877 }
1878 SYSINIT(apic_setup_local, SI_SUB_CPU, SI_ORDER_SECOND, apic_setup_local, NULL);
1879
1880 /*
1881  * Setup the I/O APICs.
1882  */
1883 static void
1884 apic_setup_io(void *dummy __unused)
1885 {
1886         int retval;
1887
1888         if (best_enum == NULL)
1889                 return;
1890
1891         /*
1892          * Local APIC must be registered before other PICs and pseudo PICs
1893          * for proper suspend/resume order.
1894          */
1895         intr_register_pic(&lapic_pic);
1896
1897         retval = best_enum->apic_setup_io();
1898         if (retval != 0)
1899                 printf("%s: Failed to setup I/O APICs: returned %d\n",
1900                     best_enum->apic_name, retval);
1901
1902         /*
1903          * Finish setting up the local APIC on the BSP once we know
1904          * how to properly program the LINT pins.  In particular, this
1905          * enables the EOI suppression mode, if LAPIC support it and
1906          * user did not disabled the mode.
1907          */
1908         lapic_setup(1);
1909         if (bootverbose)
1910                 lapic_dump("BSP");
1911
1912         /* Enable the MSI "pic". */
1913         init_ops.msi_init();
1914 }
1915 SYSINIT(apic_setup_io, SI_SUB_INTR, SI_ORDER_THIRD, apic_setup_io, NULL);
1916
1917 #ifdef SMP
1918 /*
1919  * Inter Processor Interrupt functions.  The lapic_ipi_*() functions are
1920  * private to the MD code.  The public interface for the rest of the
1921  * kernel is defined in mp_machdep.c.
1922  */
1923
1924 /*
1925  * Wait delay microseconds for IPI to be sent.  If delay is -1, we
1926  * wait forever.
1927  */
1928 static int
1929 native_lapic_ipi_wait(int delay)
1930 {
1931         uint64_t rx;
1932
1933         /* LAPIC_ICR.APIC_DELSTAT_MASK is undefined in x2APIC mode */
1934         if (x2apic_mode)
1935                 return (1);
1936
1937         for (rx = 0; delay == -1 || rx < lapic_ipi_wait_mult * delay; rx++) {
1938                 if ((lapic_read_icr_lo() & APIC_DELSTAT_MASK) ==
1939                     APIC_DELSTAT_IDLE)
1940                         return (1);
1941                 ia32_pause();
1942         }
1943         return (0);
1944 }
1945
1946 static void
1947 native_lapic_ipi_raw(register_t icrlo, u_int dest)
1948 {
1949         uint64_t icr;
1950         uint32_t vhi, vlo;
1951         register_t saveintr;
1952
1953         /* XXX: Need more sanity checking of icrlo? */
1954         KASSERT(x2apic_mode || lapic_map != NULL,
1955             ("%s called too early", __func__));
1956         KASSERT(x2apic_mode ||
1957             (dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0,
1958             ("%s: invalid dest field", __func__));
1959         KASSERT((icrlo & APIC_ICRLO_RESV_MASK) == 0,
1960             ("%s: reserved bits set in ICR LO register", __func__));
1961
1962         /* Set destination in ICR HI register if it is being used. */
1963         if (!x2apic_mode) {
1964                 saveintr = intr_disable();
1965                 icr = lapic_read_icr();
1966         }
1967
1968         if ((icrlo & APIC_DEST_MASK) == APIC_DEST_DESTFLD) {
1969                 if (x2apic_mode) {
1970                         vhi = dest;
1971                 } else {
1972                         vhi = icr >> 32;
1973                         vhi &= ~APIC_ID_MASK;
1974                         vhi |= dest << APIC_ID_SHIFT;
1975                 }
1976         } else {
1977                 vhi = 0;
1978         }
1979
1980         /* Program the contents of the IPI and dispatch it. */
1981         if (x2apic_mode) {
1982                 vlo = icrlo;
1983         } else {
1984                 vlo = icr;
1985                 vlo &= APIC_ICRLO_RESV_MASK;
1986                 vlo |= icrlo;
1987         }
1988         lapic_write_icr(vhi, vlo);
1989         if (!x2apic_mode)
1990                 intr_restore(saveintr);
1991 }
1992
1993 #define BEFORE_SPIN     50000
1994 #ifdef DETECT_DEADLOCK
1995 #define AFTER_SPIN      50
1996 #endif
1997
1998 static void
1999 native_lapic_ipi_vectored(u_int vector, int dest)
2000 {
2001         register_t icrlo, destfield;
2002
2003         KASSERT((vector & ~APIC_VECTOR_MASK) == 0,
2004             ("%s: invalid vector %d", __func__, vector));
2005
2006         icrlo = APIC_DESTMODE_PHY | APIC_TRIGMOD_EDGE | APIC_LEVEL_ASSERT;
2007
2008         /*
2009          * NMI IPIs are just fake vectors used to send a NMI.  Use special rules
2010          * regarding NMIs if passed, otherwise specify the vector.
2011          */
2012         if (vector >= IPI_NMI_FIRST)
2013                 icrlo |= APIC_DELMODE_NMI;
2014         else
2015                 icrlo |= vector | APIC_DELMODE_FIXED;
2016         destfield = 0;
2017         switch (dest) {
2018         case APIC_IPI_DEST_SELF:
2019                 icrlo |= APIC_DEST_SELF;
2020                 break;
2021         case APIC_IPI_DEST_ALL:
2022                 icrlo |= APIC_DEST_ALLISELF;
2023                 break;
2024         case APIC_IPI_DEST_OTHERS:
2025                 icrlo |= APIC_DEST_ALLESELF;
2026                 break;
2027         default:
2028                 KASSERT(x2apic_mode ||
2029                     (dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0,
2030                     ("%s: invalid destination 0x%x", __func__, dest));
2031                 destfield = dest;
2032         }
2033
2034         /* Wait for an earlier IPI to finish. */
2035         if (!lapic_ipi_wait(BEFORE_SPIN)) {
2036                 if (panicstr != NULL)
2037                         return;
2038                 else
2039                         panic("APIC: Previous IPI is stuck");
2040         }
2041
2042         lapic_ipi_raw(icrlo, destfield);
2043
2044 #ifdef DETECT_DEADLOCK
2045         /* Wait for IPI to be delivered. */
2046         if (!lapic_ipi_wait(AFTER_SPIN)) {
2047 #ifdef needsattention
2048                 /*
2049                  * XXX FIXME:
2050                  *
2051                  * The above function waits for the message to actually be
2052                  * delivered.  It breaks out after an arbitrary timeout
2053                  * since the message should eventually be delivered (at
2054                  * least in theory) and that if it wasn't we would catch
2055                  * the failure with the check above when the next IPI is
2056                  * sent.
2057                  *
2058                  * We could skip this wait entirely, EXCEPT it probably
2059                  * protects us from other routines that assume that the
2060                  * message was delivered and acted upon when this function
2061                  * returns.
2062                  */
2063                 printf("APIC: IPI might be stuck\n");
2064 #else /* !needsattention */
2065                 /* Wait until mesage is sent without a timeout. */
2066                 while (lapic_read_icr_lo() & APIC_DELSTAT_PEND)
2067                         ia32_pause();
2068 #endif /* needsattention */
2069         }
2070 #endif /* DETECT_DEADLOCK */
2071 }
2072
2073 #endif /* SMP */
2074
2075 /*
2076  * Since the IDT is shared by all CPUs the IPI slot update needs to be globally
2077  * visible.
2078  *
2079  * Consider the case where an IPI is generated immediately after allocation:
2080  *     vector = lapic_ipi_alloc(ipifunc);
2081  *     ipi_selected(other_cpus, vector);
2082  *
2083  * In xAPIC mode a write to ICR_LO has serializing semantics because the
2084  * APIC page is mapped as an uncached region. In x2APIC mode there is an
2085  * explicit 'mfence' before the ICR MSR is written. Therefore in both cases
2086  * the IDT slot update is globally visible before the IPI is delivered.
2087  */
2088 static int
2089 native_lapic_ipi_alloc(inthand_t *ipifunc)
2090 {
2091         struct gate_descriptor *ip;
2092         long func;
2093         int idx, vector;
2094
2095         KASSERT(ipifunc != &IDTVEC(rsvd), ("invalid ipifunc %p", ipifunc));
2096
2097         vector = -1;
2098         mtx_lock_spin(&icu_lock);
2099         for (idx = IPI_DYN_FIRST; idx <= IPI_DYN_LAST; idx++) {
2100                 ip = &idt[idx];
2101                 func = (ip->gd_hioffset << 16) | ip->gd_looffset;
2102                 if (func == (uintptr_t)&IDTVEC(rsvd)) {
2103                         vector = idx;
2104                         setidt(vector, ipifunc, SDT_APIC, SEL_KPL, GSEL_APIC);
2105                         break;
2106                 }
2107         }
2108         mtx_unlock_spin(&icu_lock);
2109         return (vector);
2110 }
2111
2112 static void
2113 native_lapic_ipi_free(int vector)
2114 {
2115         struct gate_descriptor *ip;
2116         long func;
2117
2118         KASSERT(vector >= IPI_DYN_FIRST && vector <= IPI_DYN_LAST,
2119             ("%s: invalid vector %d", __func__, vector));
2120
2121         mtx_lock_spin(&icu_lock);
2122         ip = &idt[vector];
2123         func = (ip->gd_hioffset << 16) | ip->gd_looffset;
2124         KASSERT(func != (uintptr_t)&IDTVEC(rsvd),
2125             ("invalid idtfunc %#lx", func));
2126         setidt(vector, &IDTVEC(rsvd), SDT_APICT, SEL_KPL, GSEL_APIC);
2127         mtx_unlock_spin(&icu_lock);
2128 }