sys/x86/x86/local_apic.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-3-Clause
   3  *
   4  * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org>
   5  * Copyright (c) 1996, by Steve Passe
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. The name of the developer may NOT be used to endorse or promote products
  14  *    derived from this software without specific prior written permission.
  15  * 3. Neither the name of the author nor the names of any co-contributors
  16  *    may be used to endorse or promote products derived from this software
  17  *    without specific prior written permission.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  29  * SUCH DAMAGE.
  30  */
  31
  32 /*
  33  * Local APIC support on Pentium and later processors.
  34  */
  35
  36 #include <sys/cdefs.h>
  37 __FBSDID("$FreeBSD$");
  38
  39 #include "opt_atpic.h"
  40 #include "opt_hwpmc_hooks.h"
  41
  42 #include "opt_ddb.h"
  43
  44 #include <sys/param.h>
  45 #include <sys/systm.h>
  46 #include <sys/bus.h>
  47 #include <sys/kernel.h>
  48 #include <sys/lock.h>
  49 #include <sys/malloc.h>
  50 #include <sys/mutex.h>
  51 #include <sys/pcpu.h>
  52 #include <sys/proc.h>
  53 #include <sys/sched.h>
  54 #include <sys/smp.h>
  55 #include <sys/sysctl.h>
  56 #include <sys/timeet.h>
  57
  58 #include <vm/vm.h>
  59 #include <vm/pmap.h>
  60
  61 #include <x86/apicreg.h>
  62 #include <machine/clock.h>
  63 #include <machine/cpufunc.h>
  64 #include <machine/cputypes.h>
  65 #include <machine/frame.h>
  66 #include <machine/intr_machdep.h>
  67 #include <x86/apicvar.h>
  68 #include <x86/mca.h>
  69 #include <machine/md_var.h>
  70 #include <machine/smp.h>
  71 #include <machine/specialreg.h>
  72 #include <x86/init.h>
  73
  74 #ifdef DDB
  75 #include <sys/interrupt.h>
  76 #include <ddb/ddb.h>
  77 #endif
  78
  79 #ifdef __amd64__
  80 #define SDT_APIC        SDT_SYSIGT
  81 #define SDT_APICT       SDT_SYSIGT
  82 #define GSEL_APIC       0
  83 #else
  84 #define SDT_APIC        SDT_SYS386IGT
  85 #define SDT_APICT       SDT_SYS386TGT
  86 #define GSEL_APIC       GSEL(GCODE_SEL, SEL_KPL)
  87 #endif
  88
  89 static MALLOC_DEFINE(M_LAPIC, "local_apic", "Local APIC items");
  90
  91 /* Sanity checks on IDT vectors. */
  92 CTASSERT(APIC_IO_INTS + APIC_NUM_IOINTS == APIC_TIMER_INT);
  93 CTASSERT(APIC_TIMER_INT < APIC_LOCAL_INTS);
  94 CTASSERT(APIC_LOCAL_INTS == 240);
  95 CTASSERT(IPI_STOP < APIC_SPURIOUS_INT);
  96
  97 /* Magic IRQ values for the timer and syscalls. */
  98 #define IRQ_TIMER       (NUM_IO_INTS + 1)
  99 #define IRQ_SYSCALL     (NUM_IO_INTS + 2)
 100 #define IRQ_DTRACE_RET  (NUM_IO_INTS + 3)
 101 #define IRQ_EVTCHN      (NUM_IO_INTS + 4)
 102
 103 enum lat_timer_mode {
 104         LAT_MODE_UNDEF =        0,
 105         LAT_MODE_PERIODIC =     1,
 106         LAT_MODE_ONESHOT =      2,
 107         LAT_MODE_DEADLINE =     3,
 108 };
 109
 110 /*
 111  * Support for local APICs.  Local APICs manage interrupts on each
 112  * individual processor as opposed to I/O APICs which receive interrupts
 113  * from I/O devices and then forward them on to the local APICs.
 114  *
 115  * Local APICs can also send interrupts to each other thus providing the
 116  * mechanism for IPIs.
 117  */
 118
 119 struct lvt {
 120         u_int lvt_edgetrigger:1;
 121         u_int lvt_activehi:1;
 122         u_int lvt_masked:1;
 123         u_int lvt_active:1;
 124         u_int lvt_mode:16;
 125         u_int lvt_vector:8;
 126 };
 127
 128 struct lapic {
 129         struct lvt la_lvts[APIC_LVT_MAX + 1];
 130         struct lvt la_elvts[APIC_ELVT_MAX + 1];;
 131         u_int la_id:8;
 132         u_int la_cluster:4;
 133         u_int la_cluster_id:2;
 134         u_int la_present:1;
 135         u_long *la_timer_count;
 136         uint64_t la_timer_period;
 137         enum lat_timer_mode la_timer_mode;
 138         uint32_t lvt_timer_base;
 139         uint32_t lvt_timer_last;
 140         /* Include IDT_SYSCALL to make indexing easier. */
 141         int la_ioint_irqs[APIC_NUM_IOINTS + 1];
 142 } static *lapics;
 143
 144 /* Global defaults for local APIC LVT entries. */
 145 static struct lvt lvts[APIC_LVT_MAX + 1] = {
 146         { 1, 1, 1, 1, APIC_LVT_DM_EXTINT, 0 },  /* LINT0: masked ExtINT */
 147         { 1, 1, 0, 1, APIC_LVT_DM_NMI, 0 },     /* LINT1: NMI */
 148         { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_TIMER_INT },      /* Timer */
 149         { 1, 1, 0, 1, APIC_LVT_DM_FIXED, APIC_ERROR_INT },      /* Error */
 150         { 1, 1, 1, 1, APIC_LVT_DM_NMI, 0 },     /* PMC */
 151         { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_THERMAL_INT },    /* Thermal */
 152         { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_CMC_INT },        /* CMCI */
 153 };
 154
 155 /* Global defaults for AMD local APIC ELVT entries. */
 156 static struct lvt elvts[APIC_ELVT_MAX + 1] = {
 157         { 1, 1, 1, 0, APIC_LVT_DM_FIXED, 0 },
 158         { 1, 1, 1, 0, APIC_LVT_DM_FIXED, APIC_CMC_INT },
 159         { 1, 1, 1, 0, APIC_LVT_DM_FIXED, 0 },
 160         { 1, 1, 1, 0, APIC_LVT_DM_FIXED, 0 },
 161 };
 162
 163 static inthand_t *ioint_handlers[] = {
 164         NULL,                   /* 0 - 31 */
 165         IDTVEC(apic_isr1),      /* 32 - 63 */
 166         IDTVEC(apic_isr2),      /* 64 - 95 */
 167         IDTVEC(apic_isr3),      /* 96 - 127 */
 168         IDTVEC(apic_isr4),      /* 128 - 159 */
 169         IDTVEC(apic_isr5),      /* 160 - 191 */
 170         IDTVEC(apic_isr6),      /* 192 - 223 */
 171         IDTVEC(apic_isr7),      /* 224 - 255 */
 172 };
 173
 174
 175 static u_int32_t lapic_timer_divisors[] = {
 176         APIC_TDCR_1, APIC_TDCR_2, APIC_TDCR_4, APIC_TDCR_8, APIC_TDCR_16,
 177         APIC_TDCR_32, APIC_TDCR_64, APIC_TDCR_128
 178 };
 179
 180 extern inthand_t IDTVEC(rsvd);
 181
 182 volatile char *lapic_map;
 183 vm_paddr_t lapic_paddr;
 184 int x2apic_mode;
 185 int lapic_eoi_suppression;
 186 static int lapic_timer_tsc_deadline;
 187 static u_long lapic_timer_divisor, count_freq;
 188 static struct eventtimer lapic_et;
 189 #ifdef SMP
 190 static uint64_t lapic_ipi_wait_mult;
 191 #endif
 192 unsigned int max_apic_id;
 193
 194 SYSCTL_NODE(_hw, OID_AUTO, apic, CTLFLAG_RD, 0, "APIC options");
 195 SYSCTL_INT(_hw_apic, OID_AUTO, x2apic_mode, CTLFLAG_RD, &x2apic_mode, 0, "");
 196 SYSCTL_INT(_hw_apic, OID_AUTO, eoi_suppression, CTLFLAG_RD,
 197     &lapic_eoi_suppression, 0, "");
 198 SYSCTL_INT(_hw_apic, OID_AUTO, timer_tsc_deadline, CTLFLAG_RD,
 199     &lapic_timer_tsc_deadline, 0, "");
 200
 201 static uint32_t
 202 lapic_read32(enum LAPIC_REGISTERS reg)
 203 {
 204         uint32_t res;
 205
 206         if (x2apic_mode) {
 207                 res = rdmsr32(MSR_APIC_000 + reg);
 208         } else {
 209                 res = *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL);
 210         }
 211         return (res);
 212 }
 213
 214 static void
 215 lapic_write32(enum LAPIC_REGISTERS reg, uint32_t val)
 216 {
 217
 218         if (x2apic_mode) {
 219                 mfence();
 220                 wrmsr(MSR_APIC_000 + reg, val);
 221         } else {
 222                 *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL) = val;
 223         }
 224 }
 225
 226 static void
 227 lapic_write32_nofence(enum LAPIC_REGISTERS reg, uint32_t val)
 228 {
 229
 230         if (x2apic_mode) {
 231                 wrmsr(MSR_APIC_000 + reg, val);
 232         } else {
 233                 *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL) = val;
 234         }
 235 }
 236
 237 #ifdef SMP
 238 static uint64_t
 239 lapic_read_icr(void)
 240 {
 241         uint64_t v;
 242         uint32_t vhi, vlo;
 243
 244         if (x2apic_mode) {
 245                 v = rdmsr(MSR_APIC_000 + LAPIC_ICR_LO);
 246         } else {
 247                 vhi = lapic_read32(LAPIC_ICR_HI);
 248                 vlo = lapic_read32(LAPIC_ICR_LO);
 249                 v = ((uint64_t)vhi << 32) | vlo;
 250         }
 251         return (v);
 252 }
 253
 254 static uint64_t
 255 lapic_read_icr_lo(void)
 256 {
 257
 258         return (lapic_read32(LAPIC_ICR_LO));
 259 }
 260
 261 static void
 262 lapic_write_icr(uint32_t vhi, uint32_t vlo)
 263 {
 264         uint64_t v;
 265
 266         if (x2apic_mode) {
 267                 v = ((uint64_t)vhi << 32) | vlo;
 268                 mfence();
 269                 wrmsr(MSR_APIC_000 + LAPIC_ICR_LO, v);
 270         } else {
 271                 lapic_write32(LAPIC_ICR_HI, vhi);
 272                 lapic_write32(LAPIC_ICR_LO, vlo);
 273         }
 274 }
 275 #endif /* SMP */
 276
 277 static void
 278 native_lapic_enable_x2apic(void)
 279 {
 280         uint64_t apic_base;
 281
 282         apic_base = rdmsr(MSR_APICBASE);
 283         apic_base |= APICBASE_X2APIC | APICBASE_ENABLED;
 284         wrmsr(MSR_APICBASE, apic_base);
 285 }
 286
 287 static bool
 288 native_lapic_is_x2apic(void)
 289 {
 290         uint64_t apic_base;
 291
 292         apic_base = rdmsr(MSR_APICBASE);
 293         return ((apic_base & (APICBASE_X2APIC | APICBASE_ENABLED)) ==
 294             (APICBASE_X2APIC | APICBASE_ENABLED));
 295 }
 296
 297 static void     lapic_enable(void);
 298 static void     lapic_resume(struct pic *pic, bool suspend_cancelled);
 299 static void     lapic_timer_oneshot(struct lapic *);
 300 static void     lapic_timer_oneshot_nointr(struct lapic *, uint32_t);
 301 static void     lapic_timer_periodic(struct lapic *);
 302 static void     lapic_timer_deadline(struct lapic *);
 303 static void     lapic_timer_stop(struct lapic *);
 304 static void     lapic_timer_set_divisor(u_int divisor);
 305 static uint32_t lvt_mode(struct lapic *la, u_int pin, uint32_t value);
 306 static int      lapic_et_start(struct eventtimer *et,
 307                     sbintime_t first, sbintime_t period);
 308 static int      lapic_et_stop(struct eventtimer *et);
 309 static u_int    apic_idt_to_irq(u_int apic_id, u_int vector);
 310 static void     lapic_set_tpr(u_int vector);
 311
 312 struct pic lapic_pic = { .pic_resume = lapic_resume };
 313
 314 /* Forward declarations for apic_ops */
 315 static void     native_lapic_create(u_int apic_id, int boot_cpu);
 316 static void     native_lapic_init(vm_paddr_t addr);
 317 static void     native_lapic_xapic_mode(void);
 318 static void     native_lapic_setup(int boot);
 319 static void     native_lapic_dump(const char *str);
 320 static void     native_lapic_disable(void);
 321 static void     native_lapic_eoi(void);
 322 static int      native_lapic_id(void);
 323 static int      native_lapic_intr_pending(u_int vector);
 324 static u_int    native_apic_cpuid(u_int apic_id);
 325 static u_int    native_apic_alloc_vector(u_int apic_id, u_int irq);
 326 static u_int    native_apic_alloc_vectors(u_int apic_id, u_int *irqs,
 327                     u_int count, u_int align);
 328 static void     native_apic_disable_vector(u_int apic_id, u_int vector);
 329 static void     native_apic_enable_vector(u_int apic_id, u_int vector);
 330 static void     native_apic_free_vector(u_int apic_id, u_int vector, u_int irq);
 331 static void     native_lapic_set_logical_id(u_int apic_id, u_int cluster,
 332                     u_int cluster_id);
 333 static int      native_lapic_enable_pmc(void);
 334 static void     native_lapic_disable_pmc(void);
 335 static void     native_lapic_reenable_pmc(void);
 336 static void     native_lapic_enable_cmc(void);
 337 static int      native_lapic_enable_mca_elvt(void);
 338 static int      native_lapic_set_lvt_mask(u_int apic_id, u_int lvt,
 339                     u_char masked);
 340 static int      native_lapic_set_lvt_mode(u_int apic_id, u_int lvt,
 341                     uint32_t mode);
 342 static int      native_lapic_set_lvt_polarity(u_int apic_id, u_int lvt,
 343                     enum intr_polarity pol);
 344 static int      native_lapic_set_lvt_triggermode(u_int apic_id, u_int lvt,
 345                     enum intr_trigger trigger);
 346 #ifdef SMP
 347 static void     native_lapic_ipi_raw(register_t icrlo, u_int dest);
 348 static void     native_lapic_ipi_vectored(u_int vector, int dest);
 349 static int      native_lapic_ipi_wait(int delay);
 350 #endif /* SMP */
 351 static int      native_lapic_ipi_alloc(inthand_t *ipifunc);
 352 static void     native_lapic_ipi_free(int vector);
 353
 354 struct apic_ops apic_ops = {
 355         .create                 = native_lapic_create,
 356         .init                   = native_lapic_init,
 357         .xapic_mode             = native_lapic_xapic_mode,
 358         .is_x2apic              = native_lapic_is_x2apic,
 359         .setup                  = native_lapic_setup,
 360         .dump                   = native_lapic_dump,
 361         .disable                = native_lapic_disable,
 362         .eoi                    = native_lapic_eoi,
 363         .id                     = native_lapic_id,
 364         .intr_pending           = native_lapic_intr_pending,
 365         .set_logical_id         = native_lapic_set_logical_id,
 366         .cpuid                  = native_apic_cpuid,
 367         .alloc_vector           = native_apic_alloc_vector,
 368         .alloc_vectors          = native_apic_alloc_vectors,
 369         .enable_vector          = native_apic_enable_vector,
 370         .disable_vector         = native_apic_disable_vector,
 371         .free_vector            = native_apic_free_vector,
 372         .enable_pmc             = native_lapic_enable_pmc,
 373         .disable_pmc            = native_lapic_disable_pmc,
 374         .reenable_pmc           = native_lapic_reenable_pmc,
 375         .enable_cmc             = native_lapic_enable_cmc,
 376         .enable_mca_elvt        = native_lapic_enable_mca_elvt,
 377 #ifdef SMP
 378         .ipi_raw                = native_lapic_ipi_raw,
 379         .ipi_vectored           = native_lapic_ipi_vectored,
 380         .ipi_wait               = native_lapic_ipi_wait,
 381 #endif
 382         .ipi_alloc              = native_lapic_ipi_alloc,
 383         .ipi_free               = native_lapic_ipi_free,
 384         .set_lvt_mask           = native_lapic_set_lvt_mask,
 385         .set_lvt_mode           = native_lapic_set_lvt_mode,
 386         .set_lvt_polarity       = native_lapic_set_lvt_polarity,
 387         .set_lvt_triggermode    = native_lapic_set_lvt_triggermode,
 388 };
 389
 390 static uint32_t
 391 lvt_mode_impl(struct lapic *la, struct lvt *lvt, u_int pin, uint32_t value)
 392 {
 393
 394         value &= ~(APIC_LVT_M | APIC_LVT_TM | APIC_LVT_IIPP | APIC_LVT_DM |
 395             APIC_LVT_VECTOR);
 396         if (lvt->lvt_edgetrigger == 0)
 397                 value |= APIC_LVT_TM;
 398         if (lvt->lvt_activehi == 0)
 399                 value |= APIC_LVT_IIPP_INTALO;
 400         if (lvt->lvt_masked)
 401                 value |= APIC_LVT_M;
 402         value |= lvt->lvt_mode;
 403         switch (lvt->lvt_mode) {
 404         case APIC_LVT_DM_NMI:
 405         case APIC_LVT_DM_SMI:
 406         case APIC_LVT_DM_INIT:
 407         case APIC_LVT_DM_EXTINT:
 408                 if (!lvt->lvt_edgetrigger && bootverbose) {
 409                         printf("lapic%u: Forcing LINT%u to edge trigger\n",
 410                             la->la_id, pin);
 411                         value &= ~APIC_LVT_TM;
 412                 }
 413                 /* Use a vector of 0. */
 414                 break;
 415         case APIC_LVT_DM_FIXED:
 416                 value |= lvt->lvt_vector;
 417                 break;
 418         default:
 419                 panic("bad APIC LVT delivery mode: %#x\n", value);
 420         }
 421         return (value);
 422 }
 423
 424 static uint32_t
 425 lvt_mode(struct lapic *la, u_int pin, uint32_t value)
 426 {
 427         struct lvt *lvt;
 428
 429         KASSERT(pin <= APIC_LVT_MAX,
 430             ("%s: pin %u out of range", __func__, pin));
 431         if (la->la_lvts[pin].lvt_active)
 432                 lvt = &la->la_lvts[pin];
 433         else
 434                 lvt = &lvts[pin];
 435
 436         return (lvt_mode_impl(la, lvt, pin, value));
 437 }
 438
 439 static uint32_t
 440 elvt_mode(struct lapic *la, u_int idx, uint32_t value)
 441 {
 442         struct lvt *elvt;
 443
 444         KASSERT(idx <= APIC_ELVT_MAX,
 445             ("%s: idx %u out of range", __func__, idx));
 446
 447         elvt = &la->la_elvts[idx];
 448         KASSERT(elvt->lvt_active, ("%s: ELVT%u is not active", __func__, idx));
 449         KASSERT(elvt->lvt_edgetrigger,
 450             ("%s: ELVT%u is not edge triggered", __func__, idx));
 451         KASSERT(elvt->lvt_activehi,
 452             ("%s: ELVT%u is not active high", __func__, idx));
 453         return (lvt_mode_impl(la, elvt, idx, value));
 454 }
 455
 456 /*
 457  * Map the local APIC and setup necessary interrupt vectors.
 458  */
 459 static void
 460 native_lapic_init(vm_paddr_t addr)
 461 {
 462 #ifdef SMP
 463         uint64_t r, r1, r2, rx;
 464 #endif
 465         uint32_t ver;
 466         u_int regs[4];
 467         int i, arat;
 468
 469         /*
 470          * Enable x2APIC mode if possible. Map the local APIC
 471          * registers page.
 472          *
 473          * Keep the LAPIC registers page mapped uncached for x2APIC
 474          * mode too, to have direct map page attribute set to
 475          * uncached.  This is needed to work around CPU errata present
 476          * on all Intel processors.
 477          */
 478         KASSERT(trunc_page(addr) == addr,
 479             ("local APIC not aligned on a page boundary"));
 480         lapic_paddr = addr;
 481         lapic_map = pmap_mapdev(addr, PAGE_SIZE);
 482         if (x2apic_mode) {
 483                 native_lapic_enable_x2apic();
 484                 lapic_map = NULL;
 485         }
 486
 487         /* Setup the spurious interrupt handler. */
 488         setidt(APIC_SPURIOUS_INT, IDTVEC(spuriousint), SDT_APIC, SEL_KPL,
 489             GSEL_APIC);
 490
 491         /* Perform basic initialization of the BSP's local APIC. */
 492         lapic_enable();
 493
 494         /* Set BSP's per-CPU local APIC ID. */
 495         PCPU_SET(apic_id, lapic_id());
 496
 497         /* Local APIC timer interrupt. */
 498         setidt(APIC_TIMER_INT, IDTVEC(timerint), SDT_APIC, SEL_KPL, GSEL_APIC);
 499
 500         /* Local APIC error interrupt. */
 501         setidt(APIC_ERROR_INT, IDTVEC(errorint), SDT_APIC, SEL_KPL, GSEL_APIC);
 502
 503         /* XXX: Thermal interrupt */
 504
 505         /* Local APIC CMCI. */
 506         setidt(APIC_CMC_INT, IDTVEC(cmcint), SDT_APICT, SEL_KPL, GSEL_APIC);
 507
 508         if ((resource_int_value("apic", 0, "clock", &i) != 0 || i != 0)) {
 509                 arat = 0;
 510                 /* Intel CPUID 0x06 EAX[2] set if APIC timer runs in C3. */
 511                 if (cpu_vendor_id == CPU_VENDOR_INTEL && cpu_high >= 6) {
 512                         do_cpuid(0x06, regs);
 513                         if ((regs[0] & CPUTPM1_ARAT) != 0)
 514                                 arat = 1;
 515                 }
 516                 bzero(&lapic_et, sizeof(lapic_et));
 517                 lapic_et.et_name = "LAPIC";
 518                 lapic_et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT |
 519                     ET_FLAGS_PERCPU;
 520                 lapic_et.et_quality = 600;
 521                 if (!arat) {
 522                         lapic_et.et_flags |= ET_FLAGS_C3STOP;
 523                         lapic_et.et_quality = 100;
 524                 }
 525                 if ((cpu_feature & CPUID_TSC) != 0 &&
 526                     (cpu_feature2 & CPUID2_TSCDLT) != 0 &&
 527                     tsc_is_invariant && tsc_freq != 0) {
 528                         lapic_timer_tsc_deadline = 1;
 529                         TUNABLE_INT_FETCH("hw.lapic_tsc_deadline",
 530                             &lapic_timer_tsc_deadline);
 531                 }
 532
 533                 lapic_et.et_frequency = 0;
 534                 /* We don't know frequency yet, so trying to guess. */
 535                 lapic_et.et_min_period = 0x00001000LL;
 536                 lapic_et.et_max_period = SBT_1S;
 537                 lapic_et.et_start = lapic_et_start;
 538                 lapic_et.et_stop = lapic_et_stop;
 539                 lapic_et.et_priv = NULL;
 540                 et_register(&lapic_et);
 541         }
 542
 543         /*
 544          * Set lapic_eoi_suppression after lapic_enable(), to not
 545          * enable suppression in the hardware prematurely.  Note that
 546          * we by default enable suppression even when system only has
 547          * one IO-APIC, since EOI is broadcasted to all APIC agents,
 548          * including CPUs, otherwise.
 549          *
 550          * It seems that at least some KVM versions report
 551          * EOI_SUPPRESSION bit, but auto-EOI does not work.
 552          */
 553         ver = lapic_read32(LAPIC_VERSION);
 554         if ((ver & APIC_VER_EOI_SUPPRESSION) != 0) {
 555                 lapic_eoi_suppression = 1;
 556                 if (vm_guest == VM_GUEST_KVM) {
 557                         if (bootverbose)
 558                                 printf(
 559                        "KVM -- disabling lapic eoi suppression\n");
 560                         lapic_eoi_suppression = 0;
 561                 }
 562                 TUNABLE_INT_FETCH("hw.lapic_eoi_suppression",
 563                     &lapic_eoi_suppression);
 564         }
 565
 566 #ifdef SMP
 567 #define LOOPS   100000
 568         /*
 569          * Calibrate the busy loop waiting for IPI ack in xAPIC mode.
 570          * lapic_ipi_wait_mult contains the number of iterations which
 571          * approximately delay execution for 1 microsecond (the
 572          * argument to native_lapic_ipi_wait() is in microseconds).
 573          *
 574          * We assume that TSC is present and already measured.
 575          * Possible TSC frequency jumps are irrelevant to the
 576          * calibration loop below, the CPU clock management code is
 577          * not yet started, and we do not enter sleep states.
 578          */
 579         KASSERT((cpu_feature & CPUID_TSC) != 0 && tsc_freq != 0,
 580             ("TSC not initialized"));
 581         if (!x2apic_mode) {
 582                 r = rdtsc();
 583                 for (rx = 0; rx < LOOPS; rx++) {
 584                         (void)lapic_read_icr_lo();
 585                         ia32_pause();
 586                 }
 587                 r = rdtsc() - r;
 588                 r1 = tsc_freq * LOOPS;
 589                 r2 = r * 1000000;
 590                 lapic_ipi_wait_mult = r1 >= r2 ? r1 / r2 : 1;
 591                 if (bootverbose) {
 592                         printf("LAPIC: ipi_wait() us multiplier %ju (r %ju "
 593                             "tsc %ju)\n", (uintmax_t)lapic_ipi_wait_mult,
 594                             (uintmax_t)r, (uintmax_t)tsc_freq);
 595                 }
 596         }
 597 #undef LOOPS
 598 #endif /* SMP */
 599 }
 600
 601 /*
 602  * Create a local APIC instance.
 603  */
 604 static void
 605 native_lapic_create(u_int apic_id, int boot_cpu)
 606 {
 607         int i;
 608
 609         if (apic_id > max_apic_id) {
 610                 printf("APIC: Ignoring local APIC with ID %d\n", apic_id);
 611                 if (boot_cpu)
 612                         panic("Can't ignore BSP");
 613                 return;
 614         }
 615         KASSERT(!lapics[apic_id].la_present, ("duplicate local APIC %u",
 616             apic_id));
 617
 618         /*
 619          * Assume no local LVT overrides and a cluster of 0 and
 620          * intra-cluster ID of 0.
 621          */
 622         lapics[apic_id].la_present = 1;
 623         lapics[apic_id].la_id = apic_id;
 624         for (i = 0; i <= APIC_LVT_MAX; i++) {
 625                 lapics[apic_id].la_lvts[i] = lvts[i];
 626                 lapics[apic_id].la_lvts[i].lvt_active = 0;
 627         }
 628         for (i = 0; i <= APIC_ELVT_MAX; i++) {
 629                 lapics[apic_id].la_elvts[i] = elvts[i];
 630                 lapics[apic_id].la_elvts[i].lvt_active = 0;
 631         }
 632         for (i = 0; i <= APIC_NUM_IOINTS; i++)
 633             lapics[apic_id].la_ioint_irqs[i] = -1;
 634         lapics[apic_id].la_ioint_irqs[IDT_SYSCALL - APIC_IO_INTS] = IRQ_SYSCALL;
 635         lapics[apic_id].la_ioint_irqs[APIC_TIMER_INT - APIC_IO_INTS] =
 636             IRQ_TIMER;
 637 #ifdef KDTRACE_HOOKS
 638         lapics[apic_id].la_ioint_irqs[IDT_DTRACE_RET - APIC_IO_INTS] =
 639             IRQ_DTRACE_RET;
 640 #endif
 641 #ifdef XENHVM
 642         lapics[apic_id].la_ioint_irqs[IDT_EVTCHN - APIC_IO_INTS] = IRQ_EVTCHN;
 643 #endif
 644
 645
 646 #ifdef SMP
 647         cpu_add(apic_id, boot_cpu);
 648 #endif
 649 }
 650
 651 static inline uint32_t
 652 amd_read_ext_features(void)
 653 {
 654         uint32_t version;
 655
 656         if (cpu_vendor_id != CPU_VENDOR_AMD)
 657                 return (0);
 658         version = lapic_read32(LAPIC_VERSION);
 659         if ((version & APIC_VER_AMD_EXT_SPACE) != 0)
 660                 return (lapic_read32(LAPIC_EXT_FEATURES));
 661         else
 662                 return (0);
 663 }
 664
 665 static inline uint32_t
 666 amd_read_elvt_count(void)
 667 {
 668         uint32_t extf;
 669         uint32_t count;
 670
 671         extf = amd_read_ext_features();
 672         count = (extf & APIC_EXTF_ELVT_MASK) >> APIC_EXTF_ELVT_SHIFT;
 673         count = min(count, APIC_ELVT_MAX + 1);
 674         return (count);
 675 }
 676
 677 /*
 678  * Dump contents of local APIC registers
 679  */
 680 static void
 681 native_lapic_dump(const char* str)
 682 {
 683         uint32_t version;
 684         uint32_t maxlvt;
 685         uint32_t extf;
 686         int elvt_count;
 687         int i;
 688
 689         version = lapic_read32(LAPIC_VERSION);
 690         maxlvt = (version & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
 691         printf("cpu%d %s:\n", PCPU_GET(cpuid), str);
 692         printf("     ID: 0x%08x   VER: 0x%08x LDR: 0x%08x DFR: 0x%08x",
 693             lapic_read32(LAPIC_ID), version,
 694             lapic_read32(LAPIC_LDR), x2apic_mode ? 0 : lapic_read32(LAPIC_DFR));
 695         if ((cpu_feature2 & CPUID2_X2APIC) != 0)
 696                 printf(" x2APIC: %d", x2apic_mode);
 697         printf("\n  lint0: 0x%08x lint1: 0x%08x TPR: 0x%08x SVR: 0x%08x\n",
 698             lapic_read32(LAPIC_LVT_LINT0), lapic_read32(LAPIC_LVT_LINT1),
 699             lapic_read32(LAPIC_TPR), lapic_read32(LAPIC_SVR));
 700         printf("  timer: 0x%08x therm: 0x%08x err: 0x%08x",
 701             lapic_read32(LAPIC_LVT_TIMER), lapic_read32(LAPIC_LVT_THERMAL),
 702             lapic_read32(LAPIC_LVT_ERROR));
 703         if (maxlvt >= APIC_LVT_PMC)
 704                 printf(" pmc: 0x%08x", lapic_read32(LAPIC_LVT_PCINT));
 705         printf("\n");
 706         if (maxlvt >= APIC_LVT_CMCI)
 707                 printf("   cmci: 0x%08x\n", lapic_read32(LAPIC_LVT_CMCI));
 708         extf = amd_read_ext_features();
 709         if (extf != 0) {
 710                 printf("   AMD ext features: 0x%08x\n", extf);
 711                 elvt_count = amd_read_elvt_count();
 712                 for (i = 0; i < elvt_count; i++)
 713                         printf("   AMD elvt%d: 0x%08x\n", i,
 714                             lapic_read32(LAPIC_EXT_LVT0 + i));
 715         }
 716 }
 717
 718 static void
 719 native_lapic_xapic_mode(void)
 720 {
 721         register_t saveintr;
 722
 723         saveintr = intr_disable();
 724         if (x2apic_mode)
 725                 native_lapic_enable_x2apic();
 726         intr_restore(saveintr);
 727 }
 728
 729 static void
 730 native_lapic_setup(int boot)
 731 {
 732         struct lapic *la;
 733         uint32_t version;
 734         uint32_t maxlvt;
 735         register_t saveintr;
 736         char buf[MAXCOMLEN + 1];
 737         int elvt_count;
 738         int i;
 739
 740         saveintr = intr_disable();
 741
 742         la = &lapics[lapic_id()];
 743         KASSERT(la->la_present, ("missing APIC structure"));
 744         version = lapic_read32(LAPIC_VERSION);
 745         maxlvt = (version & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
 746
 747         /* Initialize the TPR to allow all interrupts. */
 748         lapic_set_tpr(0);
 749
 750         /* Setup spurious vector and enable the local APIC. */
 751         lapic_enable();
 752
 753         /* Program LINT[01] LVT entries. */
 754         lapic_write32(LAPIC_LVT_LINT0, lvt_mode(la, APIC_LVT_LINT0,
 755             lapic_read32(LAPIC_LVT_LINT0)));
 756         lapic_write32(LAPIC_LVT_LINT1, lvt_mode(la, APIC_LVT_LINT1,
 757             lapic_read32(LAPIC_LVT_LINT1)));
 758
 759         /* Program the PMC LVT entry if present. */
 760         if (maxlvt >= APIC_LVT_PMC) {
 761                 lapic_write32(LAPIC_LVT_PCINT, lvt_mode(la, APIC_LVT_PMC,
 762                     LAPIC_LVT_PCINT));
 763         }
 764
 765         /* Program timer LVT and setup handler. */
 766         la->lvt_timer_base = lvt_mode(la, APIC_LVT_TIMER,
 767             lapic_read32(LAPIC_LVT_TIMER));
 768         la->lvt_timer_last = la->lvt_timer_base;
 769         lapic_write32(LAPIC_LVT_TIMER, la->lvt_timer_base);
 770         if (boot) {
 771                 snprintf(buf, sizeof(buf), "cpu%d:timer", PCPU_GET(cpuid));
 772                 intrcnt_add(buf, &la->la_timer_count);
 773         }
 774
 775         /* Setup the timer if configured. */
 776         if (la->la_timer_mode != LAT_MODE_UNDEF) {
 777                 KASSERT(la->la_timer_period != 0, ("lapic%u: zero divisor",
 778                     lapic_id()));
 779                 switch (la->la_timer_mode) {
 780                 case LAT_MODE_PERIODIC:
 781                         lapic_timer_set_divisor(lapic_timer_divisor);
 782                         lapic_timer_periodic(la);
 783                         break;
 784                 case LAT_MODE_ONESHOT:
 785                         lapic_timer_set_divisor(lapic_timer_divisor);
 786                         lapic_timer_oneshot(la);
 787                         break;
 788                 case LAT_MODE_DEADLINE:
 789                         lapic_timer_deadline(la);
 790                         break;
 791                 default:
 792                         panic("corrupted la_timer_mode %p %d", la,
 793                             la->la_timer_mode);
 794                 }
 795         }
 796
 797         /* Program error LVT and clear any existing errors. */
 798         lapic_write32(LAPIC_LVT_ERROR, lvt_mode(la, APIC_LVT_ERROR,
 799             lapic_read32(LAPIC_LVT_ERROR)));
 800         lapic_write32(LAPIC_ESR, 0);
 801
 802         /* XXX: Thermal LVT */
 803
 804         /* Program the CMCI LVT entry if present. */
 805         if (maxlvt >= APIC_LVT_CMCI) {
 806                 lapic_write32(LAPIC_LVT_CMCI, lvt_mode(la, APIC_LVT_CMCI,
 807                     lapic_read32(LAPIC_LVT_CMCI)));
 808         }
 809
 810         elvt_count = amd_read_elvt_count();
 811         for (i = 0; i < elvt_count; i++) {
 812                 if (la->la_elvts[i].lvt_active)
 813                         lapic_write32(LAPIC_EXT_LVT0 + i,
 814                             elvt_mode(la, i, lapic_read32(LAPIC_EXT_LVT0 + i)));
 815         }
 816
 817         intr_restore(saveintr);
 818 }
 819
 820 static void
 821 native_lapic_reenable_pmc(void)
 822 {
 823 #ifdef HWPMC_HOOKS
 824         uint32_t value;
 825
 826         value = lapic_read32(LAPIC_LVT_PCINT);
 827         value &= ~APIC_LVT_M;
 828         lapic_write32(LAPIC_LVT_PCINT, value);
 829 #endif
 830 }
 831
 832 #ifdef HWPMC_HOOKS
 833 static void
 834 lapic_update_pmc(void *dummy)
 835 {
 836         struct lapic *la;
 837
 838         la = &lapics[lapic_id()];
 839         lapic_write32(LAPIC_LVT_PCINT, lvt_mode(la, APIC_LVT_PMC,
 840             lapic_read32(LAPIC_LVT_PCINT)));
 841 }
 842 #endif
 843
 844 static int
 845 native_lapic_enable_pmc(void)
 846 {
 847 #ifdef HWPMC_HOOKS
 848         u_int32_t maxlvt;
 849
 850         /* Fail if the local APIC is not present. */
 851         if (!x2apic_mode && lapic_map == NULL)
 852                 return (0);
 853
 854         /* Fail if the PMC LVT is not present. */
 855         maxlvt = (lapic_read32(LAPIC_VERSION) & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
 856         if (maxlvt < APIC_LVT_PMC)
 857                 return (0);
 858
 859         lvts[APIC_LVT_PMC].lvt_masked = 0;
 860
 861 #ifdef EARLY_AP_STARTUP
 862         MPASS(mp_ncpus == 1 || smp_started);
 863         smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL);
 864 #else
 865 #ifdef SMP
 866         /*
 867          * If hwpmc was loaded at boot time then the APs may not be
 868          * started yet.  In that case, don't forward the request to
 869          * them as they will program the lvt when they start.
 870          */
 871         if (smp_started)
 872                 smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL);
 873         else
 874 #endif
 875                 lapic_update_pmc(NULL);
 876 #endif
 877         return (1);
 878 #else
 879         return (0);
 880 #endif
 881 }
 882
 883 static void
 884 native_lapic_disable_pmc(void)
 885 {
 886 #ifdef HWPMC_HOOKS
 887         u_int32_t maxlvt;
 888
 889         /* Fail if the local APIC is not present. */
 890         if (!x2apic_mode && lapic_map == NULL)
 891                 return;
 892
 893         /* Fail if the PMC LVT is not present. */
 894         maxlvt = (lapic_read32(LAPIC_VERSION) & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
 895         if (maxlvt < APIC_LVT_PMC)
 896                 return;
 897
 898         lvts[APIC_LVT_PMC].lvt_masked = 1;
 899
 900 #ifdef SMP
 901         /* The APs should always be started when hwpmc is unloaded. */
 902         KASSERT(mp_ncpus == 1 || smp_started, ("hwpmc unloaded too early"));
 903 #endif
 904         smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL);
 905 #endif
 906 }
 907
 908 static void
 909 lapic_calibrate_initcount(struct eventtimer *et, struct lapic *la)
 910 {
 911         u_long value;
 912
 913         /* Start off with a divisor of 2 (power on reset default). */
 914         lapic_timer_divisor = 2;
 915         /* Try to calibrate the local APIC timer. */
 916         do {
 917                 lapic_timer_set_divisor(lapic_timer_divisor);
 918                 lapic_timer_oneshot_nointr(la, APIC_TIMER_MAX_COUNT);
 919                 DELAY(1000000);
 920                 value = APIC_TIMER_MAX_COUNT - lapic_read32(LAPIC_CCR_TIMER);
 921                 if (value != APIC_TIMER_MAX_COUNT)
 922                         break;
 923                 lapic_timer_divisor <<= 1;
 924         } while (lapic_timer_divisor <= 128);
 925         if (lapic_timer_divisor > 128)
 926                 panic("lapic: Divisor too big");
 927         if (bootverbose) {
 928                 printf("lapic: Divisor %lu, Frequency %lu Hz\n",
 929                     lapic_timer_divisor, value);
 930         }
 931         count_freq = value;
 932 }
 933
 934 static void
 935 lapic_calibrate_deadline(struct eventtimer *et, struct lapic *la __unused)
 936 {
 937
 938         if (bootverbose) {
 939                 printf("lapic: deadline tsc mode, Frequency %ju Hz\n",
 940                     (uintmax_t)tsc_freq);
 941         }
 942 }
 943
 944 static void
 945 lapic_change_mode(struct eventtimer *et, struct lapic *la,
 946     enum lat_timer_mode newmode)
 947 {
 948
 949         if (la->la_timer_mode == newmode)
 950                 return;
 951         switch (newmode) {
 952         case LAT_MODE_PERIODIC:
 953                 lapic_timer_set_divisor(lapic_timer_divisor);
 954                 et->et_frequency = count_freq;
 955                 break;
 956         case LAT_MODE_DEADLINE:
 957                 et->et_frequency = tsc_freq;
 958                 break;
 959         case LAT_MODE_ONESHOT:
 960                 lapic_timer_set_divisor(lapic_timer_divisor);
 961                 et->et_frequency = count_freq;
 962                 break;
 963         default:
 964                 panic("lapic_change_mode %d", newmode);
 965         }
 966         la->la_timer_mode = newmode;
 967         et->et_min_period = (0x00000002LLU << 32) / et->et_frequency;
 968         et->et_max_period = (0xfffffffeLLU << 32) / et->et_frequency;
 969 }
 970
 971 static int
 972 lapic_et_start(struct eventtimer *et, sbintime_t first, sbintime_t period)
 973 {
 974         struct lapic *la;
 975
 976         la = &lapics[PCPU_GET(apic_id)];
 977         if (et->et_frequency == 0) {
 978                 lapic_calibrate_initcount(et, la);
 979                 if (lapic_timer_tsc_deadline)
 980                         lapic_calibrate_deadline(et, la);
 981         }
 982         if (period != 0) {
 983                 lapic_change_mode(et, la, LAT_MODE_PERIODIC);
 984                 la->la_timer_period = ((uint32_t)et->et_frequency * period) >>
 985                     32;
 986                 lapic_timer_periodic(la);
 987         } else if (lapic_timer_tsc_deadline) {
 988                 lapic_change_mode(et, la, LAT_MODE_DEADLINE);
 989                 la->la_timer_period = (et->et_frequency * first) >> 32;
 990                 lapic_timer_deadline(la);
 991         } else {
 992                 lapic_change_mode(et, la, LAT_MODE_ONESHOT);
 993                 la->la_timer_period = ((uint32_t)et->et_frequency * first) >>
 994                     32;
 995                 lapic_timer_oneshot(la);
 996         }
 997         return (0);
 998 }
 999
1000 static int
1001 lapic_et_stop(struct eventtimer *et)
1002 {
1003         struct lapic *la;
1004
1005         la = &lapics[PCPU_GET(apic_id)];
1006         lapic_timer_stop(la);
1007         la->la_timer_mode = LAT_MODE_UNDEF;
1008         return (0);
1009 }
1010
1011 static void
1012 native_lapic_disable(void)
1013 {
1014         uint32_t value;
1015
1016         /* Software disable the local APIC. */
1017         value = lapic_read32(LAPIC_SVR);
1018         value &= ~APIC_SVR_SWEN;
1019         lapic_write32(LAPIC_SVR, value);
1020 }
1021
1022 static void
1023 lapic_enable(void)
1024 {
1025         uint32_t value;
1026
1027         /* Program the spurious vector to enable the local APIC. */
1028         value = lapic_read32(LAPIC_SVR);
1029         value &= ~(APIC_SVR_VECTOR | APIC_SVR_FOCUS);
1030         value |= APIC_SVR_FEN | APIC_SVR_SWEN | APIC_SPURIOUS_INT;
1031         if (lapic_eoi_suppression)
1032                 value |= APIC_SVR_EOI_SUPPRESSION;
1033         lapic_write32(LAPIC_SVR, value);
1034 }
1035
1036 /* Reset the local APIC on the BSP during resume. */
1037 static void
1038 lapic_resume(struct pic *pic, bool suspend_cancelled)
1039 {
1040
1041         lapic_setup(0);
1042 }
1043
1044 static int
1045 native_lapic_id(void)
1046 {
1047         uint32_t v;
1048
1049         KASSERT(x2apic_mode || lapic_map != NULL, ("local APIC is not mapped"));
1050         v = lapic_read32(LAPIC_ID);
1051         if (!x2apic_mode)
1052                 v >>= APIC_ID_SHIFT;
1053         return (v);
1054 }
1055
1056 static int
1057 native_lapic_intr_pending(u_int vector)
1058 {
1059         uint32_t irr;
1060
1061         /*
1062          * The IRR registers are an array of registers each of which
1063          * only describes 32 interrupts in the low 32 bits.  Thus, we
1064          * divide the vector by 32 to get the register index.
1065          * Finally, we modulus the vector by 32 to determine the
1066          * individual bit to test.
1067          */
1068         irr = lapic_read32(LAPIC_IRR0 + vector / 32);
1069         return (irr & 1 << (vector % 32));
1070 }
1071
1072 static void
1073 native_lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id)
1074 {
1075         struct lapic *la;
1076
1077         KASSERT(lapics[apic_id].la_present, ("%s: APIC %u doesn't exist",
1078             __func__, apic_id));
1079         KASSERT(cluster <= APIC_MAX_CLUSTER, ("%s: cluster %u too big",
1080             __func__, cluster));
1081         KASSERT(cluster_id <= APIC_MAX_INTRACLUSTER_ID,
1082             ("%s: intra cluster id %u too big", __func__, cluster_id));
1083         la = &lapics[apic_id];
1084         la->la_cluster = cluster;
1085         la->la_cluster_id = cluster_id;
1086 }
1087
1088 static int
1089 native_lapic_set_lvt_mask(u_int apic_id, u_int pin, u_char masked)
1090 {
1091
1092         if (pin > APIC_LVT_MAX)
1093                 return (EINVAL);
1094         if (apic_id == APIC_ID_ALL) {
1095                 lvts[pin].lvt_masked = masked;
1096                 if (bootverbose)
1097                         printf("lapic:");
1098         } else {
1099                 KASSERT(lapics[apic_id].la_present,
1100                     ("%s: missing APIC %u", __func__, apic_id));
1101                 lapics[apic_id].la_lvts[pin].lvt_masked = masked;
1102                 lapics[apic_id].la_lvts[pin].lvt_active = 1;
1103                 if (bootverbose)
1104                         printf("lapic%u:", apic_id);
1105         }
1106         if (bootverbose)
1107                 printf(" LINT%u %s\n", pin, masked ? "masked" : "unmasked");
1108         return (0);
1109 }
1110
1111 static int
1112 native_lapic_set_lvt_mode(u_int apic_id, u_int pin, u_int32_t mode)
1113 {
1114         struct lvt *lvt;
1115
1116         if (pin > APIC_LVT_MAX)
1117                 return (EINVAL);
1118         if (apic_id == APIC_ID_ALL) {
1119                 lvt = &lvts[pin];
1120                 if (bootverbose)
1121                         printf("lapic:");
1122         } else {
1123                 KASSERT(lapics[apic_id].la_present,
1124                     ("%s: missing APIC %u", __func__, apic_id));
1125                 lvt = &lapics[apic_id].la_lvts[pin];
1126                 lvt->lvt_active = 1;
1127                 if (bootverbose)
1128                         printf("lapic%u:", apic_id);
1129         }
1130         lvt->lvt_mode = mode;
1131         switch (mode) {
1132         case APIC_LVT_DM_NMI:
1133         case APIC_LVT_DM_SMI:
1134         case APIC_LVT_DM_INIT:
1135         case APIC_LVT_DM_EXTINT:
1136                 lvt->lvt_edgetrigger = 1;
1137                 lvt->lvt_activehi = 1;
1138                 if (mode == APIC_LVT_DM_EXTINT)
1139                         lvt->lvt_masked = 1;
1140                 else
1141                         lvt->lvt_masked = 0;
1142                 break;
1143         default:
1144                 panic("Unsupported delivery mode: 0x%x\n", mode);
1145         }
1146         if (bootverbose) {
1147                 printf(" Routing ");
1148                 switch (mode) {
1149                 case APIC_LVT_DM_NMI:
1150                         printf("NMI");
1151                         break;
1152                 case APIC_LVT_DM_SMI:
1153                         printf("SMI");
1154                         break;
1155                 case APIC_LVT_DM_INIT:
1156                         printf("INIT");
1157                         break;
1158                 case APIC_LVT_DM_EXTINT:
1159                         printf("ExtINT");
1160                         break;
1161                 }
1162                 printf(" -> LINT%u\n", pin);
1163         }
1164         return (0);
1165 }
1166
1167 static int
1168 native_lapic_set_lvt_polarity(u_int apic_id, u_int pin, enum intr_polarity pol)
1169 {
1170
1171         if (pin > APIC_LVT_MAX || pol == INTR_POLARITY_CONFORM)
1172                 return (EINVAL);
1173         if (apic_id == APIC_ID_ALL) {
1174                 lvts[pin].lvt_activehi = (pol == INTR_POLARITY_HIGH);
1175                 if (bootverbose)
1176                         printf("lapic:");
1177         } else {
1178                 KASSERT(lapics[apic_id].la_present,
1179                     ("%s: missing APIC %u", __func__, apic_id));
1180                 lapics[apic_id].la_lvts[pin].lvt_active = 1;
1181                 lapics[apic_id].la_lvts[pin].lvt_activehi =
1182                     (pol == INTR_POLARITY_HIGH);
1183                 if (bootverbose)
1184                         printf("lapic%u:", apic_id);
1185         }
1186         if (bootverbose)
1187                 printf(" LINT%u polarity: %s\n", pin,
1188                     pol == INTR_POLARITY_HIGH ? "high" : "low");
1189         return (0);
1190 }
1191
1192 static int
1193 native_lapic_set_lvt_triggermode(u_int apic_id, u_int pin,
1194      enum intr_trigger trigger)
1195 {
1196
1197         if (pin > APIC_LVT_MAX || trigger == INTR_TRIGGER_CONFORM)
1198                 return (EINVAL);
1199         if (apic_id == APIC_ID_ALL) {
1200                 lvts[pin].lvt_edgetrigger = (trigger == INTR_TRIGGER_EDGE);
1201                 if (bootverbose)
1202                         printf("lapic:");
1203         } else {
1204                 KASSERT(lapics[apic_id].la_present,
1205                     ("%s: missing APIC %u", __func__, apic_id));
1206                 lapics[apic_id].la_lvts[pin].lvt_edgetrigger =
1207                     (trigger == INTR_TRIGGER_EDGE);
1208                 lapics[apic_id].la_lvts[pin].lvt_active = 1;
1209                 if (bootverbose)
1210                         printf("lapic%u:", apic_id);
1211         }
1212         if (bootverbose)
1213                 printf(" LINT%u trigger: %s\n", pin,
1214                     trigger == INTR_TRIGGER_EDGE ? "edge" : "level");
1215         return (0);
1216 }
1217
1218 /*
1219  * Adjust the TPR of the current CPU so that it blocks all interrupts below
1220  * the passed in vector.
1221  */
1222 static void
1223 lapic_set_tpr(u_int vector)
1224 {
1225 #ifdef CHEAP_TPR
1226         lapic_write32(LAPIC_TPR, vector);
1227 #else
1228         uint32_t tpr;
1229
1230         tpr = lapic_read32(LAPIC_TPR) & ~APIC_TPR_PRIO;
1231         tpr |= vector;
1232         lapic_write32(LAPIC_TPR, tpr);
1233 #endif
1234 }
1235
1236 static void
1237 native_lapic_eoi(void)
1238 {
1239
1240         lapic_write32_nofence(LAPIC_EOI, 0);
1241 }
1242
1243 void
1244 lapic_handle_intr(int vector, struct trapframe *frame)
1245 {
1246         struct intsrc *isrc;
1247
1248         isrc = intr_lookup_source(apic_idt_to_irq(PCPU_GET(apic_id),
1249             vector));
1250         intr_execute_handlers(isrc, frame);
1251 }
1252
1253 void
1254 lapic_handle_timer(struct trapframe *frame)
1255 {
1256         struct lapic *la;
1257         struct trapframe *oldframe;
1258         struct thread *td;
1259
1260         /* Send EOI first thing. */
1261         lapic_eoi();
1262
1263 #if defined(SMP) && !defined(SCHED_ULE)
1264         /*
1265          * Don't do any accounting for the disabled HTT cores, since it
1266          * will provide misleading numbers for the userland.
1267          *
1268          * No locking is necessary here, since even if we lose the race
1269          * when hlt_cpus_mask changes it is not a big deal, really.
1270          *
1271          * Don't do that for ULE, since ULE doesn't consider hlt_cpus_mask
1272          * and unlike other schedulers it actually schedules threads to
1273          * those CPUs.
1274          */
1275         if (CPU_ISSET(PCPU_GET(cpuid), &hlt_cpus_mask))
1276                 return;
1277 #endif
1278
1279         /* Look up our local APIC structure for the tick counters. */
1280         la = &lapics[PCPU_GET(apic_id)];
1281         (*la->la_timer_count)++;
1282         critical_enter();
1283         if (lapic_et.et_active) {
1284                 td = curthread;
1285                 td->td_intr_nesting_level++;
1286                 oldframe = td->td_intr_frame;
1287                 td->td_intr_frame = frame;
1288                 lapic_et.et_event_cb(&lapic_et, lapic_et.et_arg);
1289                 td->td_intr_frame = oldframe;
1290                 td->td_intr_nesting_level--;
1291         }
1292         critical_exit();
1293 }
1294
1295 static void
1296 lapic_timer_set_divisor(u_int divisor)
1297 {
1298
1299         KASSERT(powerof2(divisor), ("lapic: invalid divisor %u", divisor));
1300         KASSERT(ffs(divisor) <= nitems(lapic_timer_divisors),
1301                 ("lapic: invalid divisor %u", divisor));
1302         lapic_write32(LAPIC_DCR_TIMER, lapic_timer_divisors[ffs(divisor) - 1]);
1303 }
1304
1305 static void
1306 lapic_timer_oneshot(struct lapic *la)
1307 {
1308         uint32_t value;
1309
1310         value = la->lvt_timer_base;
1311         value &= ~(APIC_LVTT_TM | APIC_LVT_M);
1312         value |= APIC_LVTT_TM_ONE_SHOT;
1313         la->lvt_timer_last = value;
1314         lapic_write32(LAPIC_LVT_TIMER, value);
1315         lapic_write32(LAPIC_ICR_TIMER, la->la_timer_period);
1316 }
1317
1318 static void
1319 lapic_timer_oneshot_nointr(struct lapic *la, uint32_t count)
1320 {
1321         uint32_t value;
1322
1323         value = la->lvt_timer_base;
1324         value &= ~APIC_LVTT_TM;
1325         value |= APIC_LVTT_TM_ONE_SHOT | APIC_LVT_M;
1326         la->lvt_timer_last = value;
1327         lapic_write32(LAPIC_LVT_TIMER, value);
1328         lapic_write32(LAPIC_ICR_TIMER, count);
1329 }
1330
1331 static void
1332 lapic_timer_periodic(struct lapic *la)
1333 {
1334         uint32_t value;
1335
1336         value = la->lvt_timer_base;
1337         value &= ~(APIC_LVTT_TM | APIC_LVT_M);
1338         value |= APIC_LVTT_TM_PERIODIC;
1339         la->lvt_timer_last = value;
1340         lapic_write32(LAPIC_LVT_TIMER, value);
1341         lapic_write32(LAPIC_ICR_TIMER, la->la_timer_period);
1342 }
1343
1344 static void
1345 lapic_timer_deadline(struct lapic *la)
1346 {
1347         uint32_t value;
1348
1349         value = la->lvt_timer_base;
1350         value &= ~(APIC_LVTT_TM | APIC_LVT_M);
1351         value |= APIC_LVTT_TM_TSCDLT;
1352         if (value != la->lvt_timer_last) {
1353                 la->lvt_timer_last = value;
1354                 lapic_write32_nofence(LAPIC_LVT_TIMER, value);
1355                 if (!x2apic_mode)
1356                         mfence();
1357         }
1358         wrmsr(MSR_TSC_DEADLINE, la->la_timer_period + rdtsc());
1359 }
1360
1361 static void
1362 lapic_timer_stop(struct lapic *la)
1363 {
1364         uint32_t value;
1365
1366         if (la->la_timer_mode == LAT_MODE_DEADLINE) {
1367                 wrmsr(MSR_TSC_DEADLINE, 0);
1368                 mfence();
1369         } else {
1370                 value = la->lvt_timer_base;
1371                 value &= ~APIC_LVTT_TM;
1372                 value |= APIC_LVT_M;
1373                 la->lvt_timer_last = value;
1374                 lapic_write32(LAPIC_LVT_TIMER, value);
1375         }
1376 }
1377
1378 void
1379 lapic_handle_cmc(void)
1380 {
1381
1382         lapic_eoi();
1383         cmc_intr();
1384 }
1385
1386 /*
1387  * Called from the mca_init() to activate the CMC interrupt if this CPU is
1388  * responsible for monitoring any MC banks for CMC events.  Since mca_init()
1389  * is called prior to lapic_setup() during boot, this just needs to unmask
1390  * this CPU's LVT_CMCI entry.
1391  */
1392 static void
1393 native_lapic_enable_cmc(void)
1394 {
1395         u_int apic_id;
1396
1397 #ifdef DEV_ATPIC
1398         if (!x2apic_mode && lapic_map == NULL)
1399                 return;
1400 #endif
1401         apic_id = PCPU_GET(apic_id);
1402         KASSERT(lapics[apic_id].la_present,
1403             ("%s: missing APIC %u", __func__, apic_id));
1404         lapics[apic_id].la_lvts[APIC_LVT_CMCI].lvt_masked = 0;
1405         lapics[apic_id].la_lvts[APIC_LVT_CMCI].lvt_active = 1;
1406         if (bootverbose)
1407                 printf("lapic%u: CMCI unmasked\n", apic_id);
1408 }
1409
1410 static int
1411 native_lapic_enable_mca_elvt(void)
1412 {
1413         u_int apic_id;
1414         uint32_t value;
1415         int elvt_count;
1416
1417 #ifdef DEV_ATPIC
1418         if (lapic_map == NULL)
1419                 return (-1);
1420 #endif
1421
1422         apic_id = PCPU_GET(apic_id);
1423         KASSERT(lapics[apic_id].la_present,
1424             ("%s: missing APIC %u", __func__, apic_id));
1425         elvt_count = amd_read_elvt_count();
1426         if (elvt_count <= APIC_ELVT_MCA)
1427                 return (-1);
1428
1429         value = lapic_read32(LAPIC_EXT_LVT0 + APIC_ELVT_MCA);
1430         if ((value & APIC_LVT_M) == 0) {
1431                 if (bootverbose)
1432                         printf("AMD MCE Thresholding Extended LVT is already active\n");
1433                 return (APIC_ELVT_MCA);
1434         }
1435         lapics[apic_id].la_elvts[APIC_ELVT_MCA].lvt_masked = 0;
1436         lapics[apic_id].la_elvts[APIC_ELVT_MCA].lvt_active = 1;
1437         if (bootverbose)
1438                 printf("lapic%u: MCE Thresholding ELVT unmasked\n", apic_id);
1439         return (APIC_ELVT_MCA);
1440 }
1441
1442 void
1443 lapic_handle_error(void)
1444 {
1445         uint32_t esr;
1446
1447         /*
1448          * Read the contents of the error status register.  Write to
1449          * the register first before reading from it to force the APIC
1450          * to update its value to indicate any errors that have
1451          * occurred since the previous write to the register.
1452          */
1453         lapic_write32(LAPIC_ESR, 0);
1454         esr = lapic_read32(LAPIC_ESR);
1455
1456         printf("CPU%d: local APIC error 0x%x\n", PCPU_GET(cpuid), esr);
1457         lapic_eoi();
1458 }
1459
1460 static u_int
1461 native_apic_cpuid(u_int apic_id)
1462 {
1463 #ifdef SMP
1464         return apic_cpuids[apic_id];
1465 #else
1466         return 0;
1467 #endif
1468 }
1469
1470 /* Request a free IDT vector to be used by the specified IRQ. */
1471 static u_int
1472 native_apic_alloc_vector(u_int apic_id, u_int irq)
1473 {
1474         u_int vector;
1475
1476         KASSERT(irq < NUM_IO_INTS, ("Invalid IRQ %u", irq));
1477
1478         /*
1479          * Search for a free vector.  Currently we just use a very simple
1480          * algorithm to find the first free vector.
1481          */
1482         mtx_lock_spin(&icu_lock);
1483         for (vector = 0; vector < APIC_NUM_IOINTS; vector++) {
1484                 if (lapics[apic_id].la_ioint_irqs[vector] != -1)
1485                         continue;
1486                 lapics[apic_id].la_ioint_irqs[vector] = irq;
1487                 mtx_unlock_spin(&icu_lock);
1488                 return (vector + APIC_IO_INTS);
1489         }
1490         mtx_unlock_spin(&icu_lock);
1491         return (0);
1492 }
1493
1494 /*
1495  * Request 'count' free contiguous IDT vectors to be used by 'count'
1496  * IRQs.  'count' must be a power of two and the vectors will be
1497  * aligned on a boundary of 'align'.  If the request cannot be
1498  * satisfied, 0 is returned.
1499  */
1500 static u_int
1501 native_apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align)
1502 {
1503         u_int first, run, vector;
1504
1505         KASSERT(powerof2(count), ("bad count"));
1506         KASSERT(powerof2(align), ("bad align"));
1507         KASSERT(align >= count, ("align < count"));
1508 #ifdef INVARIANTS
1509         for (run = 0; run < count; run++)
1510                 KASSERT(irqs[run] < NUM_IO_INTS, ("Invalid IRQ %u at index %u",
1511                     irqs[run], run));
1512 #endif
1513
1514         /*
1515          * Search for 'count' free vectors.  As with apic_alloc_vector(),
1516          * this just uses a simple first fit algorithm.
1517          */
1518         run = 0;
1519         first = 0;
1520         mtx_lock_spin(&icu_lock);
1521         for (vector = 0; vector < APIC_NUM_IOINTS; vector++) {
1522
1523                 /* Vector is in use, end run. */
1524                 if (lapics[apic_id].la_ioint_irqs[vector] != -1) {
1525                         run = 0;
1526                         first = 0;
1527                         continue;
1528                 }
1529
1530                 /* Start a new run if run == 0 and vector is aligned. */
1531                 if (run == 0) {
1532                         if ((vector & (align - 1)) != 0)
1533                                 continue;
1534                         first = vector;
1535                 }
1536                 run++;
1537
1538                 /* Keep looping if the run isn't long enough yet. */
1539                 if (run < count)
1540                         continue;
1541
1542                 /* Found a run, assign IRQs and return the first vector. */
1543                 for (vector = 0; vector < count; vector++)
1544                         lapics[apic_id].la_ioint_irqs[first + vector] =
1545                             irqs[vector];
1546                 mtx_unlock_spin(&icu_lock);
1547                 return (first + APIC_IO_INTS);
1548         }
1549         mtx_unlock_spin(&icu_lock);
1550         printf("APIC: Couldn't find APIC vectors for %u IRQs\n", count);
1551         return (0);
1552 }
1553
1554 /*
1555  * Enable a vector for a particular apic_id.  Since all lapics share idt
1556  * entries and ioint_handlers this enables the vector on all lapics.  lapics
1557  * which do not have the vector configured would report spurious interrupts
1558  * should it fire.
1559  */
1560 static void
1561 native_apic_enable_vector(u_int apic_id, u_int vector)
1562 {
1563
1564         KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry"));
1565         KASSERT(ioint_handlers[vector / 32] != NULL,
1566             ("No ISR handler for vector %u", vector));
1567 #ifdef KDTRACE_HOOKS
1568         KASSERT(vector != IDT_DTRACE_RET,
1569             ("Attempt to overwrite DTrace entry"));
1570 #endif
1571         setidt(vector, ioint_handlers[vector / 32], SDT_APIC, SEL_KPL,
1572             GSEL_APIC);
1573 }
1574
1575 static void
1576 native_apic_disable_vector(u_int apic_id, u_int vector)
1577 {
1578
1579         KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry"));
1580 #ifdef KDTRACE_HOOKS
1581         KASSERT(vector != IDT_DTRACE_RET,
1582             ("Attempt to overwrite DTrace entry"));
1583 #endif
1584         KASSERT(ioint_handlers[vector / 32] != NULL,
1585             ("No ISR handler for vector %u", vector));
1586 #ifdef notyet
1587         /*
1588          * We can not currently clear the idt entry because other cpus
1589          * may have a valid vector at this offset.
1590          */
1591         setidt(vector, &IDTVEC(rsvd), SDT_APICT, SEL_KPL, GSEL_APIC);
1592 #endif
1593 }
1594
1595 /* Release an APIC vector when it's no longer in use. */
1596 static void
1597 native_apic_free_vector(u_int apic_id, u_int vector, u_int irq)
1598 {
1599         struct thread *td;
1600
1601         KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL &&
1602             vector <= APIC_IO_INTS + APIC_NUM_IOINTS,
1603             ("Vector %u does not map to an IRQ line", vector));
1604         KASSERT(irq < NUM_IO_INTS, ("Invalid IRQ %u", irq));
1605         KASSERT(lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] ==
1606             irq, ("IRQ mismatch"));
1607 #ifdef KDTRACE_HOOKS
1608         KASSERT(vector != IDT_DTRACE_RET,
1609             ("Attempt to overwrite DTrace entry"));
1610 #endif
1611
1612         /*
1613          * Bind us to the cpu that owned the vector before freeing it so
1614          * we don't lose an interrupt delivery race.
1615          */
1616         td = curthread;
1617         if (!rebooting) {
1618                 thread_lock(td);
1619                 if (sched_is_bound(td))
1620                         panic("apic_free_vector: Thread already bound.\n");
1621                 sched_bind(td, apic_cpuid(apic_id));
1622                 thread_unlock(td);
1623         }
1624         mtx_lock_spin(&icu_lock);
1625         lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] = -1;
1626         mtx_unlock_spin(&icu_lock);
1627         if (!rebooting) {
1628                 thread_lock(td);
1629                 sched_unbind(td);
1630                 thread_unlock(td);
1631         }
1632 }
1633
1634 /* Map an IDT vector (APIC) to an IRQ (interrupt source). */
1635 static u_int
1636 apic_idt_to_irq(u_int apic_id, u_int vector)
1637 {
1638         int irq;
1639
1640         KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL &&
1641             vector <= APIC_IO_INTS + APIC_NUM_IOINTS,
1642             ("Vector %u does not map to an IRQ line", vector));
1643 #ifdef KDTRACE_HOOKS
1644         KASSERT(vector != IDT_DTRACE_RET,
1645             ("Attempt to overwrite DTrace entry"));
1646 #endif
1647         irq = lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS];
1648         if (irq < 0)
1649                 irq = 0;
1650         return (irq);
1651 }
1652
1653 #ifdef DDB
1654 /*
1655  * Dump data about APIC IDT vector mappings.
1656  */
1657 DB_SHOW_COMMAND(apic, db_show_apic)
1658 {
1659         struct intsrc *isrc;
1660         int i, verbose;
1661         u_int apic_id;
1662         u_int irq;
1663
1664         if (strcmp(modif, "vv") == 0)
1665                 verbose = 2;
1666         else if (strcmp(modif, "v") == 0)
1667                 verbose = 1;
1668         else
1669                 verbose = 0;
1670         for (apic_id = 0; apic_id <= max_apic_id; apic_id++) {
1671                 if (lapics[apic_id].la_present == 0)
1672                         continue;
1673                 db_printf("Interrupts bound to lapic %u\n", apic_id);
1674                 for (i = 0; i < APIC_NUM_IOINTS + 1 && !db_pager_quit; i++) {
1675                         irq = lapics[apic_id].la_ioint_irqs[i];
1676                         if (irq == -1 || irq == IRQ_SYSCALL)
1677                                 continue;
1678 #ifdef KDTRACE_HOOKS
1679                         if (irq == IRQ_DTRACE_RET)
1680                                 continue;
1681 #endif
1682 #ifdef XENHVM
1683                         if (irq == IRQ_EVTCHN)
1684                                 continue;
1685 #endif
1686                         db_printf("vec 0x%2x -> ", i + APIC_IO_INTS);
1687                         if (irq == IRQ_TIMER)
1688                                 db_printf("lapic timer\n");
1689                         else if (irq < NUM_IO_INTS) {
1690                                 isrc = intr_lookup_source(irq);
1691                                 if (isrc == NULL || verbose == 0)
1692                                         db_printf("IRQ %u\n", irq);
1693                                 else
1694                                         db_dump_intr_event(isrc->is_event,
1695                                             verbose == 2);
1696                         } else
1697                                 db_printf("IRQ %u ???\n", irq);
1698                 }
1699         }
1700 }
1701
1702 static void
1703 dump_mask(const char *prefix, uint32_t v, int base)
1704 {
1705         int i, first;
1706
1707         first = 1;
1708         for (i = 0; i < 32; i++)
1709                 if (v & (1 << i)) {
1710                         if (first) {
1711                                 db_printf("%s:", prefix);
1712                                 first = 0;
1713                         }
1714                         db_printf(" %02x", base + i);
1715                 }
1716         if (!first)
1717                 db_printf("\n");
1718 }
1719
1720 /* Show info from the lapic regs for this CPU. */
1721 DB_SHOW_COMMAND(lapic, db_show_lapic)
1722 {
1723         uint32_t v;
1724
1725         db_printf("lapic ID = %d\n", lapic_id());
1726         v = lapic_read32(LAPIC_VERSION);
1727         db_printf("version  = %d.%d\n", (v & APIC_VER_VERSION) >> 4,
1728             v & 0xf);
1729         db_printf("max LVT  = %d\n", (v & APIC_VER_MAXLVT) >> MAXLVTSHIFT);
1730         v = lapic_read32(LAPIC_SVR);
1731         db_printf("SVR      = %02x (%s)\n", v & APIC_SVR_VECTOR,
1732             v & APIC_SVR_ENABLE ? "enabled" : "disabled");
1733         db_printf("TPR      = %02x\n", lapic_read32(LAPIC_TPR));
1734
1735 #define dump_field(prefix, regn, index)                                 \
1736         dump_mask(__XSTRING(prefix ## index),                           \
1737             lapic_read32(LAPIC_ ## regn ## index),                      \
1738             index * 32)
1739
1740         db_printf("In-service Interrupts:\n");
1741         dump_field(isr, ISR, 0);
1742         dump_field(isr, ISR, 1);
1743         dump_field(isr, ISR, 2);
1744         dump_field(isr, ISR, 3);
1745         dump_field(isr, ISR, 4);
1746         dump_field(isr, ISR, 5);
1747         dump_field(isr, ISR, 6);
1748         dump_field(isr, ISR, 7);
1749
1750         db_printf("TMR Interrupts:\n");
1751         dump_field(tmr, TMR, 0);
1752         dump_field(tmr, TMR, 1);
1753         dump_field(tmr, TMR, 2);
1754         dump_field(tmr, TMR, 3);
1755         dump_field(tmr, TMR, 4);
1756         dump_field(tmr, TMR, 5);
1757         dump_field(tmr, TMR, 6);
1758         dump_field(tmr, TMR, 7);
1759
1760         db_printf("IRR Interrupts:\n");
1761         dump_field(irr, IRR, 0);
1762         dump_field(irr, IRR, 1);
1763         dump_field(irr, IRR, 2);
1764         dump_field(irr, IRR, 3);
1765         dump_field(irr, IRR, 4);
1766         dump_field(irr, IRR, 5);
1767         dump_field(irr, IRR, 6);
1768         dump_field(irr, IRR, 7);
1769
1770 #undef dump_field
1771 }
1772 #endif
1773
1774 /*
1775  * APIC probing support code.  This includes code to manage enumerators.
1776  */
1777
1778 static SLIST_HEAD(, apic_enumerator) enumerators =
1779         SLIST_HEAD_INITIALIZER(enumerators);
1780 static struct apic_enumerator *best_enum;
1781
1782 void
1783 apic_register_enumerator(struct apic_enumerator *enumerator)
1784 {
1785 #ifdef INVARIANTS
1786         struct apic_enumerator *apic_enum;
1787
1788         SLIST_FOREACH(apic_enum, &enumerators, apic_next) {
1789                 if (apic_enum == enumerator)
1790                         panic("%s: Duplicate register of %s", __func__,
1791                             enumerator->apic_name);
1792         }
1793 #endif
1794         SLIST_INSERT_HEAD(&enumerators, enumerator, apic_next);
1795 }
1796
1797 /*
1798  * We have to look for CPU's very, very early because certain subsystems
1799  * want to know how many CPU's we have extremely early on in the boot
1800  * process.
1801  */
1802 static void
1803 apic_init(void *dummy __unused)
1804 {
1805         struct apic_enumerator *enumerator;
1806         int retval, best;
1807
1808         /* We only support built in local APICs. */
1809         if (!(cpu_feature & CPUID_APIC))
1810                 return;
1811
1812         /* Don't probe if APIC mode is disabled. */
1813         if (resource_disabled("apic", 0))
1814                 return;
1815
1816         /* Probe all the enumerators to find the best match. */
1817         best_enum = NULL;
1818         best = 0;
1819         SLIST_FOREACH(enumerator, &enumerators, apic_next) {
1820                 retval = enumerator->apic_probe();
1821                 if (retval > 0)
1822                         continue;
1823                 if (best_enum == NULL || best < retval) {
1824                         best_enum = enumerator;
1825                         best = retval;
1826                 }
1827         }
1828         if (best_enum == NULL) {
1829                 if (bootverbose)
1830                         printf("APIC: Could not find any APICs.\n");
1831 #ifndef DEV_ATPIC
1832                 panic("running without device atpic requires a local APIC");
1833 #endif
1834                 return;
1835         }
1836
1837         if (bootverbose)
1838                 printf("APIC: Using the %s enumerator.\n",
1839                     best_enum->apic_name);
1840
1841 #ifdef I686_CPU
1842         /*
1843          * To work around an errata, we disable the local APIC on some
1844          * CPUs during early startup.  We need to turn the local APIC back
1845          * on on such CPUs now.
1846          */
1847         ppro_reenable_apic();
1848 #endif
1849
1850         /* Probe the CPU's in the system. */
1851         retval = best_enum->apic_probe_cpus();
1852         if (retval != 0)
1853                 printf("%s: Failed to probe CPUs: returned %d\n",
1854                     best_enum->apic_name, retval);
1855
1856 }
1857 SYSINIT(apic_init, SI_SUB_TUNABLES - 1, SI_ORDER_SECOND, apic_init, NULL);
1858
1859 /*
1860  * Setup the local APIC.  We have to do this prior to starting up the APs
1861  * in the SMP case.
1862  */
1863 static void
1864 apic_setup_local(void *dummy __unused)
1865 {
1866         int retval;
1867
1868         if (best_enum == NULL)
1869                 return;
1870
1871         lapics = malloc(sizeof(*lapics) * (max_apic_id + 1), M_LAPIC,
1872             M_WAITOK | M_ZERO);
1873
1874         /* Initialize the local APIC. */
1875         retval = best_enum->apic_setup_local();
1876         if (retval != 0)
1877                 printf("%s: Failed to setup the local APIC: returned %d\n",
1878                     best_enum->apic_name, retval);
1879 }
1880 SYSINIT(apic_setup_local, SI_SUB_CPU, SI_ORDER_SECOND, apic_setup_local, NULL);
1881
1882 /*
1883  * Setup the I/O APICs.
1884  */
1885 static void
1886 apic_setup_io(void *dummy __unused)
1887 {
1888         int retval;
1889
1890         if (best_enum == NULL)
1891                 return;
1892
1893         /*
1894          * Local APIC must be registered before other PICs and pseudo PICs
1895          * for proper suspend/resume order.
1896          */
1897         intr_register_pic(&lapic_pic);
1898
1899         retval = best_enum->apic_setup_io();
1900         if (retval != 0)
1901                 printf("%s: Failed to setup I/O APICs: returned %d\n",
1902                     best_enum->apic_name, retval);
1903
1904         /*
1905          * Finish setting up the local APIC on the BSP once we know
1906          * how to properly program the LINT pins.  In particular, this
1907          * enables the EOI suppression mode, if LAPIC support it and
1908          * user did not disabled the mode.
1909          */
1910         lapic_setup(1);
1911         if (bootverbose)
1912                 lapic_dump("BSP");
1913
1914         /* Enable the MSI "pic". */
1915         init_ops.msi_init();
1916 }
1917 SYSINIT(apic_setup_io, SI_SUB_INTR, SI_ORDER_THIRD, apic_setup_io, NULL);
1918
1919 #ifdef SMP
1920 /*
1921  * Inter Processor Interrupt functions.  The lapic_ipi_*() functions are
1922  * private to the MD code.  The public interface for the rest of the
1923  * kernel is defined in mp_machdep.c.
1924  */
1925
1926 /*
1927  * Wait delay microseconds for IPI to be sent.  If delay is -1, we
1928  * wait forever.
1929  */
1930 static int
1931 native_lapic_ipi_wait(int delay)
1932 {
1933         uint64_t rx;
1934
1935         /* LAPIC_ICR.APIC_DELSTAT_MASK is undefined in x2APIC mode */
1936         if (x2apic_mode)
1937                 return (1);
1938
1939         for (rx = 0; delay == -1 || rx < lapic_ipi_wait_mult * delay; rx++) {
1940                 if ((lapic_read_icr_lo() & APIC_DELSTAT_MASK) ==
1941                     APIC_DELSTAT_IDLE)
1942                         return (1);
1943                 ia32_pause();
1944         }
1945         return (0);
1946 }
1947
1948 static void
1949 native_lapic_ipi_raw(register_t icrlo, u_int dest)
1950 {
1951         uint64_t icr;
1952         uint32_t vhi, vlo;
1953         register_t saveintr;
1954
1955         /* XXX: Need more sanity checking of icrlo? */
1956         KASSERT(x2apic_mode || lapic_map != NULL,
1957             ("%s called too early", __func__));
1958         KASSERT(x2apic_mode ||
1959             (dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0,
1960             ("%s: invalid dest field", __func__));
1961         KASSERT((icrlo & APIC_ICRLO_RESV_MASK) == 0,
1962             ("%s: reserved bits set in ICR LO register", __func__));
1963
1964         /* Set destination in ICR HI register if it is being used. */
1965         if (!x2apic_mode) {
1966                 saveintr = intr_disable();
1967                 icr = lapic_read_icr();
1968         }
1969
1970         if ((icrlo & APIC_DEST_MASK) == APIC_DEST_DESTFLD) {
1971                 if (x2apic_mode) {
1972                         vhi = dest;
1973                 } else {
1974                         vhi = icr >> 32;
1975                         vhi &= ~APIC_ID_MASK;
1976                         vhi |= dest << APIC_ID_SHIFT;
1977                 }
1978         } else {
1979                 vhi = 0;
1980         }
1981
1982         /* Program the contents of the IPI and dispatch it. */
1983         if (x2apic_mode) {
1984                 vlo = icrlo;
1985         } else {
1986                 vlo = icr;
1987                 vlo &= APIC_ICRLO_RESV_MASK;
1988                 vlo |= icrlo;
1989         }
1990         lapic_write_icr(vhi, vlo);
1991         if (!x2apic_mode)
1992                 intr_restore(saveintr);
1993 }
1994
1995 #define BEFORE_SPIN     50000
1996 #ifdef DETECT_DEADLOCK
1997 #define AFTER_SPIN      50
1998 #endif
1999
2000 static void
2001 native_lapic_ipi_vectored(u_int vector, int dest)
2002 {
2003         register_t icrlo, destfield;
2004
2005         KASSERT((vector & ~APIC_VECTOR_MASK) == 0,
2006             ("%s: invalid vector %d", __func__, vector));
2007
2008         icrlo = APIC_DESTMODE_PHY | APIC_TRIGMOD_EDGE | APIC_LEVEL_ASSERT;
2009
2010         /*
2011          * NMI IPIs are just fake vectors used to send a NMI.  Use special rules
2012          * regarding NMIs if passed, otherwise specify the vector.
2013          */
2014         if (vector >= IPI_NMI_FIRST)
2015                 icrlo |= APIC_DELMODE_NMI;
2016         else
2017                 icrlo |= vector | APIC_DELMODE_FIXED;
2018         destfield = 0;
2019         switch (dest) {
2020         case APIC_IPI_DEST_SELF:
2021                 icrlo |= APIC_DEST_SELF;
2022                 break;
2023         case APIC_IPI_DEST_ALL:
2024                 icrlo |= APIC_DEST_ALLISELF;
2025                 break;
2026         case APIC_IPI_DEST_OTHERS:
2027                 icrlo |= APIC_DEST_ALLESELF;
2028                 break;
2029         default:
2030                 KASSERT(x2apic_mode ||
2031                     (dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0,
2032                     ("%s: invalid destination 0x%x", __func__, dest));
2033                 destfield = dest;
2034         }
2035
2036         /* Wait for an earlier IPI to finish. */
2037         if (!lapic_ipi_wait(BEFORE_SPIN)) {
2038                 if (panicstr != NULL)
2039                         return;
2040                 else
2041                         panic("APIC: Previous IPI is stuck");
2042         }
2043
2044         lapic_ipi_raw(icrlo, destfield);
2045
2046 #ifdef DETECT_DEADLOCK
2047         /* Wait for IPI to be delivered. */
2048         if (!lapic_ipi_wait(AFTER_SPIN)) {
2049 #ifdef needsattention
2050                 /*
2051                  * XXX FIXME:
2052                  *
2053                  * The above function waits for the message to actually be
2054                  * delivered.  It breaks out after an arbitrary timeout
2055                  * since the message should eventually be delivered (at
2056                  * least in theory) and that if it wasn't we would catch
2057                  * the failure with the check above when the next IPI is
2058                  * sent.
2059                  *
2060                  * We could skip this wait entirely, EXCEPT it probably
2061                  * protects us from other routines that assume that the
2062                  * message was delivered and acted upon when this function
2063                  * returns.
2064                  */
2065                 printf("APIC: IPI might be stuck\n");
2066 #else /* !needsattention */
2067                 /* Wait until mesage is sent without a timeout. */
2068                 while (lapic_read_icr_lo() & APIC_DELSTAT_PEND)
2069                         ia32_pause();
2070 #endif /* needsattention */
2071         }
2072 #endif /* DETECT_DEADLOCK */
2073 }
2074
2075 #endif /* SMP */
2076
2077 /*
2078  * Since the IDT is shared by all CPUs the IPI slot update needs to be globally
2079  * visible.
2080  *
2081  * Consider the case where an IPI is generated immediately after allocation:
2082  *     vector = lapic_ipi_alloc(ipifunc);
2083  *     ipi_selected(other_cpus, vector);
2084  *
2085  * In xAPIC mode a write to ICR_LO has serializing semantics because the
2086  * APIC page is mapped as an uncached region. In x2APIC mode there is an
2087  * explicit 'mfence' before the ICR MSR is written. Therefore in both cases
2088  * the IDT slot update is globally visible before the IPI is delivered.
2089  */
2090 static int
2091 native_lapic_ipi_alloc(inthand_t *ipifunc)
2092 {
2093         struct gate_descriptor *ip;
2094         long func;
2095         int idx, vector;
2096
2097         KASSERT(ipifunc != &IDTVEC(rsvd), ("invalid ipifunc %p", ipifunc));
2098
2099         vector = -1;
2100         mtx_lock_spin(&icu_lock);
2101         for (idx = IPI_DYN_FIRST; idx <= IPI_DYN_LAST; idx++) {
2102                 ip = &idt[idx];
2103                 func = (ip->gd_hioffset << 16) | ip->gd_looffset;
2104                 if (func == (uintptr_t)&IDTVEC(rsvd)) {
2105                         vector = idx;
2106                         setidt(vector, ipifunc, SDT_APIC, SEL_KPL, GSEL_APIC);
2107                         break;
2108                 }
2109         }
2110         mtx_unlock_spin(&icu_lock);
2111         return (vector);
2112 }
2113
2114 static void
2115 native_lapic_ipi_free(int vector)
2116 {
2117         struct gate_descriptor *ip;
2118         long func;
2119
2120         KASSERT(vector >= IPI_DYN_FIRST && vector <= IPI_DYN_LAST,
2121             ("%s: invalid vector %d", __func__, vector));
2122
2123         mtx_lock_spin(&icu_lock);
2124         ip = &idt[vector];
2125         func = (ip->gd_hioffset << 16) | ip->gd_looffset;
2126         KASSERT(func != (uintptr_t)&IDTVEC(rsvd),
2127             ("invalid idtfunc %#lx", func));
2128         setidt(vector, &IDTVEC(rsvd), SDT_APICT, SEL_KPL, GSEL_APIC);
2129         mtx_unlock_spin(&icu_lock);
2130 }