sys/amd64/vmm/io/vlapic.c

   1 /*-
   2  * Copyright (c) 2011 NetApp, Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  *
  26  * $FreeBSD$
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include <sys/param.h>
  33 #include <sys/lock.h>
  34 #include <sys/kernel.h>
  35 #include <sys/malloc.h>
  36 #include <sys/mutex.h>
  37 #include <sys/systm.h>
  38 #include <sys/smp.h>
  39
  40 #include <x86/specialreg.h>
  41 #include <x86/apicreg.h>
  42
  43 #include <machine/clock.h>
  44 #include <machine/smp.h>
  45
  46 #include <machine/vmm.h>
  47
  48 #include "vmm_ipi.h"
  49 #include "vmm_lapic.h"
  50 #include "vmm_ktr.h"
  51 #include "vmm_stat.h"
  52
  53 #include "vlapic.h"
  54 #include "vlapic_priv.h"
  55 #include "vioapic.h"
  56
  57 #define PRIO(x)                 ((x) >> 4)
  58
  59 #define VLAPIC_VERSION          (16)
  60
  61 #define x2apic(vlapic)  (((vlapic)->msr_apicbase & APICBASE_X2APIC) ? 1 : 0)
  62
  63 /*
  64  * The 'vlapic->timer_mtx' is used to provide mutual exclusion between the
  65  * vlapic_callout_handler() and vcpu accesses to:
  66  * - timer_freq_bt, timer_period_bt, timer_fire_bt
  67  * - timer LVT register
  68  */
  69 #define VLAPIC_TIMER_LOCK(vlapic)       mtx_lock_spin(&((vlapic)->timer_mtx))
  70 #define VLAPIC_TIMER_UNLOCK(vlapic)     mtx_unlock_spin(&((vlapic)->timer_mtx))
  71 #define VLAPIC_TIMER_LOCKED(vlapic)     mtx_owned(&((vlapic)->timer_mtx))
  72
  73 /*
  74  * APIC timer frequency:
  75  * - arbitrary but chosen to be in the ballpark of contemporary hardware.
  76  * - power-of-two to avoid loss of precision when converted to a bintime.
  77  */
  78 #define VLAPIC_BUS_FREQ         (128 * 1024 * 1024)
  79
  80 static __inline uint32_t
  81 vlapic_get_id(struct vlapic *vlapic)
  82 {
  83
  84         if (x2apic(vlapic))
  85                 return (vlapic->vcpuid);
  86         else
  87                 return (vlapic->vcpuid << 24);
  88 }
  89
  90 static uint32_t
  91 x2apic_ldr(struct vlapic *vlapic)
  92 {
  93         int apicid;
  94         uint32_t ldr;
  95
  96         apicid = vlapic_get_id(vlapic);
  97         ldr = 1 << (apicid & 0xf);
  98         ldr |= (apicid & 0xffff0) << 12;
  99         return (ldr);
 100 }
 101
 102 void
 103 vlapic_dfr_write_handler(struct vlapic *vlapic)
 104 {
 105         struct LAPIC *lapic;
 106
 107         lapic = vlapic->apic_page;
 108         if (x2apic(vlapic)) {
 109                 VM_CTR1(vlapic->vm, "ignoring write to DFR in x2apic mode: %#x",
 110                     lapic->dfr);
 111                 lapic->dfr = 0;
 112                 return;
 113         }
 114
 115         lapic->dfr &= APIC_DFR_MODEL_MASK;
 116         lapic->dfr |= APIC_DFR_RESERVED;
 117
 118         if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_FLAT)
 119                 VLAPIC_CTR0(vlapic, "vlapic DFR in Flat Model");
 120         else if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_CLUSTER)
 121                 VLAPIC_CTR0(vlapic, "vlapic DFR in Cluster Model");
 122         else
 123                 VLAPIC_CTR1(vlapic, "DFR in Unknown Model %#x", lapic->dfr);
 124 }
 125
 126 void
 127 vlapic_ldr_write_handler(struct vlapic *vlapic)
 128 {
 129         struct LAPIC *lapic;
 130
 131         lapic = vlapic->apic_page;
 132
 133         /* LDR is read-only in x2apic mode */
 134         if (x2apic(vlapic)) {
 135                 VLAPIC_CTR1(vlapic, "ignoring write to LDR in x2apic mode: %#x",
 136                     lapic->ldr);
 137                 lapic->ldr = x2apic_ldr(vlapic);
 138         } else {
 139                 lapic->ldr &= ~APIC_LDR_RESERVED;
 140                 VLAPIC_CTR1(vlapic, "vlapic LDR set to %#x", lapic->ldr);
 141         }
 142 }
 143
 144 void
 145 vlapic_id_write_handler(struct vlapic *vlapic)
 146 {
 147         struct LAPIC *lapic;
 148
 149         /*
 150          * We don't allow the ID register to be modified so reset it back to
 151          * its default value.
 152          */
 153         lapic = vlapic->apic_page;
 154         lapic->id = vlapic_get_id(vlapic);
 155 }
 156
 157 static int
 158 vlapic_timer_divisor(uint32_t dcr)
 159 {
 160         switch (dcr & 0xB) {
 161         case APIC_TDCR_1:
 162                 return (1);
 163         case APIC_TDCR_2:
 164                 return (2);
 165         case APIC_TDCR_4:
 166                 return (4);
 167         case APIC_TDCR_8:
 168                 return (8);
 169         case APIC_TDCR_16:
 170                 return (16);
 171         case APIC_TDCR_32:
 172                 return (32);
 173         case APIC_TDCR_64:
 174                 return (64);
 175         case APIC_TDCR_128:
 176                 return (128);
 177         default:
 178                 panic("vlapic_timer_divisor: invalid dcr 0x%08x", dcr);
 179         }
 180 }
 181
 182 #if 0
 183 static inline void
 184 vlapic_dump_lvt(uint32_t offset, uint32_t *lvt)
 185 {
 186         printf("Offset %x: lvt %08x (V:%02x DS:%x M:%x)\n", offset,
 187             *lvt, *lvt & APIC_LVTT_VECTOR, *lvt & APIC_LVTT_DS,
 188             *lvt & APIC_LVTT_M);
 189 }
 190 #endif
 191
 192 static uint32_t
 193 vlapic_get_ccr(struct vlapic *vlapic)
 194 {
 195         struct bintime bt_now, bt_rem;
 196         struct LAPIC *lapic;
 197         uint32_t ccr;
 198
 199         ccr = 0;
 200         lapic = vlapic->apic_page;
 201
 202         VLAPIC_TIMER_LOCK(vlapic);
 203         if (callout_active(&vlapic->callout)) {
 204                 /*
 205                  * If the timer is scheduled to expire in the future then
 206                  * compute the value of 'ccr' based on the remaining time.
 207                  */
 208                 binuptime(&bt_now);
 209                 if (bintime_cmp(&vlapic->timer_fire_bt, &bt_now, >)) {
 210                         bt_rem = vlapic->timer_fire_bt;
 211                         bintime_sub(&bt_rem, &bt_now);
 212                         ccr += bt_rem.sec * BT2FREQ(&vlapic->timer_freq_bt);
 213                         ccr += bt_rem.frac / vlapic->timer_freq_bt.frac;
 214                 }
 215         }
 216         KASSERT(ccr <= lapic->icr_timer, ("vlapic_get_ccr: invalid ccr %#x, "
 217             "icr_timer is %#x", ccr, lapic->icr_timer));
 218         VLAPIC_CTR2(vlapic, "vlapic ccr_timer = %#x, icr_timer = %#x",
 219             ccr, lapic->icr_timer);
 220         VLAPIC_TIMER_UNLOCK(vlapic);
 221         return (ccr);
 222 }
 223
 224 void
 225 vlapic_dcr_write_handler(struct vlapic *vlapic)
 226 {
 227         struct LAPIC *lapic;
 228         int divisor;
 229
 230         lapic = vlapic->apic_page;
 231         VLAPIC_TIMER_LOCK(vlapic);
 232
 233         divisor = vlapic_timer_divisor(lapic->dcr_timer);
 234         VLAPIC_CTR2(vlapic, "vlapic dcr_timer=%#x, divisor=%d",
 235             lapic->dcr_timer, divisor);
 236
 237         /*
 238          * Update the timer frequency and the timer period.
 239          *
 240          * XXX changes to the frequency divider will not take effect until
 241          * the timer is reloaded.
 242          */
 243         FREQ2BT(VLAPIC_BUS_FREQ / divisor, &vlapic->timer_freq_bt);
 244         vlapic->timer_period_bt = vlapic->timer_freq_bt;
 245         bintime_mul(&vlapic->timer_period_bt, lapic->icr_timer);
 246
 247         VLAPIC_TIMER_UNLOCK(vlapic);
 248 }
 249
 250 void
 251 vlapic_esr_write_handler(struct vlapic *vlapic)
 252 {
 253         struct LAPIC *lapic;
 254
 255         lapic = vlapic->apic_page;
 256         lapic->esr = vlapic->esr_pending;
 257         vlapic->esr_pending = 0;
 258 }
 259
 260 int
 261 vlapic_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
 262 {
 263         struct LAPIC *lapic;
 264         uint32_t *irrptr, *tmrptr, mask;
 265         int idx;
 266
 267         KASSERT(vector >= 0 && vector < 256, ("invalid vector %d", vector));
 268
 269         lapic = vlapic->apic_page;
 270         if (!(lapic->svr & APIC_SVR_ENABLE)) {
 271                 VLAPIC_CTR1(vlapic, "vlapic is software disabled, ignoring "
 272                     "interrupt %d", vector);
 273                 return (0);
 274         }
 275
 276         if (vector < 16) {
 277                 vlapic_set_error(vlapic, APIC_ESR_RECEIVE_ILLEGAL_VECTOR);
 278                 VLAPIC_CTR1(vlapic, "vlapic ignoring interrupt to vector %d",
 279                     vector);
 280                 return (1);
 281         }
 282
 283         if (vlapic->ops.set_intr_ready)
 284                 return ((*vlapic->ops.set_intr_ready)(vlapic, vector, level));
 285
 286         idx = (vector / 32) * 4;
 287         mask = 1 << (vector % 32);
 288
 289         irrptr = &lapic->irr0;
 290         atomic_set_int(&irrptr[idx], mask);
 291
 292         /*
 293          * Verify that the trigger-mode of the interrupt matches with
 294          * the vlapic TMR registers.
 295          */
 296         tmrptr = &lapic->tmr0;
 297         if ((tmrptr[idx] & mask) != (level ? mask : 0)) {
 298                 VLAPIC_CTR3(vlapic, "vlapic TMR[%d] is 0x%08x but "
 299                     "interrupt is %s-triggered", idx / 4, tmrptr[idx],
 300                     level ? "level" : "edge");
 301         }
 302
 303         VLAPIC_CTR_IRR(vlapic, "vlapic_set_intr_ready");
 304         return (1);
 305 }
 306
 307 static __inline uint32_t *
 308 vlapic_get_lvtptr(struct vlapic *vlapic, uint32_t offset)
 309 {
 310         struct LAPIC    *lapic = vlapic->apic_page;
 311         int              i;
 312
 313         switch (offset) {
 314         case APIC_OFFSET_CMCI_LVT:
 315                 return (&lapic->lvt_cmci);
 316         case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
 317                 i = (offset - APIC_OFFSET_TIMER_LVT) >> 2;
 318                 return ((&lapic->lvt_timer) + i);;
 319         default:
 320                 panic("vlapic_get_lvt: invalid LVT\n");
 321         }
 322 }
 323
 324 static __inline int
 325 lvt_off_to_idx(uint32_t offset)
 326 {
 327         int index;
 328
 329         switch (offset) {
 330         case APIC_OFFSET_CMCI_LVT:
 331                 index = APIC_LVT_CMCI;
 332                 break;
 333         case APIC_OFFSET_TIMER_LVT:
 334                 index = APIC_LVT_TIMER;
 335                 break;
 336         case APIC_OFFSET_THERM_LVT:
 337                 index = APIC_LVT_THERMAL;
 338                 break;
 339         case APIC_OFFSET_PERF_LVT:
 340                 index = APIC_LVT_PMC;
 341                 break;
 342         case APIC_OFFSET_LINT0_LVT:
 343                 index = APIC_LVT_LINT0;
 344                 break;
 345         case APIC_OFFSET_LINT1_LVT:
 346                 index = APIC_LVT_LINT1;
 347                 break;
 348         case APIC_OFFSET_ERROR_LVT:
 349                 index = APIC_LVT_ERROR;
 350                 break;
 351         default:
 352                 index = -1;
 353                 break;
 354         }
 355         KASSERT(index >= 0 && index <= VLAPIC_MAXLVT_INDEX, ("lvt_off_to_idx: "
 356             "invalid lvt index %d for offset %#x", index, offset));
 357
 358         return (index);
 359 }
 360
 361 static __inline uint32_t
 362 vlapic_get_lvt(struct vlapic *vlapic, uint32_t offset)
 363 {
 364         int idx;
 365         uint32_t val;
 366
 367         idx = lvt_off_to_idx(offset);
 368         val = atomic_load_acq_32(&vlapic->lvt_last[idx]);
 369         return (val);
 370 }
 371
 372 void
 373 vlapic_lvt_write_handler(struct vlapic *vlapic, uint32_t offset)
 374 {
 375         uint32_t *lvtptr, mask, val;
 376         struct LAPIC *lapic;
 377         int idx;
 378
 379         lapic = vlapic->apic_page;
 380         lvtptr = vlapic_get_lvtptr(vlapic, offset);
 381         val = *lvtptr;
 382         idx = lvt_off_to_idx(offset);
 383
 384         if (!(lapic->svr & APIC_SVR_ENABLE))
 385                 val |= APIC_LVT_M;
 386         mask = APIC_LVT_M | APIC_LVT_DS | APIC_LVT_VECTOR;
 387         switch (offset) {
 388         case APIC_OFFSET_TIMER_LVT:
 389                 mask |= APIC_LVTT_TM;
 390                 break;
 391         case APIC_OFFSET_ERROR_LVT:
 392                 break;
 393         case APIC_OFFSET_LINT0_LVT:
 394         case APIC_OFFSET_LINT1_LVT:
 395                 mask |= APIC_LVT_TM | APIC_LVT_RIRR | APIC_LVT_IIPP;
 396                 /* FALLTHROUGH */
 397         default:
 398                 mask |= APIC_LVT_DM;
 399                 break;
 400         }
 401         val &= mask;
 402         *lvtptr = val;
 403         atomic_store_rel_32(&vlapic->lvt_last[idx], val);
 404 }
 405
 406 static void
 407 vlapic_mask_lvts(struct vlapic *vlapic)
 408 {
 409         struct LAPIC *lapic = vlapic->apic_page;
 410
 411         lapic->lvt_cmci |= APIC_LVT_M;
 412         vlapic_lvt_write_handler(vlapic, APIC_OFFSET_CMCI_LVT);
 413
 414         lapic->lvt_timer |= APIC_LVT_M;
 415         vlapic_lvt_write_handler(vlapic, APIC_OFFSET_TIMER_LVT);
 416
 417         lapic->lvt_thermal |= APIC_LVT_M;
 418         vlapic_lvt_write_handler(vlapic, APIC_OFFSET_THERM_LVT);
 419
 420         lapic->lvt_pcint |= APIC_LVT_M;
 421         vlapic_lvt_write_handler(vlapic, APIC_OFFSET_PERF_LVT);
 422
 423         lapic->lvt_lint0 |= APIC_LVT_M;
 424         vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT0_LVT);
 425
 426         lapic->lvt_lint1 |= APIC_LVT_M;
 427         vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT1_LVT);
 428
 429         lapic->lvt_error |= APIC_LVT_M;
 430         vlapic_lvt_write_handler(vlapic, APIC_OFFSET_ERROR_LVT);
 431 }
 432
 433 static int
 434 vlapic_fire_lvt(struct vlapic *vlapic, uint32_t lvt)
 435 {
 436         uint32_t vec, mode;
 437
 438         if (lvt & APIC_LVT_M)
 439                 return (0);
 440
 441         vec = lvt & APIC_LVT_VECTOR;
 442         mode = lvt & APIC_LVT_DM;
 443
 444         switch (mode) {
 445         case APIC_LVT_DM_FIXED:
 446                 if (vec < 16) {
 447                         vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR);
 448                         return (0);
 449                 }
 450                 if (vlapic_set_intr_ready(vlapic, vec, false))
 451                         vcpu_notify_event(vlapic->vm, vlapic->vcpuid, true);
 452                 break;
 453         case APIC_LVT_DM_NMI:
 454                 vm_inject_nmi(vlapic->vm, vlapic->vcpuid);
 455                 break;
 456         case APIC_LVT_DM_EXTINT:
 457                 vm_inject_extint(vlapic->vm, vlapic->vcpuid);
 458                 break;
 459         default:
 460                 // Other modes ignored
 461                 return (0);
 462         }
 463         return (1);
 464 }
 465
 466 #if 1
 467 static void
 468 dump_isrvec_stk(struct vlapic *vlapic)
 469 {
 470         int i;
 471         uint32_t *isrptr;
 472
 473         isrptr = &vlapic->apic_page->isr0;
 474         for (i = 0; i < 8; i++)
 475                 printf("ISR%d 0x%08x\n", i, isrptr[i * 4]);
 476
 477         for (i = 0; i <= vlapic->isrvec_stk_top; i++)
 478                 printf("isrvec_stk[%d] = %d\n", i, vlapic->isrvec_stk[i]);
 479 }
 480 #endif
 481
 482 /*
 483  * Algorithm adopted from section "Interrupt, Task and Processor Priority"
 484  * in Intel Architecture Manual Vol 3a.
 485  */
 486 static void
 487 vlapic_update_ppr(struct vlapic *vlapic)
 488 {
 489         int isrvec, tpr, ppr;
 490
 491         /*
 492          * Note that the value on the stack at index 0 is always 0.
 493          *
 494          * This is a placeholder for the value of ISRV when none of the
 495          * bits is set in the ISRx registers.
 496          */
 497         isrvec = vlapic->isrvec_stk[vlapic->isrvec_stk_top];
 498         tpr = vlapic->apic_page->tpr;
 499
 500 #if 1
 501         {
 502                 int i, lastprio, curprio, vector, idx;
 503                 uint32_t *isrptr;
 504
 505                 if (vlapic->isrvec_stk_top == 0 && isrvec != 0)
 506                         panic("isrvec_stk is corrupted: %d", isrvec);
 507
 508                 /*
 509                  * Make sure that the priority of the nested interrupts is
 510                  * always increasing.
 511                  */
 512                 lastprio = -1;
 513                 for (i = 1; i <= vlapic->isrvec_stk_top; i++) {
 514                         curprio = PRIO(vlapic->isrvec_stk[i]);
 515                         if (curprio <= lastprio) {
 516                                 dump_isrvec_stk(vlapic);
 517                                 panic("isrvec_stk does not satisfy invariant");
 518                         }
 519                         lastprio = curprio;
 520                 }
 521
 522                 /*
 523                  * Make sure that each bit set in the ISRx registers has a
 524                  * corresponding entry on the isrvec stack.
 525                  */
 526                 i = 1;
 527                 isrptr = &vlapic->apic_page->isr0;
 528                 for (vector = 0; vector < 256; vector++) {
 529                         idx = (vector / 32) * 4;
 530                         if (isrptr[idx] & (1 << (vector % 32))) {
 531                                 if (i > vlapic->isrvec_stk_top ||
 532                                     vlapic->isrvec_stk[i] != vector) {
 533                                         dump_isrvec_stk(vlapic);
 534                                         panic("ISR and isrvec_stk out of sync");
 535                                 }
 536                                 i++;
 537                         }
 538                 }
 539         }
 540 #endif
 541
 542         if (PRIO(tpr) >= PRIO(isrvec))
 543                 ppr = tpr;
 544         else
 545                 ppr = isrvec & 0xf0;
 546
 547         vlapic->apic_page->ppr = ppr;
 548         VLAPIC_CTR1(vlapic, "vlapic_update_ppr 0x%02x", ppr);
 549 }
 550
 551 static VMM_STAT(VLAPIC_GRATUITOUS_EOI, "EOI without any in-service interrupt");
 552
 553 static void
 554 vlapic_process_eoi(struct vlapic *vlapic)
 555 {
 556         struct LAPIC    *lapic = vlapic->apic_page;
 557         uint32_t        *isrptr, *tmrptr;
 558         int             i, idx, bitpos, vector;
 559
 560         isrptr = &lapic->isr0;
 561         tmrptr = &lapic->tmr0;
 562
 563         for (i = 7; i >= 0; i--) {
 564                 idx = i * 4;
 565                 bitpos = fls(isrptr[idx]);
 566                 if (bitpos-- != 0) {
 567                         if (vlapic->isrvec_stk_top <= 0) {
 568                                 panic("invalid vlapic isrvec_stk_top %d",
 569                                       vlapic->isrvec_stk_top);
 570                         }
 571                         isrptr[idx] &= ~(1 << bitpos);
 572                         vector = i * 32 + bitpos;
 573                         VCPU_CTR1(vlapic->vm, vlapic->vcpuid, "EOI vector %d",
 574                             vector);
 575                         VLAPIC_CTR_ISR(vlapic, "vlapic_process_eoi");
 576                         vlapic->isrvec_stk_top--;
 577                         vlapic_update_ppr(vlapic);
 578                         if ((tmrptr[idx] & (1 << bitpos)) != 0) {
 579                                 vioapic_process_eoi(vlapic->vm, vlapic->vcpuid,
 580                                     vector);
 581                         }
 582                         return;
 583                 }
 584         }
 585         VCPU_CTR0(vlapic->vm, vlapic->vcpuid, "Gratuitous EOI");
 586         vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_GRATUITOUS_EOI, 1);
 587 }
 588
 589 static __inline int
 590 vlapic_get_lvt_field(uint32_t lvt, uint32_t mask)
 591 {
 592
 593         return (lvt & mask);
 594 }
 595
 596 static __inline int
 597 vlapic_periodic_timer(struct vlapic *vlapic)
 598 {
 599         uint32_t lvt;
 600
 601         lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
 602
 603         return (vlapic_get_lvt_field(lvt, APIC_LVTT_TM_PERIODIC));
 604 }
 605
 606 static VMM_STAT(VLAPIC_INTR_ERROR, "error interrupts generated by vlapic");
 607
 608 void
 609 vlapic_set_error(struct vlapic *vlapic, uint32_t mask)
 610 {
 611         uint32_t lvt;
 612
 613         vlapic->esr_pending |= mask;
 614         if (vlapic->esr_firing)
 615                 return;
 616         vlapic->esr_firing = 1;
 617
 618         // The error LVT always uses the fixed delivery mode.
 619         lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_ERROR_LVT);
 620         if (vlapic_fire_lvt(vlapic, lvt | APIC_LVT_DM_FIXED)) {
 621                 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_ERROR, 1);
 622         }
 623         vlapic->esr_firing = 0;
 624 }
 625
 626 static VMM_STAT(VLAPIC_INTR_TIMER, "timer interrupts generated by vlapic");
 627
 628 static void
 629 vlapic_fire_timer(struct vlapic *vlapic)
 630 {
 631         uint32_t lvt;
 632
 633         KASSERT(VLAPIC_TIMER_LOCKED(vlapic), ("vlapic_fire_timer not locked"));
 634
 635         // The timer LVT always uses the fixed delivery mode.
 636         lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
 637         if (vlapic_fire_lvt(vlapic, lvt | APIC_LVT_DM_FIXED)) {
 638                 VLAPIC_CTR0(vlapic, "vlapic timer fired");
 639                 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_TIMER, 1);
 640         }
 641 }
 642
 643 static VMM_STAT(VLAPIC_INTR_CMC,
 644     "corrected machine check interrupts generated by vlapic");
 645
 646 void
 647 vlapic_fire_cmci(struct vlapic *vlapic)
 648 {
 649         uint32_t lvt;
 650
 651         lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_CMCI_LVT);
 652         if (vlapic_fire_lvt(vlapic, lvt)) {
 653                 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_CMC, 1);
 654         }
 655 }
 656
 657 static VMM_STAT_ARRAY(LVTS_TRIGGERRED, VLAPIC_MAXLVT_INDEX + 1,
 658     "lvts triggered");
 659
 660 int
 661 vlapic_trigger_lvt(struct vlapic *vlapic, int vector)
 662 {
 663         uint32_t lvt;
 664
 665         if (vlapic_enabled(vlapic) == false) {
 666                 /*
 667                  * When the local APIC is global/hardware disabled,
 668                  * LINT[1:0] pins are configured as INTR and NMI pins,
 669                  * respectively.
 670                 */
 671                 switch (vector) {
 672                         case APIC_LVT_LINT0:
 673                                 vm_inject_extint(vlapic->vm, vlapic->vcpuid);
 674                                 break;
 675                         case APIC_LVT_LINT1:
 676                                 vm_inject_nmi(vlapic->vm, vlapic->vcpuid);
 677                                 break;
 678                         default:
 679                                 break;
 680                 }
 681                 return (0);
 682         }
 683
 684         switch (vector) {
 685         case APIC_LVT_LINT0:
 686                 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_LINT0_LVT);
 687                 break;
 688         case APIC_LVT_LINT1:
 689                 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_LINT1_LVT);
 690                 break;
 691         case APIC_LVT_TIMER:
 692                 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
 693                 lvt |= APIC_LVT_DM_FIXED;
 694                 break;
 695         case APIC_LVT_ERROR:
 696                 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_ERROR_LVT);
 697                 lvt |= APIC_LVT_DM_FIXED;
 698                 break;
 699         case APIC_LVT_PMC:
 700                 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_PERF_LVT);
 701                 break;
 702         case APIC_LVT_THERMAL:
 703                 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_THERM_LVT);
 704                 break;
 705         case APIC_LVT_CMCI:
 706                 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_CMCI_LVT);
 707                 break;
 708         default:
 709                 return (EINVAL);
 710         }
 711         if (vlapic_fire_lvt(vlapic, lvt)) {
 712                 vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid,
 713                     LVTS_TRIGGERRED, vector, 1);
 714         }
 715         return (0);
 716 }
 717
 718 static void
 719 vlapic_callout_handler(void *arg)
 720 {
 721         struct vlapic *vlapic;
 722         struct bintime bt, btnow;
 723         sbintime_t rem_sbt;
 724
 725         vlapic = arg;
 726
 727         VLAPIC_TIMER_LOCK(vlapic);
 728         if (callout_pending(&vlapic->callout))  /* callout was reset */
 729                 goto done;
 730
 731         if (!callout_active(&vlapic->callout))  /* callout was stopped */
 732                 goto done;
 733
 734         callout_deactivate(&vlapic->callout);
 735
 736         vlapic_fire_timer(vlapic);
 737
 738         if (vlapic_periodic_timer(vlapic)) {
 739                 binuptime(&btnow);
 740                 KASSERT(bintime_cmp(&btnow, &vlapic->timer_fire_bt, >=),
 741                     ("vlapic callout at %#lx.%#lx, expected at %#lx.#%lx",
 742                     btnow.sec, btnow.frac, vlapic->timer_fire_bt.sec,
 743                     vlapic->timer_fire_bt.frac));
 744
 745                 /*
 746                  * Compute the delta between when the timer was supposed to
 747                  * fire and the present time.
 748                  */
 749                 bt = btnow;
 750                 bintime_sub(&bt, &vlapic->timer_fire_bt);
 751
 752                 rem_sbt = bttosbt(vlapic->timer_period_bt);
 753                 if (bintime_cmp(&bt, &vlapic->timer_period_bt, <)) {
 754                         /*
 755                          * Adjust the time until the next countdown downward
 756                          * to account for the lost time.
 757                          */
 758                         rem_sbt -= bttosbt(bt);
 759                 } else {
 760                         /*
 761                          * If the delta is greater than the timer period then
 762                          * just reset our time base instead of trying to catch
 763                          * up.
 764                          */
 765                         vlapic->timer_fire_bt = btnow;
 766                         VLAPIC_CTR2(vlapic, "vlapic timer lagging by %lu "
 767                             "usecs, period is %lu usecs - resetting time base",
 768                             bttosbt(bt) / SBT_1US,
 769                             bttosbt(vlapic->timer_period_bt) / SBT_1US);
 770                 }
 771
 772                 bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
 773                 callout_reset_sbt(&vlapic->callout, rem_sbt, 0,
 774                     vlapic_callout_handler, vlapic, 0);
 775         }
 776 done:
 777         VLAPIC_TIMER_UNLOCK(vlapic);
 778 }
 779
 780 void
 781 vlapic_icrtmr_write_handler(struct vlapic *vlapic)
 782 {
 783         struct LAPIC *lapic;
 784         sbintime_t sbt;
 785         uint32_t icr_timer;
 786
 787         VLAPIC_TIMER_LOCK(vlapic);
 788
 789         lapic = vlapic->apic_page;
 790         icr_timer = lapic->icr_timer;
 791
 792         vlapic->timer_period_bt = vlapic->timer_freq_bt;
 793         bintime_mul(&vlapic->timer_period_bt, icr_timer);
 794
 795         if (icr_timer != 0) {
 796                 binuptime(&vlapic->timer_fire_bt);
 797                 bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
 798
 799                 sbt = bttosbt(vlapic->timer_period_bt);
 800                 callout_reset_sbt(&vlapic->callout, sbt, 0,
 801                     vlapic_callout_handler, vlapic, 0);
 802         } else
 803                 callout_stop(&vlapic->callout);
 804
 805         VLAPIC_TIMER_UNLOCK(vlapic);
 806 }
 807
 808 /*
 809  * This function populates 'dmask' with the set of vcpus that match the
 810  * addressing specified by the (dest, phys, lowprio) tuple.
 811  *
 812  * 'x2apic_dest' specifies whether 'dest' is interpreted as x2APIC (32-bit)
 813  * or xAPIC (8-bit) destination field.
 814  */
 815 static void
 816 vlapic_calcdest(struct vm *vm, cpuset_t *dmask, uint32_t dest, bool phys,
 817     bool lowprio, bool x2apic_dest)
 818 {
 819         struct vlapic *vlapic;
 820         uint32_t dfr, ldr, ldest, cluster;
 821         uint32_t mda_flat_ldest, mda_cluster_ldest, mda_ldest, mda_cluster_id;
 822         cpuset_t amask;
 823         int vcpuid;
 824
 825         if ((x2apic_dest && dest == 0xffffffff) ||
 826             (!x2apic_dest && dest == 0xff)) {
 827                 /*
 828                  * Broadcast in both logical and physical modes.
 829                  */
 830                 *dmask = vm_active_cpus(vm);
 831                 return;
 832         }
 833
 834         if (phys) {
 835                 /*
 836                  * Physical mode: destination is APIC ID.
 837                  */
 838                 CPU_ZERO(dmask);
 839                 vcpuid = vm_apicid2vcpuid(vm, dest);
 840                 if (vcpuid < VM_MAXCPU)
 841                         CPU_SET(vcpuid, dmask);
 842         } else {
 843                 /*
 844                  * In the "Flat Model" the MDA is interpreted as an 8-bit wide
 845                  * bitmask. This model is only avilable in the xAPIC mode.
 846                  */
 847                 mda_flat_ldest = dest & 0xff;
 848
 849                 /*
 850                  * In the "Cluster Model" the MDA is used to identify a
 851                  * specific cluster and a set of APICs in that cluster.
 852                  */
 853                 if (x2apic_dest) {
 854                         mda_cluster_id = dest >> 16;
 855                         mda_cluster_ldest = dest & 0xffff;
 856                 } else {
 857                         mda_cluster_id = (dest >> 4) & 0xf;
 858                         mda_cluster_ldest = dest & 0xf;
 859                 }
 860
 861                 /*
 862                  * Logical mode: match each APIC that has a bit set
 863                  * in it's LDR that matches a bit in the ldest.
 864                  */
 865                 CPU_ZERO(dmask);
 866                 amask = vm_active_cpus(vm);
 867                 while ((vcpuid = CPU_FFS(&amask)) != 0) {
 868                         vcpuid--;
 869                         CPU_CLR(vcpuid, &amask);
 870
 871                         vlapic = vm_lapic(vm, vcpuid);
 872                         dfr = vlapic->apic_page->dfr;
 873                         ldr = vlapic->apic_page->ldr;
 874
 875                         if ((dfr & APIC_DFR_MODEL_MASK) ==
 876                             APIC_DFR_MODEL_FLAT) {
 877                                 ldest = ldr >> 24;
 878                                 mda_ldest = mda_flat_ldest;
 879                         } else if ((dfr & APIC_DFR_MODEL_MASK) ==
 880                             APIC_DFR_MODEL_CLUSTER) {
 881                                 if (x2apic(vlapic)) {
 882                                         cluster = ldr >> 16;
 883                                         ldest = ldr & 0xffff;
 884                                 } else {
 885                                         cluster = ldr >> 28;
 886                                         ldest = (ldr >> 24) & 0xf;
 887                                 }
 888                                 if (cluster != mda_cluster_id)
 889                                         continue;
 890                                 mda_ldest = mda_cluster_ldest;
 891                         } else {
 892                                 /*
 893                                  * Guest has configured a bad logical
 894                                  * model for this vcpu - skip it.
 895                                  */
 896                                 VLAPIC_CTR1(vlapic, "vlapic has bad logical "
 897                                     "model %x - cannot deliver interrupt", dfr);
 898                                 continue;
 899                         }
 900
 901                         if ((mda_ldest & ldest) != 0) {
 902                                 CPU_SET(vcpuid, dmask);
 903                                 if (lowprio)
 904                                         break;
 905                         }
 906                 }
 907         }
 908 }
 909
 910 static VMM_STAT_ARRAY(IPIS_SENT, VM_MAXCPU, "ipis sent to vcpu");
 911
 912 static void
 913 vlapic_set_tpr(struct vlapic *vlapic, uint8_t val)
 914 {
 915         struct LAPIC *lapic = vlapic->apic_page;
 916
 917         if (lapic->tpr != val) {
 918                 VCPU_CTR2(vlapic->vm, vlapic->vcpuid, "vlapic TPR changed "
 919                     "from %#x to %#x", lapic->tpr, val);
 920                 lapic->tpr = val;
 921                 vlapic_update_ppr(vlapic);
 922         }
 923 }
 924
 925 static uint8_t
 926 vlapic_get_tpr(struct vlapic *vlapic)
 927 {
 928         struct LAPIC *lapic = vlapic->apic_page;
 929
 930         return (lapic->tpr);
 931 }
 932
 933 void
 934 vlapic_set_cr8(struct vlapic *vlapic, uint64_t val)
 935 {
 936         uint8_t tpr;
 937
 938         if (val & ~0xf) {
 939                 vm_inject_gp(vlapic->vm, vlapic->vcpuid);
 940                 return;
 941         }
 942
 943         tpr = val << 4;
 944         vlapic_set_tpr(vlapic, tpr);
 945 }
 946
 947 uint64_t
 948 vlapic_get_cr8(struct vlapic *vlapic)
 949 {
 950         uint8_t tpr;
 951
 952         tpr = vlapic_get_tpr(vlapic);
 953         return (tpr >> 4);
 954 }
 955
 956 int
 957 vlapic_icrlo_write_handler(struct vlapic *vlapic, bool *retu)
 958 {
 959         int i;
 960         bool phys;
 961         cpuset_t dmask;
 962         uint64_t icrval;
 963         uint32_t dest, vec, mode;
 964         struct vlapic *vlapic2;
 965         struct vm_exit *vmexit;
 966         struct LAPIC *lapic;
 967
 968         lapic = vlapic->apic_page;
 969         lapic->icr_lo &= ~APIC_DELSTAT_PEND;
 970         icrval = ((uint64_t)lapic->icr_hi << 32) | lapic->icr_lo;
 971
 972         if (x2apic(vlapic))
 973                 dest = icrval >> 32;
 974         else
 975                 dest = icrval >> (32 + 24);
 976         vec = icrval & APIC_VECTOR_MASK;
 977         mode = icrval & APIC_DELMODE_MASK;
 978
 979         if (mode == APIC_DELMODE_FIXED && vec < 16) {
 980                 vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR);
 981                 VLAPIC_CTR1(vlapic, "Ignoring invalid IPI %d", vec);
 982                 return (0);
 983         }
 984
 985         VLAPIC_CTR2(vlapic, "icrlo 0x%016lx triggered ipi %d", icrval, vec);
 986
 987         if (mode == APIC_DELMODE_FIXED || mode == APIC_DELMODE_NMI) {
 988                 switch (icrval & APIC_DEST_MASK) {
 989                 case APIC_DEST_DESTFLD:
 990                         phys = ((icrval & APIC_DESTMODE_LOG) == 0);
 991                         vlapic_calcdest(vlapic->vm, &dmask, dest, phys, false,
 992                             x2apic(vlapic));
 993                         break;
 994                 case APIC_DEST_SELF:
 995                         CPU_SETOF(vlapic->vcpuid, &dmask);
 996                         break;
 997                 case APIC_DEST_ALLISELF:
 998                         dmask = vm_active_cpus(vlapic->vm);
 999                         break;
1000                 case APIC_DEST_ALLESELF:
1001                         dmask = vm_active_cpus(vlapic->vm);
1002                         CPU_CLR(vlapic->vcpuid, &dmask);
1003                         break;
1004                 default:
1005                         CPU_ZERO(&dmask);       /* satisfy gcc */
1006                         break;
1007                 }
1008
1009                 while ((i = CPU_FFS(&dmask)) != 0) {
1010                         i--;
1011                         CPU_CLR(i, &dmask);
1012                         if (mode == APIC_DELMODE_FIXED) {
1013                                 lapic_intr_edge(vlapic->vm, i, vec);
1014                                 vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid,
1015                                                     IPIS_SENT, i, 1);
1016                                 VLAPIC_CTR2(vlapic, "vlapic sending ipi %d "
1017                                     "to vcpuid %d", vec, i);
1018                         } else {
1019                                 vm_inject_nmi(vlapic->vm, i);
1020                                 VLAPIC_CTR1(vlapic, "vlapic sending ipi nmi "
1021                                     "to vcpuid %d", i);
1022                         }
1023                 }
1024
1025                 return (0);     /* handled completely in the kernel */
1026         }
1027
1028         if (mode == APIC_DELMODE_INIT) {
1029                 if ((icrval & APIC_LEVEL_MASK) == APIC_LEVEL_DEASSERT)
1030                         return (0);
1031
1032                 if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) {
1033                         vlapic2 = vm_lapic(vlapic->vm, dest);
1034
1035                         /* move from INIT to waiting-for-SIPI state */
1036                         if (vlapic2->boot_state == BS_INIT) {
1037                                 vlapic2->boot_state = BS_SIPI;
1038                         }
1039
1040                         return (0);
1041                 }
1042         }
1043
1044         if (mode == APIC_DELMODE_STARTUP) {
1045                 if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) {
1046                         vlapic2 = vm_lapic(vlapic->vm, dest);
1047
1048                         /*
1049                          * Ignore SIPIs in any state other than wait-for-SIPI
1050                          */
1051                         if (vlapic2->boot_state != BS_SIPI)
1052                                 return (0);
1053
1054                         vlapic2->boot_state = BS_RUNNING;
1055
1056                         *retu = true;
1057                         vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid);
1058                         vmexit->exitcode = VM_EXITCODE_SPINUP_AP;
1059                         vmexit->u.spinup_ap.vcpu = dest;
1060                         vmexit->u.spinup_ap.rip = vec << PAGE_SHIFT;
1061
1062                         return (0);
1063                 }
1064         }
1065
1066         /*
1067          * This will cause a return to userland.
1068          */
1069         return (1);
1070 }
1071
1072 void
1073 vlapic_self_ipi_handler(struct vlapic *vlapic, uint64_t val)
1074 {
1075         int vec;
1076
1077         KASSERT(x2apic(vlapic), ("SELF_IPI does not exist in xAPIC mode"));
1078
1079         vec = val & 0xff;
1080         lapic_intr_edge(vlapic->vm, vlapic->vcpuid, vec);
1081         vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid, IPIS_SENT,
1082             vlapic->vcpuid, 1);
1083         VLAPIC_CTR1(vlapic, "vlapic self-ipi %d", vec);
1084 }
1085
1086 int
1087 vlapic_pending_intr(struct vlapic *vlapic, int *vecptr)
1088 {
1089         struct LAPIC    *lapic = vlapic->apic_page;
1090         int              idx, i, bitpos, vector;
1091         uint32_t        *irrptr, val;
1092
1093         if (vlapic->ops.pending_intr)
1094                 return ((*vlapic->ops.pending_intr)(vlapic, vecptr));
1095
1096         irrptr = &lapic->irr0;
1097
1098         for (i = 7; i >= 0; i--) {
1099                 idx = i * 4;
1100                 val = atomic_load_acq_int(&irrptr[idx]);
1101                 bitpos = fls(val);
1102                 if (bitpos != 0) {
1103                         vector = i * 32 + (bitpos - 1);
1104                         if (PRIO(vector) > PRIO(lapic->ppr)) {
1105                                 VLAPIC_CTR1(vlapic, "pending intr %d", vector);
1106                                 if (vecptr != NULL)
1107                                         *vecptr = vector;
1108                                 return (1);
1109                         } else
1110                                 break;
1111                 }
1112         }
1113         return (0);
1114 }
1115
1116 void
1117 vlapic_intr_accepted(struct vlapic *vlapic, int vector)
1118 {
1119         struct LAPIC    *lapic = vlapic->apic_page;
1120         uint32_t        *irrptr, *isrptr;
1121         int             idx, stk_top;
1122
1123         if (vlapic->ops.intr_accepted)
1124                 return ((*vlapic->ops.intr_accepted)(vlapic, vector));
1125
1126         /*
1127          * clear the ready bit for vector being accepted in irr
1128          * and set the vector as in service in isr.
1129          */
1130         idx = (vector / 32) * 4;
1131
1132         irrptr = &lapic->irr0;
1133         atomic_clear_int(&irrptr[idx], 1 << (vector % 32));
1134         VLAPIC_CTR_IRR(vlapic, "vlapic_intr_accepted");
1135
1136         isrptr = &lapic->isr0;
1137         isrptr[idx] |= 1 << (vector % 32);
1138         VLAPIC_CTR_ISR(vlapic, "vlapic_intr_accepted");
1139
1140         /*
1141          * Update the PPR
1142          */
1143         vlapic->isrvec_stk_top++;
1144
1145         stk_top = vlapic->isrvec_stk_top;
1146         if (stk_top >= ISRVEC_STK_SIZE)
1147                 panic("isrvec_stk_top overflow %d", stk_top);
1148
1149         vlapic->isrvec_stk[stk_top] = vector;
1150         vlapic_update_ppr(vlapic);
1151 }
1152
1153 void
1154 vlapic_svr_write_handler(struct vlapic *vlapic)
1155 {
1156         struct LAPIC *lapic;
1157         uint32_t old, new, changed;
1158
1159         lapic = vlapic->apic_page;
1160
1161         new = lapic->svr;
1162         old = vlapic->svr_last;
1163         vlapic->svr_last = new;
1164
1165         changed = old ^ new;
1166         if ((changed & APIC_SVR_ENABLE) != 0) {
1167                 if ((new & APIC_SVR_ENABLE) == 0) {
1168                         /*
1169                          * The apic is now disabled so stop the apic timer
1170                          * and mask all the LVT entries.
1171                          */
1172                         VLAPIC_CTR0(vlapic, "vlapic is software-disabled");
1173                         VLAPIC_TIMER_LOCK(vlapic);
1174                         callout_stop(&vlapic->callout);
1175                         VLAPIC_TIMER_UNLOCK(vlapic);
1176                         vlapic_mask_lvts(vlapic);
1177                 } else {
1178                         /*
1179                          * The apic is now enabled so restart the apic timer
1180                          * if it is configured in periodic mode.
1181                          */
1182                         VLAPIC_CTR0(vlapic, "vlapic is software-enabled");
1183                         if (vlapic_periodic_timer(vlapic))
1184                                 vlapic_icrtmr_write_handler(vlapic);
1185                 }
1186         }
1187 }
1188
1189 int
1190 vlapic_read(struct vlapic *vlapic, int mmio_access, uint64_t offset,
1191     uint64_t *data, bool *retu)
1192 {
1193         struct LAPIC    *lapic = vlapic->apic_page;
1194         uint32_t        *reg;
1195         int              i;
1196
1197         /* Ignore MMIO accesses in x2APIC mode */
1198         if (x2apic(vlapic) && mmio_access) {
1199                 VLAPIC_CTR1(vlapic, "MMIO read from offset %#lx in x2APIC mode",
1200                     offset);
1201                 *data = 0;
1202                 goto done;
1203         }
1204
1205         if (!x2apic(vlapic) && !mmio_access) {
1206                 /*
1207                  * XXX Generate GP fault for MSR accesses in xAPIC mode
1208                  */
1209                 VLAPIC_CTR1(vlapic, "x2APIC MSR read from offset %#lx in "
1210                     "xAPIC mode", offset);
1211                 *data = 0;
1212                 goto done;
1213         }
1214
1215         if (offset > sizeof(*lapic)) {
1216                 *data = 0;
1217                 goto done;
1218         }
1219
1220         offset &= ~3;
1221         switch(offset)
1222         {
1223                 case APIC_OFFSET_ID:
1224                         *data = lapic->id;
1225                         break;
1226                 case APIC_OFFSET_VER:
1227                         *data = lapic->version;
1228                         break;
1229                 case APIC_OFFSET_TPR:
1230                         *data = vlapic_get_tpr(vlapic);
1231                         break;
1232                 case APIC_OFFSET_APR:
1233                         *data = lapic->apr;
1234                         break;
1235                 case APIC_OFFSET_PPR:
1236                         *data = lapic->ppr;
1237                         break;
1238                 case APIC_OFFSET_EOI:
1239                         *data = lapic->eoi;
1240                         break;
1241                 case APIC_OFFSET_LDR:
1242                         *data = lapic->ldr;
1243                         break;
1244                 case APIC_OFFSET_DFR:
1245                         *data = lapic->dfr;
1246                         break;
1247                 case APIC_OFFSET_SVR:
1248                         *data = lapic->svr;
1249                         break;
1250                 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
1251                         i = (offset - APIC_OFFSET_ISR0) >> 2;
1252                         reg = &lapic->isr0;
1253                         *data = *(reg + i);
1254                         break;
1255                 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
1256                         i = (offset - APIC_OFFSET_TMR0) >> 2;
1257                         reg = &lapic->tmr0;
1258                         *data = *(reg + i);
1259                         break;
1260                 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
1261                         i = (offset - APIC_OFFSET_IRR0) >> 2;
1262                         reg = &lapic->irr0;
1263                         *data = atomic_load_acq_int(reg + i);
1264                         break;
1265                 case APIC_OFFSET_ESR:
1266                         *data = lapic->esr;
1267                         break;
1268                 case APIC_OFFSET_ICR_LOW:
1269                         *data = lapic->icr_lo;
1270                         if (x2apic(vlapic))
1271                                 *data |= (uint64_t)lapic->icr_hi << 32;
1272                         break;
1273                 case APIC_OFFSET_ICR_HI:
1274                         *data = lapic->icr_hi;
1275                         break;
1276                 case APIC_OFFSET_CMCI_LVT:
1277                 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
1278                         *data = vlapic_get_lvt(vlapic, offset);
1279 #ifdef INVARIANTS
1280                         reg = vlapic_get_lvtptr(vlapic, offset);
1281                         KASSERT(*data == *reg, ("inconsistent lvt value at "
1282                             "offset %#lx: %#lx/%#x", offset, *data, *reg));
1283 #endif
1284                         break;
1285                 case APIC_OFFSET_TIMER_ICR:
1286                         *data = lapic->icr_timer;
1287                         break;
1288                 case APIC_OFFSET_TIMER_CCR:
1289                         *data = vlapic_get_ccr(vlapic);
1290                         break;
1291                 case APIC_OFFSET_TIMER_DCR:
1292                         *data = lapic->dcr_timer;
1293                         break;
1294                 case APIC_OFFSET_SELF_IPI:
1295                         /*
1296                          * XXX generate a GP fault if vlapic is in x2apic mode
1297                          */
1298                         *data = 0;
1299                         break;
1300                 case APIC_OFFSET_RRR:
1301                 default:
1302                         *data = 0;
1303                         break;
1304         }
1305 done:
1306         VLAPIC_CTR2(vlapic, "vlapic read offset %#x, data %#lx", offset, *data);
1307         return 0;
1308 }
1309
1310 int
1311 vlapic_write(struct vlapic *vlapic, int mmio_access, uint64_t offset,
1312     uint64_t data, bool *retu)
1313 {
1314         struct LAPIC    *lapic = vlapic->apic_page;
1315         uint32_t        *regptr;
1316         int             retval;
1317
1318         KASSERT((offset & 0xf) == 0 && offset < PAGE_SIZE,
1319             ("vlapic_write: invalid offset %#lx", offset));
1320
1321         VLAPIC_CTR2(vlapic, "vlapic write offset %#lx, data %#lx",
1322             offset, data);
1323
1324         if (offset > sizeof(*lapic))
1325                 return (0);
1326
1327         /* Ignore MMIO accesses in x2APIC mode */
1328         if (x2apic(vlapic) && mmio_access) {
1329                 VLAPIC_CTR2(vlapic, "MMIO write of %#lx to offset %#lx "
1330                     "in x2APIC mode", data, offset);
1331                 return (0);
1332         }
1333
1334         /*
1335          * XXX Generate GP fault for MSR accesses in xAPIC mode
1336          */
1337         if (!x2apic(vlapic) && !mmio_access) {
1338                 VLAPIC_CTR2(vlapic, "x2APIC MSR write of %#lx to offset %#lx "
1339                     "in xAPIC mode", data, offset);
1340                 return (0);
1341         }
1342
1343         retval = 0;
1344         switch(offset)
1345         {
1346                 case APIC_OFFSET_ID:
1347                         lapic->id = data;
1348                         vlapic_id_write_handler(vlapic);
1349                         break;
1350                 case APIC_OFFSET_TPR:
1351                         vlapic_set_tpr(vlapic, data & 0xff);
1352                         break;
1353                 case APIC_OFFSET_EOI:
1354                         vlapic_process_eoi(vlapic);
1355                         break;
1356                 case APIC_OFFSET_LDR:
1357                         lapic->ldr = data;
1358                         vlapic_ldr_write_handler(vlapic);
1359                         break;
1360                 case APIC_OFFSET_DFR:
1361                         lapic->dfr = data;
1362                         vlapic_dfr_write_handler(vlapic);
1363                         break;
1364                 case APIC_OFFSET_SVR:
1365                         lapic->svr = data;
1366                         vlapic_svr_write_handler(vlapic);
1367                         break;
1368                 case APIC_OFFSET_ICR_LOW:
1369                         lapic->icr_lo = data;
1370                         if (x2apic(vlapic))
1371                                 lapic->icr_hi = data >> 32;
1372                         retval = vlapic_icrlo_write_handler(vlapic, retu);
1373                         break;
1374                 case APIC_OFFSET_ICR_HI:
1375                         lapic->icr_hi = data;
1376                         break;
1377                 case APIC_OFFSET_CMCI_LVT:
1378                 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
1379                         regptr = vlapic_get_lvtptr(vlapic, offset);
1380                         *regptr = data;
1381                         vlapic_lvt_write_handler(vlapic, offset);
1382                         break;
1383                 case APIC_OFFSET_TIMER_ICR:
1384                         lapic->icr_timer = data;
1385                         vlapic_icrtmr_write_handler(vlapic);
1386                         break;
1387
1388                 case APIC_OFFSET_TIMER_DCR:
1389                         lapic->dcr_timer = data;
1390                         vlapic_dcr_write_handler(vlapic);
1391                         break;
1392
1393                 case APIC_OFFSET_ESR:
1394                         vlapic_esr_write_handler(vlapic);
1395                         break;
1396
1397                 case APIC_OFFSET_SELF_IPI:
1398                         if (x2apic(vlapic))
1399                                 vlapic_self_ipi_handler(vlapic, data);
1400                         break;
1401
1402                 case APIC_OFFSET_VER:
1403                 case APIC_OFFSET_APR:
1404                 case APIC_OFFSET_PPR:
1405                 case APIC_OFFSET_RRR:
1406                 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
1407                 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
1408                 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
1409                 case APIC_OFFSET_TIMER_CCR:
1410                 default:
1411                         // Read only.
1412                         break;
1413         }
1414
1415         return (retval);
1416 }
1417
1418 static void
1419 vlapic_reset(struct vlapic *vlapic)
1420 {
1421         struct LAPIC *lapic;
1422
1423         lapic = vlapic->apic_page;
1424         bzero(lapic, sizeof(struct LAPIC));
1425
1426         lapic->id = vlapic_get_id(vlapic);
1427         lapic->version = VLAPIC_VERSION;
1428         lapic->version |= (VLAPIC_MAXLVT_INDEX << MAXLVTSHIFT);
1429         lapic->dfr = 0xffffffff;
1430         lapic->svr = APIC_SVR_VECTOR;
1431         vlapic_mask_lvts(vlapic);
1432         vlapic_reset_tmr(vlapic);
1433
1434         lapic->dcr_timer = 0;
1435         vlapic_dcr_write_handler(vlapic);
1436
1437         if (vlapic->vcpuid == 0)
1438                 vlapic->boot_state = BS_RUNNING;        /* BSP */
1439         else
1440                 vlapic->boot_state = BS_INIT;           /* AP */
1441
1442         vlapic->svr_last = lapic->svr;
1443 }
1444
1445 void
1446 vlapic_init(struct vlapic *vlapic)
1447 {
1448         KASSERT(vlapic->vm != NULL, ("vlapic_init: vm is not initialized"));
1449         KASSERT(vlapic->vcpuid >= 0 && vlapic->vcpuid < VM_MAXCPU,
1450             ("vlapic_init: vcpuid is not initialized"));
1451         KASSERT(vlapic->apic_page != NULL, ("vlapic_init: apic_page is not "
1452             "initialized"));
1453
1454         /*
1455          * If the vlapic is configured in x2apic mode then it will be
1456          * accessed in the critical section via the MSR emulation code.
1457          *
1458          * Therefore the timer mutex must be a spinlock because blockable
1459          * mutexes cannot be acquired in a critical section.
1460          */
1461         mtx_init(&vlapic->timer_mtx, "vlapic timer mtx", NULL, MTX_SPIN);
1462         callout_init(&vlapic->callout, 1);
1463
1464         vlapic->msr_apicbase = DEFAULT_APIC_BASE | APICBASE_ENABLED;
1465
1466         if (vlapic->vcpuid == 0)
1467                 vlapic->msr_apicbase |= APICBASE_BSP;
1468
1469         vlapic_reset(vlapic);
1470 }
1471
1472 void
1473 vlapic_cleanup(struct vlapic *vlapic)
1474 {
1475
1476         callout_drain(&vlapic->callout);
1477 }
1478
1479 uint64_t
1480 vlapic_get_apicbase(struct vlapic *vlapic)
1481 {
1482
1483         return (vlapic->msr_apicbase);
1484 }
1485
1486 int
1487 vlapic_set_apicbase(struct vlapic *vlapic, uint64_t new)
1488 {
1489
1490         if (vlapic->msr_apicbase != new) {
1491                 VLAPIC_CTR2(vlapic, "Changing APIC_BASE MSR from %#lx to %#lx "
1492                     "not supported", vlapic->msr_apicbase, new);
1493                 return (-1);
1494         }
1495
1496         return (0);
1497 }
1498
1499 void
1500 vlapic_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
1501 {
1502         struct vlapic *vlapic;
1503         struct LAPIC *lapic;
1504
1505         vlapic = vm_lapic(vm, vcpuid);
1506
1507         if (state == X2APIC_DISABLED)
1508                 vlapic->msr_apicbase &= ~APICBASE_X2APIC;
1509         else
1510                 vlapic->msr_apicbase |= APICBASE_X2APIC;
1511
1512         /*
1513          * Reset the local APIC registers whose values are mode-dependent.
1514          *
1515          * XXX this works because the APIC mode can be changed only at vcpu
1516          * initialization time.
1517          */
1518         lapic = vlapic->apic_page;
1519         lapic->id = vlapic_get_id(vlapic);
1520         if (x2apic(vlapic)) {
1521                 lapic->ldr = x2apic_ldr(vlapic);
1522                 lapic->dfr = 0;
1523         } else {
1524                 lapic->ldr = 0;
1525                 lapic->dfr = 0xffffffff;
1526         }
1527
1528         if (state == X2APIC_ENABLED) {
1529                 if (vlapic->ops.enable_x2apic_mode)
1530                         (*vlapic->ops.enable_x2apic_mode)(vlapic);
1531         }
1532 }
1533
1534 void
1535 vlapic_deliver_intr(struct vm *vm, bool level, uint32_t dest, bool phys,
1536     int delmode, int vec)
1537 {
1538         bool lowprio;
1539         int vcpuid;
1540         cpuset_t dmask;
1541
1542         if (delmode != IOART_DELFIXED &&
1543             delmode != IOART_DELLOPRI &&
1544             delmode != IOART_DELEXINT) {
1545                 VM_CTR1(vm, "vlapic intr invalid delmode %#x", delmode);
1546                 return;
1547         }
1548         lowprio = (delmode == IOART_DELLOPRI);
1549
1550         /*
1551          * We don't provide any virtual interrupt redirection hardware so
1552          * all interrupts originating from the ioapic or MSI specify the
1553          * 'dest' in the legacy xAPIC format.
1554          */
1555         vlapic_calcdest(vm, &dmask, dest, phys, lowprio, false);
1556
1557         while ((vcpuid = CPU_FFS(&dmask)) != 0) {
1558                 vcpuid--;
1559                 CPU_CLR(vcpuid, &dmask);
1560                 if (delmode == IOART_DELEXINT) {
1561                         vm_inject_extint(vm, vcpuid);
1562                 } else {
1563                         lapic_set_intr(vm, vcpuid, vec, level);
1564                 }
1565         }
1566 }
1567
1568 void
1569 vlapic_post_intr(struct vlapic *vlapic, int hostcpu, int ipinum)
1570 {
1571         /*
1572          * Post an interrupt to the vcpu currently running on 'hostcpu'.
1573          *
1574          * This is done by leveraging features like Posted Interrupts (Intel)
1575          * Doorbell MSR (AMD AVIC) that avoid a VM exit.
1576          *
1577          * If neither of these features are available then fallback to
1578          * sending an IPI to 'hostcpu'.
1579          */
1580         if (vlapic->ops.post_intr)
1581                 (*vlapic->ops.post_intr)(vlapic, hostcpu);
1582         else
1583                 ipi_cpu(hostcpu, ipinum);
1584 }
1585
1586 bool
1587 vlapic_enabled(struct vlapic *vlapic)
1588 {
1589         struct LAPIC *lapic = vlapic->apic_page;
1590
1591         if ((vlapic->msr_apicbase & APICBASE_ENABLED) != 0 &&
1592             (lapic->svr & APIC_SVR_ENABLE) != 0)
1593                 return (true);
1594         else
1595                 return (false);
1596 }
1597
1598 static void
1599 vlapic_set_tmr(struct vlapic *vlapic, int vector, bool level)
1600 {
1601         struct LAPIC *lapic;
1602         uint32_t *tmrptr, mask;
1603         int idx;
1604
1605         lapic = vlapic->apic_page;
1606         tmrptr = &lapic->tmr0;
1607         idx = (vector / 32) * 4;
1608         mask = 1 << (vector % 32);
1609         if (level)
1610                 tmrptr[idx] |= mask;
1611         else
1612                 tmrptr[idx] &= ~mask;
1613
1614         if (vlapic->ops.set_tmr != NULL)
1615                 (*vlapic->ops.set_tmr)(vlapic, vector, level);
1616 }
1617
1618 void
1619 vlapic_reset_tmr(struct vlapic *vlapic)
1620 {
1621         int vector;
1622
1623         VLAPIC_CTR0(vlapic, "vlapic resetting all vectors to edge-triggered");
1624
1625         for (vector = 0; vector <= 255; vector++)
1626                 vlapic_set_tmr(vlapic, vector, false);
1627 }
1628
1629 void
1630 vlapic_set_tmr_level(struct vlapic *vlapic, uint32_t dest, bool phys,
1631     int delmode, int vector)
1632 {
1633         cpuset_t dmask;
1634         bool lowprio;
1635
1636         KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
1637
1638         /*
1639          * A level trigger is valid only for fixed and lowprio delivery modes.
1640          */
1641         if (delmode != APIC_DELMODE_FIXED && delmode != APIC_DELMODE_LOWPRIO) {
1642                 VLAPIC_CTR1(vlapic, "Ignoring level trigger-mode for "
1643                     "delivery-mode %d", delmode);
1644                 return;
1645         }
1646
1647         lowprio = (delmode == APIC_DELMODE_LOWPRIO);
1648         vlapic_calcdest(vlapic->vm, &dmask, dest, phys, lowprio, false);
1649
1650         if (!CPU_ISSET(vlapic->vcpuid, &dmask))
1651                 return;
1652
1653         VLAPIC_CTR1(vlapic, "vector %d set to level-triggered", vector);
1654         vlapic_set_tmr(vlapic, vector, true);
1655 }