sys/amd64/vmm/io/vlapic.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2011 NetApp, Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  *
  28  * $FreeBSD$
  29  */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33
  34 #include <sys/param.h>
  35 #include <sys/lock.h>
  36 #include <sys/kernel.h>
  37 #include <sys/malloc.h>
  38 #include <sys/mutex.h>
  39 #include <sys/systm.h>
  40 #include <sys/smp.h>
  41
  42 #include <x86/specialreg.h>
  43 #include <x86/apicreg.h>
  44
  45 #include <machine/clock.h>
  46 #include <machine/smp.h>
  47
  48 #include <machine/vmm.h>
  49
  50 #include "vmm_lapic.h"
  51 #include "vmm_ktr.h"
  52 #include "vmm_stat.h"
  53
  54 #include "vlapic.h"
  55 #include "vlapic_priv.h"
  56 #include "vioapic.h"
  57
  58 #define PRIO(x)                 ((x) >> 4)
  59
  60 #define VLAPIC_VERSION          (16)
  61
  62 #define x2apic(vlapic)  (((vlapic)->msr_apicbase & APICBASE_X2APIC) ? 1 : 0)
  63
  64 /*
  65  * The 'vlapic->timer_mtx' is used to provide mutual exclusion between the
  66  * vlapic_callout_handler() and vcpu accesses to:
  67  * - timer_freq_bt, timer_period_bt, timer_fire_bt
  68  * - timer LVT register
  69  */
  70 #define VLAPIC_TIMER_LOCK(vlapic)       mtx_lock_spin(&((vlapic)->timer_mtx))
  71 #define VLAPIC_TIMER_UNLOCK(vlapic)     mtx_unlock_spin(&((vlapic)->timer_mtx))
  72 #define VLAPIC_TIMER_LOCKED(vlapic)     mtx_owned(&((vlapic)->timer_mtx))
  73
  74 /*
  75  * APIC timer frequency:
  76  * - arbitrary but chosen to be in the ballpark of contemporary hardware.
  77  * - power-of-two to avoid loss of precision when converted to a bintime.
  78  */
  79 #define VLAPIC_BUS_FREQ         (128 * 1024 * 1024)
  80
  81 static __inline uint32_t
  82 vlapic_get_id(struct vlapic *vlapic)
  83 {
  84
  85         if (x2apic(vlapic))
  86                 return (vlapic->vcpuid);
  87         else
  88                 return (vlapic->vcpuid << 24);
  89 }
  90
  91 static uint32_t
  92 x2apic_ldr(struct vlapic *vlapic)
  93 {
  94         int apicid;
  95         uint32_t ldr;
  96
  97         apicid = vlapic_get_id(vlapic);
  98         ldr = 1 << (apicid & 0xf);
  99         ldr |= (apicid & 0xffff0) << 12;
 100         return (ldr);
 101 }
 102
 103 void
 104 vlapic_dfr_write_handler(struct vlapic *vlapic)
 105 {
 106         struct LAPIC *lapic;
 107
 108         lapic = vlapic->apic_page;
 109         if (x2apic(vlapic)) {
 110                 VM_CTR1(vlapic->vm, "ignoring write to DFR in x2apic mode: %#x",
 111                     lapic->dfr);
 112                 lapic->dfr = 0;
 113                 return;
 114         }
 115
 116         lapic->dfr &= APIC_DFR_MODEL_MASK;
 117         lapic->dfr |= APIC_DFR_RESERVED;
 118
 119         if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_FLAT)
 120                 VLAPIC_CTR0(vlapic, "vlapic DFR in Flat Model");
 121         else if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_CLUSTER)
 122                 VLAPIC_CTR0(vlapic, "vlapic DFR in Cluster Model");
 123         else
 124                 VLAPIC_CTR1(vlapic, "DFR in Unknown Model %#x", lapic->dfr);
 125 }
 126
 127 void
 128 vlapic_ldr_write_handler(struct vlapic *vlapic)
 129 {
 130         struct LAPIC *lapic;
 131
 132         lapic = vlapic->apic_page;
 133
 134         /* LDR is read-only in x2apic mode */
 135         if (x2apic(vlapic)) {
 136                 VLAPIC_CTR1(vlapic, "ignoring write to LDR in x2apic mode: %#x",
 137                     lapic->ldr);
 138                 lapic->ldr = x2apic_ldr(vlapic);
 139         } else {
 140                 lapic->ldr &= ~APIC_LDR_RESERVED;
 141                 VLAPIC_CTR1(vlapic, "vlapic LDR set to %#x", lapic->ldr);
 142         }
 143 }
 144
 145 void
 146 vlapic_id_write_handler(struct vlapic *vlapic)
 147 {
 148         struct LAPIC *lapic;
 149
 150         /*
 151          * We don't allow the ID register to be modified so reset it back to
 152          * its default value.
 153          */
 154         lapic = vlapic->apic_page;
 155         lapic->id = vlapic_get_id(vlapic);
 156 }
 157
 158 static int
 159 vlapic_timer_divisor(uint32_t dcr)
 160 {
 161         switch (dcr & 0xB) {
 162         case APIC_TDCR_1:
 163                 return (1);
 164         case APIC_TDCR_2:
 165                 return (2);
 166         case APIC_TDCR_4:
 167                 return (4);
 168         case APIC_TDCR_8:
 169                 return (8);
 170         case APIC_TDCR_16:
 171                 return (16);
 172         case APIC_TDCR_32:
 173                 return (32);
 174         case APIC_TDCR_64:
 175                 return (64);
 176         case APIC_TDCR_128:
 177                 return (128);
 178         default:
 179                 panic("vlapic_timer_divisor: invalid dcr 0x%08x", dcr);
 180         }
 181 }
 182
 183 #if 0
 184 static inline void
 185 vlapic_dump_lvt(uint32_t offset, uint32_t *lvt)
 186 {
 187         printf("Offset %x: lvt %08x (V:%02x DS:%x M:%x)\n", offset,
 188             *lvt, *lvt & APIC_LVTT_VECTOR, *lvt & APIC_LVTT_DS,
 189             *lvt & APIC_LVTT_M);
 190 }
 191 #endif
 192
 193 static uint32_t
 194 vlapic_get_ccr(struct vlapic *vlapic)
 195 {
 196         struct bintime bt_now, bt_rem;
 197         struct LAPIC *lapic;
 198         uint32_t ccr;
 199
 200         ccr = 0;
 201         lapic = vlapic->apic_page;
 202
 203         VLAPIC_TIMER_LOCK(vlapic);
 204         if (callout_active(&vlapic->callout)) {
 205                 /*
 206                  * If the timer is scheduled to expire in the future then
 207                  * compute the value of 'ccr' based on the remaining time.
 208                  */
 209                 binuptime(&bt_now);
 210                 if (bintime_cmp(&vlapic->timer_fire_bt, &bt_now, >)) {
 211                         bt_rem = vlapic->timer_fire_bt;
 212                         bintime_sub(&bt_rem, &bt_now);
 213                         ccr += bt_rem.sec * BT2FREQ(&vlapic->timer_freq_bt);
 214                         ccr += bt_rem.frac / vlapic->timer_freq_bt.frac;
 215                 }
 216         }
 217         KASSERT(ccr <= lapic->icr_timer, ("vlapic_get_ccr: invalid ccr %#x, "
 218             "icr_timer is %#x", ccr, lapic->icr_timer));
 219         VLAPIC_CTR2(vlapic, "vlapic ccr_timer = %#x, icr_timer = %#x",
 220             ccr, lapic->icr_timer);
 221         VLAPIC_TIMER_UNLOCK(vlapic);
 222         return (ccr);
 223 }
 224
 225 void
 226 vlapic_dcr_write_handler(struct vlapic *vlapic)
 227 {
 228         struct LAPIC *lapic;
 229         int divisor;
 230
 231         lapic = vlapic->apic_page;
 232         VLAPIC_TIMER_LOCK(vlapic);
 233
 234         divisor = vlapic_timer_divisor(lapic->dcr_timer);
 235         VLAPIC_CTR2(vlapic, "vlapic dcr_timer=%#x, divisor=%d",
 236             lapic->dcr_timer, divisor);
 237
 238         /*
 239          * Update the timer frequency and the timer period.
 240          *
 241          * XXX changes to the frequency divider will not take effect until
 242          * the timer is reloaded.
 243          */
 244         FREQ2BT(VLAPIC_BUS_FREQ / divisor, &vlapic->timer_freq_bt);
 245         vlapic->timer_period_bt = vlapic->timer_freq_bt;
 246         bintime_mul(&vlapic->timer_period_bt, lapic->icr_timer);
 247
 248         VLAPIC_TIMER_UNLOCK(vlapic);
 249 }
 250
 251 void
 252 vlapic_esr_write_handler(struct vlapic *vlapic)
 253 {
 254         struct LAPIC *lapic;
 255
 256         lapic = vlapic->apic_page;
 257         lapic->esr = vlapic->esr_pending;
 258         vlapic->esr_pending = 0;
 259 }
 260
 261 int
 262 vlapic_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
 263 {
 264         struct LAPIC *lapic;
 265         uint32_t *irrptr, *tmrptr, mask;
 266         int idx;
 267
 268         KASSERT(vector >= 0 && vector < 256, ("invalid vector %d", vector));
 269
 270         lapic = vlapic->apic_page;
 271         if (!(lapic->svr & APIC_SVR_ENABLE)) {
 272                 VLAPIC_CTR1(vlapic, "vlapic is software disabled, ignoring "
 273                     "interrupt %d", vector);
 274                 return (0);
 275         }
 276
 277         if (vector < 16) {
 278                 vlapic_set_error(vlapic, APIC_ESR_RECEIVE_ILLEGAL_VECTOR);
 279                 VLAPIC_CTR1(vlapic, "vlapic ignoring interrupt to vector %d",
 280                     vector);
 281                 return (1);
 282         }
 283
 284         if (vlapic->ops.set_intr_ready)
 285                 return ((*vlapic->ops.set_intr_ready)(vlapic, vector, level));
 286
 287         idx = (vector / 32) * 4;
 288         mask = 1 << (vector % 32);
 289
 290         irrptr = &lapic->irr0;
 291         atomic_set_int(&irrptr[idx], mask);
 292
 293         /*
 294          * Verify that the trigger-mode of the interrupt matches with
 295          * the vlapic TMR registers.
 296          */
 297         tmrptr = &lapic->tmr0;
 298         if ((tmrptr[idx] & mask) != (level ? mask : 0)) {
 299                 VLAPIC_CTR3(vlapic, "vlapic TMR[%d] is 0x%08x but "
 300                     "interrupt is %s-triggered", idx / 4, tmrptr[idx],
 301                     level ? "level" : "edge");
 302         }
 303
 304         VLAPIC_CTR_IRR(vlapic, "vlapic_set_intr_ready");
 305         return (1);
 306 }
 307
 308 static __inline uint32_t *
 309 vlapic_get_lvtptr(struct vlapic *vlapic, uint32_t offset)
 310 {
 311         struct LAPIC    *lapic = vlapic->apic_page;
 312         int              i;
 313
 314         switch (offset) {
 315         case APIC_OFFSET_CMCI_LVT:
 316                 return (&lapic->lvt_cmci);
 317         case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
 318                 i = (offset - APIC_OFFSET_TIMER_LVT) >> 2;
 319                 return ((&lapic->lvt_timer) + i);
 320         default:
 321                 panic("vlapic_get_lvt: invalid LVT\n");
 322         }
 323 }
 324
 325 static __inline int
 326 lvt_off_to_idx(uint32_t offset)
 327 {
 328         int index;
 329
 330         switch (offset) {
 331         case APIC_OFFSET_CMCI_LVT:
 332                 index = APIC_LVT_CMCI;
 333                 break;
 334         case APIC_OFFSET_TIMER_LVT:
 335                 index = APIC_LVT_TIMER;
 336                 break;
 337         case APIC_OFFSET_THERM_LVT:
 338                 index = APIC_LVT_THERMAL;
 339                 break;
 340         case APIC_OFFSET_PERF_LVT:
 341                 index = APIC_LVT_PMC;
 342                 break;
 343         case APIC_OFFSET_LINT0_LVT:
 344                 index = APIC_LVT_LINT0;
 345                 break;
 346         case APIC_OFFSET_LINT1_LVT:
 347                 index = APIC_LVT_LINT1;
 348                 break;
 349         case APIC_OFFSET_ERROR_LVT:
 350                 index = APIC_LVT_ERROR;
 351                 break;
 352         default:
 353                 index = -1;
 354                 break;
 355         }
 356         KASSERT(index >= 0 && index <= VLAPIC_MAXLVT_INDEX, ("lvt_off_to_idx: "
 357             "invalid lvt index %d for offset %#x", index, offset));
 358
 359         return (index);
 360 }
 361
 362 static __inline uint32_t
 363 vlapic_get_lvt(struct vlapic *vlapic, uint32_t offset)
 364 {
 365         int idx;
 366         uint32_t val;
 367
 368         idx = lvt_off_to_idx(offset);
 369         val = atomic_load_acq_32(&vlapic->lvt_last[idx]);
 370         return (val);
 371 }
 372
 373 void
 374 vlapic_lvt_write_handler(struct vlapic *vlapic, uint32_t offset)
 375 {
 376         uint32_t *lvtptr, mask, val;
 377         struct LAPIC *lapic;
 378         int idx;
 379
 380         lapic = vlapic->apic_page;
 381         lvtptr = vlapic_get_lvtptr(vlapic, offset);
 382         val = *lvtptr;
 383         idx = lvt_off_to_idx(offset);
 384
 385         if (!(lapic->svr & APIC_SVR_ENABLE))
 386                 val |= APIC_LVT_M;
 387         mask = APIC_LVT_M | APIC_LVT_DS | APIC_LVT_VECTOR;
 388         switch (offset) {
 389         case APIC_OFFSET_TIMER_LVT:
 390                 mask |= APIC_LVTT_TM;
 391                 break;
 392         case APIC_OFFSET_ERROR_LVT:
 393                 break;
 394         case APIC_OFFSET_LINT0_LVT:
 395         case APIC_OFFSET_LINT1_LVT:
 396                 mask |= APIC_LVT_TM | APIC_LVT_RIRR | APIC_LVT_IIPP;
 397                 /* FALLTHROUGH */
 398         default:
 399                 mask |= APIC_LVT_DM;
 400                 break;
 401         }
 402         val &= mask;
 403         *lvtptr = val;
 404         atomic_store_rel_32(&vlapic->lvt_last[idx], val);
 405 }
 406
 407 static void
 408 vlapic_mask_lvts(struct vlapic *vlapic)
 409 {
 410         struct LAPIC *lapic = vlapic->apic_page;
 411
 412         lapic->lvt_cmci |= APIC_LVT_M;
 413         vlapic_lvt_write_handler(vlapic, APIC_OFFSET_CMCI_LVT);
 414
 415         lapic->lvt_timer |= APIC_LVT_M;
 416         vlapic_lvt_write_handler(vlapic, APIC_OFFSET_TIMER_LVT);
 417
 418         lapic->lvt_thermal |= APIC_LVT_M;
 419         vlapic_lvt_write_handler(vlapic, APIC_OFFSET_THERM_LVT);
 420
 421         lapic->lvt_pcint |= APIC_LVT_M;
 422         vlapic_lvt_write_handler(vlapic, APIC_OFFSET_PERF_LVT);
 423
 424         lapic->lvt_lint0 |= APIC_LVT_M;
 425         vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT0_LVT);
 426
 427         lapic->lvt_lint1 |= APIC_LVT_M;
 428         vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT1_LVT);
 429
 430         lapic->lvt_error |= APIC_LVT_M;
 431         vlapic_lvt_write_handler(vlapic, APIC_OFFSET_ERROR_LVT);
 432 }
 433
 434 static int
 435 vlapic_fire_lvt(struct vlapic *vlapic, uint32_t lvt)
 436 {
 437         uint32_t vec, mode;
 438
 439         if (lvt & APIC_LVT_M)
 440                 return (0);
 441
 442         vec = lvt & APIC_LVT_VECTOR;
 443         mode = lvt & APIC_LVT_DM;
 444
 445         switch (mode) {
 446         case APIC_LVT_DM_FIXED:
 447                 if (vec < 16) {
 448                         vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR);
 449                         return (0);
 450                 }
 451                 if (vlapic_set_intr_ready(vlapic, vec, false))
 452                         vcpu_notify_event(vlapic->vm, vlapic->vcpuid, true);
 453                 break;
 454         case APIC_LVT_DM_NMI:
 455                 vm_inject_nmi(vlapic->vm, vlapic->vcpuid);
 456                 break;
 457         case APIC_LVT_DM_EXTINT:
 458                 vm_inject_extint(vlapic->vm, vlapic->vcpuid);
 459                 break;
 460         default:
 461                 // Other modes ignored
 462                 return (0);
 463         }
 464         return (1);
 465 }
 466
 467 #if 1
 468 static void
 469 dump_isrvec_stk(struct vlapic *vlapic)
 470 {
 471         int i;
 472         uint32_t *isrptr;
 473
 474         isrptr = &vlapic->apic_page->isr0;
 475         for (i = 0; i < 8; i++)
 476                 printf("ISR%d 0x%08x\n", i, isrptr[i * 4]);
 477
 478         for (i = 0; i <= vlapic->isrvec_stk_top; i++)
 479                 printf("isrvec_stk[%d] = %d\n", i, vlapic->isrvec_stk[i]);
 480 }
 481 #endif
 482
 483 /*
 484  * Algorithm adopted from section "Interrupt, Task and Processor Priority"
 485  * in Intel Architecture Manual Vol 3a.
 486  */
 487 static void
 488 vlapic_update_ppr(struct vlapic *vlapic)
 489 {
 490         int isrvec, tpr, ppr;
 491
 492         /*
 493          * Note that the value on the stack at index 0 is always 0.
 494          *
 495          * This is a placeholder for the value of ISRV when none of the
 496          * bits is set in the ISRx registers.
 497          */
 498         isrvec = vlapic->isrvec_stk[vlapic->isrvec_stk_top];
 499         tpr = vlapic->apic_page->tpr;
 500
 501 #if 1
 502         {
 503                 int i, lastprio, curprio, vector, idx;
 504                 uint32_t *isrptr;
 505
 506                 if (vlapic->isrvec_stk_top == 0 && isrvec != 0)
 507                         panic("isrvec_stk is corrupted: %d", isrvec);
 508
 509                 /*
 510                  * Make sure that the priority of the nested interrupts is
 511                  * always increasing.
 512                  */
 513                 lastprio = -1;
 514                 for (i = 1; i <= vlapic->isrvec_stk_top; i++) {
 515                         curprio = PRIO(vlapic->isrvec_stk[i]);
 516                         if (curprio <= lastprio) {
 517                                 dump_isrvec_stk(vlapic);
 518                                 panic("isrvec_stk does not satisfy invariant");
 519                         }
 520                         lastprio = curprio;
 521                 }
 522
 523                 /*
 524                  * Make sure that each bit set in the ISRx registers has a
 525                  * corresponding entry on the isrvec stack.
 526                  */
 527                 i = 1;
 528                 isrptr = &vlapic->apic_page->isr0;
 529                 for (vector = 0; vector < 256; vector++) {
 530                         idx = (vector / 32) * 4;
 531                         if (isrptr[idx] & (1 << (vector % 32))) {
 532                                 if (i > vlapic->isrvec_stk_top ||
 533                                     vlapic->isrvec_stk[i] != vector) {
 534                                         dump_isrvec_stk(vlapic);
 535                                         panic("ISR and isrvec_stk out of sync");
 536                                 }
 537                                 i++;
 538                         }
 539                 }
 540         }
 541 #endif
 542
 543         if (PRIO(tpr) >= PRIO(isrvec))
 544                 ppr = tpr;
 545         else
 546                 ppr = isrvec & 0xf0;
 547
 548         vlapic->apic_page->ppr = ppr;
 549         VLAPIC_CTR1(vlapic, "vlapic_update_ppr 0x%02x", ppr);
 550 }
 551
 552 void
 553 vlapic_sync_tpr(struct vlapic *vlapic)
 554 {
 555         vlapic_update_ppr(vlapic);
 556 }
 557
 558 static VMM_STAT(VLAPIC_GRATUITOUS_EOI, "EOI without any in-service interrupt");
 559
 560 static void
 561 vlapic_process_eoi(struct vlapic *vlapic)
 562 {
 563         struct LAPIC    *lapic = vlapic->apic_page;
 564         uint32_t        *isrptr, *tmrptr;
 565         int             i, idx, bitpos, vector;
 566
 567         isrptr = &lapic->isr0;
 568         tmrptr = &lapic->tmr0;
 569
 570         for (i = 7; i >= 0; i--) {
 571                 idx = i * 4;
 572                 bitpos = fls(isrptr[idx]);
 573                 if (bitpos-- != 0) {
 574                         if (vlapic->isrvec_stk_top <= 0) {
 575                                 panic("invalid vlapic isrvec_stk_top %d",
 576                                       vlapic->isrvec_stk_top);
 577                         }
 578                         isrptr[idx] &= ~(1 << bitpos);
 579                         vector = i * 32 + bitpos;
 580                         VCPU_CTR1(vlapic->vm, vlapic->vcpuid, "EOI vector %d",
 581                             vector);
 582                         VLAPIC_CTR_ISR(vlapic, "vlapic_process_eoi");
 583                         vlapic->isrvec_stk_top--;
 584                         vlapic_update_ppr(vlapic);
 585                         if ((tmrptr[idx] & (1 << bitpos)) != 0) {
 586                                 vioapic_process_eoi(vlapic->vm, vlapic->vcpuid,
 587                                     vector);
 588                         }
 589                         return;
 590                 }
 591         }
 592         VCPU_CTR0(vlapic->vm, vlapic->vcpuid, "Gratuitous EOI");
 593         vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_GRATUITOUS_EOI, 1);
 594 }
 595
 596 static __inline int
 597 vlapic_get_lvt_field(uint32_t lvt, uint32_t mask)
 598 {
 599
 600         return (lvt & mask);
 601 }
 602
 603 static __inline int
 604 vlapic_periodic_timer(struct vlapic *vlapic)
 605 {
 606         uint32_t lvt;
 607
 608         lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
 609
 610         return (vlapic_get_lvt_field(lvt, APIC_LVTT_TM_PERIODIC));
 611 }
 612
 613 static VMM_STAT(VLAPIC_INTR_ERROR, "error interrupts generated by vlapic");
 614
 615 void
 616 vlapic_set_error(struct vlapic *vlapic, uint32_t mask)
 617 {
 618         uint32_t lvt;
 619
 620         vlapic->esr_pending |= mask;
 621         if (vlapic->esr_firing)
 622                 return;
 623         vlapic->esr_firing = 1;
 624
 625         // The error LVT always uses the fixed delivery mode.
 626         lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_ERROR_LVT);
 627         if (vlapic_fire_lvt(vlapic, lvt | APIC_LVT_DM_FIXED)) {
 628                 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_ERROR, 1);
 629         }
 630         vlapic->esr_firing = 0;
 631 }
 632
 633 static VMM_STAT(VLAPIC_INTR_TIMER, "timer interrupts generated by vlapic");
 634
 635 static void
 636 vlapic_fire_timer(struct vlapic *vlapic)
 637 {
 638         uint32_t lvt;
 639
 640         KASSERT(VLAPIC_TIMER_LOCKED(vlapic), ("vlapic_fire_timer not locked"));
 641
 642         // The timer LVT always uses the fixed delivery mode.
 643         lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
 644         if (vlapic_fire_lvt(vlapic, lvt | APIC_LVT_DM_FIXED)) {
 645                 VLAPIC_CTR0(vlapic, "vlapic timer fired");
 646                 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_TIMER, 1);
 647         }
 648 }
 649
 650 static VMM_STAT(VLAPIC_INTR_CMC,
 651     "corrected machine check interrupts generated by vlapic");
 652
 653 void
 654 vlapic_fire_cmci(struct vlapic *vlapic)
 655 {
 656         uint32_t lvt;
 657
 658         lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_CMCI_LVT);
 659         if (vlapic_fire_lvt(vlapic, lvt)) {
 660                 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_CMC, 1);
 661         }
 662 }
 663
 664 static VMM_STAT_ARRAY(LVTS_TRIGGERRED, VLAPIC_MAXLVT_INDEX + 1,
 665     "lvts triggered");
 666
 667 int
 668 vlapic_trigger_lvt(struct vlapic *vlapic, int vector)
 669 {
 670         uint32_t lvt;
 671
 672         if (vlapic_enabled(vlapic) == false) {
 673                 /*
 674                  * When the local APIC is global/hardware disabled,
 675                  * LINT[1:0] pins are configured as INTR and NMI pins,
 676                  * respectively.
 677                 */
 678                 switch (vector) {
 679                         case APIC_LVT_LINT0:
 680                                 vm_inject_extint(vlapic->vm, vlapic->vcpuid);
 681                                 break;
 682                         case APIC_LVT_LINT1:
 683                                 vm_inject_nmi(vlapic->vm, vlapic->vcpuid);
 684                                 break;
 685                         default:
 686                                 break;
 687                 }
 688                 return (0);
 689         }
 690
 691         switch (vector) {
 692         case APIC_LVT_LINT0:
 693                 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_LINT0_LVT);
 694                 break;
 695         case APIC_LVT_LINT1:
 696                 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_LINT1_LVT);
 697                 break;
 698         case APIC_LVT_TIMER:
 699                 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
 700                 lvt |= APIC_LVT_DM_FIXED;
 701                 break;
 702         case APIC_LVT_ERROR:
 703                 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_ERROR_LVT);
 704                 lvt |= APIC_LVT_DM_FIXED;
 705                 break;
 706         case APIC_LVT_PMC:
 707                 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_PERF_LVT);
 708                 break;
 709         case APIC_LVT_THERMAL:
 710                 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_THERM_LVT);
 711                 break;
 712         case APIC_LVT_CMCI:
 713                 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_CMCI_LVT);
 714                 break;
 715         default:
 716                 return (EINVAL);
 717         }
 718         if (vlapic_fire_lvt(vlapic, lvt)) {
 719                 vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid,
 720                     LVTS_TRIGGERRED, vector, 1);
 721         }
 722         return (0);
 723 }
 724
 725 static void
 726 vlapic_callout_handler(void *arg)
 727 {
 728         struct vlapic *vlapic;
 729         struct bintime bt, btnow;
 730         sbintime_t rem_sbt;
 731
 732         vlapic = arg;
 733
 734         VLAPIC_TIMER_LOCK(vlapic);
 735         if (callout_pending(&vlapic->callout))  /* callout was reset */
 736                 goto done;
 737
 738         if (!callout_active(&vlapic->callout))  /* callout was stopped */
 739                 goto done;
 740
 741         callout_deactivate(&vlapic->callout);
 742
 743         vlapic_fire_timer(vlapic);
 744
 745         if (vlapic_periodic_timer(vlapic)) {
 746                 binuptime(&btnow);
 747                 KASSERT(bintime_cmp(&btnow, &vlapic->timer_fire_bt, >=),
 748                     ("vlapic callout at %#lx.%#lx, expected at %#lx.#%lx",
 749                     btnow.sec, btnow.frac, vlapic->timer_fire_bt.sec,
 750                     vlapic->timer_fire_bt.frac));
 751
 752                 /*
 753                  * Compute the delta between when the timer was supposed to
 754                  * fire and the present time.
 755                  */
 756                 bt = btnow;
 757                 bintime_sub(&bt, &vlapic->timer_fire_bt);
 758
 759                 rem_sbt = bttosbt(vlapic->timer_period_bt);
 760                 if (bintime_cmp(&bt, &vlapic->timer_period_bt, <)) {
 761                         /*
 762                          * Adjust the time until the next countdown downward
 763                          * to account for the lost time.
 764                          */
 765                         rem_sbt -= bttosbt(bt);
 766                 } else {
 767                         /*
 768                          * If the delta is greater than the timer period then
 769                          * just reset our time base instead of trying to catch
 770                          * up.
 771                          */
 772                         vlapic->timer_fire_bt = btnow;
 773                         VLAPIC_CTR2(vlapic, "vlapic timer lagging by %lu "
 774                             "usecs, period is %lu usecs - resetting time base",
 775                             bttosbt(bt) / SBT_1US,
 776                             bttosbt(vlapic->timer_period_bt) / SBT_1US);
 777                 }
 778
 779                 bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
 780                 callout_reset_sbt(&vlapic->callout, rem_sbt, 0,
 781                     vlapic_callout_handler, vlapic, 0);
 782         }
 783 done:
 784         VLAPIC_TIMER_UNLOCK(vlapic);
 785 }
 786
 787 void
 788 vlapic_icrtmr_write_handler(struct vlapic *vlapic)
 789 {
 790         struct LAPIC *lapic;
 791         sbintime_t sbt;
 792         uint32_t icr_timer;
 793
 794         VLAPIC_TIMER_LOCK(vlapic);
 795
 796         lapic = vlapic->apic_page;
 797         icr_timer = lapic->icr_timer;
 798
 799         vlapic->timer_period_bt = vlapic->timer_freq_bt;
 800         bintime_mul(&vlapic->timer_period_bt, icr_timer);
 801
 802         if (icr_timer != 0) {
 803                 binuptime(&vlapic->timer_fire_bt);
 804                 bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
 805
 806                 sbt = bttosbt(vlapic->timer_period_bt);
 807                 callout_reset_sbt(&vlapic->callout, sbt, 0,
 808                     vlapic_callout_handler, vlapic, 0);
 809         } else
 810                 callout_stop(&vlapic->callout);
 811
 812         VLAPIC_TIMER_UNLOCK(vlapic);
 813 }
 814
 815 /*
 816  * This function populates 'dmask' with the set of vcpus that match the
 817  * addressing specified by the (dest, phys, lowprio) tuple.
 818  *
 819  * 'x2apic_dest' specifies whether 'dest' is interpreted as x2APIC (32-bit)
 820  * or xAPIC (8-bit) destination field.
 821  */
 822 static void
 823 vlapic_calcdest(struct vm *vm, cpuset_t *dmask, uint32_t dest, bool phys,
 824     bool lowprio, bool x2apic_dest)
 825 {
 826         struct vlapic *vlapic;
 827         uint32_t dfr, ldr, ldest, cluster;
 828         uint32_t mda_flat_ldest, mda_cluster_ldest, mda_ldest, mda_cluster_id;
 829         cpuset_t amask;
 830         int vcpuid;
 831
 832         if ((x2apic_dest && dest == 0xffffffff) ||
 833             (!x2apic_dest && dest == 0xff)) {
 834                 /*
 835                  * Broadcast in both logical and physical modes.
 836                  */
 837                 *dmask = vm_active_cpus(vm);
 838                 return;
 839         }
 840
 841         if (phys) {
 842                 /*
 843                  * Physical mode: destination is APIC ID.
 844                  */
 845                 CPU_ZERO(dmask);
 846                 vcpuid = vm_apicid2vcpuid(vm, dest);
 847                 amask = vm_active_cpus(vm);
 848                 if (vcpuid < vm_get_maxcpus(vm) && CPU_ISSET(vcpuid, &amask))
 849                         CPU_SET(vcpuid, dmask);
 850         } else {
 851                 /*
 852                  * In the "Flat Model" the MDA is interpreted as an 8-bit wide
 853                  * bitmask. This model is only available in the xAPIC mode.
 854                  */
 855                 mda_flat_ldest = dest & 0xff;
 856
 857                 /*
 858                  * In the "Cluster Model" the MDA is used to identify a
 859                  * specific cluster and a set of APICs in that cluster.
 860                  */
 861                 if (x2apic_dest) {
 862                         mda_cluster_id = dest >> 16;
 863                         mda_cluster_ldest = dest & 0xffff;
 864                 } else {
 865                         mda_cluster_id = (dest >> 4) & 0xf;
 866                         mda_cluster_ldest = dest & 0xf;
 867                 }
 868
 869                 /*
 870                  * Logical mode: match each APIC that has a bit set
 871                  * in its LDR that matches a bit in the ldest.
 872                  */
 873                 CPU_ZERO(dmask);
 874                 amask = vm_active_cpus(vm);
 875                 while ((vcpuid = CPU_FFS(&amask)) != 0) {
 876                         vcpuid--;
 877                         CPU_CLR(vcpuid, &amask);
 878
 879                         vlapic = vm_lapic(vm, vcpuid);
 880                         dfr = vlapic->apic_page->dfr;
 881                         ldr = vlapic->apic_page->ldr;
 882
 883                         if ((dfr & APIC_DFR_MODEL_MASK) ==
 884                             APIC_DFR_MODEL_FLAT) {
 885                                 ldest = ldr >> 24;
 886                                 mda_ldest = mda_flat_ldest;
 887                         } else if ((dfr & APIC_DFR_MODEL_MASK) ==
 888                             APIC_DFR_MODEL_CLUSTER) {
 889                                 if (x2apic(vlapic)) {
 890                                         cluster = ldr >> 16;
 891                                         ldest = ldr & 0xffff;
 892                                 } else {
 893                                         cluster = ldr >> 28;
 894                                         ldest = (ldr >> 24) & 0xf;
 895                                 }
 896                                 if (cluster != mda_cluster_id)
 897                                         continue;
 898                                 mda_ldest = mda_cluster_ldest;
 899                         } else {
 900                                 /*
 901                                  * Guest has configured a bad logical
 902                                  * model for this vcpu - skip it.
 903                                  */
 904                                 VLAPIC_CTR1(vlapic, "vlapic has bad logical "
 905                                     "model %x - cannot deliver interrupt", dfr);
 906                                 continue;
 907                         }
 908
 909                         if ((mda_ldest & ldest) != 0) {
 910                                 CPU_SET(vcpuid, dmask);
 911                                 if (lowprio)
 912                                         break;
 913                         }
 914                 }
 915         }
 916 }
 917
 918 static VMM_STAT_ARRAY(IPIS_SENT, VM_MAXCPU, "ipis sent to vcpu");
 919
 920 static void
 921 vlapic_set_tpr(struct vlapic *vlapic, uint8_t val)
 922 {
 923         struct LAPIC *lapic = vlapic->apic_page;
 924
 925         if (lapic->tpr != val) {
 926                 VCPU_CTR2(vlapic->vm, vlapic->vcpuid, "vlapic TPR changed "
 927                     "from %#x to %#x", lapic->tpr, val);
 928                 lapic->tpr = val;
 929                 vlapic_update_ppr(vlapic);
 930         }
 931 }
 932
 933 static uint8_t
 934 vlapic_get_tpr(struct vlapic *vlapic)
 935 {
 936         struct LAPIC *lapic = vlapic->apic_page;
 937
 938         return (lapic->tpr);
 939 }
 940
 941 void
 942 vlapic_set_cr8(struct vlapic *vlapic, uint64_t val)
 943 {
 944         uint8_t tpr;
 945
 946         if (val & ~0xf) {
 947                 vm_inject_gp(vlapic->vm, vlapic->vcpuid);
 948                 return;
 949         }
 950
 951         tpr = val << 4;
 952         vlapic_set_tpr(vlapic, tpr);
 953 }
 954
 955 uint64_t
 956 vlapic_get_cr8(struct vlapic *vlapic)
 957 {
 958         uint8_t tpr;
 959
 960         tpr = vlapic_get_tpr(vlapic);
 961         return (tpr >> 4);
 962 }
 963
 964 int
 965 vlapic_icrlo_write_handler(struct vlapic *vlapic, bool *retu)
 966 {
 967         int i;
 968         bool phys;
 969         cpuset_t dmask;
 970         uint64_t icrval;
 971         uint32_t dest, vec, mode;
 972         struct vlapic *vlapic2;
 973         struct vm_exit *vmexit;
 974         struct LAPIC *lapic;
 975         uint16_t maxcpus;
 976
 977         lapic = vlapic->apic_page;
 978         lapic->icr_lo &= ~APIC_DELSTAT_PEND;
 979         icrval = ((uint64_t)lapic->icr_hi << 32) | lapic->icr_lo;
 980
 981         if (x2apic(vlapic))
 982                 dest = icrval >> 32;
 983         else
 984                 dest = icrval >> (32 + 24);
 985         vec = icrval & APIC_VECTOR_MASK;
 986         mode = icrval & APIC_DELMODE_MASK;
 987
 988         if (mode == APIC_DELMODE_FIXED && vec < 16) {
 989                 vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR);
 990                 VLAPIC_CTR1(vlapic, "Ignoring invalid IPI %d", vec);
 991                 return (0);
 992         }
 993
 994         VLAPIC_CTR2(vlapic, "icrlo 0x%016lx triggered ipi %d", icrval, vec);
 995
 996         if (mode == APIC_DELMODE_FIXED || mode == APIC_DELMODE_NMI) {
 997                 switch (icrval & APIC_DEST_MASK) {
 998                 case APIC_DEST_DESTFLD:
 999                         phys = ((icrval & APIC_DESTMODE_LOG) == 0);
1000                         vlapic_calcdest(vlapic->vm, &dmask, dest, phys, false,
1001                             x2apic(vlapic));
1002                         break;
1003                 case APIC_DEST_SELF:
1004                         CPU_SETOF(vlapic->vcpuid, &dmask);
1005                         break;
1006                 case APIC_DEST_ALLISELF:
1007                         dmask = vm_active_cpus(vlapic->vm);
1008                         break;
1009                 case APIC_DEST_ALLESELF:
1010                         dmask = vm_active_cpus(vlapic->vm);
1011                         CPU_CLR(vlapic->vcpuid, &dmask);
1012                         break;
1013                 default:
1014                         CPU_ZERO(&dmask);       /* satisfy gcc */
1015                         break;
1016                 }
1017
1018                 while ((i = CPU_FFS(&dmask)) != 0) {
1019                         i--;
1020                         CPU_CLR(i, &dmask);
1021                         if (mode == APIC_DELMODE_FIXED) {
1022                                 lapic_intr_edge(vlapic->vm, i, vec);
1023                                 vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid,
1024                                                     IPIS_SENT, i, 1);
1025                                 VLAPIC_CTR2(vlapic, "vlapic sending ipi %d "
1026                                     "to vcpuid %d", vec, i);
1027                         } else {
1028                                 vm_inject_nmi(vlapic->vm, i);
1029                                 VLAPIC_CTR1(vlapic, "vlapic sending ipi nmi "
1030                                     "to vcpuid %d", i);
1031                         }
1032                 }
1033
1034                 return (0);     /* handled completely in the kernel */
1035         }
1036
1037         maxcpus = vm_get_maxcpus(vlapic->vm);
1038         if (mode == APIC_DELMODE_INIT) {
1039                 if ((icrval & APIC_LEVEL_MASK) == APIC_LEVEL_DEASSERT)
1040                         return (0);
1041
1042                 if (vlapic->vcpuid == 0 && dest != 0 && dest < maxcpus) {
1043                         vlapic2 = vm_lapic(vlapic->vm, dest);
1044
1045                         /* move from INIT to waiting-for-SIPI state */
1046                         if (vlapic2->boot_state == BS_INIT) {
1047                                 vlapic2->boot_state = BS_SIPI;
1048                         }
1049
1050                         return (0);
1051                 }
1052         }
1053
1054         if (mode == APIC_DELMODE_STARTUP) {
1055                 if (vlapic->vcpuid == 0 && dest != 0 && dest < maxcpus) {
1056                         vlapic2 = vm_lapic(vlapic->vm, dest);
1057
1058                         /*
1059                          * Ignore SIPIs in any state other than wait-for-SIPI
1060                          */
1061                         if (vlapic2->boot_state != BS_SIPI)
1062                                 return (0);
1063
1064                         vlapic2->boot_state = BS_RUNNING;
1065
1066                         *retu = true;
1067                         vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid);
1068                         vmexit->exitcode = VM_EXITCODE_SPINUP_AP;
1069                         vmexit->u.spinup_ap.vcpu = dest;
1070                         vmexit->u.spinup_ap.rip = vec << PAGE_SHIFT;
1071
1072                         return (0);
1073                 }
1074         }
1075
1076         /*
1077          * This will cause a return to userland.
1078          */
1079         return (1);
1080 }
1081
1082 void
1083 vlapic_self_ipi_handler(struct vlapic *vlapic, uint64_t val)
1084 {
1085         int vec;
1086
1087         KASSERT(x2apic(vlapic), ("SELF_IPI does not exist in xAPIC mode"));
1088
1089         vec = val & 0xff;
1090         lapic_intr_edge(vlapic->vm, vlapic->vcpuid, vec);
1091         vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid, IPIS_SENT,
1092             vlapic->vcpuid, 1);
1093         VLAPIC_CTR1(vlapic, "vlapic self-ipi %d", vec);
1094 }
1095
1096 int
1097 vlapic_pending_intr(struct vlapic *vlapic, int *vecptr)
1098 {
1099         struct LAPIC    *lapic = vlapic->apic_page;
1100         int              idx, i, bitpos, vector;
1101         uint32_t        *irrptr, val;
1102
1103         vlapic_update_ppr(vlapic);
1104
1105         if (vlapic->ops.pending_intr)
1106                 return ((*vlapic->ops.pending_intr)(vlapic, vecptr));
1107
1108         irrptr = &lapic->irr0;
1109
1110         for (i = 7; i >= 0; i--) {
1111                 idx = i * 4;
1112                 val = atomic_load_acq_int(&irrptr[idx]);
1113                 bitpos = fls(val);
1114                 if (bitpos != 0) {
1115                         vector = i * 32 + (bitpos - 1);
1116                         if (PRIO(vector) > PRIO(lapic->ppr)) {
1117                                 VLAPIC_CTR1(vlapic, "pending intr %d", vector);
1118                                 if (vecptr != NULL)
1119                                         *vecptr = vector;
1120                                 return (1);
1121                         } else
1122                                 break;
1123                 }
1124         }
1125         return (0);
1126 }
1127
1128 void
1129 vlapic_intr_accepted(struct vlapic *vlapic, int vector)
1130 {
1131         struct LAPIC    *lapic = vlapic->apic_page;
1132         uint32_t        *irrptr, *isrptr;
1133         int             idx, stk_top;
1134
1135         if (vlapic->ops.intr_accepted)
1136                 return ((*vlapic->ops.intr_accepted)(vlapic, vector));
1137
1138         /*
1139          * clear the ready bit for vector being accepted in irr
1140          * and set the vector as in service in isr.
1141          */
1142         idx = (vector / 32) * 4;
1143
1144         irrptr = &lapic->irr0;
1145         atomic_clear_int(&irrptr[idx], 1 << (vector % 32));
1146         VLAPIC_CTR_IRR(vlapic, "vlapic_intr_accepted");
1147
1148         isrptr = &lapic->isr0;
1149         isrptr[idx] |= 1 << (vector % 32);
1150         VLAPIC_CTR_ISR(vlapic, "vlapic_intr_accepted");
1151
1152         /*
1153          * Update the PPR
1154          */
1155         vlapic->isrvec_stk_top++;
1156
1157         stk_top = vlapic->isrvec_stk_top;
1158         if (stk_top >= ISRVEC_STK_SIZE)
1159                 panic("isrvec_stk_top overflow %d", stk_top);
1160
1161         vlapic->isrvec_stk[stk_top] = vector;
1162 }
1163
1164 void
1165 vlapic_svr_write_handler(struct vlapic *vlapic)
1166 {
1167         struct LAPIC *lapic;
1168         uint32_t old, new, changed;
1169
1170         lapic = vlapic->apic_page;
1171
1172         new = lapic->svr;
1173         old = vlapic->svr_last;
1174         vlapic->svr_last = new;
1175
1176         changed = old ^ new;
1177         if ((changed & APIC_SVR_ENABLE) != 0) {
1178                 if ((new & APIC_SVR_ENABLE) == 0) {
1179                         /*
1180                          * The apic is now disabled so stop the apic timer
1181                          * and mask all the LVT entries.
1182                          */
1183                         VLAPIC_CTR0(vlapic, "vlapic is software-disabled");
1184                         VLAPIC_TIMER_LOCK(vlapic);
1185                         callout_stop(&vlapic->callout);
1186                         VLAPIC_TIMER_UNLOCK(vlapic);
1187                         vlapic_mask_lvts(vlapic);
1188                 } else {
1189                         /*
1190                          * The apic is now enabled so restart the apic timer
1191                          * if it is configured in periodic mode.
1192                          */
1193                         VLAPIC_CTR0(vlapic, "vlapic is software-enabled");
1194                         if (vlapic_periodic_timer(vlapic))
1195                                 vlapic_icrtmr_write_handler(vlapic);
1196                 }
1197         }
1198 }
1199
1200 int
1201 vlapic_read(struct vlapic *vlapic, int mmio_access, uint64_t offset,
1202     uint64_t *data, bool *retu)
1203 {
1204         struct LAPIC    *lapic = vlapic->apic_page;
1205         uint32_t        *reg;
1206         int              i;
1207
1208         /* Ignore MMIO accesses in x2APIC mode */
1209         if (x2apic(vlapic) && mmio_access) {
1210                 VLAPIC_CTR1(vlapic, "MMIO read from offset %#lx in x2APIC mode",
1211                     offset);
1212                 *data = 0;
1213                 goto done;
1214         }
1215
1216         if (!x2apic(vlapic) && !mmio_access) {
1217                 /*
1218                  * XXX Generate GP fault for MSR accesses in xAPIC mode
1219                  */
1220                 VLAPIC_CTR1(vlapic, "x2APIC MSR read from offset %#lx in "
1221                     "xAPIC mode", offset);
1222                 *data = 0;
1223                 goto done;
1224         }
1225
1226         if (offset > sizeof(*lapic)) {
1227                 *data = 0;
1228                 goto done;
1229         }
1230
1231         offset &= ~3;
1232         switch(offset)
1233         {
1234                 case APIC_OFFSET_ID:
1235                         *data = lapic->id;
1236                         break;
1237                 case APIC_OFFSET_VER:
1238                         *data = lapic->version;
1239                         break;
1240                 case APIC_OFFSET_TPR:
1241                         *data = vlapic_get_tpr(vlapic);
1242                         break;
1243                 case APIC_OFFSET_APR:
1244                         *data = lapic->apr;
1245                         break;
1246                 case APIC_OFFSET_PPR:
1247                         *data = lapic->ppr;
1248                         break;
1249                 case APIC_OFFSET_EOI:
1250                         *data = lapic->eoi;
1251                         break;
1252                 case APIC_OFFSET_LDR:
1253                         *data = lapic->ldr;
1254                         break;
1255                 case APIC_OFFSET_DFR:
1256                         *data = lapic->dfr;
1257                         break;
1258                 case APIC_OFFSET_SVR:
1259                         *data = lapic->svr;
1260                         break;
1261                 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
1262                         i = (offset - APIC_OFFSET_ISR0) >> 2;
1263                         reg = &lapic->isr0;
1264                         *data = *(reg + i);
1265                         break;
1266                 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
1267                         i = (offset - APIC_OFFSET_TMR0) >> 2;
1268                         reg = &lapic->tmr0;
1269                         *data = *(reg + i);
1270                         break;
1271                 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
1272                         i = (offset - APIC_OFFSET_IRR0) >> 2;
1273                         reg = &lapic->irr0;
1274                         *data = atomic_load_acq_int(reg + i);
1275                         break;
1276                 case APIC_OFFSET_ESR:
1277                         *data = lapic->esr;
1278                         break;
1279                 case APIC_OFFSET_ICR_LOW:
1280                         *data = lapic->icr_lo;
1281                         if (x2apic(vlapic))
1282                                 *data |= (uint64_t)lapic->icr_hi << 32;
1283                         break;
1284                 case APIC_OFFSET_ICR_HI:
1285                         *data = lapic->icr_hi;
1286                         break;
1287                 case APIC_OFFSET_CMCI_LVT:
1288                 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
1289                         *data = vlapic_get_lvt(vlapic, offset);
1290 #ifdef INVARIANTS
1291                         reg = vlapic_get_lvtptr(vlapic, offset);
1292                         KASSERT(*data == *reg, ("inconsistent lvt value at "
1293                             "offset %#lx: %#lx/%#x", offset, *data, *reg));
1294 #endif
1295                         break;
1296                 case APIC_OFFSET_TIMER_ICR:
1297                         *data = lapic->icr_timer;
1298                         break;
1299                 case APIC_OFFSET_TIMER_CCR:
1300                         *data = vlapic_get_ccr(vlapic);
1301                         break;
1302                 case APIC_OFFSET_TIMER_DCR:
1303                         *data = lapic->dcr_timer;
1304                         break;
1305                 case APIC_OFFSET_SELF_IPI:
1306                         /*
1307                          * XXX generate a GP fault if vlapic is in x2apic mode
1308                          */
1309                         *data = 0;
1310                         break;
1311                 case APIC_OFFSET_RRR:
1312                 default:
1313                         *data = 0;
1314                         break;
1315         }
1316 done:
1317         VLAPIC_CTR2(vlapic, "vlapic read offset %#x, data %#lx", offset, *data);
1318         return 0;
1319 }
1320
1321 int
1322 vlapic_write(struct vlapic *vlapic, int mmio_access, uint64_t offset,
1323     uint64_t data, bool *retu)
1324 {
1325         struct LAPIC    *lapic = vlapic->apic_page;
1326         uint32_t        *regptr;
1327         int             retval;
1328
1329         KASSERT((offset & 0xf) == 0 && offset < PAGE_SIZE,
1330             ("vlapic_write: invalid offset %#lx", offset));
1331
1332         VLAPIC_CTR2(vlapic, "vlapic write offset %#lx, data %#lx",
1333             offset, data);
1334
1335         if (offset > sizeof(*lapic))
1336                 return (0);
1337
1338         /* Ignore MMIO accesses in x2APIC mode */
1339         if (x2apic(vlapic) && mmio_access) {
1340                 VLAPIC_CTR2(vlapic, "MMIO write of %#lx to offset %#lx "
1341                     "in x2APIC mode", data, offset);
1342                 return (0);
1343         }
1344
1345         /*
1346          * XXX Generate GP fault for MSR accesses in xAPIC mode
1347          */
1348         if (!x2apic(vlapic) && !mmio_access) {
1349                 VLAPIC_CTR2(vlapic, "x2APIC MSR write of %#lx to offset %#lx "
1350                     "in xAPIC mode", data, offset);
1351                 return (0);
1352         }
1353
1354         retval = 0;
1355         switch(offset)
1356         {
1357                 case APIC_OFFSET_ID:
1358                         lapic->id = data;
1359                         vlapic_id_write_handler(vlapic);
1360                         break;
1361                 case APIC_OFFSET_TPR:
1362                         vlapic_set_tpr(vlapic, data & 0xff);
1363                         break;
1364                 case APIC_OFFSET_EOI:
1365                         vlapic_process_eoi(vlapic);
1366                         break;
1367                 case APIC_OFFSET_LDR:
1368                         lapic->ldr = data;
1369                         vlapic_ldr_write_handler(vlapic);
1370                         break;
1371                 case APIC_OFFSET_DFR:
1372                         lapic->dfr = data;
1373                         vlapic_dfr_write_handler(vlapic);
1374                         break;
1375                 case APIC_OFFSET_SVR:
1376                         lapic->svr = data;
1377                         vlapic_svr_write_handler(vlapic);
1378                         break;
1379                 case APIC_OFFSET_ICR_LOW:
1380                         lapic->icr_lo = data;
1381                         if (x2apic(vlapic))
1382                                 lapic->icr_hi = data >> 32;
1383                         retval = vlapic_icrlo_write_handler(vlapic, retu);
1384                         break;
1385                 case APIC_OFFSET_ICR_HI:
1386                         lapic->icr_hi = data;
1387                         break;
1388                 case APIC_OFFSET_CMCI_LVT:
1389                 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
1390                         regptr = vlapic_get_lvtptr(vlapic, offset);
1391                         *regptr = data;
1392                         vlapic_lvt_write_handler(vlapic, offset);
1393                         break;
1394                 case APIC_OFFSET_TIMER_ICR:
1395                         lapic->icr_timer = data;
1396                         vlapic_icrtmr_write_handler(vlapic);
1397                         break;
1398
1399                 case APIC_OFFSET_TIMER_DCR:
1400                         lapic->dcr_timer = data;
1401                         vlapic_dcr_write_handler(vlapic);
1402                         break;
1403
1404                 case APIC_OFFSET_ESR:
1405                         vlapic_esr_write_handler(vlapic);
1406                         break;
1407
1408                 case APIC_OFFSET_SELF_IPI:
1409                         if (x2apic(vlapic))
1410                                 vlapic_self_ipi_handler(vlapic, data);
1411                         break;
1412
1413                 case APIC_OFFSET_VER:
1414                 case APIC_OFFSET_APR:
1415                 case APIC_OFFSET_PPR:
1416                 case APIC_OFFSET_RRR:
1417                 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
1418                 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
1419                 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
1420                 case APIC_OFFSET_TIMER_CCR:
1421                 default:
1422                         // Read only.
1423                         break;
1424         }
1425
1426         return (retval);
1427 }
1428
1429 static void
1430 vlapic_reset(struct vlapic *vlapic)
1431 {
1432         struct LAPIC *lapic;
1433
1434         lapic = vlapic->apic_page;
1435         bzero(lapic, sizeof(struct LAPIC));
1436
1437         lapic->id = vlapic_get_id(vlapic);
1438         lapic->version = VLAPIC_VERSION;
1439         lapic->version |= (VLAPIC_MAXLVT_INDEX << MAXLVTSHIFT);
1440         lapic->dfr = 0xffffffff;
1441         lapic->svr = APIC_SVR_VECTOR;
1442         vlapic_mask_lvts(vlapic);
1443         vlapic_reset_tmr(vlapic);
1444
1445         lapic->dcr_timer = 0;
1446         vlapic_dcr_write_handler(vlapic);
1447
1448         if (vlapic->vcpuid == 0)
1449                 vlapic->boot_state = BS_RUNNING;        /* BSP */
1450         else
1451                 vlapic->boot_state = BS_INIT;           /* AP */
1452
1453         vlapic->svr_last = lapic->svr;
1454 }
1455
1456 void
1457 vlapic_init(struct vlapic *vlapic)
1458 {
1459         KASSERT(vlapic->vm != NULL, ("vlapic_init: vm is not initialized"));
1460         KASSERT(vlapic->vcpuid >= 0 &&
1461             vlapic->vcpuid < vm_get_maxcpus(vlapic->vm),
1462             ("vlapic_init: vcpuid is not initialized"));
1463         KASSERT(vlapic->apic_page != NULL, ("vlapic_init: apic_page is not "
1464             "initialized"));
1465
1466         /*
1467          * If the vlapic is configured in x2apic mode then it will be
1468          * accessed in the critical section via the MSR emulation code.
1469          *
1470          * Therefore the timer mutex must be a spinlock because blockable
1471          * mutexes cannot be acquired in a critical section.
1472          */
1473         mtx_init(&vlapic->timer_mtx, "vlapic timer mtx", NULL, MTX_SPIN);
1474         callout_init(&vlapic->callout, 1);
1475
1476         vlapic->msr_apicbase = DEFAULT_APIC_BASE | APICBASE_ENABLED;
1477
1478         if (vlapic->vcpuid == 0)
1479                 vlapic->msr_apicbase |= APICBASE_BSP;
1480
1481         vlapic_reset(vlapic);
1482 }
1483
1484 void
1485 vlapic_cleanup(struct vlapic *vlapic)
1486 {
1487
1488         callout_drain(&vlapic->callout);
1489 }
1490
1491 uint64_t
1492 vlapic_get_apicbase(struct vlapic *vlapic)
1493 {
1494
1495         return (vlapic->msr_apicbase);
1496 }
1497
1498 int
1499 vlapic_set_apicbase(struct vlapic *vlapic, uint64_t new)
1500 {
1501
1502         if (vlapic->msr_apicbase != new) {
1503                 VLAPIC_CTR2(vlapic, "Changing APIC_BASE MSR from %#lx to %#lx "
1504                     "not supported", vlapic->msr_apicbase, new);
1505                 return (-1);
1506         }
1507
1508         return (0);
1509 }
1510
1511 void
1512 vlapic_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
1513 {
1514         struct vlapic *vlapic;
1515         struct LAPIC *lapic;
1516
1517         vlapic = vm_lapic(vm, vcpuid);
1518
1519         if (state == X2APIC_DISABLED)
1520                 vlapic->msr_apicbase &= ~APICBASE_X2APIC;
1521         else
1522                 vlapic->msr_apicbase |= APICBASE_X2APIC;
1523
1524         /*
1525          * Reset the local APIC registers whose values are mode-dependent.
1526          *
1527          * XXX this works because the APIC mode can be changed only at vcpu
1528          * initialization time.
1529          */
1530         lapic = vlapic->apic_page;
1531         lapic->id = vlapic_get_id(vlapic);
1532         if (x2apic(vlapic)) {
1533                 lapic->ldr = x2apic_ldr(vlapic);
1534                 lapic->dfr = 0;
1535         } else {
1536                 lapic->ldr = 0;
1537                 lapic->dfr = 0xffffffff;
1538         }
1539
1540         if (state == X2APIC_ENABLED) {
1541                 if (vlapic->ops.enable_x2apic_mode)
1542                         (*vlapic->ops.enable_x2apic_mode)(vlapic);
1543         }
1544 }
1545
1546 void
1547 vlapic_deliver_intr(struct vm *vm, bool level, uint32_t dest, bool phys,
1548     int delmode, int vec)
1549 {
1550         bool lowprio;
1551         int vcpuid;
1552         cpuset_t dmask;
1553
1554         if (delmode != IOART_DELFIXED &&
1555             delmode != IOART_DELLOPRI &&
1556             delmode != IOART_DELEXINT) {
1557                 VM_CTR1(vm, "vlapic intr invalid delmode %#x", delmode);
1558                 return;
1559         }
1560         lowprio = (delmode == IOART_DELLOPRI);
1561
1562         /*
1563          * We don't provide any virtual interrupt redirection hardware so
1564          * all interrupts originating from the ioapic or MSI specify the
1565          * 'dest' in the legacy xAPIC format.
1566          */
1567         vlapic_calcdest(vm, &dmask, dest, phys, lowprio, false);
1568
1569         while ((vcpuid = CPU_FFS(&dmask)) != 0) {
1570                 vcpuid--;
1571                 CPU_CLR(vcpuid, &dmask);
1572                 if (delmode == IOART_DELEXINT) {
1573                         vm_inject_extint(vm, vcpuid);
1574                 } else {
1575                         lapic_set_intr(vm, vcpuid, vec, level);
1576                 }
1577         }
1578 }
1579
1580 void
1581 vlapic_post_intr(struct vlapic *vlapic, int hostcpu, int ipinum)
1582 {
1583         /*
1584          * Post an interrupt to the vcpu currently running on 'hostcpu'.
1585          *
1586          * This is done by leveraging features like Posted Interrupts (Intel)
1587          * Doorbell MSR (AMD AVIC) that avoid a VM exit.
1588          *
1589          * If neither of these features are available then fallback to
1590          * sending an IPI to 'hostcpu'.
1591          */
1592         if (vlapic->ops.post_intr)
1593                 (*vlapic->ops.post_intr)(vlapic, hostcpu);
1594         else
1595                 ipi_cpu(hostcpu, ipinum);
1596 }
1597
1598 bool
1599 vlapic_enabled(struct vlapic *vlapic)
1600 {
1601         struct LAPIC *lapic = vlapic->apic_page;
1602
1603         if ((vlapic->msr_apicbase & APICBASE_ENABLED) != 0 &&
1604             (lapic->svr & APIC_SVR_ENABLE) != 0)
1605                 return (true);
1606         else
1607                 return (false);
1608 }
1609
1610 static void
1611 vlapic_set_tmr(struct vlapic *vlapic, int vector, bool level)
1612 {
1613         struct LAPIC *lapic;
1614         uint32_t *tmrptr, mask;
1615         int idx;
1616
1617         lapic = vlapic->apic_page;
1618         tmrptr = &lapic->tmr0;
1619         idx = (vector / 32) * 4;
1620         mask = 1 << (vector % 32);
1621         if (level)
1622                 tmrptr[idx] |= mask;
1623         else
1624                 tmrptr[idx] &= ~mask;
1625
1626         if (vlapic->ops.set_tmr != NULL)
1627                 (*vlapic->ops.set_tmr)(vlapic, vector, level);
1628 }
1629
1630 void
1631 vlapic_reset_tmr(struct vlapic *vlapic)
1632 {
1633         int vector;
1634
1635         VLAPIC_CTR0(vlapic, "vlapic resetting all vectors to edge-triggered");
1636
1637         for (vector = 0; vector <= 255; vector++)
1638                 vlapic_set_tmr(vlapic, vector, false);
1639 }
1640
1641 void
1642 vlapic_set_tmr_level(struct vlapic *vlapic, uint32_t dest, bool phys,
1643     int delmode, int vector)
1644 {
1645         cpuset_t dmask;
1646         bool lowprio;
1647
1648         KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
1649
1650         /*
1651          * A level trigger is valid only for fixed and lowprio delivery modes.
1652          */
1653         if (delmode != APIC_DELMODE_FIXED && delmode != APIC_DELMODE_LOWPRIO) {
1654                 VLAPIC_CTR1(vlapic, "Ignoring level trigger-mode for "
1655                     "delivery-mode %d", delmode);
1656                 return;
1657         }
1658
1659         lowprio = (delmode == APIC_DELMODE_LOWPRIO);
1660         vlapic_calcdest(vlapic->vm, &dmask, dest, phys, lowprio, false);
1661
1662         if (!CPU_ISSET(vlapic->vcpuid, &dmask))
1663                 return;
1664
1665         VLAPIC_CTR1(vlapic, "vector %d set to level-triggered", vector);
1666         vlapic_set_tmr(vlapic, vector, true);
1667 }