sys/amd64/vmm/io/vlapic.c

   1 /*-
   2  * Copyright (c) 2011 NetApp, Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  *
  26  * $FreeBSD$
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include <sys/param.h>
  33 #include <sys/kernel.h>
  34 #include <sys/malloc.h>
  35 #include <sys/systm.h>
  36 #include <sys/smp.h>
  37
  38 #include <machine/clock.h>
  39 #include <x86/specialreg.h>
  40 #include <x86/apicreg.h>
  41
  42 #include <machine/vmm.h>
  43
  44 #include "vmm_stat.h"
  45 #include "vmm_lapic.h"
  46 #include "vmm_ktr.h"
  47 #include "vdev.h"
  48 #include "vlapic.h"
  49
  50 #define VLAPIC_CTR0(vlapic, format)                                     \
  51         VMM_CTR0((vlapic)->vm, (vlapic)->vcpuid, format)
  52
  53 #define VLAPIC_CTR1(vlapic, format, p1)                                 \
  54         VMM_CTR1((vlapic)->vm, (vlapic)->vcpuid, format, p1)
  55
  56 #define VLAPIC_CTR_IRR(vlapic, msg)                                     \
  57 do {                                                                    \
  58         uint32_t *irrptr = &(vlapic)->apic.irr0;                        \
  59         irrptr[0] = irrptr[0];  /* silence compiler */                  \
  60         VLAPIC_CTR1((vlapic), msg " irr0 0x%08x", irrptr[0 << 2]);      \
  61         VLAPIC_CTR1((vlapic), msg " irr1 0x%08x", irrptr[1 << 2]);      \
  62         VLAPIC_CTR1((vlapic), msg " irr2 0x%08x", irrptr[2 << 2]);      \
  63         VLAPIC_CTR1((vlapic), msg " irr3 0x%08x", irrptr[3 << 2]);      \
  64         VLAPIC_CTR1((vlapic), msg " irr4 0x%08x", irrptr[4 << 2]);      \
  65         VLAPIC_CTR1((vlapic), msg " irr5 0x%08x", irrptr[5 << 2]);      \
  66         VLAPIC_CTR1((vlapic), msg " irr6 0x%08x", irrptr[6 << 2]);      \
  67         VLAPIC_CTR1((vlapic), msg " irr7 0x%08x", irrptr[7 << 2]);      \
  68 } while (0)
  69
  70 #define VLAPIC_CTR_ISR(vlapic, msg)                                     \
  71 do {                                                                    \
  72         uint32_t *isrptr = &(vlapic)->apic.isr0;                        \
  73         isrptr[0] = isrptr[0];  /* silence compiler */                  \
  74         VLAPIC_CTR1((vlapic), msg " isr0 0x%08x", isrptr[0 << 2]);      \
  75         VLAPIC_CTR1((vlapic), msg " isr1 0x%08x", isrptr[1 << 2]);      \
  76         VLAPIC_CTR1((vlapic), msg " isr2 0x%08x", isrptr[2 << 2]);      \
  77         VLAPIC_CTR1((vlapic), msg " isr3 0x%08x", isrptr[3 << 2]);      \
  78         VLAPIC_CTR1((vlapic), msg " isr4 0x%08x", isrptr[4 << 2]);      \
  79         VLAPIC_CTR1((vlapic), msg " isr5 0x%08x", isrptr[5 << 2]);      \
  80         VLAPIC_CTR1((vlapic), msg " isr6 0x%08x", isrptr[6 << 2]);      \
  81         VLAPIC_CTR1((vlapic), msg " isr7 0x%08x", isrptr[7 << 2]);      \
  82 } while (0)
  83
  84 static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic");
  85
  86 #define PRIO(x)                 ((x) >> 4)
  87
  88 #define VLAPIC_VERSION          (16)
  89 #define VLAPIC_MAXLVT_ENTRIES   (5)
  90
  91 #define x2apic(vlapic)  (((vlapic)->msr_apicbase & APICBASE_X2APIC) ? 1 : 0)
  92
  93 enum boot_state {
  94         BS_INIT,
  95         BS_SIPI,
  96         BS_RUNNING
  97 };
  98
  99 struct vlapic {
 100         struct vm               *vm;
 101         int                     vcpuid;
 102
 103         struct io_region        *mmio;
 104         struct vdev_ops         *ops;
 105         struct LAPIC             apic;
 106
 107         int                      esr_update;
 108
 109         int                      divisor;
 110         int                      ccr_ticks;
 111
 112         /*
 113          * The 'isrvec_stk' is a stack of vectors injected by the local apic.
 114          * A vector is popped from the stack when the processor does an EOI.
 115          * The vector on the top of the stack is used to compute the
 116          * Processor Priority in conjunction with the TPR.
 117          */
 118         uint8_t                  isrvec_stk[ISRVEC_STK_SIZE];
 119         int                      isrvec_stk_top;
 120
 121         uint64_t                msr_apicbase;
 122         enum boot_state         boot_state;
 123 };
 124
 125 #define VLAPIC_BUS_FREQ tsc_freq
 126
 127 static int
 128 vlapic_timer_divisor(uint32_t dcr)
 129 {
 130         switch (dcr & 0xB) {
 131         case APIC_TDCR_1:
 132                 return (1);
 133         case APIC_TDCR_2:
 134                 return (2);
 135         case APIC_TDCR_4:
 136                 return (4);
 137         case APIC_TDCR_8:
 138                 return (8);
 139         case APIC_TDCR_16:
 140                 return (16);
 141         case APIC_TDCR_32:
 142                 return (32);
 143         case APIC_TDCR_64:
 144                 return (64);
 145         case APIC_TDCR_128:
 146                 return (128);
 147         default:
 148                 panic("vlapic_timer_divisor: invalid dcr 0x%08x", dcr);
 149         }
 150 }
 151
 152 static void
 153 vlapic_mask_lvts(uint32_t *lvts, int num_lvt)
 154 {
 155         int i;
 156         for (i = 0; i < num_lvt; i++) {
 157                 *lvts |= APIC_LVT_M;
 158                 lvts += 4;
 159         }
 160 }
 161
 162 #if 0
 163 static inline void
 164 vlapic_dump_lvt(uint32_t offset, uint32_t *lvt)
 165 {
 166         printf("Offset %x: lvt %08x (V:%02x DS:%x M:%x)\n", offset,
 167             *lvt, *lvt & APIC_LVTT_VECTOR, *lvt & APIC_LVTT_DS,
 168             *lvt & APIC_LVTT_M);
 169 }
 170 #endif
 171
 172 static uint64_t
 173 vlapic_get_ccr(struct vlapic *vlapic)
 174 {
 175         struct LAPIC    *lapic = &vlapic->apic;
 176         return lapic->ccr_timer;
 177 }
 178
 179 static void
 180 vlapic_update_errors(struct vlapic *vlapic)
 181 {
 182         struct LAPIC    *lapic = &vlapic->apic;
 183         lapic->esr = 0; // XXX
 184 }
 185
 186 static void
 187 vlapic_init_ipi(struct vlapic *vlapic)
 188 {
 189         struct LAPIC    *lapic = &vlapic->apic;
 190         lapic->version = VLAPIC_VERSION;
 191         lapic->version |= (VLAPIC_MAXLVT_ENTRIES < MAXLVTSHIFT);
 192         lapic->dfr = 0xffffffff;
 193         lapic->svr = APIC_SVR_VECTOR;
 194         vlapic_mask_lvts(&lapic->lvt_timer, VLAPIC_MAXLVT_ENTRIES+1);
 195 }
 196
 197 static int
 198 vlapic_op_reset(void* dev)
 199 {
 200         struct vlapic   *vlapic = (struct vlapic*)dev;
 201         struct LAPIC    *lapic = &vlapic->apic;
 202
 203         memset(lapic, 0, sizeof(*lapic));
 204         lapic->apr = vlapic->vcpuid;
 205         vlapic_init_ipi(vlapic);
 206         vlapic->divisor = vlapic_timer_divisor(lapic->dcr_timer);
 207
 208         if (vlapic->vcpuid == 0)
 209                 vlapic->boot_state = BS_RUNNING;        /* BSP */
 210         else
 211                 vlapic->boot_state = BS_INIT;           /* AP */
 212
 213         return 0;
 214
 215 }
 216
 217 static int
 218 vlapic_op_init(void* dev)
 219 {
 220         struct vlapic *vlapic = (struct vlapic*)dev;
 221         vdev_register_region(vlapic->ops, vlapic, vlapic->mmio);
 222         return vlapic_op_reset(dev);
 223 }
 224
 225 static int
 226 vlapic_op_halt(void* dev)
 227 {
 228         struct vlapic *vlapic = (struct vlapic*)dev;
 229         vdev_unregister_region(vlapic, vlapic->mmio);
 230         return 0;
 231
 232 }
 233
 234 void
 235 vlapic_set_intr_ready(struct vlapic *vlapic, int vector)
 236 {
 237         struct LAPIC    *lapic = &vlapic->apic;
 238         uint32_t        *irrptr;
 239         int             idx;
 240
 241         if (vector < 0 || vector >= 256)
 242                 panic("vlapic_set_intr_ready: invalid vector %d\n", vector);
 243
 244         idx = (vector / 32) * 4;
 245         irrptr = &lapic->irr0;
 246         atomic_set_int(&irrptr[idx], 1 << (vector % 32));
 247         VLAPIC_CTR_IRR(vlapic, "vlapic_set_intr_ready");
 248 }
 249
 250 static void
 251 vlapic_start_timer(struct vlapic *vlapic, uint32_t elapsed)
 252 {
 253         uint32_t icr_timer;
 254
 255         icr_timer = vlapic->apic.icr_timer;
 256
 257         vlapic->ccr_ticks = ticks;
 258         if (elapsed < icr_timer)
 259                 vlapic->apic.ccr_timer = icr_timer - elapsed;
 260         else {
 261                 /*
 262                  * This can happen when the guest is trying to run its local
 263                  * apic timer higher that the setting of 'hz' in the host.
 264                  *
 265                  * We deal with this by running the guest local apic timer
 266                  * at the rate of the host's 'hz' setting.
 267                  */
 268                 vlapic->apic.ccr_timer = 0;
 269         }
 270 }
 271
 272 static __inline uint32_t *
 273 vlapic_get_lvt(struct vlapic *vlapic, uint32_t offset)
 274 {
 275         struct LAPIC    *lapic = &vlapic->apic;
 276         int              i;
 277
 278         if (offset < APIC_OFFSET_TIMER_LVT || offset > APIC_OFFSET_ERROR_LVT) {
 279                 panic("vlapic_get_lvt: invalid LVT\n");
 280         }
 281         i = (offset - APIC_OFFSET_TIMER_LVT) >> 2;
 282         return ((&lapic->lvt_timer) + i);;
 283 }
 284
 285 #if 1
 286 static void
 287 dump_isrvec_stk(struct vlapic *vlapic)
 288 {
 289         int i;
 290         uint32_t *isrptr;
 291
 292         isrptr = &vlapic->apic.isr0;
 293         for (i = 0; i < 8; i++)
 294                 printf("ISR%d 0x%08x\n", i, isrptr[i * 4]);
 295
 296         for (i = 0; i <= vlapic->isrvec_stk_top; i++)
 297                 printf("isrvec_stk[%d] = %d\n", i, vlapic->isrvec_stk[i]);
 298 }
 299 #endif
 300
 301 /*
 302  * Algorithm adopted from section "Interrupt, Task and Processor Priority"
 303  * in Intel Architecture Manual Vol 3a.
 304  */
 305 static void
 306 vlapic_update_ppr(struct vlapic *vlapic)
 307 {
 308         int isrvec, tpr, ppr;
 309
 310         /*
 311          * Note that the value on the stack at index 0 is always 0.
 312          *
 313          * This is a placeholder for the value of ISRV when none of the
 314          * bits is set in the ISRx registers.
 315          */
 316         isrvec = vlapic->isrvec_stk[vlapic->isrvec_stk_top];
 317         tpr = vlapic->apic.tpr;
 318
 319 #if 1
 320         {
 321                 int i, lastprio, curprio, vector, idx;
 322                 uint32_t *isrptr;
 323
 324                 if (vlapic->isrvec_stk_top == 0 && isrvec != 0)
 325                         panic("isrvec_stk is corrupted: %d", isrvec);
 326
 327                 /*
 328                  * Make sure that the priority of the nested interrupts is
 329                  * always increasing.
 330                  */
 331                 lastprio = -1;
 332                 for (i = 1; i <= vlapic->isrvec_stk_top; i++) {
 333                         curprio = PRIO(vlapic->isrvec_stk[i]);
 334                         if (curprio <= lastprio) {
 335                                 dump_isrvec_stk(vlapic);
 336                                 panic("isrvec_stk does not satisfy invariant");
 337                         }
 338                         lastprio = curprio;
 339                 }
 340
 341                 /*
 342                  * Make sure that each bit set in the ISRx registers has a
 343                  * corresponding entry on the isrvec stack.
 344                  */
 345                 i = 1;
 346                 isrptr = &vlapic->apic.isr0;
 347                 for (vector = 0; vector < 256; vector++) {
 348                         idx = (vector / 32) * 4;
 349                         if (isrptr[idx] & (1 << (vector % 32))) {
 350                                 if (i > vlapic->isrvec_stk_top ||
 351                                     vlapic->isrvec_stk[i] != vector) {
 352                                         dump_isrvec_stk(vlapic);
 353                                         panic("ISR and isrvec_stk out of sync");
 354                                 }
 355                                 i++;
 356                         }
 357                 }
 358         }
 359 #endif
 360
 361         if (PRIO(tpr) >= PRIO(isrvec))
 362                 ppr = tpr;
 363         else
 364                 ppr = isrvec & 0xf0;
 365
 366         vlapic->apic.ppr = ppr;
 367         VLAPIC_CTR1(vlapic, "vlapic_update_ppr 0x%02x", ppr);
 368 }
 369
 370 static void
 371 vlapic_process_eoi(struct vlapic *vlapic)
 372 {
 373         struct LAPIC    *lapic = &vlapic->apic;
 374         uint32_t        *isrptr;
 375         int             i, idx, bitpos;
 376
 377         isrptr = &lapic->isr0;
 378
 379         /*
 380          * The x86 architecture reserves the the first 32 vectors for use
 381          * by the processor.
 382          */
 383         for (i = 7; i > 0; i--) {
 384                 idx = i * 4;
 385                 bitpos = fls(isrptr[idx]);
 386                 if (bitpos != 0) {
 387                         if (vlapic->isrvec_stk_top <= 0) {
 388                                 panic("invalid vlapic isrvec_stk_top %d",
 389                                       vlapic->isrvec_stk_top);
 390                         }
 391                         isrptr[idx] &= ~(1 << (bitpos - 1));
 392                         VLAPIC_CTR_ISR(vlapic, "vlapic_process_eoi");
 393                         vlapic->isrvec_stk_top--;
 394                         vlapic_update_ppr(vlapic);
 395                         return;
 396                 }
 397         }
 398 }
 399
 400 static __inline int
 401 vlapic_get_lvt_field(uint32_t *lvt, uint32_t mask)
 402 {
 403         return (*lvt & mask);
 404 }
 405
 406 static __inline int
 407 vlapic_periodic_timer(struct vlapic *vlapic)
 408 {
 409         uint32_t *lvt;
 410
 411         lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
 412
 413         return (vlapic_get_lvt_field(lvt, APIC_LVTT_TM_PERIODIC));
 414 }
 415
 416 static VMM_STAT(VLAPIC_INTR_TIMER, "timer interrupts generated by vlapic");
 417
 418 static void
 419 vlapic_fire_timer(struct vlapic *vlapic)
 420 {
 421         int vector;
 422         uint32_t *lvt;
 423
 424         lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
 425
 426         if (!vlapic_get_lvt_field(lvt, APIC_LVTT_M)) {
 427                 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_TIMER, 1);
 428                 vector = vlapic_get_lvt_field(lvt,APIC_LVTT_VECTOR);
 429                 vlapic_set_intr_ready(vlapic, vector);
 430         }
 431 }
 432
 433 static VMM_STAT_ARRAY(IPIS_SENT, VM_MAXCPU, "ipis sent to vcpu");
 434
 435 static int
 436 lapic_process_icr(struct vlapic *vlapic, uint64_t icrval)
 437 {
 438         int i;
 439         cpuset_t dmask;
 440         uint32_t dest, vec, mode;
 441         struct vlapic *vlapic2;
 442         struct vm_exit *vmexit;
 443
 444         if (x2apic(vlapic))
 445                 dest = icrval >> 32;
 446         else
 447                 dest = icrval >> (32 + 24);
 448         vec = icrval & APIC_VECTOR_MASK;
 449         mode = icrval & APIC_DELMODE_MASK;
 450
 451         if (mode == APIC_DELMODE_FIXED || mode == APIC_DELMODE_NMI) {
 452                 switch (icrval & APIC_DEST_MASK) {
 453                 case APIC_DEST_DESTFLD:
 454                         CPU_SETOF(dest, &dmask);
 455                         break;
 456                 case APIC_DEST_SELF:
 457                         CPU_SETOF(vlapic->vcpuid, &dmask);
 458                         break;
 459                 case APIC_DEST_ALLISELF:
 460                         dmask = vm_active_cpus(vlapic->vm);
 461                         break;
 462                 case APIC_DEST_ALLESELF:
 463                         dmask = vm_active_cpus(vlapic->vm);
 464                         CPU_CLR(vlapic->vcpuid, &dmask);
 465                         break;
 466                 }
 467
 468                 while ((i = cpusetobj_ffs(&dmask)) != 0) {
 469                         i--;
 470                         CPU_CLR(i, &dmask);
 471                         if (mode == APIC_DELMODE_FIXED) {
 472                                 lapic_set_intr(vlapic->vm, i, vec);
 473                                 vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid,
 474                                                     IPIS_SENT, i, 1);
 475                         } else
 476                                 vm_inject_nmi(vlapic->vm, i);
 477                 }
 478
 479                 return (0);     /* handled completely in the kernel */
 480         }
 481
 482         if (mode == APIC_DELMODE_INIT) {
 483                 if ((icrval & APIC_LEVEL_MASK) == APIC_LEVEL_DEASSERT)
 484                         return (0);
 485
 486                 if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) {
 487                         vlapic2 = vm_lapic(vlapic->vm, dest);
 488
 489                         /* move from INIT to waiting-for-SIPI state */
 490                         if (vlapic2->boot_state == BS_INIT) {
 491                                 vlapic2->boot_state = BS_SIPI;
 492                         }
 493
 494                         return (0);
 495                 }
 496         }
 497
 498         if (mode == APIC_DELMODE_STARTUP) {
 499                 if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) {
 500                         vlapic2 = vm_lapic(vlapic->vm, dest);
 501
 502                         /*
 503                          * Ignore SIPIs in any state other than wait-for-SIPI
 504                          */
 505                         if (vlapic2->boot_state != BS_SIPI)
 506                                 return (0);
 507
 508                         vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid);
 509                         vmexit->exitcode = VM_EXITCODE_SPINUP_AP;
 510                         vmexit->u.spinup_ap.vcpu = dest;
 511                         vmexit->u.spinup_ap.rip = vec << PAGE_SHIFT;
 512
 513                         /*
 514                          * XXX this assumes that the startup IPI always succeeds
 515                          */
 516                         vlapic2->boot_state = BS_RUNNING;
 517                         vm_activate_cpu(vlapic2->vm, dest);
 518
 519                         return (0);
 520                 }
 521         }
 522
 523         /*
 524          * This will cause a return to userland.
 525          */
 526         return (1);
 527 }
 528
 529 int
 530 vlapic_pending_intr(struct vlapic *vlapic)
 531 {
 532         struct LAPIC    *lapic = &vlapic->apic;
 533         int              idx, i, bitpos, vector;
 534         uint32_t        *irrptr, val;
 535
 536         irrptr = &lapic->irr0;
 537
 538         /*
 539          * The x86 architecture reserves the the first 32 vectors for use
 540          * by the processor.
 541          */
 542         for (i = 7; i > 0; i--) {
 543                 idx = i * 4;
 544                 val = atomic_load_acq_int(&irrptr[idx]);
 545                 bitpos = fls(val);
 546                 if (bitpos != 0) {
 547                         vector = i * 32 + (bitpos - 1);
 548                         if (PRIO(vector) > PRIO(lapic->ppr)) {
 549                                 VLAPIC_CTR1(vlapic, "pending intr %d", vector);
 550                                 return (vector);
 551                         } else
 552                                 break;
 553                 }
 554         }
 555         VLAPIC_CTR0(vlapic, "no pending intr");
 556         return (-1);
 557 }
 558
 559 void
 560 vlapic_intr_accepted(struct vlapic *vlapic, int vector)
 561 {
 562         struct LAPIC    *lapic = &vlapic->apic;
 563         uint32_t        *irrptr, *isrptr;
 564         int             idx, stk_top;
 565
 566         /*
 567          * clear the ready bit for vector being accepted in irr
 568          * and set the vector as in service in isr.
 569          */
 570         idx = (vector / 32) * 4;
 571
 572         irrptr = &lapic->irr0;
 573         atomic_clear_int(&irrptr[idx], 1 << (vector % 32));
 574         VLAPIC_CTR_IRR(vlapic, "vlapic_intr_accepted");
 575
 576         isrptr = &lapic->isr0;
 577         isrptr[idx] |= 1 << (vector % 32);
 578         VLAPIC_CTR_ISR(vlapic, "vlapic_intr_accepted");
 579
 580         /*
 581          * Update the PPR
 582          */
 583         vlapic->isrvec_stk_top++;
 584
 585         stk_top = vlapic->isrvec_stk_top;
 586         if (stk_top >= ISRVEC_STK_SIZE)
 587                 panic("isrvec_stk_top overflow %d", stk_top);
 588
 589         vlapic->isrvec_stk[stk_top] = vector;
 590         vlapic_update_ppr(vlapic);
 591 }
 592
 593 int
 594 vlapic_op_mem_read(void* dev, uint64_t gpa, opsize_t size, uint64_t *data)
 595 {
 596         struct vlapic   *vlapic = (struct vlapic*)dev;
 597         struct LAPIC    *lapic = &vlapic->apic;
 598         uint64_t         offset = gpa & ~(PAGE_SIZE);
 599         uint32_t        *reg;
 600         int              i;
 601
 602         if (offset > sizeof(*lapic)) {
 603                 *data = 0;
 604                 return 0;
 605         }
 606
 607         offset &= ~3;
 608         switch(offset)
 609         {
 610                 case APIC_OFFSET_ID:
 611                         if (x2apic(vlapic))
 612                                 *data = vlapic->vcpuid;
 613                         else
 614                                 *data = vlapic->vcpuid << 24;
 615                         break;
 616                 case APIC_OFFSET_VER:
 617                         *data = lapic->version;
 618                         break;
 619                 case APIC_OFFSET_TPR:
 620                         *data = lapic->tpr;
 621                         break;
 622                 case APIC_OFFSET_APR:
 623                         *data = lapic->apr;
 624                         break;
 625                 case APIC_OFFSET_PPR:
 626                         *data = lapic->ppr;
 627                         break;
 628                 case APIC_OFFSET_EOI:
 629                         *data = lapic->eoi;
 630                         break;
 631                 case APIC_OFFSET_LDR:
 632                         *data = lapic->ldr;
 633                         break;
 634                 case APIC_OFFSET_DFR:
 635                         *data = lapic->dfr;
 636                         break;
 637                 case APIC_OFFSET_SVR:
 638                         *data = lapic->svr;
 639                         break;
 640                 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
 641                         i = (offset - APIC_OFFSET_ISR0) >> 2;
 642                         reg = &lapic->isr0;
 643                         *data = *(reg + i);
 644                         break;
 645                 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
 646                         i = (offset - APIC_OFFSET_TMR0) >> 2;
 647                         reg = &lapic->tmr0;
 648                         *data = *(reg + i);
 649                         break;
 650                 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
 651                         i = (offset - APIC_OFFSET_IRR0) >> 2;
 652                         reg = &lapic->irr0;
 653                         *data = atomic_load_acq_int(reg + i);
 654                         break;
 655                 case APIC_OFFSET_ESR:
 656                         *data = lapic->esr;
 657                         break;
 658                 case APIC_OFFSET_ICR_LOW:
 659                         *data = lapic->icr_lo;
 660                         break;
 661                 case APIC_OFFSET_ICR_HI:
 662                         *data = lapic->icr_hi;
 663                         break;
 664                 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
 665                         reg = vlapic_get_lvt(vlapic, offset);
 666                         *data = *(reg);
 667                         break;
 668                 case APIC_OFFSET_ICR:
 669                         *data = lapic->icr_timer;
 670                         break;
 671                 case APIC_OFFSET_CCR:
 672                         *data = vlapic_get_ccr(vlapic);
 673                         break;
 674                 case APIC_OFFSET_DCR:
 675                         *data = lapic->dcr_timer;
 676                         break;
 677                 case APIC_OFFSET_RRR:
 678                 default:
 679                         *data = 0;
 680                         break;
 681         }
 682         return 0;
 683 }
 684
 685 int
 686 vlapic_op_mem_write(void* dev, uint64_t gpa, opsize_t size, uint64_t data)
 687 {
 688         struct vlapic   *vlapic = (struct vlapic*)dev;
 689         struct LAPIC    *lapic = &vlapic->apic;
 690         uint64_t         offset = gpa & ~(PAGE_SIZE);
 691         uint32_t        *reg;
 692         int             retval;
 693
 694         if (offset > sizeof(*lapic)) {
 695                 return 0;
 696         }
 697
 698         retval = 0;
 699         offset &= ~3;
 700         switch(offset)
 701         {
 702                 case APIC_OFFSET_ID:
 703                         break;
 704                 case APIC_OFFSET_TPR:
 705                         lapic->tpr = data & 0xff;
 706                         vlapic_update_ppr(vlapic);
 707                         break;
 708                 case APIC_OFFSET_EOI:
 709                         vlapic_process_eoi(vlapic);
 710                         break;
 711                 case APIC_OFFSET_LDR:
 712                         break;
 713                 case APIC_OFFSET_DFR:
 714                         break;
 715                 case APIC_OFFSET_SVR:
 716                         lapic->svr = data;
 717                         break;
 718                 case APIC_OFFSET_ICR_LOW:
 719                         if (!x2apic(vlapic)) {
 720                                 data &= 0xffffffff;
 721                                 data |= (uint64_t)lapic->icr_hi << 32;
 722                         }
 723                         retval = lapic_process_icr(vlapic, data);
 724                         break;
 725                 case APIC_OFFSET_ICR_HI:
 726                         if (!x2apic(vlapic)) {
 727                                 retval = 0;
 728                                 lapic->icr_hi = data;
 729                         }
 730                         break;
 731                 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
 732                         reg = vlapic_get_lvt(vlapic, offset);
 733                         if (!(lapic->svr & APIC_SVR_ENABLE)) {
 734                                 data |= APIC_LVT_M;
 735                         }
 736                         *reg = data;
 737                         // vlapic_dump_lvt(offset, reg);
 738                         break;
 739                 case APIC_OFFSET_ICR:
 740                         lapic->icr_timer = data;
 741                         vlapic_start_timer(vlapic, 0);
 742                         break;
 743
 744                 case APIC_OFFSET_DCR:
 745                         lapic->dcr_timer = data;
 746                         vlapic->divisor = vlapic_timer_divisor(data);
 747                         break;
 748
 749                 case APIC_OFFSET_ESR:
 750                         vlapic_update_errors(vlapic);
 751                         break;
 752                 case APIC_OFFSET_VER:
 753                 case APIC_OFFSET_APR:
 754                 case APIC_OFFSET_PPR:
 755                 case APIC_OFFSET_RRR:
 756                 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
 757                 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
 758                 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
 759                 case APIC_OFFSET_CCR:
 760                 default:
 761                         // Read only.
 762                         break;
 763         }
 764
 765         return (retval);
 766 }
 767
 768 int
 769 vlapic_timer_tick(struct vlapic *vlapic)
 770 {
 771         int curticks, delta, periodic, fired;
 772         uint32_t ccr;
 773         uint32_t decrement, leftover;
 774
 775 restart:
 776         curticks = ticks;
 777         delta = curticks - vlapic->ccr_ticks;
 778
 779         /* Local APIC timer is disabled */
 780         if (vlapic->apic.icr_timer == 0)
 781                 return (-1);
 782
 783         /* One-shot mode and timer has already counted down to zero */
 784         periodic = vlapic_periodic_timer(vlapic);
 785         if (!periodic && vlapic->apic.ccr_timer == 0)
 786                 return (-1);
 787         /*
 788          * The 'curticks' and 'ccr_ticks' are out of sync by more than
 789          * 2^31 ticks. We deal with this by restarting the timer.
 790          */
 791         if (delta < 0) {
 792                 vlapic_start_timer(vlapic, 0);
 793                 goto restart;
 794         }
 795
 796         fired = 0;
 797         decrement = (VLAPIC_BUS_FREQ / vlapic->divisor) / hz;
 798
 799         vlapic->ccr_ticks = curticks;
 800         ccr = vlapic->apic.ccr_timer;
 801
 802         while (delta-- > 0) {
 803                 if (ccr > decrement) {
 804                         ccr -= decrement;
 805                         continue;
 806                 }
 807
 808                 /* Trigger the local apic timer interrupt */
 809                 vlapic_fire_timer(vlapic);
 810                 if (periodic) {
 811                         leftover = decrement - ccr;
 812                         vlapic_start_timer(vlapic, leftover);
 813                         ccr = vlapic->apic.ccr_timer;
 814                 } else {
 815                         /*
 816                          * One-shot timer has counted down to zero.
 817                          */
 818                         ccr = 0;
 819                 }
 820                 fired = 1;
 821                 break;
 822         }
 823
 824         vlapic->apic.ccr_timer = ccr;
 825
 826         if (!fired)
 827                 return ((ccr / decrement) + 1);
 828         else
 829                 return (0);
 830 }
 831
 832 struct vdev_ops vlapic_dev_ops = {
 833         .name = "vlapic",
 834         .init = vlapic_op_init,
 835         .reset = vlapic_op_reset,
 836         .halt = vlapic_op_halt,
 837         .memread = vlapic_op_mem_read,
 838         .memwrite = vlapic_op_mem_write,
 839 };
 840 static struct io_region vlapic_mmio[VM_MAXCPU];
 841
 842 struct vlapic *
 843 vlapic_init(struct vm *vm, int vcpuid)
 844 {
 845         struct vlapic           *vlapic;
 846
 847         vlapic = malloc(sizeof(struct vlapic), M_VLAPIC, M_WAITOK | M_ZERO);
 848         vlapic->vm = vm;
 849         vlapic->vcpuid = vcpuid;
 850
 851         vlapic->msr_apicbase = DEFAULT_APIC_BASE | APICBASE_ENABLED;
 852
 853         if (vcpuid == 0)
 854                 vlapic->msr_apicbase |= APICBASE_BSP;
 855
 856         vlapic->ops = &vlapic_dev_ops;
 857
 858         vlapic->mmio = vlapic_mmio + vcpuid;
 859         vlapic->mmio->base = DEFAULT_APIC_BASE;
 860         vlapic->mmio->len = PAGE_SIZE;
 861         vlapic->mmio->attr = MMIO_READ|MMIO_WRITE;
 862         vlapic->mmio->vcpu = vcpuid;
 863
 864         vdev_register(&vlapic_dev_ops, vlapic);
 865
 866         vlapic_op_init(vlapic);
 867
 868         return (vlapic);
 869 }
 870
 871 void
 872 vlapic_cleanup(struct vlapic *vlapic)
 873 {
 874         vlapic_op_halt(vlapic);
 875         vdev_unregister(vlapic);
 876         free(vlapic, M_VLAPIC);
 877 }
 878
 879 uint64_t
 880 vlapic_get_apicbase(struct vlapic *vlapic)
 881 {
 882
 883         return (vlapic->msr_apicbase);
 884 }
 885
 886 void
 887 vlapic_set_apicbase(struct vlapic *vlapic, uint64_t val)
 888 {
 889         int err;
 890         enum x2apic_state state;
 891
 892         err = vm_get_x2apic_state(vlapic->vm, vlapic->vcpuid, &state);
 893         if (err)
 894                 panic("vlapic_set_apicbase: err %d fetching x2apic state", err);
 895
 896         if (state == X2APIC_DISABLED)
 897                 val &= ~APICBASE_X2APIC;
 898
 899         vlapic->msr_apicbase = val;
 900 }
 901
 902 void
 903 vlapic_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
 904 {
 905         struct vlapic *vlapic;
 906
 907         vlapic = vm_lapic(vm, vcpuid);
 908
 909         if (state == X2APIC_DISABLED)
 910                 vlapic->msr_apicbase &= ~APICBASE_X2APIC;
 911 }