sys/amd64/vmm/io/vlapic.c

   1 /*-
   2  * Copyright (c) 2011 NetApp, Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  *
  26  * $FreeBSD$
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include <sys/param.h>
  33 #include <sys/kernel.h>
  34 #include <sys/malloc.h>
  35 #include <sys/systm.h>
  36 #include <sys/smp.h>
  37
  38 #include <machine/clock.h>
  39 #include <x86/specialreg.h>
  40 #include <x86/apicreg.h>
  41
  42 #include <machine/vmm.h>
  43
  44 #include "vmm_stat.h"
  45 #include "vmm_lapic.h"
  46 #include "vmm_ktr.h"
  47 #include "vdev.h"
  48 #include "vlapic.h"
  49
  50 #define VLAPIC_CTR0(vlapic, format)                                     \
  51         VMM_CTR0((vlapic)->vm, (vlapic)->vcpuid, format)
  52
  53 #define VLAPIC_CTR1(vlapic, format, p1)                                 \
  54         VMM_CTR1((vlapic)->vm, (vlapic)->vcpuid, format, p1)
  55
  56 #define VLAPIC_CTR_IRR(vlapic, msg)                                     \
  57 do {                                                                    \
  58         uint32_t *irrptr = &(vlapic)->apic.irr0;                        \
  59         irrptr[0] = irrptr[0];  /* silence compiler */                  \
  60         VLAPIC_CTR1((vlapic), msg " irr0 0x%08x", irrptr[0 << 2]);      \
  61         VLAPIC_CTR1((vlapic), msg " irr1 0x%08x", irrptr[1 << 2]);      \
  62         VLAPIC_CTR1((vlapic), msg " irr2 0x%08x", irrptr[2 << 2]);      \
  63         VLAPIC_CTR1((vlapic), msg " irr3 0x%08x", irrptr[3 << 2]);      \
  64         VLAPIC_CTR1((vlapic), msg " irr4 0x%08x", irrptr[4 << 2]);      \
  65         VLAPIC_CTR1((vlapic), msg " irr5 0x%08x", irrptr[5 << 2]);      \
  66         VLAPIC_CTR1((vlapic), msg " irr6 0x%08x", irrptr[6 << 2]);      \
  67         VLAPIC_CTR1((vlapic), msg " irr7 0x%08x", irrptr[7 << 2]);      \
  68 } while (0)
  69
  70 #define VLAPIC_CTR_ISR(vlapic, msg)                                     \
  71 do {                                                                    \
  72         uint32_t *isrptr = &(vlapic)->apic.isr0;                        \
  73         isrptr[0] = isrptr[0];  /* silence compiler */                  \
  74         VLAPIC_CTR1((vlapic), msg " isr0 0x%08x", isrptr[0 << 2]);      \
  75         VLAPIC_CTR1((vlapic), msg " isr1 0x%08x", isrptr[1 << 2]);      \
  76         VLAPIC_CTR1((vlapic), msg " isr2 0x%08x", isrptr[2 << 2]);      \
  77         VLAPIC_CTR1((vlapic), msg " isr3 0x%08x", isrptr[3 << 2]);      \
  78         VLAPIC_CTR1((vlapic), msg " isr4 0x%08x", isrptr[4 << 2]);      \
  79         VLAPIC_CTR1((vlapic), msg " isr5 0x%08x", isrptr[5 << 2]);      \
  80         VLAPIC_CTR1((vlapic), msg " isr6 0x%08x", isrptr[6 << 2]);      \
  81         VLAPIC_CTR1((vlapic), msg " isr7 0x%08x", isrptr[7 << 2]);      \
  82 } while (0)
  83
  84 static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic");
  85
  86 #define PRIO(x)                 ((x) >> 4)
  87
  88 #define VLAPIC_VERSION          (16)
  89 #define VLAPIC_MAXLVT_ENTRIES   (5)
  90
  91 #define x2apic(vlapic)  (((vlapic)->msr_apicbase & APICBASE_X2APIC) ? 1 : 0)
  92
  93 enum boot_state {
  94         BS_INIT,
  95         BS_SIPI,
  96         BS_RUNNING
  97 };
  98
  99 struct vlapic {
 100         struct vm               *vm;
 101         int                     vcpuid;
 102
 103         struct io_region        *mmio;
 104         struct vdev_ops         *ops;
 105         struct LAPIC             apic;
 106
 107         int                      esr_update;
 108
 109         int                      divisor;
 110         int                      ccr_ticks;
 111
 112         /*
 113          * The 'isrvec_stk' is a stack of vectors injected by the local apic.
 114          * A vector is popped from the stack when the processor does an EOI.
 115          * The vector on the top of the stack is used to compute the
 116          * Processor Priority in conjunction with the TPR.
 117          */
 118         uint8_t                  isrvec_stk[ISRVEC_STK_SIZE];
 119         int                      isrvec_stk_top;
 120
 121         uint64_t                msr_apicbase;
 122         enum boot_state         boot_state;
 123 };
 124
 125 #define VLAPIC_BUS_FREQ tsc_freq
 126
 127 static int
 128 vlapic_timer_divisor(uint32_t dcr)
 129 {
 130         switch (dcr & 0xB) {
 131         case APIC_TDCR_1:
 132                 return (1);
 133         case APIC_TDCR_2:
 134                 return (2);
 135         case APIC_TDCR_4:
 136                 return (4);
 137         case APIC_TDCR_8:
 138                 return (8);
 139         case APIC_TDCR_16:
 140                 return (16);
 141         case APIC_TDCR_32:
 142                 return (32);
 143         case APIC_TDCR_64:
 144                 return (64);
 145         case APIC_TDCR_128:
 146                 return (128);
 147         default:
 148                 panic("vlapic_timer_divisor: invalid dcr 0x%08x", dcr);
 149         }
 150 }
 151
 152 static void
 153 vlapic_mask_lvts(uint32_t *lvts, int num_lvt)
 154 {
 155         int i;
 156         for (i = 0; i < num_lvt; i++) {
 157                 *lvts |= APIC_LVT_M;
 158                 lvts += 4;
 159         }
 160 }
 161
 162 #if 0
 163 static inline void
 164 vlapic_dump_lvt(uint32_t offset, uint32_t *lvt)
 165 {
 166         printf("Offset %x: lvt %08x (V:%02x DS:%x M:%x)\n", offset,
 167             *lvt, *lvt & APIC_LVTT_VECTOR, *lvt & APIC_LVTT_DS,
 168             *lvt & APIC_LVTT_M);
 169 }
 170 #endif
 171
 172 static uint64_t
 173 vlapic_get_ccr(struct vlapic *vlapic)
 174 {
 175         struct LAPIC    *lapic = &vlapic->apic;
 176         return lapic->ccr_timer;
 177 }
 178
 179 static void
 180 vlapic_update_errors(struct vlapic *vlapic)
 181 {
 182         struct LAPIC    *lapic = &vlapic->apic;
 183         lapic->esr = 0; // XXX
 184 }
 185
 186 static void
 187 vlapic_init_ipi(struct vlapic *vlapic)
 188 {
 189         struct LAPIC    *lapic = &vlapic->apic;
 190         lapic->version = VLAPIC_VERSION;
 191         lapic->version |= (VLAPIC_MAXLVT_ENTRIES < MAXLVTSHIFT);
 192         lapic->dfr = 0xffffffff;
 193         lapic->svr = APIC_SVR_VECTOR;
 194         vlapic_mask_lvts(&lapic->lvt_timer, VLAPIC_MAXLVT_ENTRIES+1);
 195 }
 196
 197 static int
 198 vlapic_op_reset(void* dev)
 199 {
 200         struct vlapic   *vlapic = (struct vlapic*)dev;
 201         struct LAPIC    *lapic = &vlapic->apic;
 202
 203         memset(lapic, 0, sizeof(*lapic));
 204         lapic->apr = vlapic->vcpuid;
 205         vlapic_init_ipi(vlapic);
 206         vlapic->divisor = vlapic_timer_divisor(lapic->dcr_timer);
 207
 208         if (vlapic->vcpuid == 0)
 209                 vlapic->boot_state = BS_RUNNING;        /* BSP */
 210         else
 211                 vlapic->boot_state = BS_INIT;           /* AP */
 212
 213         return 0;
 214
 215 }
 216
 217 static int
 218 vlapic_op_init(void* dev)
 219 {
 220         struct vlapic *vlapic = (struct vlapic*)dev;
 221         vdev_register_region(vlapic->ops, vlapic, vlapic->mmio);
 222         return vlapic_op_reset(dev);
 223 }
 224
 225 static int
 226 vlapic_op_halt(void* dev)
 227 {
 228         struct vlapic *vlapic = (struct vlapic*)dev;
 229         vdev_unregister_region(vlapic, vlapic->mmio);
 230         return 0;
 231
 232 }
 233
 234 void
 235 vlapic_set_intr_ready(struct vlapic *vlapic, int vector)
 236 {
 237         struct LAPIC    *lapic = &vlapic->apic;
 238         uint32_t        *irrptr;
 239         int             idx;
 240
 241         if (vector < 0 || vector >= 256)
 242                 panic("vlapic_set_intr_ready: invalid vector %d\n", vector);
 243
 244         idx = (vector / 32) * 4;
 245         irrptr = &lapic->irr0;
 246         atomic_set_int(&irrptr[idx], 1 << (vector % 32));
 247         VLAPIC_CTR_IRR(vlapic, "vlapic_set_intr_ready");
 248 }
 249
 250 static void
 251 vlapic_start_timer(struct vlapic *vlapic, uint32_t elapsed)
 252 {
 253         uint32_t icr_timer;
 254
 255         icr_timer = vlapic->apic.icr_timer;
 256
 257         vlapic->ccr_ticks = ticks;
 258         if (elapsed < icr_timer)
 259                 vlapic->apic.ccr_timer = icr_timer - elapsed;
 260         else {
 261                 /*
 262                  * This can happen when the guest is trying to run its local
 263                  * apic timer higher that the setting of 'hz' in the host.
 264                  *
 265                  * We deal with this by running the guest local apic timer
 266                  * at the rate of the host's 'hz' setting.
 267                  */
 268                 vlapic->apic.ccr_timer = 0;
 269         }
 270 }
 271
 272 static __inline uint32_t *
 273 vlapic_get_lvt(struct vlapic *vlapic, uint32_t offset)
 274 {
 275         struct LAPIC    *lapic = &vlapic->apic;
 276         int              i;
 277
 278         if (offset < APIC_OFFSET_TIMER_LVT || offset > APIC_OFFSET_ERROR_LVT) {
 279                 panic("vlapic_get_lvt: invalid LVT\n");
 280         }
 281         i = (offset - APIC_OFFSET_TIMER_LVT) >> 2;
 282         return ((&lapic->lvt_timer) + i);;
 283 }
 284
 285 #if 1
 286 static void
 287 dump_isrvec_stk(struct vlapic *vlapic)
 288 {
 289         int i;
 290         uint32_t *isrptr;
 291
 292         isrptr = &vlapic->apic.isr0;
 293         for (i = 0; i < 8; i++)
 294                 printf("ISR%d 0x%08x\n", i, isrptr[i * 4]);
 295
 296         for (i = 0; i <= vlapic->isrvec_stk_top; i++)
 297                 printf("isrvec_stk[%d] = %d\n", i, vlapic->isrvec_stk[i]);
 298 }
 299 #endif
 300
 301 /*
 302  * Algorithm adopted from section "Interrupt, Task and Processor Priority"
 303  * in Intel Architecture Manual Vol 3a.
 304  */
 305 static void
 306 vlapic_update_ppr(struct vlapic *vlapic)
 307 {
 308         int isrvec, tpr, ppr;
 309
 310         /*
 311          * Note that the value on the stack at index 0 is always 0.
 312          *
 313          * This is a placeholder for the value of ISRV when none of the
 314          * bits is set in the ISRx registers.
 315          */
 316         isrvec = vlapic->isrvec_stk[vlapic->isrvec_stk_top];
 317         tpr = vlapic->apic.tpr;
 318
 319 #if 1
 320         {
 321                 int i, lastprio, curprio, vector, idx;
 322                 uint32_t *isrptr;
 323
 324                 if (vlapic->isrvec_stk_top == 0 && isrvec != 0)
 325                         panic("isrvec_stk is corrupted: %d", isrvec);
 326
 327                 /*
 328                  * Make sure that the priority of the nested interrupts is
 329                  * always increasing.
 330                  */
 331                 lastprio = -1;
 332                 for (i = 1; i <= vlapic->isrvec_stk_top; i++) {
 333                         curprio = PRIO(vlapic->isrvec_stk[i]);
 334                         if (curprio <= lastprio) {
 335                                 dump_isrvec_stk(vlapic);
 336                                 panic("isrvec_stk does not satisfy invariant");
 337                         }
 338                         lastprio = curprio;
 339                 }
 340
 341                 /*
 342                  * Make sure that each bit set in the ISRx registers has a
 343                  * corresponding entry on the isrvec stack.
 344                  */
 345                 i = 1;
 346                 isrptr = &vlapic->apic.isr0;
 347                 for (vector = 0; vector < 256; vector++) {
 348                         idx = (vector / 32) * 4;
 349                         if (isrptr[idx] & (1 << (vector % 32))) {
 350                                 if (i > vlapic->isrvec_stk_top ||
 351                                     vlapic->isrvec_stk[i] != vector) {
 352                                         dump_isrvec_stk(vlapic);
 353                                         panic("ISR and isrvec_stk out of sync");
 354                                 }
 355                                 i++;
 356                         }
 357                 }
 358         }
 359 #endif
 360
 361         if (PRIO(tpr) >= PRIO(isrvec))
 362                 ppr = tpr;
 363         else
 364                 ppr = isrvec & 0xf0;
 365
 366         vlapic->apic.ppr = ppr;
 367         VLAPIC_CTR1(vlapic, "vlapic_update_ppr 0x%02x", ppr);
 368 }
 369
 370 static void
 371 vlapic_process_eoi(struct vlapic *vlapic)
 372 {
 373         struct LAPIC    *lapic = &vlapic->apic;
 374         uint32_t        *isrptr;
 375         int             i, idx, bitpos;
 376
 377         isrptr = &lapic->isr0;
 378
 379         /*
 380          * The x86 architecture reserves the the first 32 vectors for use
 381          * by the processor.
 382          */
 383         for (i = 7; i > 0; i--) {
 384                 idx = i * 4;
 385                 bitpos = fls(isrptr[idx]);
 386                 if (bitpos != 0) {
 387                         if (vlapic->isrvec_stk_top <= 0) {
 388                                 panic("invalid vlapic isrvec_stk_top %d",
 389                                       vlapic->isrvec_stk_top);
 390                         }
 391                         isrptr[idx] &= ~(1 << (bitpos - 1));
 392                         VLAPIC_CTR_ISR(vlapic, "vlapic_process_eoi");
 393                         vlapic->isrvec_stk_top--;
 394                         vlapic_update_ppr(vlapic);
 395                         return;
 396                 }
 397         }
 398 }
 399
 400 static __inline int
 401 vlapic_get_lvt_field(uint32_t *lvt, uint32_t mask)
 402 {
 403         return (*lvt & mask);
 404 }
 405
 406 static __inline int
 407 vlapic_periodic_timer(struct vlapic *vlapic)
 408 {
 409         uint32_t *lvt;
 410
 411         lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
 412
 413         return (vlapic_get_lvt_field(lvt, APIC_LVTT_TM_PERIODIC));
 414 }
 415
 416 static VMM_STAT(VLAPIC_INTR_TIMER, "timer interrupts generated by vlapic");
 417
 418 static void
 419 vlapic_fire_timer(struct vlapic *vlapic)
 420 {
 421         int vector;
 422         uint32_t *lvt;
 423
 424         lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
 425
 426         if (!vlapic_get_lvt_field(lvt, APIC_LVTT_M)) {
 427                 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_TIMER, 1);
 428                 vector = vlapic_get_lvt_field(lvt,APIC_LVTT_VECTOR);
 429                 vlapic_set_intr_ready(vlapic, vector);
 430         }
 431 }
 432
 433 static VMM_STAT_ARRAY(IPIS_SENT, VM_MAXCPU, "ipis sent to vcpu");
 434
 435 static int
 436 lapic_process_icr(struct vlapic *vlapic, uint64_t icrval)
 437 {
 438         int i;
 439         cpuset_t dmask;
 440         uint32_t dest, vec, mode;
 441         struct vlapic *vlapic2;
 442         struct vm_exit *vmexit;
 443
 444         if (x2apic(vlapic))
 445                 dest = icrval >> 32;
 446         else
 447                 dest = icrval >> (32 + 24);
 448         vec = icrval & APIC_VECTOR_MASK;
 449         mode = icrval & APIC_DELMODE_MASK;
 450
 451         if (mode == APIC_DELMODE_FIXED || mode == APIC_DELMODE_NMI) {
 452                 switch (icrval & APIC_DEST_MASK) {
 453                 case APIC_DEST_DESTFLD:
 454                         CPU_SETOF(dest, &dmask);
 455                         break;
 456                 case APIC_DEST_SELF:
 457                         CPU_SETOF(vlapic->vcpuid, &dmask);
 458                         break;
 459                 case APIC_DEST_ALLISELF:
 460                         dmask = vm_active_cpus(vlapic->vm);
 461                         break;
 462                 case APIC_DEST_ALLESELF:
 463                         dmask = vm_active_cpus(vlapic->vm);
 464                         CPU_CLR(vlapic->vcpuid, &dmask);
 465                         break;
 466                 default:
 467                         CPU_ZERO(&dmask);       /* satisfy gcc */
 468                         break;
 469                 }
 470
 471                 while ((i = CPU_FFS(&dmask)) != 0) {
 472                         i--;
 473                         CPU_CLR(i, &dmask);
 474                         if (mode == APIC_DELMODE_FIXED) {
 475                                 lapic_set_intr(vlapic->vm, i, vec);
 476                                 vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid,
 477                                                     IPIS_SENT, i, 1);
 478                         } else
 479                                 vm_inject_nmi(vlapic->vm, i);
 480                 }
 481
 482                 return (0);     /* handled completely in the kernel */
 483         }
 484
 485         if (mode == APIC_DELMODE_INIT) {
 486                 if ((icrval & APIC_LEVEL_MASK) == APIC_LEVEL_DEASSERT)
 487                         return (0);
 488
 489                 if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) {
 490                         vlapic2 = vm_lapic(vlapic->vm, dest);
 491
 492                         /* move from INIT to waiting-for-SIPI state */
 493                         if (vlapic2->boot_state == BS_INIT) {
 494                                 vlapic2->boot_state = BS_SIPI;
 495                         }
 496
 497                         return (0);
 498                 }
 499         }
 500
 501         if (mode == APIC_DELMODE_STARTUP) {
 502                 if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) {
 503                         vlapic2 = vm_lapic(vlapic->vm, dest);
 504
 505                         /*
 506                          * Ignore SIPIs in any state other than wait-for-SIPI
 507                          */
 508                         if (vlapic2->boot_state != BS_SIPI)
 509                                 return (0);
 510
 511                         vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid);
 512                         vmexit->exitcode = VM_EXITCODE_SPINUP_AP;
 513                         vmexit->u.spinup_ap.vcpu = dest;
 514                         vmexit->u.spinup_ap.rip = vec << PAGE_SHIFT;
 515
 516                         /*
 517                          * XXX this assumes that the startup IPI always succeeds
 518                          */
 519                         vlapic2->boot_state = BS_RUNNING;
 520                         vm_activate_cpu(vlapic2->vm, dest);
 521
 522                         return (0);
 523                 }
 524         }
 525
 526         /*
 527          * This will cause a return to userland.
 528          */
 529         return (1);
 530 }
 531
 532 int
 533 vlapic_pending_intr(struct vlapic *vlapic)
 534 {
 535         struct LAPIC    *lapic = &vlapic->apic;
 536         int              idx, i, bitpos, vector;
 537         uint32_t        *irrptr, val;
 538
 539         irrptr = &lapic->irr0;
 540
 541         /*
 542          * The x86 architecture reserves the the first 32 vectors for use
 543          * by the processor.
 544          */
 545         for (i = 7; i > 0; i--) {
 546                 idx = i * 4;
 547                 val = atomic_load_acq_int(&irrptr[idx]);
 548                 bitpos = fls(val);
 549                 if (bitpos != 0) {
 550                         vector = i * 32 + (bitpos - 1);
 551                         if (PRIO(vector) > PRIO(lapic->ppr)) {
 552                                 VLAPIC_CTR1(vlapic, "pending intr %d", vector);
 553                                 return (vector);
 554                         } else
 555                                 break;
 556                 }
 557         }
 558         VLAPIC_CTR0(vlapic, "no pending intr");
 559         return (-1);
 560 }
 561
 562 void
 563 vlapic_intr_accepted(struct vlapic *vlapic, int vector)
 564 {
 565         struct LAPIC    *lapic = &vlapic->apic;
 566         uint32_t        *irrptr, *isrptr;
 567         int             idx, stk_top;
 568
 569         /*
 570          * clear the ready bit for vector being accepted in irr
 571          * and set the vector as in service in isr.
 572          */
 573         idx = (vector / 32) * 4;
 574
 575         irrptr = &lapic->irr0;
 576         atomic_clear_int(&irrptr[idx], 1 << (vector % 32));
 577         VLAPIC_CTR_IRR(vlapic, "vlapic_intr_accepted");
 578
 579         isrptr = &lapic->isr0;
 580         isrptr[idx] |= 1 << (vector % 32);
 581         VLAPIC_CTR_ISR(vlapic, "vlapic_intr_accepted");
 582
 583         /*
 584          * Update the PPR
 585          */
 586         vlapic->isrvec_stk_top++;
 587
 588         stk_top = vlapic->isrvec_stk_top;
 589         if (stk_top >= ISRVEC_STK_SIZE)
 590                 panic("isrvec_stk_top overflow %d", stk_top);
 591
 592         vlapic->isrvec_stk[stk_top] = vector;
 593         vlapic_update_ppr(vlapic);
 594 }
 595
 596 int
 597 vlapic_op_mem_read(void* dev, uint64_t gpa, opsize_t size, uint64_t *data)
 598 {
 599         struct vlapic   *vlapic = (struct vlapic*)dev;
 600         struct LAPIC    *lapic = &vlapic->apic;
 601         uint64_t         offset = gpa & ~(PAGE_SIZE);
 602         uint32_t        *reg;
 603         int              i;
 604
 605         if (offset > sizeof(*lapic)) {
 606                 *data = 0;
 607                 return 0;
 608         }
 609
 610         offset &= ~3;
 611         switch(offset)
 612         {
 613                 case APIC_OFFSET_ID:
 614                         if (x2apic(vlapic))
 615                                 *data = vlapic->vcpuid;
 616                         else
 617                                 *data = vlapic->vcpuid << 24;
 618                         break;
 619                 case APIC_OFFSET_VER:
 620                         *data = lapic->version;
 621                         break;
 622                 case APIC_OFFSET_TPR:
 623                         *data = lapic->tpr;
 624                         break;
 625                 case APIC_OFFSET_APR:
 626                         *data = lapic->apr;
 627                         break;
 628                 case APIC_OFFSET_PPR:
 629                         *data = lapic->ppr;
 630                         break;
 631                 case APIC_OFFSET_EOI:
 632                         *data = lapic->eoi;
 633                         break;
 634                 case APIC_OFFSET_LDR:
 635                         *data = lapic->ldr;
 636                         break;
 637                 case APIC_OFFSET_DFR:
 638                         *data = lapic->dfr;
 639                         break;
 640                 case APIC_OFFSET_SVR:
 641                         *data = lapic->svr;
 642                         break;
 643                 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
 644                         i = (offset - APIC_OFFSET_ISR0) >> 2;
 645                         reg = &lapic->isr0;
 646                         *data = *(reg + i);
 647                         break;
 648                 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
 649                         i = (offset - APIC_OFFSET_TMR0) >> 2;
 650                         reg = &lapic->tmr0;
 651                         *data = *(reg + i);
 652                         break;
 653                 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
 654                         i = (offset - APIC_OFFSET_IRR0) >> 2;
 655                         reg = &lapic->irr0;
 656                         *data = atomic_load_acq_int(reg + i);
 657                         break;
 658                 case APIC_OFFSET_ESR:
 659                         *data = lapic->esr;
 660                         break;
 661                 case APIC_OFFSET_ICR_LOW:
 662                         *data = lapic->icr_lo;
 663                         break;
 664                 case APIC_OFFSET_ICR_HI:
 665                         *data = lapic->icr_hi;
 666                         break;
 667                 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
 668                         reg = vlapic_get_lvt(vlapic, offset);
 669                         *data = *(reg);
 670                         break;
 671                 case APIC_OFFSET_ICR:
 672                         *data = lapic->icr_timer;
 673                         break;
 674                 case APIC_OFFSET_CCR:
 675                         *data = vlapic_get_ccr(vlapic);
 676                         break;
 677                 case APIC_OFFSET_DCR:
 678                         *data = lapic->dcr_timer;
 679                         break;
 680                 case APIC_OFFSET_RRR:
 681                 default:
 682                         *data = 0;
 683                         break;
 684         }
 685         return 0;
 686 }
 687
 688 int
 689 vlapic_op_mem_write(void* dev, uint64_t gpa, opsize_t size, uint64_t data)
 690 {
 691         struct vlapic   *vlapic = (struct vlapic*)dev;
 692         struct LAPIC    *lapic = &vlapic->apic;
 693         uint64_t         offset = gpa & ~(PAGE_SIZE);
 694         uint32_t        *reg;
 695         int             retval;
 696
 697         if (offset > sizeof(*lapic)) {
 698                 return 0;
 699         }
 700
 701         retval = 0;
 702         offset &= ~3;
 703         switch(offset)
 704         {
 705                 case APIC_OFFSET_ID:
 706                         break;
 707                 case APIC_OFFSET_TPR:
 708                         lapic->tpr = data & 0xff;
 709                         vlapic_update_ppr(vlapic);
 710                         break;
 711                 case APIC_OFFSET_EOI:
 712                         vlapic_process_eoi(vlapic);
 713                         break;
 714                 case APIC_OFFSET_LDR:
 715                         break;
 716                 case APIC_OFFSET_DFR:
 717                         break;
 718                 case APIC_OFFSET_SVR:
 719                         lapic->svr = data;
 720                         break;
 721                 case APIC_OFFSET_ICR_LOW:
 722                         if (!x2apic(vlapic)) {
 723                                 data &= 0xffffffff;
 724                                 data |= (uint64_t)lapic->icr_hi << 32;
 725                         }
 726                         retval = lapic_process_icr(vlapic, data);
 727                         break;
 728                 case APIC_OFFSET_ICR_HI:
 729                         if (!x2apic(vlapic)) {
 730                                 retval = 0;
 731                                 lapic->icr_hi = data;
 732                         }
 733                         break;
 734                 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
 735                         reg = vlapic_get_lvt(vlapic, offset);
 736                         if (!(lapic->svr & APIC_SVR_ENABLE)) {
 737                                 data |= APIC_LVT_M;
 738                         }
 739                         *reg = data;
 740                         // vlapic_dump_lvt(offset, reg);
 741                         break;
 742                 case APIC_OFFSET_ICR:
 743                         lapic->icr_timer = data;
 744                         vlapic_start_timer(vlapic, 0);
 745                         break;
 746
 747                 case APIC_OFFSET_DCR:
 748                         lapic->dcr_timer = data;
 749                         vlapic->divisor = vlapic_timer_divisor(data);
 750                         break;
 751
 752                 case APIC_OFFSET_ESR:
 753                         vlapic_update_errors(vlapic);
 754                         break;
 755                 case APIC_OFFSET_VER:
 756                 case APIC_OFFSET_APR:
 757                 case APIC_OFFSET_PPR:
 758                 case APIC_OFFSET_RRR:
 759                 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
 760                 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
 761                 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
 762                 case APIC_OFFSET_CCR:
 763                 default:
 764                         // Read only.
 765                         break;
 766         }
 767
 768         return (retval);
 769 }
 770
 771 int
 772 vlapic_timer_tick(struct vlapic *vlapic)
 773 {
 774         int curticks, delta, periodic, fired;
 775         uint32_t ccr;
 776         uint32_t decrement, leftover;
 777
 778 restart:
 779         curticks = ticks;
 780         delta = curticks - vlapic->ccr_ticks;
 781
 782         /* Local APIC timer is disabled */
 783         if (vlapic->apic.icr_timer == 0)
 784                 return (-1);
 785
 786         /* One-shot mode and timer has already counted down to zero */
 787         periodic = vlapic_periodic_timer(vlapic);
 788         if (!periodic && vlapic->apic.ccr_timer == 0)
 789                 return (-1);
 790         /*
 791          * The 'curticks' and 'ccr_ticks' are out of sync by more than
 792          * 2^31 ticks. We deal with this by restarting the timer.
 793          */
 794         if (delta < 0) {
 795                 vlapic_start_timer(vlapic, 0);
 796                 goto restart;
 797         }
 798
 799         fired = 0;
 800         decrement = (VLAPIC_BUS_FREQ / vlapic->divisor) / hz;
 801
 802         vlapic->ccr_ticks = curticks;
 803         ccr = vlapic->apic.ccr_timer;
 804
 805         while (delta-- > 0) {
 806                 if (ccr > decrement) {
 807                         ccr -= decrement;
 808                         continue;
 809                 }
 810
 811                 /* Trigger the local apic timer interrupt */
 812                 vlapic_fire_timer(vlapic);
 813                 if (periodic) {
 814                         leftover = decrement - ccr;
 815                         vlapic_start_timer(vlapic, leftover);
 816                         ccr = vlapic->apic.ccr_timer;
 817                 } else {
 818                         /*
 819                          * One-shot timer has counted down to zero.
 820                          */
 821                         ccr = 0;
 822                 }
 823                 fired = 1;
 824                 break;
 825         }
 826
 827         vlapic->apic.ccr_timer = ccr;
 828
 829         if (!fired)
 830                 return ((ccr / decrement) + 1);
 831         else
 832                 return (0);
 833 }
 834
 835 struct vdev_ops vlapic_dev_ops = {
 836         .name = "vlapic",
 837         .init = vlapic_op_init,
 838         .reset = vlapic_op_reset,
 839         .halt = vlapic_op_halt,
 840         .memread = vlapic_op_mem_read,
 841         .memwrite = vlapic_op_mem_write,
 842 };
 843 static struct io_region vlapic_mmio[VM_MAXCPU];
 844
 845 struct vlapic *
 846 vlapic_init(struct vm *vm, int vcpuid)
 847 {
 848         struct vlapic           *vlapic;
 849
 850         vlapic = malloc(sizeof(struct vlapic), M_VLAPIC, M_WAITOK | M_ZERO);
 851         vlapic->vm = vm;
 852         vlapic->vcpuid = vcpuid;
 853
 854         vlapic->msr_apicbase = DEFAULT_APIC_BASE | APICBASE_ENABLED;
 855
 856         if (vcpuid == 0)
 857                 vlapic->msr_apicbase |= APICBASE_BSP;
 858
 859         vlapic->ops = &vlapic_dev_ops;
 860
 861         vlapic->mmio = vlapic_mmio + vcpuid;
 862         vlapic->mmio->base = DEFAULT_APIC_BASE;
 863         vlapic->mmio->len = PAGE_SIZE;
 864         vlapic->mmio->attr = MMIO_READ|MMIO_WRITE;
 865         vlapic->mmio->vcpu = vcpuid;
 866
 867         vdev_register(&vlapic_dev_ops, vlapic);
 868
 869         vlapic_op_init(vlapic);
 870
 871         return (vlapic);
 872 }
 873
 874 void
 875 vlapic_cleanup(struct vlapic *vlapic)
 876 {
 877         vlapic_op_halt(vlapic);
 878         vdev_unregister(vlapic);
 879         free(vlapic, M_VLAPIC);
 880 }
 881
 882 uint64_t
 883 vlapic_get_apicbase(struct vlapic *vlapic)
 884 {
 885
 886         return (vlapic->msr_apicbase);
 887 }
 888
 889 void
 890 vlapic_set_apicbase(struct vlapic *vlapic, uint64_t val)
 891 {
 892         int err;
 893         enum x2apic_state state;
 894
 895         err = vm_get_x2apic_state(vlapic->vm, vlapic->vcpuid, &state);
 896         if (err)
 897                 panic("vlapic_set_apicbase: err %d fetching x2apic state", err);
 898
 899         if (state == X2APIC_DISABLED)
 900                 val &= ~APICBASE_X2APIC;
 901
 902         vlapic->msr_apicbase = val;
 903 }
 904
 905 void
 906 vlapic_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
 907 {
 908         struct vlapic *vlapic;
 909
 910         vlapic = vm_lapic(vm, vcpuid);
 911
 912         if (state == X2APIC_DISABLED)
 913                 vlapic->msr_apicbase &= ~APICBASE_X2APIC;
 914 }