sys/amd64/vmm/intel/vmx.c

   1 /*-
   2  * Copyright (c) 2011 NetApp, Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  *
  26  * $FreeBSD$
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include <sys/param.h>
  33 #include <sys/systm.h>
  34 #include <sys/smp.h>
  35 #include <sys/kernel.h>
  36 #include <sys/malloc.h>
  37 #include <sys/pcpu.h>
  38 #include <sys/proc.h>
  39 #include <sys/sysctl.h>
  40
  41 #include <vm/vm.h>
  42 #include <vm/pmap.h>
  43
  44 #include <machine/psl.h>
  45 #include <machine/cpufunc.h>
  46 #include <machine/md_var.h>
  47 #include <machine/segments.h>
  48 #include <machine/smp.h>
  49 #include <machine/specialreg.h>
  50 #include <machine/vmparam.h>
  51
  52 #include <machine/vmm.h>
  53 #include <machine/vmm_dev.h>
  54 #include <machine/vmm_instruction_emul.h>
  55 #include "vmm_lapic.h"
  56 #include "vmm_host.h"
  57 #include "vmm_ioport.h"
  58 #include "vmm_ktr.h"
  59 #include "vmm_stat.h"
  60 #include "vatpic.h"
  61 #include "vlapic.h"
  62 #include "vlapic_priv.h"
  63
  64 #include "ept.h"
  65 #include "vmx_cpufunc.h"
  66 #include "vmx.h"
  67 #include "vmx_msr.h"
  68 #include "x86.h"
  69 #include "vmx_controls.h"
  70
  71 #define PINBASED_CTLS_ONE_SETTING                                       \
  72         (PINBASED_EXTINT_EXITING        |                               \
  73          PINBASED_NMI_EXITING           |                               \
  74          PINBASED_VIRTUAL_NMI)
  75 #define PINBASED_CTLS_ZERO_SETTING      0
  76
  77 #define PROCBASED_CTLS_WINDOW_SETTING                                   \
  78         (PROCBASED_INT_WINDOW_EXITING   |                               \
  79          PROCBASED_NMI_WINDOW_EXITING)
  80
  81 #define PROCBASED_CTLS_ONE_SETTING                                      \
  82         (PROCBASED_SECONDARY_CONTROLS   |                               \
  83          PROCBASED_MWAIT_EXITING        |                               \
  84          PROCBASED_MONITOR_EXITING      |                               \
  85          PROCBASED_IO_EXITING           |                               \
  86          PROCBASED_MSR_BITMAPS          |                               \
  87          PROCBASED_CTLS_WINDOW_SETTING  |                               \
  88          PROCBASED_CR8_LOAD_EXITING     |                               \
  89          PROCBASED_CR8_STORE_EXITING)
  90 #define PROCBASED_CTLS_ZERO_SETTING     \
  91         (PROCBASED_CR3_LOAD_EXITING |   \
  92         PROCBASED_CR3_STORE_EXITING |   \
  93         PROCBASED_IO_BITMAPS)
  94
  95 #define PROCBASED_CTLS2_ONE_SETTING     PROCBASED2_ENABLE_EPT
  96 #define PROCBASED_CTLS2_ZERO_SETTING    0
  97
  98 #define VM_EXIT_CTLS_ONE_SETTING                                        \
  99         (VM_EXIT_SAVE_DEBUG_CONTROLS            |                       \
 100         VM_EXIT_HOST_LMA                        |                       \
 101         VM_EXIT_SAVE_EFER                       |                       \
 102         VM_EXIT_LOAD_EFER                       |                       \
 103         VM_EXIT_ACKNOWLEDGE_INTERRUPT)
 104
 105 #define VM_EXIT_CTLS_ZERO_SETTING       0
 106
 107 #define VM_ENTRY_CTLS_ONE_SETTING                                       \
 108         (VM_ENTRY_LOAD_DEBUG_CONTROLS           |                       \
 109         VM_ENTRY_LOAD_EFER)
 110
 111 #define VM_ENTRY_CTLS_ZERO_SETTING                                      \
 112         (VM_ENTRY_INTO_SMM                      |                       \
 113         VM_ENTRY_DEACTIVATE_DUAL_MONITOR)
 114
 115 #define HANDLED         1
 116 #define UNHANDLED       0
 117
 118 static MALLOC_DEFINE(M_VMX, "vmx", "vmx");
 119 static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic");
 120
 121 SYSCTL_DECL(_hw_vmm);
 122 SYSCTL_NODE(_hw_vmm, OID_AUTO, vmx, CTLFLAG_RW, NULL, NULL);
 123
 124 int vmxon_enabled[MAXCPU];
 125 static char vmxon_region[MAXCPU][PAGE_SIZE] __aligned(PAGE_SIZE);
 126
 127 static uint32_t pinbased_ctls, procbased_ctls, procbased_ctls2;
 128 static uint32_t exit_ctls, entry_ctls;
 129
 130 static uint64_t cr0_ones_mask, cr0_zeros_mask;
 131 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_ones_mask, CTLFLAG_RD,
 132              &cr0_ones_mask, 0, NULL);
 133 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_zeros_mask, CTLFLAG_RD,
 134              &cr0_zeros_mask, 0, NULL);
 135
 136 static uint64_t cr4_ones_mask, cr4_zeros_mask;
 137 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_ones_mask, CTLFLAG_RD,
 138              &cr4_ones_mask, 0, NULL);
 139 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_zeros_mask, CTLFLAG_RD,
 140              &cr4_zeros_mask, 0, NULL);
 141
 142 static int vmx_initialized;
 143 SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, initialized, CTLFLAG_RD,
 144            &vmx_initialized, 0, "Intel VMX initialized");
 145
 146 /*
 147  * Optional capabilities
 148  */
 149 static SYSCTL_NODE(_hw_vmm_vmx, OID_AUTO, cap, CTLFLAG_RW, NULL, NULL);
 150
 151 static int cap_halt_exit;
 152 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, halt_exit, CTLFLAG_RD, &cap_halt_exit, 0,
 153     "HLT triggers a VM-exit");
 154
 155 static int cap_pause_exit;
 156 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, pause_exit, CTLFLAG_RD, &cap_pause_exit,
 157     0, "PAUSE triggers a VM-exit");
 158
 159 static int cap_unrestricted_guest;
 160 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, unrestricted_guest, CTLFLAG_RD,
 161     &cap_unrestricted_guest, 0, "Unrestricted guests");
 162
 163 static int cap_monitor_trap;
 164 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, monitor_trap, CTLFLAG_RD,
 165     &cap_monitor_trap, 0, "Monitor trap flag");
 166
 167 static int cap_invpcid;
 168 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, invpcid, CTLFLAG_RD, &cap_invpcid,
 169     0, "Guests are allowed to use INVPCID");
 170
 171 static int virtual_interrupt_delivery;
 172 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, virtual_interrupt_delivery, CTLFLAG_RD,
 173     &virtual_interrupt_delivery, 0, "APICv virtual interrupt delivery support");
 174
 175 static int posted_interrupts;
 176 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, posted_interrupts, CTLFLAG_RD,
 177     &posted_interrupts, 0, "APICv posted interrupt support");
 178
 179 static int pirvec = -1;
 180 SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupt_vector, CTLFLAG_RD,
 181     &pirvec, 0, "APICv posted interrupt vector");
 182
 183 static struct unrhdr *vpid_unr;
 184 static u_int vpid_alloc_failed;
 185 SYSCTL_UINT(_hw_vmm_vmx, OID_AUTO, vpid_alloc_failed, CTLFLAG_RD,
 186             &vpid_alloc_failed, 0, NULL);
 187
 188 static int guest_l1d_flush;
 189 SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, l1d_flush, CTLFLAG_RD,
 190     &guest_l1d_flush, 0, NULL);
 191
 192 uint64_t vmx_msr_flush_cmd;
 193
 194 /*
 195  * Use the last page below 4GB as the APIC access address. This address is
 196  * occupied by the boot firmware so it is guaranteed that it will not conflict
 197  * with a page in system memory.
 198  */
 199 #define APIC_ACCESS_ADDRESS     0xFFFFF000
 200
 201 static int vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc);
 202 static int vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval);
 203 static int vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val);
 204 static void vmx_inject_pir(struct vlapic *vlapic);
 205
 206 #ifdef KTR
 207 static const char *
 208 exit_reason_to_str(int reason)
 209 {
 210         static char reasonbuf[32];
 211
 212         switch (reason) {
 213         case EXIT_REASON_EXCEPTION:
 214                 return "exception";
 215         case EXIT_REASON_EXT_INTR:
 216                 return "extint";
 217         case EXIT_REASON_TRIPLE_FAULT:
 218                 return "triplefault";
 219         case EXIT_REASON_INIT:
 220                 return "init";
 221         case EXIT_REASON_SIPI:
 222                 return "sipi";
 223         case EXIT_REASON_IO_SMI:
 224                 return "iosmi";
 225         case EXIT_REASON_SMI:
 226                 return "smi";
 227         case EXIT_REASON_INTR_WINDOW:
 228                 return "intrwindow";
 229         case EXIT_REASON_NMI_WINDOW:
 230                 return "nmiwindow";
 231         case EXIT_REASON_TASK_SWITCH:
 232                 return "taskswitch";
 233         case EXIT_REASON_CPUID:
 234                 return "cpuid";
 235         case EXIT_REASON_GETSEC:
 236                 return "getsec";
 237         case EXIT_REASON_HLT:
 238                 return "hlt";
 239         case EXIT_REASON_INVD:
 240                 return "invd";
 241         case EXIT_REASON_INVLPG:
 242                 return "invlpg";
 243         case EXIT_REASON_RDPMC:
 244                 return "rdpmc";
 245         case EXIT_REASON_RDTSC:
 246                 return "rdtsc";
 247         case EXIT_REASON_RSM:
 248                 return "rsm";
 249         case EXIT_REASON_VMCALL:
 250                 return "vmcall";
 251         case EXIT_REASON_VMCLEAR:
 252                 return "vmclear";
 253         case EXIT_REASON_VMLAUNCH:
 254                 return "vmlaunch";
 255         case EXIT_REASON_VMPTRLD:
 256                 return "vmptrld";
 257         case EXIT_REASON_VMPTRST:
 258                 return "vmptrst";
 259         case EXIT_REASON_VMREAD:
 260                 return "vmread";
 261         case EXIT_REASON_VMRESUME:
 262                 return "vmresume";
 263         case EXIT_REASON_VMWRITE:
 264                 return "vmwrite";
 265         case EXIT_REASON_VMXOFF:
 266                 return "vmxoff";
 267         case EXIT_REASON_VMXON:
 268                 return "vmxon";
 269         case EXIT_REASON_CR_ACCESS:
 270                 return "craccess";
 271         case EXIT_REASON_DR_ACCESS:
 272                 return "draccess";
 273         case EXIT_REASON_INOUT:
 274                 return "inout";
 275         case EXIT_REASON_RDMSR:
 276                 return "rdmsr";
 277         case EXIT_REASON_WRMSR:
 278                 return "wrmsr";
 279         case EXIT_REASON_INVAL_VMCS:
 280                 return "invalvmcs";
 281         case EXIT_REASON_INVAL_MSR:
 282                 return "invalmsr";
 283         case EXIT_REASON_MWAIT:
 284                 return "mwait";
 285         case EXIT_REASON_MTF:
 286                 return "mtf";
 287         case EXIT_REASON_MONITOR:
 288                 return "monitor";
 289         case EXIT_REASON_PAUSE:
 290                 return "pause";
 291         case EXIT_REASON_MCE_DURING_ENTRY:
 292                 return "mce-during-entry";
 293         case EXIT_REASON_TPR:
 294                 return "tpr";
 295         case EXIT_REASON_APIC_ACCESS:
 296                 return "apic-access";
 297         case EXIT_REASON_GDTR_IDTR:
 298                 return "gdtridtr";
 299         case EXIT_REASON_LDTR_TR:
 300                 return "ldtrtr";
 301         case EXIT_REASON_EPT_FAULT:
 302                 return "eptfault";
 303         case EXIT_REASON_EPT_MISCONFIG:
 304                 return "eptmisconfig";
 305         case EXIT_REASON_INVEPT:
 306                 return "invept";
 307         case EXIT_REASON_RDTSCP:
 308                 return "rdtscp";
 309         case EXIT_REASON_VMX_PREEMPT:
 310                 return "vmxpreempt";
 311         case EXIT_REASON_INVVPID:
 312                 return "invvpid";
 313         case EXIT_REASON_WBINVD:
 314                 return "wbinvd";
 315         case EXIT_REASON_XSETBV:
 316                 return "xsetbv";
 317         case EXIT_REASON_APIC_WRITE:
 318                 return "apic-write";
 319         default:
 320                 snprintf(reasonbuf, sizeof(reasonbuf), "%d", reason);
 321                 return (reasonbuf);
 322         }
 323 }
 324 #endif  /* KTR */
 325
 326 static int
 327 vmx_allow_x2apic_msrs(struct vmx *vmx)
 328 {
 329         int i, error;
 330
 331         error = 0;
 332
 333         /*
 334          * Allow readonly access to the following x2APIC MSRs from the guest.
 335          */
 336         error += guest_msr_ro(vmx, MSR_APIC_ID);
 337         error += guest_msr_ro(vmx, MSR_APIC_VERSION);
 338         error += guest_msr_ro(vmx, MSR_APIC_LDR);
 339         error += guest_msr_ro(vmx, MSR_APIC_SVR);
 340
 341         for (i = 0; i < 8; i++)
 342                 error += guest_msr_ro(vmx, MSR_APIC_ISR0 + i);
 343
 344         for (i = 0; i < 8; i++)
 345                 error += guest_msr_ro(vmx, MSR_APIC_TMR0 + i);
 346
 347         for (i = 0; i < 8; i++)
 348                 error += guest_msr_ro(vmx, MSR_APIC_IRR0 + i);
 349
 350         error += guest_msr_ro(vmx, MSR_APIC_ESR);
 351         error += guest_msr_ro(vmx, MSR_APIC_LVT_TIMER);
 352         error += guest_msr_ro(vmx, MSR_APIC_LVT_THERMAL);
 353         error += guest_msr_ro(vmx, MSR_APIC_LVT_PCINT);
 354         error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT0);
 355         error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT1);
 356         error += guest_msr_ro(vmx, MSR_APIC_LVT_ERROR);
 357         error += guest_msr_ro(vmx, MSR_APIC_ICR_TIMER);
 358         error += guest_msr_ro(vmx, MSR_APIC_DCR_TIMER);
 359         error += guest_msr_ro(vmx, MSR_APIC_ICR);
 360
 361         /*
 362          * Allow TPR, EOI and SELF_IPI MSRs to be read and written by the guest.
 363          *
 364          * These registers get special treatment described in the section
 365          * "Virtualizing MSR-Based APIC Accesses".
 366          */
 367         error += guest_msr_rw(vmx, MSR_APIC_TPR);
 368         error += guest_msr_rw(vmx, MSR_APIC_EOI);
 369         error += guest_msr_rw(vmx, MSR_APIC_SELF_IPI);
 370
 371         return (error);
 372 }
 373
 374 u_long
 375 vmx_fix_cr0(u_long cr0)
 376 {
 377
 378         return ((cr0 | cr0_ones_mask) & ~cr0_zeros_mask);
 379 }
 380
 381 u_long
 382 vmx_fix_cr4(u_long cr4)
 383 {
 384
 385         return ((cr4 | cr4_ones_mask) & ~cr4_zeros_mask);
 386 }
 387
 388 static void
 389 vpid_free(int vpid)
 390 {
 391         if (vpid < 0 || vpid > 0xffff)
 392                 panic("vpid_free: invalid vpid %d", vpid);
 393
 394         /*
 395          * VPIDs [0,VM_MAXCPU] are special and are not allocated from
 396          * the unit number allocator.
 397          */
 398
 399         if (vpid > VM_MAXCPU)
 400                 free_unr(vpid_unr, vpid);
 401 }
 402
 403 static void
 404 vpid_alloc(uint16_t *vpid, int num)
 405 {
 406         int i, x;
 407
 408         if (num <= 0 || num > VM_MAXCPU)
 409                 panic("invalid number of vpids requested: %d", num);
 410
 411         /*
 412          * If the "enable vpid" execution control is not enabled then the
 413          * VPID is required to be 0 for all vcpus.
 414          */
 415         if ((procbased_ctls2 & PROCBASED2_ENABLE_VPID) == 0) {
 416                 for (i = 0; i < num; i++)
 417                         vpid[i] = 0;
 418                 return;
 419         }
 420
 421         /*
 422          * Allocate a unique VPID for each vcpu from the unit number allocator.
 423          */
 424         for (i = 0; i < num; i++) {
 425                 x = alloc_unr(vpid_unr);
 426                 if (x == -1)
 427                         break;
 428                 else
 429                         vpid[i] = x;
 430         }
 431
 432         if (i < num) {
 433                 atomic_add_int(&vpid_alloc_failed, 1);
 434
 435                 /*
 436                  * If the unit number allocator does not have enough unique
 437                  * VPIDs then we need to allocate from the [1,VM_MAXCPU] range.
 438                  *
 439                  * These VPIDs are not be unique across VMs but this does not
 440                  * affect correctness because the combined mappings are also
 441                  * tagged with the EP4TA which is unique for each VM.
 442                  *
 443                  * It is still sub-optimal because the invvpid will invalidate
 444                  * combined mappings for a particular VPID across all EP4TAs.
 445                  */
 446                 while (i-- > 0)
 447                         vpid_free(vpid[i]);
 448
 449                 for (i = 0; i < num; i++)
 450                         vpid[i] = i + 1;
 451         }
 452 }
 453
 454 static void
 455 vpid_init(void)
 456 {
 457         /*
 458          * VPID 0 is required when the "enable VPID" execution control is
 459          * disabled.
 460          *
 461          * VPIDs [1,VM_MAXCPU] are used as the "overflow namespace" when the
 462          * unit number allocator does not have sufficient unique VPIDs to
 463          * satisfy the allocation.
 464          *
 465          * The remaining VPIDs are managed by the unit number allocator.
 466          */
 467         vpid_unr = new_unrhdr(VM_MAXCPU + 1, 0xffff, NULL);
 468 }
 469
 470 static void
 471 vmx_disable(void *arg __unused)
 472 {
 473         struct invvpid_desc invvpid_desc = { 0 };
 474         struct invept_desc invept_desc = { 0 };
 475
 476         if (vmxon_enabled[curcpu]) {
 477                 /*
 478                  * See sections 25.3.3.3 and 25.3.3.4 in Intel Vol 3b.
 479                  *
 480                  * VMXON or VMXOFF are not required to invalidate any TLB
 481                  * caching structures. This prevents potential retention of
 482                  * cached information in the TLB between distinct VMX episodes.
 483                  */
 484                 invvpid(INVVPID_TYPE_ALL_CONTEXTS, invvpid_desc);
 485                 invept(INVEPT_TYPE_ALL_CONTEXTS, invept_desc);
 486                 vmxoff();
 487         }
 488         load_cr4(rcr4() & ~CR4_VMXE);
 489 }
 490
 491 static int
 492 vmx_cleanup(void)
 493 {
 494
 495         if (pirvec >= 0)
 496                 lapic_ipi_free(pirvec);
 497
 498         if (vpid_unr != NULL) {
 499                 delete_unrhdr(vpid_unr);
 500                 vpid_unr = NULL;
 501         }
 502
 503         smp_rendezvous(NULL, vmx_disable, NULL, NULL);
 504
 505         return (0);
 506 }
 507
 508 static void
 509 vmx_enable(void *arg __unused)
 510 {
 511         int error;
 512         uint64_t feature_control;
 513
 514         feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
 515         if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 0 ||
 516             (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) {
 517                 wrmsr(MSR_IA32_FEATURE_CONTROL,
 518                     feature_control | IA32_FEATURE_CONTROL_VMX_EN |
 519                     IA32_FEATURE_CONTROL_LOCK);
 520         }
 521
 522         load_cr4(rcr4() | CR4_VMXE);
 523
 524         *(uint32_t *)vmxon_region[curcpu] = vmx_revision();
 525         error = vmxon(vmxon_region[curcpu]);
 526         if (error == 0)
 527                 vmxon_enabled[curcpu] = 1;
 528 }
 529
 530 static void
 531 vmx_restore(void)
 532 {
 533
 534         if (vmxon_enabled[curcpu])
 535                 vmxon(vmxon_region[curcpu]);
 536 }
 537
 538 static int
 539 vmx_init(int ipinum)
 540 {
 541         int error, use_tpr_shadow;
 542         uint64_t basic, fixed0, fixed1, feature_control;
 543         uint32_t tmp, procbased2_vid_bits;
 544
 545         /* CPUID.1:ECX[bit 5] must be 1 for processor to support VMX */
 546         if (!(cpu_feature2 & CPUID2_VMX)) {
 547                 printf("vmx_init: processor does not support VMX operation\n");
 548                 return (ENXIO);
 549         }
 550
 551         /*
 552          * Verify that MSR_IA32_FEATURE_CONTROL lock and VMXON enable bits
 553          * are set (bits 0 and 2 respectively).
 554          */
 555         feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
 556         if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 1 &&
 557             (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) {
 558                 printf("vmx_init: VMX operation disabled by BIOS\n");
 559                 return (ENXIO);
 560         }
 561
 562         /*
 563          * Verify capabilities MSR_VMX_BASIC:
 564          * - bit 54 indicates support for INS/OUTS decoding
 565          */
 566         basic = rdmsr(MSR_VMX_BASIC);
 567         if ((basic & (1UL << 54)) == 0) {
 568                 printf("vmx_init: processor does not support desired basic "
 569                     "capabilities\n");
 570                 return (EINVAL);
 571         }
 572
 573         /* Check support for primary processor-based VM-execution controls */
 574         error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 575                                MSR_VMX_TRUE_PROCBASED_CTLS,
 576                                PROCBASED_CTLS_ONE_SETTING,
 577                                PROCBASED_CTLS_ZERO_SETTING, &procbased_ctls);
 578         if (error) {
 579                 printf("vmx_init: processor does not support desired primary "
 580                        "processor-based controls\n");
 581                 return (error);
 582         }
 583
 584         /* Clear the processor-based ctl bits that are set on demand */
 585         procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING;
 586
 587         /* Check support for secondary processor-based VM-execution controls */
 588         error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
 589                                MSR_VMX_PROCBASED_CTLS2,
 590                                PROCBASED_CTLS2_ONE_SETTING,
 591                                PROCBASED_CTLS2_ZERO_SETTING, &procbased_ctls2);
 592         if (error) {
 593                 printf("vmx_init: processor does not support desired secondary "
 594                        "processor-based controls\n");
 595                 return (error);
 596         }
 597
 598         /* Check support for VPID */
 599         error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
 600                                PROCBASED2_ENABLE_VPID, 0, &tmp);
 601         if (error == 0)
 602                 procbased_ctls2 |= PROCBASED2_ENABLE_VPID;
 603
 604         /* Check support for pin-based VM-execution controls */
 605         error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
 606                                MSR_VMX_TRUE_PINBASED_CTLS,
 607                                PINBASED_CTLS_ONE_SETTING,
 608                                PINBASED_CTLS_ZERO_SETTING, &pinbased_ctls);
 609         if (error) {
 610                 printf("vmx_init: processor does not support desired "
 611                        "pin-based controls\n");
 612                 return (error);
 613         }
 614
 615         /* Check support for VM-exit controls */
 616         error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, MSR_VMX_TRUE_EXIT_CTLS,
 617                                VM_EXIT_CTLS_ONE_SETTING,
 618                                VM_EXIT_CTLS_ZERO_SETTING,
 619                                &exit_ctls);
 620         if (error) {
 621                 printf("vmx_init: processor does not support desired "
 622                     "exit controls\n");
 623                 return (error);
 624         }
 625
 626         /* Check support for VM-entry controls */
 627         error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS, MSR_VMX_TRUE_ENTRY_CTLS,
 628             VM_ENTRY_CTLS_ONE_SETTING, VM_ENTRY_CTLS_ZERO_SETTING,
 629             &entry_ctls);
 630         if (error) {
 631                 printf("vmx_init: processor does not support desired "
 632                     "entry controls\n");
 633                 return (error);
 634         }
 635
 636         /*
 637          * Check support for optional features by testing them
 638          * as individual bits
 639          */
 640         cap_halt_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 641                                         MSR_VMX_TRUE_PROCBASED_CTLS,
 642                                         PROCBASED_HLT_EXITING, 0,
 643                                         &tmp) == 0);
 644
 645         cap_monitor_trap = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 646                                         MSR_VMX_PROCBASED_CTLS,
 647                                         PROCBASED_MTF, 0,
 648                                         &tmp) == 0);
 649
 650         cap_pause_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 651                                          MSR_VMX_TRUE_PROCBASED_CTLS,
 652                                          PROCBASED_PAUSE_EXITING, 0,
 653                                          &tmp) == 0);
 654
 655         cap_unrestricted_guest = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
 656                                         MSR_VMX_PROCBASED_CTLS2,
 657                                         PROCBASED2_UNRESTRICTED_GUEST, 0,
 658                                         &tmp) == 0);
 659
 660         cap_invpcid = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
 661             MSR_VMX_PROCBASED_CTLS2, PROCBASED2_ENABLE_INVPCID, 0,
 662             &tmp) == 0);
 663
 664         /*
 665          * Check support for virtual interrupt delivery.
 666          */
 667         procbased2_vid_bits = (PROCBASED2_VIRTUALIZE_APIC_ACCESSES |
 668             PROCBASED2_VIRTUALIZE_X2APIC_MODE |
 669             PROCBASED2_APIC_REGISTER_VIRTUALIZATION |
 670             PROCBASED2_VIRTUAL_INTERRUPT_DELIVERY);
 671
 672         use_tpr_shadow = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 673             MSR_VMX_TRUE_PROCBASED_CTLS, PROCBASED_USE_TPR_SHADOW, 0,
 674             &tmp) == 0);
 675
 676         error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
 677             procbased2_vid_bits, 0, &tmp);
 678         if (error == 0 && use_tpr_shadow) {
 679                 virtual_interrupt_delivery = 1;
 680                 TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_vid",
 681                     &virtual_interrupt_delivery);
 682         }
 683
 684         if (virtual_interrupt_delivery) {
 685                 procbased_ctls |= PROCBASED_USE_TPR_SHADOW;
 686                 procbased_ctls2 |= procbased2_vid_bits;
 687                 procbased_ctls2 &= ~PROCBASED2_VIRTUALIZE_X2APIC_MODE;
 688
 689                 /*
 690                  * No need to emulate accesses to %CR8 if virtual
 691                  * interrupt delivery is enabled.
 692                  */
 693                 procbased_ctls &= ~PROCBASED_CR8_LOAD_EXITING;
 694                 procbased_ctls &= ~PROCBASED_CR8_STORE_EXITING;
 695
 696                 /*
 697                  * Check for Posted Interrupts only if Virtual Interrupt
 698                  * Delivery is enabled.
 699                  */
 700                 error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
 701                     MSR_VMX_TRUE_PINBASED_CTLS, PINBASED_POSTED_INTERRUPT, 0,
 702                     &tmp);
 703                 if (error == 0) {
 704                         pirvec = lapic_ipi_alloc(pti ? &IDTVEC(justreturn1_pti) :
 705                             &IDTVEC(justreturn));
 706                         if (pirvec < 0) {
 707                                 if (bootverbose) {
 708                                         printf("vmx_init: unable to allocate "
 709                                             "posted interrupt vector\n");
 710                                 }
 711                         } else {
 712                                 posted_interrupts = 1;
 713                                 TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_pir",
 714                                     &posted_interrupts);
 715                         }
 716                 }
 717         }
 718
 719         if (posted_interrupts)
 720                     pinbased_ctls |= PINBASED_POSTED_INTERRUPT;
 721
 722         /* Initialize EPT */
 723         error = ept_init(ipinum);
 724         if (error) {
 725                 printf("vmx_init: ept initialization failed (%d)\n", error);
 726                 return (error);
 727         }
 728
 729         guest_l1d_flush = (cpu_ia32_arch_caps & IA32_ARCH_CAP_RDCL_NO) == 0;
 730         TUNABLE_INT_FETCH("hw.vmm.l1d_flush", &guest_l1d_flush);
 731         if (guest_l1d_flush &&
 732             (cpu_stdext_feature3 & CPUID_STDEXT3_L1D_FLUSH) != 0)
 733                 vmx_msr_flush_cmd = IA32_FLUSH_CMD_L1D;
 734
 735         /*
 736          * Stash the cr0 and cr4 bits that must be fixed to 0 or 1
 737          */
 738         fixed0 = rdmsr(MSR_VMX_CR0_FIXED0);
 739         fixed1 = rdmsr(MSR_VMX_CR0_FIXED1);
 740         cr0_ones_mask = fixed0 & fixed1;
 741         cr0_zeros_mask = ~fixed0 & ~fixed1;
 742
 743         /*
 744          * CR0_PE and CR0_PG can be set to zero in VMX non-root operation
 745          * if unrestricted guest execution is allowed.
 746          */
 747         if (cap_unrestricted_guest)
 748                 cr0_ones_mask &= ~(CR0_PG | CR0_PE);
 749
 750         /*
 751          * Do not allow the guest to set CR0_NW or CR0_CD.
 752          */
 753         cr0_zeros_mask |= (CR0_NW | CR0_CD);
 754
 755         fixed0 = rdmsr(MSR_VMX_CR4_FIXED0);
 756         fixed1 = rdmsr(MSR_VMX_CR4_FIXED1);
 757         cr4_ones_mask = fixed0 & fixed1;
 758         cr4_zeros_mask = ~fixed0 & ~fixed1;
 759
 760         vpid_init();
 761
 762         vmx_msr_init();
 763
 764         /* enable VMX operation */
 765         smp_rendezvous(NULL, vmx_enable, NULL, NULL);
 766
 767         vmx_initialized = 1;
 768
 769         return (0);
 770 }
 771
 772 static void
 773 vmx_trigger_hostintr(int vector)
 774 {
 775         uintptr_t func;
 776         struct gate_descriptor *gd;
 777
 778         gd = &idt[vector];
 779
 780         KASSERT(vector >= 32 && vector <= 255, ("vmx_trigger_hostintr: "
 781             "invalid vector %d", vector));
 782         KASSERT(gd->gd_p == 1, ("gate descriptor for vector %d not present",
 783             vector));
 784         KASSERT(gd->gd_type == SDT_SYSIGT, ("gate descriptor for vector %d "
 785             "has invalid type %d", vector, gd->gd_type));
 786         KASSERT(gd->gd_dpl == SEL_KPL, ("gate descriptor for vector %d "
 787             "has invalid dpl %d", vector, gd->gd_dpl));
 788         KASSERT(gd->gd_selector == GSEL(GCODE_SEL, SEL_KPL), ("gate descriptor "
 789             "for vector %d has invalid selector %d", vector, gd->gd_selector));
 790         KASSERT(gd->gd_ist == 0, ("gate descriptor for vector %d has invalid "
 791             "IST %d", vector, gd->gd_ist));
 792
 793         func = ((long)gd->gd_hioffset << 16 | gd->gd_looffset);
 794         vmx_call_isr(func);
 795 }
 796
 797 static int
 798 vmx_setup_cr_shadow(int which, struct vmcs *vmcs, uint32_t initial)
 799 {
 800         int error, mask_ident, shadow_ident;
 801         uint64_t mask_value;
 802
 803         if (which != 0 && which != 4)
 804                 panic("vmx_setup_cr_shadow: unknown cr%d", which);
 805
 806         if (which == 0) {
 807                 mask_ident = VMCS_CR0_MASK;
 808                 mask_value = cr0_ones_mask | cr0_zeros_mask;
 809                 shadow_ident = VMCS_CR0_SHADOW;
 810         } else {
 811                 mask_ident = VMCS_CR4_MASK;
 812                 mask_value = cr4_ones_mask | cr4_zeros_mask;
 813                 shadow_ident = VMCS_CR4_SHADOW;
 814         }
 815
 816         error = vmcs_setreg(vmcs, 0, VMCS_IDENT(mask_ident), mask_value);
 817         if (error)
 818                 return (error);
 819
 820         error = vmcs_setreg(vmcs, 0, VMCS_IDENT(shadow_ident), initial);
 821         if (error)
 822                 return (error);
 823
 824         return (0);
 825 }
 826 #define vmx_setup_cr0_shadow(vmcs,init) vmx_setup_cr_shadow(0, (vmcs), (init))
 827 #define vmx_setup_cr4_shadow(vmcs,init) vmx_setup_cr_shadow(4, (vmcs), (init))
 828
 829 static void *
 830 vmx_vminit(struct vm *vm, pmap_t pmap)
 831 {
 832         uint16_t vpid[VM_MAXCPU];
 833         int i, error;
 834         struct vmx *vmx;
 835         struct vmcs *vmcs;
 836         uint32_t exc_bitmap;
 837
 838         vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO);
 839         if ((uintptr_t)vmx & PAGE_MASK) {
 840                 panic("malloc of struct vmx not aligned on %d byte boundary",
 841                       PAGE_SIZE);
 842         }
 843         vmx->vm = vm;
 844
 845         vmx->eptp = eptp(vtophys((vm_offset_t)pmap->pm_pml4));
 846
 847         /*
 848          * Clean up EPTP-tagged guest physical and combined mappings
 849          *
 850          * VMX transitions are not required to invalidate any guest physical
 851          * mappings. So, it may be possible for stale guest physical mappings
 852          * to be present in the processor TLBs.
 853          *
 854          * Combined mappings for this EP4TA are also invalidated for all VPIDs.
 855          */
 856         ept_invalidate_mappings(vmx->eptp);
 857
 858         msr_bitmap_initialize(vmx->msr_bitmap);
 859
 860         /*
 861          * It is safe to allow direct access to MSR_GSBASE and MSR_FSBASE.
 862          * The guest FSBASE and GSBASE are saved and restored during
 863          * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are
 864          * always restored from the vmcs host state area on vm-exit.
 865          *
 866          * The SYSENTER_CS/ESP/EIP MSRs are identical to FS/GSBASE in
 867          * how they are saved/restored so can be directly accessed by the
 868          * guest.
 869          *
 870          * MSR_EFER is saved and restored in the guest VMCS area on a
 871          * VM exit and entry respectively. It is also restored from the
 872          * host VMCS area on a VM exit.
 873          *
 874          * The TSC MSR is exposed read-only. Writes are disallowed as
 875          * that will impact the host TSC.  If the guest does a write
 876          * the "use TSC offsetting" execution control is enabled and the
 877          * difference between the host TSC and the guest TSC is written
 878          * into the TSC offset in the VMCS.
 879          */
 880         if (guest_msr_rw(vmx, MSR_GSBASE) ||
 881             guest_msr_rw(vmx, MSR_FSBASE) ||
 882             guest_msr_rw(vmx, MSR_SYSENTER_CS_MSR) ||
 883             guest_msr_rw(vmx, MSR_SYSENTER_ESP_MSR) ||
 884             guest_msr_rw(vmx, MSR_SYSENTER_EIP_MSR) ||
 885             guest_msr_rw(vmx, MSR_EFER) ||
 886             guest_msr_ro(vmx, MSR_TSC))
 887                 panic("vmx_vminit: error setting guest msr access");
 888
 889         vpid_alloc(vpid, VM_MAXCPU);
 890
 891         if (virtual_interrupt_delivery) {
 892                 error = vm_map_mmio(vm, DEFAULT_APIC_BASE, PAGE_SIZE,
 893                     APIC_ACCESS_ADDRESS);
 894                 /* XXX this should really return an error to the caller */
 895                 KASSERT(error == 0, ("vm_map_mmio(apicbase) error %d", error));
 896         }
 897
 898         for (i = 0; i < VM_MAXCPU; i++) {
 899                 vmcs = &vmx->vmcs[i];
 900                 vmcs->identifier = vmx_revision();
 901                 error = vmclear(vmcs);
 902                 if (error != 0) {
 903                         panic("vmx_vminit: vmclear error %d on vcpu %d\n",
 904                               error, i);
 905                 }
 906
 907                 vmx_msr_guest_init(vmx, i);
 908
 909                 error = vmcs_init(vmcs);
 910                 KASSERT(error == 0, ("vmcs_init error %d", error));
 911
 912                 VMPTRLD(vmcs);
 913                 error = 0;
 914                 error += vmwrite(VMCS_HOST_RSP, (u_long)&vmx->ctx[i]);
 915                 error += vmwrite(VMCS_EPTP, vmx->eptp);
 916                 error += vmwrite(VMCS_PIN_BASED_CTLS, pinbased_ctls);
 917                 error += vmwrite(VMCS_PRI_PROC_BASED_CTLS, procbased_ctls);
 918                 error += vmwrite(VMCS_SEC_PROC_BASED_CTLS, procbased_ctls2);
 919                 error += vmwrite(VMCS_EXIT_CTLS, exit_ctls);
 920                 error += vmwrite(VMCS_ENTRY_CTLS, entry_ctls);
 921                 error += vmwrite(VMCS_MSR_BITMAP, vtophys(vmx->msr_bitmap));
 922                 error += vmwrite(VMCS_VPID, vpid[i]);
 923
 924                 /* exception bitmap */
 925                 if (vcpu_trace_exceptions(vm, i))
 926                         exc_bitmap = 0xffffffff;
 927                 else
 928                         exc_bitmap = 1 << IDT_MC;
 929                 error += vmwrite(VMCS_EXCEPTION_BITMAP, exc_bitmap);
 930
 931                 vmx->ctx[i].guest_dr6 = 0xffff0ff0;
 932                 error += vmwrite(VMCS_GUEST_DR7, 0x400);
 933
 934                 if (virtual_interrupt_delivery) {
 935                         error += vmwrite(VMCS_APIC_ACCESS, APIC_ACCESS_ADDRESS);
 936                         error += vmwrite(VMCS_VIRTUAL_APIC,
 937                             vtophys(&vmx->apic_page[i]));
 938                         error += vmwrite(VMCS_EOI_EXIT0, 0);
 939                         error += vmwrite(VMCS_EOI_EXIT1, 0);
 940                         error += vmwrite(VMCS_EOI_EXIT2, 0);
 941                         error += vmwrite(VMCS_EOI_EXIT3, 0);
 942                 }
 943                 if (posted_interrupts) {
 944                         error += vmwrite(VMCS_PIR_VECTOR, pirvec);
 945                         error += vmwrite(VMCS_PIR_DESC,
 946                             vtophys(&vmx->pir_desc[i]));
 947                 }
 948                 VMCLEAR(vmcs);
 949                 KASSERT(error == 0, ("vmx_vminit: error customizing the vmcs"));
 950
 951                 vmx->cap[i].set = 0;
 952                 vmx->cap[i].proc_ctls = procbased_ctls;
 953                 vmx->cap[i].proc_ctls2 = procbased_ctls2;
 954
 955                 vmx->state[i].nextrip = ~0;
 956                 vmx->state[i].lastcpu = NOCPU;
 957                 vmx->state[i].vpid = vpid[i];
 958
 959                 /*
 960                  * Set up the CR0/4 shadows, and init the read shadow
 961                  * to the power-on register value from the Intel Sys Arch.
 962                  *  CR0 - 0x60000010
 963                  *  CR4 - 0
 964                  */
 965                 error = vmx_setup_cr0_shadow(vmcs, 0x60000010);
 966                 if (error != 0)
 967                         panic("vmx_setup_cr0_shadow %d", error);
 968
 969                 error = vmx_setup_cr4_shadow(vmcs, 0);
 970                 if (error != 0)
 971                         panic("vmx_setup_cr4_shadow %d", error);
 972
 973                 vmx->ctx[i].pmap = pmap;
 974         }
 975
 976         return (vmx);
 977 }
 978
 979 static int
 980 vmx_handle_cpuid(struct vm *vm, int vcpu, struct vmxctx *vmxctx)
 981 {
 982         int handled, func;
 983
 984         func = vmxctx->guest_rax;
 985
 986         handled = x86_emulate_cpuid(vm, vcpu,
 987                                     (uint32_t*)(&vmxctx->guest_rax),
 988                                     (uint32_t*)(&vmxctx->guest_rbx),
 989                                     (uint32_t*)(&vmxctx->guest_rcx),
 990                                     (uint32_t*)(&vmxctx->guest_rdx));
 991         return (handled);
 992 }
 993
 994 static __inline void
 995 vmx_run_trace(struct vmx *vmx, int vcpu)
 996 {
 997 #ifdef KTR
 998         VCPU_CTR1(vmx->vm, vcpu, "Resume execution at %#lx", vmcs_guest_rip());
 999 #endif
1000 }
1001
1002 static __inline void
1003 vmx_exit_trace(struct vmx *vmx, int vcpu, uint64_t rip, uint32_t exit_reason,
1004                int handled)
1005 {
1006 #ifdef KTR
1007         VCPU_CTR3(vmx->vm, vcpu, "%s %s vmexit at 0x%0lx",
1008                  handled ? "handled" : "unhandled",
1009                  exit_reason_to_str(exit_reason), rip);
1010 #endif
1011 }
1012
1013 static __inline void
1014 vmx_astpending_trace(struct vmx *vmx, int vcpu, uint64_t rip)
1015 {
1016 #ifdef KTR
1017         VCPU_CTR1(vmx->vm, vcpu, "astpending vmexit at 0x%0lx", rip);
1018 #endif
1019 }
1020
1021 static VMM_STAT_INTEL(VCPU_INVVPID_SAVED, "Number of vpid invalidations saved");
1022 static VMM_STAT_INTEL(VCPU_INVVPID_DONE, "Number of vpid invalidations done");
1023
1024 /*
1025  * Invalidate guest mappings identified by its vpid from the TLB.
1026  */
1027 static __inline void
1028 vmx_invvpid(struct vmx *vmx, int vcpu, pmap_t pmap, int running)
1029 {
1030         struct vmxstate *vmxstate;
1031         struct invvpid_desc invvpid_desc;
1032
1033         vmxstate = &vmx->state[vcpu];
1034         if (vmxstate->vpid == 0)
1035                 return;
1036
1037         if (!running) {
1038                 /*
1039                  * Set the 'lastcpu' to an invalid host cpu.
1040                  *
1041                  * This will invalidate TLB entries tagged with the vcpu's
1042                  * vpid the next time it runs via vmx_set_pcpu_defaults().
1043                  */
1044                 vmxstate->lastcpu = NOCPU;
1045                 return;
1046         }
1047
1048         KASSERT(curthread->td_critnest > 0, ("%s: vcpu %d running outside "
1049             "critical section", __func__, vcpu));
1050
1051         /*
1052          * Invalidate all mappings tagged with 'vpid'
1053          *
1054          * We do this because this vcpu was executing on a different host
1055          * cpu when it last ran. We do not track whether it invalidated
1056          * mappings associated with its 'vpid' during that run. So we must
1057          * assume that the mappings associated with 'vpid' on 'curcpu' are
1058          * stale and invalidate them.
1059          *
1060          * Note that we incur this penalty only when the scheduler chooses to
1061          * move the thread associated with this vcpu between host cpus.
1062          *
1063          * Note also that this will invalidate mappings tagged with 'vpid'
1064          * for "all" EP4TAs.
1065          */
1066         if (pmap->pm_eptgen == vmx->eptgen[curcpu]) {
1067                 invvpid_desc._res1 = 0;
1068                 invvpid_desc._res2 = 0;
1069                 invvpid_desc.vpid = vmxstate->vpid;
1070                 invvpid_desc.linear_addr = 0;
1071                 invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc);
1072                 vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_DONE, 1);
1073         } else {
1074                 /*
1075                  * The invvpid can be skipped if an invept is going to
1076                  * be performed before entering the guest. The invept
1077                  * will invalidate combined mappings tagged with
1078                  * 'vmx->eptp' for all vpids.
1079                  */
1080                 vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_SAVED, 1);
1081         }
1082 }
1083
1084 static void
1085 vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu, pmap_t pmap)
1086 {
1087         struct vmxstate *vmxstate;
1088
1089         vmxstate = &vmx->state[vcpu];
1090         if (vmxstate->lastcpu == curcpu)
1091                 return;
1092
1093         vmxstate->lastcpu = curcpu;
1094
1095         vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1);
1096
1097         vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase());
1098         vmcs_write(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase());
1099         vmcs_write(VMCS_HOST_GS_BASE, vmm_get_host_gsbase());
1100         vmx_invvpid(vmx, vcpu, pmap, 1);
1101 }
1102
1103 /*
1104  * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set.
1105  */
1106 CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0);
1107
1108 static void __inline
1109 vmx_set_int_window_exiting(struct vmx *vmx, int vcpu)
1110 {
1111
1112         if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) == 0) {
1113                 vmx->cap[vcpu].proc_ctls |= PROCBASED_INT_WINDOW_EXITING;
1114                 vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1115                 VCPU_CTR0(vmx->vm, vcpu, "Enabling interrupt window exiting");
1116         }
1117 }
1118
1119 static void __inline
1120 vmx_clear_int_window_exiting(struct vmx *vmx, int vcpu)
1121 {
1122
1123         KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0,
1124             ("intr_window_exiting not set: %#x", vmx->cap[vcpu].proc_ctls));
1125         vmx->cap[vcpu].proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING;
1126         vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1127         VCPU_CTR0(vmx->vm, vcpu, "Disabling interrupt window exiting");
1128 }
1129
1130 static void __inline
1131 vmx_set_nmi_window_exiting(struct vmx *vmx, int vcpu)
1132 {
1133
1134         if ((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) == 0) {
1135                 vmx->cap[vcpu].proc_ctls |= PROCBASED_NMI_WINDOW_EXITING;
1136                 vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1137                 VCPU_CTR0(vmx->vm, vcpu, "Enabling NMI window exiting");
1138         }
1139 }
1140
1141 static void __inline
1142 vmx_clear_nmi_window_exiting(struct vmx *vmx, int vcpu)
1143 {
1144
1145         KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) != 0,
1146             ("nmi_window_exiting not set %#x", vmx->cap[vcpu].proc_ctls));
1147         vmx->cap[vcpu].proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING;
1148         vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1149         VCPU_CTR0(vmx->vm, vcpu, "Disabling NMI window exiting");
1150 }
1151
1152 int
1153 vmx_set_tsc_offset(struct vmx *vmx, int vcpu, uint64_t offset)
1154 {
1155         int error;
1156
1157         if ((vmx->cap[vcpu].proc_ctls & PROCBASED_TSC_OFFSET) == 0) {
1158                 vmx->cap[vcpu].proc_ctls |= PROCBASED_TSC_OFFSET;
1159                 vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1160                 VCPU_CTR0(vmx->vm, vcpu, "Enabling TSC offsetting");
1161         }
1162
1163         error = vmwrite(VMCS_TSC_OFFSET, offset);
1164
1165         return (error);
1166 }
1167
1168 #define NMI_BLOCKING    (VMCS_INTERRUPTIBILITY_NMI_BLOCKING |           \
1169                          VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
1170 #define HWINTR_BLOCKING (VMCS_INTERRUPTIBILITY_STI_BLOCKING |           \
1171                          VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
1172
1173 static void
1174 vmx_inject_nmi(struct vmx *vmx, int vcpu)
1175 {
1176         uint32_t gi, info;
1177
1178         gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1179         KASSERT((gi & NMI_BLOCKING) == 0, ("vmx_inject_nmi: invalid guest "
1180             "interruptibility-state %#x", gi));
1181
1182         info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1183         KASSERT((info & VMCS_INTR_VALID) == 0, ("vmx_inject_nmi: invalid "
1184             "VM-entry interruption information %#x", info));
1185
1186         /*
1187          * Inject the virtual NMI. The vector must be the NMI IDT entry
1188          * or the VMCS entry check will fail.
1189          */
1190         info = IDT_NMI | VMCS_INTR_T_NMI | VMCS_INTR_VALID;
1191         vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1192
1193         VCPU_CTR0(vmx->vm, vcpu, "Injecting vNMI");
1194
1195         /* Clear the request */
1196         vm_nmi_clear(vmx->vm, vcpu);
1197 }
1198
1199 static void
1200 vmx_inject_interrupts(struct vmx *vmx, int vcpu, struct vlapic *vlapic,
1201     uint64_t guestrip)
1202 {
1203         int vector, need_nmi_exiting, extint_pending;
1204         uint64_t rflags, entryinfo;
1205         uint32_t gi, info;
1206
1207         if (vmx->state[vcpu].nextrip != guestrip) {
1208                 gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1209                 if (gi & HWINTR_BLOCKING) {
1210                         VCPU_CTR2(vmx->vm, vcpu, "Guest interrupt blocking "
1211                             "cleared due to rip change: %#lx/%#lx",
1212                             vmx->state[vcpu].nextrip, guestrip);
1213                         gi &= ~HWINTR_BLOCKING;
1214                         vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1215                 }
1216         }
1217
1218         if (vm_entry_intinfo(vmx->vm, vcpu, &entryinfo)) {
1219                 KASSERT((entryinfo & VMCS_INTR_VALID) != 0, ("%s: entry "
1220                     "intinfo is not valid: %#lx", __func__, entryinfo));
1221
1222                 info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1223                 KASSERT((info & VMCS_INTR_VALID) == 0, ("%s: cannot inject "
1224                      "pending exception: %#lx/%#x", __func__, entryinfo, info));
1225
1226                 info = entryinfo;
1227                 vector = info & 0xff;
1228                 if (vector == IDT_BP || vector == IDT_OF) {
1229                         /*
1230                          * VT-x requires #BP and #OF to be injected as software
1231                          * exceptions.
1232                          */
1233                         info &= ~VMCS_INTR_T_MASK;
1234                         info |= VMCS_INTR_T_SWEXCEPTION;
1235                 }
1236
1237                 if (info & VMCS_INTR_DEL_ERRCODE)
1238                         vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR, entryinfo >> 32);
1239
1240                 vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1241         }
1242
1243         if (vm_nmi_pending(vmx->vm, vcpu)) {
1244                 /*
1245                  * If there are no conditions blocking NMI injection then
1246                  * inject it directly here otherwise enable "NMI window
1247                  * exiting" to inject it as soon as we can.
1248                  *
1249                  * We also check for STI_BLOCKING because some implementations
1250                  * don't allow NMI injection in this case. If we are running
1251                  * on a processor that doesn't have this restriction it will
1252                  * immediately exit and the NMI will be injected in the
1253                  * "NMI window exiting" handler.
1254                  */
1255                 need_nmi_exiting = 1;
1256                 gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1257                 if ((gi & (HWINTR_BLOCKING | NMI_BLOCKING)) == 0) {
1258                         info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1259                         if ((info & VMCS_INTR_VALID) == 0) {
1260                                 vmx_inject_nmi(vmx, vcpu);
1261                                 need_nmi_exiting = 0;
1262                         } else {
1263                                 VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI "
1264                                     "due to VM-entry intr info %#x", info);
1265                         }
1266                 } else {
1267                         VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI due to "
1268                             "Guest Interruptibility-state %#x", gi);
1269                 }
1270
1271                 if (need_nmi_exiting)
1272                         vmx_set_nmi_window_exiting(vmx, vcpu);
1273         }
1274
1275         extint_pending = vm_extint_pending(vmx->vm, vcpu);
1276
1277         if (!extint_pending && virtual_interrupt_delivery) {
1278                 vmx_inject_pir(vlapic);
1279                 return;
1280         }
1281
1282         /*
1283          * If interrupt-window exiting is already in effect then don't bother
1284          * checking for pending interrupts. This is just an optimization and
1285          * not needed for correctness.
1286          */
1287         if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0) {
1288                 VCPU_CTR0(vmx->vm, vcpu, "Skip interrupt injection due to "
1289                     "pending int_window_exiting");
1290                 return;
1291         }
1292
1293         if (!extint_pending) {
1294                 /* Ask the local apic for a vector to inject */
1295                 if (!vlapic_pending_intr(vlapic, &vector))
1296                         return;
1297
1298                 /*
1299                  * From the Intel SDM, Volume 3, Section "Maskable
1300                  * Hardware Interrupts":
1301                  * - maskable interrupt vectors [16,255] can be delivered
1302                  *   through the local APIC.
1303                 */
1304                 KASSERT(vector >= 16 && vector <= 255,
1305                     ("invalid vector %d from local APIC", vector));
1306         } else {
1307                 /* Ask the legacy pic for a vector to inject */
1308                 vatpic_pending_intr(vmx->vm, &vector);
1309
1310                 /*
1311                  * From the Intel SDM, Volume 3, Section "Maskable
1312                  * Hardware Interrupts":
1313                  * - maskable interrupt vectors [0,255] can be delivered
1314                  *   through the INTR pin.
1315                  */
1316                 KASSERT(vector >= 0 && vector <= 255,
1317                     ("invalid vector %d from INTR", vector));
1318         }
1319
1320         /* Check RFLAGS.IF and the interruptibility state of the guest */
1321         rflags = vmcs_read(VMCS_GUEST_RFLAGS);
1322         if ((rflags & PSL_I) == 0) {
1323                 VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
1324                     "rflags %#lx", vector, rflags);
1325                 goto cantinject;
1326         }
1327
1328         gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1329         if (gi & HWINTR_BLOCKING) {
1330                 VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
1331                     "Guest Interruptibility-state %#x", vector, gi);
1332                 goto cantinject;
1333         }
1334
1335         info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1336         if (info & VMCS_INTR_VALID) {
1337                 /*
1338                  * This is expected and could happen for multiple reasons:
1339                  * - A vectoring VM-entry was aborted due to astpending
1340                  * - A VM-exit happened during event injection.
1341                  * - An exception was injected above.
1342                  * - An NMI was injected above or after "NMI window exiting"
1343                  */
1344                 VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
1345                     "VM-entry intr info %#x", vector, info);
1346                 goto cantinject;
1347         }
1348
1349         /* Inject the interrupt */
1350         info = VMCS_INTR_T_HWINTR | VMCS_INTR_VALID;
1351         info |= vector;
1352         vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1353
1354         if (!extint_pending) {
1355                 /* Update the Local APIC ISR */
1356                 vlapic_intr_accepted(vlapic, vector);
1357         } else {
1358                 vm_extint_clear(vmx->vm, vcpu);
1359                 vatpic_intr_accepted(vmx->vm, vector);
1360
1361                 /*
1362                  * After we accepted the current ExtINT the PIC may
1363                  * have posted another one.  If that is the case, set
1364                  * the Interrupt Window Exiting execution control so
1365                  * we can inject that one too.
1366                  *
1367                  * Also, interrupt window exiting allows us to inject any
1368                  * pending APIC vector that was preempted by the ExtINT
1369                  * as soon as possible. This applies both for the software
1370                  * emulated vlapic and the hardware assisted virtual APIC.
1371                  */
1372                 vmx_set_int_window_exiting(vmx, vcpu);
1373         }
1374
1375         VCPU_CTR1(vmx->vm, vcpu, "Injecting hwintr at vector %d", vector);
1376
1377         return;
1378
1379 cantinject:
1380         /*
1381          * Set the Interrupt Window Exiting execution control so we can inject
1382          * the interrupt as soon as blocking condition goes away.
1383          */
1384         vmx_set_int_window_exiting(vmx, vcpu);
1385 }
1386
1387 /*
1388  * If the Virtual NMIs execution control is '1' then the logical processor
1389  * tracks virtual-NMI blocking in the Guest Interruptibility-state field of
1390  * the VMCS. An IRET instruction in VMX non-root operation will remove any
1391  * virtual-NMI blocking.
1392  *
1393  * This unblocking occurs even if the IRET causes a fault. In this case the
1394  * hypervisor needs to restore virtual-NMI blocking before resuming the guest.
1395  */
1396 static void
1397 vmx_restore_nmi_blocking(struct vmx *vmx, int vcpuid)
1398 {
1399         uint32_t gi;
1400
1401         VCPU_CTR0(vmx->vm, vcpuid, "Restore Virtual-NMI blocking");
1402         gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1403         gi |= VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
1404         vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1405 }
1406
1407 static void
1408 vmx_clear_nmi_blocking(struct vmx *vmx, int vcpuid)
1409 {
1410         uint32_t gi;
1411
1412         VCPU_CTR0(vmx->vm, vcpuid, "Clear Virtual-NMI blocking");
1413         gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1414         gi &= ~VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
1415         vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1416 }
1417
1418 static void
1419 vmx_assert_nmi_blocking(struct vmx *vmx, int vcpuid)
1420 {
1421         uint32_t gi;
1422
1423         gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1424         KASSERT(gi & VMCS_INTERRUPTIBILITY_NMI_BLOCKING,
1425             ("NMI blocking is not in effect %#x", gi));
1426 }
1427
1428 static int
1429 vmx_emulate_xsetbv(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
1430 {
1431         struct vmxctx *vmxctx;
1432         uint64_t xcrval;
1433         const struct xsave_limits *limits;
1434
1435         vmxctx = &vmx->ctx[vcpu];
1436         limits = vmm_get_xsave_limits();
1437
1438         /*
1439          * Note that the processor raises a GP# fault on its own if
1440          * xsetbv is executed for CPL != 0, so we do not have to
1441          * emulate that fault here.
1442          */
1443
1444         /* Only xcr0 is supported. */
1445         if (vmxctx->guest_rcx != 0) {
1446                 vm_inject_gp(vmx->vm, vcpu);
1447                 return (HANDLED);
1448         }
1449
1450         /* We only handle xcr0 if both the host and guest have XSAVE enabled. */
1451         if (!limits->xsave_enabled || !(vmcs_read(VMCS_GUEST_CR4) & CR4_XSAVE)) {
1452                 vm_inject_ud(vmx->vm, vcpu);
1453                 return (HANDLED);
1454         }
1455
1456         xcrval = vmxctx->guest_rdx << 32 | (vmxctx->guest_rax & 0xffffffff);
1457         if ((xcrval & ~limits->xcr0_allowed) != 0) {
1458                 vm_inject_gp(vmx->vm, vcpu);
1459                 return (HANDLED);
1460         }
1461
1462         if (!(xcrval & XFEATURE_ENABLED_X87)) {
1463                 vm_inject_gp(vmx->vm, vcpu);
1464                 return (HANDLED);
1465         }
1466
1467         /* AVX (YMM_Hi128) requires SSE. */
1468         if (xcrval & XFEATURE_ENABLED_AVX &&
1469             (xcrval & XFEATURE_AVX) != XFEATURE_AVX) {
1470                 vm_inject_gp(vmx->vm, vcpu);
1471                 return (HANDLED);
1472         }
1473
1474         /*
1475          * AVX512 requires base AVX (YMM_Hi128) as well as OpMask,
1476          * ZMM_Hi256, and Hi16_ZMM.
1477          */
1478         if (xcrval & XFEATURE_AVX512 &&
1479             (xcrval & (XFEATURE_AVX512 | XFEATURE_AVX)) !=
1480             (XFEATURE_AVX512 | XFEATURE_AVX)) {
1481                 vm_inject_gp(vmx->vm, vcpu);
1482                 return (HANDLED);
1483         }
1484
1485         /*
1486          * Intel MPX requires both bound register state flags to be
1487          * set.
1488          */
1489         if (((xcrval & XFEATURE_ENABLED_BNDREGS) != 0) !=
1490             ((xcrval & XFEATURE_ENABLED_BNDCSR) != 0)) {
1491                 vm_inject_gp(vmx->vm, vcpu);
1492                 return (HANDLED);
1493         }
1494
1495         /*
1496          * This runs "inside" vmrun() with the guest's FPU state, so
1497          * modifying xcr0 directly modifies the guest's xcr0, not the
1498          * host's.
1499          */
1500         load_xcr(0, xcrval);
1501         return (HANDLED);
1502 }
1503
1504 static uint64_t
1505 vmx_get_guest_reg(struct vmx *vmx, int vcpu, int ident)
1506 {
1507         const struct vmxctx *vmxctx;
1508
1509         vmxctx = &vmx->ctx[vcpu];
1510
1511         switch (ident) {
1512         case 0:
1513                 return (vmxctx->guest_rax);
1514         case 1:
1515                 return (vmxctx->guest_rcx);
1516         case 2:
1517                 return (vmxctx->guest_rdx);
1518         case 3:
1519                 return (vmxctx->guest_rbx);
1520         case 4:
1521                 return (vmcs_read(VMCS_GUEST_RSP));
1522         case 5:
1523                 return (vmxctx->guest_rbp);
1524         case 6:
1525                 return (vmxctx->guest_rsi);
1526         case 7:
1527                 return (vmxctx->guest_rdi);
1528         case 8:
1529                 return (vmxctx->guest_r8);
1530         case 9:
1531                 return (vmxctx->guest_r9);
1532         case 10:
1533                 return (vmxctx->guest_r10);
1534         case 11:
1535                 return (vmxctx->guest_r11);
1536         case 12:
1537                 return (vmxctx->guest_r12);
1538         case 13:
1539                 return (vmxctx->guest_r13);
1540         case 14:
1541                 return (vmxctx->guest_r14);
1542         case 15:
1543                 return (vmxctx->guest_r15);
1544         default:
1545                 panic("invalid vmx register %d", ident);
1546         }
1547 }
1548
1549 static void
1550 vmx_set_guest_reg(struct vmx *vmx, int vcpu, int ident, uint64_t regval)
1551 {
1552         struct vmxctx *vmxctx;
1553
1554         vmxctx = &vmx->ctx[vcpu];
1555
1556         switch (ident) {
1557         case 0:
1558                 vmxctx->guest_rax = regval;
1559                 break;
1560         case 1:
1561                 vmxctx->guest_rcx = regval;
1562                 break;
1563         case 2:
1564                 vmxctx->guest_rdx = regval;
1565                 break;
1566         case 3:
1567                 vmxctx->guest_rbx = regval;
1568                 break;
1569         case 4:
1570                 vmcs_write(VMCS_GUEST_RSP, regval);
1571                 break;
1572         case 5:
1573                 vmxctx->guest_rbp = regval;
1574                 break;
1575         case 6:
1576                 vmxctx->guest_rsi = regval;
1577                 break;
1578         case 7:
1579                 vmxctx->guest_rdi = regval;
1580                 break;
1581         case 8:
1582                 vmxctx->guest_r8 = regval;
1583                 break;
1584         case 9:
1585                 vmxctx->guest_r9 = regval;
1586                 break;
1587         case 10:
1588                 vmxctx->guest_r10 = regval;
1589                 break;
1590         case 11:
1591                 vmxctx->guest_r11 = regval;
1592                 break;
1593         case 12:
1594                 vmxctx->guest_r12 = regval;
1595                 break;
1596         case 13:
1597                 vmxctx->guest_r13 = regval;
1598                 break;
1599         case 14:
1600                 vmxctx->guest_r14 = regval;
1601                 break;
1602         case 15:
1603                 vmxctx->guest_r15 = regval;
1604                 break;
1605         default:
1606                 panic("invalid vmx register %d", ident);
1607         }
1608 }
1609
1610 static int
1611 vmx_emulate_cr0_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
1612 {
1613         uint64_t crval, regval;
1614
1615         /* We only handle mov to %cr0 at this time */
1616         if ((exitqual & 0xf0) != 0x00)
1617                 return (UNHANDLED);
1618
1619         regval = vmx_get_guest_reg(vmx, vcpu, (exitqual >> 8) & 0xf);
1620
1621         vmcs_write(VMCS_CR0_SHADOW, regval);
1622
1623         crval = regval | cr0_ones_mask;
1624         crval &= ~cr0_zeros_mask;
1625         vmcs_write(VMCS_GUEST_CR0, crval);
1626
1627         if (regval & CR0_PG) {
1628                 uint64_t efer, entry_ctls;
1629
1630                 /*
1631                  * If CR0.PG is 1 and EFER.LME is 1 then EFER.LMA and
1632                  * the "IA-32e mode guest" bit in VM-entry control must be
1633                  * equal.
1634                  */
1635                 efer = vmcs_read(VMCS_GUEST_IA32_EFER);
1636                 if (efer & EFER_LME) {
1637                         efer |= EFER_LMA;
1638                         vmcs_write(VMCS_GUEST_IA32_EFER, efer);
1639                         entry_ctls = vmcs_read(VMCS_ENTRY_CTLS);
1640                         entry_ctls |= VM_ENTRY_GUEST_LMA;
1641                         vmcs_write(VMCS_ENTRY_CTLS, entry_ctls);
1642                 }
1643         }
1644
1645         return (HANDLED);
1646 }
1647
1648 static int
1649 vmx_emulate_cr4_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
1650 {
1651         uint64_t crval, regval;
1652
1653         /* We only handle mov to %cr4 at this time */
1654         if ((exitqual & 0xf0) != 0x00)
1655                 return (UNHANDLED);
1656
1657         regval = vmx_get_guest_reg(vmx, vcpu, (exitqual >> 8) & 0xf);
1658
1659         vmcs_write(VMCS_CR4_SHADOW, regval);
1660
1661         crval = regval | cr4_ones_mask;
1662         crval &= ~cr4_zeros_mask;
1663         vmcs_write(VMCS_GUEST_CR4, crval);
1664
1665         return (HANDLED);
1666 }
1667
1668 static int
1669 vmx_emulate_cr8_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
1670 {
1671         struct vlapic *vlapic;
1672         uint64_t cr8;
1673         int regnum;
1674
1675         /* We only handle mov %cr8 to/from a register at this time. */
1676         if ((exitqual & 0xe0) != 0x00) {
1677                 return (UNHANDLED);
1678         }
1679
1680         vlapic = vm_lapic(vmx->vm, vcpu);
1681         regnum = (exitqual >> 8) & 0xf;
1682         if (exitqual & 0x10) {
1683                 cr8 = vlapic_get_cr8(vlapic);
1684                 vmx_set_guest_reg(vmx, vcpu, regnum, cr8);
1685         } else {
1686                 cr8 = vmx_get_guest_reg(vmx, vcpu, regnum);
1687                 vlapic_set_cr8(vlapic, cr8);
1688         }
1689
1690         return (HANDLED);
1691 }
1692
1693 /*
1694  * From section "Guest Register State" in the Intel SDM: CPL = SS.DPL
1695  */
1696 static int
1697 vmx_cpl(void)
1698 {
1699         uint32_t ssar;
1700
1701         ssar = vmcs_read(VMCS_GUEST_SS_ACCESS_RIGHTS);
1702         return ((ssar >> 5) & 0x3);
1703 }
1704
1705 static enum vm_cpu_mode
1706 vmx_cpu_mode(void)
1707 {
1708         uint32_t csar;
1709
1710         if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LMA) {
1711                 csar = vmcs_read(VMCS_GUEST_CS_ACCESS_RIGHTS);
1712                 if (csar & 0x2000)
1713                         return (CPU_MODE_64BIT);        /* CS.L = 1 */
1714                 else
1715                         return (CPU_MODE_COMPATIBILITY);
1716         } else if (vmcs_read(VMCS_GUEST_CR0) & CR0_PE) {
1717                 return (CPU_MODE_PROTECTED);
1718         } else {
1719                 return (CPU_MODE_REAL);
1720         }
1721 }
1722
1723 static enum vm_paging_mode
1724 vmx_paging_mode(void)
1725 {
1726
1727         if (!(vmcs_read(VMCS_GUEST_CR0) & CR0_PG))
1728                 return (PAGING_MODE_FLAT);
1729         if (!(vmcs_read(VMCS_GUEST_CR4) & CR4_PAE))
1730                 return (PAGING_MODE_32);
1731         if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LME)
1732                 return (PAGING_MODE_64);
1733         else
1734                 return (PAGING_MODE_PAE);
1735 }
1736
1737 static uint64_t
1738 inout_str_index(struct vmx *vmx, int vcpuid, int in)
1739 {
1740         uint64_t val;
1741         int error;
1742         enum vm_reg_name reg;
1743
1744         reg = in ? VM_REG_GUEST_RDI : VM_REG_GUEST_RSI;
1745         error = vmx_getreg(vmx, vcpuid, reg, &val);
1746         KASSERT(error == 0, ("%s: vmx_getreg error %d", __func__, error));
1747         return (val);
1748 }
1749
1750 static uint64_t
1751 inout_str_count(struct vmx *vmx, int vcpuid, int rep)
1752 {
1753         uint64_t val;
1754         int error;
1755
1756         if (rep) {
1757                 error = vmx_getreg(vmx, vcpuid, VM_REG_GUEST_RCX, &val);
1758                 KASSERT(!error, ("%s: vmx_getreg error %d", __func__, error));
1759         } else {
1760                 val = 1;
1761         }
1762         return (val);
1763 }
1764
1765 static int
1766 inout_str_addrsize(uint32_t inst_info)
1767 {
1768         uint32_t size;
1769
1770         size = (inst_info >> 7) & 0x7;
1771         switch (size) {
1772         case 0:
1773                 return (2);     /* 16 bit */
1774         case 1:
1775                 return (4);     /* 32 bit */
1776         case 2:
1777                 return (8);     /* 64 bit */
1778         default:
1779                 panic("%s: invalid size encoding %d", __func__, size);
1780         }
1781 }
1782
1783 static void
1784 inout_str_seginfo(struct vmx *vmx, int vcpuid, uint32_t inst_info, int in,
1785     struct vm_inout_str *vis)
1786 {
1787         int error, s;
1788
1789         if (in) {
1790                 vis->seg_name = VM_REG_GUEST_ES;
1791         } else {
1792                 s = (inst_info >> 15) & 0x7;
1793                 vis->seg_name = vm_segment_name(s);
1794         }
1795
1796         error = vmx_getdesc(vmx, vcpuid, vis->seg_name, &vis->seg_desc);
1797         KASSERT(error == 0, ("%s: vmx_getdesc error %d", __func__, error));
1798 }
1799
1800 static void
1801 vmx_paging_info(struct vm_guest_paging *paging)
1802 {
1803         paging->cr3 = vmcs_guest_cr3();
1804         paging->cpl = vmx_cpl();
1805         paging->cpu_mode = vmx_cpu_mode();
1806         paging->paging_mode = vmx_paging_mode();
1807 }
1808
1809 static void
1810 vmexit_inst_emul(struct vm_exit *vmexit, uint64_t gpa, uint64_t gla)
1811 {
1812         struct vm_guest_paging *paging;
1813         uint32_t csar;
1814
1815         paging = &vmexit->u.inst_emul.paging;
1816
1817         vmexit->exitcode = VM_EXITCODE_INST_EMUL;
1818         vmexit->inst_length = 0;
1819         vmexit->u.inst_emul.gpa = gpa;
1820         vmexit->u.inst_emul.gla = gla;
1821         vmx_paging_info(paging);
1822         switch (paging->cpu_mode) {
1823         case CPU_MODE_REAL:
1824                 vmexit->u.inst_emul.cs_base = vmcs_read(VMCS_GUEST_CS_BASE);
1825                 vmexit->u.inst_emul.cs_d = 0;
1826                 break;
1827         case CPU_MODE_PROTECTED:
1828         case CPU_MODE_COMPATIBILITY:
1829                 vmexit->u.inst_emul.cs_base = vmcs_read(VMCS_GUEST_CS_BASE);
1830                 csar = vmcs_read(VMCS_GUEST_CS_ACCESS_RIGHTS);
1831                 vmexit->u.inst_emul.cs_d = SEG_DESC_DEF32(csar);
1832                 break;
1833         default:
1834                 vmexit->u.inst_emul.cs_base = 0;
1835                 vmexit->u.inst_emul.cs_d = 0;
1836                 break;
1837         }
1838         vie_init(&vmexit->u.inst_emul.vie, NULL, 0);
1839 }
1840
1841 static int
1842 ept_fault_type(uint64_t ept_qual)
1843 {
1844         int fault_type;
1845
1846         if (ept_qual & EPT_VIOLATION_DATA_WRITE)
1847                 fault_type = VM_PROT_WRITE;
1848         else if (ept_qual & EPT_VIOLATION_INST_FETCH)
1849                 fault_type = VM_PROT_EXECUTE;
1850         else
1851                 fault_type= VM_PROT_READ;
1852
1853         return (fault_type);
1854 }
1855
1856 static boolean_t
1857 ept_emulation_fault(uint64_t ept_qual)
1858 {
1859         int read, write;
1860
1861         /* EPT fault on an instruction fetch doesn't make sense here */
1862         if (ept_qual & EPT_VIOLATION_INST_FETCH)
1863                 return (FALSE);
1864
1865         /* EPT fault must be a read fault or a write fault */
1866         read = ept_qual & EPT_VIOLATION_DATA_READ ? 1 : 0;
1867         write = ept_qual & EPT_VIOLATION_DATA_WRITE ? 1 : 0;
1868         if ((read | write) == 0)
1869                 return (FALSE);
1870
1871         /*
1872          * The EPT violation must have been caused by accessing a
1873          * guest-physical address that is a translation of a guest-linear
1874          * address.
1875          */
1876         if ((ept_qual & EPT_VIOLATION_GLA_VALID) == 0 ||
1877             (ept_qual & EPT_VIOLATION_XLAT_VALID) == 0) {
1878                 return (FALSE);
1879         }
1880
1881         return (TRUE);
1882 }
1883
1884 static __inline int
1885 apic_access_virtualization(struct vmx *vmx, int vcpuid)
1886 {
1887         uint32_t proc_ctls2;
1888
1889         proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
1890         return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) ? 1 : 0);
1891 }
1892
1893 static __inline int
1894 x2apic_virtualization(struct vmx *vmx, int vcpuid)
1895 {
1896         uint32_t proc_ctls2;
1897
1898         proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
1899         return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_X2APIC_MODE) ? 1 : 0);
1900 }
1901
1902 static int
1903 vmx_handle_apic_write(struct vmx *vmx, int vcpuid, struct vlapic *vlapic,
1904     uint64_t qual)
1905 {
1906         int error, handled, offset;
1907         uint32_t *apic_regs, vector;
1908         bool retu;
1909
1910         handled = HANDLED;
1911         offset = APIC_WRITE_OFFSET(qual);
1912
1913         if (!apic_access_virtualization(vmx, vcpuid)) {
1914                 /*
1915                  * In general there should not be any APIC write VM-exits
1916                  * unless APIC-access virtualization is enabled.
1917                  *
1918                  * However self-IPI virtualization can legitimately trigger
1919                  * an APIC-write VM-exit so treat it specially.
1920                  */
1921                 if (x2apic_virtualization(vmx, vcpuid) &&
1922                     offset == APIC_OFFSET_SELF_IPI) {
1923                         apic_regs = (uint32_t *)(vlapic->apic_page);
1924                         vector = apic_regs[APIC_OFFSET_SELF_IPI / 4];
1925                         vlapic_self_ipi_handler(vlapic, vector);
1926                         return (HANDLED);
1927                 } else
1928                         return (UNHANDLED);
1929         }
1930
1931         switch (offset) {
1932         case APIC_OFFSET_ID:
1933                 vlapic_id_write_handler(vlapic);
1934                 break;
1935         case APIC_OFFSET_LDR:
1936                 vlapic_ldr_write_handler(vlapic);
1937                 break;
1938         case APIC_OFFSET_DFR:
1939                 vlapic_dfr_write_handler(vlapic);
1940                 break;
1941         case APIC_OFFSET_SVR:
1942                 vlapic_svr_write_handler(vlapic);
1943                 break;
1944         case APIC_OFFSET_ESR:
1945                 vlapic_esr_write_handler(vlapic);
1946                 break;
1947         case APIC_OFFSET_ICR_LOW:
1948                 retu = false;
1949                 error = vlapic_icrlo_write_handler(vlapic, &retu);
1950                 if (error != 0 || retu)
1951                         handled = UNHANDLED;
1952                 break;
1953         case APIC_OFFSET_CMCI_LVT:
1954         case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
1955                 vlapic_lvt_write_handler(vlapic, offset);
1956                 break;
1957         case APIC_OFFSET_TIMER_ICR:
1958                 vlapic_icrtmr_write_handler(vlapic);
1959                 break;
1960         case APIC_OFFSET_TIMER_DCR:
1961                 vlapic_dcr_write_handler(vlapic);
1962                 break;
1963         default:
1964                 handled = UNHANDLED;
1965                 break;
1966         }
1967         return (handled);
1968 }
1969
1970 static bool
1971 apic_access_fault(struct vmx *vmx, int vcpuid, uint64_t gpa)
1972 {
1973
1974         if (apic_access_virtualization(vmx, vcpuid) &&
1975             (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE))
1976                 return (true);
1977         else
1978                 return (false);
1979 }
1980
1981 static int
1982 vmx_handle_apic_access(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit)
1983 {
1984         uint64_t qual;
1985         int access_type, offset, allowed;
1986
1987         if (!apic_access_virtualization(vmx, vcpuid))
1988                 return (UNHANDLED);
1989
1990         qual = vmexit->u.vmx.exit_qualification;
1991         access_type = APIC_ACCESS_TYPE(qual);
1992         offset = APIC_ACCESS_OFFSET(qual);
1993
1994         allowed = 0;
1995         if (access_type == 0) {
1996                 /*
1997                  * Read data access to the following registers is expected.
1998                  */
1999                 switch (offset) {
2000                 case APIC_OFFSET_APR:
2001                 case APIC_OFFSET_PPR:
2002                 case APIC_OFFSET_RRR:
2003                 case APIC_OFFSET_CMCI_LVT:
2004                 case APIC_OFFSET_TIMER_CCR:
2005                         allowed = 1;
2006                         break;
2007                 default:
2008                         break;
2009                 }
2010         } else if (access_type == 1) {
2011                 /*
2012                  * Write data access to the following registers is expected.
2013                  */
2014                 switch (offset) {
2015                 case APIC_OFFSET_VER:
2016                 case APIC_OFFSET_APR:
2017                 case APIC_OFFSET_PPR:
2018                 case APIC_OFFSET_RRR:
2019                 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
2020                 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
2021                 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
2022                 case APIC_OFFSET_CMCI_LVT:
2023                 case APIC_OFFSET_TIMER_CCR:
2024                         allowed = 1;
2025                         break;
2026                 default:
2027                         break;
2028                 }
2029         }
2030
2031         if (allowed) {
2032                 vmexit_inst_emul(vmexit, DEFAULT_APIC_BASE + offset,
2033                     VIE_INVALID_GLA);
2034         }
2035
2036         /*
2037          * Regardless of whether the APIC-access is allowed this handler
2038          * always returns UNHANDLED:
2039          * - if the access is allowed then it is handled by emulating the
2040          *   instruction that caused the VM-exit (outside the critical section)
2041          * - if the access is not allowed then it will be converted to an
2042          *   exitcode of VM_EXITCODE_VMX and will be dealt with in userland.
2043          */
2044         return (UNHANDLED);
2045 }
2046
2047 static enum task_switch_reason
2048 vmx_task_switch_reason(uint64_t qual)
2049 {
2050         int reason;
2051
2052         reason = (qual >> 30) & 0x3;
2053         switch (reason) {
2054         case 0:
2055                 return (TSR_CALL);
2056         case 1:
2057                 return (TSR_IRET);
2058         case 2:
2059                 return (TSR_JMP);
2060         case 3:
2061                 return (TSR_IDT_GATE);
2062         default:
2063                 panic("%s: invalid reason %d", __func__, reason);
2064         }
2065 }
2066
2067 static int
2068 emulate_wrmsr(struct vmx *vmx, int vcpuid, u_int num, uint64_t val, bool *retu)
2069 {
2070         int error;
2071
2072         if (lapic_msr(num))
2073                 error = lapic_wrmsr(vmx->vm, vcpuid, num, val, retu);
2074         else
2075                 error = vmx_wrmsr(vmx, vcpuid, num, val, retu);
2076
2077         return (error);
2078 }
2079
2080 static int
2081 emulate_rdmsr(struct vmx *vmx, int vcpuid, u_int num, bool *retu)
2082 {
2083         struct vmxctx *vmxctx;
2084         uint64_t result;
2085         uint32_t eax, edx;
2086         int error;
2087
2088         if (lapic_msr(num))
2089                 error = lapic_rdmsr(vmx->vm, vcpuid, num, &result, retu);
2090         else
2091                 error = vmx_rdmsr(vmx, vcpuid, num, &result, retu);
2092
2093         if (error == 0) {
2094                 eax = result;
2095                 vmxctx = &vmx->ctx[vcpuid];
2096                 error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RAX, eax);
2097                 KASSERT(error == 0, ("vmxctx_setreg(rax) error %d", error));
2098
2099                 edx = result >> 32;
2100                 error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RDX, edx);
2101                 KASSERT(error == 0, ("vmxctx_setreg(rdx) error %d", error));
2102         }
2103
2104         return (error);
2105 }
2106
2107 static int
2108 vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
2109 {
2110         int error, errcode, errcode_valid, handled, in;
2111         struct vmxctx *vmxctx;
2112         struct vlapic *vlapic;
2113         struct vm_inout_str *vis;
2114         struct vm_task_switch *ts;
2115         uint32_t eax, ecx, edx, idtvec_info, idtvec_err, intr_info, inst_info;
2116         uint32_t intr_type, intr_vec, reason;
2117         uint64_t exitintinfo, qual, gpa;
2118         bool retu;
2119
2120         CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_VIRTUAL_NMI) != 0);
2121         CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_NMI_EXITING) != 0);
2122
2123         handled = UNHANDLED;
2124         vmxctx = &vmx->ctx[vcpu];
2125
2126         qual = vmexit->u.vmx.exit_qualification;
2127         reason = vmexit->u.vmx.exit_reason;
2128         vmexit->exitcode = VM_EXITCODE_BOGUS;
2129
2130         vmm_stat_incr(vmx->vm, vcpu, VMEXIT_COUNT, 1);
2131
2132         /*
2133          * VM-entry failures during or after loading guest state.
2134          *
2135          * These VM-exits are uncommon but must be handled specially
2136          * as most VM-exit fields are not populated as usual.
2137          */
2138         if (__predict_false(reason == EXIT_REASON_MCE_DURING_ENTRY)) {
2139                 VCPU_CTR0(vmx->vm, vcpu, "Handling MCE during VM-entry");
2140                 __asm __volatile("int $18");
2141                 return (1);
2142         }
2143
2144         /*
2145          * VM exits that can be triggered during event delivery need to
2146          * be handled specially by re-injecting the event if the IDT
2147          * vectoring information field's valid bit is set.
2148          *
2149          * See "Information for VM Exits During Event Delivery" in Intel SDM
2150          * for details.
2151          */
2152         idtvec_info = vmcs_idt_vectoring_info();
2153         if (idtvec_info & VMCS_IDT_VEC_VALID) {
2154                 idtvec_info &= ~(1 << 12); /* clear undefined bit */
2155                 exitintinfo = idtvec_info;
2156                 if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) {
2157                         idtvec_err = vmcs_idt_vectoring_err();
2158                         exitintinfo |= (uint64_t)idtvec_err << 32;
2159                 }
2160                 error = vm_exit_intinfo(vmx->vm, vcpu, exitintinfo);
2161                 KASSERT(error == 0, ("%s: vm_set_intinfo error %d",
2162                     __func__, error));
2163
2164                 /*
2165                  * If 'virtual NMIs' are being used and the VM-exit
2166                  * happened while injecting an NMI during the previous
2167                  * VM-entry, then clear "blocking by NMI" in the
2168                  * Guest Interruptibility-State so the NMI can be
2169                  * reinjected on the subsequent VM-entry.
2170                  *
2171                  * However, if the NMI was being delivered through a task
2172                  * gate, then the new task must start execution with NMIs
2173                  * blocked so don't clear NMI blocking in this case.
2174                  */
2175                 intr_type = idtvec_info & VMCS_INTR_T_MASK;
2176                 if (intr_type == VMCS_INTR_T_NMI) {
2177                         if (reason != EXIT_REASON_TASK_SWITCH)
2178                                 vmx_clear_nmi_blocking(vmx, vcpu);
2179                         else
2180                                 vmx_assert_nmi_blocking(vmx, vcpu);
2181                 }
2182
2183                 /*
2184                  * Update VM-entry instruction length if the event being
2185                  * delivered was a software interrupt or software exception.
2186                  */
2187                 if (intr_type == VMCS_INTR_T_SWINTR ||
2188                     intr_type == VMCS_INTR_T_PRIV_SWEXCEPTION ||
2189                     intr_type == VMCS_INTR_T_SWEXCEPTION) {
2190                         vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length);
2191                 }
2192         }
2193
2194         switch (reason) {
2195         case EXIT_REASON_TASK_SWITCH:
2196                 ts = &vmexit->u.task_switch;
2197                 ts->tsssel = qual & 0xffff;
2198                 ts->reason = vmx_task_switch_reason(qual);
2199                 ts->ext = 0;
2200                 ts->errcode_valid = 0;
2201                 vmx_paging_info(&ts->paging);
2202                 /*
2203                  * If the task switch was due to a CALL, JMP, IRET, software
2204                  * interrupt (INT n) or software exception (INT3, INTO),
2205                  * then the saved %rip references the instruction that caused
2206                  * the task switch. The instruction length field in the VMCS
2207                  * is valid in this case.
2208                  *
2209                  * In all other cases (e.g., NMI, hardware exception) the
2210                  * saved %rip is one that would have been saved in the old TSS
2211                  * had the task switch completed normally so the instruction
2212                  * length field is not needed in this case and is explicitly
2213                  * set to 0.
2214                  */
2215                 if (ts->reason == TSR_IDT_GATE) {
2216                         KASSERT(idtvec_info & VMCS_IDT_VEC_VALID,
2217                             ("invalid idtvec_info %#x for IDT task switch",
2218                             idtvec_info));
2219                         intr_type = idtvec_info & VMCS_INTR_T_MASK;
2220                         if (intr_type != VMCS_INTR_T_SWINTR &&
2221                             intr_type != VMCS_INTR_T_SWEXCEPTION &&
2222                             intr_type != VMCS_INTR_T_PRIV_SWEXCEPTION) {
2223                                 /* Task switch triggered by external event */
2224                                 ts->ext = 1;
2225                                 vmexit->inst_length = 0;
2226                                 if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) {
2227                                         ts->errcode_valid = 1;
2228                                         ts->errcode = vmcs_idt_vectoring_err();
2229                                 }
2230                         }
2231                 }
2232                 vmexit->exitcode = VM_EXITCODE_TASK_SWITCH;
2233                 VCPU_CTR4(vmx->vm, vcpu, "task switch reason %d, tss 0x%04x, "
2234                     "%s errcode 0x%016lx", ts->reason, ts->tsssel,
2235                     ts->ext ? "external" : "internal",
2236                     ((uint64_t)ts->errcode << 32) | ts->errcode_valid);
2237                 break;
2238         case EXIT_REASON_CR_ACCESS:
2239                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CR_ACCESS, 1);
2240                 switch (qual & 0xf) {
2241                 case 0:
2242                         handled = vmx_emulate_cr0_access(vmx, vcpu, qual);
2243                         break;
2244                 case 4:
2245                         handled = vmx_emulate_cr4_access(vmx, vcpu, qual);
2246                         break;
2247                 case 8:
2248                         handled = vmx_emulate_cr8_access(vmx, vcpu, qual);
2249                         break;
2250                 }
2251                 break;
2252         case EXIT_REASON_RDMSR:
2253                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RDMSR, 1);
2254                 retu = false;
2255                 ecx = vmxctx->guest_rcx;
2256                 VCPU_CTR1(vmx->vm, vcpu, "rdmsr 0x%08x", ecx);
2257                 error = emulate_rdmsr(vmx, vcpu, ecx, &retu);
2258                 if (error) {
2259                         vmexit->exitcode = VM_EXITCODE_RDMSR;
2260                         vmexit->u.msr.code = ecx;
2261                 } else if (!retu) {
2262                         handled = HANDLED;
2263                 } else {
2264                         /* Return to userspace with a valid exitcode */
2265                         KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
2266                             ("emulate_rdmsr retu with bogus exitcode"));
2267                 }
2268                 break;
2269         case EXIT_REASON_WRMSR:
2270                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_WRMSR, 1);
2271                 retu = false;
2272                 eax = vmxctx->guest_rax;
2273                 ecx = vmxctx->guest_rcx;
2274                 edx = vmxctx->guest_rdx;
2275                 VCPU_CTR2(vmx->vm, vcpu, "wrmsr 0x%08x value 0x%016lx",
2276                     ecx, (uint64_t)edx << 32 | eax);
2277                 error = emulate_wrmsr(vmx, vcpu, ecx,
2278                     (uint64_t)edx << 32 | eax, &retu);
2279                 if (error) {
2280                         vmexit->exitcode = VM_EXITCODE_WRMSR;
2281                         vmexit->u.msr.code = ecx;
2282                         vmexit->u.msr.wval = (uint64_t)edx << 32 | eax;
2283                 } else if (!retu) {
2284                         handled = HANDLED;
2285                 } else {
2286                         /* Return to userspace with a valid exitcode */
2287                         KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
2288                             ("emulate_wrmsr retu with bogus exitcode"));
2289                 }
2290                 break;
2291         case EXIT_REASON_HLT:
2292                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_HLT, 1);
2293                 vmexit->exitcode = VM_EXITCODE_HLT;
2294                 vmexit->u.hlt.rflags = vmcs_read(VMCS_GUEST_RFLAGS);
2295                 break;
2296         case EXIT_REASON_MTF:
2297                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_MTRAP, 1);
2298                 vmexit->exitcode = VM_EXITCODE_MTRAP;
2299                 vmexit->inst_length = 0;
2300                 break;
2301         case EXIT_REASON_PAUSE:
2302                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_PAUSE, 1);
2303                 vmexit->exitcode = VM_EXITCODE_PAUSE;
2304                 break;
2305         case EXIT_REASON_INTR_WINDOW:
2306                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INTR_WINDOW, 1);
2307                 vmx_clear_int_window_exiting(vmx, vcpu);
2308                 return (1);
2309         case EXIT_REASON_EXT_INTR:
2310                 /*
2311                  * External interrupts serve only to cause VM exits and allow
2312                  * the host interrupt handler to run.
2313                  *
2314                  * If this external interrupt triggers a virtual interrupt
2315                  * to a VM, then that state will be recorded by the
2316                  * host interrupt handler in the VM's softc. We will inject
2317                  * this virtual interrupt during the subsequent VM enter.
2318                  */
2319                 intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
2320
2321                 /*
2322                  * XXX: Ignore this exit if VMCS_INTR_VALID is not set.
2323                  * This appears to be a bug in VMware Fusion?
2324                  */
2325                 if (!(intr_info & VMCS_INTR_VALID))
2326                         return (1);
2327                 KASSERT((intr_info & VMCS_INTR_VALID) != 0 &&
2328                     (intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_HWINTR,
2329                     ("VM exit interruption info invalid: %#x", intr_info));
2330                 vmx_trigger_hostintr(intr_info & 0xff);
2331
2332                 /*
2333                  * This is special. We want to treat this as an 'handled'
2334                  * VM-exit but not increment the instruction pointer.
2335                  */
2336                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXTINT, 1);
2337                 return (1);
2338         case EXIT_REASON_NMI_WINDOW:
2339                 /* Exit to allow the pending virtual NMI to be injected */
2340                 if (vm_nmi_pending(vmx->vm, vcpu))
2341                         vmx_inject_nmi(vmx, vcpu);
2342                 vmx_clear_nmi_window_exiting(vmx, vcpu);
2343                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NMI_WINDOW, 1);
2344                 return (1);
2345         case EXIT_REASON_INOUT:
2346                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INOUT, 1);
2347                 vmexit->exitcode = VM_EXITCODE_INOUT;
2348                 vmexit->u.inout.bytes = (qual & 0x7) + 1;
2349                 vmexit->u.inout.in = in = (qual & 0x8) ? 1 : 0;
2350                 vmexit->u.inout.string = (qual & 0x10) ? 1 : 0;
2351                 vmexit->u.inout.rep = (qual & 0x20) ? 1 : 0;
2352                 vmexit->u.inout.port = (uint16_t)(qual >> 16);
2353                 vmexit->u.inout.eax = (uint32_t)(vmxctx->guest_rax);
2354                 if (vmexit->u.inout.string) {
2355                         inst_info = vmcs_read(VMCS_EXIT_INSTRUCTION_INFO);
2356                         vmexit->exitcode = VM_EXITCODE_INOUT_STR;
2357                         vis = &vmexit->u.inout_str;
2358                         vmx_paging_info(&vis->paging);
2359                         vis->rflags = vmcs_read(VMCS_GUEST_RFLAGS);
2360                         vis->cr0 = vmcs_read(VMCS_GUEST_CR0);
2361                         vis->index = inout_str_index(vmx, vcpu, in);
2362                         vis->count = inout_str_count(vmx, vcpu, vis->inout.rep);
2363                         vis->addrsize = inout_str_addrsize(inst_info);
2364                         inout_str_seginfo(vmx, vcpu, inst_info, in, vis);
2365                 }
2366                 break;
2367         case EXIT_REASON_CPUID:
2368                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CPUID, 1);
2369                 handled = vmx_handle_cpuid(vmx->vm, vcpu, vmxctx);
2370                 break;
2371         case EXIT_REASON_EXCEPTION:
2372                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXCEPTION, 1);
2373                 intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
2374                 KASSERT((intr_info & VMCS_INTR_VALID) != 0,
2375                     ("VM exit interruption info invalid: %#x", intr_info));
2376
2377                 intr_vec = intr_info & 0xff;
2378                 intr_type = intr_info & VMCS_INTR_T_MASK;
2379
2380                 /*
2381                  * If Virtual NMIs control is 1 and the VM-exit is due to a
2382                  * fault encountered during the execution of IRET then we must
2383                  * restore the state of "virtual-NMI blocking" before resuming
2384                  * the guest.
2385                  *
2386                  * See "Resuming Guest Software after Handling an Exception".
2387                  * See "Information for VM Exits Due to Vectored Events".
2388                  */
2389                 if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
2390                     (intr_vec != IDT_DF) &&
2391                     (intr_info & EXIT_QUAL_NMIUDTI) != 0)
2392                         vmx_restore_nmi_blocking(vmx, vcpu);
2393
2394                 /*
2395                  * The NMI has already been handled in vmx_exit_handle_nmi().
2396                  */
2397                 if (intr_type == VMCS_INTR_T_NMI)
2398                         return (1);
2399
2400                 /*
2401                  * Call the machine check handler by hand. Also don't reflect
2402                  * the machine check back into the guest.
2403                  */
2404                 if (intr_vec == IDT_MC) {
2405                         VCPU_CTR0(vmx->vm, vcpu, "Vectoring to MCE handler");
2406                         __asm __volatile("int $18");
2407                         return (1);
2408                 }
2409
2410                 if (intr_vec == IDT_PF) {
2411                         error = vmxctx_setreg(vmxctx, VM_REG_GUEST_CR2, qual);
2412                         KASSERT(error == 0, ("%s: vmxctx_setreg(cr2) error %d",
2413                             __func__, error));
2414                 }
2415
2416                 /*
2417                  * Software exceptions exhibit trap-like behavior. This in
2418                  * turn requires populating the VM-entry instruction length
2419                  * so that the %rip in the trap frame is past the INT3/INTO
2420                  * instruction.
2421                  */
2422                 if (intr_type == VMCS_INTR_T_SWEXCEPTION)
2423                         vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length);
2424
2425                 /* Reflect all other exceptions back into the guest */
2426                 errcode_valid = errcode = 0;
2427                 if (intr_info & VMCS_INTR_DEL_ERRCODE) {
2428                         errcode_valid = 1;
2429                         errcode = vmcs_read(VMCS_EXIT_INTR_ERRCODE);
2430                 }
2431                 VCPU_CTR2(vmx->vm, vcpu, "Reflecting exception %d/%#x into "
2432                     "the guest", intr_vec, errcode);
2433                 error = vm_inject_exception(vmx->vm, vcpu, intr_vec,
2434                     errcode_valid, errcode, 0);
2435                 KASSERT(error == 0, ("%s: vm_inject_exception error %d",
2436                     __func__, error));
2437                 return (1);
2438
2439         case EXIT_REASON_EPT_FAULT:
2440                 /*
2441                  * If 'gpa' lies within the address space allocated to
2442                  * memory then this must be a nested page fault otherwise
2443                  * this must be an instruction that accesses MMIO space.
2444                  */
2445                 gpa = vmcs_gpa();
2446                 if (vm_mem_allocated(vmx->vm, vcpu, gpa) ||
2447                     apic_access_fault(vmx, vcpu, gpa)) {
2448                         vmexit->exitcode = VM_EXITCODE_PAGING;
2449                         vmexit->inst_length = 0;
2450                         vmexit->u.paging.gpa = gpa;
2451                         vmexit->u.paging.fault_type = ept_fault_type(qual);
2452                         vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NESTED_FAULT, 1);
2453                 } else if (ept_emulation_fault(qual)) {
2454                         vmexit_inst_emul(vmexit, gpa, vmcs_gla());
2455                         vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INST_EMUL, 1);
2456                 }
2457                 /*
2458                  * If Virtual NMIs control is 1 and the VM-exit is due to an
2459                  * EPT fault during the execution of IRET then we must restore
2460                  * the state of "virtual-NMI blocking" before resuming.
2461                  *
2462                  * See description of "NMI unblocking due to IRET" in
2463                  * "Exit Qualification for EPT Violations".
2464                  */
2465                 if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
2466                     (qual & EXIT_QUAL_NMIUDTI) != 0)
2467                         vmx_restore_nmi_blocking(vmx, vcpu);
2468                 break;
2469         case EXIT_REASON_VIRTUALIZED_EOI:
2470                 vmexit->exitcode = VM_EXITCODE_IOAPIC_EOI;
2471                 vmexit->u.ioapic_eoi.vector = qual & 0xFF;
2472                 vmexit->inst_length = 0;        /* trap-like */
2473                 break;
2474         case EXIT_REASON_APIC_ACCESS:
2475                 handled = vmx_handle_apic_access(vmx, vcpu, vmexit);
2476                 break;
2477         case EXIT_REASON_APIC_WRITE:
2478                 /*
2479                  * APIC-write VM exit is trap-like so the %rip is already
2480                  * pointing to the next instruction.
2481                  */
2482                 vmexit->inst_length = 0;
2483                 vlapic = vm_lapic(vmx->vm, vcpu);
2484                 handled = vmx_handle_apic_write(vmx, vcpu, vlapic, qual);
2485                 break;
2486         case EXIT_REASON_XSETBV:
2487                 handled = vmx_emulate_xsetbv(vmx, vcpu, vmexit);
2488                 break;
2489         case EXIT_REASON_MONITOR:
2490                 vmexit->exitcode = VM_EXITCODE_MONITOR;
2491                 break;
2492         case EXIT_REASON_MWAIT:
2493                 vmexit->exitcode = VM_EXITCODE_MWAIT;
2494                 break;
2495         default:
2496                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_UNKNOWN, 1);
2497                 break;
2498         }
2499
2500         if (handled) {
2501                 /*
2502                  * It is possible that control is returned to userland
2503                  * even though we were able to handle the VM exit in the
2504                  * kernel.
2505                  *
2506                  * In such a case we want to make sure that the userland
2507                  * restarts guest execution at the instruction *after*
2508                  * the one we just processed. Therefore we update the
2509                  * guest rip in the VMCS and in 'vmexit'.
2510                  */
2511                 vmexit->rip += vmexit->inst_length;
2512                 vmexit->inst_length = 0;
2513                 vmcs_write(VMCS_GUEST_RIP, vmexit->rip);
2514         } else {
2515                 if (vmexit->exitcode == VM_EXITCODE_BOGUS) {
2516                         /*
2517                          * If this VM exit was not claimed by anybody then
2518                          * treat it as a generic VMX exit.
2519                          */
2520                         vmexit->exitcode = VM_EXITCODE_VMX;
2521                         vmexit->u.vmx.status = VM_SUCCESS;
2522                         vmexit->u.vmx.inst_type = 0;
2523                         vmexit->u.vmx.inst_error = 0;
2524                 } else {
2525                         /*
2526                          * The exitcode and collateral have been populated.
2527                          * The VM exit will be processed further in userland.
2528                          */
2529                 }
2530         }
2531         return (handled);
2532 }
2533
2534 static __inline void
2535 vmx_exit_inst_error(struct vmxctx *vmxctx, int rc, struct vm_exit *vmexit)
2536 {
2537
2538         KASSERT(vmxctx->inst_fail_status != VM_SUCCESS,
2539             ("vmx_exit_inst_error: invalid inst_fail_status %d",
2540             vmxctx->inst_fail_status));
2541
2542         vmexit->inst_length = 0;
2543         vmexit->exitcode = VM_EXITCODE_VMX;
2544         vmexit->u.vmx.status = vmxctx->inst_fail_status;
2545         vmexit->u.vmx.inst_error = vmcs_instruction_error();
2546         vmexit->u.vmx.exit_reason = ~0;
2547         vmexit->u.vmx.exit_qualification = ~0;
2548
2549         switch (rc) {
2550         case VMX_VMRESUME_ERROR:
2551         case VMX_VMLAUNCH_ERROR:
2552         case VMX_INVEPT_ERROR:
2553                 vmexit->u.vmx.inst_type = rc;
2554                 break;
2555         default:
2556                 panic("vm_exit_inst_error: vmx_enter_guest returned %d", rc);
2557         }
2558 }
2559
2560 /*
2561  * If the NMI-exiting VM execution control is set to '1' then an NMI in
2562  * non-root operation causes a VM-exit. NMI blocking is in effect so it is
2563  * sufficient to simply vector to the NMI handler via a software interrupt.
2564  * However, this must be done before maskable interrupts are enabled
2565  * otherwise the "iret" issued by an interrupt handler will incorrectly
2566  * clear NMI blocking.
2567  */
2568 static __inline void
2569 vmx_exit_handle_nmi(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit)
2570 {
2571         uint32_t intr_info;
2572
2573         KASSERT((read_rflags() & PSL_I) == 0, ("interrupts enabled"));
2574
2575         if (vmexit->u.vmx.exit_reason != EXIT_REASON_EXCEPTION)
2576                 return;
2577
2578         intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
2579         KASSERT((intr_info & VMCS_INTR_VALID) != 0,
2580             ("VM exit interruption info invalid: %#x", intr_info));
2581
2582         if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI) {
2583                 KASSERT((intr_info & 0xff) == IDT_NMI, ("VM exit due "
2584                     "to NMI has invalid vector: %#x", intr_info));
2585                 VCPU_CTR0(vmx->vm, vcpuid, "Vectoring to NMI handler");
2586                 __asm __volatile("int $2");
2587         }
2588 }
2589
2590 static __inline void
2591 vmx_dr_enter_guest(struct vmxctx *vmxctx)
2592 {
2593         register_t rflags;
2594
2595         /* Save host control debug registers. */
2596         vmxctx->host_dr7 = rdr7();
2597         vmxctx->host_debugctl = rdmsr(MSR_DEBUGCTLMSR);
2598
2599         /*
2600          * Disable debugging in DR7 and DEBUGCTL to avoid triggering
2601          * exceptions in the host based on the guest DRx values.  The
2602          * guest DR7 and DEBUGCTL are saved/restored in the VMCS.
2603          */
2604         load_dr7(0);
2605         wrmsr(MSR_DEBUGCTLMSR, 0);
2606
2607         /*
2608          * Disable single stepping the kernel to avoid corrupting the
2609          * guest DR6.  A debugger might still be able to corrupt the
2610          * guest DR6 by setting a breakpoint after this point and then
2611          * single stepping.
2612          */
2613         rflags = read_rflags();
2614         vmxctx->host_tf = rflags & PSL_T;
2615         write_rflags(rflags & ~PSL_T);
2616
2617         /* Save host debug registers. */
2618         vmxctx->host_dr0 = rdr0();
2619         vmxctx->host_dr1 = rdr1();
2620         vmxctx->host_dr2 = rdr2();
2621         vmxctx->host_dr3 = rdr3();
2622         vmxctx->host_dr6 = rdr6();
2623
2624         /* Restore guest debug registers. */
2625         load_dr0(vmxctx->guest_dr0);
2626         load_dr1(vmxctx->guest_dr1);
2627         load_dr2(vmxctx->guest_dr2);
2628         load_dr3(vmxctx->guest_dr3);
2629         load_dr6(vmxctx->guest_dr6);
2630 }
2631
2632 static __inline void
2633 vmx_dr_leave_guest(struct vmxctx *vmxctx)
2634 {
2635
2636         /* Save guest debug registers. */
2637         vmxctx->guest_dr0 = rdr0();
2638         vmxctx->guest_dr1 = rdr1();
2639         vmxctx->guest_dr2 = rdr2();
2640         vmxctx->guest_dr3 = rdr3();
2641         vmxctx->guest_dr6 = rdr6();
2642
2643         /*
2644          * Restore host debug registers.  Restore DR7, DEBUGCTL, and
2645          * PSL_T last.
2646          */
2647         load_dr0(vmxctx->host_dr0);
2648         load_dr1(vmxctx->host_dr1);
2649         load_dr2(vmxctx->host_dr2);
2650         load_dr3(vmxctx->host_dr3);
2651         load_dr6(vmxctx->host_dr6);
2652         wrmsr(MSR_DEBUGCTLMSR, vmxctx->host_debugctl);
2653         load_dr7(vmxctx->host_dr7);
2654         write_rflags(read_rflags() | vmxctx->host_tf);
2655 }
2656
2657 static int
2658 vmx_run(void *arg, int vcpu, register_t rip, pmap_t pmap,
2659     struct vm_eventinfo *evinfo)
2660 {
2661         int rc, handled, launched;
2662         struct vmx *vmx;
2663         struct vm *vm;
2664         struct vmxctx *vmxctx;
2665         struct vmcs *vmcs;
2666         struct vm_exit *vmexit;
2667         struct vlapic *vlapic;
2668         uint32_t exit_reason;
2669
2670         vmx = arg;
2671         vm = vmx->vm;
2672         vmcs = &vmx->vmcs[vcpu];
2673         vmxctx = &vmx->ctx[vcpu];
2674         vlapic = vm_lapic(vm, vcpu);
2675         vmexit = vm_exitinfo(vm, vcpu);
2676         launched = 0;
2677
2678         KASSERT(vmxctx->pmap == pmap,
2679             ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap));
2680
2681         vmx_msr_guest_enter(vmx, vcpu);
2682
2683         VMPTRLD(vmcs);
2684
2685         /*
2686          * XXX
2687          * We do this every time because we may setup the virtual machine
2688          * from a different process than the one that actually runs it.
2689          *
2690          * If the life of a virtual machine was spent entirely in the context
2691          * of a single process we could do this once in vmx_vminit().
2692          */
2693         vmcs_write(VMCS_HOST_CR3, rcr3());
2694
2695         vmcs_write(VMCS_GUEST_RIP, rip);
2696         vmx_set_pcpu_defaults(vmx, vcpu, pmap);
2697         do {
2698                 KASSERT(vmcs_guest_rip() == rip, ("%s: vmcs guest rip mismatch "
2699                     "%#lx/%#lx", __func__, vmcs_guest_rip(), rip));
2700
2701                 handled = UNHANDLED;
2702                 /*
2703                  * Interrupts are disabled from this point on until the
2704                  * guest starts executing. This is done for the following
2705                  * reasons:
2706                  *
2707                  * If an AST is asserted on this thread after the check below,
2708                  * then the IPI_AST notification will not be lost, because it
2709                  * will cause a VM exit due to external interrupt as soon as
2710                  * the guest state is loaded.
2711                  *
2712                  * A posted interrupt after 'vmx_inject_interrupts()' will
2713                  * not be "lost" because it will be held pending in the host
2714                  * APIC because interrupts are disabled. The pending interrupt
2715                  * will be recognized as soon as the guest state is loaded.
2716                  *
2717                  * The same reasoning applies to the IPI generated by
2718                  * pmap_invalidate_ept().
2719                  */
2720                 disable_intr();
2721                 vmx_inject_interrupts(vmx, vcpu, vlapic, rip);
2722
2723                 /*
2724                  * Check for vcpu suspension after injecting events because
2725                  * vmx_inject_interrupts() can suspend the vcpu due to a
2726                  * triple fault.
2727                  */
2728                 if (vcpu_suspended(evinfo)) {
2729                         enable_intr();
2730                         vm_exit_suspended(vmx->vm, vcpu, rip);
2731                         break;
2732                 }
2733
2734                 if (vcpu_rendezvous_pending(evinfo)) {
2735                         enable_intr();
2736                         vm_exit_rendezvous(vmx->vm, vcpu, rip);
2737                         break;
2738                 }
2739
2740                 if (vcpu_reqidle(evinfo)) {
2741                         enable_intr();
2742                         vm_exit_reqidle(vmx->vm, vcpu, rip);
2743                         break;
2744                 }
2745
2746                 if (vcpu_should_yield(vm, vcpu)) {
2747                         enable_intr();
2748                         vm_exit_astpending(vmx->vm, vcpu, rip);
2749                         vmx_astpending_trace(vmx, vcpu, rip);
2750                         handled = HANDLED;
2751                         break;
2752                 }
2753
2754                 vmx_run_trace(vmx, vcpu);
2755                 vmx_dr_enter_guest(vmxctx);
2756                 rc = vmx_enter_guest(vmxctx, vmx, launched);
2757                 vmx_dr_leave_guest(vmxctx);
2758
2759                 /* Collect some information for VM exit processing */
2760                 vmexit->rip = rip = vmcs_guest_rip();
2761                 vmexit->inst_length = vmexit_instruction_length();
2762                 vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason();
2763                 vmexit->u.vmx.exit_qualification = vmcs_exit_qualification();
2764
2765                 /* Update 'nextrip' */
2766                 vmx->state[vcpu].nextrip = rip;
2767
2768                 if (rc == VMX_GUEST_VMEXIT) {
2769                         vmx_exit_handle_nmi(vmx, vcpu, vmexit);
2770                         enable_intr();
2771                         handled = vmx_exit_process(vmx, vcpu, vmexit);
2772                 } else {
2773                         enable_intr();
2774                         vmx_exit_inst_error(vmxctx, rc, vmexit);
2775                 }
2776                 launched = 1;
2777                 vmx_exit_trace(vmx, vcpu, rip, exit_reason, handled);
2778                 rip = vmexit->rip;
2779         } while (handled);
2780
2781         /*
2782          * If a VM exit has been handled then the exitcode must be BOGUS
2783          * If a VM exit is not handled then the exitcode must not be BOGUS
2784          */
2785         if ((handled && vmexit->exitcode != VM_EXITCODE_BOGUS) ||
2786             (!handled && vmexit->exitcode == VM_EXITCODE_BOGUS)) {
2787                 panic("Mismatch between handled (%d) and exitcode (%d)",
2788                       handled, vmexit->exitcode);
2789         }
2790
2791         if (!handled)
2792                 vmm_stat_incr(vm, vcpu, VMEXIT_USERSPACE, 1);
2793
2794         VCPU_CTR1(vm, vcpu, "returning from vmx_run: exitcode %d",
2795             vmexit->exitcode);
2796
2797         VMCLEAR(vmcs);
2798         vmx_msr_guest_exit(vmx, vcpu);
2799
2800         return (0);
2801 }
2802
2803 static void
2804 vmx_vmcleanup(void *arg)
2805 {
2806         int i;
2807         struct vmx *vmx = arg;
2808
2809         if (apic_access_virtualization(vmx, 0))
2810                 vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
2811
2812         for (i = 0; i < VM_MAXCPU; i++)
2813                 vpid_free(vmx->state[i].vpid);
2814
2815         free(vmx, M_VMX);
2816
2817         return;
2818 }
2819
2820 static register_t *
2821 vmxctx_regptr(struct vmxctx *vmxctx, int reg)
2822 {
2823
2824         switch (reg) {
2825         case VM_REG_GUEST_RAX:
2826                 return (&vmxctx->guest_rax);
2827         case VM_REG_GUEST_RBX:
2828                 return (&vmxctx->guest_rbx);
2829         case VM_REG_GUEST_RCX:
2830                 return (&vmxctx->guest_rcx);
2831         case VM_REG_GUEST_RDX:
2832                 return (&vmxctx->guest_rdx);
2833         case VM_REG_GUEST_RSI:
2834                 return (&vmxctx->guest_rsi);
2835         case VM_REG_GUEST_RDI:
2836                 return (&vmxctx->guest_rdi);
2837         case VM_REG_GUEST_RBP:
2838                 return (&vmxctx->guest_rbp);
2839         case VM_REG_GUEST_R8:
2840                 return (&vmxctx->guest_r8);
2841         case VM_REG_GUEST_R9:
2842                 return (&vmxctx->guest_r9);
2843         case VM_REG_GUEST_R10:
2844                 return (&vmxctx->guest_r10);
2845         case VM_REG_GUEST_R11:
2846                 return (&vmxctx->guest_r11);
2847         case VM_REG_GUEST_R12:
2848                 return (&vmxctx->guest_r12);
2849         case VM_REG_GUEST_R13:
2850                 return (&vmxctx->guest_r13);
2851         case VM_REG_GUEST_R14:
2852                 return (&vmxctx->guest_r14);
2853         case VM_REG_GUEST_R15:
2854                 return (&vmxctx->guest_r15);
2855         case VM_REG_GUEST_CR2:
2856                 return (&vmxctx->guest_cr2);
2857         case VM_REG_GUEST_DR0:
2858                 return (&vmxctx->guest_dr0);
2859         case VM_REG_GUEST_DR1:
2860                 return (&vmxctx->guest_dr1);
2861         case VM_REG_GUEST_DR2:
2862                 return (&vmxctx->guest_dr2);
2863         case VM_REG_GUEST_DR3:
2864                 return (&vmxctx->guest_dr3);
2865         case VM_REG_GUEST_DR6:
2866                 return (&vmxctx->guest_dr6);
2867         default:
2868                 break;
2869         }
2870         return (NULL);
2871 }
2872
2873 static int
2874 vmxctx_getreg(struct vmxctx *vmxctx, int reg, uint64_t *retval)
2875 {
2876         register_t *regp;
2877
2878         if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
2879                 *retval = *regp;
2880                 return (0);
2881         } else
2882                 return (EINVAL);
2883 }
2884
2885 static int
2886 vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val)
2887 {
2888         register_t *regp;
2889
2890         if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
2891                 *regp = val;
2892                 return (0);
2893         } else
2894                 return (EINVAL);
2895 }
2896
2897 static int
2898 vmx_get_intr_shadow(struct vmx *vmx, int vcpu, int running, uint64_t *retval)
2899 {
2900         uint64_t gi;
2901         int error;
2902
2903         error = vmcs_getreg(&vmx->vmcs[vcpu], running,
2904             VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY), &gi);
2905         *retval = (gi & HWINTR_BLOCKING) ? 1 : 0;
2906         return (error);
2907 }
2908
2909 static int
2910 vmx_modify_intr_shadow(struct vmx *vmx, int vcpu, int running, uint64_t val)
2911 {
2912         struct vmcs *vmcs;
2913         uint64_t gi;
2914         int error, ident;
2915
2916         /*
2917          * Forcing the vcpu into an interrupt shadow is not supported.
2918          */
2919         if (val) {
2920                 error = EINVAL;
2921                 goto done;
2922         }
2923
2924         vmcs = &vmx->vmcs[vcpu];
2925         ident = VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY);
2926         error = vmcs_getreg(vmcs, running, ident, &gi);
2927         if (error == 0) {
2928                 gi &= ~HWINTR_BLOCKING;
2929                 error = vmcs_setreg(vmcs, running, ident, gi);
2930         }
2931 done:
2932         VCPU_CTR2(vmx->vm, vcpu, "Setting intr_shadow to %#lx %s", val,
2933             error ? "failed" : "succeeded");
2934         return (error);
2935 }
2936
2937 static int
2938 vmx_shadow_reg(int reg)
2939 {
2940         int shreg;
2941
2942         shreg = -1;
2943
2944         switch (reg) {
2945         case VM_REG_GUEST_CR0:
2946                 shreg = VMCS_CR0_SHADOW;
2947                 break;
2948         case VM_REG_GUEST_CR4:
2949                 shreg = VMCS_CR4_SHADOW;
2950                 break;
2951         default:
2952                 break;
2953         }
2954
2955         return (shreg);
2956 }
2957
2958 static int
2959 vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval)
2960 {
2961         int running, hostcpu;
2962         struct vmx *vmx = arg;
2963
2964         running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
2965         if (running && hostcpu != curcpu)
2966                 panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), vcpu);
2967
2968         if (reg == VM_REG_GUEST_INTR_SHADOW)
2969                 return (vmx_get_intr_shadow(vmx, vcpu, running, retval));
2970
2971         if (vmxctx_getreg(&vmx->ctx[vcpu], reg, retval) == 0)
2972                 return (0);
2973
2974         return (vmcs_getreg(&vmx->vmcs[vcpu], running, reg, retval));
2975 }
2976
2977 static int
2978 vmx_setreg(void *arg, int vcpu, int reg, uint64_t val)
2979 {
2980         int error, hostcpu, running, shadow;
2981         uint64_t ctls;
2982         pmap_t pmap;
2983         struct vmx *vmx = arg;
2984
2985         running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
2986         if (running && hostcpu != curcpu)
2987                 panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), vcpu);
2988
2989         if (reg == VM_REG_GUEST_INTR_SHADOW)
2990                 return (vmx_modify_intr_shadow(vmx, vcpu, running, val));
2991
2992         if (vmxctx_setreg(&vmx->ctx[vcpu], reg, val) == 0)
2993                 return (0);
2994
2995         error = vmcs_setreg(&vmx->vmcs[vcpu], running, reg, val);
2996
2997         if (error == 0) {
2998                 /*
2999                  * If the "load EFER" VM-entry control is 1 then the
3000                  * value of EFER.LMA must be identical to "IA-32e mode guest"
3001                  * bit in the VM-entry control.
3002                  */
3003                 if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 &&
3004                     (reg == VM_REG_GUEST_EFER)) {
3005                         vmcs_getreg(&vmx->vmcs[vcpu], running,
3006                                     VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls);
3007                         if (val & EFER_LMA)
3008                                 ctls |= VM_ENTRY_GUEST_LMA;
3009                         else
3010                                 ctls &= ~VM_ENTRY_GUEST_LMA;
3011                         vmcs_setreg(&vmx->vmcs[vcpu], running,
3012                                     VMCS_IDENT(VMCS_ENTRY_CTLS), ctls);
3013                 }
3014
3015                 shadow = vmx_shadow_reg(reg);
3016                 if (shadow > 0) {
3017                         /*
3018                          * Store the unmodified value in the shadow
3019                          */
3020                         error = vmcs_setreg(&vmx->vmcs[vcpu], running,
3021                                     VMCS_IDENT(shadow), val);
3022                 }
3023
3024                 if (reg == VM_REG_GUEST_CR3) {
3025                         /*
3026                          * Invalidate the guest vcpu's TLB mappings to emulate
3027                          * the behavior of updating %cr3.
3028                          *
3029                          * XXX the processor retains global mappings when %cr3
3030                          * is updated but vmx_invvpid() does not.
3031                          */
3032                         pmap = vmx->ctx[vcpu].pmap;
3033                         vmx_invvpid(vmx, vcpu, pmap, running);
3034                 }
3035         }
3036
3037         return (error);
3038 }
3039
3040 static int
3041 vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
3042 {
3043         int hostcpu, running;
3044         struct vmx *vmx = arg;
3045
3046         running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
3047         if (running && hostcpu != curcpu)
3048                 panic("vmx_getdesc: %s%d is running", vm_name(vmx->vm), vcpu);
3049
3050         return (vmcs_getdesc(&vmx->vmcs[vcpu], running, reg, desc));
3051 }
3052
3053 static int
3054 vmx_setdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
3055 {
3056         int hostcpu, running;
3057         struct vmx *vmx = arg;
3058
3059         running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
3060         if (running && hostcpu != curcpu)
3061                 panic("vmx_setdesc: %s%d is running", vm_name(vmx->vm), vcpu);
3062
3063         return (vmcs_setdesc(&vmx->vmcs[vcpu], running, reg, desc));
3064 }
3065
3066 static int
3067 vmx_getcap(void *arg, int vcpu, int type, int *retval)
3068 {
3069         struct vmx *vmx = arg;
3070         int vcap;
3071         int ret;
3072
3073         ret = ENOENT;
3074
3075         vcap = vmx->cap[vcpu].set;
3076
3077         switch (type) {
3078         case VM_CAP_HALT_EXIT:
3079                 if (cap_halt_exit)
3080                         ret = 0;
3081                 break;
3082         case VM_CAP_PAUSE_EXIT:
3083                 if (cap_pause_exit)
3084                         ret = 0;
3085                 break;
3086         case VM_CAP_MTRAP_EXIT:
3087                 if (cap_monitor_trap)
3088                         ret = 0;
3089                 break;
3090         case VM_CAP_UNRESTRICTED_GUEST:
3091                 if (cap_unrestricted_guest)
3092                         ret = 0;
3093                 break;
3094         case VM_CAP_ENABLE_INVPCID:
3095                 if (cap_invpcid)
3096                         ret = 0;
3097                 break;
3098         default:
3099                 break;
3100         }
3101
3102         if (ret == 0)
3103                 *retval = (vcap & (1 << type)) ? 1 : 0;
3104
3105         return (ret);
3106 }
3107
3108 static int
3109 vmx_setcap(void *arg, int vcpu, int type, int val)
3110 {
3111         struct vmx *vmx = arg;
3112         struct vmcs *vmcs = &vmx->vmcs[vcpu];
3113         uint32_t baseval;
3114         uint32_t *pptr;
3115         int error;
3116         int flag;
3117         int reg;
3118         int retval;
3119
3120         retval = ENOENT;
3121         pptr = NULL;
3122
3123         switch (type) {
3124         case VM_CAP_HALT_EXIT:
3125                 if (cap_halt_exit) {
3126                         retval = 0;
3127                         pptr = &vmx->cap[vcpu].proc_ctls;
3128                         baseval = *pptr;
3129                         flag = PROCBASED_HLT_EXITING;
3130                         reg = VMCS_PRI_PROC_BASED_CTLS;
3131                 }
3132                 break;
3133         case VM_CAP_MTRAP_EXIT:
3134                 if (cap_monitor_trap) {
3135                         retval = 0;
3136                         pptr = &vmx->cap[vcpu].proc_ctls;
3137                         baseval = *pptr;
3138                         flag = PROCBASED_MTF;
3139                         reg = VMCS_PRI_PROC_BASED_CTLS;
3140                 }
3141                 break;
3142         case VM_CAP_PAUSE_EXIT:
3143                 if (cap_pause_exit) {
3144                         retval = 0;
3145                         pptr = &vmx->cap[vcpu].proc_ctls;
3146                         baseval = *pptr;
3147                         flag = PROCBASED_PAUSE_EXITING;
3148                         reg = VMCS_PRI_PROC_BASED_CTLS;
3149                 }
3150                 break;
3151         case VM_CAP_UNRESTRICTED_GUEST:
3152                 if (cap_unrestricted_guest) {
3153                         retval = 0;
3154                         pptr = &vmx->cap[vcpu].proc_ctls2;
3155                         baseval = *pptr;
3156                         flag = PROCBASED2_UNRESTRICTED_GUEST;
3157                         reg = VMCS_SEC_PROC_BASED_CTLS;
3158                 }
3159                 break;
3160         case VM_CAP_ENABLE_INVPCID:
3161                 if (cap_invpcid) {
3162                         retval = 0;
3163                         pptr = &vmx->cap[vcpu].proc_ctls2;
3164                         baseval = *pptr;
3165                         flag = PROCBASED2_ENABLE_INVPCID;
3166                         reg = VMCS_SEC_PROC_BASED_CTLS;
3167                 }
3168                 break;
3169         default:
3170                 break;
3171         }
3172
3173         if (retval == 0) {
3174                 if (val) {
3175                         baseval |= flag;
3176                 } else {
3177                         baseval &= ~flag;
3178                 }
3179                 VMPTRLD(vmcs);
3180                 error = vmwrite(reg, baseval);
3181                 VMCLEAR(vmcs);
3182
3183                 if (error) {
3184                         retval = error;
3185                 } else {
3186                         /*
3187                          * Update optional stored flags, and record
3188                          * setting
3189                          */
3190                         if (pptr != NULL) {
3191                                 *pptr = baseval;
3192                         }
3193
3194                         if (val) {
3195                                 vmx->cap[vcpu].set |= (1 << type);
3196                         } else {
3197                                 vmx->cap[vcpu].set &= ~(1 << type);
3198                         }
3199                 }
3200         }
3201
3202         return (retval);
3203 }
3204
3205 struct vlapic_vtx {
3206         struct vlapic   vlapic;
3207         struct pir_desc *pir_desc;
3208         struct vmx      *vmx;
3209 };
3210
3211 #define VMX_CTR_PIR(vm, vcpuid, pir_desc, notify, vector, level, msg)   \
3212 do {                                                                    \
3213         VCPU_CTR2(vm, vcpuid, msg " assert %s-triggered vector %d",     \
3214             level ? "level" : "edge", vector);                          \
3215         VCPU_CTR1(vm, vcpuid, msg " pir0 0x%016lx", pir_desc->pir[0]);  \
3216         VCPU_CTR1(vm, vcpuid, msg " pir1 0x%016lx", pir_desc->pir[1]);  \
3217         VCPU_CTR1(vm, vcpuid, msg " pir2 0x%016lx", pir_desc->pir[2]);  \
3218         VCPU_CTR1(vm, vcpuid, msg " pir3 0x%016lx", pir_desc->pir[3]);  \
3219         VCPU_CTR1(vm, vcpuid, msg " notify: %s", notify ? "yes" : "no");\
3220 } while (0)
3221
3222 /*
3223  * vlapic->ops handlers that utilize the APICv hardware assist described in
3224  * Chapter 29 of the Intel SDM.
3225  */
3226 static int
3227 vmx_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
3228 {
3229         struct vlapic_vtx *vlapic_vtx;
3230         struct pir_desc *pir_desc;
3231         uint64_t mask;
3232         int idx, notify;
3233
3234         vlapic_vtx = (struct vlapic_vtx *)vlapic;
3235         pir_desc = vlapic_vtx->pir_desc;
3236
3237         /*
3238          * Keep track of interrupt requests in the PIR descriptor. This is
3239          * because the virtual APIC page pointed to by the VMCS cannot be
3240          * modified if the vcpu is running.
3241          */
3242         idx = vector / 64;
3243         mask = 1UL << (vector % 64);
3244         atomic_set_long(&pir_desc->pir[idx], mask);
3245         notify = atomic_cmpset_long(&pir_desc->pending, 0, 1);
3246
3247         VMX_CTR_PIR(vlapic->vm, vlapic->vcpuid, pir_desc, notify, vector,
3248             level, "vmx_set_intr_ready");
3249         return (notify);
3250 }
3251
3252 static int
3253 vmx_pending_intr(struct vlapic *vlapic, int *vecptr)
3254 {
3255         struct vlapic_vtx *vlapic_vtx;
3256         struct pir_desc *pir_desc;
3257         struct LAPIC *lapic;
3258         uint64_t pending, pirval;
3259         uint32_t ppr, vpr;
3260         int i;
3261
3262         /*
3263          * This function is only expected to be called from the 'HLT' exit
3264          * handler which does not care about the vector that is pending.
3265          */
3266         KASSERT(vecptr == NULL, ("vmx_pending_intr: vecptr must be NULL"));
3267
3268         vlapic_vtx = (struct vlapic_vtx *)vlapic;
3269         pir_desc = vlapic_vtx->pir_desc;
3270
3271         pending = atomic_load_acq_long(&pir_desc->pending);
3272         if (!pending)
3273                 return (0);     /* common case */
3274
3275         /*
3276          * If there is an interrupt pending then it will be recognized only
3277          * if its priority is greater than the processor priority.
3278          *
3279          * Special case: if the processor priority is zero then any pending
3280          * interrupt will be recognized.
3281          */
3282         lapic = vlapic->apic_page;
3283         ppr = lapic->ppr & 0xf0;
3284         if (ppr == 0)
3285                 return (1);
3286
3287         VCPU_CTR1(vlapic->vm, vlapic->vcpuid, "HLT with non-zero PPR %d",
3288             lapic->ppr);
3289
3290         for (i = 3; i >= 0; i--) {
3291                 pirval = pir_desc->pir[i];
3292                 if (pirval != 0) {
3293                         vpr = (i * 64 + flsl(pirval) - 1) & 0xf0;
3294                         return (vpr > ppr);
3295                 }
3296         }
3297         return (0);
3298 }
3299
3300 static void
3301 vmx_intr_accepted(struct vlapic *vlapic, int vector)
3302 {
3303
3304         panic("vmx_intr_accepted: not expected to be called");
3305 }
3306
3307 static void
3308 vmx_set_tmr(struct vlapic *vlapic, int vector, bool level)
3309 {
3310         struct vlapic_vtx *vlapic_vtx;
3311         struct vmx *vmx;
3312         struct vmcs *vmcs;
3313         uint64_t mask, val;
3314
3315         KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
3316         KASSERT(!vcpu_is_running(vlapic->vm, vlapic->vcpuid, NULL),
3317             ("vmx_set_tmr: vcpu cannot be running"));
3318
3319         vlapic_vtx = (struct vlapic_vtx *)vlapic;
3320         vmx = vlapic_vtx->vmx;
3321         vmcs = &vmx->vmcs[vlapic->vcpuid];
3322         mask = 1UL << (vector % 64);
3323
3324         VMPTRLD(vmcs);
3325         val = vmcs_read(VMCS_EOI_EXIT(vector));
3326         if (level)
3327                 val |= mask;
3328         else
3329                 val &= ~mask;
3330         vmcs_write(VMCS_EOI_EXIT(vector), val);
3331         VMCLEAR(vmcs);
3332 }
3333
3334 static void
3335 vmx_enable_x2apic_mode(struct vlapic *vlapic)
3336 {
3337         struct vmx *vmx;
3338         struct vmcs *vmcs;
3339         uint32_t proc_ctls2;
3340         int vcpuid, error;
3341
3342         vcpuid = vlapic->vcpuid;
3343         vmx = ((struct vlapic_vtx *)vlapic)->vmx;
3344         vmcs = &vmx->vmcs[vcpuid];
3345
3346         proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
3347         KASSERT((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) != 0,
3348             ("%s: invalid proc_ctls2 %#x", __func__, proc_ctls2));
3349
3350         proc_ctls2 &= ~PROCBASED2_VIRTUALIZE_APIC_ACCESSES;
3351         proc_ctls2 |= PROCBASED2_VIRTUALIZE_X2APIC_MODE;
3352         vmx->cap[vcpuid].proc_ctls2 = proc_ctls2;
3353
3354         VMPTRLD(vmcs);
3355         vmcs_write(VMCS_SEC_PROC_BASED_CTLS, proc_ctls2);
3356         VMCLEAR(vmcs);
3357
3358         if (vlapic->vcpuid == 0) {
3359                 /*
3360                  * The nested page table mappings are shared by all vcpus
3361                  * so unmap the APIC access page just once.
3362                  */
3363                 error = vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
3364                 KASSERT(error == 0, ("%s: vm_unmap_mmio error %d",
3365                     __func__, error));
3366
3367                 /*
3368                  * The MSR bitmap is shared by all vcpus so modify it only
3369                  * once in the context of vcpu 0.
3370                  */
3371                 error = vmx_allow_x2apic_msrs(vmx);
3372                 KASSERT(error == 0, ("%s: vmx_allow_x2apic_msrs error %d",
3373                     __func__, error));
3374         }
3375 }
3376
3377 static void
3378 vmx_post_intr(struct vlapic *vlapic, int hostcpu)
3379 {
3380
3381         ipi_cpu(hostcpu, pirvec);
3382 }
3383
3384 /*
3385  * Transfer the pending interrupts in the PIR descriptor to the IRR
3386  * in the virtual APIC page.
3387  */
3388 static void
3389 vmx_inject_pir(struct vlapic *vlapic)
3390 {
3391         struct vlapic_vtx *vlapic_vtx;
3392         struct pir_desc *pir_desc;
3393         struct LAPIC *lapic;
3394         uint64_t val, pirval;
3395         int rvi, pirbase = -1;
3396         uint16_t intr_status_old, intr_status_new;
3397
3398         vlapic_vtx = (struct vlapic_vtx *)vlapic;
3399         pir_desc = vlapic_vtx->pir_desc;
3400         if (atomic_cmpset_long(&pir_desc->pending, 1, 0) == 0) {
3401                 VCPU_CTR0(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: "
3402                     "no posted interrupt pending");
3403                 return;
3404         }
3405
3406         pirval = 0;
3407         pirbase = -1;
3408         lapic = vlapic->apic_page;
3409
3410         val = atomic_readandclear_long(&pir_desc->pir[0]);
3411         if (val != 0) {
3412                 lapic->irr0 |= val;
3413                 lapic->irr1 |= val >> 32;
3414                 pirbase = 0;
3415                 pirval = val;
3416         }
3417
3418         val = atomic_readandclear_long(&pir_desc->pir[1]);
3419         if (val != 0) {
3420                 lapic->irr2 |= val;
3421                 lapic->irr3 |= val >> 32;
3422                 pirbase = 64;
3423                 pirval = val;
3424         }
3425
3426         val = atomic_readandclear_long(&pir_desc->pir[2]);
3427         if (val != 0) {
3428                 lapic->irr4 |= val;
3429                 lapic->irr5 |= val >> 32;
3430                 pirbase = 128;
3431                 pirval = val;
3432         }
3433
3434         val = atomic_readandclear_long(&pir_desc->pir[3]);
3435         if (val != 0) {
3436                 lapic->irr6 |= val;
3437                 lapic->irr7 |= val >> 32;
3438                 pirbase = 192;
3439                 pirval = val;
3440         }
3441
3442         VLAPIC_CTR_IRR(vlapic, "vmx_inject_pir");
3443
3444         /*
3445          * Update RVI so the processor can evaluate pending virtual
3446          * interrupts on VM-entry.
3447          *
3448          * It is possible for pirval to be 0 here, even though the
3449          * pending bit has been set. The scenario is:
3450          * CPU-Y is sending a posted interrupt to CPU-X, which
3451          * is running a guest and processing posted interrupts in h/w.
3452          * CPU-X will eventually exit and the state seen in s/w is
3453          * the pending bit set, but no PIR bits set.
3454          *
3455          *      CPU-X                      CPU-Y
3456          *   (vm running)                (host running)
3457          *   rx posted interrupt
3458          *   CLEAR pending bit
3459          *                               SET PIR bit
3460          *   READ/CLEAR PIR bits
3461          *                               SET pending bit
3462          *   (vm exit)
3463          *   pending bit set, PIR 0
3464          */
3465         if (pirval != 0) {
3466                 rvi = pirbase + flsl(pirval) - 1;
3467                 intr_status_old = vmcs_read(VMCS_GUEST_INTR_STATUS);
3468                 intr_status_new = (intr_status_old & 0xFF00) | rvi;
3469                 if (intr_status_new > intr_status_old) {
3470                         vmcs_write(VMCS_GUEST_INTR_STATUS, intr_status_new);
3471                         VCPU_CTR2(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: "
3472                             "guest_intr_status changed from 0x%04x to 0x%04x",
3473                             intr_status_old, intr_status_new);
3474                 }
3475         }
3476 }
3477
3478 static struct vlapic *
3479 vmx_vlapic_init(void *arg, int vcpuid)
3480 {
3481         struct vmx *vmx;
3482         struct vlapic *vlapic;
3483         struct vlapic_vtx *vlapic_vtx;
3484
3485         vmx = arg;
3486
3487         vlapic = malloc(sizeof(struct vlapic_vtx), M_VLAPIC, M_WAITOK | M_ZERO);
3488         vlapic->vm = vmx->vm;
3489         vlapic->vcpuid = vcpuid;
3490         vlapic->apic_page = (struct LAPIC *)&vmx->apic_page[vcpuid];
3491
3492         vlapic_vtx = (struct vlapic_vtx *)vlapic;
3493         vlapic_vtx->pir_desc = &vmx->pir_desc[vcpuid];
3494         vlapic_vtx->vmx = vmx;
3495
3496         if (virtual_interrupt_delivery) {
3497                 vlapic->ops.set_intr_ready = vmx_set_intr_ready;
3498                 vlapic->ops.pending_intr = vmx_pending_intr;
3499                 vlapic->ops.intr_accepted = vmx_intr_accepted;
3500                 vlapic->ops.set_tmr = vmx_set_tmr;
3501                 vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode;
3502         }
3503
3504         if (posted_interrupts)
3505                 vlapic->ops.post_intr = vmx_post_intr;
3506
3507         vlapic_init(vlapic);
3508
3509         return (vlapic);
3510 }
3511
3512 static void
3513 vmx_vlapic_cleanup(void *arg, struct vlapic *vlapic)
3514 {
3515
3516         vlapic_cleanup(vlapic);
3517         free(vlapic, M_VLAPIC);
3518 }
3519
3520 struct vmm_ops vmm_ops_intel = {
3521         vmx_init,
3522         vmx_cleanup,
3523         vmx_restore,
3524         vmx_vminit,
3525         vmx_run,
3526         vmx_vmcleanup,
3527         vmx_getreg,
3528         vmx_setreg,
3529         vmx_getdesc,
3530         vmx_setdesc,
3531         vmx_getcap,
3532         vmx_setcap,
3533         ept_vmspace_alloc,
3534         ept_vmspace_free,
3535         vmx_vlapic_init,
3536         vmx_vlapic_cleanup,
3537 };