sys/amd64/vmm/intel/vmx.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2011 NetApp, Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  *
  28  * $FreeBSD$
  29  */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33
  34 #include <sys/param.h>
  35 #include <sys/systm.h>
  36 #include <sys/smp.h>
  37 #include <sys/kernel.h>
  38 #include <sys/malloc.h>
  39 #include <sys/pcpu.h>
  40 #include <sys/proc.h>
  41 #include <sys/sysctl.h>
  42
  43 #include <vm/vm.h>
  44 #include <vm/pmap.h>
  45
  46 #include <machine/psl.h>
  47 #include <machine/cpufunc.h>
  48 #include <machine/md_var.h>
  49 #include <machine/segments.h>
  50 #include <machine/smp.h>
  51 #include <machine/specialreg.h>
  52 #include <machine/vmparam.h>
  53
  54 #include <machine/vmm.h>
  55 #include <machine/vmm_dev.h>
  56 #include <machine/vmm_instruction_emul.h>
  57 #include "vmm_lapic.h"
  58 #include "vmm_host.h"
  59 #include "vmm_ioport.h"
  60 #include "vmm_ktr.h"
  61 #include "vmm_stat.h"
  62 #include "vatpic.h"
  63 #include "vlapic.h"
  64 #include "vlapic_priv.h"
  65
  66 #include "ept.h"
  67 #include "vmx_cpufunc.h"
  68 #include "vmx.h"
  69 #include "vmx_msr.h"
  70 #include "x86.h"
  71 #include "vmx_controls.h"
  72
  73 #define PINBASED_CTLS_ONE_SETTING                                       \
  74         (PINBASED_EXTINT_EXITING        |                               \
  75          PINBASED_NMI_EXITING           |                               \
  76          PINBASED_VIRTUAL_NMI)
  77 #define PINBASED_CTLS_ZERO_SETTING      0
  78
  79 #define PROCBASED_CTLS_WINDOW_SETTING                                   \
  80         (PROCBASED_INT_WINDOW_EXITING   |                               \
  81          PROCBASED_NMI_WINDOW_EXITING)
  82
  83 #define PROCBASED_CTLS_ONE_SETTING                                      \
  84         (PROCBASED_SECONDARY_CONTROLS   |                               \
  85          PROCBASED_MWAIT_EXITING        |                               \
  86          PROCBASED_MONITOR_EXITING      |                               \
  87          PROCBASED_IO_EXITING           |                               \
  88          PROCBASED_MSR_BITMAPS          |                               \
  89          PROCBASED_CTLS_WINDOW_SETTING  |                               \
  90          PROCBASED_CR8_LOAD_EXITING     |                               \
  91          PROCBASED_CR8_STORE_EXITING)
  92 #define PROCBASED_CTLS_ZERO_SETTING     \
  93         (PROCBASED_CR3_LOAD_EXITING |   \
  94         PROCBASED_CR3_STORE_EXITING |   \
  95         PROCBASED_IO_BITMAPS)
  96
  97 #define PROCBASED_CTLS2_ONE_SETTING     PROCBASED2_ENABLE_EPT
  98 #define PROCBASED_CTLS2_ZERO_SETTING    0
  99
 100 #define VM_EXIT_CTLS_ONE_SETTING                                        \
 101         (VM_EXIT_SAVE_DEBUG_CONTROLS            |                       \
 102         VM_EXIT_HOST_LMA                        |                       \
 103         VM_EXIT_SAVE_EFER                       |                       \
 104         VM_EXIT_LOAD_EFER                       |                       \
 105         VM_EXIT_ACKNOWLEDGE_INTERRUPT)
 106
 107 #define VM_EXIT_CTLS_ZERO_SETTING       0
 108
 109 #define VM_ENTRY_CTLS_ONE_SETTING                                       \
 110         (VM_ENTRY_LOAD_DEBUG_CONTROLS           |                       \
 111         VM_ENTRY_LOAD_EFER)
 112
 113 #define VM_ENTRY_CTLS_ZERO_SETTING                                      \
 114         (VM_ENTRY_INTO_SMM                      |                       \
 115         VM_ENTRY_DEACTIVATE_DUAL_MONITOR)
 116
 117 #define HANDLED         1
 118 #define UNHANDLED       0
 119
 120 static MALLOC_DEFINE(M_VMX, "vmx", "vmx");
 121 static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic");
 122
 123 SYSCTL_DECL(_hw_vmm);
 124 SYSCTL_NODE(_hw_vmm, OID_AUTO, vmx, CTLFLAG_RW, NULL, NULL);
 125
 126 int vmxon_enabled[MAXCPU];
 127 static char vmxon_region[MAXCPU][PAGE_SIZE] __aligned(PAGE_SIZE);
 128
 129 static uint32_t pinbased_ctls, procbased_ctls, procbased_ctls2;
 130 static uint32_t exit_ctls, entry_ctls;
 131
 132 static uint64_t cr0_ones_mask, cr0_zeros_mask;
 133 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_ones_mask, CTLFLAG_RD,
 134              &cr0_ones_mask, 0, NULL);
 135 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_zeros_mask, CTLFLAG_RD,
 136              &cr0_zeros_mask, 0, NULL);
 137
 138 static uint64_t cr4_ones_mask, cr4_zeros_mask;
 139 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_ones_mask, CTLFLAG_RD,
 140              &cr4_ones_mask, 0, NULL);
 141 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_zeros_mask, CTLFLAG_RD,
 142              &cr4_zeros_mask, 0, NULL);
 143
 144 static int vmx_initialized;
 145 SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, initialized, CTLFLAG_RD,
 146            &vmx_initialized, 0, "Intel VMX initialized");
 147
 148 /*
 149  * Optional capabilities
 150  */
 151 static SYSCTL_NODE(_hw_vmm_vmx, OID_AUTO, cap, CTLFLAG_RW, NULL, NULL);
 152
 153 static int cap_halt_exit;
 154 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, halt_exit, CTLFLAG_RD, &cap_halt_exit, 0,
 155     "HLT triggers a VM-exit");
 156
 157 static int cap_pause_exit;
 158 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, pause_exit, CTLFLAG_RD, &cap_pause_exit,
 159     0, "PAUSE triggers a VM-exit");
 160
 161 static int cap_unrestricted_guest;
 162 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, unrestricted_guest, CTLFLAG_RD,
 163     &cap_unrestricted_guest, 0, "Unrestricted guests");
 164
 165 static int cap_monitor_trap;
 166 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, monitor_trap, CTLFLAG_RD,
 167     &cap_monitor_trap, 0, "Monitor trap flag");
 168
 169 static int cap_invpcid;
 170 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, invpcid, CTLFLAG_RD, &cap_invpcid,
 171     0, "Guests are allowed to use INVPCID");
 172
 173 static int virtual_interrupt_delivery;
 174 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, virtual_interrupt_delivery, CTLFLAG_RD,
 175     &virtual_interrupt_delivery, 0, "APICv virtual interrupt delivery support");
 176
 177 static int posted_interrupts;
 178 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, posted_interrupts, CTLFLAG_RD,
 179     &posted_interrupts, 0, "APICv posted interrupt support");
 180
 181 static int pirvec = -1;
 182 SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupt_vector, CTLFLAG_RD,
 183     &pirvec, 0, "APICv posted interrupt vector");
 184
 185 static struct unrhdr *vpid_unr;
 186 static u_int vpid_alloc_failed;
 187 SYSCTL_UINT(_hw_vmm_vmx, OID_AUTO, vpid_alloc_failed, CTLFLAG_RD,
 188             &vpid_alloc_failed, 0, NULL);
 189
 190 /*
 191  * Use the last page below 4GB as the APIC access address. This address is
 192  * occupied by the boot firmware so it is guaranteed that it will not conflict
 193  * with a page in system memory.
 194  */
 195 #define APIC_ACCESS_ADDRESS     0xFFFFF000
 196
 197 static int vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc);
 198 static int vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval);
 199 static int vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val);
 200 static void vmx_inject_pir(struct vlapic *vlapic);
 201
 202 #ifdef KTR
 203 static const char *
 204 exit_reason_to_str(int reason)
 205 {
 206         static char reasonbuf[32];
 207
 208         switch (reason) {
 209         case EXIT_REASON_EXCEPTION:
 210                 return "exception";
 211         case EXIT_REASON_EXT_INTR:
 212                 return "extint";
 213         case EXIT_REASON_TRIPLE_FAULT:
 214                 return "triplefault";
 215         case EXIT_REASON_INIT:
 216                 return "init";
 217         case EXIT_REASON_SIPI:
 218                 return "sipi";
 219         case EXIT_REASON_IO_SMI:
 220                 return "iosmi";
 221         case EXIT_REASON_SMI:
 222                 return "smi";
 223         case EXIT_REASON_INTR_WINDOW:
 224                 return "intrwindow";
 225         case EXIT_REASON_NMI_WINDOW:
 226                 return "nmiwindow";
 227         case EXIT_REASON_TASK_SWITCH:
 228                 return "taskswitch";
 229         case EXIT_REASON_CPUID:
 230                 return "cpuid";
 231         case EXIT_REASON_GETSEC:
 232                 return "getsec";
 233         case EXIT_REASON_HLT:
 234                 return "hlt";
 235         case EXIT_REASON_INVD:
 236                 return "invd";
 237         case EXIT_REASON_INVLPG:
 238                 return "invlpg";
 239         case EXIT_REASON_RDPMC:
 240                 return "rdpmc";
 241         case EXIT_REASON_RDTSC:
 242                 return "rdtsc";
 243         case EXIT_REASON_RSM:
 244                 return "rsm";
 245         case EXIT_REASON_VMCALL:
 246                 return "vmcall";
 247         case EXIT_REASON_VMCLEAR:
 248                 return "vmclear";
 249         case EXIT_REASON_VMLAUNCH:
 250                 return "vmlaunch";
 251         case EXIT_REASON_VMPTRLD:
 252                 return "vmptrld";
 253         case EXIT_REASON_VMPTRST:
 254                 return "vmptrst";
 255         case EXIT_REASON_VMREAD:
 256                 return "vmread";
 257         case EXIT_REASON_VMRESUME:
 258                 return "vmresume";
 259         case EXIT_REASON_VMWRITE:
 260                 return "vmwrite";
 261         case EXIT_REASON_VMXOFF:
 262                 return "vmxoff";
 263         case EXIT_REASON_VMXON:
 264                 return "vmxon";
 265         case EXIT_REASON_CR_ACCESS:
 266                 return "craccess";
 267         case EXIT_REASON_DR_ACCESS:
 268                 return "draccess";
 269         case EXIT_REASON_INOUT:
 270                 return "inout";
 271         case EXIT_REASON_RDMSR:
 272                 return "rdmsr";
 273         case EXIT_REASON_WRMSR:
 274                 return "wrmsr";
 275         case EXIT_REASON_INVAL_VMCS:
 276                 return "invalvmcs";
 277         case EXIT_REASON_INVAL_MSR:
 278                 return "invalmsr";
 279         case EXIT_REASON_MWAIT:
 280                 return "mwait";
 281         case EXIT_REASON_MTF:
 282                 return "mtf";
 283         case EXIT_REASON_MONITOR:
 284                 return "monitor";
 285         case EXIT_REASON_PAUSE:
 286                 return "pause";
 287         case EXIT_REASON_MCE_DURING_ENTRY:
 288                 return "mce-during-entry";
 289         case EXIT_REASON_TPR:
 290                 return "tpr";
 291         case EXIT_REASON_APIC_ACCESS:
 292                 return "apic-access";
 293         case EXIT_REASON_GDTR_IDTR:
 294                 return "gdtridtr";
 295         case EXIT_REASON_LDTR_TR:
 296                 return "ldtrtr";
 297         case EXIT_REASON_EPT_FAULT:
 298                 return "eptfault";
 299         case EXIT_REASON_EPT_MISCONFIG:
 300                 return "eptmisconfig";
 301         case EXIT_REASON_INVEPT:
 302                 return "invept";
 303         case EXIT_REASON_RDTSCP:
 304                 return "rdtscp";
 305         case EXIT_REASON_VMX_PREEMPT:
 306                 return "vmxpreempt";
 307         case EXIT_REASON_INVVPID:
 308                 return "invvpid";
 309         case EXIT_REASON_WBINVD:
 310                 return "wbinvd";
 311         case EXIT_REASON_XSETBV:
 312                 return "xsetbv";
 313         case EXIT_REASON_APIC_WRITE:
 314                 return "apic-write";
 315         default:
 316                 snprintf(reasonbuf, sizeof(reasonbuf), "%d", reason);
 317                 return (reasonbuf);
 318         }
 319 }
 320 #endif  /* KTR */
 321
 322 static int
 323 vmx_allow_x2apic_msrs(struct vmx *vmx)
 324 {
 325         int i, error;
 326
 327         error = 0;
 328
 329         /*
 330          * Allow readonly access to the following x2APIC MSRs from the guest.
 331          */
 332         error += guest_msr_ro(vmx, MSR_APIC_ID);
 333         error += guest_msr_ro(vmx, MSR_APIC_VERSION);
 334         error += guest_msr_ro(vmx, MSR_APIC_LDR);
 335         error += guest_msr_ro(vmx, MSR_APIC_SVR);
 336
 337         for (i = 0; i < 8; i++)
 338                 error += guest_msr_ro(vmx, MSR_APIC_ISR0 + i);
 339
 340         for (i = 0; i < 8; i++)
 341                 error += guest_msr_ro(vmx, MSR_APIC_TMR0 + i);
 342
 343         for (i = 0; i < 8; i++)
 344                 error += guest_msr_ro(vmx, MSR_APIC_IRR0 + i);
 345
 346         error += guest_msr_ro(vmx, MSR_APIC_ESR);
 347         error += guest_msr_ro(vmx, MSR_APIC_LVT_TIMER);
 348         error += guest_msr_ro(vmx, MSR_APIC_LVT_THERMAL);
 349         error += guest_msr_ro(vmx, MSR_APIC_LVT_PCINT);
 350         error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT0);
 351         error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT1);
 352         error += guest_msr_ro(vmx, MSR_APIC_LVT_ERROR);
 353         error += guest_msr_ro(vmx, MSR_APIC_ICR_TIMER);
 354         error += guest_msr_ro(vmx, MSR_APIC_DCR_TIMER);
 355         error += guest_msr_ro(vmx, MSR_APIC_ICR);
 356
 357         /*
 358          * Allow TPR, EOI and SELF_IPI MSRs to be read and written by the guest.
 359          *
 360          * These registers get special treatment described in the section
 361          * "Virtualizing MSR-Based APIC Accesses".
 362          */
 363         error += guest_msr_rw(vmx, MSR_APIC_TPR);
 364         error += guest_msr_rw(vmx, MSR_APIC_EOI);
 365         error += guest_msr_rw(vmx, MSR_APIC_SELF_IPI);
 366
 367         return (error);
 368 }
 369
 370 u_long
 371 vmx_fix_cr0(u_long cr0)
 372 {
 373
 374         return ((cr0 | cr0_ones_mask) & ~cr0_zeros_mask);
 375 }
 376
 377 u_long
 378 vmx_fix_cr4(u_long cr4)
 379 {
 380
 381         return ((cr4 | cr4_ones_mask) & ~cr4_zeros_mask);
 382 }
 383
 384 static void
 385 vpid_free(int vpid)
 386 {
 387         if (vpid < 0 || vpid > 0xffff)
 388                 panic("vpid_free: invalid vpid %d", vpid);
 389
 390         /*
 391          * VPIDs [0,VM_MAXCPU] are special and are not allocated from
 392          * the unit number allocator.
 393          */
 394
 395         if (vpid > VM_MAXCPU)
 396                 free_unr(vpid_unr, vpid);
 397 }
 398
 399 static void
 400 vpid_alloc(uint16_t *vpid, int num)
 401 {
 402         int i, x;
 403
 404         if (num <= 0 || num > VM_MAXCPU)
 405                 panic("invalid number of vpids requested: %d", num);
 406
 407         /*
 408          * If the "enable vpid" execution control is not enabled then the
 409          * VPID is required to be 0 for all vcpus.
 410          */
 411         if ((procbased_ctls2 & PROCBASED2_ENABLE_VPID) == 0) {
 412                 for (i = 0; i < num; i++)
 413                         vpid[i] = 0;
 414                 return;
 415         }
 416
 417         /*
 418          * Allocate a unique VPID for each vcpu from the unit number allocator.
 419          */
 420         for (i = 0; i < num; i++) {
 421                 x = alloc_unr(vpid_unr);
 422                 if (x == -1)
 423                         break;
 424                 else
 425                         vpid[i] = x;
 426         }
 427
 428         if (i < num) {
 429                 atomic_add_int(&vpid_alloc_failed, 1);
 430
 431                 /*
 432                  * If the unit number allocator does not have enough unique
 433                  * VPIDs then we need to allocate from the [1,VM_MAXCPU] range.
 434                  *
 435                  * These VPIDs are not be unique across VMs but this does not
 436                  * affect correctness because the combined mappings are also
 437                  * tagged with the EP4TA which is unique for each VM.
 438                  *
 439                  * It is still sub-optimal because the invvpid will invalidate
 440                  * combined mappings for a particular VPID across all EP4TAs.
 441                  */
 442                 while (i-- > 0)
 443                         vpid_free(vpid[i]);
 444
 445                 for (i = 0; i < num; i++)
 446                         vpid[i] = i + 1;
 447         }
 448 }
 449
 450 static void
 451 vpid_init(void)
 452 {
 453         /*
 454          * VPID 0 is required when the "enable VPID" execution control is
 455          * disabled.
 456          *
 457          * VPIDs [1,VM_MAXCPU] are used as the "overflow namespace" when the
 458          * unit number allocator does not have sufficient unique VPIDs to
 459          * satisfy the allocation.
 460          *
 461          * The remaining VPIDs are managed by the unit number allocator.
 462          */
 463         vpid_unr = new_unrhdr(VM_MAXCPU + 1, 0xffff, NULL);
 464 }
 465
 466 static void
 467 vmx_disable(void *arg __unused)
 468 {
 469         struct invvpid_desc invvpid_desc = { 0 };
 470         struct invept_desc invept_desc = { 0 };
 471
 472         if (vmxon_enabled[curcpu]) {
 473                 /*
 474                  * See sections 25.3.3.3 and 25.3.3.4 in Intel Vol 3b.
 475                  *
 476                  * VMXON or VMXOFF are not required to invalidate any TLB
 477                  * caching structures. This prevents potential retention of
 478                  * cached information in the TLB between distinct VMX episodes.
 479                  */
 480                 invvpid(INVVPID_TYPE_ALL_CONTEXTS, invvpid_desc);
 481                 invept(INVEPT_TYPE_ALL_CONTEXTS, invept_desc);
 482                 vmxoff();
 483         }
 484         load_cr4(rcr4() & ~CR4_VMXE);
 485 }
 486
 487 static int
 488 vmx_cleanup(void)
 489 {
 490
 491         if (pirvec >= 0)
 492                 lapic_ipi_free(pirvec);
 493
 494         if (vpid_unr != NULL) {
 495                 delete_unrhdr(vpid_unr);
 496                 vpid_unr = NULL;
 497         }
 498
 499         smp_rendezvous(NULL, vmx_disable, NULL, NULL);
 500
 501         return (0);
 502 }
 503
 504 static void
 505 vmx_enable(void *arg __unused)
 506 {
 507         int error;
 508         uint64_t feature_control;
 509
 510         feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
 511         if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 0 ||
 512             (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) {
 513                 wrmsr(MSR_IA32_FEATURE_CONTROL,
 514                     feature_control | IA32_FEATURE_CONTROL_VMX_EN |
 515                     IA32_FEATURE_CONTROL_LOCK);
 516         }
 517
 518         load_cr4(rcr4() | CR4_VMXE);
 519
 520         *(uint32_t *)vmxon_region[curcpu] = vmx_revision();
 521         error = vmxon(vmxon_region[curcpu]);
 522         if (error == 0)
 523                 vmxon_enabled[curcpu] = 1;
 524 }
 525
 526 static void
 527 vmx_restore(void)
 528 {
 529
 530         if (vmxon_enabled[curcpu])
 531                 vmxon(vmxon_region[curcpu]);
 532 }
 533
 534 static int
 535 vmx_init(int ipinum)
 536 {
 537         int error, use_tpr_shadow;
 538         uint64_t basic, fixed0, fixed1, feature_control;
 539         uint32_t tmp, procbased2_vid_bits;
 540
 541         /* CPUID.1:ECX[bit 5] must be 1 for processor to support VMX */
 542         if (!(cpu_feature2 & CPUID2_VMX)) {
 543                 printf("vmx_init: processor does not support VMX operation\n");
 544                 return (ENXIO);
 545         }
 546
 547         /*
 548          * Verify that MSR_IA32_FEATURE_CONTROL lock and VMXON enable bits
 549          * are set (bits 0 and 2 respectively).
 550          */
 551         feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
 552         if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 1 &&
 553             (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) {
 554                 printf("vmx_init: VMX operation disabled by BIOS\n");
 555                 return (ENXIO);
 556         }
 557
 558         /*
 559          * Verify capabilities MSR_VMX_BASIC:
 560          * - bit 54 indicates support for INS/OUTS decoding
 561          */
 562         basic = rdmsr(MSR_VMX_BASIC);
 563         if ((basic & (1UL << 54)) == 0) {
 564                 printf("vmx_init: processor does not support desired basic "
 565                     "capabilities\n");
 566                 return (EINVAL);
 567         }
 568
 569         /* Check support for primary processor-based VM-execution controls */
 570         error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 571                                MSR_VMX_TRUE_PROCBASED_CTLS,
 572                                PROCBASED_CTLS_ONE_SETTING,
 573                                PROCBASED_CTLS_ZERO_SETTING, &procbased_ctls);
 574         if (error) {
 575                 printf("vmx_init: processor does not support desired primary "
 576                        "processor-based controls\n");
 577                 return (error);
 578         }
 579
 580         /* Clear the processor-based ctl bits that are set on demand */
 581         procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING;
 582
 583         /* Check support for secondary processor-based VM-execution controls */
 584         error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
 585                                MSR_VMX_PROCBASED_CTLS2,
 586                                PROCBASED_CTLS2_ONE_SETTING,
 587                                PROCBASED_CTLS2_ZERO_SETTING, &procbased_ctls2);
 588         if (error) {
 589                 printf("vmx_init: processor does not support desired secondary "
 590                        "processor-based controls\n");
 591                 return (error);
 592         }
 593
 594         /* Check support for VPID */
 595         error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
 596                                PROCBASED2_ENABLE_VPID, 0, &tmp);
 597         if (error == 0)
 598                 procbased_ctls2 |= PROCBASED2_ENABLE_VPID;
 599
 600         /* Check support for pin-based VM-execution controls */
 601         error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
 602                                MSR_VMX_TRUE_PINBASED_CTLS,
 603                                PINBASED_CTLS_ONE_SETTING,
 604                                PINBASED_CTLS_ZERO_SETTING, &pinbased_ctls);
 605         if (error) {
 606                 printf("vmx_init: processor does not support desired "
 607                        "pin-based controls\n");
 608                 return (error);
 609         }
 610
 611         /* Check support for VM-exit controls */
 612         error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, MSR_VMX_TRUE_EXIT_CTLS,
 613                                VM_EXIT_CTLS_ONE_SETTING,
 614                                VM_EXIT_CTLS_ZERO_SETTING,
 615                                &exit_ctls);
 616         if (error) {
 617                 printf("vmx_init: processor does not support desired "
 618                     "exit controls\n");
 619                 return (error);
 620         }
 621
 622         /* Check support for VM-entry controls */
 623         error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS, MSR_VMX_TRUE_ENTRY_CTLS,
 624             VM_ENTRY_CTLS_ONE_SETTING, VM_ENTRY_CTLS_ZERO_SETTING,
 625             &entry_ctls);
 626         if (error) {
 627                 printf("vmx_init: processor does not support desired "
 628                     "entry controls\n");
 629                 return (error);
 630         }
 631
 632         /*
 633          * Check support for optional features by testing them
 634          * as individual bits
 635          */
 636         cap_halt_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 637                                         MSR_VMX_TRUE_PROCBASED_CTLS,
 638                                         PROCBASED_HLT_EXITING, 0,
 639                                         &tmp) == 0);
 640
 641         cap_monitor_trap = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 642                                         MSR_VMX_PROCBASED_CTLS,
 643                                         PROCBASED_MTF, 0,
 644                                         &tmp) == 0);
 645
 646         cap_pause_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 647                                          MSR_VMX_TRUE_PROCBASED_CTLS,
 648                                          PROCBASED_PAUSE_EXITING, 0,
 649                                          &tmp) == 0);
 650
 651         cap_unrestricted_guest = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
 652                                         MSR_VMX_PROCBASED_CTLS2,
 653                                         PROCBASED2_UNRESTRICTED_GUEST, 0,
 654                                         &tmp) == 0);
 655
 656         cap_invpcid = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
 657             MSR_VMX_PROCBASED_CTLS2, PROCBASED2_ENABLE_INVPCID, 0,
 658             &tmp) == 0);
 659
 660         /*
 661          * Check support for virtual interrupt delivery.
 662          */
 663         procbased2_vid_bits = (PROCBASED2_VIRTUALIZE_APIC_ACCESSES |
 664             PROCBASED2_VIRTUALIZE_X2APIC_MODE |
 665             PROCBASED2_APIC_REGISTER_VIRTUALIZATION |
 666             PROCBASED2_VIRTUAL_INTERRUPT_DELIVERY);
 667
 668         use_tpr_shadow = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 669             MSR_VMX_TRUE_PROCBASED_CTLS, PROCBASED_USE_TPR_SHADOW, 0,
 670             &tmp) == 0);
 671
 672         error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
 673             procbased2_vid_bits, 0, &tmp);
 674         if (error == 0 && use_tpr_shadow) {
 675                 virtual_interrupt_delivery = 1;
 676                 TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_vid",
 677                     &virtual_interrupt_delivery);
 678         }
 679
 680         if (virtual_interrupt_delivery) {
 681                 procbased_ctls |= PROCBASED_USE_TPR_SHADOW;
 682                 procbased_ctls2 |= procbased2_vid_bits;
 683                 procbased_ctls2 &= ~PROCBASED2_VIRTUALIZE_X2APIC_MODE;
 684
 685                 /*
 686                  * No need to emulate accesses to %CR8 if virtual
 687                  * interrupt delivery is enabled.
 688                  */
 689                 procbased_ctls &= ~PROCBASED_CR8_LOAD_EXITING;
 690                 procbased_ctls &= ~PROCBASED_CR8_STORE_EXITING;
 691
 692                 /*
 693                  * Check for Posted Interrupts only if Virtual Interrupt
 694                  * Delivery is enabled.
 695                  */
 696                 error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
 697                     MSR_VMX_TRUE_PINBASED_CTLS, PINBASED_POSTED_INTERRUPT, 0,
 698                     &tmp);
 699                 if (error == 0) {
 700                         pirvec = lapic_ipi_alloc(pti ? &IDTVEC(justreturn1_pti) :
 701                             &IDTVEC(justreturn));
 702                         if (pirvec < 0) {
 703                                 if (bootverbose) {
 704                                         printf("vmx_init: unable to allocate "
 705                                             "posted interrupt vector\n");
 706                                 }
 707                         } else {
 708                                 posted_interrupts = 1;
 709                                 TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_pir",
 710                                     &posted_interrupts);
 711                         }
 712                 }
 713         }
 714
 715         if (posted_interrupts)
 716                     pinbased_ctls |= PINBASED_POSTED_INTERRUPT;
 717
 718         /* Initialize EPT */
 719         error = ept_init(ipinum);
 720         if (error) {
 721                 printf("vmx_init: ept initialization failed (%d)\n", error);
 722                 return (error);
 723         }
 724
 725         /*
 726          * Stash the cr0 and cr4 bits that must be fixed to 0 or 1
 727          */
 728         fixed0 = rdmsr(MSR_VMX_CR0_FIXED0);
 729         fixed1 = rdmsr(MSR_VMX_CR0_FIXED1);
 730         cr0_ones_mask = fixed0 & fixed1;
 731         cr0_zeros_mask = ~fixed0 & ~fixed1;
 732
 733         /*
 734          * CR0_PE and CR0_PG can be set to zero in VMX non-root operation
 735          * if unrestricted guest execution is allowed.
 736          */
 737         if (cap_unrestricted_guest)
 738                 cr0_ones_mask &= ~(CR0_PG | CR0_PE);
 739
 740         /*
 741          * Do not allow the guest to set CR0_NW or CR0_CD.
 742          */
 743         cr0_zeros_mask |= (CR0_NW | CR0_CD);
 744
 745         fixed0 = rdmsr(MSR_VMX_CR4_FIXED0);
 746         fixed1 = rdmsr(MSR_VMX_CR4_FIXED1);
 747         cr4_ones_mask = fixed0 & fixed1;
 748         cr4_zeros_mask = ~fixed0 & ~fixed1;
 749
 750         vpid_init();
 751
 752         vmx_msr_init();
 753
 754         /* enable VMX operation */
 755         smp_rendezvous(NULL, vmx_enable, NULL, NULL);
 756
 757         vmx_initialized = 1;
 758
 759         return (0);
 760 }
 761
 762 static void
 763 vmx_trigger_hostintr(int vector)
 764 {
 765         uintptr_t func;
 766         struct gate_descriptor *gd;
 767
 768         gd = &idt[vector];
 769
 770         KASSERT(vector >= 32 && vector <= 255, ("vmx_trigger_hostintr: "
 771             "invalid vector %d", vector));
 772         KASSERT(gd->gd_p == 1, ("gate descriptor for vector %d not present",
 773             vector));
 774         KASSERT(gd->gd_type == SDT_SYSIGT, ("gate descriptor for vector %d "
 775             "has invalid type %d", vector, gd->gd_type));
 776         KASSERT(gd->gd_dpl == SEL_KPL, ("gate descriptor for vector %d "
 777             "has invalid dpl %d", vector, gd->gd_dpl));
 778         KASSERT(gd->gd_selector == GSEL(GCODE_SEL, SEL_KPL), ("gate descriptor "
 779             "for vector %d has invalid selector %d", vector, gd->gd_selector));
 780         KASSERT(gd->gd_ist == 0, ("gate descriptor for vector %d has invalid "
 781             "IST %d", vector, gd->gd_ist));
 782
 783         func = ((long)gd->gd_hioffset << 16 | gd->gd_looffset);
 784         vmx_call_isr(func);
 785 }
 786
 787 static int
 788 vmx_setup_cr_shadow(int which, struct vmcs *vmcs, uint32_t initial)
 789 {
 790         int error, mask_ident, shadow_ident;
 791         uint64_t mask_value;
 792
 793         if (which != 0 && which != 4)
 794                 panic("vmx_setup_cr_shadow: unknown cr%d", which);
 795
 796         if (which == 0) {
 797                 mask_ident = VMCS_CR0_MASK;
 798                 mask_value = cr0_ones_mask | cr0_zeros_mask;
 799                 shadow_ident = VMCS_CR0_SHADOW;
 800         } else {
 801                 mask_ident = VMCS_CR4_MASK;
 802                 mask_value = cr4_ones_mask | cr4_zeros_mask;
 803                 shadow_ident = VMCS_CR4_SHADOW;
 804         }
 805
 806         error = vmcs_setreg(vmcs, 0, VMCS_IDENT(mask_ident), mask_value);
 807         if (error)
 808                 return (error);
 809
 810         error = vmcs_setreg(vmcs, 0, VMCS_IDENT(shadow_ident), initial);
 811         if (error)
 812                 return (error);
 813
 814         return (0);
 815 }
 816 #define vmx_setup_cr0_shadow(vmcs,init) vmx_setup_cr_shadow(0, (vmcs), (init))
 817 #define vmx_setup_cr4_shadow(vmcs,init) vmx_setup_cr_shadow(4, (vmcs), (init))
 818
 819 static void *
 820 vmx_vminit(struct vm *vm, pmap_t pmap)
 821 {
 822         uint16_t vpid[VM_MAXCPU];
 823         int i, error;
 824         struct vmx *vmx;
 825         struct vmcs *vmcs;
 826         uint32_t exc_bitmap;
 827
 828         vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO);
 829         if ((uintptr_t)vmx & PAGE_MASK) {
 830                 panic("malloc of struct vmx not aligned on %d byte boundary",
 831                       PAGE_SIZE);
 832         }
 833         vmx->vm = vm;
 834
 835         vmx->eptp = eptp(vtophys((vm_offset_t)pmap->pm_pml4));
 836
 837         /*
 838          * Clean up EPTP-tagged guest physical and combined mappings
 839          *
 840          * VMX transitions are not required to invalidate any guest physical
 841          * mappings. So, it may be possible for stale guest physical mappings
 842          * to be present in the processor TLBs.
 843          *
 844          * Combined mappings for this EP4TA are also invalidated for all VPIDs.
 845          */
 846         ept_invalidate_mappings(vmx->eptp);
 847
 848         msr_bitmap_initialize(vmx->msr_bitmap);
 849
 850         /*
 851          * It is safe to allow direct access to MSR_GSBASE and MSR_FSBASE.
 852          * The guest FSBASE and GSBASE are saved and restored during
 853          * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are
 854          * always restored from the vmcs host state area on vm-exit.
 855          *
 856          * The SYSENTER_CS/ESP/EIP MSRs are identical to FS/GSBASE in
 857          * how they are saved/restored so can be directly accessed by the
 858          * guest.
 859          *
 860          * MSR_EFER is saved and restored in the guest VMCS area on a
 861          * VM exit and entry respectively. It is also restored from the
 862          * host VMCS area on a VM exit.
 863          *
 864          * The TSC MSR is exposed read-only. Writes are disallowed as
 865          * that will impact the host TSC.  If the guest does a write
 866          * the "use TSC offsetting" execution control is enabled and the
 867          * difference between the host TSC and the guest TSC is written
 868          * into the TSC offset in the VMCS.
 869          */
 870         if (guest_msr_rw(vmx, MSR_GSBASE) ||
 871             guest_msr_rw(vmx, MSR_FSBASE) ||
 872             guest_msr_rw(vmx, MSR_SYSENTER_CS_MSR) ||
 873             guest_msr_rw(vmx, MSR_SYSENTER_ESP_MSR) ||
 874             guest_msr_rw(vmx, MSR_SYSENTER_EIP_MSR) ||
 875             guest_msr_rw(vmx, MSR_EFER) ||
 876             guest_msr_ro(vmx, MSR_TSC))
 877                 panic("vmx_vminit: error setting guest msr access");
 878
 879         vpid_alloc(vpid, VM_MAXCPU);
 880
 881         if (virtual_interrupt_delivery) {
 882                 error = vm_map_mmio(vm, DEFAULT_APIC_BASE, PAGE_SIZE,
 883                     APIC_ACCESS_ADDRESS);
 884                 /* XXX this should really return an error to the caller */
 885                 KASSERT(error == 0, ("vm_map_mmio(apicbase) error %d", error));
 886         }
 887
 888         for (i = 0; i < VM_MAXCPU; i++) {
 889                 vmcs = &vmx->vmcs[i];
 890                 vmcs->identifier = vmx_revision();
 891                 error = vmclear(vmcs);
 892                 if (error != 0) {
 893                         panic("vmx_vminit: vmclear error %d on vcpu %d\n",
 894                               error, i);
 895                 }
 896
 897                 vmx_msr_guest_init(vmx, i);
 898
 899                 error = vmcs_init(vmcs);
 900                 KASSERT(error == 0, ("vmcs_init error %d", error));
 901
 902                 VMPTRLD(vmcs);
 903                 error = 0;
 904                 error += vmwrite(VMCS_HOST_RSP, (u_long)&vmx->ctx[i]);
 905                 error += vmwrite(VMCS_EPTP, vmx->eptp);
 906                 error += vmwrite(VMCS_PIN_BASED_CTLS, pinbased_ctls);
 907                 error += vmwrite(VMCS_PRI_PROC_BASED_CTLS, procbased_ctls);
 908                 error += vmwrite(VMCS_SEC_PROC_BASED_CTLS, procbased_ctls2);
 909                 error += vmwrite(VMCS_EXIT_CTLS, exit_ctls);
 910                 error += vmwrite(VMCS_ENTRY_CTLS, entry_ctls);
 911                 error += vmwrite(VMCS_MSR_BITMAP, vtophys(vmx->msr_bitmap));
 912                 error += vmwrite(VMCS_VPID, vpid[i]);
 913
 914                 /* exception bitmap */
 915                 if (vcpu_trace_exceptions(vm, i))
 916                         exc_bitmap = 0xffffffff;
 917                 else
 918                         exc_bitmap = 1 << IDT_MC;
 919                 error += vmwrite(VMCS_EXCEPTION_BITMAP, exc_bitmap);
 920
 921                 vmx->ctx[i].guest_dr6 = 0xffff0ff0;
 922                 error += vmwrite(VMCS_GUEST_DR7, 0x400);
 923
 924                 if (virtual_interrupt_delivery) {
 925                         error += vmwrite(VMCS_APIC_ACCESS, APIC_ACCESS_ADDRESS);
 926                         error += vmwrite(VMCS_VIRTUAL_APIC,
 927                             vtophys(&vmx->apic_page[i]));
 928                         error += vmwrite(VMCS_EOI_EXIT0, 0);
 929                         error += vmwrite(VMCS_EOI_EXIT1, 0);
 930                         error += vmwrite(VMCS_EOI_EXIT2, 0);
 931                         error += vmwrite(VMCS_EOI_EXIT3, 0);
 932                 }
 933                 if (posted_interrupts) {
 934                         error += vmwrite(VMCS_PIR_VECTOR, pirvec);
 935                         error += vmwrite(VMCS_PIR_DESC,
 936                             vtophys(&vmx->pir_desc[i]));
 937                 }
 938                 VMCLEAR(vmcs);
 939                 KASSERT(error == 0, ("vmx_vminit: error customizing the vmcs"));
 940
 941                 vmx->cap[i].set = 0;
 942                 vmx->cap[i].proc_ctls = procbased_ctls;
 943                 vmx->cap[i].proc_ctls2 = procbased_ctls2;
 944
 945                 vmx->state[i].nextrip = ~0;
 946                 vmx->state[i].lastcpu = NOCPU;
 947                 vmx->state[i].vpid = vpid[i];
 948
 949                 /*
 950                  * Set up the CR0/4 shadows, and init the read shadow
 951                  * to the power-on register value from the Intel Sys Arch.
 952                  *  CR0 - 0x60000010
 953                  *  CR4 - 0
 954                  */
 955                 error = vmx_setup_cr0_shadow(vmcs, 0x60000010);
 956                 if (error != 0)
 957                         panic("vmx_setup_cr0_shadow %d", error);
 958
 959                 error = vmx_setup_cr4_shadow(vmcs, 0);
 960                 if (error != 0)
 961                         panic("vmx_setup_cr4_shadow %d", error);
 962
 963                 vmx->ctx[i].pmap = pmap;
 964         }
 965
 966         return (vmx);
 967 }
 968
 969 static int
 970 vmx_handle_cpuid(struct vm *vm, int vcpu, struct vmxctx *vmxctx)
 971 {
 972         int handled, func;
 973
 974         func = vmxctx->guest_rax;
 975
 976         handled = x86_emulate_cpuid(vm, vcpu,
 977                                     (uint32_t*)(&vmxctx->guest_rax),
 978                                     (uint32_t*)(&vmxctx->guest_rbx),
 979                                     (uint32_t*)(&vmxctx->guest_rcx),
 980                                     (uint32_t*)(&vmxctx->guest_rdx));
 981         return (handled);
 982 }
 983
 984 static __inline void
 985 vmx_run_trace(struct vmx *vmx, int vcpu)
 986 {
 987 #ifdef KTR
 988         VCPU_CTR1(vmx->vm, vcpu, "Resume execution at %#lx", vmcs_guest_rip());
 989 #endif
 990 }
 991
 992 static __inline void
 993 vmx_exit_trace(struct vmx *vmx, int vcpu, uint64_t rip, uint32_t exit_reason,
 994                int handled)
 995 {
 996 #ifdef KTR
 997         VCPU_CTR3(vmx->vm, vcpu, "%s %s vmexit at 0x%0lx",
 998                  handled ? "handled" : "unhandled",
 999                  exit_reason_to_str(exit_reason), rip);
1000 #endif
1001 }
1002
1003 static __inline void
1004 vmx_astpending_trace(struct vmx *vmx, int vcpu, uint64_t rip)
1005 {
1006 #ifdef KTR
1007         VCPU_CTR1(vmx->vm, vcpu, "astpending vmexit at 0x%0lx", rip);
1008 #endif
1009 }
1010
1011 static VMM_STAT_INTEL(VCPU_INVVPID_SAVED, "Number of vpid invalidations saved");
1012 static VMM_STAT_INTEL(VCPU_INVVPID_DONE, "Number of vpid invalidations done");
1013
1014 /*
1015  * Invalidate guest mappings identified by its vpid from the TLB.
1016  */
1017 static __inline void
1018 vmx_invvpid(struct vmx *vmx, int vcpu, pmap_t pmap, int running)
1019 {
1020         struct vmxstate *vmxstate;
1021         struct invvpid_desc invvpid_desc;
1022
1023         vmxstate = &vmx->state[vcpu];
1024         if (vmxstate->vpid == 0)
1025                 return;
1026
1027         if (!running) {
1028                 /*
1029                  * Set the 'lastcpu' to an invalid host cpu.
1030                  *
1031                  * This will invalidate TLB entries tagged with the vcpu's
1032                  * vpid the next time it runs via vmx_set_pcpu_defaults().
1033                  */
1034                 vmxstate->lastcpu = NOCPU;
1035                 return;
1036         }
1037
1038         KASSERT(curthread->td_critnest > 0, ("%s: vcpu %d running outside "
1039             "critical section", __func__, vcpu));
1040
1041         /*
1042          * Invalidate all mappings tagged with 'vpid'
1043          *
1044          * We do this because this vcpu was executing on a different host
1045          * cpu when it last ran. We do not track whether it invalidated
1046          * mappings associated with its 'vpid' during that run. So we must
1047          * assume that the mappings associated with 'vpid' on 'curcpu' are
1048          * stale and invalidate them.
1049          *
1050          * Note that we incur this penalty only when the scheduler chooses to
1051          * move the thread associated with this vcpu between host cpus.
1052          *
1053          * Note also that this will invalidate mappings tagged with 'vpid'
1054          * for "all" EP4TAs.
1055          */
1056         if (pmap->pm_eptgen == vmx->eptgen[curcpu]) {
1057                 invvpid_desc._res1 = 0;
1058                 invvpid_desc._res2 = 0;
1059                 invvpid_desc.vpid = vmxstate->vpid;
1060                 invvpid_desc.linear_addr = 0;
1061                 invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc);
1062                 vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_DONE, 1);
1063         } else {
1064                 /*
1065                  * The invvpid can be skipped if an invept is going to
1066                  * be performed before entering the guest. The invept
1067                  * will invalidate combined mappings tagged with
1068                  * 'vmx->eptp' for all vpids.
1069                  */
1070                 vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_SAVED, 1);
1071         }
1072 }
1073
1074 static void
1075 vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu, pmap_t pmap)
1076 {
1077         struct vmxstate *vmxstate;
1078
1079         vmxstate = &vmx->state[vcpu];
1080         if (vmxstate->lastcpu == curcpu)
1081                 return;
1082
1083         vmxstate->lastcpu = curcpu;
1084
1085         vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1);
1086
1087         vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase());
1088         vmcs_write(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase());
1089         vmcs_write(VMCS_HOST_GS_BASE, vmm_get_host_gsbase());
1090         vmx_invvpid(vmx, vcpu, pmap, 1);
1091 }
1092
1093 /*
1094  * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set.
1095  */
1096 CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0);
1097
1098 static void __inline
1099 vmx_set_int_window_exiting(struct vmx *vmx, int vcpu)
1100 {
1101
1102         if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) == 0) {
1103                 vmx->cap[vcpu].proc_ctls |= PROCBASED_INT_WINDOW_EXITING;
1104                 vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1105                 VCPU_CTR0(vmx->vm, vcpu, "Enabling interrupt window exiting");
1106         }
1107 }
1108
1109 static void __inline
1110 vmx_clear_int_window_exiting(struct vmx *vmx, int vcpu)
1111 {
1112
1113         KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0,
1114             ("intr_window_exiting not set: %#x", vmx->cap[vcpu].proc_ctls));
1115         vmx->cap[vcpu].proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING;
1116         vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1117         VCPU_CTR0(vmx->vm, vcpu, "Disabling interrupt window exiting");
1118 }
1119
1120 static void __inline
1121 vmx_set_nmi_window_exiting(struct vmx *vmx, int vcpu)
1122 {
1123
1124         if ((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) == 0) {
1125                 vmx->cap[vcpu].proc_ctls |= PROCBASED_NMI_WINDOW_EXITING;
1126                 vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1127                 VCPU_CTR0(vmx->vm, vcpu, "Enabling NMI window exiting");
1128         }
1129 }
1130
1131 static void __inline
1132 vmx_clear_nmi_window_exiting(struct vmx *vmx, int vcpu)
1133 {
1134
1135         KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) != 0,
1136             ("nmi_window_exiting not set %#x", vmx->cap[vcpu].proc_ctls));
1137         vmx->cap[vcpu].proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING;
1138         vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1139         VCPU_CTR0(vmx->vm, vcpu, "Disabling NMI window exiting");
1140 }
1141
1142 int
1143 vmx_set_tsc_offset(struct vmx *vmx, int vcpu, uint64_t offset)
1144 {
1145         int error;
1146
1147         if ((vmx->cap[vcpu].proc_ctls & PROCBASED_TSC_OFFSET) == 0) {
1148                 vmx->cap[vcpu].proc_ctls |= PROCBASED_TSC_OFFSET;
1149                 vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1150                 VCPU_CTR0(vmx->vm, vcpu, "Enabling TSC offsetting");
1151         }
1152
1153         error = vmwrite(VMCS_TSC_OFFSET, offset);
1154
1155         return (error);
1156 }
1157
1158 #define NMI_BLOCKING    (VMCS_INTERRUPTIBILITY_NMI_BLOCKING |           \
1159                          VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
1160 #define HWINTR_BLOCKING (VMCS_INTERRUPTIBILITY_STI_BLOCKING |           \
1161                          VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
1162
1163 static void
1164 vmx_inject_nmi(struct vmx *vmx, int vcpu)
1165 {
1166         uint32_t gi, info;
1167
1168         gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1169         KASSERT((gi & NMI_BLOCKING) == 0, ("vmx_inject_nmi: invalid guest "
1170             "interruptibility-state %#x", gi));
1171
1172         info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1173         KASSERT((info & VMCS_INTR_VALID) == 0, ("vmx_inject_nmi: invalid "
1174             "VM-entry interruption information %#x", info));
1175
1176         /*
1177          * Inject the virtual NMI. The vector must be the NMI IDT entry
1178          * or the VMCS entry check will fail.
1179          */
1180         info = IDT_NMI | VMCS_INTR_T_NMI | VMCS_INTR_VALID;
1181         vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1182
1183         VCPU_CTR0(vmx->vm, vcpu, "Injecting vNMI");
1184
1185         /* Clear the request */
1186         vm_nmi_clear(vmx->vm, vcpu);
1187 }
1188
1189 static void
1190 vmx_inject_interrupts(struct vmx *vmx, int vcpu, struct vlapic *vlapic,
1191     uint64_t guestrip)
1192 {
1193         int vector, need_nmi_exiting, extint_pending;
1194         uint64_t rflags, entryinfo;
1195         uint32_t gi, info;
1196
1197         if (vmx->state[vcpu].nextrip != guestrip) {
1198                 gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1199                 if (gi & HWINTR_BLOCKING) {
1200                         VCPU_CTR2(vmx->vm, vcpu, "Guest interrupt blocking "
1201                             "cleared due to rip change: %#lx/%#lx",
1202                             vmx->state[vcpu].nextrip, guestrip);
1203                         gi &= ~HWINTR_BLOCKING;
1204                         vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1205                 }
1206         }
1207
1208         if (vm_entry_intinfo(vmx->vm, vcpu, &entryinfo)) {
1209                 KASSERT((entryinfo & VMCS_INTR_VALID) != 0, ("%s: entry "
1210                     "intinfo is not valid: %#lx", __func__, entryinfo));
1211
1212                 info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1213                 KASSERT((info & VMCS_INTR_VALID) == 0, ("%s: cannot inject "
1214                      "pending exception: %#lx/%#x", __func__, entryinfo, info));
1215
1216                 info = entryinfo;
1217                 vector = info & 0xff;
1218                 if (vector == IDT_BP || vector == IDT_OF) {
1219                         /*
1220                          * VT-x requires #BP and #OF to be injected as software
1221                          * exceptions.
1222                          */
1223                         info &= ~VMCS_INTR_T_MASK;
1224                         info |= VMCS_INTR_T_SWEXCEPTION;
1225                 }
1226
1227                 if (info & VMCS_INTR_DEL_ERRCODE)
1228                         vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR, entryinfo >> 32);
1229
1230                 vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1231         }
1232
1233         if (vm_nmi_pending(vmx->vm, vcpu)) {
1234                 /*
1235                  * If there are no conditions blocking NMI injection then
1236                  * inject it directly here otherwise enable "NMI window
1237                  * exiting" to inject it as soon as we can.
1238                  *
1239                  * We also check for STI_BLOCKING because some implementations
1240                  * don't allow NMI injection in this case. If we are running
1241                  * on a processor that doesn't have this restriction it will
1242                  * immediately exit and the NMI will be injected in the
1243                  * "NMI window exiting" handler.
1244                  */
1245                 need_nmi_exiting = 1;
1246                 gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1247                 if ((gi & (HWINTR_BLOCKING | NMI_BLOCKING)) == 0) {
1248                         info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1249                         if ((info & VMCS_INTR_VALID) == 0) {
1250                                 vmx_inject_nmi(vmx, vcpu);
1251                                 need_nmi_exiting = 0;
1252                         } else {
1253                                 VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI "
1254                                     "due to VM-entry intr info %#x", info);
1255                         }
1256                 } else {
1257                         VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI due to "
1258                             "Guest Interruptibility-state %#x", gi);
1259                 }
1260
1261                 if (need_nmi_exiting)
1262                         vmx_set_nmi_window_exiting(vmx, vcpu);
1263         }
1264
1265         extint_pending = vm_extint_pending(vmx->vm, vcpu);
1266
1267         if (!extint_pending && virtual_interrupt_delivery) {
1268                 vmx_inject_pir(vlapic);
1269                 return;
1270         }
1271
1272         /*
1273          * If interrupt-window exiting is already in effect then don't bother
1274          * checking for pending interrupts. This is just an optimization and
1275          * not needed for correctness.
1276          */
1277         if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0) {
1278                 VCPU_CTR0(vmx->vm, vcpu, "Skip interrupt injection due to "
1279                     "pending int_window_exiting");
1280                 return;
1281         }
1282
1283         if (!extint_pending) {
1284                 /* Ask the local apic for a vector to inject */
1285                 if (!vlapic_pending_intr(vlapic, &vector))
1286                         return;
1287
1288                 /*
1289                  * From the Intel SDM, Volume 3, Section "Maskable
1290                  * Hardware Interrupts":
1291                  * - maskable interrupt vectors [16,255] can be delivered
1292                  *   through the local APIC.
1293                 */
1294                 KASSERT(vector >= 16 && vector <= 255,
1295                     ("invalid vector %d from local APIC", vector));
1296         } else {
1297                 /* Ask the legacy pic for a vector to inject */
1298                 vatpic_pending_intr(vmx->vm, &vector);
1299
1300                 /*
1301                  * From the Intel SDM, Volume 3, Section "Maskable
1302                  * Hardware Interrupts":
1303                  * - maskable interrupt vectors [0,255] can be delivered
1304                  *   through the INTR pin.
1305                  */
1306                 KASSERT(vector >= 0 && vector <= 255,
1307                     ("invalid vector %d from INTR", vector));
1308         }
1309
1310         /* Check RFLAGS.IF and the interruptibility state of the guest */
1311         rflags = vmcs_read(VMCS_GUEST_RFLAGS);
1312         if ((rflags & PSL_I) == 0) {
1313                 VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
1314                     "rflags %#lx", vector, rflags);
1315                 goto cantinject;
1316         }
1317
1318         gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1319         if (gi & HWINTR_BLOCKING) {
1320                 VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
1321                     "Guest Interruptibility-state %#x", vector, gi);
1322                 goto cantinject;
1323         }
1324
1325         info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1326         if (info & VMCS_INTR_VALID) {
1327                 /*
1328                  * This is expected and could happen for multiple reasons:
1329                  * - A vectoring VM-entry was aborted due to astpending
1330                  * - A VM-exit happened during event injection.
1331                  * - An exception was injected above.
1332                  * - An NMI was injected above or after "NMI window exiting"
1333                  */
1334                 VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
1335                     "VM-entry intr info %#x", vector, info);
1336                 goto cantinject;
1337         }
1338
1339         /* Inject the interrupt */
1340         info = VMCS_INTR_T_HWINTR | VMCS_INTR_VALID;
1341         info |= vector;
1342         vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1343
1344         if (!extint_pending) {
1345                 /* Update the Local APIC ISR */
1346                 vlapic_intr_accepted(vlapic, vector);
1347         } else {
1348                 vm_extint_clear(vmx->vm, vcpu);
1349                 vatpic_intr_accepted(vmx->vm, vector);
1350
1351                 /*
1352                  * After we accepted the current ExtINT the PIC may
1353                  * have posted another one.  If that is the case, set
1354                  * the Interrupt Window Exiting execution control so
1355                  * we can inject that one too.
1356                  *
1357                  * Also, interrupt window exiting allows us to inject any
1358                  * pending APIC vector that was preempted by the ExtINT
1359                  * as soon as possible. This applies both for the software
1360                  * emulated vlapic and the hardware assisted virtual APIC.
1361                  */
1362                 vmx_set_int_window_exiting(vmx, vcpu);
1363         }
1364
1365         VCPU_CTR1(vmx->vm, vcpu, "Injecting hwintr at vector %d", vector);
1366
1367         return;
1368
1369 cantinject:
1370         /*
1371          * Set the Interrupt Window Exiting execution control so we can inject
1372          * the interrupt as soon as blocking condition goes away.
1373          */
1374         vmx_set_int_window_exiting(vmx, vcpu);
1375 }
1376
1377 /*
1378  * If the Virtual NMIs execution control is '1' then the logical processor
1379  * tracks virtual-NMI blocking in the Guest Interruptibility-state field of
1380  * the VMCS. An IRET instruction in VMX non-root operation will remove any
1381  * virtual-NMI blocking.
1382  *
1383  * This unblocking occurs even if the IRET causes a fault. In this case the
1384  * hypervisor needs to restore virtual-NMI blocking before resuming the guest.
1385  */
1386 static void
1387 vmx_restore_nmi_blocking(struct vmx *vmx, int vcpuid)
1388 {
1389         uint32_t gi;
1390
1391         VCPU_CTR0(vmx->vm, vcpuid, "Restore Virtual-NMI blocking");
1392         gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1393         gi |= VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
1394         vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1395 }
1396
1397 static void
1398 vmx_clear_nmi_blocking(struct vmx *vmx, int vcpuid)
1399 {
1400         uint32_t gi;
1401
1402         VCPU_CTR0(vmx->vm, vcpuid, "Clear Virtual-NMI blocking");
1403         gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1404         gi &= ~VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
1405         vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1406 }
1407
1408 static void
1409 vmx_assert_nmi_blocking(struct vmx *vmx, int vcpuid)
1410 {
1411         uint32_t gi;
1412
1413         gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1414         KASSERT(gi & VMCS_INTERRUPTIBILITY_NMI_BLOCKING,
1415             ("NMI blocking is not in effect %#x", gi));
1416 }
1417
1418 static int
1419 vmx_emulate_xsetbv(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
1420 {
1421         struct vmxctx *vmxctx;
1422         uint64_t xcrval;
1423         const struct xsave_limits *limits;
1424
1425         vmxctx = &vmx->ctx[vcpu];
1426         limits = vmm_get_xsave_limits();
1427
1428         /*
1429          * Note that the processor raises a GP# fault on its own if
1430          * xsetbv is executed for CPL != 0, so we do not have to
1431          * emulate that fault here.
1432          */
1433
1434         /* Only xcr0 is supported. */
1435         if (vmxctx->guest_rcx != 0) {
1436                 vm_inject_gp(vmx->vm, vcpu);
1437                 return (HANDLED);
1438         }
1439
1440         /* We only handle xcr0 if both the host and guest have XSAVE enabled. */
1441         if (!limits->xsave_enabled || !(vmcs_read(VMCS_GUEST_CR4) & CR4_XSAVE)) {
1442                 vm_inject_ud(vmx->vm, vcpu);
1443                 return (HANDLED);
1444         }
1445
1446         xcrval = vmxctx->guest_rdx << 32 | (vmxctx->guest_rax & 0xffffffff);
1447         if ((xcrval & ~limits->xcr0_allowed) != 0) {
1448                 vm_inject_gp(vmx->vm, vcpu);
1449                 return (HANDLED);
1450         }
1451
1452         if (!(xcrval & XFEATURE_ENABLED_X87)) {
1453                 vm_inject_gp(vmx->vm, vcpu);
1454                 return (HANDLED);
1455         }
1456
1457         /* AVX (YMM_Hi128) requires SSE. */
1458         if (xcrval & XFEATURE_ENABLED_AVX &&
1459             (xcrval & XFEATURE_AVX) != XFEATURE_AVX) {
1460                 vm_inject_gp(vmx->vm, vcpu);
1461                 return (HANDLED);
1462         }
1463
1464         /*
1465          * AVX512 requires base AVX (YMM_Hi128) as well as OpMask,
1466          * ZMM_Hi256, and Hi16_ZMM.
1467          */
1468         if (xcrval & XFEATURE_AVX512 &&
1469             (xcrval & (XFEATURE_AVX512 | XFEATURE_AVX)) !=
1470             (XFEATURE_AVX512 | XFEATURE_AVX)) {
1471                 vm_inject_gp(vmx->vm, vcpu);
1472                 return (HANDLED);
1473         }
1474
1475         /*
1476          * Intel MPX requires both bound register state flags to be
1477          * set.
1478          */
1479         if (((xcrval & XFEATURE_ENABLED_BNDREGS) != 0) !=
1480             ((xcrval & XFEATURE_ENABLED_BNDCSR) != 0)) {
1481                 vm_inject_gp(vmx->vm, vcpu);
1482                 return (HANDLED);
1483         }
1484
1485         /*
1486          * This runs "inside" vmrun() with the guest's FPU state, so
1487          * modifying xcr0 directly modifies the guest's xcr0, not the
1488          * host's.
1489          */
1490         load_xcr(0, xcrval);
1491         return (HANDLED);
1492 }
1493
1494 static uint64_t
1495 vmx_get_guest_reg(struct vmx *vmx, int vcpu, int ident)
1496 {
1497         const struct vmxctx *vmxctx;
1498
1499         vmxctx = &vmx->ctx[vcpu];
1500
1501         switch (ident) {
1502         case 0:
1503                 return (vmxctx->guest_rax);
1504         case 1:
1505                 return (vmxctx->guest_rcx);
1506         case 2:
1507                 return (vmxctx->guest_rdx);
1508         case 3:
1509                 return (vmxctx->guest_rbx);
1510         case 4:
1511                 return (vmcs_read(VMCS_GUEST_RSP));
1512         case 5:
1513                 return (vmxctx->guest_rbp);
1514         case 6:
1515                 return (vmxctx->guest_rsi);
1516         case 7:
1517                 return (vmxctx->guest_rdi);
1518         case 8:
1519                 return (vmxctx->guest_r8);
1520         case 9:
1521                 return (vmxctx->guest_r9);
1522         case 10:
1523                 return (vmxctx->guest_r10);
1524         case 11:
1525                 return (vmxctx->guest_r11);
1526         case 12:
1527                 return (vmxctx->guest_r12);
1528         case 13:
1529                 return (vmxctx->guest_r13);
1530         case 14:
1531                 return (vmxctx->guest_r14);
1532         case 15:
1533                 return (vmxctx->guest_r15);
1534         default:
1535                 panic("invalid vmx register %d", ident);
1536         }
1537 }
1538
1539 static void
1540 vmx_set_guest_reg(struct vmx *vmx, int vcpu, int ident, uint64_t regval)
1541 {
1542         struct vmxctx *vmxctx;
1543
1544         vmxctx = &vmx->ctx[vcpu];
1545
1546         switch (ident) {
1547         case 0:
1548                 vmxctx->guest_rax = regval;
1549                 break;
1550         case 1:
1551                 vmxctx->guest_rcx = regval;
1552                 break;
1553         case 2:
1554                 vmxctx->guest_rdx = regval;
1555                 break;
1556         case 3:
1557                 vmxctx->guest_rbx = regval;
1558                 break;
1559         case 4:
1560                 vmcs_write(VMCS_GUEST_RSP, regval);
1561                 break;
1562         case 5:
1563                 vmxctx->guest_rbp = regval;
1564                 break;
1565         case 6:
1566                 vmxctx->guest_rsi = regval;
1567                 break;
1568         case 7:
1569                 vmxctx->guest_rdi = regval;
1570                 break;
1571         case 8:
1572                 vmxctx->guest_r8 = regval;
1573                 break;
1574         case 9:
1575                 vmxctx->guest_r9 = regval;
1576                 break;
1577         case 10:
1578                 vmxctx->guest_r10 = regval;
1579                 break;
1580         case 11:
1581                 vmxctx->guest_r11 = regval;
1582                 break;
1583         case 12:
1584                 vmxctx->guest_r12 = regval;
1585                 break;
1586         case 13:
1587                 vmxctx->guest_r13 = regval;
1588                 break;
1589         case 14:
1590                 vmxctx->guest_r14 = regval;
1591                 break;
1592         case 15:
1593                 vmxctx->guest_r15 = regval;
1594                 break;
1595         default:
1596                 panic("invalid vmx register %d", ident);
1597         }
1598 }
1599
1600 static int
1601 vmx_emulate_cr0_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
1602 {
1603         uint64_t crval, regval;
1604
1605         /* We only handle mov to %cr0 at this time */
1606         if ((exitqual & 0xf0) != 0x00)
1607                 return (UNHANDLED);
1608
1609         regval = vmx_get_guest_reg(vmx, vcpu, (exitqual >> 8) & 0xf);
1610
1611         vmcs_write(VMCS_CR0_SHADOW, regval);
1612
1613         crval = regval | cr0_ones_mask;
1614         crval &= ~cr0_zeros_mask;
1615         vmcs_write(VMCS_GUEST_CR0, crval);
1616
1617         if (regval & CR0_PG) {
1618                 uint64_t efer, entry_ctls;
1619
1620                 /*
1621                  * If CR0.PG is 1 and EFER.LME is 1 then EFER.LMA and
1622                  * the "IA-32e mode guest" bit in VM-entry control must be
1623                  * equal.
1624                  */
1625                 efer = vmcs_read(VMCS_GUEST_IA32_EFER);
1626                 if (efer & EFER_LME) {
1627                         efer |= EFER_LMA;
1628                         vmcs_write(VMCS_GUEST_IA32_EFER, efer);
1629                         entry_ctls = vmcs_read(VMCS_ENTRY_CTLS);
1630                         entry_ctls |= VM_ENTRY_GUEST_LMA;
1631                         vmcs_write(VMCS_ENTRY_CTLS, entry_ctls);
1632                 }
1633         }
1634
1635         return (HANDLED);
1636 }
1637
1638 static int
1639 vmx_emulate_cr4_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
1640 {
1641         uint64_t crval, regval;
1642
1643         /* We only handle mov to %cr4 at this time */
1644         if ((exitqual & 0xf0) != 0x00)
1645                 return (UNHANDLED);
1646
1647         regval = vmx_get_guest_reg(vmx, vcpu, (exitqual >> 8) & 0xf);
1648
1649         vmcs_write(VMCS_CR4_SHADOW, regval);
1650
1651         crval = regval | cr4_ones_mask;
1652         crval &= ~cr4_zeros_mask;
1653         vmcs_write(VMCS_GUEST_CR4, crval);
1654
1655         return (HANDLED);
1656 }
1657
1658 static int
1659 vmx_emulate_cr8_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
1660 {
1661         struct vlapic *vlapic;
1662         uint64_t cr8;
1663         int regnum;
1664
1665         /* We only handle mov %cr8 to/from a register at this time. */
1666         if ((exitqual & 0xe0) != 0x00) {
1667                 return (UNHANDLED);
1668         }
1669
1670         vlapic = vm_lapic(vmx->vm, vcpu);
1671         regnum = (exitqual >> 8) & 0xf;
1672         if (exitqual & 0x10) {
1673                 cr8 = vlapic_get_cr8(vlapic);
1674                 vmx_set_guest_reg(vmx, vcpu, regnum, cr8);
1675         } else {
1676                 cr8 = vmx_get_guest_reg(vmx, vcpu, regnum);
1677                 vlapic_set_cr8(vlapic, cr8);
1678         }
1679
1680         return (HANDLED);
1681 }
1682
1683 /*
1684  * From section "Guest Register State" in the Intel SDM: CPL = SS.DPL
1685  */
1686 static int
1687 vmx_cpl(void)
1688 {
1689         uint32_t ssar;
1690
1691         ssar = vmcs_read(VMCS_GUEST_SS_ACCESS_RIGHTS);
1692         return ((ssar >> 5) & 0x3);
1693 }
1694
1695 static enum vm_cpu_mode
1696 vmx_cpu_mode(void)
1697 {
1698         uint32_t csar;
1699
1700         if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LMA) {
1701                 csar = vmcs_read(VMCS_GUEST_CS_ACCESS_RIGHTS);
1702                 if (csar & 0x2000)
1703                         return (CPU_MODE_64BIT);        /* CS.L = 1 */
1704                 else
1705                         return (CPU_MODE_COMPATIBILITY);
1706         } else if (vmcs_read(VMCS_GUEST_CR0) & CR0_PE) {
1707                 return (CPU_MODE_PROTECTED);
1708         } else {
1709                 return (CPU_MODE_REAL);
1710         }
1711 }
1712
1713 static enum vm_paging_mode
1714 vmx_paging_mode(void)
1715 {
1716
1717         if (!(vmcs_read(VMCS_GUEST_CR0) & CR0_PG))
1718                 return (PAGING_MODE_FLAT);
1719         if (!(vmcs_read(VMCS_GUEST_CR4) & CR4_PAE))
1720                 return (PAGING_MODE_32);
1721         if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LME)
1722                 return (PAGING_MODE_64);
1723         else
1724                 return (PAGING_MODE_PAE);
1725 }
1726
1727 static uint64_t
1728 inout_str_index(struct vmx *vmx, int vcpuid, int in)
1729 {
1730         uint64_t val;
1731         int error;
1732         enum vm_reg_name reg;
1733
1734         reg = in ? VM_REG_GUEST_RDI : VM_REG_GUEST_RSI;
1735         error = vmx_getreg(vmx, vcpuid, reg, &val);
1736         KASSERT(error == 0, ("%s: vmx_getreg error %d", __func__, error));
1737         return (val);
1738 }
1739
1740 static uint64_t
1741 inout_str_count(struct vmx *vmx, int vcpuid, int rep)
1742 {
1743         uint64_t val;
1744         int error;
1745
1746         if (rep) {
1747                 error = vmx_getreg(vmx, vcpuid, VM_REG_GUEST_RCX, &val);
1748                 KASSERT(!error, ("%s: vmx_getreg error %d", __func__, error));
1749         } else {
1750                 val = 1;
1751         }
1752         return (val);
1753 }
1754
1755 static int
1756 inout_str_addrsize(uint32_t inst_info)
1757 {
1758         uint32_t size;
1759
1760         size = (inst_info >> 7) & 0x7;
1761         switch (size) {
1762         case 0:
1763                 return (2);     /* 16 bit */
1764         case 1:
1765                 return (4);     /* 32 bit */
1766         case 2:
1767                 return (8);     /* 64 bit */
1768         default:
1769                 panic("%s: invalid size encoding %d", __func__, size);
1770         }
1771 }
1772
1773 static void
1774 inout_str_seginfo(struct vmx *vmx, int vcpuid, uint32_t inst_info, int in,
1775     struct vm_inout_str *vis)
1776 {
1777         int error, s;
1778
1779         if (in) {
1780                 vis->seg_name = VM_REG_GUEST_ES;
1781         } else {
1782                 s = (inst_info >> 15) & 0x7;
1783                 vis->seg_name = vm_segment_name(s);
1784         }
1785
1786         error = vmx_getdesc(vmx, vcpuid, vis->seg_name, &vis->seg_desc);
1787         KASSERT(error == 0, ("%s: vmx_getdesc error %d", __func__, error));
1788 }
1789
1790 static void
1791 vmx_paging_info(struct vm_guest_paging *paging)
1792 {
1793         paging->cr3 = vmcs_guest_cr3();
1794         paging->cpl = vmx_cpl();
1795         paging->cpu_mode = vmx_cpu_mode();
1796         paging->paging_mode = vmx_paging_mode();
1797 }
1798
1799 static void
1800 vmexit_inst_emul(struct vm_exit *vmexit, uint64_t gpa, uint64_t gla)
1801 {
1802         struct vm_guest_paging *paging;
1803         uint32_t csar;
1804
1805         paging = &vmexit->u.inst_emul.paging;
1806
1807         vmexit->exitcode = VM_EXITCODE_INST_EMUL;
1808         vmexit->inst_length = 0;
1809         vmexit->u.inst_emul.gpa = gpa;
1810         vmexit->u.inst_emul.gla = gla;
1811         vmx_paging_info(paging);
1812         switch (paging->cpu_mode) {
1813         case CPU_MODE_REAL:
1814                 vmexit->u.inst_emul.cs_base = vmcs_read(VMCS_GUEST_CS_BASE);
1815                 vmexit->u.inst_emul.cs_d = 0;
1816                 break;
1817         case CPU_MODE_PROTECTED:
1818         case CPU_MODE_COMPATIBILITY:
1819                 vmexit->u.inst_emul.cs_base = vmcs_read(VMCS_GUEST_CS_BASE);
1820                 csar = vmcs_read(VMCS_GUEST_CS_ACCESS_RIGHTS);
1821                 vmexit->u.inst_emul.cs_d = SEG_DESC_DEF32(csar);
1822                 break;
1823         default:
1824                 vmexit->u.inst_emul.cs_base = 0;
1825                 vmexit->u.inst_emul.cs_d = 0;
1826                 break;
1827         }
1828         vie_init(&vmexit->u.inst_emul.vie, NULL, 0);
1829 }
1830
1831 static int
1832 ept_fault_type(uint64_t ept_qual)
1833 {
1834         int fault_type;
1835
1836         if (ept_qual & EPT_VIOLATION_DATA_WRITE)
1837                 fault_type = VM_PROT_WRITE;
1838         else if (ept_qual & EPT_VIOLATION_INST_FETCH)
1839                 fault_type = VM_PROT_EXECUTE;
1840         else
1841                 fault_type= VM_PROT_READ;
1842
1843         return (fault_type);
1844 }
1845
1846 static boolean_t
1847 ept_emulation_fault(uint64_t ept_qual)
1848 {
1849         int read, write;
1850
1851         /* EPT fault on an instruction fetch doesn't make sense here */
1852         if (ept_qual & EPT_VIOLATION_INST_FETCH)
1853                 return (FALSE);
1854
1855         /* EPT fault must be a read fault or a write fault */
1856         read = ept_qual & EPT_VIOLATION_DATA_READ ? 1 : 0;
1857         write = ept_qual & EPT_VIOLATION_DATA_WRITE ? 1 : 0;
1858         if ((read | write) == 0)
1859                 return (FALSE);
1860
1861         /*
1862          * The EPT violation must have been caused by accessing a
1863          * guest-physical address that is a translation of a guest-linear
1864          * address.
1865          */
1866         if ((ept_qual & EPT_VIOLATION_GLA_VALID) == 0 ||
1867             (ept_qual & EPT_VIOLATION_XLAT_VALID) == 0) {
1868                 return (FALSE);
1869         }
1870
1871         return (TRUE);
1872 }
1873
1874 static __inline int
1875 apic_access_virtualization(struct vmx *vmx, int vcpuid)
1876 {
1877         uint32_t proc_ctls2;
1878
1879         proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
1880         return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) ? 1 : 0);
1881 }
1882
1883 static __inline int
1884 x2apic_virtualization(struct vmx *vmx, int vcpuid)
1885 {
1886         uint32_t proc_ctls2;
1887
1888         proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
1889         return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_X2APIC_MODE) ? 1 : 0);
1890 }
1891
1892 static int
1893 vmx_handle_apic_write(struct vmx *vmx, int vcpuid, struct vlapic *vlapic,
1894     uint64_t qual)
1895 {
1896         int error, handled, offset;
1897         uint32_t *apic_regs, vector;
1898         bool retu;
1899
1900         handled = HANDLED;
1901         offset = APIC_WRITE_OFFSET(qual);
1902
1903         if (!apic_access_virtualization(vmx, vcpuid)) {
1904                 /*
1905                  * In general there should not be any APIC write VM-exits
1906                  * unless APIC-access virtualization is enabled.
1907                  *
1908                  * However self-IPI virtualization can legitimately trigger
1909                  * an APIC-write VM-exit so treat it specially.
1910                  */
1911                 if (x2apic_virtualization(vmx, vcpuid) &&
1912                     offset == APIC_OFFSET_SELF_IPI) {
1913                         apic_regs = (uint32_t *)(vlapic->apic_page);
1914                         vector = apic_regs[APIC_OFFSET_SELF_IPI / 4];
1915                         vlapic_self_ipi_handler(vlapic, vector);
1916                         return (HANDLED);
1917                 } else
1918                         return (UNHANDLED);
1919         }
1920
1921         switch (offset) {
1922         case APIC_OFFSET_ID:
1923                 vlapic_id_write_handler(vlapic);
1924                 break;
1925         case APIC_OFFSET_LDR:
1926                 vlapic_ldr_write_handler(vlapic);
1927                 break;
1928         case APIC_OFFSET_DFR:
1929                 vlapic_dfr_write_handler(vlapic);
1930                 break;
1931         case APIC_OFFSET_SVR:
1932                 vlapic_svr_write_handler(vlapic);
1933                 break;
1934         case APIC_OFFSET_ESR:
1935                 vlapic_esr_write_handler(vlapic);
1936                 break;
1937         case APIC_OFFSET_ICR_LOW:
1938                 retu = false;
1939                 error = vlapic_icrlo_write_handler(vlapic, &retu);
1940                 if (error != 0 || retu)
1941                         handled = UNHANDLED;
1942                 break;
1943         case APIC_OFFSET_CMCI_LVT:
1944         case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
1945                 vlapic_lvt_write_handler(vlapic, offset);
1946                 break;
1947         case APIC_OFFSET_TIMER_ICR:
1948                 vlapic_icrtmr_write_handler(vlapic);
1949                 break;
1950         case APIC_OFFSET_TIMER_DCR:
1951                 vlapic_dcr_write_handler(vlapic);
1952                 break;
1953         default:
1954                 handled = UNHANDLED;
1955                 break;
1956         }
1957         return (handled);
1958 }
1959
1960 static bool
1961 apic_access_fault(struct vmx *vmx, int vcpuid, uint64_t gpa)
1962 {
1963
1964         if (apic_access_virtualization(vmx, vcpuid) &&
1965             (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE))
1966                 return (true);
1967         else
1968                 return (false);
1969 }
1970
1971 static int
1972 vmx_handle_apic_access(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit)
1973 {
1974         uint64_t qual;
1975         int access_type, offset, allowed;
1976
1977         if (!apic_access_virtualization(vmx, vcpuid))
1978                 return (UNHANDLED);
1979
1980         qual = vmexit->u.vmx.exit_qualification;
1981         access_type = APIC_ACCESS_TYPE(qual);
1982         offset = APIC_ACCESS_OFFSET(qual);
1983
1984         allowed = 0;
1985         if (access_type == 0) {
1986                 /*
1987                  * Read data access to the following registers is expected.
1988                  */
1989                 switch (offset) {
1990                 case APIC_OFFSET_APR:
1991                 case APIC_OFFSET_PPR:
1992                 case APIC_OFFSET_RRR:
1993                 case APIC_OFFSET_CMCI_LVT:
1994                 case APIC_OFFSET_TIMER_CCR:
1995                         allowed = 1;
1996                         break;
1997                 default:
1998                         break;
1999                 }
2000         } else if (access_type == 1) {
2001                 /*
2002                  * Write data access to the following registers is expected.
2003                  */
2004                 switch (offset) {
2005                 case APIC_OFFSET_VER:
2006                 case APIC_OFFSET_APR:
2007                 case APIC_OFFSET_PPR:
2008                 case APIC_OFFSET_RRR:
2009                 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
2010                 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
2011                 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
2012                 case APIC_OFFSET_CMCI_LVT:
2013                 case APIC_OFFSET_TIMER_CCR:
2014                         allowed = 1;
2015                         break;
2016                 default:
2017                         break;
2018                 }
2019         }
2020
2021         if (allowed) {
2022                 vmexit_inst_emul(vmexit, DEFAULT_APIC_BASE + offset,
2023                     VIE_INVALID_GLA);
2024         }
2025
2026         /*
2027          * Regardless of whether the APIC-access is allowed this handler
2028          * always returns UNHANDLED:
2029          * - if the access is allowed then it is handled by emulating the
2030          *   instruction that caused the VM-exit (outside the critical section)
2031          * - if the access is not allowed then it will be converted to an
2032          *   exitcode of VM_EXITCODE_VMX and will be dealt with in userland.
2033          */
2034         return (UNHANDLED);
2035 }
2036
2037 static enum task_switch_reason
2038 vmx_task_switch_reason(uint64_t qual)
2039 {
2040         int reason;
2041
2042         reason = (qual >> 30) & 0x3;
2043         switch (reason) {
2044         case 0:
2045                 return (TSR_CALL);
2046         case 1:
2047                 return (TSR_IRET);
2048         case 2:
2049                 return (TSR_JMP);
2050         case 3:
2051                 return (TSR_IDT_GATE);
2052         default:
2053                 panic("%s: invalid reason %d", __func__, reason);
2054         }
2055 }
2056
2057 static int
2058 emulate_wrmsr(struct vmx *vmx, int vcpuid, u_int num, uint64_t val, bool *retu)
2059 {
2060         int error;
2061
2062         if (lapic_msr(num))
2063                 error = lapic_wrmsr(vmx->vm, vcpuid, num, val, retu);
2064         else
2065                 error = vmx_wrmsr(vmx, vcpuid, num, val, retu);
2066
2067         return (error);
2068 }
2069
2070 static int
2071 emulate_rdmsr(struct vmx *vmx, int vcpuid, u_int num, bool *retu)
2072 {
2073         struct vmxctx *vmxctx;
2074         uint64_t result;
2075         uint32_t eax, edx;
2076         int error;
2077
2078         if (lapic_msr(num))
2079                 error = lapic_rdmsr(vmx->vm, vcpuid, num, &result, retu);
2080         else
2081                 error = vmx_rdmsr(vmx, vcpuid, num, &result, retu);
2082
2083         if (error == 0) {
2084                 eax = result;
2085                 vmxctx = &vmx->ctx[vcpuid];
2086                 error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RAX, eax);
2087                 KASSERT(error == 0, ("vmxctx_setreg(rax) error %d", error));
2088
2089                 edx = result >> 32;
2090                 error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RDX, edx);
2091                 KASSERT(error == 0, ("vmxctx_setreg(rdx) error %d", error));
2092         }
2093
2094         return (error);
2095 }
2096
2097 static int
2098 vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
2099 {
2100         int error, errcode, errcode_valid, handled, in;
2101         struct vmxctx *vmxctx;
2102         struct vlapic *vlapic;
2103         struct vm_inout_str *vis;
2104         struct vm_task_switch *ts;
2105         uint32_t eax, ecx, edx, idtvec_info, idtvec_err, intr_info, inst_info;
2106         uint32_t intr_type, intr_vec, reason;
2107         uint64_t exitintinfo, qual, gpa;
2108         bool retu;
2109
2110         CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_VIRTUAL_NMI) != 0);
2111         CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_NMI_EXITING) != 0);
2112
2113         handled = UNHANDLED;
2114         vmxctx = &vmx->ctx[vcpu];
2115
2116         qual = vmexit->u.vmx.exit_qualification;
2117         reason = vmexit->u.vmx.exit_reason;
2118         vmexit->exitcode = VM_EXITCODE_BOGUS;
2119
2120         vmm_stat_incr(vmx->vm, vcpu, VMEXIT_COUNT, 1);
2121
2122         /*
2123          * VM-entry failures during or after loading guest state.
2124          *
2125          * These VM-exits are uncommon but must be handled specially
2126          * as most VM-exit fields are not populated as usual.
2127          */
2128         if (__predict_false(reason == EXIT_REASON_MCE_DURING_ENTRY)) {
2129                 VCPU_CTR0(vmx->vm, vcpu, "Handling MCE during VM-entry");
2130                 __asm __volatile("int $18");
2131                 return (1);
2132         }
2133
2134         /*
2135          * VM exits that can be triggered during event delivery need to
2136          * be handled specially by re-injecting the event if the IDT
2137          * vectoring information field's valid bit is set.
2138          *
2139          * See "Information for VM Exits During Event Delivery" in Intel SDM
2140          * for details.
2141          */
2142         idtvec_info = vmcs_idt_vectoring_info();
2143         if (idtvec_info & VMCS_IDT_VEC_VALID) {
2144                 idtvec_info &= ~(1 << 12); /* clear undefined bit */
2145                 exitintinfo = idtvec_info;
2146                 if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) {
2147                         idtvec_err = vmcs_idt_vectoring_err();
2148                         exitintinfo |= (uint64_t)idtvec_err << 32;
2149                 }
2150                 error = vm_exit_intinfo(vmx->vm, vcpu, exitintinfo);
2151                 KASSERT(error == 0, ("%s: vm_set_intinfo error %d",
2152                     __func__, error));
2153
2154                 /*
2155                  * If 'virtual NMIs' are being used and the VM-exit
2156                  * happened while injecting an NMI during the previous
2157                  * VM-entry, then clear "blocking by NMI" in the
2158                  * Guest Interruptibility-State so the NMI can be
2159                  * reinjected on the subsequent VM-entry.
2160                  *
2161                  * However, if the NMI was being delivered through a task
2162                  * gate, then the new task must start execution with NMIs
2163                  * blocked so don't clear NMI blocking in this case.
2164                  */
2165                 intr_type = idtvec_info & VMCS_INTR_T_MASK;
2166                 if (intr_type == VMCS_INTR_T_NMI) {
2167                         if (reason != EXIT_REASON_TASK_SWITCH)
2168                                 vmx_clear_nmi_blocking(vmx, vcpu);
2169                         else
2170                                 vmx_assert_nmi_blocking(vmx, vcpu);
2171                 }
2172
2173                 /*
2174                  * Update VM-entry instruction length if the event being
2175                  * delivered was a software interrupt or software exception.
2176                  */
2177                 if (intr_type == VMCS_INTR_T_SWINTR ||
2178                     intr_type == VMCS_INTR_T_PRIV_SWEXCEPTION ||
2179                     intr_type == VMCS_INTR_T_SWEXCEPTION) {
2180                         vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length);
2181                 }
2182         }
2183
2184         switch (reason) {
2185         case EXIT_REASON_TASK_SWITCH:
2186                 ts = &vmexit->u.task_switch;
2187                 ts->tsssel = qual & 0xffff;
2188                 ts->reason = vmx_task_switch_reason(qual);
2189                 ts->ext = 0;
2190                 ts->errcode_valid = 0;
2191                 vmx_paging_info(&ts->paging);
2192                 /*
2193                  * If the task switch was due to a CALL, JMP, IRET, software
2194                  * interrupt (INT n) or software exception (INT3, INTO),
2195                  * then the saved %rip references the instruction that caused
2196                  * the task switch. The instruction length field in the VMCS
2197                  * is valid in this case.
2198                  *
2199                  * In all other cases (e.g., NMI, hardware exception) the
2200                  * saved %rip is one that would have been saved in the old TSS
2201                  * had the task switch completed normally so the instruction
2202                  * length field is not needed in this case and is explicitly
2203                  * set to 0.
2204                  */
2205                 if (ts->reason == TSR_IDT_GATE) {
2206                         KASSERT(idtvec_info & VMCS_IDT_VEC_VALID,
2207                             ("invalid idtvec_info %#x for IDT task switch",
2208                             idtvec_info));
2209                         intr_type = idtvec_info & VMCS_INTR_T_MASK;
2210                         if (intr_type != VMCS_INTR_T_SWINTR &&
2211                             intr_type != VMCS_INTR_T_SWEXCEPTION &&
2212                             intr_type != VMCS_INTR_T_PRIV_SWEXCEPTION) {
2213                                 /* Task switch triggered by external event */
2214                                 ts->ext = 1;
2215                                 vmexit->inst_length = 0;
2216                                 if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) {
2217                                         ts->errcode_valid = 1;
2218                                         ts->errcode = vmcs_idt_vectoring_err();
2219                                 }
2220                         }
2221                 }
2222                 vmexit->exitcode = VM_EXITCODE_TASK_SWITCH;
2223                 VCPU_CTR4(vmx->vm, vcpu, "task switch reason %d, tss 0x%04x, "
2224                     "%s errcode 0x%016lx", ts->reason, ts->tsssel,
2225                     ts->ext ? "external" : "internal",
2226                     ((uint64_t)ts->errcode << 32) | ts->errcode_valid);
2227                 break;
2228         case EXIT_REASON_CR_ACCESS:
2229                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CR_ACCESS, 1);
2230                 switch (qual & 0xf) {
2231                 case 0:
2232                         handled = vmx_emulate_cr0_access(vmx, vcpu, qual);
2233                         break;
2234                 case 4:
2235                         handled = vmx_emulate_cr4_access(vmx, vcpu, qual);
2236                         break;
2237                 case 8:
2238                         handled = vmx_emulate_cr8_access(vmx, vcpu, qual);
2239                         break;
2240                 }
2241                 break;
2242         case EXIT_REASON_RDMSR:
2243                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RDMSR, 1);
2244                 retu = false;
2245                 ecx = vmxctx->guest_rcx;
2246                 VCPU_CTR1(vmx->vm, vcpu, "rdmsr 0x%08x", ecx);
2247                 error = emulate_rdmsr(vmx, vcpu, ecx, &retu);
2248                 if (error) {
2249                         vmexit->exitcode = VM_EXITCODE_RDMSR;
2250                         vmexit->u.msr.code = ecx;
2251                 } else if (!retu) {
2252                         handled = HANDLED;
2253                 } else {
2254                         /* Return to userspace with a valid exitcode */
2255                         KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
2256                             ("emulate_rdmsr retu with bogus exitcode"));
2257                 }
2258                 break;
2259         case EXIT_REASON_WRMSR:
2260                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_WRMSR, 1);
2261                 retu = false;
2262                 eax = vmxctx->guest_rax;
2263                 ecx = vmxctx->guest_rcx;
2264                 edx = vmxctx->guest_rdx;
2265                 VCPU_CTR2(vmx->vm, vcpu, "wrmsr 0x%08x value 0x%016lx",
2266                     ecx, (uint64_t)edx << 32 | eax);
2267                 error = emulate_wrmsr(vmx, vcpu, ecx,
2268                     (uint64_t)edx << 32 | eax, &retu);
2269                 if (error) {
2270                         vmexit->exitcode = VM_EXITCODE_WRMSR;
2271                         vmexit->u.msr.code = ecx;
2272                         vmexit->u.msr.wval = (uint64_t)edx << 32 | eax;
2273                 } else if (!retu) {
2274                         handled = HANDLED;
2275                 } else {
2276                         /* Return to userspace with a valid exitcode */
2277                         KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
2278                             ("emulate_wrmsr retu with bogus exitcode"));
2279                 }
2280                 break;
2281         case EXIT_REASON_HLT:
2282                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_HLT, 1);
2283                 vmexit->exitcode = VM_EXITCODE_HLT;
2284                 vmexit->u.hlt.rflags = vmcs_read(VMCS_GUEST_RFLAGS);
2285                 break;
2286         case EXIT_REASON_MTF:
2287                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_MTRAP, 1);
2288                 vmexit->exitcode = VM_EXITCODE_MTRAP;
2289                 vmexit->inst_length = 0;
2290                 break;
2291         case EXIT_REASON_PAUSE:
2292                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_PAUSE, 1);
2293                 vmexit->exitcode = VM_EXITCODE_PAUSE;
2294                 break;
2295         case EXIT_REASON_INTR_WINDOW:
2296                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INTR_WINDOW, 1);
2297                 vmx_clear_int_window_exiting(vmx, vcpu);
2298                 return (1);
2299         case EXIT_REASON_EXT_INTR:
2300                 /*
2301                  * External interrupts serve only to cause VM exits and allow
2302                  * the host interrupt handler to run.
2303                  *
2304                  * If this external interrupt triggers a virtual interrupt
2305                  * to a VM, then that state will be recorded by the
2306                  * host interrupt handler in the VM's softc. We will inject
2307                  * this virtual interrupt during the subsequent VM enter.
2308                  */
2309                 intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
2310
2311                 /*
2312                  * XXX: Ignore this exit if VMCS_INTR_VALID is not set.
2313                  * This appears to be a bug in VMware Fusion?
2314                  */
2315                 if (!(intr_info & VMCS_INTR_VALID))
2316                         return (1);
2317                 KASSERT((intr_info & VMCS_INTR_VALID) != 0 &&
2318                     (intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_HWINTR,
2319                     ("VM exit interruption info invalid: %#x", intr_info));
2320                 vmx_trigger_hostintr(intr_info & 0xff);
2321
2322                 /*
2323                  * This is special. We want to treat this as an 'handled'
2324                  * VM-exit but not increment the instruction pointer.
2325                  */
2326                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXTINT, 1);
2327                 return (1);
2328         case EXIT_REASON_NMI_WINDOW:
2329                 /* Exit to allow the pending virtual NMI to be injected */
2330                 if (vm_nmi_pending(vmx->vm, vcpu))
2331                         vmx_inject_nmi(vmx, vcpu);
2332                 vmx_clear_nmi_window_exiting(vmx, vcpu);
2333                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NMI_WINDOW, 1);
2334                 return (1);
2335         case EXIT_REASON_INOUT:
2336                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INOUT, 1);
2337                 vmexit->exitcode = VM_EXITCODE_INOUT;
2338                 vmexit->u.inout.bytes = (qual & 0x7) + 1;
2339                 vmexit->u.inout.in = in = (qual & 0x8) ? 1 : 0;
2340                 vmexit->u.inout.string = (qual & 0x10) ? 1 : 0;
2341                 vmexit->u.inout.rep = (qual & 0x20) ? 1 : 0;
2342                 vmexit->u.inout.port = (uint16_t)(qual >> 16);
2343                 vmexit->u.inout.eax = (uint32_t)(vmxctx->guest_rax);
2344                 if (vmexit->u.inout.string) {
2345                         inst_info = vmcs_read(VMCS_EXIT_INSTRUCTION_INFO);
2346                         vmexit->exitcode = VM_EXITCODE_INOUT_STR;
2347                         vis = &vmexit->u.inout_str;
2348                         vmx_paging_info(&vis->paging);
2349                         vis->rflags = vmcs_read(VMCS_GUEST_RFLAGS);
2350                         vis->cr0 = vmcs_read(VMCS_GUEST_CR0);
2351                         vis->index = inout_str_index(vmx, vcpu, in);
2352                         vis->count = inout_str_count(vmx, vcpu, vis->inout.rep);
2353                         vis->addrsize = inout_str_addrsize(inst_info);
2354                         inout_str_seginfo(vmx, vcpu, inst_info, in, vis);
2355                 }
2356                 break;
2357         case EXIT_REASON_CPUID:
2358                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CPUID, 1);
2359                 handled = vmx_handle_cpuid(vmx->vm, vcpu, vmxctx);
2360                 break;
2361         case EXIT_REASON_EXCEPTION:
2362                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXCEPTION, 1);
2363                 intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
2364                 KASSERT((intr_info & VMCS_INTR_VALID) != 0,
2365                     ("VM exit interruption info invalid: %#x", intr_info));
2366
2367                 intr_vec = intr_info & 0xff;
2368                 intr_type = intr_info & VMCS_INTR_T_MASK;
2369
2370                 /*
2371                  * If Virtual NMIs control is 1 and the VM-exit is due to a
2372                  * fault encountered during the execution of IRET then we must
2373                  * restore the state of "virtual-NMI blocking" before resuming
2374                  * the guest.
2375                  *
2376                  * See "Resuming Guest Software after Handling an Exception".
2377                  * See "Information for VM Exits Due to Vectored Events".
2378                  */
2379                 if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
2380                     (intr_vec != IDT_DF) &&
2381                     (intr_info & EXIT_QUAL_NMIUDTI) != 0)
2382                         vmx_restore_nmi_blocking(vmx, vcpu);
2383
2384                 /*
2385                  * The NMI has already been handled in vmx_exit_handle_nmi().
2386                  */
2387                 if (intr_type == VMCS_INTR_T_NMI)
2388                         return (1);
2389
2390                 /*
2391                  * Call the machine check handler by hand. Also don't reflect
2392                  * the machine check back into the guest.
2393                  */
2394                 if (intr_vec == IDT_MC) {
2395                         VCPU_CTR0(vmx->vm, vcpu, "Vectoring to MCE handler");
2396                         __asm __volatile("int $18");
2397                         return (1);
2398                 }
2399
2400                 if (intr_vec == IDT_PF) {
2401                         error = vmxctx_setreg(vmxctx, VM_REG_GUEST_CR2, qual);
2402                         KASSERT(error == 0, ("%s: vmxctx_setreg(cr2) error %d",
2403                             __func__, error));
2404                 }
2405
2406                 /*
2407                  * Software exceptions exhibit trap-like behavior. This in
2408                  * turn requires populating the VM-entry instruction length
2409                  * so that the %rip in the trap frame is past the INT3/INTO
2410                  * instruction.
2411                  */
2412                 if (intr_type == VMCS_INTR_T_SWEXCEPTION)
2413                         vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length);
2414
2415                 /* Reflect all other exceptions back into the guest */
2416                 errcode_valid = errcode = 0;
2417                 if (intr_info & VMCS_INTR_DEL_ERRCODE) {
2418                         errcode_valid = 1;
2419                         errcode = vmcs_read(VMCS_EXIT_INTR_ERRCODE);
2420                 }
2421                 VCPU_CTR2(vmx->vm, vcpu, "Reflecting exception %d/%#x into "
2422                     "the guest", intr_vec, errcode);
2423                 error = vm_inject_exception(vmx->vm, vcpu, intr_vec,
2424                     errcode_valid, errcode, 0);
2425                 KASSERT(error == 0, ("%s: vm_inject_exception error %d",
2426                     __func__, error));
2427                 return (1);
2428
2429         case EXIT_REASON_EPT_FAULT:
2430                 /*
2431                  * If 'gpa' lies within the address space allocated to
2432                  * memory then this must be a nested page fault otherwise
2433                  * this must be an instruction that accesses MMIO space.
2434                  */
2435                 gpa = vmcs_gpa();
2436                 if (vm_mem_allocated(vmx->vm, vcpu, gpa) ||
2437                     apic_access_fault(vmx, vcpu, gpa)) {
2438                         vmexit->exitcode = VM_EXITCODE_PAGING;
2439                         vmexit->inst_length = 0;
2440                         vmexit->u.paging.gpa = gpa;
2441                         vmexit->u.paging.fault_type = ept_fault_type(qual);
2442                         vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NESTED_FAULT, 1);
2443                 } else if (ept_emulation_fault(qual)) {
2444                         vmexit_inst_emul(vmexit, gpa, vmcs_gla());
2445                         vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INST_EMUL, 1);
2446                 }
2447                 /*
2448                  * If Virtual NMIs control is 1 and the VM-exit is due to an
2449                  * EPT fault during the execution of IRET then we must restore
2450                  * the state of "virtual-NMI blocking" before resuming.
2451                  *
2452                  * See description of "NMI unblocking due to IRET" in
2453                  * "Exit Qualification for EPT Violations".
2454                  */
2455                 if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
2456                     (qual & EXIT_QUAL_NMIUDTI) != 0)
2457                         vmx_restore_nmi_blocking(vmx, vcpu);
2458                 break;
2459         case EXIT_REASON_VIRTUALIZED_EOI:
2460                 vmexit->exitcode = VM_EXITCODE_IOAPIC_EOI;
2461                 vmexit->u.ioapic_eoi.vector = qual & 0xFF;
2462                 vmexit->inst_length = 0;        /* trap-like */
2463                 break;
2464         case EXIT_REASON_APIC_ACCESS:
2465                 handled = vmx_handle_apic_access(vmx, vcpu, vmexit);
2466                 break;
2467         case EXIT_REASON_APIC_WRITE:
2468                 /*
2469                  * APIC-write VM exit is trap-like so the %rip is already
2470                  * pointing to the next instruction.
2471                  */
2472                 vmexit->inst_length = 0;
2473                 vlapic = vm_lapic(vmx->vm, vcpu);
2474                 handled = vmx_handle_apic_write(vmx, vcpu, vlapic, qual);
2475                 break;
2476         case EXIT_REASON_XSETBV:
2477                 handled = vmx_emulate_xsetbv(vmx, vcpu, vmexit);
2478                 break;
2479         case EXIT_REASON_MONITOR:
2480                 vmexit->exitcode = VM_EXITCODE_MONITOR;
2481                 break;
2482         case EXIT_REASON_MWAIT:
2483                 vmexit->exitcode = VM_EXITCODE_MWAIT;
2484                 break;
2485         default:
2486                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_UNKNOWN, 1);
2487                 break;
2488         }
2489
2490         if (handled) {
2491                 /*
2492                  * It is possible that control is returned to userland
2493                  * even though we were able to handle the VM exit in the
2494                  * kernel.
2495                  *
2496                  * In such a case we want to make sure that the userland
2497                  * restarts guest execution at the instruction *after*
2498                  * the one we just processed. Therefore we update the
2499                  * guest rip in the VMCS and in 'vmexit'.
2500                  */
2501                 vmexit->rip += vmexit->inst_length;
2502                 vmexit->inst_length = 0;
2503                 vmcs_write(VMCS_GUEST_RIP, vmexit->rip);
2504         } else {
2505                 if (vmexit->exitcode == VM_EXITCODE_BOGUS) {
2506                         /*
2507                          * If this VM exit was not claimed by anybody then
2508                          * treat it as a generic VMX exit.
2509                          */
2510                         vmexit->exitcode = VM_EXITCODE_VMX;
2511                         vmexit->u.vmx.status = VM_SUCCESS;
2512                         vmexit->u.vmx.inst_type = 0;
2513                         vmexit->u.vmx.inst_error = 0;
2514                 } else {
2515                         /*
2516                          * The exitcode and collateral have been populated.
2517                          * The VM exit will be processed further in userland.
2518                          */
2519                 }
2520         }
2521         return (handled);
2522 }
2523
2524 static __inline void
2525 vmx_exit_inst_error(struct vmxctx *vmxctx, int rc, struct vm_exit *vmexit)
2526 {
2527
2528         KASSERT(vmxctx->inst_fail_status != VM_SUCCESS,
2529             ("vmx_exit_inst_error: invalid inst_fail_status %d",
2530             vmxctx->inst_fail_status));
2531
2532         vmexit->inst_length = 0;
2533         vmexit->exitcode = VM_EXITCODE_VMX;
2534         vmexit->u.vmx.status = vmxctx->inst_fail_status;
2535         vmexit->u.vmx.inst_error = vmcs_instruction_error();
2536         vmexit->u.vmx.exit_reason = ~0;
2537         vmexit->u.vmx.exit_qualification = ~0;
2538
2539         switch (rc) {
2540         case VMX_VMRESUME_ERROR:
2541         case VMX_VMLAUNCH_ERROR:
2542         case VMX_INVEPT_ERROR:
2543                 vmexit->u.vmx.inst_type = rc;
2544                 break;
2545         default:
2546                 panic("vm_exit_inst_error: vmx_enter_guest returned %d", rc);
2547         }
2548 }
2549
2550 /*
2551  * If the NMI-exiting VM execution control is set to '1' then an NMI in
2552  * non-root operation causes a VM-exit. NMI blocking is in effect so it is
2553  * sufficient to simply vector to the NMI handler via a software interrupt.
2554  * However, this must be done before maskable interrupts are enabled
2555  * otherwise the "iret" issued by an interrupt handler will incorrectly
2556  * clear NMI blocking.
2557  */
2558 static __inline void
2559 vmx_exit_handle_nmi(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit)
2560 {
2561         uint32_t intr_info;
2562
2563         KASSERT((read_rflags() & PSL_I) == 0, ("interrupts enabled"));
2564
2565         if (vmexit->u.vmx.exit_reason != EXIT_REASON_EXCEPTION)
2566                 return;
2567
2568         intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
2569         KASSERT((intr_info & VMCS_INTR_VALID) != 0,
2570             ("VM exit interruption info invalid: %#x", intr_info));
2571
2572         if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI) {
2573                 KASSERT((intr_info & 0xff) == IDT_NMI, ("VM exit due "
2574                     "to NMI has invalid vector: %#x", intr_info));
2575                 VCPU_CTR0(vmx->vm, vcpuid, "Vectoring to NMI handler");
2576                 __asm __volatile("int $2");
2577         }
2578 }
2579
2580 static __inline void
2581 vmx_dr_enter_guest(struct vmxctx *vmxctx)
2582 {
2583         register_t rflags;
2584
2585         /* Save host control debug registers. */
2586         vmxctx->host_dr7 = rdr7();
2587         vmxctx->host_debugctl = rdmsr(MSR_DEBUGCTLMSR);
2588
2589         /*
2590          * Disable debugging in DR7 and DEBUGCTL to avoid triggering
2591          * exceptions in the host based on the guest DRx values.  The
2592          * guest DR7 and DEBUGCTL are saved/restored in the VMCS.
2593          */
2594         load_dr7(0);
2595         wrmsr(MSR_DEBUGCTLMSR, 0);
2596
2597         /*
2598          * Disable single stepping the kernel to avoid corrupting the
2599          * guest DR6.  A debugger might still be able to corrupt the
2600          * guest DR6 by setting a breakpoint after this point and then
2601          * single stepping.
2602          */
2603         rflags = read_rflags();
2604         vmxctx->host_tf = rflags & PSL_T;
2605         write_rflags(rflags & ~PSL_T);
2606
2607         /* Save host debug registers. */
2608         vmxctx->host_dr0 = rdr0();
2609         vmxctx->host_dr1 = rdr1();
2610         vmxctx->host_dr2 = rdr2();
2611         vmxctx->host_dr3 = rdr3();
2612         vmxctx->host_dr6 = rdr6();
2613
2614         /* Restore guest debug registers. */
2615         load_dr0(vmxctx->guest_dr0);
2616         load_dr1(vmxctx->guest_dr1);
2617         load_dr2(vmxctx->guest_dr2);
2618         load_dr3(vmxctx->guest_dr3);
2619         load_dr6(vmxctx->guest_dr6);
2620 }
2621
2622 static __inline void
2623 vmx_dr_leave_guest(struct vmxctx *vmxctx)
2624 {
2625
2626         /* Save guest debug registers. */
2627         vmxctx->guest_dr0 = rdr0();
2628         vmxctx->guest_dr1 = rdr1();
2629         vmxctx->guest_dr2 = rdr2();
2630         vmxctx->guest_dr3 = rdr3();
2631         vmxctx->guest_dr6 = rdr6();
2632
2633         /*
2634          * Restore host debug registers.  Restore DR7, DEBUGCTL, and
2635          * PSL_T last.
2636          */
2637         load_dr0(vmxctx->host_dr0);
2638         load_dr1(vmxctx->host_dr1);
2639         load_dr2(vmxctx->host_dr2);
2640         load_dr3(vmxctx->host_dr3);
2641         load_dr6(vmxctx->host_dr6);
2642         wrmsr(MSR_DEBUGCTLMSR, vmxctx->host_debugctl);
2643         load_dr7(vmxctx->host_dr7);
2644         write_rflags(read_rflags() | vmxctx->host_tf);
2645 }
2646
2647 static int
2648 vmx_run(void *arg, int vcpu, register_t rip, pmap_t pmap,
2649     struct vm_eventinfo *evinfo)
2650 {
2651         int rc, handled, launched;
2652         struct vmx *vmx;
2653         struct vm *vm;
2654         struct vmxctx *vmxctx;
2655         struct vmcs *vmcs;
2656         struct vm_exit *vmexit;
2657         struct vlapic *vlapic;
2658         uint32_t exit_reason;
2659
2660         vmx = arg;
2661         vm = vmx->vm;
2662         vmcs = &vmx->vmcs[vcpu];
2663         vmxctx = &vmx->ctx[vcpu];
2664         vlapic = vm_lapic(vm, vcpu);
2665         vmexit = vm_exitinfo(vm, vcpu);
2666         launched = 0;
2667
2668         KASSERT(vmxctx->pmap == pmap,
2669             ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap));
2670
2671         vmx_msr_guest_enter(vmx, vcpu);
2672
2673         VMPTRLD(vmcs);
2674
2675         /*
2676          * XXX
2677          * We do this every time because we may setup the virtual machine
2678          * from a different process than the one that actually runs it.
2679          *
2680          * If the life of a virtual machine was spent entirely in the context
2681          * of a single process we could do this once in vmx_vminit().
2682          */
2683         vmcs_write(VMCS_HOST_CR3, rcr3());
2684
2685         vmcs_write(VMCS_GUEST_RIP, rip);
2686         vmx_set_pcpu_defaults(vmx, vcpu, pmap);
2687         do {
2688                 KASSERT(vmcs_guest_rip() == rip, ("%s: vmcs guest rip mismatch "
2689                     "%#lx/%#lx", __func__, vmcs_guest_rip(), rip));
2690
2691                 handled = UNHANDLED;
2692                 /*
2693                  * Interrupts are disabled from this point on until the
2694                  * guest starts executing. This is done for the following
2695                  * reasons:
2696                  *
2697                  * If an AST is asserted on this thread after the check below,
2698                  * then the IPI_AST notification will not be lost, because it
2699                  * will cause a VM exit due to external interrupt as soon as
2700                  * the guest state is loaded.
2701                  *
2702                  * A posted interrupt after 'vmx_inject_interrupts()' will
2703                  * not be "lost" because it will be held pending in the host
2704                  * APIC because interrupts are disabled. The pending interrupt
2705                  * will be recognized as soon as the guest state is loaded.
2706                  *
2707                  * The same reasoning applies to the IPI generated by
2708                  * pmap_invalidate_ept().
2709                  */
2710                 disable_intr();
2711                 vmx_inject_interrupts(vmx, vcpu, vlapic, rip);
2712
2713                 /*
2714                  * Check for vcpu suspension after injecting events because
2715                  * vmx_inject_interrupts() can suspend the vcpu due to a
2716                  * triple fault.
2717                  */
2718                 if (vcpu_suspended(evinfo)) {
2719                         enable_intr();
2720                         vm_exit_suspended(vmx->vm, vcpu, rip);
2721                         break;
2722                 }
2723
2724                 if (vcpu_rendezvous_pending(evinfo)) {
2725                         enable_intr();
2726                         vm_exit_rendezvous(vmx->vm, vcpu, rip);
2727                         break;
2728                 }
2729
2730                 if (vcpu_reqidle(evinfo)) {
2731                         enable_intr();
2732                         vm_exit_reqidle(vmx->vm, vcpu, rip);
2733                         break;
2734                 }
2735
2736                 if (vcpu_should_yield(vm, vcpu)) {
2737                         enable_intr();
2738                         vm_exit_astpending(vmx->vm, vcpu, rip);
2739                         vmx_astpending_trace(vmx, vcpu, rip);
2740                         handled = HANDLED;
2741                         break;
2742                 }
2743
2744                 vmx_run_trace(vmx, vcpu);
2745                 vmx_dr_enter_guest(vmxctx);
2746                 rc = vmx_enter_guest(vmxctx, vmx, launched);
2747                 vmx_dr_leave_guest(vmxctx);
2748
2749                 /* Collect some information for VM exit processing */
2750                 vmexit->rip = rip = vmcs_guest_rip();
2751                 vmexit->inst_length = vmexit_instruction_length();
2752                 vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason();
2753                 vmexit->u.vmx.exit_qualification = vmcs_exit_qualification();
2754
2755                 /* Update 'nextrip' */
2756                 vmx->state[vcpu].nextrip = rip;
2757
2758                 if (rc == VMX_GUEST_VMEXIT) {
2759                         vmx_exit_handle_nmi(vmx, vcpu, vmexit);
2760                         enable_intr();
2761                         handled = vmx_exit_process(vmx, vcpu, vmexit);
2762                 } else {
2763                         enable_intr();
2764                         vmx_exit_inst_error(vmxctx, rc, vmexit);
2765                 }
2766                 launched = 1;
2767                 vmx_exit_trace(vmx, vcpu, rip, exit_reason, handled);
2768                 rip = vmexit->rip;
2769         } while (handled);
2770
2771         /*
2772          * If a VM exit has been handled then the exitcode must be BOGUS
2773          * If a VM exit is not handled then the exitcode must not be BOGUS
2774          */
2775         if ((handled && vmexit->exitcode != VM_EXITCODE_BOGUS) ||
2776             (!handled && vmexit->exitcode == VM_EXITCODE_BOGUS)) {
2777                 panic("Mismatch between handled (%d) and exitcode (%d)",
2778                       handled, vmexit->exitcode);
2779         }
2780
2781         if (!handled)
2782                 vmm_stat_incr(vm, vcpu, VMEXIT_USERSPACE, 1);
2783
2784         VCPU_CTR1(vm, vcpu, "returning from vmx_run: exitcode %d",
2785             vmexit->exitcode);
2786
2787         VMCLEAR(vmcs);
2788         vmx_msr_guest_exit(vmx, vcpu);
2789
2790         return (0);
2791 }
2792
2793 static void
2794 vmx_vmcleanup(void *arg)
2795 {
2796         int i;
2797         struct vmx *vmx = arg;
2798
2799         if (apic_access_virtualization(vmx, 0))
2800                 vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
2801
2802         for (i = 0; i < VM_MAXCPU; i++)
2803                 vpid_free(vmx->state[i].vpid);
2804
2805         free(vmx, M_VMX);
2806
2807         return;
2808 }
2809
2810 static register_t *
2811 vmxctx_regptr(struct vmxctx *vmxctx, int reg)
2812 {
2813
2814         switch (reg) {
2815         case VM_REG_GUEST_RAX:
2816                 return (&vmxctx->guest_rax);
2817         case VM_REG_GUEST_RBX:
2818                 return (&vmxctx->guest_rbx);
2819         case VM_REG_GUEST_RCX:
2820                 return (&vmxctx->guest_rcx);
2821         case VM_REG_GUEST_RDX:
2822                 return (&vmxctx->guest_rdx);
2823         case VM_REG_GUEST_RSI:
2824                 return (&vmxctx->guest_rsi);
2825         case VM_REG_GUEST_RDI:
2826                 return (&vmxctx->guest_rdi);
2827         case VM_REG_GUEST_RBP:
2828                 return (&vmxctx->guest_rbp);
2829         case VM_REG_GUEST_R8:
2830                 return (&vmxctx->guest_r8);
2831         case VM_REG_GUEST_R9:
2832                 return (&vmxctx->guest_r9);
2833         case VM_REG_GUEST_R10:
2834                 return (&vmxctx->guest_r10);
2835         case VM_REG_GUEST_R11:
2836                 return (&vmxctx->guest_r11);
2837         case VM_REG_GUEST_R12:
2838                 return (&vmxctx->guest_r12);
2839         case VM_REG_GUEST_R13:
2840                 return (&vmxctx->guest_r13);
2841         case VM_REG_GUEST_R14:
2842                 return (&vmxctx->guest_r14);
2843         case VM_REG_GUEST_R15:
2844                 return (&vmxctx->guest_r15);
2845         case VM_REG_GUEST_CR2:
2846                 return (&vmxctx->guest_cr2);
2847         case VM_REG_GUEST_DR0:
2848                 return (&vmxctx->guest_dr0);
2849         case VM_REG_GUEST_DR1:
2850                 return (&vmxctx->guest_dr1);
2851         case VM_REG_GUEST_DR2:
2852                 return (&vmxctx->guest_dr2);
2853         case VM_REG_GUEST_DR3:
2854                 return (&vmxctx->guest_dr3);
2855         case VM_REG_GUEST_DR6:
2856                 return (&vmxctx->guest_dr6);
2857         default:
2858                 break;
2859         }
2860         return (NULL);
2861 }
2862
2863 static int
2864 vmxctx_getreg(struct vmxctx *vmxctx, int reg, uint64_t *retval)
2865 {
2866         register_t *regp;
2867
2868         if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
2869                 *retval = *regp;
2870                 return (0);
2871         } else
2872                 return (EINVAL);
2873 }
2874
2875 static int
2876 vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val)
2877 {
2878         register_t *regp;
2879
2880         if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
2881                 *regp = val;
2882                 return (0);
2883         } else
2884                 return (EINVAL);
2885 }
2886
2887 static int
2888 vmx_get_intr_shadow(struct vmx *vmx, int vcpu, int running, uint64_t *retval)
2889 {
2890         uint64_t gi;
2891         int error;
2892
2893         error = vmcs_getreg(&vmx->vmcs[vcpu], running,
2894             VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY), &gi);
2895         *retval = (gi & HWINTR_BLOCKING) ? 1 : 0;
2896         return (error);
2897 }
2898
2899 static int
2900 vmx_modify_intr_shadow(struct vmx *vmx, int vcpu, int running, uint64_t val)
2901 {
2902         struct vmcs *vmcs;
2903         uint64_t gi;
2904         int error, ident;
2905
2906         /*
2907          * Forcing the vcpu into an interrupt shadow is not supported.
2908          */
2909         if (val) {
2910                 error = EINVAL;
2911                 goto done;
2912         }
2913
2914         vmcs = &vmx->vmcs[vcpu];
2915         ident = VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY);
2916         error = vmcs_getreg(vmcs, running, ident, &gi);
2917         if (error == 0) {
2918                 gi &= ~HWINTR_BLOCKING;
2919                 error = vmcs_setreg(vmcs, running, ident, gi);
2920         }
2921 done:
2922         VCPU_CTR2(vmx->vm, vcpu, "Setting intr_shadow to %#lx %s", val,
2923             error ? "failed" : "succeeded");
2924         return (error);
2925 }
2926
2927 static int
2928 vmx_shadow_reg(int reg)
2929 {
2930         int shreg;
2931
2932         shreg = -1;
2933
2934         switch (reg) {
2935         case VM_REG_GUEST_CR0:
2936                 shreg = VMCS_CR0_SHADOW;
2937                 break;
2938         case VM_REG_GUEST_CR4:
2939                 shreg = VMCS_CR4_SHADOW;
2940                 break;
2941         default:
2942                 break;
2943         }
2944
2945         return (shreg);
2946 }
2947
2948 static int
2949 vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval)
2950 {
2951         int running, hostcpu;
2952         struct vmx *vmx = arg;
2953
2954         running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
2955         if (running && hostcpu != curcpu)
2956                 panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), vcpu);
2957
2958         if (reg == VM_REG_GUEST_INTR_SHADOW)
2959                 return (vmx_get_intr_shadow(vmx, vcpu, running, retval));
2960
2961         if (vmxctx_getreg(&vmx->ctx[vcpu], reg, retval) == 0)
2962                 return (0);
2963
2964         return (vmcs_getreg(&vmx->vmcs[vcpu], running, reg, retval));
2965 }
2966
2967 static int
2968 vmx_setreg(void *arg, int vcpu, int reg, uint64_t val)
2969 {
2970         int error, hostcpu, running, shadow;
2971         uint64_t ctls;
2972         pmap_t pmap;
2973         struct vmx *vmx = arg;
2974
2975         running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
2976         if (running && hostcpu != curcpu)
2977                 panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), vcpu);
2978
2979         if (reg == VM_REG_GUEST_INTR_SHADOW)
2980                 return (vmx_modify_intr_shadow(vmx, vcpu, running, val));
2981
2982         if (vmxctx_setreg(&vmx->ctx[vcpu], reg, val) == 0)
2983                 return (0);
2984
2985         error = vmcs_setreg(&vmx->vmcs[vcpu], running, reg, val);
2986
2987         if (error == 0) {
2988                 /*
2989                  * If the "load EFER" VM-entry control is 1 then the
2990                  * value of EFER.LMA must be identical to "IA-32e mode guest"
2991                  * bit in the VM-entry control.
2992                  */
2993                 if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 &&
2994                     (reg == VM_REG_GUEST_EFER)) {
2995                         vmcs_getreg(&vmx->vmcs[vcpu], running,
2996                                     VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls);
2997                         if (val & EFER_LMA)
2998                                 ctls |= VM_ENTRY_GUEST_LMA;
2999                         else
3000                                 ctls &= ~VM_ENTRY_GUEST_LMA;
3001                         vmcs_setreg(&vmx->vmcs[vcpu], running,
3002                                     VMCS_IDENT(VMCS_ENTRY_CTLS), ctls);
3003                 }
3004
3005                 shadow = vmx_shadow_reg(reg);
3006                 if (shadow > 0) {
3007                         /*
3008                          * Store the unmodified value in the shadow
3009                          */
3010                         error = vmcs_setreg(&vmx->vmcs[vcpu], running,
3011                                     VMCS_IDENT(shadow), val);
3012                 }
3013
3014                 if (reg == VM_REG_GUEST_CR3) {
3015                         /*
3016                          * Invalidate the guest vcpu's TLB mappings to emulate
3017                          * the behavior of updating %cr3.
3018                          *
3019                          * XXX the processor retains global mappings when %cr3
3020                          * is updated but vmx_invvpid() does not.
3021                          */
3022                         pmap = vmx->ctx[vcpu].pmap;
3023                         vmx_invvpid(vmx, vcpu, pmap, running);
3024                 }
3025         }
3026
3027         return (error);
3028 }
3029
3030 static int
3031 vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
3032 {
3033         int hostcpu, running;
3034         struct vmx *vmx = arg;
3035
3036         running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
3037         if (running && hostcpu != curcpu)
3038                 panic("vmx_getdesc: %s%d is running", vm_name(vmx->vm), vcpu);
3039
3040         return (vmcs_getdesc(&vmx->vmcs[vcpu], running, reg, desc));
3041 }
3042
3043 static int
3044 vmx_setdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
3045 {
3046         int hostcpu, running;
3047         struct vmx *vmx = arg;
3048
3049         running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
3050         if (running && hostcpu != curcpu)
3051                 panic("vmx_setdesc: %s%d is running", vm_name(vmx->vm), vcpu);
3052
3053         return (vmcs_setdesc(&vmx->vmcs[vcpu], running, reg, desc));
3054 }
3055
3056 static int
3057 vmx_getcap(void *arg, int vcpu, int type, int *retval)
3058 {
3059         struct vmx *vmx = arg;
3060         int vcap;
3061         int ret;
3062
3063         ret = ENOENT;
3064
3065         vcap = vmx->cap[vcpu].set;
3066
3067         switch (type) {
3068         case VM_CAP_HALT_EXIT:
3069                 if (cap_halt_exit)
3070                         ret = 0;
3071                 break;
3072         case VM_CAP_PAUSE_EXIT:
3073                 if (cap_pause_exit)
3074                         ret = 0;
3075                 break;
3076         case VM_CAP_MTRAP_EXIT:
3077                 if (cap_monitor_trap)
3078                         ret = 0;
3079                 break;
3080         case VM_CAP_UNRESTRICTED_GUEST:
3081                 if (cap_unrestricted_guest)
3082                         ret = 0;
3083                 break;
3084         case VM_CAP_ENABLE_INVPCID:
3085                 if (cap_invpcid)
3086                         ret = 0;
3087                 break;
3088         default:
3089                 break;
3090         }
3091
3092         if (ret == 0)
3093                 *retval = (vcap & (1 << type)) ? 1 : 0;
3094
3095         return (ret);
3096 }
3097
3098 static int
3099 vmx_setcap(void *arg, int vcpu, int type, int val)
3100 {
3101         struct vmx *vmx = arg;
3102         struct vmcs *vmcs = &vmx->vmcs[vcpu];
3103         uint32_t baseval;
3104         uint32_t *pptr;
3105         int error;
3106         int flag;
3107         int reg;
3108         int retval;
3109
3110         retval = ENOENT;
3111         pptr = NULL;
3112
3113         switch (type) {
3114         case VM_CAP_HALT_EXIT:
3115                 if (cap_halt_exit) {
3116                         retval = 0;
3117                         pptr = &vmx->cap[vcpu].proc_ctls;
3118                         baseval = *pptr;
3119                         flag = PROCBASED_HLT_EXITING;
3120                         reg = VMCS_PRI_PROC_BASED_CTLS;
3121                 }
3122                 break;
3123         case VM_CAP_MTRAP_EXIT:
3124                 if (cap_monitor_trap) {
3125                         retval = 0;
3126                         pptr = &vmx->cap[vcpu].proc_ctls;
3127                         baseval = *pptr;
3128                         flag = PROCBASED_MTF;
3129                         reg = VMCS_PRI_PROC_BASED_CTLS;
3130                 }
3131                 break;
3132         case VM_CAP_PAUSE_EXIT:
3133                 if (cap_pause_exit) {
3134                         retval = 0;
3135                         pptr = &vmx->cap[vcpu].proc_ctls;
3136                         baseval = *pptr;
3137                         flag = PROCBASED_PAUSE_EXITING;
3138                         reg = VMCS_PRI_PROC_BASED_CTLS;
3139                 }
3140                 break;
3141         case VM_CAP_UNRESTRICTED_GUEST:
3142                 if (cap_unrestricted_guest) {
3143                         retval = 0;
3144                         pptr = &vmx->cap[vcpu].proc_ctls2;
3145                         baseval = *pptr;
3146                         flag = PROCBASED2_UNRESTRICTED_GUEST;
3147                         reg = VMCS_SEC_PROC_BASED_CTLS;
3148                 }
3149                 break;
3150         case VM_CAP_ENABLE_INVPCID:
3151                 if (cap_invpcid) {
3152                         retval = 0;
3153                         pptr = &vmx->cap[vcpu].proc_ctls2;
3154                         baseval = *pptr;
3155                         flag = PROCBASED2_ENABLE_INVPCID;
3156                         reg = VMCS_SEC_PROC_BASED_CTLS;
3157                 }
3158                 break;
3159         default:
3160                 break;
3161         }
3162
3163         if (retval == 0) {
3164                 if (val) {
3165                         baseval |= flag;
3166                 } else {
3167                         baseval &= ~flag;
3168                 }
3169                 VMPTRLD(vmcs);
3170                 error = vmwrite(reg, baseval);
3171                 VMCLEAR(vmcs);
3172
3173                 if (error) {
3174                         retval = error;
3175                 } else {
3176                         /*
3177                          * Update optional stored flags, and record
3178                          * setting
3179                          */
3180                         if (pptr != NULL) {
3181                                 *pptr = baseval;
3182                         }
3183
3184                         if (val) {
3185                                 vmx->cap[vcpu].set |= (1 << type);
3186                         } else {
3187                                 vmx->cap[vcpu].set &= ~(1 << type);
3188                         }
3189                 }
3190         }
3191
3192         return (retval);
3193 }
3194
3195 struct vlapic_vtx {
3196         struct vlapic   vlapic;
3197         struct pir_desc *pir_desc;
3198         struct vmx      *vmx;
3199 };
3200
3201 #define VMX_CTR_PIR(vm, vcpuid, pir_desc, notify, vector, level, msg)   \
3202 do {                                                                    \
3203         VCPU_CTR2(vm, vcpuid, msg " assert %s-triggered vector %d",     \
3204             level ? "level" : "edge", vector);                          \
3205         VCPU_CTR1(vm, vcpuid, msg " pir0 0x%016lx", pir_desc->pir[0]);  \
3206         VCPU_CTR1(vm, vcpuid, msg " pir1 0x%016lx", pir_desc->pir[1]);  \
3207         VCPU_CTR1(vm, vcpuid, msg " pir2 0x%016lx", pir_desc->pir[2]);  \
3208         VCPU_CTR1(vm, vcpuid, msg " pir3 0x%016lx", pir_desc->pir[3]);  \
3209         VCPU_CTR1(vm, vcpuid, msg " notify: %s", notify ? "yes" : "no");\
3210 } while (0)
3211
3212 /*
3213  * vlapic->ops handlers that utilize the APICv hardware assist described in
3214  * Chapter 29 of the Intel SDM.
3215  */
3216 static int
3217 vmx_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
3218 {
3219         struct vlapic_vtx *vlapic_vtx;
3220         struct pir_desc *pir_desc;
3221         uint64_t mask;
3222         int idx, notify;
3223
3224         vlapic_vtx = (struct vlapic_vtx *)vlapic;
3225         pir_desc = vlapic_vtx->pir_desc;
3226
3227         /*
3228          * Keep track of interrupt requests in the PIR descriptor. This is
3229          * because the virtual APIC page pointed to by the VMCS cannot be
3230          * modified if the vcpu is running.
3231          */
3232         idx = vector / 64;
3233         mask = 1UL << (vector % 64);
3234         atomic_set_long(&pir_desc->pir[idx], mask);
3235         notify = atomic_cmpset_long(&pir_desc->pending, 0, 1);
3236
3237         VMX_CTR_PIR(vlapic->vm, vlapic->vcpuid, pir_desc, notify, vector,
3238             level, "vmx_set_intr_ready");
3239         return (notify);
3240 }
3241
3242 static int
3243 vmx_pending_intr(struct vlapic *vlapic, int *vecptr)
3244 {
3245         struct vlapic_vtx *vlapic_vtx;
3246         struct pir_desc *pir_desc;
3247         struct LAPIC *lapic;
3248         uint64_t pending, pirval;
3249         uint32_t ppr, vpr;
3250         int i;
3251
3252         /*
3253          * This function is only expected to be called from the 'HLT' exit
3254          * handler which does not care about the vector that is pending.
3255          */
3256         KASSERT(vecptr == NULL, ("vmx_pending_intr: vecptr must be NULL"));
3257
3258         vlapic_vtx = (struct vlapic_vtx *)vlapic;
3259         pir_desc = vlapic_vtx->pir_desc;
3260
3261         pending = atomic_load_acq_long(&pir_desc->pending);
3262         if (!pending)
3263                 return (0);     /* common case */
3264
3265         /*
3266          * If there is an interrupt pending then it will be recognized only
3267          * if its priority is greater than the processor priority.
3268          *
3269          * Special case: if the processor priority is zero then any pending
3270          * interrupt will be recognized.
3271          */
3272         lapic = vlapic->apic_page;
3273         ppr = lapic->ppr & 0xf0;
3274         if (ppr == 0)
3275                 return (1);
3276
3277         VCPU_CTR1(vlapic->vm, vlapic->vcpuid, "HLT with non-zero PPR %d",
3278             lapic->ppr);
3279
3280         for (i = 3; i >= 0; i--) {
3281                 pirval = pir_desc->pir[i];
3282                 if (pirval != 0) {
3283                         vpr = (i * 64 + flsl(pirval) - 1) & 0xf0;
3284                         return (vpr > ppr);
3285                 }
3286         }
3287         return (0);
3288 }
3289
3290 static void
3291 vmx_intr_accepted(struct vlapic *vlapic, int vector)
3292 {
3293
3294         panic("vmx_intr_accepted: not expected to be called");
3295 }
3296
3297 static void
3298 vmx_set_tmr(struct vlapic *vlapic, int vector, bool level)
3299 {
3300         struct vlapic_vtx *vlapic_vtx;
3301         struct vmx *vmx;
3302         struct vmcs *vmcs;
3303         uint64_t mask, val;
3304
3305         KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
3306         KASSERT(!vcpu_is_running(vlapic->vm, vlapic->vcpuid, NULL),
3307             ("vmx_set_tmr: vcpu cannot be running"));
3308
3309         vlapic_vtx = (struct vlapic_vtx *)vlapic;
3310         vmx = vlapic_vtx->vmx;
3311         vmcs = &vmx->vmcs[vlapic->vcpuid];
3312         mask = 1UL << (vector % 64);
3313
3314         VMPTRLD(vmcs);
3315         val = vmcs_read(VMCS_EOI_EXIT(vector));
3316         if (level)
3317                 val |= mask;
3318         else
3319                 val &= ~mask;
3320         vmcs_write(VMCS_EOI_EXIT(vector), val);
3321         VMCLEAR(vmcs);
3322 }
3323
3324 static void
3325 vmx_enable_x2apic_mode(struct vlapic *vlapic)
3326 {
3327         struct vmx *vmx;
3328         struct vmcs *vmcs;
3329         uint32_t proc_ctls2;
3330         int vcpuid, error;
3331
3332         vcpuid = vlapic->vcpuid;
3333         vmx = ((struct vlapic_vtx *)vlapic)->vmx;
3334         vmcs = &vmx->vmcs[vcpuid];
3335
3336         proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
3337         KASSERT((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) != 0,
3338             ("%s: invalid proc_ctls2 %#x", __func__, proc_ctls2));
3339
3340         proc_ctls2 &= ~PROCBASED2_VIRTUALIZE_APIC_ACCESSES;
3341         proc_ctls2 |= PROCBASED2_VIRTUALIZE_X2APIC_MODE;
3342         vmx->cap[vcpuid].proc_ctls2 = proc_ctls2;
3343
3344         VMPTRLD(vmcs);
3345         vmcs_write(VMCS_SEC_PROC_BASED_CTLS, proc_ctls2);
3346         VMCLEAR(vmcs);
3347
3348         if (vlapic->vcpuid == 0) {
3349                 /*
3350                  * The nested page table mappings are shared by all vcpus
3351                  * so unmap the APIC access page just once.
3352                  */
3353                 error = vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
3354                 KASSERT(error == 0, ("%s: vm_unmap_mmio error %d",
3355                     __func__, error));
3356
3357                 /*
3358                  * The MSR bitmap is shared by all vcpus so modify it only
3359                  * once in the context of vcpu 0.
3360                  */
3361                 error = vmx_allow_x2apic_msrs(vmx);
3362                 KASSERT(error == 0, ("%s: vmx_allow_x2apic_msrs error %d",
3363                     __func__, error));
3364         }
3365 }
3366
3367 static void
3368 vmx_post_intr(struct vlapic *vlapic, int hostcpu)
3369 {
3370
3371         ipi_cpu(hostcpu, pirvec);
3372 }
3373
3374 /*
3375  * Transfer the pending interrupts in the PIR descriptor to the IRR
3376  * in the virtual APIC page.
3377  */
3378 static void
3379 vmx_inject_pir(struct vlapic *vlapic)
3380 {
3381         struct vlapic_vtx *vlapic_vtx;
3382         struct pir_desc *pir_desc;
3383         struct LAPIC *lapic;
3384         uint64_t val, pirval;
3385         int rvi, pirbase = -1;
3386         uint16_t intr_status_old, intr_status_new;
3387
3388         vlapic_vtx = (struct vlapic_vtx *)vlapic;
3389         pir_desc = vlapic_vtx->pir_desc;
3390         if (atomic_cmpset_long(&pir_desc->pending, 1, 0) == 0) {
3391                 VCPU_CTR0(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: "
3392                     "no posted interrupt pending");
3393                 return;
3394         }
3395
3396         pirval = 0;
3397         pirbase = -1;
3398         lapic = vlapic->apic_page;
3399
3400         val = atomic_readandclear_long(&pir_desc->pir[0]);
3401         if (val != 0) {
3402                 lapic->irr0 |= val;
3403                 lapic->irr1 |= val >> 32;
3404                 pirbase = 0;
3405                 pirval = val;
3406         }
3407
3408         val = atomic_readandclear_long(&pir_desc->pir[1]);
3409         if (val != 0) {
3410                 lapic->irr2 |= val;
3411                 lapic->irr3 |= val >> 32;
3412                 pirbase = 64;
3413                 pirval = val;
3414         }
3415
3416         val = atomic_readandclear_long(&pir_desc->pir[2]);
3417         if (val != 0) {
3418                 lapic->irr4 |= val;
3419                 lapic->irr5 |= val >> 32;
3420                 pirbase = 128;
3421                 pirval = val;
3422         }
3423
3424         val = atomic_readandclear_long(&pir_desc->pir[3]);
3425         if (val != 0) {
3426                 lapic->irr6 |= val;
3427                 lapic->irr7 |= val >> 32;
3428                 pirbase = 192;
3429                 pirval = val;
3430         }
3431
3432         VLAPIC_CTR_IRR(vlapic, "vmx_inject_pir");
3433
3434         /*
3435          * Update RVI so the processor can evaluate pending virtual
3436          * interrupts on VM-entry.
3437          *
3438          * It is possible for pirval to be 0 here, even though the
3439          * pending bit has been set. The scenario is:
3440          * CPU-Y is sending a posted interrupt to CPU-X, which
3441          * is running a guest and processing posted interrupts in h/w.
3442          * CPU-X will eventually exit and the state seen in s/w is
3443          * the pending bit set, but no PIR bits set.
3444          *
3445          *      CPU-X                      CPU-Y
3446          *   (vm running)                (host running)
3447          *   rx posted interrupt
3448          *   CLEAR pending bit
3449          *                               SET PIR bit
3450          *   READ/CLEAR PIR bits
3451          *                               SET pending bit
3452          *   (vm exit)
3453          *   pending bit set, PIR 0
3454          */
3455         if (pirval != 0) {
3456                 rvi = pirbase + flsl(pirval) - 1;
3457                 intr_status_old = vmcs_read(VMCS_GUEST_INTR_STATUS);
3458                 intr_status_new = (intr_status_old & 0xFF00) | rvi;
3459                 if (intr_status_new > intr_status_old) {
3460                         vmcs_write(VMCS_GUEST_INTR_STATUS, intr_status_new);
3461                         VCPU_CTR2(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: "
3462                             "guest_intr_status changed from 0x%04x to 0x%04x",
3463                             intr_status_old, intr_status_new);
3464                 }
3465         }
3466 }
3467
3468 static struct vlapic *
3469 vmx_vlapic_init(void *arg, int vcpuid)
3470 {
3471         struct vmx *vmx;
3472         struct vlapic *vlapic;
3473         struct vlapic_vtx *vlapic_vtx;
3474
3475         vmx = arg;
3476
3477         vlapic = malloc(sizeof(struct vlapic_vtx), M_VLAPIC, M_WAITOK | M_ZERO);
3478         vlapic->vm = vmx->vm;
3479         vlapic->vcpuid = vcpuid;
3480         vlapic->apic_page = (struct LAPIC *)&vmx->apic_page[vcpuid];
3481
3482         vlapic_vtx = (struct vlapic_vtx *)vlapic;
3483         vlapic_vtx->pir_desc = &vmx->pir_desc[vcpuid];
3484         vlapic_vtx->vmx = vmx;
3485
3486         if (virtual_interrupt_delivery) {
3487                 vlapic->ops.set_intr_ready = vmx_set_intr_ready;
3488                 vlapic->ops.pending_intr = vmx_pending_intr;
3489                 vlapic->ops.intr_accepted = vmx_intr_accepted;
3490                 vlapic->ops.set_tmr = vmx_set_tmr;
3491                 vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode;
3492         }
3493
3494         if (posted_interrupts)
3495                 vlapic->ops.post_intr = vmx_post_intr;
3496
3497         vlapic_init(vlapic);
3498
3499         return (vlapic);
3500 }
3501
3502 static void
3503 vmx_vlapic_cleanup(void *arg, struct vlapic *vlapic)
3504 {
3505
3506         vlapic_cleanup(vlapic);
3507         free(vlapic, M_VLAPIC);
3508 }
3509
3510 struct vmm_ops vmm_ops_intel = {
3511         vmx_init,
3512         vmx_cleanup,
3513         vmx_restore,
3514         vmx_vminit,
3515         vmx_run,
3516         vmx_vmcleanup,
3517         vmx_getreg,
3518         vmx_setreg,
3519         vmx_getdesc,
3520         vmx_setdesc,
3521         vmx_getcap,
3522         vmx_setcap,
3523         ept_vmspace_alloc,
3524         ept_vmspace_free,
3525         vmx_vlapic_init,
3526         vmx_vlapic_cleanup,
3527 };