sys/amd64/vmm/intel/vmx.c

   1 /*-
   2  * Copyright (c) 2011 NetApp, Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  *
  26  * $FreeBSD$
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include <sys/param.h>
  33 #include <sys/systm.h>
  34 #include <sys/smp.h>
  35 #include <sys/kernel.h>
  36 #include <sys/malloc.h>
  37 #include <sys/pcpu.h>
  38 #include <sys/proc.h>
  39 #include <sys/sysctl.h>
  40
  41 #include <vm/vm.h>
  42 #include <vm/pmap.h>
  43
  44 #include <machine/psl.h>
  45 #include <machine/cpufunc.h>
  46 #include <machine/md_var.h>
  47 #include <machine/segments.h>
  48 #include <machine/smp.h>
  49 #include <machine/specialreg.h>
  50 #include <machine/vmparam.h>
  51
  52 #include <machine/vmm.h>
  53 #include <machine/vmm_dev.h>
  54 #include "vmm_host.h"
  55 #include "vmm_ipi.h"
  56 #include "vmm_msr.h"
  57 #include "vmm_ktr.h"
  58 #include "vmm_stat.h"
  59 #include "vlapic.h"
  60 #include "vlapic_priv.h"
  61
  62 #include "vmx_msr.h"
  63 #include "ept.h"
  64 #include "vmx_cpufunc.h"
  65 #include "vmx.h"
  66 #include "x86.h"
  67 #include "vmx_controls.h"
  68
  69 #define PINBASED_CTLS_ONE_SETTING                                       \
  70         (PINBASED_EXTINT_EXITING        |                               \
  71          PINBASED_NMI_EXITING           |                               \
  72          PINBASED_VIRTUAL_NMI)
  73 #define PINBASED_CTLS_ZERO_SETTING      0
  74
  75 #define PROCBASED_CTLS_WINDOW_SETTING                                   \
  76         (PROCBASED_INT_WINDOW_EXITING   |                               \
  77          PROCBASED_NMI_WINDOW_EXITING)
  78
  79 #define PROCBASED_CTLS_ONE_SETTING                                      \
  80         (PROCBASED_SECONDARY_CONTROLS   |                               \
  81          PROCBASED_IO_EXITING           |                               \
  82          PROCBASED_MSR_BITMAPS          |                               \
  83          PROCBASED_CTLS_WINDOW_SETTING)
  84 #define PROCBASED_CTLS_ZERO_SETTING     \
  85         (PROCBASED_CR3_LOAD_EXITING |   \
  86         PROCBASED_CR3_STORE_EXITING |   \
  87         PROCBASED_IO_BITMAPS)
  88
  89 #define PROCBASED_CTLS2_ONE_SETTING     PROCBASED2_ENABLE_EPT
  90 #define PROCBASED_CTLS2_ZERO_SETTING    0
  91
  92 #define VM_EXIT_CTLS_ONE_SETTING_NO_PAT                                 \
  93         (VM_EXIT_HOST_LMA                       |                       \
  94         VM_EXIT_SAVE_EFER                       |                       \
  95         VM_EXIT_LOAD_EFER)
  96
  97 #define VM_EXIT_CTLS_ONE_SETTING                                        \
  98         (VM_EXIT_CTLS_ONE_SETTING_NO_PAT        |                       \
  99         VM_EXIT_ACKNOWLEDGE_INTERRUPT           |                       \
 100         VM_EXIT_SAVE_PAT                        |                       \
 101         VM_EXIT_LOAD_PAT)
 102 #define VM_EXIT_CTLS_ZERO_SETTING       VM_EXIT_SAVE_DEBUG_CONTROLS
 103
 104 #define VM_ENTRY_CTLS_ONE_SETTING_NO_PAT        VM_ENTRY_LOAD_EFER
 105
 106 #define VM_ENTRY_CTLS_ONE_SETTING                                       \
 107         (VM_ENTRY_CTLS_ONE_SETTING_NO_PAT       |                       \
 108         VM_ENTRY_LOAD_PAT)
 109 #define VM_ENTRY_CTLS_ZERO_SETTING                                      \
 110         (VM_ENTRY_LOAD_DEBUG_CONTROLS           |                       \
 111         VM_ENTRY_INTO_SMM                       |                       \
 112         VM_ENTRY_DEACTIVATE_DUAL_MONITOR)
 113
 114 #define guest_msr_rw(vmx, msr) \
 115         msr_bitmap_change_access((vmx)->msr_bitmap, (msr), MSR_BITMAP_ACCESS_RW)
 116
 117 #define guest_msr_ro(vmx, msr) \
 118     msr_bitmap_change_access((vmx)->msr_bitmap, (msr), MSR_BITMAP_ACCESS_READ)
 119
 120 #define HANDLED         1
 121 #define UNHANDLED       0
 122
 123 static MALLOC_DEFINE(M_VMX, "vmx", "vmx");
 124 static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic");
 125
 126 SYSCTL_DECL(_hw_vmm);
 127 SYSCTL_NODE(_hw_vmm, OID_AUTO, vmx, CTLFLAG_RW, NULL, NULL);
 128
 129 int vmxon_enabled[MAXCPU];
 130 static char vmxon_region[MAXCPU][PAGE_SIZE] __aligned(PAGE_SIZE);
 131
 132 static uint32_t pinbased_ctls, procbased_ctls, procbased_ctls2;
 133 static uint32_t exit_ctls, entry_ctls;
 134
 135 static uint64_t cr0_ones_mask, cr0_zeros_mask;
 136 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_ones_mask, CTLFLAG_RD,
 137              &cr0_ones_mask, 0, NULL);
 138 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_zeros_mask, CTLFLAG_RD,
 139              &cr0_zeros_mask, 0, NULL);
 140
 141 static uint64_t cr4_ones_mask, cr4_zeros_mask;
 142 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_ones_mask, CTLFLAG_RD,
 143              &cr4_ones_mask, 0, NULL);
 144 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_zeros_mask, CTLFLAG_RD,
 145              &cr4_zeros_mask, 0, NULL);
 146
 147 static int vmx_no_patmsr;
 148
 149 static int vmx_initialized;
 150 SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, initialized, CTLFLAG_RD,
 151            &vmx_initialized, 0, "Intel VMX initialized");
 152
 153 /*
 154  * Optional capabilities
 155  */
 156 static int cap_halt_exit;
 157 static int cap_pause_exit;
 158 static int cap_unrestricted_guest;
 159 static int cap_monitor_trap;
 160 static int cap_invpcid;
 161
 162 static int virtual_interrupt_delivery;
 163 SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, virtual_interrupt_delivery, CTLFLAG_RD,
 164     &virtual_interrupt_delivery, 0, "APICv virtual interrupt delivery support");
 165
 166 static int posted_interrupts;
 167 SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupts, CTLFLAG_RD,
 168     &posted_interrupts, 0, "APICv posted interrupt support");
 169
 170 static int pirvec;
 171 SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupt_vector, CTLFLAG_RD,
 172     &pirvec, 0, "APICv posted interrupt vector");
 173
 174 static struct unrhdr *vpid_unr;
 175 static u_int vpid_alloc_failed;
 176 SYSCTL_UINT(_hw_vmm_vmx, OID_AUTO, vpid_alloc_failed, CTLFLAG_RD,
 177             &vpid_alloc_failed, 0, NULL);
 178
 179 /*
 180  * Use the last page below 4GB as the APIC access address. This address is
 181  * occupied by the boot firmware so it is guaranteed that it will not conflict
 182  * with a page in system memory.
 183  */
 184 #define APIC_ACCESS_ADDRESS     0xFFFFF000
 185
 186 static void vmx_inject_pir(struct vlapic *vlapic);
 187
 188 #ifdef KTR
 189 static const char *
 190 exit_reason_to_str(int reason)
 191 {
 192         static char reasonbuf[32];
 193
 194         switch (reason) {
 195         case EXIT_REASON_EXCEPTION:
 196                 return "exception";
 197         case EXIT_REASON_EXT_INTR:
 198                 return "extint";
 199         case EXIT_REASON_TRIPLE_FAULT:
 200                 return "triplefault";
 201         case EXIT_REASON_INIT:
 202                 return "init";
 203         case EXIT_REASON_SIPI:
 204                 return "sipi";
 205         case EXIT_REASON_IO_SMI:
 206                 return "iosmi";
 207         case EXIT_REASON_SMI:
 208                 return "smi";
 209         case EXIT_REASON_INTR_WINDOW:
 210                 return "intrwindow";
 211         case EXIT_REASON_NMI_WINDOW:
 212                 return "nmiwindow";
 213         case EXIT_REASON_TASK_SWITCH:
 214                 return "taskswitch";
 215         case EXIT_REASON_CPUID:
 216                 return "cpuid";
 217         case EXIT_REASON_GETSEC:
 218                 return "getsec";
 219         case EXIT_REASON_HLT:
 220                 return "hlt";
 221         case EXIT_REASON_INVD:
 222                 return "invd";
 223         case EXIT_REASON_INVLPG:
 224                 return "invlpg";
 225         case EXIT_REASON_RDPMC:
 226                 return "rdpmc";
 227         case EXIT_REASON_RDTSC:
 228                 return "rdtsc";
 229         case EXIT_REASON_RSM:
 230                 return "rsm";
 231         case EXIT_REASON_VMCALL:
 232                 return "vmcall";
 233         case EXIT_REASON_VMCLEAR:
 234                 return "vmclear";
 235         case EXIT_REASON_VMLAUNCH:
 236                 return "vmlaunch";
 237         case EXIT_REASON_VMPTRLD:
 238                 return "vmptrld";
 239         case EXIT_REASON_VMPTRST:
 240                 return "vmptrst";
 241         case EXIT_REASON_VMREAD:
 242                 return "vmread";
 243         case EXIT_REASON_VMRESUME:
 244                 return "vmresume";
 245         case EXIT_REASON_VMWRITE:
 246                 return "vmwrite";
 247         case EXIT_REASON_VMXOFF:
 248                 return "vmxoff";
 249         case EXIT_REASON_VMXON:
 250                 return "vmxon";
 251         case EXIT_REASON_CR_ACCESS:
 252                 return "craccess";
 253         case EXIT_REASON_DR_ACCESS:
 254                 return "draccess";
 255         case EXIT_REASON_INOUT:
 256                 return "inout";
 257         case EXIT_REASON_RDMSR:
 258                 return "rdmsr";
 259         case EXIT_REASON_WRMSR:
 260                 return "wrmsr";
 261         case EXIT_REASON_INVAL_VMCS:
 262                 return "invalvmcs";
 263         case EXIT_REASON_INVAL_MSR:
 264                 return "invalmsr";
 265         case EXIT_REASON_MWAIT:
 266                 return "mwait";
 267         case EXIT_REASON_MTF:
 268                 return "mtf";
 269         case EXIT_REASON_MONITOR:
 270                 return "monitor";
 271         case EXIT_REASON_PAUSE:
 272                 return "pause";
 273         case EXIT_REASON_MCE:
 274                 return "mce";
 275         case EXIT_REASON_TPR:
 276                 return "tpr";
 277         case EXIT_REASON_APIC_ACCESS:
 278                 return "apic-access";
 279         case EXIT_REASON_GDTR_IDTR:
 280                 return "gdtridtr";
 281         case EXIT_REASON_LDTR_TR:
 282                 return "ldtrtr";
 283         case EXIT_REASON_EPT_FAULT:
 284                 return "eptfault";
 285         case EXIT_REASON_EPT_MISCONFIG:
 286                 return "eptmisconfig";
 287         case EXIT_REASON_INVEPT:
 288                 return "invept";
 289         case EXIT_REASON_RDTSCP:
 290                 return "rdtscp";
 291         case EXIT_REASON_VMX_PREEMPT:
 292                 return "vmxpreempt";
 293         case EXIT_REASON_INVVPID:
 294                 return "invvpid";
 295         case EXIT_REASON_WBINVD:
 296                 return "wbinvd";
 297         case EXIT_REASON_XSETBV:
 298                 return "xsetbv";
 299         case EXIT_REASON_APIC_WRITE:
 300                 return "apic-write";
 301         default:
 302                 snprintf(reasonbuf, sizeof(reasonbuf), "%d", reason);
 303                 return (reasonbuf);
 304         }
 305 }
 306 #endif  /* KTR */
 307
 308 static int
 309 vmx_allow_x2apic_msrs(struct vmx *vmx)
 310 {
 311         int i, error;
 312
 313         error = 0;
 314
 315         /*
 316          * Allow readonly access to the following x2APIC MSRs from the guest.
 317          */
 318         error += guest_msr_ro(vmx, MSR_APIC_ID);
 319         error += guest_msr_ro(vmx, MSR_APIC_VERSION);
 320         error += guest_msr_ro(vmx, MSR_APIC_LDR);
 321         error += guest_msr_ro(vmx, MSR_APIC_SVR);
 322
 323         for (i = 0; i < 8; i++)
 324                 error += guest_msr_ro(vmx, MSR_APIC_ISR0 + i);
 325
 326         for (i = 0; i < 8; i++)
 327                 error += guest_msr_ro(vmx, MSR_APIC_TMR0 + i);
 328
 329         for (i = 0; i < 8; i++)
 330                 error += guest_msr_ro(vmx, MSR_APIC_IRR0 + i);
 331
 332         error += guest_msr_ro(vmx, MSR_APIC_ESR);
 333         error += guest_msr_ro(vmx, MSR_APIC_LVT_TIMER);
 334         error += guest_msr_ro(vmx, MSR_APIC_LVT_THERMAL);
 335         error += guest_msr_ro(vmx, MSR_APIC_LVT_PCINT);
 336         error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT0);
 337         error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT1);
 338         error += guest_msr_ro(vmx, MSR_APIC_LVT_ERROR);
 339         error += guest_msr_ro(vmx, MSR_APIC_ICR_TIMER);
 340         error += guest_msr_ro(vmx, MSR_APIC_DCR_TIMER);
 341         error += guest_msr_ro(vmx, MSR_APIC_ICR);
 342
 343         /*
 344          * Allow TPR, EOI and SELF_IPI MSRs to be read and written by the guest.
 345          *
 346          * These registers get special treatment described in the section
 347          * "Virtualizing MSR-Based APIC Accesses".
 348          */
 349         error += guest_msr_rw(vmx, MSR_APIC_TPR);
 350         error += guest_msr_rw(vmx, MSR_APIC_EOI);
 351         error += guest_msr_rw(vmx, MSR_APIC_SELF_IPI);
 352
 353         return (error);
 354 }
 355
 356 u_long
 357 vmx_fix_cr0(u_long cr0)
 358 {
 359
 360         return ((cr0 | cr0_ones_mask) & ~cr0_zeros_mask);
 361 }
 362
 363 u_long
 364 vmx_fix_cr4(u_long cr4)
 365 {
 366
 367         return ((cr4 | cr4_ones_mask) & ~cr4_zeros_mask);
 368 }
 369
 370 static void
 371 vpid_free(int vpid)
 372 {
 373         if (vpid < 0 || vpid > 0xffff)
 374                 panic("vpid_free: invalid vpid %d", vpid);
 375
 376         /*
 377          * VPIDs [0,VM_MAXCPU] are special and are not allocated from
 378          * the unit number allocator.
 379          */
 380
 381         if (vpid > VM_MAXCPU)
 382                 free_unr(vpid_unr, vpid);
 383 }
 384
 385 static void
 386 vpid_alloc(uint16_t *vpid, int num)
 387 {
 388         int i, x;
 389
 390         if (num <= 0 || num > VM_MAXCPU)
 391                 panic("invalid number of vpids requested: %d", num);
 392
 393         /*
 394          * If the "enable vpid" execution control is not enabled then the
 395          * VPID is required to be 0 for all vcpus.
 396          */
 397         if ((procbased_ctls2 & PROCBASED2_ENABLE_VPID) == 0) {
 398                 for (i = 0; i < num; i++)
 399                         vpid[i] = 0;
 400                 return;
 401         }
 402
 403         /*
 404          * Allocate a unique VPID for each vcpu from the unit number allocator.
 405          */
 406         for (i = 0; i < num; i++) {
 407                 x = alloc_unr(vpid_unr);
 408                 if (x == -1)
 409                         break;
 410                 else
 411                         vpid[i] = x;
 412         }
 413
 414         if (i < num) {
 415                 atomic_add_int(&vpid_alloc_failed, 1);
 416
 417                 /*
 418                  * If the unit number allocator does not have enough unique
 419                  * VPIDs then we need to allocate from the [1,VM_MAXCPU] range.
 420                  *
 421                  * These VPIDs are not be unique across VMs but this does not
 422                  * affect correctness because the combined mappings are also
 423                  * tagged with the EP4TA which is unique for each VM.
 424                  *
 425                  * It is still sub-optimal because the invvpid will invalidate
 426                  * combined mappings for a particular VPID across all EP4TAs.
 427                  */
 428                 while (i-- > 0)
 429                         vpid_free(vpid[i]);
 430
 431                 for (i = 0; i < num; i++)
 432                         vpid[i] = i + 1;
 433         }
 434 }
 435
 436 static void
 437 vpid_init(void)
 438 {
 439         /*
 440          * VPID 0 is required when the "enable VPID" execution control is
 441          * disabled.
 442          *
 443          * VPIDs [1,VM_MAXCPU] are used as the "overflow namespace" when the
 444          * unit number allocator does not have sufficient unique VPIDs to
 445          * satisfy the allocation.
 446          *
 447          * The remaining VPIDs are managed by the unit number allocator.
 448          */
 449         vpid_unr = new_unrhdr(VM_MAXCPU + 1, 0xffff, NULL);
 450 }
 451
 452 static void
 453 msr_save_area_init(struct msr_entry *g_area, int *g_count)
 454 {
 455         int cnt;
 456
 457         static struct msr_entry guest_msrs[] = {
 458                 { MSR_KGSBASE, 0, 0 },
 459         };
 460
 461         cnt = sizeof(guest_msrs) / sizeof(guest_msrs[0]);
 462         if (cnt > GUEST_MSR_MAX_ENTRIES)
 463                 panic("guest msr save area overrun");
 464         bcopy(guest_msrs, g_area, sizeof(guest_msrs));
 465         *g_count = cnt;
 466 }
 467
 468 static void
 469 vmx_disable(void *arg __unused)
 470 {
 471         struct invvpid_desc invvpid_desc = { 0 };
 472         struct invept_desc invept_desc = { 0 };
 473
 474         if (vmxon_enabled[curcpu]) {
 475                 /*
 476                  * See sections 25.3.3.3 and 25.3.3.4 in Intel Vol 3b.
 477                  *
 478                  * VMXON or VMXOFF are not required to invalidate any TLB
 479                  * caching structures. This prevents potential retention of
 480                  * cached information in the TLB between distinct VMX episodes.
 481                  */
 482                 invvpid(INVVPID_TYPE_ALL_CONTEXTS, invvpid_desc);
 483                 invept(INVEPT_TYPE_ALL_CONTEXTS, invept_desc);
 484                 vmxoff();
 485         }
 486         load_cr4(rcr4() & ~CR4_VMXE);
 487 }
 488
 489 static int
 490 vmx_cleanup(void)
 491 {
 492
 493         if (pirvec != 0)
 494                 vmm_ipi_free(pirvec);
 495
 496         if (vpid_unr != NULL) {
 497                 delete_unrhdr(vpid_unr);
 498                 vpid_unr = NULL;
 499         }
 500
 501         smp_rendezvous(NULL, vmx_disable, NULL, NULL);
 502
 503         return (0);
 504 }
 505
 506 static void
 507 vmx_enable(void *arg __unused)
 508 {
 509         int error;
 510
 511         load_cr4(rcr4() | CR4_VMXE);
 512
 513         *(uint32_t *)vmxon_region[curcpu] = vmx_revision();
 514         error = vmxon(vmxon_region[curcpu]);
 515         if (error == 0)
 516                 vmxon_enabled[curcpu] = 1;
 517 }
 518
 519 static void
 520 vmx_restore(void)
 521 {
 522
 523         if (vmxon_enabled[curcpu])
 524                 vmxon(vmxon_region[curcpu]);
 525 }
 526
 527 static int
 528 vmx_init(int ipinum)
 529 {
 530         int error, use_tpr_shadow;
 531         uint64_t fixed0, fixed1, feature_control;
 532         uint32_t tmp, procbased2_vid_bits;
 533
 534         /* CPUID.1:ECX[bit 5] must be 1 for processor to support VMX */
 535         if (!(cpu_feature2 & CPUID2_VMX)) {
 536                 printf("vmx_init: processor does not support VMX operation\n");
 537                 return (ENXIO);
 538         }
 539
 540         /*
 541          * Verify that MSR_IA32_FEATURE_CONTROL lock and VMXON enable bits
 542          * are set (bits 0 and 2 respectively).
 543          */
 544         feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
 545         if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 0 ||
 546             (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) {
 547                 printf("vmx_init: VMX operation disabled by BIOS\n");
 548                 return (ENXIO);
 549         }
 550
 551         /* Check support for primary processor-based VM-execution controls */
 552         error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 553                                MSR_VMX_TRUE_PROCBASED_CTLS,
 554                                PROCBASED_CTLS_ONE_SETTING,
 555                                PROCBASED_CTLS_ZERO_SETTING, &procbased_ctls);
 556         if (error) {
 557                 printf("vmx_init: processor does not support desired primary "
 558                        "processor-based controls\n");
 559                 return (error);
 560         }
 561
 562         /* Clear the processor-based ctl bits that are set on demand */
 563         procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING;
 564
 565         /* Check support for secondary processor-based VM-execution controls */
 566         error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
 567                                MSR_VMX_PROCBASED_CTLS2,
 568                                PROCBASED_CTLS2_ONE_SETTING,
 569                                PROCBASED_CTLS2_ZERO_SETTING, &procbased_ctls2);
 570         if (error) {
 571                 printf("vmx_init: processor does not support desired secondary "
 572                        "processor-based controls\n");
 573                 return (error);
 574         }
 575
 576         /* Check support for VPID */
 577         error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
 578                                PROCBASED2_ENABLE_VPID, 0, &tmp);
 579         if (error == 0)
 580                 procbased_ctls2 |= PROCBASED2_ENABLE_VPID;
 581
 582         /* Check support for pin-based VM-execution controls */
 583         error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
 584                                MSR_VMX_TRUE_PINBASED_CTLS,
 585                                PINBASED_CTLS_ONE_SETTING,
 586                                PINBASED_CTLS_ZERO_SETTING, &pinbased_ctls);
 587         if (error) {
 588                 printf("vmx_init: processor does not support desired "
 589                        "pin-based controls\n");
 590                 return (error);
 591         }
 592
 593         /* Check support for VM-exit controls */
 594         error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, MSR_VMX_TRUE_EXIT_CTLS,
 595                                VM_EXIT_CTLS_ONE_SETTING,
 596                                VM_EXIT_CTLS_ZERO_SETTING,
 597                                &exit_ctls);
 598         if (error) {
 599                 /* Try again without the PAT MSR bits */
 600                 error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS,
 601                                        MSR_VMX_TRUE_EXIT_CTLS,
 602                                        VM_EXIT_CTLS_ONE_SETTING_NO_PAT,
 603                                        VM_EXIT_CTLS_ZERO_SETTING,
 604                                        &exit_ctls);
 605                 if (error) {
 606                         printf("vmx_init: processor does not support desired "
 607                                "exit controls\n");
 608                         return (error);
 609                 } else {
 610                         if (bootverbose)
 611                                 printf("vmm: PAT MSR access not supported\n");
 612                         guest_msr_valid(MSR_PAT);
 613                         vmx_no_patmsr = 1;
 614                 }
 615         }
 616
 617         /* Check support for VM-entry controls */
 618         if (!vmx_no_patmsr) {
 619                 error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS,
 620                                        MSR_VMX_TRUE_ENTRY_CTLS,
 621                                        VM_ENTRY_CTLS_ONE_SETTING,
 622                                        VM_ENTRY_CTLS_ZERO_SETTING,
 623                                        &entry_ctls);
 624         } else {
 625                 error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS,
 626                                        MSR_VMX_TRUE_ENTRY_CTLS,
 627                                        VM_ENTRY_CTLS_ONE_SETTING_NO_PAT,
 628                                        VM_ENTRY_CTLS_ZERO_SETTING,
 629                                        &entry_ctls);
 630         }
 631
 632         if (error) {
 633                 printf("vmx_init: processor does not support desired "
 634                        "entry controls\n");
 635                        return (error);
 636         }
 637
 638         /*
 639          * Check support for optional features by testing them
 640          * as individual bits
 641          */
 642         cap_halt_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 643                                         MSR_VMX_TRUE_PROCBASED_CTLS,
 644                                         PROCBASED_HLT_EXITING, 0,
 645                                         &tmp) == 0);
 646
 647         cap_monitor_trap = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 648                                         MSR_VMX_PROCBASED_CTLS,
 649                                         PROCBASED_MTF, 0,
 650                                         &tmp) == 0);
 651
 652         cap_pause_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 653                                          MSR_VMX_TRUE_PROCBASED_CTLS,
 654                                          PROCBASED_PAUSE_EXITING, 0,
 655                                          &tmp) == 0);
 656
 657         cap_unrestricted_guest = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
 658                                         MSR_VMX_PROCBASED_CTLS2,
 659                                         PROCBASED2_UNRESTRICTED_GUEST, 0,
 660                                         &tmp) == 0);
 661
 662         cap_invpcid = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
 663             MSR_VMX_PROCBASED_CTLS2, PROCBASED2_ENABLE_INVPCID, 0,
 664             &tmp) == 0);
 665
 666         /*
 667          * Check support for virtual interrupt delivery.
 668          */
 669         procbased2_vid_bits = (PROCBASED2_VIRTUALIZE_APIC_ACCESSES |
 670             PROCBASED2_VIRTUALIZE_X2APIC_MODE |
 671             PROCBASED2_APIC_REGISTER_VIRTUALIZATION |
 672             PROCBASED2_VIRTUAL_INTERRUPT_DELIVERY);
 673
 674         use_tpr_shadow = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 675             MSR_VMX_TRUE_PROCBASED_CTLS, PROCBASED_USE_TPR_SHADOW, 0,
 676             &tmp) == 0);
 677
 678         error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
 679             procbased2_vid_bits, 0, &tmp);
 680         if (error == 0 && use_tpr_shadow) {
 681                 virtual_interrupt_delivery = 1;
 682                 TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_vid",
 683                     &virtual_interrupt_delivery);
 684         }
 685
 686         if (virtual_interrupt_delivery) {
 687                 procbased_ctls |= PROCBASED_USE_TPR_SHADOW;
 688                 procbased_ctls2 |= procbased2_vid_bits;
 689                 procbased_ctls2 &= ~PROCBASED2_VIRTUALIZE_X2APIC_MODE;
 690
 691                 /*
 692                  * Check for Posted Interrupts only if Virtual Interrupt
 693                  * Delivery is enabled.
 694                  */
 695                 error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
 696                     MSR_VMX_TRUE_PINBASED_CTLS, PINBASED_POSTED_INTERRUPT, 0,
 697                     &tmp);
 698                 if (error == 0) {
 699                         pirvec = vmm_ipi_alloc();
 700                         if (pirvec == 0) {
 701                                 if (bootverbose) {
 702                                         printf("vmx_init: unable to allocate "
 703                                             "posted interrupt vector\n");
 704                                 }
 705                         } else {
 706                                 posted_interrupts = 1;
 707                                 TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_pir",
 708                                     &posted_interrupts);
 709                         }
 710                 }
 711         }
 712
 713         if (posted_interrupts)
 714                     pinbased_ctls |= PINBASED_POSTED_INTERRUPT;
 715
 716         /* Initialize EPT */
 717         error = ept_init(ipinum);
 718         if (error) {
 719                 printf("vmx_init: ept initialization failed (%d)\n", error);
 720                 return (error);
 721         }
 722
 723         /*
 724          * Stash the cr0 and cr4 bits that must be fixed to 0 or 1
 725          */
 726         fixed0 = rdmsr(MSR_VMX_CR0_FIXED0);
 727         fixed1 = rdmsr(MSR_VMX_CR0_FIXED1);
 728         cr0_ones_mask = fixed0 & fixed1;
 729         cr0_zeros_mask = ~fixed0 & ~fixed1;
 730
 731         /*
 732          * CR0_PE and CR0_PG can be set to zero in VMX non-root operation
 733          * if unrestricted guest execution is allowed.
 734          */
 735         if (cap_unrestricted_guest)
 736                 cr0_ones_mask &= ~(CR0_PG | CR0_PE);
 737
 738         /*
 739          * Do not allow the guest to set CR0_NW or CR0_CD.
 740          */
 741         cr0_zeros_mask |= (CR0_NW | CR0_CD);
 742
 743         fixed0 = rdmsr(MSR_VMX_CR4_FIXED0);
 744         fixed1 = rdmsr(MSR_VMX_CR4_FIXED1);
 745         cr4_ones_mask = fixed0 & fixed1;
 746         cr4_zeros_mask = ~fixed0 & ~fixed1;
 747
 748         vpid_init();
 749
 750         /* enable VMX operation */
 751         smp_rendezvous(NULL, vmx_enable, NULL, NULL);
 752
 753         vmx_initialized = 1;
 754
 755         return (0);
 756 }
 757
 758 static void
 759 vmx_trigger_hostintr(int vector)
 760 {
 761         uintptr_t func;
 762         struct gate_descriptor *gd;
 763
 764         gd = &idt[vector];
 765
 766         KASSERT(vector >= 32 && vector <= 255, ("vmx_trigger_hostintr: "
 767             "invalid vector %d", vector));
 768         KASSERT(gd->gd_p == 1, ("gate descriptor for vector %d not present",
 769             vector));
 770         KASSERT(gd->gd_type == SDT_SYSIGT, ("gate descriptor for vector %d "
 771             "has invalid type %d", vector, gd->gd_type));
 772         KASSERT(gd->gd_dpl == SEL_KPL, ("gate descriptor for vector %d "
 773             "has invalid dpl %d", vector, gd->gd_dpl));
 774         KASSERT(gd->gd_selector == GSEL(GCODE_SEL, SEL_KPL), ("gate descriptor "
 775             "for vector %d has invalid selector %d", vector, gd->gd_selector));
 776         KASSERT(gd->gd_ist == 0, ("gate descriptor for vector %d has invalid "
 777             "IST %d", vector, gd->gd_ist));
 778
 779         func = ((long)gd->gd_hioffset << 16 | gd->gd_looffset);
 780         vmx_call_isr(func);
 781 }
 782
 783 static int
 784 vmx_setup_cr_shadow(int which, struct vmcs *vmcs, uint32_t initial)
 785 {
 786         int error, mask_ident, shadow_ident;
 787         uint64_t mask_value;
 788
 789         if (which != 0 && which != 4)
 790                 panic("vmx_setup_cr_shadow: unknown cr%d", which);
 791
 792         if (which == 0) {
 793                 mask_ident = VMCS_CR0_MASK;
 794                 mask_value = cr0_ones_mask | cr0_zeros_mask;
 795                 shadow_ident = VMCS_CR0_SHADOW;
 796         } else {
 797                 mask_ident = VMCS_CR4_MASK;
 798                 mask_value = cr4_ones_mask | cr4_zeros_mask;
 799                 shadow_ident = VMCS_CR4_SHADOW;
 800         }
 801
 802         error = vmcs_setreg(vmcs, 0, VMCS_IDENT(mask_ident), mask_value);
 803         if (error)
 804                 return (error);
 805
 806         error = vmcs_setreg(vmcs, 0, VMCS_IDENT(shadow_ident), initial);
 807         if (error)
 808                 return (error);
 809
 810         return (0);
 811 }
 812 #define vmx_setup_cr0_shadow(vmcs,init) vmx_setup_cr_shadow(0, (vmcs), (init))
 813 #define vmx_setup_cr4_shadow(vmcs,init) vmx_setup_cr_shadow(4, (vmcs), (init))
 814
 815 static void *
 816 vmx_vminit(struct vm *vm, pmap_t pmap)
 817 {
 818         uint16_t vpid[VM_MAXCPU];
 819         int i, error, guest_msr_count;
 820         struct vmx *vmx;
 821         struct vmcs *vmcs;
 822
 823         vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO);
 824         if ((uintptr_t)vmx & PAGE_MASK) {
 825                 panic("malloc of struct vmx not aligned on %d byte boundary",
 826                       PAGE_SIZE);
 827         }
 828         vmx->vm = vm;
 829
 830         vmx->eptp = eptp(vtophys((vm_offset_t)pmap->pm_pml4));
 831
 832         /*
 833          * Clean up EPTP-tagged guest physical and combined mappings
 834          *
 835          * VMX transitions are not required to invalidate any guest physical
 836          * mappings. So, it may be possible for stale guest physical mappings
 837          * to be present in the processor TLBs.
 838          *
 839          * Combined mappings for this EP4TA are also invalidated for all VPIDs.
 840          */
 841         ept_invalidate_mappings(vmx->eptp);
 842
 843         msr_bitmap_initialize(vmx->msr_bitmap);
 844
 845         /*
 846          * It is safe to allow direct access to MSR_GSBASE and MSR_FSBASE.
 847          * The guest FSBASE and GSBASE are saved and restored during
 848          * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are
 849          * always restored from the vmcs host state area on vm-exit.
 850          *
 851          * The SYSENTER_CS/ESP/EIP MSRs are identical to FS/GSBASE in
 852          * how they are saved/restored so can be directly accessed by the
 853          * guest.
 854          *
 855          * Guest KGSBASE is saved and restored in the guest MSR save area.
 856          * Host KGSBASE is restored before returning to userland from the pcb.
 857          * There will be a window of time when we are executing in the host
 858          * kernel context with a value of KGSBASE from the guest. This is ok
 859          * because the value of KGSBASE is inconsequential in kernel context.
 860          *
 861          * MSR_EFER is saved and restored in the guest VMCS area on a
 862          * VM exit and entry respectively. It is also restored from the
 863          * host VMCS area on a VM exit.
 864          */
 865         if (guest_msr_rw(vmx, MSR_GSBASE) ||
 866             guest_msr_rw(vmx, MSR_FSBASE) ||
 867             guest_msr_rw(vmx, MSR_SYSENTER_CS_MSR) ||
 868             guest_msr_rw(vmx, MSR_SYSENTER_ESP_MSR) ||
 869             guest_msr_rw(vmx, MSR_SYSENTER_EIP_MSR) ||
 870             guest_msr_rw(vmx, MSR_KGSBASE) ||
 871             guest_msr_rw(vmx, MSR_EFER))
 872                 panic("vmx_vminit: error setting guest msr access");
 873
 874         /*
 875          * MSR_PAT is saved and restored in the guest VMCS are on a VM exit
 876          * and entry respectively. It is also restored from the host VMCS
 877          * area on a VM exit. However, if running on a system with no
 878          * MSR_PAT save/restore support, leave access disabled so accesses
 879          * will be trapped.
 880          */
 881         if (!vmx_no_patmsr && guest_msr_rw(vmx, MSR_PAT))
 882                 panic("vmx_vminit: error setting guest pat msr access");
 883
 884         vpid_alloc(vpid, VM_MAXCPU);
 885
 886         if (virtual_interrupt_delivery) {
 887                 error = vm_map_mmio(vm, DEFAULT_APIC_BASE, PAGE_SIZE,
 888                     APIC_ACCESS_ADDRESS);
 889                 /* XXX this should really return an error to the caller */
 890                 KASSERT(error == 0, ("vm_map_mmio(apicbase) error %d", error));
 891         }
 892
 893         for (i = 0; i < VM_MAXCPU; i++) {
 894                 vmcs = &vmx->vmcs[i];
 895                 vmcs->identifier = vmx_revision();
 896                 error = vmclear(vmcs);
 897                 if (error != 0) {
 898                         panic("vmx_vminit: vmclear error %d on vcpu %d\n",
 899                               error, i);
 900                 }
 901
 902                 error = vmcs_init(vmcs);
 903                 KASSERT(error == 0, ("vmcs_init error %d", error));
 904
 905                 VMPTRLD(vmcs);
 906                 error = 0;
 907                 error += vmwrite(VMCS_HOST_RSP, (u_long)&vmx->ctx[i]);
 908                 error += vmwrite(VMCS_EPTP, vmx->eptp);
 909                 error += vmwrite(VMCS_PIN_BASED_CTLS, pinbased_ctls);
 910                 error += vmwrite(VMCS_PRI_PROC_BASED_CTLS, procbased_ctls);
 911                 error += vmwrite(VMCS_SEC_PROC_BASED_CTLS, procbased_ctls2);
 912                 error += vmwrite(VMCS_EXIT_CTLS, exit_ctls);
 913                 error += vmwrite(VMCS_ENTRY_CTLS, entry_ctls);
 914                 error += vmwrite(VMCS_MSR_BITMAP, vtophys(vmx->msr_bitmap));
 915                 error += vmwrite(VMCS_VPID, vpid[i]);
 916                 if (virtual_interrupt_delivery) {
 917                         error += vmwrite(VMCS_APIC_ACCESS, APIC_ACCESS_ADDRESS);
 918                         error += vmwrite(VMCS_VIRTUAL_APIC,
 919                             vtophys(&vmx->apic_page[i]));
 920                         error += vmwrite(VMCS_EOI_EXIT0, 0);
 921                         error += vmwrite(VMCS_EOI_EXIT1, 0);
 922                         error += vmwrite(VMCS_EOI_EXIT2, 0);
 923                         error += vmwrite(VMCS_EOI_EXIT3, 0);
 924                 }
 925                 if (posted_interrupts) {
 926                         error += vmwrite(VMCS_PIR_VECTOR, pirvec);
 927                         error += vmwrite(VMCS_PIR_DESC,
 928                             vtophys(&vmx->pir_desc[i]));
 929                 }
 930                 VMCLEAR(vmcs);
 931                 KASSERT(error == 0, ("vmx_vminit: error customizing the vmcs"));
 932
 933                 vmx->cap[i].set = 0;
 934                 vmx->cap[i].proc_ctls = procbased_ctls;
 935                 vmx->cap[i].proc_ctls2 = procbased_ctls2;
 936
 937                 vmx->state[i].lastcpu = -1;
 938                 vmx->state[i].vpid = vpid[i];
 939
 940                 msr_save_area_init(vmx->guest_msrs[i], &guest_msr_count);
 941
 942                 error = vmcs_set_msr_save(vmcs, vtophys(vmx->guest_msrs[i]),
 943                     guest_msr_count);
 944                 if (error != 0)
 945                         panic("vmcs_set_msr_save error %d", error);
 946
 947                 /*
 948                  * Set up the CR0/4 shadows, and init the read shadow
 949                  * to the power-on register value from the Intel Sys Arch.
 950                  *  CR0 - 0x60000010
 951                  *  CR4 - 0
 952                  */
 953                 error = vmx_setup_cr0_shadow(vmcs, 0x60000010);
 954                 if (error != 0)
 955                         panic("vmx_setup_cr0_shadow %d", error);
 956
 957                 error = vmx_setup_cr4_shadow(vmcs, 0);
 958                 if (error != 0)
 959                         panic("vmx_setup_cr4_shadow %d", error);
 960
 961                 vmx->ctx[i].pmap = pmap;
 962         }
 963
 964         return (vmx);
 965 }
 966
 967 static int
 968 vmx_handle_cpuid(struct vm *vm, int vcpu, struct vmxctx *vmxctx)
 969 {
 970         int handled, func;
 971
 972         func = vmxctx->guest_rax;
 973
 974         handled = x86_emulate_cpuid(vm, vcpu,
 975                                     (uint32_t*)(&vmxctx->guest_rax),
 976                                     (uint32_t*)(&vmxctx->guest_rbx),
 977                                     (uint32_t*)(&vmxctx->guest_rcx),
 978                                     (uint32_t*)(&vmxctx->guest_rdx));
 979         return (handled);
 980 }
 981
 982 static __inline void
 983 vmx_run_trace(struct vmx *vmx, int vcpu)
 984 {
 985 #ifdef KTR
 986         VCPU_CTR1(vmx->vm, vcpu, "Resume execution at %#lx", vmcs_guest_rip());
 987 #endif
 988 }
 989
 990 static __inline void
 991 vmx_exit_trace(struct vmx *vmx, int vcpu, uint64_t rip, uint32_t exit_reason,
 992                int handled)
 993 {
 994 #ifdef KTR
 995         VCPU_CTR3(vmx->vm, vcpu, "%s %s vmexit at 0x%0lx",
 996                  handled ? "handled" : "unhandled",
 997                  exit_reason_to_str(exit_reason), rip);
 998 #endif
 999 }
1000
1001 static __inline void
1002 vmx_astpending_trace(struct vmx *vmx, int vcpu, uint64_t rip)
1003 {
1004 #ifdef KTR
1005         VCPU_CTR1(vmx->vm, vcpu, "astpending vmexit at 0x%0lx", rip);
1006 #endif
1007 }
1008
1009 static VMM_STAT_INTEL(VCPU_INVVPID_SAVED, "Number of vpid invalidations saved");
1010
1011 static void
1012 vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu, pmap_t pmap)
1013 {
1014         struct vmxstate *vmxstate;
1015         struct invvpid_desc invvpid_desc;
1016
1017         vmxstate = &vmx->state[vcpu];
1018         if (vmxstate->lastcpu == curcpu)
1019                 return;
1020
1021         vmxstate->lastcpu = curcpu;
1022
1023         vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1);
1024
1025         vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase());
1026         vmcs_write(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase());
1027         vmcs_write(VMCS_HOST_GS_BASE, vmm_get_host_gsbase());
1028
1029         /*
1030          * If we are using VPIDs then invalidate all mappings tagged with 'vpid'
1031          *
1032          * We do this because this vcpu was executing on a different host
1033          * cpu when it last ran. We do not track whether it invalidated
1034          * mappings associated with its 'vpid' during that run. So we must
1035          * assume that the mappings associated with 'vpid' on 'curcpu' are
1036          * stale and invalidate them.
1037          *
1038          * Note that we incur this penalty only when the scheduler chooses to
1039          * move the thread associated with this vcpu between host cpus.
1040          *
1041          * Note also that this will invalidate mappings tagged with 'vpid'
1042          * for "all" EP4TAs.
1043          */
1044         if (vmxstate->vpid != 0) {
1045                 if (pmap->pm_eptgen == vmx->eptgen[curcpu]) {
1046                         invvpid_desc._res1 = 0;
1047                         invvpid_desc._res2 = 0;
1048                         invvpid_desc.vpid = vmxstate->vpid;
1049                         invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc);
1050                 } else {
1051                         /*
1052                          * The invvpid can be skipped if an invept is going to
1053                          * be performed before entering the guest. The invept
1054                          * will invalidate combined mappings tagged with
1055                          * 'vmx->eptp' for all vpids.
1056                          */
1057                         vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_SAVED, 1);
1058                 }
1059         }
1060 }
1061
1062 /*
1063  * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set.
1064  */
1065 CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0);
1066
1067 static void __inline
1068 vmx_set_int_window_exiting(struct vmx *vmx, int vcpu)
1069 {
1070
1071         if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) == 0) {
1072                 vmx->cap[vcpu].proc_ctls |= PROCBASED_INT_WINDOW_EXITING;
1073                 vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1074                 VCPU_CTR0(vmx->vm, vcpu, "Enabling interrupt window exiting");
1075         }
1076 }
1077
1078 static void __inline
1079 vmx_clear_int_window_exiting(struct vmx *vmx, int vcpu)
1080 {
1081
1082         KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0,
1083             ("intr_window_exiting not set: %#x", vmx->cap[vcpu].proc_ctls));
1084         vmx->cap[vcpu].proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING;
1085         vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1086         VCPU_CTR0(vmx->vm, vcpu, "Disabling interrupt window exiting");
1087 }
1088
1089 static void __inline
1090 vmx_set_nmi_window_exiting(struct vmx *vmx, int vcpu)
1091 {
1092
1093         if ((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) == 0) {
1094                 vmx->cap[vcpu].proc_ctls |= PROCBASED_NMI_WINDOW_EXITING;
1095                 vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1096                 VCPU_CTR0(vmx->vm, vcpu, "Enabling NMI window exiting");
1097         }
1098 }
1099
1100 static void __inline
1101 vmx_clear_nmi_window_exiting(struct vmx *vmx, int vcpu)
1102 {
1103
1104         KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) != 0,
1105             ("nmi_window_exiting not set %#x", vmx->cap[vcpu].proc_ctls));
1106         vmx->cap[vcpu].proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING;
1107         vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1108         VCPU_CTR0(vmx->vm, vcpu, "Disabling NMI window exiting");
1109 }
1110
1111 #define NMI_BLOCKING    (VMCS_INTERRUPTIBILITY_NMI_BLOCKING |           \
1112                          VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
1113 #define HWINTR_BLOCKING (VMCS_INTERRUPTIBILITY_STI_BLOCKING |           \
1114                          VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
1115
1116 static void
1117 vmx_inject_nmi(struct vmx *vmx, int vcpu)
1118 {
1119         uint32_t gi, info;
1120
1121         gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1122         KASSERT((gi & NMI_BLOCKING) == 0, ("vmx_inject_nmi: invalid guest "
1123             "interruptibility-state %#x", gi));
1124
1125         info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1126         KASSERT((info & VMCS_INTR_VALID) == 0, ("vmx_inject_nmi: invalid "
1127             "VM-entry interruption information %#x", info));
1128
1129         /*
1130          * Inject the virtual NMI. The vector must be the NMI IDT entry
1131          * or the VMCS entry check will fail.
1132          */
1133         info = IDT_NMI | VMCS_INTR_T_NMI | VMCS_INTR_VALID;
1134         vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1135
1136         VCPU_CTR0(vmx->vm, vcpu, "Injecting vNMI");
1137
1138         /* Clear the request */
1139         vm_nmi_clear(vmx->vm, vcpu);
1140 }
1141
1142 static void
1143 vmx_inject_interrupts(struct vmx *vmx, int vcpu, struct vlapic *vlapic)
1144 {
1145         struct vm_exception exc;
1146         int vector, need_nmi_exiting;
1147         uint64_t rflags;
1148         uint32_t gi, info;
1149
1150         if (vm_exception_pending(vmx->vm, vcpu, &exc)) {
1151                 KASSERT(exc.vector >= 0 && exc.vector < 32,
1152                     ("%s: invalid exception vector %d", __func__, exc.vector));
1153
1154                 info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1155                 KASSERT((info & VMCS_INTR_VALID) == 0, ("%s: cannot inject "
1156                      "pending exception %d: %#x", __func__, exc.vector, info));
1157
1158                 info = exc.vector | VMCS_INTR_T_HWEXCEPTION | VMCS_INTR_VALID;
1159                 if (exc.error_code_valid) {
1160                         info |= VMCS_INTR_DEL_ERRCODE;
1161                         vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR, exc.error_code);
1162                 }
1163                 vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1164         }
1165
1166         if (vm_nmi_pending(vmx->vm, vcpu)) {
1167                 /*
1168                  * If there are no conditions blocking NMI injection then
1169                  * inject it directly here otherwise enable "NMI window
1170                  * exiting" to inject it as soon as we can.
1171                  *
1172                  * We also check for STI_BLOCKING because some implementations
1173                  * don't allow NMI injection in this case. If we are running
1174                  * on a processor that doesn't have this restriction it will
1175                  * immediately exit and the NMI will be injected in the
1176                  * "NMI window exiting" handler.
1177                  */
1178                 need_nmi_exiting = 1;
1179                 gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1180                 if ((gi & (HWINTR_BLOCKING | NMI_BLOCKING)) == 0) {
1181                         info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1182                         if ((info & VMCS_INTR_VALID) == 0) {
1183                                 vmx_inject_nmi(vmx, vcpu);
1184                                 need_nmi_exiting = 0;
1185                         } else {
1186                                 VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI "
1187                                     "due to VM-entry intr info %#x", info);
1188                         }
1189                 } else {
1190                         VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI due to "
1191                             "Guest Interruptibility-state %#x", gi);
1192                 }
1193
1194                 if (need_nmi_exiting)
1195                         vmx_set_nmi_window_exiting(vmx, vcpu);
1196         }
1197
1198         if (virtual_interrupt_delivery) {
1199                 vmx_inject_pir(vlapic);
1200                 return;
1201         }
1202
1203         /*
1204          * If interrupt-window exiting is already in effect then don't bother
1205          * checking for pending interrupts. This is just an optimization and
1206          * not needed for correctness.
1207          */
1208         if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0) {
1209                 VCPU_CTR0(vmx->vm, vcpu, "Skip interrupt injection due to "
1210                     "pending int_window_exiting");
1211                 return;
1212         }
1213
1214         /* Ask the local apic for a vector to inject */
1215         if (!vlapic_pending_intr(vlapic, &vector))
1216                 return;
1217
1218         KASSERT(vector >= 32 && vector <= 255, ("invalid vector %d", vector));
1219
1220         /* Check RFLAGS.IF and the interruptibility state of the guest */
1221         rflags = vmcs_read(VMCS_GUEST_RFLAGS);
1222         if ((rflags & PSL_I) == 0) {
1223                 VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
1224                     "rflags %#lx", vector, rflags);
1225                 goto cantinject;
1226         }
1227
1228         gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1229         if (gi & HWINTR_BLOCKING) {
1230                 VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
1231                     "Guest Interruptibility-state %#x", vector, gi);
1232                 goto cantinject;
1233         }
1234
1235         info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1236         if (info & VMCS_INTR_VALID) {
1237                 /*
1238                  * This is expected and could happen for multiple reasons:
1239                  * - A vectoring VM-entry was aborted due to astpending
1240                  * - A VM-exit happened during event injection.
1241                  * - An exception was injected above.
1242                  * - An NMI was injected above or after "NMI window exiting"
1243                  */
1244                 VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
1245                     "VM-entry intr info %#x", vector, info);
1246                 goto cantinject;
1247         }
1248
1249         /* Inject the interrupt */
1250         info = VMCS_INTR_T_HWINTR | VMCS_INTR_VALID;
1251         info |= vector;
1252         vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1253
1254         /* Update the Local APIC ISR */
1255         vlapic_intr_accepted(vlapic, vector);
1256
1257         VCPU_CTR1(vmx->vm, vcpu, "Injecting hwintr at vector %d", vector);
1258
1259         return;
1260
1261 cantinject:
1262         /*
1263          * Set the Interrupt Window Exiting execution control so we can inject
1264          * the interrupt as soon as blocking condition goes away.
1265          */
1266         vmx_set_int_window_exiting(vmx, vcpu);
1267 }
1268
1269 /*
1270  * If the Virtual NMIs execution control is '1' then the logical processor
1271  * tracks virtual-NMI blocking in the Guest Interruptibility-state field of
1272  * the VMCS. An IRET instruction in VMX non-root operation will remove any
1273  * virtual-NMI blocking.
1274  *
1275  * This unblocking occurs even if the IRET causes a fault. In this case the
1276  * hypervisor needs to restore virtual-NMI blocking before resuming the guest.
1277  */
1278 static void
1279 vmx_restore_nmi_blocking(struct vmx *vmx, int vcpuid)
1280 {
1281         uint32_t gi;
1282
1283         VCPU_CTR0(vmx->vm, vcpuid, "Restore Virtual-NMI blocking");
1284         gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1285         gi |= VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
1286         vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1287 }
1288
1289 static void
1290 vmx_clear_nmi_blocking(struct vmx *vmx, int vcpuid)
1291 {
1292         uint32_t gi;
1293
1294         VCPU_CTR0(vmx->vm, vcpuid, "Clear Virtual-NMI blocking");
1295         gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1296         gi &= ~VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
1297         vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1298 }
1299
1300 static int
1301 vmx_emulate_xsetbv(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
1302 {
1303         struct vmxctx *vmxctx;
1304         uint64_t xcrval;
1305         const struct xsave_limits *limits;
1306
1307         vmxctx = &vmx->ctx[vcpu];
1308         limits = vmm_get_xsave_limits();
1309
1310         /*
1311          * Note that the processor raises a GP# fault on its own if
1312          * xsetbv is executed for CPL != 0, so we do not have to
1313          * emulate that fault here.
1314          */
1315
1316         /* Only xcr0 is supported. */
1317         if (vmxctx->guest_rcx != 0) {
1318                 vm_inject_gp(vmx->vm, vcpu);
1319                 return (HANDLED);
1320         }
1321
1322         /* We only handle xcr0 if both the host and guest have XSAVE enabled. */
1323         if (!limits->xsave_enabled || !(vmcs_read(VMCS_GUEST_CR4) & CR4_XSAVE)) {
1324                 vm_inject_ud(vmx->vm, vcpu);
1325                 return (HANDLED);
1326         }
1327
1328         xcrval = vmxctx->guest_rdx << 32 | (vmxctx->guest_rax & 0xffffffff);
1329         if ((xcrval & ~limits->xcr0_allowed) != 0) {
1330                 vm_inject_gp(vmx->vm, vcpu);
1331                 return (HANDLED);
1332         }
1333
1334         if (!(xcrval & XFEATURE_ENABLED_X87)) {
1335                 vm_inject_gp(vmx->vm, vcpu);
1336                 return (HANDLED);
1337         }
1338
1339         if ((xcrval & (XFEATURE_ENABLED_AVX | XFEATURE_ENABLED_SSE)) ==
1340             XFEATURE_ENABLED_AVX) {
1341                 vm_inject_gp(vmx->vm, vcpu);
1342                 return (HANDLED);
1343         }
1344
1345         /*
1346          * This runs "inside" vmrun() with the guest's FPU state, so
1347          * modifying xcr0 directly modifies the guest's xcr0, not the
1348          * host's.
1349          */
1350         load_xcr(0, xcrval);
1351         return (HANDLED);
1352 }
1353
1354 static int
1355 vmx_emulate_cr_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
1356 {
1357         int cr, vmcs_guest_cr, vmcs_shadow_cr;
1358         uint64_t crval, regval, ones_mask, zeros_mask;
1359         const struct vmxctx *vmxctx;
1360
1361         /* We only handle mov to %cr0 or %cr4 at this time */
1362         if ((exitqual & 0xf0) != 0x00)
1363                 return (UNHANDLED);
1364
1365         cr = exitqual & 0xf;
1366         if (cr != 0 && cr != 4)
1367                 return (UNHANDLED);
1368
1369         regval = 0; /* silence gcc */
1370         vmxctx = &vmx->ctx[vcpu];
1371
1372         /*
1373          * We must use vmcs_write() directly here because vmcs_setreg() will
1374          * call vmclear(vmcs) as a side-effect which we certainly don't want.
1375          */
1376         switch ((exitqual >> 8) & 0xf) {
1377         case 0:
1378                 regval = vmxctx->guest_rax;
1379                 break;
1380         case 1:
1381                 regval = vmxctx->guest_rcx;
1382                 break;
1383         case 2:
1384                 regval = vmxctx->guest_rdx;
1385                 break;
1386         case 3:
1387                 regval = vmxctx->guest_rbx;
1388                 break;
1389         case 4:
1390                 regval = vmcs_read(VMCS_GUEST_RSP);
1391                 break;
1392         case 5:
1393                 regval = vmxctx->guest_rbp;
1394                 break;
1395         case 6:
1396                 regval = vmxctx->guest_rsi;
1397                 break;
1398         case 7:
1399                 regval = vmxctx->guest_rdi;
1400                 break;
1401         case 8:
1402                 regval = vmxctx->guest_r8;
1403                 break;
1404         case 9:
1405                 regval = vmxctx->guest_r9;
1406                 break;
1407         case 10:
1408                 regval = vmxctx->guest_r10;
1409                 break;
1410         case 11:
1411                 regval = vmxctx->guest_r11;
1412                 break;
1413         case 12:
1414                 regval = vmxctx->guest_r12;
1415                 break;
1416         case 13:
1417                 regval = vmxctx->guest_r13;
1418                 break;
1419         case 14:
1420                 regval = vmxctx->guest_r14;
1421                 break;
1422         case 15:
1423                 regval = vmxctx->guest_r15;
1424                 break;
1425         }
1426
1427         if (cr == 0) {
1428                 ones_mask = cr0_ones_mask;
1429                 zeros_mask = cr0_zeros_mask;
1430                 vmcs_guest_cr = VMCS_GUEST_CR0;
1431                 vmcs_shadow_cr = VMCS_CR0_SHADOW;
1432         } else {
1433                 ones_mask = cr4_ones_mask;
1434                 zeros_mask = cr4_zeros_mask;
1435                 vmcs_guest_cr = VMCS_GUEST_CR4;
1436                 vmcs_shadow_cr = VMCS_CR4_SHADOW;
1437         }
1438         vmcs_write(vmcs_shadow_cr, regval);
1439
1440         crval = regval | ones_mask;
1441         crval &= ~zeros_mask;
1442         vmcs_write(vmcs_guest_cr, crval);
1443
1444         if (cr == 0 && regval & CR0_PG) {
1445                 uint64_t efer, entry_ctls;
1446
1447                 /*
1448                  * If CR0.PG is 1 and EFER.LME is 1 then EFER.LMA and
1449                  * the "IA-32e mode guest" bit in VM-entry control must be
1450                  * equal.
1451                  */
1452                 efer = vmcs_read(VMCS_GUEST_IA32_EFER);
1453                 if (efer & EFER_LME) {
1454                         efer |= EFER_LMA;
1455                         vmcs_write(VMCS_GUEST_IA32_EFER, efer);
1456                         entry_ctls = vmcs_read(VMCS_ENTRY_CTLS);
1457                         entry_ctls |= VM_ENTRY_GUEST_LMA;
1458                         vmcs_write(VMCS_ENTRY_CTLS, entry_ctls);
1459                 }
1460         }
1461
1462         return (HANDLED);
1463 }
1464
1465 static enum vie_cpu_mode
1466 vmx_cpu_mode(void)
1467 {
1468
1469         if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LMA)
1470                 return (CPU_MODE_64BIT);
1471         else
1472                 return (CPU_MODE_COMPATIBILITY);
1473 }
1474
1475 static enum vie_paging_mode
1476 vmx_paging_mode(void)
1477 {
1478
1479         if (!(vmcs_read(VMCS_GUEST_CR0) & CR0_PG))
1480                 return (PAGING_MODE_FLAT);
1481         if (!(vmcs_read(VMCS_GUEST_CR4) & CR4_PAE))
1482                 return (PAGING_MODE_32);
1483         if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LME)
1484                 return (PAGING_MODE_64);
1485         else
1486                 return (PAGING_MODE_PAE);
1487 }
1488
1489 static int
1490 ept_fault_type(uint64_t ept_qual)
1491 {
1492         int fault_type;
1493
1494         if (ept_qual & EPT_VIOLATION_DATA_WRITE)
1495                 fault_type = VM_PROT_WRITE;
1496         else if (ept_qual & EPT_VIOLATION_INST_FETCH)
1497                 fault_type = VM_PROT_EXECUTE;
1498         else
1499                 fault_type= VM_PROT_READ;
1500
1501         return (fault_type);
1502 }
1503
1504 static boolean_t
1505 ept_emulation_fault(uint64_t ept_qual)
1506 {
1507         int read, write;
1508
1509         /* EPT fault on an instruction fetch doesn't make sense here */
1510         if (ept_qual & EPT_VIOLATION_INST_FETCH)
1511                 return (FALSE);
1512
1513         /* EPT fault must be a read fault or a write fault */
1514         read = ept_qual & EPT_VIOLATION_DATA_READ ? 1 : 0;
1515         write = ept_qual & EPT_VIOLATION_DATA_WRITE ? 1 : 0;
1516         if ((read | write) == 0)
1517                 return (FALSE);
1518
1519         /*
1520          * The EPT violation must have been caused by accessing a
1521          * guest-physical address that is a translation of a guest-linear
1522          * address.
1523          */
1524         if ((ept_qual & EPT_VIOLATION_GLA_VALID) == 0 ||
1525             (ept_qual & EPT_VIOLATION_XLAT_VALID) == 0) {
1526                 return (FALSE);
1527         }
1528
1529         return (TRUE);
1530 }
1531
1532 static __inline int
1533 apic_access_virtualization(struct vmx *vmx, int vcpuid)
1534 {
1535         uint32_t proc_ctls2;
1536
1537         proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
1538         return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) ? 1 : 0);
1539 }
1540
1541 static __inline int
1542 x2apic_virtualization(struct vmx *vmx, int vcpuid)
1543 {
1544         uint32_t proc_ctls2;
1545
1546         proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
1547         return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_X2APIC_MODE) ? 1 : 0);
1548 }
1549
1550 static int
1551 vmx_handle_apic_write(struct vmx *vmx, int vcpuid, struct vlapic *vlapic,
1552     uint64_t qual)
1553 {
1554         int error, handled, offset;
1555         uint32_t *apic_regs, vector;
1556         bool retu;
1557
1558         handled = HANDLED;
1559         offset = APIC_WRITE_OFFSET(qual);
1560
1561         if (!apic_access_virtualization(vmx, vcpuid)) {
1562                 /*
1563                  * In general there should not be any APIC write VM-exits
1564                  * unless APIC-access virtualization is enabled.
1565                  *
1566                  * However self-IPI virtualization can legitimately trigger
1567                  * an APIC-write VM-exit so treat it specially.
1568                  */
1569                 if (x2apic_virtualization(vmx, vcpuid) &&
1570                     offset == APIC_OFFSET_SELF_IPI) {
1571                         apic_regs = (uint32_t *)(vlapic->apic_page);
1572                         vector = apic_regs[APIC_OFFSET_SELF_IPI / 4];
1573                         vlapic_self_ipi_handler(vlapic, vector);
1574                         return (HANDLED);
1575                 } else
1576                         return (UNHANDLED);
1577         }
1578
1579         switch (offset) {
1580         case APIC_OFFSET_ID:
1581                 vlapic_id_write_handler(vlapic);
1582                 break;
1583         case APIC_OFFSET_LDR:
1584                 vlapic_ldr_write_handler(vlapic);
1585                 break;
1586         case APIC_OFFSET_DFR:
1587                 vlapic_dfr_write_handler(vlapic);
1588                 break;
1589         case APIC_OFFSET_SVR:
1590                 vlapic_svr_write_handler(vlapic);
1591                 break;
1592         case APIC_OFFSET_ESR:
1593                 vlapic_esr_write_handler(vlapic);
1594                 break;
1595         case APIC_OFFSET_ICR_LOW:
1596                 retu = false;
1597                 error = vlapic_icrlo_write_handler(vlapic, &retu);
1598                 if (error != 0 || retu)
1599                         handled = UNHANDLED;
1600                 break;
1601         case APIC_OFFSET_CMCI_LVT:
1602         case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
1603                 vlapic_lvt_write_handler(vlapic, offset);
1604                 break;
1605         case APIC_OFFSET_TIMER_ICR:
1606                 vlapic_icrtmr_write_handler(vlapic);
1607                 break;
1608         case APIC_OFFSET_TIMER_DCR:
1609                 vlapic_dcr_write_handler(vlapic);
1610                 break;
1611         default:
1612                 handled = UNHANDLED;
1613                 break;
1614         }
1615         return (handled);
1616 }
1617
1618 static bool
1619 apic_access_fault(struct vmx *vmx, int vcpuid, uint64_t gpa)
1620 {
1621
1622         if (apic_access_virtualization(vmx, vcpuid) &&
1623             (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE))
1624                 return (true);
1625         else
1626                 return (false);
1627 }
1628
1629 static int
1630 vmx_handle_apic_access(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit)
1631 {
1632         uint64_t qual;
1633         int access_type, offset, allowed;
1634
1635         if (!apic_access_virtualization(vmx, vcpuid))
1636                 return (UNHANDLED);
1637
1638         qual = vmexit->u.vmx.exit_qualification;
1639         access_type = APIC_ACCESS_TYPE(qual);
1640         offset = APIC_ACCESS_OFFSET(qual);
1641
1642         allowed = 0;
1643         if (access_type == 0) {
1644                 /*
1645                  * Read data access to the following registers is expected.
1646                  */
1647                 switch (offset) {
1648                 case APIC_OFFSET_APR:
1649                 case APIC_OFFSET_PPR:
1650                 case APIC_OFFSET_RRR:
1651                 case APIC_OFFSET_CMCI_LVT:
1652                 case APIC_OFFSET_TIMER_CCR:
1653                         allowed = 1;
1654                         break;
1655                 default:
1656                         break;
1657                 }
1658         } else if (access_type == 1) {
1659                 /*
1660                  * Write data access to the following registers is expected.
1661                  */
1662                 switch (offset) {
1663                 case APIC_OFFSET_VER:
1664                 case APIC_OFFSET_APR:
1665                 case APIC_OFFSET_PPR:
1666                 case APIC_OFFSET_RRR:
1667                 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
1668                 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
1669                 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
1670                 case APIC_OFFSET_CMCI_LVT:
1671                 case APIC_OFFSET_TIMER_CCR:
1672                         allowed = 1;
1673                         break;
1674                 default:
1675                         break;
1676                 }
1677         }
1678
1679         if (allowed) {
1680                 vmexit->exitcode = VM_EXITCODE_INST_EMUL;
1681                 vmexit->u.inst_emul.gpa = DEFAULT_APIC_BASE + offset;
1682                 vmexit->u.inst_emul.gla = VIE_INVALID_GLA;
1683                 vmexit->u.inst_emul.cr3 = vmcs_guest_cr3();
1684                 vmexit->u.inst_emul.cpu_mode = vmx_cpu_mode();
1685                 vmexit->u.inst_emul.paging_mode = vmx_paging_mode();
1686         }
1687
1688         /*
1689          * Regardless of whether the APIC-access is allowed this handler
1690          * always returns UNHANDLED:
1691          * - if the access is allowed then it is handled by emulating the
1692          *   instruction that caused the VM-exit (outside the critical section)
1693          * - if the access is not allowed then it will be converted to an
1694          *   exitcode of VM_EXITCODE_VMX and will be dealt with in userland.
1695          */
1696         return (UNHANDLED);
1697 }
1698
1699 static int
1700 vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
1701 {
1702         int error, handled;
1703         struct vmxctx *vmxctx;
1704         struct vlapic *vlapic;
1705         uint32_t eax, ecx, edx, idtvec_info, idtvec_err, intr_info, reason;
1706         uint64_t qual, gpa;
1707         bool retu;
1708
1709         CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_VIRTUAL_NMI) != 0);
1710         CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_NMI_EXITING) != 0);
1711
1712         handled = UNHANDLED;
1713         vmxctx = &vmx->ctx[vcpu];
1714
1715         qual = vmexit->u.vmx.exit_qualification;
1716         reason = vmexit->u.vmx.exit_reason;
1717         vmexit->exitcode = VM_EXITCODE_BOGUS;
1718
1719         vmm_stat_incr(vmx->vm, vcpu, VMEXIT_COUNT, 1);
1720
1721         /*
1722          * VM exits that could be triggered during event injection on the
1723          * previous VM entry need to be handled specially by re-injecting
1724          * the event.
1725          *
1726          * See "Information for VM Exits During Event Delivery" in Intel SDM
1727          * for details.
1728          */
1729         switch (reason) {
1730         case EXIT_REASON_EPT_FAULT:
1731         case EXIT_REASON_EPT_MISCONFIG:
1732         case EXIT_REASON_APIC_ACCESS:
1733         case EXIT_REASON_TASK_SWITCH:
1734         case EXIT_REASON_EXCEPTION:
1735                 idtvec_info = vmcs_idt_vectoring_info();
1736                 if (idtvec_info & VMCS_IDT_VEC_VALID) {
1737                         idtvec_info &= ~(1 << 12); /* clear undefined bit */
1738                         vmcs_write(VMCS_ENTRY_INTR_INFO, idtvec_info);
1739                         if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) {
1740                                 idtvec_err = vmcs_idt_vectoring_err();
1741                                 vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR,
1742                                     idtvec_err);
1743                         }
1744                         /*
1745                          * If 'virtual NMIs' are being used and the VM-exit
1746                          * happened while injecting an NMI during the previous
1747                          * VM-entry, then clear "blocking by NMI" in the Guest
1748                          * Interruptibility-state.
1749                          */
1750                         if ((idtvec_info & VMCS_INTR_T_MASK) ==
1751                             VMCS_INTR_T_NMI) {
1752                                  vmx_clear_nmi_blocking(vmx, vcpu);
1753                         }
1754                         vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length);
1755                 }
1756         default:
1757                 idtvec_info = 0;
1758                 break;
1759         }
1760
1761         switch (reason) {
1762         case EXIT_REASON_CR_ACCESS:
1763                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CR_ACCESS, 1);
1764                 handled = vmx_emulate_cr_access(vmx, vcpu, qual);
1765                 break;
1766         case EXIT_REASON_RDMSR:
1767                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RDMSR, 1);
1768                 retu = false;
1769                 ecx = vmxctx->guest_rcx;
1770                 error = emulate_rdmsr(vmx->vm, vcpu, ecx, &retu);
1771                 if (error) {
1772                         vmexit->exitcode = VM_EXITCODE_RDMSR;
1773                         vmexit->u.msr.code = ecx;
1774                 } else if (!retu) {
1775                         handled = HANDLED;
1776                 } else {
1777                         /* Return to userspace with a valid exitcode */
1778                         KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
1779                             ("emulate_wrmsr retu with bogus exitcode"));
1780                 }
1781                 break;
1782         case EXIT_REASON_WRMSR:
1783                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_WRMSR, 1);
1784                 retu = false;
1785                 eax = vmxctx->guest_rax;
1786                 ecx = vmxctx->guest_rcx;
1787                 edx = vmxctx->guest_rdx;
1788                 error = emulate_wrmsr(vmx->vm, vcpu, ecx,
1789                     (uint64_t)edx << 32 | eax, &retu);
1790                 if (error) {
1791                         vmexit->exitcode = VM_EXITCODE_WRMSR;
1792                         vmexit->u.msr.code = ecx;
1793                         vmexit->u.msr.wval = (uint64_t)edx << 32 | eax;
1794                 } else if (!retu) {
1795                         handled = HANDLED;
1796                 } else {
1797                         /* Return to userspace with a valid exitcode */
1798                         KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
1799                             ("emulate_wrmsr retu with bogus exitcode"));
1800                 }
1801                 break;
1802         case EXIT_REASON_HLT:
1803                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_HLT, 1);
1804                 vmexit->exitcode = VM_EXITCODE_HLT;
1805                 vmexit->u.hlt.rflags = vmcs_read(VMCS_GUEST_RFLAGS);
1806                 break;
1807         case EXIT_REASON_MTF:
1808                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_MTRAP, 1);
1809                 vmexit->exitcode = VM_EXITCODE_MTRAP;
1810                 break;
1811         case EXIT_REASON_PAUSE:
1812                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_PAUSE, 1);
1813                 vmexit->exitcode = VM_EXITCODE_PAUSE;
1814                 break;
1815         case EXIT_REASON_INTR_WINDOW:
1816                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INTR_WINDOW, 1);
1817                 vmx_clear_int_window_exiting(vmx, vcpu);
1818                 return (1);
1819         case EXIT_REASON_EXT_INTR:
1820                 /*
1821                  * External interrupts serve only to cause VM exits and allow
1822                  * the host interrupt handler to run.
1823                  *
1824                  * If this external interrupt triggers a virtual interrupt
1825                  * to a VM, then that state will be recorded by the
1826                  * host interrupt handler in the VM's softc. We will inject
1827                  * this virtual interrupt during the subsequent VM enter.
1828                  */
1829                 intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
1830
1831                 /*
1832                  * XXX: Ignore this exit if VMCS_INTR_VALID is not set.
1833                  * This appears to be a bug in VMware Fusion?
1834                  */
1835                 if (!(intr_info & VMCS_INTR_VALID))
1836                         return (1);
1837                 KASSERT((intr_info & VMCS_INTR_VALID) != 0 &&
1838                     (intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_HWINTR,
1839                     ("VM exit interruption info invalid: %#x", intr_info));
1840                 vmx_trigger_hostintr(intr_info & 0xff);
1841
1842                 /*
1843                  * This is special. We want to treat this as an 'handled'
1844                  * VM-exit but not increment the instruction pointer.
1845                  */
1846                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXTINT, 1);
1847                 return (1);
1848         case EXIT_REASON_NMI_WINDOW:
1849                 /* Exit to allow the pending virtual NMI to be injected */
1850                 if (vm_nmi_pending(vmx->vm, vcpu))
1851                         vmx_inject_nmi(vmx, vcpu);
1852                 vmx_clear_nmi_window_exiting(vmx, vcpu);
1853                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NMI_WINDOW, 1);
1854                 return (1);
1855         case EXIT_REASON_INOUT:
1856                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INOUT, 1);
1857                 vmexit->exitcode = VM_EXITCODE_INOUT;
1858                 vmexit->u.inout.bytes = (qual & 0x7) + 1;
1859                 vmexit->u.inout.in = (qual & 0x8) ? 1 : 0;
1860                 vmexit->u.inout.string = (qual & 0x10) ? 1 : 0;
1861                 vmexit->u.inout.rep = (qual & 0x20) ? 1 : 0;
1862                 vmexit->u.inout.port = (uint16_t)(qual >> 16);
1863                 vmexit->u.inout.eax = (uint32_t)(vmxctx->guest_rax);
1864                 break;
1865         case EXIT_REASON_CPUID:
1866                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CPUID, 1);
1867                 handled = vmx_handle_cpuid(vmx->vm, vcpu, vmxctx);
1868                 break;
1869         case EXIT_REASON_EXCEPTION:
1870                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXCEPTION, 1);
1871                 intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
1872                 KASSERT((intr_info & VMCS_INTR_VALID) != 0,
1873                     ("VM exit interruption info invalid: %#x", intr_info));
1874
1875                 /*
1876                  * If Virtual NMIs control is 1 and the VM-exit is due to a
1877                  * fault encountered during the execution of IRET then we must
1878                  * restore the state of "virtual-NMI blocking" before resuming
1879                  * the guest.
1880                  *
1881                  * See "Resuming Guest Software after Handling an Exception".
1882                  */
1883                 if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
1884                     (intr_info & 0xff) != IDT_DF &&
1885                     (intr_info & EXIT_QUAL_NMIUDTI) != 0)
1886                         vmx_restore_nmi_blocking(vmx, vcpu);
1887
1888                 /*
1889                  * The NMI has already been handled in vmx_exit_handle_nmi().
1890                  */
1891                 if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI)
1892                         return (1);
1893                 break;
1894         case EXIT_REASON_EPT_FAULT:
1895                 /*
1896                  * If 'gpa' lies within the address space allocated to
1897                  * memory then this must be a nested page fault otherwise
1898                  * this must be an instruction that accesses MMIO space.
1899                  */
1900                 gpa = vmcs_gpa();
1901                 if (vm_mem_allocated(vmx->vm, gpa) ||
1902                     apic_access_fault(vmx, vcpu, gpa)) {
1903                         vmexit->exitcode = VM_EXITCODE_PAGING;
1904                         vmexit->u.paging.gpa = gpa;
1905                         vmexit->u.paging.fault_type = ept_fault_type(qual);
1906                         vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NESTED_FAULT, 1);
1907                 } else if (ept_emulation_fault(qual)) {
1908                         vmexit->exitcode = VM_EXITCODE_INST_EMUL;
1909                         vmexit->u.inst_emul.gpa = gpa;
1910                         vmexit->u.inst_emul.gla = vmcs_gla();
1911                         vmexit->u.inst_emul.cr3 = vmcs_guest_cr3();
1912                         vmexit->u.inst_emul.cpu_mode = vmx_cpu_mode();
1913                         vmexit->u.inst_emul.paging_mode = vmx_paging_mode();
1914                         vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INST_EMUL, 1);
1915                 }
1916                 /*
1917                  * If Virtual NMIs control is 1 and the VM-exit is due to an
1918                  * EPT fault during the execution of IRET then we must restore
1919                  * the state of "virtual-NMI blocking" before resuming.
1920                  *
1921                  * See description of "NMI unblocking due to IRET" in
1922                  * "Exit Qualification for EPT Violations".
1923                  */
1924                 if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
1925                     (qual & EXIT_QUAL_NMIUDTI) != 0)
1926                         vmx_restore_nmi_blocking(vmx, vcpu);
1927                 break;
1928         case EXIT_REASON_VIRTUALIZED_EOI:
1929                 vmexit->exitcode = VM_EXITCODE_IOAPIC_EOI;
1930                 vmexit->u.ioapic_eoi.vector = qual & 0xFF;
1931                 vmexit->inst_length = 0;        /* trap-like */
1932                 break;
1933         case EXIT_REASON_APIC_ACCESS:
1934                 handled = vmx_handle_apic_access(vmx, vcpu, vmexit);
1935                 break;
1936         case EXIT_REASON_APIC_WRITE:
1937                 /*
1938                  * APIC-write VM exit is trap-like so the %rip is already
1939                  * pointing to the next instruction.
1940                  */
1941                 vmexit->inst_length = 0;
1942                 vlapic = vm_lapic(vmx->vm, vcpu);
1943                 handled = vmx_handle_apic_write(vmx, vcpu, vlapic, qual);
1944                 break;
1945         case EXIT_REASON_XSETBV:
1946                 handled = vmx_emulate_xsetbv(vmx, vcpu, vmexit);
1947                 break;
1948         default:
1949                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_UNKNOWN, 1);
1950                 break;
1951         }
1952
1953         if (handled) {
1954                 /*
1955                  * It is possible that control is returned to userland
1956                  * even though we were able to handle the VM exit in the
1957                  * kernel.
1958                  *
1959                  * In such a case we want to make sure that the userland
1960                  * restarts guest execution at the instruction *after*
1961                  * the one we just processed. Therefore we update the
1962                  * guest rip in the VMCS and in 'vmexit'.
1963                  */
1964                 vmexit->rip += vmexit->inst_length;
1965                 vmexit->inst_length = 0;
1966                 vmcs_write(VMCS_GUEST_RIP, vmexit->rip);
1967         } else {
1968                 if (vmexit->exitcode == VM_EXITCODE_BOGUS) {
1969                         /*
1970                          * If this VM exit was not claimed by anybody then
1971                          * treat it as a generic VMX exit.
1972                          */
1973                         vmexit->exitcode = VM_EXITCODE_VMX;
1974                         vmexit->u.vmx.status = VM_SUCCESS;
1975                         vmexit->u.vmx.inst_type = 0;
1976                         vmexit->u.vmx.inst_error = 0;
1977                 } else {
1978                         /*
1979                          * The exitcode and collateral have been populated.
1980                          * The VM exit will be processed further in userland.
1981                          */
1982                 }
1983         }
1984         return (handled);
1985 }
1986
1987 static __inline int
1988 vmx_exit_astpending(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
1989 {
1990
1991         vmexit->rip = vmcs_guest_rip();
1992         vmexit->inst_length = 0;
1993         vmexit->exitcode = VM_EXITCODE_BOGUS;
1994         vmx_astpending_trace(vmx, vcpu, vmexit->rip);
1995         vmm_stat_incr(vmx->vm, vcpu, VMEXIT_ASTPENDING, 1);
1996
1997         return (HANDLED);
1998 }
1999
2000 static __inline int
2001 vmx_exit_rendezvous(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
2002 {
2003
2004         vmexit->rip = vmcs_guest_rip();
2005         vmexit->inst_length = 0;
2006         vmexit->exitcode = VM_EXITCODE_RENDEZVOUS;
2007         vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RENDEZVOUS, 1);
2008
2009         return (UNHANDLED);
2010 }
2011
2012 static __inline int
2013 vmx_exit_inst_error(struct vmxctx *vmxctx, int rc, struct vm_exit *vmexit)
2014 {
2015
2016         KASSERT(vmxctx->inst_fail_status != VM_SUCCESS,
2017             ("vmx_exit_inst_error: invalid inst_fail_status %d",
2018             vmxctx->inst_fail_status));
2019
2020         vmexit->inst_length = 0;
2021         vmexit->exitcode = VM_EXITCODE_VMX;
2022         vmexit->u.vmx.status = vmxctx->inst_fail_status;
2023         vmexit->u.vmx.inst_error = vmcs_instruction_error();
2024         vmexit->u.vmx.exit_reason = ~0;
2025         vmexit->u.vmx.exit_qualification = ~0;
2026
2027         switch (rc) {
2028         case VMX_VMRESUME_ERROR:
2029         case VMX_VMLAUNCH_ERROR:
2030         case VMX_INVEPT_ERROR:
2031                 vmexit->u.vmx.inst_type = rc;
2032                 break;
2033         default:
2034                 panic("vm_exit_inst_error: vmx_enter_guest returned %d", rc);
2035         }
2036
2037         return (UNHANDLED);
2038 }
2039
2040 /*
2041  * If the NMI-exiting VM execution control is set to '1' then an NMI in
2042  * non-root operation causes a VM-exit. NMI blocking is in effect so it is
2043  * sufficient to simply vector to the NMI handler via a software interrupt.
2044  * However, this must be done before maskable interrupts are enabled
2045  * otherwise the "iret" issued by an interrupt handler will incorrectly
2046  * clear NMI blocking.
2047  */
2048 static __inline void
2049 vmx_exit_handle_nmi(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit)
2050 {
2051         uint32_t intr_info;
2052
2053         KASSERT((read_rflags() & PSL_I) == 0, ("interrupts enabled"));
2054
2055         if (vmexit->u.vmx.exit_reason != EXIT_REASON_EXCEPTION)
2056                 return;
2057
2058         intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
2059         KASSERT((intr_info & VMCS_INTR_VALID) != 0,
2060             ("VM exit interruption info invalid: %#x", intr_info));
2061
2062         if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI) {
2063                 KASSERT((intr_info & 0xff) == IDT_NMI, ("VM exit due "
2064                     "to NMI has invalid vector: %#x", intr_info));
2065                 VCPU_CTR0(vmx->vm, vcpuid, "Vectoring to NMI handler");
2066                 __asm __volatile("int $2");
2067         }
2068 }
2069
2070 static int
2071 vmx_run(void *arg, int vcpu, register_t startrip, pmap_t pmap,
2072     void *rendezvous_cookie)
2073 {
2074         int rc, handled, launched;
2075         struct vmx *vmx;
2076         struct vm *vm;
2077         struct vmxctx *vmxctx;
2078         struct vmcs *vmcs;
2079         struct vm_exit *vmexit;
2080         struct vlapic *vlapic;
2081         uint64_t rip;
2082         uint32_t exit_reason;
2083
2084         vmx = arg;
2085         vm = vmx->vm;
2086         vmcs = &vmx->vmcs[vcpu];
2087         vmxctx = &vmx->ctx[vcpu];
2088         vlapic = vm_lapic(vm, vcpu);
2089         vmexit = vm_exitinfo(vm, vcpu);
2090         launched = 0;
2091
2092         KASSERT(vmxctx->pmap == pmap,
2093             ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap));
2094
2095         VMPTRLD(vmcs);
2096
2097         /*
2098          * XXX
2099          * We do this every time because we may setup the virtual machine
2100          * from a different process than the one that actually runs it.
2101          *
2102          * If the life of a virtual machine was spent entirely in the context
2103          * of a single process we could do this once in vmx_vminit().
2104          */
2105         vmcs_write(VMCS_HOST_CR3, rcr3());
2106
2107         vmcs_write(VMCS_GUEST_RIP, startrip);
2108         vmx_set_pcpu_defaults(vmx, vcpu, pmap);
2109         do {
2110                 /*
2111                  * Interrupts are disabled from this point on until the
2112                  * guest starts executing. This is done for the following
2113                  * reasons:
2114                  *
2115                  * If an AST is asserted on this thread after the check below,
2116                  * then the IPI_AST notification will not be lost, because it
2117                  * will cause a VM exit due to external interrupt as soon as
2118                  * the guest state is loaded.
2119                  *
2120                  * A posted interrupt after 'vmx_inject_interrupts()' will
2121                  * not be "lost" because it will be held pending in the host
2122                  * APIC because interrupts are disabled. The pending interrupt
2123                  * will be recognized as soon as the guest state is loaded.
2124                  *
2125                  * The same reasoning applies to the IPI generated by
2126                  * pmap_invalidate_ept().
2127                  */
2128                 disable_intr();
2129                 if (curthread->td_flags & (TDF_ASTPENDING | TDF_NEEDRESCHED)) {
2130                         enable_intr();
2131                         handled = vmx_exit_astpending(vmx, vcpu, vmexit);
2132                         break;
2133                 }
2134
2135                 if (vcpu_rendezvous_pending(rendezvous_cookie)) {
2136                         enable_intr();
2137                         handled = vmx_exit_rendezvous(vmx, vcpu, vmexit);
2138                         break;
2139                 }
2140
2141                 vmx_inject_interrupts(vmx, vcpu, vlapic);
2142                 vmx_run_trace(vmx, vcpu);
2143                 rc = vmx_enter_guest(vmxctx, vmx, launched);
2144
2145                 /* Collect some information for VM exit processing */
2146                 vmexit->rip = rip = vmcs_guest_rip();
2147                 vmexit->inst_length = vmexit_instruction_length();
2148                 vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason();
2149                 vmexit->u.vmx.exit_qualification = vmcs_exit_qualification();
2150
2151                 if (rc == VMX_GUEST_VMEXIT) {
2152                         vmx_exit_handle_nmi(vmx, vcpu, vmexit);
2153                         enable_intr();
2154                         handled = vmx_exit_process(vmx, vcpu, vmexit);
2155                 } else {
2156                         enable_intr();
2157                         handled = vmx_exit_inst_error(vmxctx, rc, vmexit);
2158                 }
2159                 launched = 1;
2160                 vmx_exit_trace(vmx, vcpu, rip, exit_reason, handled);
2161         } while (handled);
2162
2163         /*
2164          * If a VM exit has been handled then the exitcode must be BOGUS
2165          * If a VM exit is not handled then the exitcode must not be BOGUS
2166          */
2167         if ((handled && vmexit->exitcode != VM_EXITCODE_BOGUS) ||
2168             (!handled && vmexit->exitcode == VM_EXITCODE_BOGUS)) {
2169                 panic("Mismatch between handled (%d) and exitcode (%d)",
2170                       handled, vmexit->exitcode);
2171         }
2172
2173         if (!handled)
2174                 vmm_stat_incr(vm, vcpu, VMEXIT_USERSPACE, 1);
2175
2176         VCPU_CTR1(vm, vcpu, "returning from vmx_run: exitcode %d",
2177             vmexit->exitcode);
2178
2179         VMCLEAR(vmcs);
2180         return (0);
2181 }
2182
2183 static void
2184 vmx_vmcleanup(void *arg)
2185 {
2186         int i, error;
2187         struct vmx *vmx = arg;
2188
2189         if (apic_access_virtualization(vmx, 0))
2190                 vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
2191
2192         for (i = 0; i < VM_MAXCPU; i++)
2193                 vpid_free(vmx->state[i].vpid);
2194
2195         /*
2196          * XXXSMP we also need to clear the VMCS active on the other vcpus.
2197          */
2198         error = vmclear(&vmx->vmcs[0]);
2199         if (error != 0)
2200                 panic("vmx_vmcleanup: vmclear error %d on vcpu 0", error);
2201
2202         free(vmx, M_VMX);
2203
2204         return;
2205 }
2206
2207 static register_t *
2208 vmxctx_regptr(struct vmxctx *vmxctx, int reg)
2209 {
2210
2211         switch (reg) {
2212         case VM_REG_GUEST_RAX:
2213                 return (&vmxctx->guest_rax);
2214         case VM_REG_GUEST_RBX:
2215                 return (&vmxctx->guest_rbx);
2216         case VM_REG_GUEST_RCX:
2217                 return (&vmxctx->guest_rcx);
2218         case VM_REG_GUEST_RDX:
2219                 return (&vmxctx->guest_rdx);
2220         case VM_REG_GUEST_RSI:
2221                 return (&vmxctx->guest_rsi);
2222         case VM_REG_GUEST_RDI:
2223                 return (&vmxctx->guest_rdi);
2224         case VM_REG_GUEST_RBP:
2225                 return (&vmxctx->guest_rbp);
2226         case VM_REG_GUEST_R8:
2227                 return (&vmxctx->guest_r8);
2228         case VM_REG_GUEST_R9:
2229                 return (&vmxctx->guest_r9);
2230         case VM_REG_GUEST_R10:
2231                 return (&vmxctx->guest_r10);
2232         case VM_REG_GUEST_R11:
2233                 return (&vmxctx->guest_r11);
2234         case VM_REG_GUEST_R12:
2235                 return (&vmxctx->guest_r12);
2236         case VM_REG_GUEST_R13:
2237                 return (&vmxctx->guest_r13);
2238         case VM_REG_GUEST_R14:
2239                 return (&vmxctx->guest_r14);
2240         case VM_REG_GUEST_R15:
2241                 return (&vmxctx->guest_r15);
2242         default:
2243                 break;
2244         }
2245         return (NULL);
2246 }
2247
2248 static int
2249 vmxctx_getreg(struct vmxctx *vmxctx, int reg, uint64_t *retval)
2250 {
2251         register_t *regp;
2252
2253         if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
2254                 *retval = *regp;
2255                 return (0);
2256         } else
2257                 return (EINVAL);
2258 }
2259
2260 static int
2261 vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val)
2262 {
2263         register_t *regp;
2264
2265         if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
2266                 *regp = val;
2267                 return (0);
2268         } else
2269                 return (EINVAL);
2270 }
2271
2272 static int
2273 vmx_shadow_reg(int reg)
2274 {
2275         int shreg;
2276
2277         shreg = -1;
2278
2279         switch (reg) {
2280         case VM_REG_GUEST_CR0:
2281                 shreg = VMCS_CR0_SHADOW;
2282                 break;
2283         case VM_REG_GUEST_CR4:
2284                 shreg = VMCS_CR4_SHADOW;
2285                 break;
2286         default:
2287                 break;
2288         }
2289
2290         return (shreg);
2291 }
2292
2293 static int
2294 vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval)
2295 {
2296         int running, hostcpu;
2297         struct vmx *vmx = arg;
2298
2299         running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
2300         if (running && hostcpu != curcpu)
2301                 panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), vcpu);
2302
2303         if (vmxctx_getreg(&vmx->ctx[vcpu], reg, retval) == 0)
2304                 return (0);
2305
2306         return (vmcs_getreg(&vmx->vmcs[vcpu], running, reg, retval));
2307 }
2308
2309 static int
2310 vmx_setreg(void *arg, int vcpu, int reg, uint64_t val)
2311 {
2312         int error, hostcpu, running, shadow;
2313         uint64_t ctls;
2314         struct vmx *vmx = arg;
2315
2316         running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
2317         if (running && hostcpu != curcpu)
2318                 panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), vcpu);
2319
2320         if (vmxctx_setreg(&vmx->ctx[vcpu], reg, val) == 0)
2321                 return (0);
2322
2323         error = vmcs_setreg(&vmx->vmcs[vcpu], running, reg, val);
2324
2325         if (error == 0) {
2326                 /*
2327                  * If the "load EFER" VM-entry control is 1 then the
2328                  * value of EFER.LMA must be identical to "IA-32e mode guest"
2329                  * bit in the VM-entry control.
2330                  */
2331                 if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 &&
2332                     (reg == VM_REG_GUEST_EFER)) {
2333                         vmcs_getreg(&vmx->vmcs[vcpu], running,
2334                                     VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls);
2335                         if (val & EFER_LMA)
2336                                 ctls |= VM_ENTRY_GUEST_LMA;
2337                         else
2338                                 ctls &= ~VM_ENTRY_GUEST_LMA;
2339                         vmcs_setreg(&vmx->vmcs[vcpu], running,
2340                                     VMCS_IDENT(VMCS_ENTRY_CTLS), ctls);
2341                 }
2342
2343                 shadow = vmx_shadow_reg(reg);
2344                 if (shadow > 0) {
2345                         /*
2346                          * Store the unmodified value in the shadow
2347                          */
2348                         error = vmcs_setreg(&vmx->vmcs[vcpu], running,
2349                                     VMCS_IDENT(shadow), val);
2350                 }
2351         }
2352
2353         return (error);
2354 }
2355
2356 static int
2357 vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
2358 {
2359         struct vmx *vmx = arg;
2360
2361         return (vmcs_getdesc(&vmx->vmcs[vcpu], reg, desc));
2362 }
2363
2364 static int
2365 vmx_setdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
2366 {
2367         struct vmx *vmx = arg;
2368
2369         return (vmcs_setdesc(&vmx->vmcs[vcpu], reg, desc));
2370 }
2371
2372 static int
2373 vmx_getcap(void *arg, int vcpu, int type, int *retval)
2374 {
2375         struct vmx *vmx = arg;
2376         int vcap;
2377         int ret;
2378
2379         ret = ENOENT;
2380
2381         vcap = vmx->cap[vcpu].set;
2382
2383         switch (type) {
2384         case VM_CAP_HALT_EXIT:
2385                 if (cap_halt_exit)
2386                         ret = 0;
2387                 break;
2388         case VM_CAP_PAUSE_EXIT:
2389                 if (cap_pause_exit)
2390                         ret = 0;
2391                 break;
2392         case VM_CAP_MTRAP_EXIT:
2393                 if (cap_monitor_trap)
2394                         ret = 0;
2395                 break;
2396         case VM_CAP_UNRESTRICTED_GUEST:
2397                 if (cap_unrestricted_guest)
2398                         ret = 0;
2399                 break;
2400         case VM_CAP_ENABLE_INVPCID:
2401                 if (cap_invpcid)
2402                         ret = 0;
2403                 break;
2404         default:
2405                 break;
2406         }
2407
2408         if (ret == 0)
2409                 *retval = (vcap & (1 << type)) ? 1 : 0;
2410
2411         return (ret);
2412 }
2413
2414 static int
2415 vmx_setcap(void *arg, int vcpu, int type, int val)
2416 {
2417         struct vmx *vmx = arg;
2418         struct vmcs *vmcs = &vmx->vmcs[vcpu];
2419         uint32_t baseval;
2420         uint32_t *pptr;
2421         int error;
2422         int flag;
2423         int reg;
2424         int retval;
2425
2426         retval = ENOENT;
2427         pptr = NULL;
2428
2429         switch (type) {
2430         case VM_CAP_HALT_EXIT:
2431                 if (cap_halt_exit) {
2432                         retval = 0;
2433                         pptr = &vmx->cap[vcpu].proc_ctls;
2434                         baseval = *pptr;
2435                         flag = PROCBASED_HLT_EXITING;
2436                         reg = VMCS_PRI_PROC_BASED_CTLS;
2437                 }
2438                 break;
2439         case VM_CAP_MTRAP_EXIT:
2440                 if (cap_monitor_trap) {
2441                         retval = 0;
2442                         pptr = &vmx->cap[vcpu].proc_ctls;
2443                         baseval = *pptr;
2444                         flag = PROCBASED_MTF;
2445                         reg = VMCS_PRI_PROC_BASED_CTLS;
2446                 }
2447                 break;
2448         case VM_CAP_PAUSE_EXIT:
2449                 if (cap_pause_exit) {
2450                         retval = 0;
2451                         pptr = &vmx->cap[vcpu].proc_ctls;
2452                         baseval = *pptr;
2453                         flag = PROCBASED_PAUSE_EXITING;
2454                         reg = VMCS_PRI_PROC_BASED_CTLS;
2455                 }
2456                 break;
2457         case VM_CAP_UNRESTRICTED_GUEST:
2458                 if (cap_unrestricted_guest) {
2459                         retval = 0;
2460                         pptr = &vmx->cap[vcpu].proc_ctls2;
2461                         baseval = *pptr;
2462                         flag = PROCBASED2_UNRESTRICTED_GUEST;
2463                         reg = VMCS_SEC_PROC_BASED_CTLS;
2464                 }
2465                 break;
2466         case VM_CAP_ENABLE_INVPCID:
2467                 if (cap_invpcid) {
2468                         retval = 0;
2469                         pptr = &vmx->cap[vcpu].proc_ctls2;
2470                         baseval = *pptr;
2471                         flag = PROCBASED2_ENABLE_INVPCID;
2472                         reg = VMCS_SEC_PROC_BASED_CTLS;
2473                 }
2474                 break;
2475         default:
2476                 break;
2477         }
2478
2479         if (retval == 0) {
2480                 if (val) {
2481                         baseval |= flag;
2482                 } else {
2483                         baseval &= ~flag;
2484                 }
2485                 VMPTRLD(vmcs);
2486                 error = vmwrite(reg, baseval);
2487                 VMCLEAR(vmcs);
2488
2489                 if (error) {
2490                         retval = error;
2491                 } else {
2492                         /*
2493                          * Update optional stored flags, and record
2494                          * setting
2495                          */
2496                         if (pptr != NULL) {
2497                                 *pptr = baseval;
2498                         }
2499
2500                         if (val) {
2501                                 vmx->cap[vcpu].set |= (1 << type);
2502                         } else {
2503                                 vmx->cap[vcpu].set &= ~(1 << type);
2504                         }
2505                 }
2506         }
2507
2508         return (retval);
2509 }
2510
2511 struct vlapic_vtx {
2512         struct vlapic   vlapic;
2513         struct pir_desc *pir_desc;
2514         struct vmx      *vmx;
2515 };
2516
2517 #define VMX_CTR_PIR(vm, vcpuid, pir_desc, notify, vector, level, msg)   \
2518 do {                                                                    \
2519         VCPU_CTR2(vm, vcpuid, msg " assert %s-triggered vector %d",     \
2520             level ? "level" : "edge", vector);                          \
2521         VCPU_CTR1(vm, vcpuid, msg " pir0 0x%016lx", pir_desc->pir[0]);  \
2522         VCPU_CTR1(vm, vcpuid, msg " pir1 0x%016lx", pir_desc->pir[1]);  \
2523         VCPU_CTR1(vm, vcpuid, msg " pir2 0x%016lx", pir_desc->pir[2]);  \
2524         VCPU_CTR1(vm, vcpuid, msg " pir3 0x%016lx", pir_desc->pir[3]);  \
2525         VCPU_CTR1(vm, vcpuid, msg " notify: %s", notify ? "yes" : "no");\
2526 } while (0)
2527
2528 /*
2529  * vlapic->ops handlers that utilize the APICv hardware assist described in
2530  * Chapter 29 of the Intel SDM.
2531  */
2532 static int
2533 vmx_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
2534 {
2535         struct vlapic_vtx *vlapic_vtx;
2536         struct pir_desc *pir_desc;
2537         uint64_t mask;
2538         int idx, notify;
2539
2540         vlapic_vtx = (struct vlapic_vtx *)vlapic;
2541         pir_desc = vlapic_vtx->pir_desc;
2542
2543         /*
2544          * Keep track of interrupt requests in the PIR descriptor. This is
2545          * because the virtual APIC page pointed to by the VMCS cannot be
2546          * modified if the vcpu is running.
2547          */
2548         idx = vector / 64;
2549         mask = 1UL << (vector % 64);
2550         atomic_set_long(&pir_desc->pir[idx], mask);
2551         notify = atomic_cmpset_long(&pir_desc->pending, 0, 1);
2552
2553         VMX_CTR_PIR(vlapic->vm, vlapic->vcpuid, pir_desc, notify, vector,
2554             level, "vmx_set_intr_ready");
2555         return (notify);
2556 }
2557
2558 static int
2559 vmx_pending_intr(struct vlapic *vlapic, int *vecptr)
2560 {
2561         struct vlapic_vtx *vlapic_vtx;
2562         struct pir_desc *pir_desc;
2563         struct LAPIC *lapic;
2564         uint64_t pending, pirval;
2565         uint32_t ppr, vpr;
2566         int i;
2567
2568         /*
2569          * This function is only expected to be called from the 'HLT' exit
2570          * handler which does not care about the vector that is pending.
2571          */
2572         KASSERT(vecptr == NULL, ("vmx_pending_intr: vecptr must be NULL"));
2573
2574         vlapic_vtx = (struct vlapic_vtx *)vlapic;
2575         pir_desc = vlapic_vtx->pir_desc;
2576
2577         pending = atomic_load_acq_long(&pir_desc->pending);
2578         if (!pending)
2579                 return (0);     /* common case */
2580
2581         /*
2582          * If there is an interrupt pending then it will be recognized only
2583          * if its priority is greater than the processor priority.
2584          *
2585          * Special case: if the processor priority is zero then any pending
2586          * interrupt will be recognized.
2587          */
2588         lapic = vlapic->apic_page;
2589         ppr = lapic->ppr & 0xf0;
2590         if (ppr == 0)
2591                 return (1);
2592
2593         VCPU_CTR1(vlapic->vm, vlapic->vcpuid, "HLT with non-zero PPR %d",
2594             lapic->ppr);
2595
2596         for (i = 3; i >= 0; i--) {
2597                 pirval = pir_desc->pir[i];
2598                 if (pirval != 0) {
2599                         vpr = (i * 64 + flsl(pirval) - 1) & 0xf0;
2600                         return (vpr > ppr);
2601                 }
2602         }
2603         return (0);
2604 }
2605
2606 static void
2607 vmx_intr_accepted(struct vlapic *vlapic, int vector)
2608 {
2609
2610         panic("vmx_intr_accepted: not expected to be called");
2611 }
2612
2613 static void
2614 vmx_set_tmr(struct vlapic *vlapic, int vector, bool level)
2615 {
2616         struct vlapic_vtx *vlapic_vtx;
2617         struct vmx *vmx;
2618         struct vmcs *vmcs;
2619         uint64_t mask, val;
2620
2621         KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
2622         KASSERT(!vcpu_is_running(vlapic->vm, vlapic->vcpuid, NULL),
2623             ("vmx_set_tmr: vcpu cannot be running"));
2624
2625         vlapic_vtx = (struct vlapic_vtx *)vlapic;
2626         vmx = vlapic_vtx->vmx;
2627         vmcs = &vmx->vmcs[vlapic->vcpuid];
2628         mask = 1UL << (vector % 64);
2629
2630         VMPTRLD(vmcs);
2631         val = vmcs_read(VMCS_EOI_EXIT(vector));
2632         if (level)
2633                 val |= mask;
2634         else
2635                 val &= ~mask;
2636         vmcs_write(VMCS_EOI_EXIT(vector), val);
2637         VMCLEAR(vmcs);
2638 }
2639
2640 static void
2641 vmx_enable_x2apic_mode(struct vlapic *vlapic)
2642 {
2643         struct vmx *vmx;
2644         struct vmcs *vmcs;
2645         uint32_t proc_ctls2;
2646         int vcpuid, error;
2647
2648         vcpuid = vlapic->vcpuid;
2649         vmx = ((struct vlapic_vtx *)vlapic)->vmx;
2650         vmcs = &vmx->vmcs[vcpuid];
2651
2652         proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
2653         KASSERT((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) != 0,
2654             ("%s: invalid proc_ctls2 %#x", __func__, proc_ctls2));
2655
2656         proc_ctls2 &= ~PROCBASED2_VIRTUALIZE_APIC_ACCESSES;
2657         proc_ctls2 |= PROCBASED2_VIRTUALIZE_X2APIC_MODE;
2658         vmx->cap[vcpuid].proc_ctls2 = proc_ctls2;
2659
2660         VMPTRLD(vmcs);
2661         vmcs_write(VMCS_SEC_PROC_BASED_CTLS, proc_ctls2);
2662         VMCLEAR(vmcs);
2663
2664         if (vlapic->vcpuid == 0) {
2665                 /*
2666                  * The nested page table mappings are shared by all vcpus
2667                  * so unmap the APIC access page just once.
2668                  */
2669                 error = vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
2670                 KASSERT(error == 0, ("%s: vm_unmap_mmio error %d",
2671                     __func__, error));
2672
2673                 /*
2674                  * The MSR bitmap is shared by all vcpus so modify it only
2675                  * once in the context of vcpu 0.
2676                  */
2677                 error = vmx_allow_x2apic_msrs(vmx);
2678                 KASSERT(error == 0, ("%s: vmx_allow_x2apic_msrs error %d",
2679                     __func__, error));
2680         }
2681 }
2682
2683 static void
2684 vmx_post_intr(struct vlapic *vlapic, int hostcpu)
2685 {
2686
2687         ipi_cpu(hostcpu, pirvec);
2688 }
2689
2690 /*
2691  * Transfer the pending interrupts in the PIR descriptor to the IRR
2692  * in the virtual APIC page.
2693  */
2694 static void
2695 vmx_inject_pir(struct vlapic *vlapic)
2696 {
2697         struct vlapic_vtx *vlapic_vtx;
2698         struct pir_desc *pir_desc;
2699         struct LAPIC *lapic;
2700         uint64_t val, pirval;
2701         int rvi, pirbase;
2702         uint16_t intr_status_old, intr_status_new;
2703
2704         vlapic_vtx = (struct vlapic_vtx *)vlapic;
2705         pir_desc = vlapic_vtx->pir_desc;
2706         if (atomic_cmpset_long(&pir_desc->pending, 1, 0) == 0) {
2707                 VCPU_CTR0(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: "
2708                     "no posted interrupt pending");
2709                 return;
2710         }
2711
2712         pirval = 0;
2713         lapic = vlapic->apic_page;
2714
2715         val = atomic_readandclear_long(&pir_desc->pir[0]);
2716         if (val != 0) {
2717                 lapic->irr0 |= val;
2718                 lapic->irr1 |= val >> 32;
2719                 pirbase = 0;
2720                 pirval = val;
2721         }
2722
2723         val = atomic_readandclear_long(&pir_desc->pir[1]);
2724         if (val != 0) {
2725                 lapic->irr2 |= val;
2726                 lapic->irr3 |= val >> 32;
2727                 pirbase = 64;
2728                 pirval = val;
2729         }
2730
2731         val = atomic_readandclear_long(&pir_desc->pir[2]);
2732         if (val != 0) {
2733                 lapic->irr4 |= val;
2734                 lapic->irr5 |= val >> 32;
2735                 pirbase = 128;
2736                 pirval = val;
2737         }
2738
2739         val = atomic_readandclear_long(&pir_desc->pir[3]);
2740         if (val != 0) {
2741                 lapic->irr6 |= val;
2742                 lapic->irr7 |= val >> 32;
2743                 pirbase = 192;
2744                 pirval = val;
2745         }
2746         VLAPIC_CTR_IRR(vlapic, "vmx_inject_pir");
2747
2748         /*
2749          * Update RVI so the processor can evaluate pending virtual
2750          * interrupts on VM-entry.
2751          */
2752         if (pirval != 0) {
2753                 rvi = pirbase + flsl(pirval) - 1;
2754                 intr_status_old = vmcs_read(VMCS_GUEST_INTR_STATUS);
2755                 intr_status_new = (intr_status_old & 0xFF00) | rvi;
2756                 if (intr_status_new > intr_status_old) {
2757                         vmcs_write(VMCS_GUEST_INTR_STATUS, intr_status_new);
2758                         VCPU_CTR2(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: "
2759                             "guest_intr_status changed from 0x%04x to 0x%04x",
2760                             intr_status_old, intr_status_new);
2761                 }
2762         }
2763 }
2764
2765 static struct vlapic *
2766 vmx_vlapic_init(void *arg, int vcpuid)
2767 {
2768         struct vmx *vmx;
2769         struct vlapic *vlapic;
2770         struct vlapic_vtx *vlapic_vtx;
2771
2772         vmx = arg;
2773
2774         vlapic = malloc(sizeof(struct vlapic_vtx), M_VLAPIC, M_WAITOK | M_ZERO);
2775         vlapic->vm = vmx->vm;
2776         vlapic->vcpuid = vcpuid;
2777         vlapic->apic_page = (struct LAPIC *)&vmx->apic_page[vcpuid];
2778
2779         vlapic_vtx = (struct vlapic_vtx *)vlapic;
2780         vlapic_vtx->pir_desc = &vmx->pir_desc[vcpuid];
2781         vlapic_vtx->vmx = vmx;
2782
2783         if (virtual_interrupt_delivery) {
2784                 vlapic->ops.set_intr_ready = vmx_set_intr_ready;
2785                 vlapic->ops.pending_intr = vmx_pending_intr;
2786                 vlapic->ops.intr_accepted = vmx_intr_accepted;
2787                 vlapic->ops.set_tmr = vmx_set_tmr;
2788                 vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode;
2789         }
2790
2791         if (posted_interrupts)
2792                 vlapic->ops.post_intr = vmx_post_intr;
2793
2794         vlapic_init(vlapic);
2795
2796         return (vlapic);
2797 }
2798
2799 static void
2800 vmx_vlapic_cleanup(void *arg, struct vlapic *vlapic)
2801 {
2802
2803         vlapic_cleanup(vlapic);
2804         free(vlapic, M_VLAPIC);
2805 }
2806
2807 struct vmm_ops vmm_ops_intel = {
2808         vmx_init,
2809         vmx_cleanup,
2810         vmx_restore,
2811         vmx_vminit,
2812         vmx_run,
2813         vmx_vmcleanup,
2814         vmx_getreg,
2815         vmx_setreg,
2816         vmx_getdesc,
2817         vmx_setdesc,
2818         vmx_getcap,
2819         vmx_setcap,
2820         ept_vmspace_alloc,
2821         ept_vmspace_free,
2822         vmx_vlapic_init,
2823         vmx_vlapic_cleanup,
2824 };