sys/amd64/vmm/intel/vmx.c

   1 /*-
   2  * Copyright (c) 2011 NetApp, Inc.
   3  * All rights reserved.
   4  * Copyright (c) 2018 Joyent, Inc.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  *
  27  * $FreeBSD$
  28  */
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include <sys/param.h>
  34 #include <sys/systm.h>
  35 #include <sys/smp.h>
  36 #include <sys/kernel.h>
  37 #include <sys/malloc.h>
  38 #include <sys/pcpu.h>
  39 #include <sys/proc.h>
  40 #include <sys/sysctl.h>
  41
  42 #include <vm/vm.h>
  43 #include <vm/pmap.h>
  44
  45 #include <machine/psl.h>
  46 #include <machine/cpufunc.h>
  47 #include <machine/md_var.h>
  48 #include <machine/reg.h>
  49 #include <machine/segments.h>
  50 #include <machine/smp.h>
  51 #include <machine/specialreg.h>
  52 #include <machine/vmparam.h>
  53
  54 #include <machine/vmm.h>
  55 #include <machine/vmm_dev.h>
  56 #include <machine/vmm_instruction_emul.h>
  57 #include "vmm_lapic.h"
  58 #include "vmm_host.h"
  59 #include "vmm_ioport.h"
  60 #include "vmm_ktr.h"
  61 #include "vmm_stat.h"
  62 #include "vatpic.h"
  63 #include "vlapic.h"
  64 #include "vlapic_priv.h"
  65
  66 #include "ept.h"
  67 #include "vmx_cpufunc.h"
  68 #include "vmx.h"
  69 #include "vmx_msr.h"
  70 #include "x86.h"
  71 #include "vmx_controls.h"
  72
  73 #define PINBASED_CTLS_ONE_SETTING                                       \
  74         (PINBASED_EXTINT_EXITING        |                               \
  75          PINBASED_NMI_EXITING           |                               \
  76          PINBASED_VIRTUAL_NMI)
  77 #define PINBASED_CTLS_ZERO_SETTING      0
  78
  79 #define PROCBASED_CTLS_WINDOW_SETTING                                   \
  80         (PROCBASED_INT_WINDOW_EXITING   |                               \
  81          PROCBASED_NMI_WINDOW_EXITING)
  82
  83 #define PROCBASED_CTLS_ONE_SETTING                                      \
  84         (PROCBASED_SECONDARY_CONTROLS   |                               \
  85          PROCBASED_MWAIT_EXITING        |                               \
  86          PROCBASED_MONITOR_EXITING      |                               \
  87          PROCBASED_IO_EXITING           |                               \
  88          PROCBASED_MSR_BITMAPS          |                               \
  89          PROCBASED_CTLS_WINDOW_SETTING  |                               \
  90          PROCBASED_CR8_LOAD_EXITING     |                               \
  91          PROCBASED_CR8_STORE_EXITING)
  92 #define PROCBASED_CTLS_ZERO_SETTING     \
  93         (PROCBASED_CR3_LOAD_EXITING |   \
  94         PROCBASED_CR3_STORE_EXITING |   \
  95         PROCBASED_IO_BITMAPS)
  96
  97 #define PROCBASED_CTLS2_ONE_SETTING     PROCBASED2_ENABLE_EPT
  98 #define PROCBASED_CTLS2_ZERO_SETTING    0
  99
 100 #define VM_EXIT_CTLS_ONE_SETTING                                        \
 101         (VM_EXIT_SAVE_DEBUG_CONTROLS            |                       \
 102         VM_EXIT_HOST_LMA                        |                       \
 103         VM_EXIT_SAVE_EFER                       |                       \
 104         VM_EXIT_LOAD_EFER                       |                       \
 105         VM_EXIT_ACKNOWLEDGE_INTERRUPT)
 106
 107 #define VM_EXIT_CTLS_ZERO_SETTING       0
 108
 109 #define VM_ENTRY_CTLS_ONE_SETTING                                       \
 110         (VM_ENTRY_LOAD_DEBUG_CONTROLS           |                       \
 111         VM_ENTRY_LOAD_EFER)
 112
 113 #define VM_ENTRY_CTLS_ZERO_SETTING                                      \
 114         (VM_ENTRY_INTO_SMM                      |                       \
 115         VM_ENTRY_DEACTIVATE_DUAL_MONITOR)
 116
 117 #define HANDLED         1
 118 #define UNHANDLED       0
 119
 120 static MALLOC_DEFINE(M_VMX, "vmx", "vmx");
 121 static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic");
 122
 123 SYSCTL_DECL(_hw_vmm);
 124 SYSCTL_NODE(_hw_vmm, OID_AUTO, vmx, CTLFLAG_RW, NULL, NULL);
 125
 126 int vmxon_enabled[MAXCPU];
 127 static char vmxon_region[MAXCPU][PAGE_SIZE] __aligned(PAGE_SIZE);
 128
 129 static uint32_t pinbased_ctls, procbased_ctls, procbased_ctls2;
 130 static uint32_t exit_ctls, entry_ctls;
 131
 132 static uint64_t cr0_ones_mask, cr0_zeros_mask;
 133 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_ones_mask, CTLFLAG_RD,
 134              &cr0_ones_mask, 0, NULL);
 135 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_zeros_mask, CTLFLAG_RD,
 136              &cr0_zeros_mask, 0, NULL);
 137
 138 static uint64_t cr4_ones_mask, cr4_zeros_mask;
 139 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_ones_mask, CTLFLAG_RD,
 140              &cr4_ones_mask, 0, NULL);
 141 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_zeros_mask, CTLFLAG_RD,
 142              &cr4_zeros_mask, 0, NULL);
 143
 144 static int vmx_initialized;
 145 SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, initialized, CTLFLAG_RD,
 146            &vmx_initialized, 0, "Intel VMX initialized");
 147
 148 /*
 149  * Optional capabilities
 150  */
 151 static SYSCTL_NODE(_hw_vmm_vmx, OID_AUTO, cap, CTLFLAG_RW, NULL, NULL);
 152
 153 static int cap_halt_exit;
 154 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, halt_exit, CTLFLAG_RD, &cap_halt_exit, 0,
 155     "HLT triggers a VM-exit");
 156
 157 static int cap_pause_exit;
 158 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, pause_exit, CTLFLAG_RD, &cap_pause_exit,
 159     0, "PAUSE triggers a VM-exit");
 160
 161 static int cap_unrestricted_guest;
 162 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, unrestricted_guest, CTLFLAG_RD,
 163     &cap_unrestricted_guest, 0, "Unrestricted guests");
 164
 165 static int cap_monitor_trap;
 166 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, monitor_trap, CTLFLAG_RD,
 167     &cap_monitor_trap, 0, "Monitor trap flag");
 168
 169 static int cap_invpcid;
 170 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, invpcid, CTLFLAG_RD, &cap_invpcid,
 171     0, "Guests are allowed to use INVPCID");
 172
 173 static int virtual_interrupt_delivery;
 174 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, virtual_interrupt_delivery, CTLFLAG_RD,
 175     &virtual_interrupt_delivery, 0, "APICv virtual interrupt delivery support");
 176
 177 static int posted_interrupts;
 178 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, posted_interrupts, CTLFLAG_RD,
 179     &posted_interrupts, 0, "APICv posted interrupt support");
 180
 181 static int pirvec = -1;
 182 SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupt_vector, CTLFLAG_RD,
 183     &pirvec, 0, "APICv posted interrupt vector");
 184
 185 static struct unrhdr *vpid_unr;
 186 static u_int vpid_alloc_failed;
 187 SYSCTL_UINT(_hw_vmm_vmx, OID_AUTO, vpid_alloc_failed, CTLFLAG_RD,
 188             &vpid_alloc_failed, 0, NULL);
 189
 190 static int guest_l1d_flush;
 191 SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, l1d_flush, CTLFLAG_RD,
 192     &guest_l1d_flush, 0, NULL);
 193 static int guest_l1d_flush_sw;
 194 SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, l1d_flush_sw, CTLFLAG_RD,
 195     &guest_l1d_flush_sw, 0, NULL);
 196
 197 static struct msr_entry msr_load_list[1] __aligned(16);
 198
 199 /*
 200  * The definitions of SDT probes for VMX.
 201  */
 202
 203 SDT_PROBE_DEFINE3(vmm, vmx, exit, entry,
 204     "struct vmx *", "int", "struct vm_exit *");
 205
 206 SDT_PROBE_DEFINE4(vmm, vmx, exit, taskswitch,
 207     "struct vmx *", "int", "struct vm_exit *", "struct vm_task_switch *");
 208
 209 SDT_PROBE_DEFINE4(vmm, vmx, exit, craccess,
 210     "struct vmx *", "int", "struct vm_exit *", "uint64_t");
 211
 212 SDT_PROBE_DEFINE4(vmm, vmx, exit, rdmsr,
 213     "struct vmx *", "int", "struct vm_exit *", "uint32_t");
 214
 215 SDT_PROBE_DEFINE5(vmm, vmx, exit, wrmsr,
 216     "struct vmx *", "int", "struct vm_exit *", "uint32_t", "uint64_t");
 217
 218 SDT_PROBE_DEFINE3(vmm, vmx, exit, halt,
 219     "struct vmx *", "int", "struct vm_exit *");
 220
 221 SDT_PROBE_DEFINE3(vmm, vmx, exit, mtrap,
 222     "struct vmx *", "int", "struct vm_exit *");
 223
 224 SDT_PROBE_DEFINE3(vmm, vmx, exit, pause,
 225     "struct vmx *", "int", "struct vm_exit *");
 226
 227 SDT_PROBE_DEFINE3(vmm, vmx, exit, intrwindow,
 228     "struct vmx *", "int", "struct vm_exit *");
 229
 230 SDT_PROBE_DEFINE4(vmm, vmx, exit, interrupt,
 231     "struct vmx *", "int", "struct vm_exit *", "uint32_t");
 232
 233 SDT_PROBE_DEFINE3(vmm, vmx, exit, nmiwindow,
 234     "struct vmx *", "int", "struct vm_exit *");
 235
 236 SDT_PROBE_DEFINE3(vmm, vmx, exit, inout,
 237     "struct vmx *", "int", "struct vm_exit *");
 238
 239 SDT_PROBE_DEFINE3(vmm, vmx, exit, cpuid,
 240     "struct vmx *", "int", "struct vm_exit *");
 241
 242 SDT_PROBE_DEFINE5(vmm, vmx, exit, exception,
 243     "struct vmx *", "int", "struct vm_exit *", "uint32_t", "int");
 244
 245 SDT_PROBE_DEFINE5(vmm, vmx, exit, nestedfault,
 246     "struct vmx *", "int", "struct vm_exit *", "uint64_t", "uint64_t");
 247
 248 SDT_PROBE_DEFINE4(vmm, vmx, exit, mmiofault,
 249     "struct vmx *", "int", "struct vm_exit *", "uint64_t");
 250
 251 SDT_PROBE_DEFINE3(vmm, vmx, exit, eoi,
 252     "struct vmx *", "int", "struct vm_exit *");
 253
 254 SDT_PROBE_DEFINE3(vmm, vmx, exit, apicaccess,
 255     "struct vmx *", "int", "struct vm_exit *");
 256
 257 SDT_PROBE_DEFINE4(vmm, vmx, exit, apicwrite,
 258     "struct vmx *", "int", "struct vm_exit *", "struct vlapic *");
 259
 260 SDT_PROBE_DEFINE3(vmm, vmx, exit, xsetbv,
 261     "struct vmx *", "int", "struct vm_exit *");
 262
 263 SDT_PROBE_DEFINE3(vmm, vmx, exit, monitor,
 264     "struct vmx *", "int", "struct vm_exit *");
 265
 266 SDT_PROBE_DEFINE3(vmm, vmx, exit, mwait,
 267     "struct vmx *", "int", "struct vm_exit *");
 268
 269 SDT_PROBE_DEFINE3(vmm, vmx, exit, vminsn,
 270     "struct vmx *", "int", "struct vm_exit *");
 271
 272 SDT_PROBE_DEFINE4(vmm, vmx, exit, unknown,
 273     "struct vmx *", "int", "struct vm_exit *", "uint32_t");
 274
 275 SDT_PROBE_DEFINE4(vmm, vmx, exit, return,
 276     "struct vmx *", "int", "struct vm_exit *", "int");
 277
 278 /*
 279  * Use the last page below 4GB as the APIC access address. This address is
 280  * occupied by the boot firmware so it is guaranteed that it will not conflict
 281  * with a page in system memory.
 282  */
 283 #define APIC_ACCESS_ADDRESS     0xFFFFF000
 284
 285 static int vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc);
 286 static int vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval);
 287 static int vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val);
 288 static void vmx_inject_pir(struct vlapic *vlapic);
 289
 290 #ifdef KTR
 291 static const char *
 292 exit_reason_to_str(int reason)
 293 {
 294         static char reasonbuf[32];
 295
 296         switch (reason) {
 297         case EXIT_REASON_EXCEPTION:
 298                 return "exception";
 299         case EXIT_REASON_EXT_INTR:
 300                 return "extint";
 301         case EXIT_REASON_TRIPLE_FAULT:
 302                 return "triplefault";
 303         case EXIT_REASON_INIT:
 304                 return "init";
 305         case EXIT_REASON_SIPI:
 306                 return "sipi";
 307         case EXIT_REASON_IO_SMI:
 308                 return "iosmi";
 309         case EXIT_REASON_SMI:
 310                 return "smi";
 311         case EXIT_REASON_INTR_WINDOW:
 312                 return "intrwindow";
 313         case EXIT_REASON_NMI_WINDOW:
 314                 return "nmiwindow";
 315         case EXIT_REASON_TASK_SWITCH:
 316                 return "taskswitch";
 317         case EXIT_REASON_CPUID:
 318                 return "cpuid";
 319         case EXIT_REASON_GETSEC:
 320                 return "getsec";
 321         case EXIT_REASON_HLT:
 322                 return "hlt";
 323         case EXIT_REASON_INVD:
 324                 return "invd";
 325         case EXIT_REASON_INVLPG:
 326                 return "invlpg";
 327         case EXIT_REASON_RDPMC:
 328                 return "rdpmc";
 329         case EXIT_REASON_RDTSC:
 330                 return "rdtsc";
 331         case EXIT_REASON_RSM:
 332                 return "rsm";
 333         case EXIT_REASON_VMCALL:
 334                 return "vmcall";
 335         case EXIT_REASON_VMCLEAR:
 336                 return "vmclear";
 337         case EXIT_REASON_VMLAUNCH:
 338                 return "vmlaunch";
 339         case EXIT_REASON_VMPTRLD:
 340                 return "vmptrld";
 341         case EXIT_REASON_VMPTRST:
 342                 return "vmptrst";
 343         case EXIT_REASON_VMREAD:
 344                 return "vmread";
 345         case EXIT_REASON_VMRESUME:
 346                 return "vmresume";
 347         case EXIT_REASON_VMWRITE:
 348                 return "vmwrite";
 349         case EXIT_REASON_VMXOFF:
 350                 return "vmxoff";
 351         case EXIT_REASON_VMXON:
 352                 return "vmxon";
 353         case EXIT_REASON_CR_ACCESS:
 354                 return "craccess";
 355         case EXIT_REASON_DR_ACCESS:
 356                 return "draccess";
 357         case EXIT_REASON_INOUT:
 358                 return "inout";
 359         case EXIT_REASON_RDMSR:
 360                 return "rdmsr";
 361         case EXIT_REASON_WRMSR:
 362                 return "wrmsr";
 363         case EXIT_REASON_INVAL_VMCS:
 364                 return "invalvmcs";
 365         case EXIT_REASON_INVAL_MSR:
 366                 return "invalmsr";
 367         case EXIT_REASON_MWAIT:
 368                 return "mwait";
 369         case EXIT_REASON_MTF:
 370                 return "mtf";
 371         case EXIT_REASON_MONITOR:
 372                 return "monitor";
 373         case EXIT_REASON_PAUSE:
 374                 return "pause";
 375         case EXIT_REASON_MCE_DURING_ENTRY:
 376                 return "mce-during-entry";
 377         case EXIT_REASON_TPR:
 378                 return "tpr";
 379         case EXIT_REASON_APIC_ACCESS:
 380                 return "apic-access";
 381         case EXIT_REASON_GDTR_IDTR:
 382                 return "gdtridtr";
 383         case EXIT_REASON_LDTR_TR:
 384                 return "ldtrtr";
 385         case EXIT_REASON_EPT_FAULT:
 386                 return "eptfault";
 387         case EXIT_REASON_EPT_MISCONFIG:
 388                 return "eptmisconfig";
 389         case EXIT_REASON_INVEPT:
 390                 return "invept";
 391         case EXIT_REASON_RDTSCP:
 392                 return "rdtscp";
 393         case EXIT_REASON_VMX_PREEMPT:
 394                 return "vmxpreempt";
 395         case EXIT_REASON_INVVPID:
 396                 return "invvpid";
 397         case EXIT_REASON_WBINVD:
 398                 return "wbinvd";
 399         case EXIT_REASON_XSETBV:
 400                 return "xsetbv";
 401         case EXIT_REASON_APIC_WRITE:
 402                 return "apic-write";
 403         default:
 404                 snprintf(reasonbuf, sizeof(reasonbuf), "%d", reason);
 405                 return (reasonbuf);
 406         }
 407 }
 408 #endif  /* KTR */
 409
 410 static int
 411 vmx_allow_x2apic_msrs(struct vmx *vmx)
 412 {
 413         int i, error;
 414
 415         error = 0;
 416
 417         /*
 418          * Allow readonly access to the following x2APIC MSRs from the guest.
 419          */
 420         error += guest_msr_ro(vmx, MSR_APIC_ID);
 421         error += guest_msr_ro(vmx, MSR_APIC_VERSION);
 422         error += guest_msr_ro(vmx, MSR_APIC_LDR);
 423         error += guest_msr_ro(vmx, MSR_APIC_SVR);
 424
 425         for (i = 0; i < 8; i++)
 426                 error += guest_msr_ro(vmx, MSR_APIC_ISR0 + i);
 427
 428         for (i = 0; i < 8; i++)
 429                 error += guest_msr_ro(vmx, MSR_APIC_TMR0 + i);
 430
 431         for (i = 0; i < 8; i++)
 432                 error += guest_msr_ro(vmx, MSR_APIC_IRR0 + i);
 433
 434         error += guest_msr_ro(vmx, MSR_APIC_ESR);
 435         error += guest_msr_ro(vmx, MSR_APIC_LVT_TIMER);
 436         error += guest_msr_ro(vmx, MSR_APIC_LVT_THERMAL);
 437         error += guest_msr_ro(vmx, MSR_APIC_LVT_PCINT);
 438         error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT0);
 439         error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT1);
 440         error += guest_msr_ro(vmx, MSR_APIC_LVT_ERROR);
 441         error += guest_msr_ro(vmx, MSR_APIC_ICR_TIMER);
 442         error += guest_msr_ro(vmx, MSR_APIC_DCR_TIMER);
 443         error += guest_msr_ro(vmx, MSR_APIC_ICR);
 444
 445         /*
 446          * Allow TPR, EOI and SELF_IPI MSRs to be read and written by the guest.
 447          *
 448          * These registers get special treatment described in the section
 449          * "Virtualizing MSR-Based APIC Accesses".
 450          */
 451         error += guest_msr_rw(vmx, MSR_APIC_TPR);
 452         error += guest_msr_rw(vmx, MSR_APIC_EOI);
 453         error += guest_msr_rw(vmx, MSR_APIC_SELF_IPI);
 454
 455         return (error);
 456 }
 457
 458 u_long
 459 vmx_fix_cr0(u_long cr0)
 460 {
 461
 462         return ((cr0 | cr0_ones_mask) & ~cr0_zeros_mask);
 463 }
 464
 465 u_long
 466 vmx_fix_cr4(u_long cr4)
 467 {
 468
 469         return ((cr4 | cr4_ones_mask) & ~cr4_zeros_mask);
 470 }
 471
 472 static void
 473 vpid_free(int vpid)
 474 {
 475         if (vpid < 0 || vpid > 0xffff)
 476                 panic("vpid_free: invalid vpid %d", vpid);
 477
 478         /*
 479          * VPIDs [0,VM_MAXCPU] are special and are not allocated from
 480          * the unit number allocator.
 481          */
 482
 483         if (vpid > VM_MAXCPU)
 484                 free_unr(vpid_unr, vpid);
 485 }
 486
 487 static void
 488 vpid_alloc(uint16_t *vpid, int num)
 489 {
 490         int i, x;
 491
 492         if (num <= 0 || num > VM_MAXCPU)
 493                 panic("invalid number of vpids requested: %d", num);
 494
 495         /*
 496          * If the "enable vpid" execution control is not enabled then the
 497          * VPID is required to be 0 for all vcpus.
 498          */
 499         if ((procbased_ctls2 & PROCBASED2_ENABLE_VPID) == 0) {
 500                 for (i = 0; i < num; i++)
 501                         vpid[i] = 0;
 502                 return;
 503         }
 504
 505         /*
 506          * Allocate a unique VPID for each vcpu from the unit number allocator.
 507          */
 508         for (i = 0; i < num; i++) {
 509                 x = alloc_unr(vpid_unr);
 510                 if (x == -1)
 511                         break;
 512                 else
 513                         vpid[i] = x;
 514         }
 515
 516         if (i < num) {
 517                 atomic_add_int(&vpid_alloc_failed, 1);
 518
 519                 /*
 520                  * If the unit number allocator does not have enough unique
 521                  * VPIDs then we need to allocate from the [1,VM_MAXCPU] range.
 522                  *
 523                  * These VPIDs are not be unique across VMs but this does not
 524                  * affect correctness because the combined mappings are also
 525                  * tagged with the EP4TA which is unique for each VM.
 526                  *
 527                  * It is still sub-optimal because the invvpid will invalidate
 528                  * combined mappings for a particular VPID across all EP4TAs.
 529                  */
 530                 while (i-- > 0)
 531                         vpid_free(vpid[i]);
 532
 533                 for (i = 0; i < num; i++)
 534                         vpid[i] = i + 1;
 535         }
 536 }
 537
 538 static void
 539 vpid_init(void)
 540 {
 541         /*
 542          * VPID 0 is required when the "enable VPID" execution control is
 543          * disabled.
 544          *
 545          * VPIDs [1,VM_MAXCPU] are used as the "overflow namespace" when the
 546          * unit number allocator does not have sufficient unique VPIDs to
 547          * satisfy the allocation.
 548          *
 549          * The remaining VPIDs are managed by the unit number allocator.
 550          */
 551         vpid_unr = new_unrhdr(VM_MAXCPU + 1, 0xffff, NULL);
 552 }
 553
 554 static void
 555 vmx_disable(void *arg __unused)
 556 {
 557         struct invvpid_desc invvpid_desc = { 0 };
 558         struct invept_desc invept_desc = { 0 };
 559
 560         if (vmxon_enabled[curcpu]) {
 561                 /*
 562                  * See sections 25.3.3.3 and 25.3.3.4 in Intel Vol 3b.
 563                  *
 564                  * VMXON or VMXOFF are not required to invalidate any TLB
 565                  * caching structures. This prevents potential retention of
 566                  * cached information in the TLB between distinct VMX episodes.
 567                  */
 568                 invvpid(INVVPID_TYPE_ALL_CONTEXTS, invvpid_desc);
 569                 invept(INVEPT_TYPE_ALL_CONTEXTS, invept_desc);
 570                 vmxoff();
 571         }
 572         load_cr4(rcr4() & ~CR4_VMXE);
 573 }
 574
 575 static int
 576 vmx_cleanup(void)
 577 {
 578
 579         if (pirvec >= 0)
 580                 lapic_ipi_free(pirvec);
 581
 582         if (vpid_unr != NULL) {
 583                 delete_unrhdr(vpid_unr);
 584                 vpid_unr = NULL;
 585         }
 586
 587         if (nmi_flush_l1d_sw == 1)
 588                 nmi_flush_l1d_sw = 0;
 589
 590         smp_rendezvous(NULL, vmx_disable, NULL, NULL);
 591
 592         return (0);
 593 }
 594
 595 static void
 596 vmx_enable(void *arg __unused)
 597 {
 598         int error;
 599         uint64_t feature_control;
 600
 601         feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
 602         if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 0 ||
 603             (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) {
 604                 wrmsr(MSR_IA32_FEATURE_CONTROL,
 605                     feature_control | IA32_FEATURE_CONTROL_VMX_EN |
 606                     IA32_FEATURE_CONTROL_LOCK);
 607         }
 608
 609         load_cr4(rcr4() | CR4_VMXE);
 610
 611         *(uint32_t *)vmxon_region[curcpu] = vmx_revision();
 612         error = vmxon(vmxon_region[curcpu]);
 613         if (error == 0)
 614                 vmxon_enabled[curcpu] = 1;
 615 }
 616
 617 static void
 618 vmx_restore(void)
 619 {
 620
 621         if (vmxon_enabled[curcpu])
 622                 vmxon(vmxon_region[curcpu]);
 623 }
 624
 625 static int
 626 vmx_init(int ipinum)
 627 {
 628         int error, use_tpr_shadow;
 629         uint64_t basic, fixed0, fixed1, feature_control;
 630         uint32_t tmp, procbased2_vid_bits;
 631
 632         /* CPUID.1:ECX[bit 5] must be 1 for processor to support VMX */
 633         if (!(cpu_feature2 & CPUID2_VMX)) {
 634                 printf("vmx_init: processor does not support VMX operation\n");
 635                 return (ENXIO);
 636         }
 637
 638         /*
 639          * Verify that MSR_IA32_FEATURE_CONTROL lock and VMXON enable bits
 640          * are set (bits 0 and 2 respectively).
 641          */
 642         feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
 643         if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 1 &&
 644             (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) {
 645                 printf("vmx_init: VMX operation disabled by BIOS\n");
 646                 return (ENXIO);
 647         }
 648
 649         /*
 650          * Verify capabilities MSR_VMX_BASIC:
 651          * - bit 54 indicates support for INS/OUTS decoding
 652          */
 653         basic = rdmsr(MSR_VMX_BASIC);
 654         if ((basic & (1UL << 54)) == 0) {
 655                 printf("vmx_init: processor does not support desired basic "
 656                     "capabilities\n");
 657                 return (EINVAL);
 658         }
 659
 660         /* Check support for primary processor-based VM-execution controls */
 661         error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 662                                MSR_VMX_TRUE_PROCBASED_CTLS,
 663                                PROCBASED_CTLS_ONE_SETTING,
 664                                PROCBASED_CTLS_ZERO_SETTING, &procbased_ctls);
 665         if (error) {
 666                 printf("vmx_init: processor does not support desired primary "
 667                        "processor-based controls\n");
 668                 return (error);
 669         }
 670
 671         /* Clear the processor-based ctl bits that are set on demand */
 672         procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING;
 673
 674         /* Check support for secondary processor-based VM-execution controls */
 675         error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
 676                                MSR_VMX_PROCBASED_CTLS2,
 677                                PROCBASED_CTLS2_ONE_SETTING,
 678                                PROCBASED_CTLS2_ZERO_SETTING, &procbased_ctls2);
 679         if (error) {
 680                 printf("vmx_init: processor does not support desired secondary "
 681                        "processor-based controls\n");
 682                 return (error);
 683         }
 684
 685         /* Check support for VPID */
 686         error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
 687                                PROCBASED2_ENABLE_VPID, 0, &tmp);
 688         if (error == 0)
 689                 procbased_ctls2 |= PROCBASED2_ENABLE_VPID;
 690
 691         /* Check support for pin-based VM-execution controls */
 692         error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
 693                                MSR_VMX_TRUE_PINBASED_CTLS,
 694                                PINBASED_CTLS_ONE_SETTING,
 695                                PINBASED_CTLS_ZERO_SETTING, &pinbased_ctls);
 696         if (error) {
 697                 printf("vmx_init: processor does not support desired "
 698                        "pin-based controls\n");
 699                 return (error);
 700         }
 701
 702         /* Check support for VM-exit controls */
 703         error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, MSR_VMX_TRUE_EXIT_CTLS,
 704                                VM_EXIT_CTLS_ONE_SETTING,
 705                                VM_EXIT_CTLS_ZERO_SETTING,
 706                                &exit_ctls);
 707         if (error) {
 708                 printf("vmx_init: processor does not support desired "
 709                     "exit controls\n");
 710                 return (error);
 711         }
 712
 713         /* Check support for VM-entry controls */
 714         error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS, MSR_VMX_TRUE_ENTRY_CTLS,
 715             VM_ENTRY_CTLS_ONE_SETTING, VM_ENTRY_CTLS_ZERO_SETTING,
 716             &entry_ctls);
 717         if (error) {
 718                 printf("vmx_init: processor does not support desired "
 719                     "entry controls\n");
 720                 return (error);
 721         }
 722
 723         /*
 724          * Check support for optional features by testing them
 725          * as individual bits
 726          */
 727         cap_halt_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 728                                         MSR_VMX_TRUE_PROCBASED_CTLS,
 729                                         PROCBASED_HLT_EXITING, 0,
 730                                         &tmp) == 0);
 731
 732         cap_monitor_trap = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 733                                         MSR_VMX_PROCBASED_CTLS,
 734                                         PROCBASED_MTF, 0,
 735                                         &tmp) == 0);
 736
 737         cap_pause_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 738                                          MSR_VMX_TRUE_PROCBASED_CTLS,
 739                                          PROCBASED_PAUSE_EXITING, 0,
 740                                          &tmp) == 0);
 741
 742         cap_unrestricted_guest = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
 743                                         MSR_VMX_PROCBASED_CTLS2,
 744                                         PROCBASED2_UNRESTRICTED_GUEST, 0,
 745                                         &tmp) == 0);
 746
 747         cap_invpcid = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
 748             MSR_VMX_PROCBASED_CTLS2, PROCBASED2_ENABLE_INVPCID, 0,
 749             &tmp) == 0);
 750
 751         /*
 752          * Check support for virtual interrupt delivery.
 753          */
 754         procbased2_vid_bits = (PROCBASED2_VIRTUALIZE_APIC_ACCESSES |
 755             PROCBASED2_VIRTUALIZE_X2APIC_MODE |
 756             PROCBASED2_APIC_REGISTER_VIRTUALIZATION |
 757             PROCBASED2_VIRTUAL_INTERRUPT_DELIVERY);
 758
 759         use_tpr_shadow = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 760             MSR_VMX_TRUE_PROCBASED_CTLS, PROCBASED_USE_TPR_SHADOW, 0,
 761             &tmp) == 0);
 762
 763         error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
 764             procbased2_vid_bits, 0, &tmp);
 765         if (error == 0 && use_tpr_shadow) {
 766                 virtual_interrupt_delivery = 1;
 767                 TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_vid",
 768                     &virtual_interrupt_delivery);
 769         }
 770
 771         if (virtual_interrupt_delivery) {
 772                 procbased_ctls |= PROCBASED_USE_TPR_SHADOW;
 773                 procbased_ctls2 |= procbased2_vid_bits;
 774                 procbased_ctls2 &= ~PROCBASED2_VIRTUALIZE_X2APIC_MODE;
 775
 776                 /*
 777                  * No need to emulate accesses to %CR8 if virtual
 778                  * interrupt delivery is enabled.
 779                  */
 780                 procbased_ctls &= ~PROCBASED_CR8_LOAD_EXITING;
 781                 procbased_ctls &= ~PROCBASED_CR8_STORE_EXITING;
 782
 783                 /*
 784                  * Check for Posted Interrupts only if Virtual Interrupt
 785                  * Delivery is enabled.
 786                  */
 787                 error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
 788                     MSR_VMX_TRUE_PINBASED_CTLS, PINBASED_POSTED_INTERRUPT, 0,
 789                     &tmp);
 790                 if (error == 0) {
 791                         pirvec = lapic_ipi_alloc(pti ? &IDTVEC(justreturn1_pti) :
 792                             &IDTVEC(justreturn));
 793                         if (pirvec < 0) {
 794                                 if (bootverbose) {
 795                                         printf("vmx_init: unable to allocate "
 796                                             "posted interrupt vector\n");
 797                                 }
 798                         } else {
 799                                 posted_interrupts = 1;
 800                                 TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_pir",
 801                                     &posted_interrupts);
 802                         }
 803                 }
 804         }
 805
 806         if (posted_interrupts)
 807                     pinbased_ctls |= PINBASED_POSTED_INTERRUPT;
 808
 809         /* Initialize EPT */
 810         error = ept_init(ipinum);
 811         if (error) {
 812                 printf("vmx_init: ept initialization failed (%d)\n", error);
 813                 return (error);
 814         }
 815
 816         guest_l1d_flush = (cpu_ia32_arch_caps &
 817             IA32_ARCH_CAP_SKIP_L1DFL_VMENTRY) == 0;
 818         TUNABLE_INT_FETCH("hw.vmm.l1d_flush", &guest_l1d_flush);
 819
 820         /*
 821          * L1D cache flush is enabled.  Use IA32_FLUSH_CMD MSR when
 822          * available.  Otherwise fall back to the software flush
 823          * method which loads enough data from the kernel text to
 824          * flush existing L1D content, both on VMX entry and on NMI
 825          * return.
 826          */
 827         if (guest_l1d_flush) {
 828                 if ((cpu_stdext_feature3 & CPUID_STDEXT3_L1D_FLUSH) == 0) {
 829                         guest_l1d_flush_sw = 1;
 830                         TUNABLE_INT_FETCH("hw.vmm.l1d_flush_sw",
 831                             &guest_l1d_flush_sw);
 832                 }
 833                 if (guest_l1d_flush_sw) {
 834                         if (nmi_flush_l1d_sw <= 1)
 835                                 nmi_flush_l1d_sw = 1;
 836                 } else {
 837                         msr_load_list[0].index = MSR_IA32_FLUSH_CMD;
 838                         msr_load_list[0].val = IA32_FLUSH_CMD_L1D;
 839                 }
 840         }
 841
 842         /*
 843          * Stash the cr0 and cr4 bits that must be fixed to 0 or 1
 844          */
 845         fixed0 = rdmsr(MSR_VMX_CR0_FIXED0);
 846         fixed1 = rdmsr(MSR_VMX_CR0_FIXED1);
 847         cr0_ones_mask = fixed0 & fixed1;
 848         cr0_zeros_mask = ~fixed0 & ~fixed1;
 849
 850         /*
 851          * CR0_PE and CR0_PG can be set to zero in VMX non-root operation
 852          * if unrestricted guest execution is allowed.
 853          */
 854         if (cap_unrestricted_guest)
 855                 cr0_ones_mask &= ~(CR0_PG | CR0_PE);
 856
 857         /*
 858          * Do not allow the guest to set CR0_NW or CR0_CD.
 859          */
 860         cr0_zeros_mask |= (CR0_NW | CR0_CD);
 861
 862         fixed0 = rdmsr(MSR_VMX_CR4_FIXED0);
 863         fixed1 = rdmsr(MSR_VMX_CR4_FIXED1);
 864         cr4_ones_mask = fixed0 & fixed1;
 865         cr4_zeros_mask = ~fixed0 & ~fixed1;
 866
 867         vpid_init();
 868
 869         vmx_msr_init();
 870
 871         /* enable VMX operation */
 872         smp_rendezvous(NULL, vmx_enable, NULL, NULL);
 873
 874         vmx_initialized = 1;
 875
 876         return (0);
 877 }
 878
 879 static void
 880 vmx_trigger_hostintr(int vector)
 881 {
 882         uintptr_t func;
 883         struct gate_descriptor *gd;
 884
 885         gd = &idt[vector];
 886
 887         KASSERT(vector >= 32 && vector <= 255, ("vmx_trigger_hostintr: "
 888             "invalid vector %d", vector));
 889         KASSERT(gd->gd_p == 1, ("gate descriptor for vector %d not present",
 890             vector));
 891         KASSERT(gd->gd_type == SDT_SYSIGT, ("gate descriptor for vector %d "
 892             "has invalid type %d", vector, gd->gd_type));
 893         KASSERT(gd->gd_dpl == SEL_KPL, ("gate descriptor for vector %d "
 894             "has invalid dpl %d", vector, gd->gd_dpl));
 895         KASSERT(gd->gd_selector == GSEL(GCODE_SEL, SEL_KPL), ("gate descriptor "
 896             "for vector %d has invalid selector %d", vector, gd->gd_selector));
 897         KASSERT(gd->gd_ist == 0, ("gate descriptor for vector %d has invalid "
 898             "IST %d", vector, gd->gd_ist));
 899
 900         func = ((long)gd->gd_hioffset << 16 | gd->gd_looffset);
 901         vmx_call_isr(func);
 902 }
 903
 904 static int
 905 vmx_setup_cr_shadow(int which, struct vmcs *vmcs, uint32_t initial)
 906 {
 907         int error, mask_ident, shadow_ident;
 908         uint64_t mask_value;
 909
 910         if (which != 0 && which != 4)
 911                 panic("vmx_setup_cr_shadow: unknown cr%d", which);
 912
 913         if (which == 0) {
 914                 mask_ident = VMCS_CR0_MASK;
 915                 mask_value = cr0_ones_mask | cr0_zeros_mask;
 916                 shadow_ident = VMCS_CR0_SHADOW;
 917         } else {
 918                 mask_ident = VMCS_CR4_MASK;
 919                 mask_value = cr4_ones_mask | cr4_zeros_mask;
 920                 shadow_ident = VMCS_CR4_SHADOW;
 921         }
 922
 923         error = vmcs_setreg(vmcs, 0, VMCS_IDENT(mask_ident), mask_value);
 924         if (error)
 925                 return (error);
 926
 927         error = vmcs_setreg(vmcs, 0, VMCS_IDENT(shadow_ident), initial);
 928         if (error)
 929                 return (error);
 930
 931         return (0);
 932 }
 933 #define vmx_setup_cr0_shadow(vmcs,init) vmx_setup_cr_shadow(0, (vmcs), (init))
 934 #define vmx_setup_cr4_shadow(vmcs,init) vmx_setup_cr_shadow(4, (vmcs), (init))
 935
 936 static void *
 937 vmx_vminit(struct vm *vm, pmap_t pmap)
 938 {
 939         uint16_t vpid[VM_MAXCPU];
 940         int i, error;
 941         struct vmx *vmx;
 942         struct vmcs *vmcs;
 943         uint32_t exc_bitmap;
 944         uint16_t maxcpus;
 945
 946         vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO);
 947         if ((uintptr_t)vmx & PAGE_MASK) {
 948                 panic("malloc of struct vmx not aligned on %d byte boundary",
 949                       PAGE_SIZE);
 950         }
 951         vmx->vm = vm;
 952
 953         vmx->eptp = eptp(vtophys((vm_offset_t)pmap->pm_pml4));
 954
 955         /*
 956          * Clean up EPTP-tagged guest physical and combined mappings
 957          *
 958          * VMX transitions are not required to invalidate any guest physical
 959          * mappings. So, it may be possible for stale guest physical mappings
 960          * to be present in the processor TLBs.
 961          *
 962          * Combined mappings for this EP4TA are also invalidated for all VPIDs.
 963          */
 964         ept_invalidate_mappings(vmx->eptp);
 965
 966         msr_bitmap_initialize(vmx->msr_bitmap);
 967
 968         /*
 969          * It is safe to allow direct access to MSR_GSBASE and MSR_FSBASE.
 970          * The guest FSBASE and GSBASE are saved and restored during
 971          * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are
 972          * always restored from the vmcs host state area on vm-exit.
 973          *
 974          * The SYSENTER_CS/ESP/EIP MSRs are identical to FS/GSBASE in
 975          * how they are saved/restored so can be directly accessed by the
 976          * guest.
 977          *
 978          * MSR_EFER is saved and restored in the guest VMCS area on a
 979          * VM exit and entry respectively. It is also restored from the
 980          * host VMCS area on a VM exit.
 981          *
 982          * The TSC MSR is exposed read-only. Writes are disallowed as
 983          * that will impact the host TSC.  If the guest does a write
 984          * the "use TSC offsetting" execution control is enabled and the
 985          * difference between the host TSC and the guest TSC is written
 986          * into the TSC offset in the VMCS.
 987          */
 988         if (guest_msr_rw(vmx, MSR_GSBASE) ||
 989             guest_msr_rw(vmx, MSR_FSBASE) ||
 990             guest_msr_rw(vmx, MSR_SYSENTER_CS_MSR) ||
 991             guest_msr_rw(vmx, MSR_SYSENTER_ESP_MSR) ||
 992             guest_msr_rw(vmx, MSR_SYSENTER_EIP_MSR) ||
 993             guest_msr_rw(vmx, MSR_EFER) ||
 994             guest_msr_ro(vmx, MSR_TSC))
 995                 panic("vmx_vminit: error setting guest msr access");
 996
 997         vpid_alloc(vpid, VM_MAXCPU);
 998
 999         if (virtual_interrupt_delivery) {
1000                 error = vm_map_mmio(vm, DEFAULT_APIC_BASE, PAGE_SIZE,
1001                     APIC_ACCESS_ADDRESS);
1002                 /* XXX this should really return an error to the caller */
1003                 KASSERT(error == 0, ("vm_map_mmio(apicbase) error %d", error));
1004         }
1005
1006         maxcpus = vm_get_maxcpus(vm);
1007         for (i = 0; i < maxcpus; i++) {
1008                 vmcs = &vmx->vmcs[i];
1009                 vmcs->identifier = vmx_revision();
1010                 error = vmclear(vmcs);
1011                 if (error != 0) {
1012                         panic("vmx_vminit: vmclear error %d on vcpu %d\n",
1013                               error, i);
1014                 }
1015
1016                 vmx_msr_guest_init(vmx, i);
1017
1018                 error = vmcs_init(vmcs);
1019                 KASSERT(error == 0, ("vmcs_init error %d", error));
1020
1021                 VMPTRLD(vmcs);
1022                 error = 0;
1023                 error += vmwrite(VMCS_HOST_RSP, (u_long)&vmx->ctx[i]);
1024                 error += vmwrite(VMCS_EPTP, vmx->eptp);
1025                 error += vmwrite(VMCS_PIN_BASED_CTLS, pinbased_ctls);
1026                 error += vmwrite(VMCS_PRI_PROC_BASED_CTLS, procbased_ctls);
1027                 error += vmwrite(VMCS_SEC_PROC_BASED_CTLS, procbased_ctls2);
1028                 error += vmwrite(VMCS_EXIT_CTLS, exit_ctls);
1029                 error += vmwrite(VMCS_ENTRY_CTLS, entry_ctls);
1030                 error += vmwrite(VMCS_MSR_BITMAP, vtophys(vmx->msr_bitmap));
1031                 error += vmwrite(VMCS_VPID, vpid[i]);
1032
1033                 if (guest_l1d_flush && !guest_l1d_flush_sw) {
1034                         vmcs_write(VMCS_ENTRY_MSR_LOAD, pmap_kextract(
1035                             (vm_offset_t)&msr_load_list[0]));
1036                         vmcs_write(VMCS_ENTRY_MSR_LOAD_COUNT,
1037                             nitems(msr_load_list));
1038                         vmcs_write(VMCS_EXIT_MSR_STORE, 0);
1039                         vmcs_write(VMCS_EXIT_MSR_STORE_COUNT, 0);
1040                 }
1041
1042                 /* exception bitmap */
1043                 if (vcpu_trace_exceptions(vm, i))
1044                         exc_bitmap = 0xffffffff;
1045                 else
1046                         exc_bitmap = 1 << IDT_MC;
1047                 error += vmwrite(VMCS_EXCEPTION_BITMAP, exc_bitmap);
1048
1049                 vmx->ctx[i].guest_dr6 = DBREG_DR6_RESERVED1;
1050                 error += vmwrite(VMCS_GUEST_DR7, DBREG_DR7_RESERVED1);
1051
1052                 if (virtual_interrupt_delivery) {
1053                         error += vmwrite(VMCS_APIC_ACCESS, APIC_ACCESS_ADDRESS);
1054                         error += vmwrite(VMCS_VIRTUAL_APIC,
1055                             vtophys(&vmx->apic_page[i]));
1056                         error += vmwrite(VMCS_EOI_EXIT0, 0);
1057                         error += vmwrite(VMCS_EOI_EXIT1, 0);
1058                         error += vmwrite(VMCS_EOI_EXIT2, 0);
1059                         error += vmwrite(VMCS_EOI_EXIT3, 0);
1060                 }
1061                 if (posted_interrupts) {
1062                         error += vmwrite(VMCS_PIR_VECTOR, pirvec);
1063                         error += vmwrite(VMCS_PIR_DESC,
1064                             vtophys(&vmx->pir_desc[i]));
1065                 }
1066                 VMCLEAR(vmcs);
1067                 KASSERT(error == 0, ("vmx_vminit: error customizing the vmcs"));
1068
1069                 vmx->cap[i].set = 0;
1070                 vmx->cap[i].proc_ctls = procbased_ctls;
1071                 vmx->cap[i].proc_ctls2 = procbased_ctls2;
1072
1073                 vmx->state[i].nextrip = ~0;
1074                 vmx->state[i].lastcpu = NOCPU;
1075                 vmx->state[i].vpid = vpid[i];
1076
1077                 /*
1078                  * Set up the CR0/4 shadows, and init the read shadow
1079                  * to the power-on register value from the Intel Sys Arch.
1080                  *  CR0 - 0x60000010
1081                  *  CR4 - 0
1082                  */
1083                 error = vmx_setup_cr0_shadow(vmcs, 0x60000010);
1084                 if (error != 0)
1085                         panic("vmx_setup_cr0_shadow %d", error);
1086
1087                 error = vmx_setup_cr4_shadow(vmcs, 0);
1088                 if (error != 0)
1089                         panic("vmx_setup_cr4_shadow %d", error);
1090
1091                 vmx->ctx[i].pmap = pmap;
1092         }
1093
1094         return (vmx);
1095 }
1096
1097 static int
1098 vmx_handle_cpuid(struct vm *vm, int vcpu, struct vmxctx *vmxctx)
1099 {
1100         int handled, func;
1101
1102         func = vmxctx->guest_rax;
1103
1104         handled = x86_emulate_cpuid(vm, vcpu,
1105                                     (uint32_t*)(&vmxctx->guest_rax),
1106                                     (uint32_t*)(&vmxctx->guest_rbx),
1107                                     (uint32_t*)(&vmxctx->guest_rcx),
1108                                     (uint32_t*)(&vmxctx->guest_rdx));
1109         return (handled);
1110 }
1111
1112 static __inline void
1113 vmx_run_trace(struct vmx *vmx, int vcpu)
1114 {
1115 #ifdef KTR
1116         VCPU_CTR1(vmx->vm, vcpu, "Resume execution at %#lx", vmcs_guest_rip());
1117 #endif
1118 }
1119
1120 static __inline void
1121 vmx_exit_trace(struct vmx *vmx, int vcpu, uint64_t rip, uint32_t exit_reason,
1122                int handled)
1123 {
1124 #ifdef KTR
1125         VCPU_CTR3(vmx->vm, vcpu, "%s %s vmexit at 0x%0lx",
1126                  handled ? "handled" : "unhandled",
1127                  exit_reason_to_str(exit_reason), rip);
1128 #endif
1129 }
1130
1131 static __inline void
1132 vmx_astpending_trace(struct vmx *vmx, int vcpu, uint64_t rip)
1133 {
1134 #ifdef KTR
1135         VCPU_CTR1(vmx->vm, vcpu, "astpending vmexit at 0x%0lx", rip);
1136 #endif
1137 }
1138
1139 static VMM_STAT_INTEL(VCPU_INVVPID_SAVED, "Number of vpid invalidations saved");
1140 static VMM_STAT_INTEL(VCPU_INVVPID_DONE, "Number of vpid invalidations done");
1141
1142 /*
1143  * Invalidate guest mappings identified by its vpid from the TLB.
1144  */
1145 static __inline void
1146 vmx_invvpid(struct vmx *vmx, int vcpu, pmap_t pmap, int running)
1147 {
1148         struct vmxstate *vmxstate;
1149         struct invvpid_desc invvpid_desc;
1150
1151         vmxstate = &vmx->state[vcpu];
1152         if (vmxstate->vpid == 0)
1153                 return;
1154
1155         if (!running) {
1156                 /*
1157                  * Set the 'lastcpu' to an invalid host cpu.
1158                  *
1159                  * This will invalidate TLB entries tagged with the vcpu's
1160                  * vpid the next time it runs via vmx_set_pcpu_defaults().
1161                  */
1162                 vmxstate->lastcpu = NOCPU;
1163                 return;
1164         }
1165
1166         KASSERT(curthread->td_critnest > 0, ("%s: vcpu %d running outside "
1167             "critical section", __func__, vcpu));
1168
1169         /*
1170          * Invalidate all mappings tagged with 'vpid'
1171          *
1172          * We do this because this vcpu was executing on a different host
1173          * cpu when it last ran. We do not track whether it invalidated
1174          * mappings associated with its 'vpid' during that run. So we must
1175          * assume that the mappings associated with 'vpid' on 'curcpu' are
1176          * stale and invalidate them.
1177          *
1178          * Note that we incur this penalty only when the scheduler chooses to
1179          * move the thread associated with this vcpu between host cpus.
1180          *
1181          * Note also that this will invalidate mappings tagged with 'vpid'
1182          * for "all" EP4TAs.
1183          */
1184         if (pmap->pm_eptgen == vmx->eptgen[curcpu]) {
1185                 invvpid_desc._res1 = 0;
1186                 invvpid_desc._res2 = 0;
1187                 invvpid_desc.vpid = vmxstate->vpid;
1188                 invvpid_desc.linear_addr = 0;
1189                 invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc);
1190                 vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_DONE, 1);
1191         } else {
1192                 /*
1193                  * The invvpid can be skipped if an invept is going to
1194                  * be performed before entering the guest. The invept
1195                  * will invalidate combined mappings tagged with
1196                  * 'vmx->eptp' for all vpids.
1197                  */
1198                 vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_SAVED, 1);
1199         }
1200 }
1201
1202 static void
1203 vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu, pmap_t pmap)
1204 {
1205         struct vmxstate *vmxstate;
1206
1207         vmxstate = &vmx->state[vcpu];
1208         if (vmxstate->lastcpu == curcpu)
1209                 return;
1210
1211         vmxstate->lastcpu = curcpu;
1212
1213         vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1);
1214
1215         vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase());
1216         vmcs_write(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase());
1217         vmcs_write(VMCS_HOST_GS_BASE, vmm_get_host_gsbase());
1218         vmx_invvpid(vmx, vcpu, pmap, 1);
1219 }
1220
1221 /*
1222  * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set.
1223  */
1224 CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0);
1225
1226 static void __inline
1227 vmx_set_int_window_exiting(struct vmx *vmx, int vcpu)
1228 {
1229
1230         if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) == 0) {
1231                 vmx->cap[vcpu].proc_ctls |= PROCBASED_INT_WINDOW_EXITING;
1232                 vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1233                 VCPU_CTR0(vmx->vm, vcpu, "Enabling interrupt window exiting");
1234         }
1235 }
1236
1237 static void __inline
1238 vmx_clear_int_window_exiting(struct vmx *vmx, int vcpu)
1239 {
1240
1241         KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0,
1242             ("intr_window_exiting not set: %#x", vmx->cap[vcpu].proc_ctls));
1243         vmx->cap[vcpu].proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING;
1244         vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1245         VCPU_CTR0(vmx->vm, vcpu, "Disabling interrupt window exiting");
1246 }
1247
1248 static void __inline
1249 vmx_set_nmi_window_exiting(struct vmx *vmx, int vcpu)
1250 {
1251
1252         if ((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) == 0) {
1253                 vmx->cap[vcpu].proc_ctls |= PROCBASED_NMI_WINDOW_EXITING;
1254                 vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1255                 VCPU_CTR0(vmx->vm, vcpu, "Enabling NMI window exiting");
1256         }
1257 }
1258
1259 static void __inline
1260 vmx_clear_nmi_window_exiting(struct vmx *vmx, int vcpu)
1261 {
1262
1263         KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) != 0,
1264             ("nmi_window_exiting not set %#x", vmx->cap[vcpu].proc_ctls));
1265         vmx->cap[vcpu].proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING;
1266         vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1267         VCPU_CTR0(vmx->vm, vcpu, "Disabling NMI window exiting");
1268 }
1269
1270 int
1271 vmx_set_tsc_offset(struct vmx *vmx, int vcpu, uint64_t offset)
1272 {
1273         int error;
1274
1275         if ((vmx->cap[vcpu].proc_ctls & PROCBASED_TSC_OFFSET) == 0) {
1276                 vmx->cap[vcpu].proc_ctls |= PROCBASED_TSC_OFFSET;
1277                 vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1278                 VCPU_CTR0(vmx->vm, vcpu, "Enabling TSC offsetting");
1279         }
1280
1281         error = vmwrite(VMCS_TSC_OFFSET, offset);
1282
1283         return (error);
1284 }
1285
1286 #define NMI_BLOCKING    (VMCS_INTERRUPTIBILITY_NMI_BLOCKING |           \
1287                          VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
1288 #define HWINTR_BLOCKING (VMCS_INTERRUPTIBILITY_STI_BLOCKING |           \
1289                          VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
1290
1291 static void
1292 vmx_inject_nmi(struct vmx *vmx, int vcpu)
1293 {
1294         uint32_t gi, info;
1295
1296         gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1297         KASSERT((gi & NMI_BLOCKING) == 0, ("vmx_inject_nmi: invalid guest "
1298             "interruptibility-state %#x", gi));
1299
1300         info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1301         KASSERT((info & VMCS_INTR_VALID) == 0, ("vmx_inject_nmi: invalid "
1302             "VM-entry interruption information %#x", info));
1303
1304         /*
1305          * Inject the virtual NMI. The vector must be the NMI IDT entry
1306          * or the VMCS entry check will fail.
1307          */
1308         info = IDT_NMI | VMCS_INTR_T_NMI | VMCS_INTR_VALID;
1309         vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1310
1311         VCPU_CTR0(vmx->vm, vcpu, "Injecting vNMI");
1312
1313         /* Clear the request */
1314         vm_nmi_clear(vmx->vm, vcpu);
1315 }
1316
1317 static void
1318 vmx_inject_interrupts(struct vmx *vmx, int vcpu, struct vlapic *vlapic,
1319     uint64_t guestrip)
1320 {
1321         int vector, need_nmi_exiting, extint_pending;
1322         uint64_t rflags, entryinfo;
1323         uint32_t gi, info;
1324
1325         if (vmx->state[vcpu].nextrip != guestrip) {
1326                 gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1327                 if (gi & HWINTR_BLOCKING) {
1328                         VCPU_CTR2(vmx->vm, vcpu, "Guest interrupt blocking "
1329                             "cleared due to rip change: %#lx/%#lx",
1330                             vmx->state[vcpu].nextrip, guestrip);
1331                         gi &= ~HWINTR_BLOCKING;
1332                         vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1333                 }
1334         }
1335
1336         if (vm_entry_intinfo(vmx->vm, vcpu, &entryinfo)) {
1337                 KASSERT((entryinfo & VMCS_INTR_VALID) != 0, ("%s: entry "
1338                     "intinfo is not valid: %#lx", __func__, entryinfo));
1339
1340                 info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1341                 KASSERT((info & VMCS_INTR_VALID) == 0, ("%s: cannot inject "
1342                      "pending exception: %#lx/%#x", __func__, entryinfo, info));
1343
1344                 info = entryinfo;
1345                 vector = info & 0xff;
1346                 if (vector == IDT_BP || vector == IDT_OF) {
1347                         /*
1348                          * VT-x requires #BP and #OF to be injected as software
1349                          * exceptions.
1350                          */
1351                         info &= ~VMCS_INTR_T_MASK;
1352                         info |= VMCS_INTR_T_SWEXCEPTION;
1353                 }
1354
1355                 if (info & VMCS_INTR_DEL_ERRCODE)
1356                         vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR, entryinfo >> 32);
1357
1358                 vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1359         }
1360
1361         if (vm_nmi_pending(vmx->vm, vcpu)) {
1362                 /*
1363                  * If there are no conditions blocking NMI injection then
1364                  * inject it directly here otherwise enable "NMI window
1365                  * exiting" to inject it as soon as we can.
1366                  *
1367                  * We also check for STI_BLOCKING because some implementations
1368                  * don't allow NMI injection in this case. If we are running
1369                  * on a processor that doesn't have this restriction it will
1370                  * immediately exit and the NMI will be injected in the
1371                  * "NMI window exiting" handler.
1372                  */
1373                 need_nmi_exiting = 1;
1374                 gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1375                 if ((gi & (HWINTR_BLOCKING | NMI_BLOCKING)) == 0) {
1376                         info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1377                         if ((info & VMCS_INTR_VALID) == 0) {
1378                                 vmx_inject_nmi(vmx, vcpu);
1379                                 need_nmi_exiting = 0;
1380                         } else {
1381                                 VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI "
1382                                     "due to VM-entry intr info %#x", info);
1383                         }
1384                 } else {
1385                         VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI due to "
1386                             "Guest Interruptibility-state %#x", gi);
1387                 }
1388
1389                 if (need_nmi_exiting)
1390                         vmx_set_nmi_window_exiting(vmx, vcpu);
1391         }
1392
1393         extint_pending = vm_extint_pending(vmx->vm, vcpu);
1394
1395         if (!extint_pending && virtual_interrupt_delivery) {
1396                 vmx_inject_pir(vlapic);
1397                 return;
1398         }
1399
1400         /*
1401          * If interrupt-window exiting is already in effect then don't bother
1402          * checking for pending interrupts. This is just an optimization and
1403          * not needed for correctness.
1404          */
1405         if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0) {
1406                 VCPU_CTR0(vmx->vm, vcpu, "Skip interrupt injection due to "
1407                     "pending int_window_exiting");
1408                 return;
1409         }
1410
1411         if (!extint_pending) {
1412                 /* Ask the local apic for a vector to inject */
1413                 if (!vlapic_pending_intr(vlapic, &vector))
1414                         return;
1415
1416                 /*
1417                  * From the Intel SDM, Volume 3, Section "Maskable
1418                  * Hardware Interrupts":
1419                  * - maskable interrupt vectors [16,255] can be delivered
1420                  *   through the local APIC.
1421                 */
1422                 KASSERT(vector >= 16 && vector <= 255,
1423                     ("invalid vector %d from local APIC", vector));
1424         } else {
1425                 /* Ask the legacy pic for a vector to inject */
1426                 vatpic_pending_intr(vmx->vm, &vector);
1427
1428                 /*
1429                  * From the Intel SDM, Volume 3, Section "Maskable
1430                  * Hardware Interrupts":
1431                  * - maskable interrupt vectors [0,255] can be delivered
1432                  *   through the INTR pin.
1433                  */
1434                 KASSERT(vector >= 0 && vector <= 255,
1435                     ("invalid vector %d from INTR", vector));
1436         }
1437
1438         /* Check RFLAGS.IF and the interruptibility state of the guest */
1439         rflags = vmcs_read(VMCS_GUEST_RFLAGS);
1440         if ((rflags & PSL_I) == 0) {
1441                 VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
1442                     "rflags %#lx", vector, rflags);
1443                 goto cantinject;
1444         }
1445
1446         gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1447         if (gi & HWINTR_BLOCKING) {
1448                 VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
1449                     "Guest Interruptibility-state %#x", vector, gi);
1450                 goto cantinject;
1451         }
1452
1453         info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1454         if (info & VMCS_INTR_VALID) {
1455                 /*
1456                  * This is expected and could happen for multiple reasons:
1457                  * - A vectoring VM-entry was aborted due to astpending
1458                  * - A VM-exit happened during event injection.
1459                  * - An exception was injected above.
1460                  * - An NMI was injected above or after "NMI window exiting"
1461                  */
1462                 VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
1463                     "VM-entry intr info %#x", vector, info);
1464                 goto cantinject;
1465         }
1466
1467         /* Inject the interrupt */
1468         info = VMCS_INTR_T_HWINTR | VMCS_INTR_VALID;
1469         info |= vector;
1470         vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1471
1472         if (!extint_pending) {
1473                 /* Update the Local APIC ISR */
1474                 vlapic_intr_accepted(vlapic, vector);
1475         } else {
1476                 vm_extint_clear(vmx->vm, vcpu);
1477                 vatpic_intr_accepted(vmx->vm, vector);
1478
1479                 /*
1480                  * After we accepted the current ExtINT the PIC may
1481                  * have posted another one.  If that is the case, set
1482                  * the Interrupt Window Exiting execution control so
1483                  * we can inject that one too.
1484                  *
1485                  * Also, interrupt window exiting allows us to inject any
1486                  * pending APIC vector that was preempted by the ExtINT
1487                  * as soon as possible. This applies both for the software
1488                  * emulated vlapic and the hardware assisted virtual APIC.
1489                  */
1490                 vmx_set_int_window_exiting(vmx, vcpu);
1491         }
1492
1493         VCPU_CTR1(vmx->vm, vcpu, "Injecting hwintr at vector %d", vector);
1494
1495         return;
1496
1497 cantinject:
1498         /*
1499          * Set the Interrupt Window Exiting execution control so we can inject
1500          * the interrupt as soon as blocking condition goes away.
1501          */
1502         vmx_set_int_window_exiting(vmx, vcpu);
1503 }
1504
1505 /*
1506  * If the Virtual NMIs execution control is '1' then the logical processor
1507  * tracks virtual-NMI blocking in the Guest Interruptibility-state field of
1508  * the VMCS. An IRET instruction in VMX non-root operation will remove any
1509  * virtual-NMI blocking.
1510  *
1511  * This unblocking occurs even if the IRET causes a fault. In this case the
1512  * hypervisor needs to restore virtual-NMI blocking before resuming the guest.
1513  */
1514 static void
1515 vmx_restore_nmi_blocking(struct vmx *vmx, int vcpuid)
1516 {
1517         uint32_t gi;
1518
1519         VCPU_CTR0(vmx->vm, vcpuid, "Restore Virtual-NMI blocking");
1520         gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1521         gi |= VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
1522         vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1523 }
1524
1525 static void
1526 vmx_clear_nmi_blocking(struct vmx *vmx, int vcpuid)
1527 {
1528         uint32_t gi;
1529
1530         VCPU_CTR0(vmx->vm, vcpuid, "Clear Virtual-NMI blocking");
1531         gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1532         gi &= ~VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
1533         vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1534 }
1535
1536 static void
1537 vmx_assert_nmi_blocking(struct vmx *vmx, int vcpuid)
1538 {
1539         uint32_t gi;
1540
1541         gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1542         KASSERT(gi & VMCS_INTERRUPTIBILITY_NMI_BLOCKING,
1543             ("NMI blocking is not in effect %#x", gi));
1544 }
1545
1546 static int
1547 vmx_emulate_xsetbv(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
1548 {
1549         struct vmxctx *vmxctx;
1550         uint64_t xcrval;
1551         const struct xsave_limits *limits;
1552
1553         vmxctx = &vmx->ctx[vcpu];
1554         limits = vmm_get_xsave_limits();
1555
1556         /*
1557          * Note that the processor raises a GP# fault on its own if
1558          * xsetbv is executed for CPL != 0, so we do not have to
1559          * emulate that fault here.
1560          */
1561
1562         /* Only xcr0 is supported. */
1563         if (vmxctx->guest_rcx != 0) {
1564                 vm_inject_gp(vmx->vm, vcpu);
1565                 return (HANDLED);
1566         }
1567
1568         /* We only handle xcr0 if both the host and guest have XSAVE enabled. */
1569         if (!limits->xsave_enabled || !(vmcs_read(VMCS_GUEST_CR4) & CR4_XSAVE)) {
1570                 vm_inject_ud(vmx->vm, vcpu);
1571                 return (HANDLED);
1572         }
1573
1574         xcrval = vmxctx->guest_rdx << 32 | (vmxctx->guest_rax & 0xffffffff);
1575         if ((xcrval & ~limits->xcr0_allowed) != 0) {
1576                 vm_inject_gp(vmx->vm, vcpu);
1577                 return (HANDLED);
1578         }
1579
1580         if (!(xcrval & XFEATURE_ENABLED_X87)) {
1581                 vm_inject_gp(vmx->vm, vcpu);
1582                 return (HANDLED);
1583         }
1584
1585         /* AVX (YMM_Hi128) requires SSE. */
1586         if (xcrval & XFEATURE_ENABLED_AVX &&
1587             (xcrval & XFEATURE_AVX) != XFEATURE_AVX) {
1588                 vm_inject_gp(vmx->vm, vcpu);
1589                 return (HANDLED);
1590         }
1591
1592         /*
1593          * AVX512 requires base AVX (YMM_Hi128) as well as OpMask,
1594          * ZMM_Hi256, and Hi16_ZMM.
1595          */
1596         if (xcrval & XFEATURE_AVX512 &&
1597             (xcrval & (XFEATURE_AVX512 | XFEATURE_AVX)) !=
1598             (XFEATURE_AVX512 | XFEATURE_AVX)) {
1599                 vm_inject_gp(vmx->vm, vcpu);
1600                 return (HANDLED);
1601         }
1602
1603         /*
1604          * Intel MPX requires both bound register state flags to be
1605          * set.
1606          */
1607         if (((xcrval & XFEATURE_ENABLED_BNDREGS) != 0) !=
1608             ((xcrval & XFEATURE_ENABLED_BNDCSR) != 0)) {
1609                 vm_inject_gp(vmx->vm, vcpu);
1610                 return (HANDLED);
1611         }
1612
1613         /*
1614          * This runs "inside" vmrun() with the guest's FPU state, so
1615          * modifying xcr0 directly modifies the guest's xcr0, not the
1616          * host's.
1617          */
1618         load_xcr(0, xcrval);
1619         return (HANDLED);
1620 }
1621
1622 static uint64_t
1623 vmx_get_guest_reg(struct vmx *vmx, int vcpu, int ident)
1624 {
1625         const struct vmxctx *vmxctx;
1626
1627         vmxctx = &vmx->ctx[vcpu];
1628
1629         switch (ident) {
1630         case 0:
1631                 return (vmxctx->guest_rax);
1632         case 1:
1633                 return (vmxctx->guest_rcx);
1634         case 2:
1635                 return (vmxctx->guest_rdx);
1636         case 3:
1637                 return (vmxctx->guest_rbx);
1638         case 4:
1639                 return (vmcs_read(VMCS_GUEST_RSP));
1640         case 5:
1641                 return (vmxctx->guest_rbp);
1642         case 6:
1643                 return (vmxctx->guest_rsi);
1644         case 7:
1645                 return (vmxctx->guest_rdi);
1646         case 8:
1647                 return (vmxctx->guest_r8);
1648         case 9:
1649                 return (vmxctx->guest_r9);
1650         case 10:
1651                 return (vmxctx->guest_r10);
1652         case 11:
1653                 return (vmxctx->guest_r11);
1654         case 12:
1655                 return (vmxctx->guest_r12);
1656         case 13:
1657                 return (vmxctx->guest_r13);
1658         case 14:
1659                 return (vmxctx->guest_r14);
1660         case 15:
1661                 return (vmxctx->guest_r15);
1662         default:
1663                 panic("invalid vmx register %d", ident);
1664         }
1665 }
1666
1667 static void
1668 vmx_set_guest_reg(struct vmx *vmx, int vcpu, int ident, uint64_t regval)
1669 {
1670         struct vmxctx *vmxctx;
1671
1672         vmxctx = &vmx->ctx[vcpu];
1673
1674         switch (ident) {
1675         case 0:
1676                 vmxctx->guest_rax = regval;
1677                 break;
1678         case 1:
1679                 vmxctx->guest_rcx = regval;
1680                 break;
1681         case 2:
1682                 vmxctx->guest_rdx = regval;
1683                 break;
1684         case 3:
1685                 vmxctx->guest_rbx = regval;
1686                 break;
1687         case 4:
1688                 vmcs_write(VMCS_GUEST_RSP, regval);
1689                 break;
1690         case 5:
1691                 vmxctx->guest_rbp = regval;
1692                 break;
1693         case 6:
1694                 vmxctx->guest_rsi = regval;
1695                 break;
1696         case 7:
1697                 vmxctx->guest_rdi = regval;
1698                 break;
1699         case 8:
1700                 vmxctx->guest_r8 = regval;
1701                 break;
1702         case 9:
1703                 vmxctx->guest_r9 = regval;
1704                 break;
1705         case 10:
1706                 vmxctx->guest_r10 = regval;
1707                 break;
1708         case 11:
1709                 vmxctx->guest_r11 = regval;
1710                 break;
1711         case 12:
1712                 vmxctx->guest_r12 = regval;
1713                 break;
1714         case 13:
1715                 vmxctx->guest_r13 = regval;
1716                 break;
1717         case 14:
1718                 vmxctx->guest_r14 = regval;
1719                 break;
1720         case 15:
1721                 vmxctx->guest_r15 = regval;
1722                 break;
1723         default:
1724                 panic("invalid vmx register %d", ident);
1725         }
1726 }
1727
1728 static int
1729 vmx_emulate_cr0_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
1730 {
1731         uint64_t crval, regval;
1732
1733         /* We only handle mov to %cr0 at this time */
1734         if ((exitqual & 0xf0) != 0x00)
1735                 return (UNHANDLED);
1736
1737         regval = vmx_get_guest_reg(vmx, vcpu, (exitqual >> 8) & 0xf);
1738
1739         vmcs_write(VMCS_CR0_SHADOW, regval);
1740
1741         crval = regval | cr0_ones_mask;
1742         crval &= ~cr0_zeros_mask;
1743         vmcs_write(VMCS_GUEST_CR0, crval);
1744
1745         if (regval & CR0_PG) {
1746                 uint64_t efer, entry_ctls;
1747
1748                 /*
1749                  * If CR0.PG is 1 and EFER.LME is 1 then EFER.LMA and
1750                  * the "IA-32e mode guest" bit in VM-entry control must be
1751                  * equal.
1752                  */
1753                 efer = vmcs_read(VMCS_GUEST_IA32_EFER);
1754                 if (efer & EFER_LME) {
1755                         efer |= EFER_LMA;
1756                         vmcs_write(VMCS_GUEST_IA32_EFER, efer);
1757                         entry_ctls = vmcs_read(VMCS_ENTRY_CTLS);
1758                         entry_ctls |= VM_ENTRY_GUEST_LMA;
1759                         vmcs_write(VMCS_ENTRY_CTLS, entry_ctls);
1760                 }
1761         }
1762
1763         return (HANDLED);
1764 }
1765
1766 static int
1767 vmx_emulate_cr4_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
1768 {
1769         uint64_t crval, regval;
1770
1771         /* We only handle mov to %cr4 at this time */
1772         if ((exitqual & 0xf0) != 0x00)
1773                 return (UNHANDLED);
1774
1775         regval = vmx_get_guest_reg(vmx, vcpu, (exitqual >> 8) & 0xf);
1776
1777         vmcs_write(VMCS_CR4_SHADOW, regval);
1778
1779         crval = regval | cr4_ones_mask;
1780         crval &= ~cr4_zeros_mask;
1781         vmcs_write(VMCS_GUEST_CR4, crval);
1782
1783         return (HANDLED);
1784 }
1785
1786 static int
1787 vmx_emulate_cr8_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
1788 {
1789         struct vlapic *vlapic;
1790         uint64_t cr8;
1791         int regnum;
1792
1793         /* We only handle mov %cr8 to/from a register at this time. */
1794         if ((exitqual & 0xe0) != 0x00) {
1795                 return (UNHANDLED);
1796         }
1797
1798         vlapic = vm_lapic(vmx->vm, vcpu);
1799         regnum = (exitqual >> 8) & 0xf;
1800         if (exitqual & 0x10) {
1801                 cr8 = vlapic_get_cr8(vlapic);
1802                 vmx_set_guest_reg(vmx, vcpu, regnum, cr8);
1803         } else {
1804                 cr8 = vmx_get_guest_reg(vmx, vcpu, regnum);
1805                 vlapic_set_cr8(vlapic, cr8);
1806         }
1807
1808         return (HANDLED);
1809 }
1810
1811 /*
1812  * From section "Guest Register State" in the Intel SDM: CPL = SS.DPL
1813  */
1814 static int
1815 vmx_cpl(void)
1816 {
1817         uint32_t ssar;
1818
1819         ssar = vmcs_read(VMCS_GUEST_SS_ACCESS_RIGHTS);
1820         return ((ssar >> 5) & 0x3);
1821 }
1822
1823 static enum vm_cpu_mode
1824 vmx_cpu_mode(void)
1825 {
1826         uint32_t csar;
1827
1828         if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LMA) {
1829                 csar = vmcs_read(VMCS_GUEST_CS_ACCESS_RIGHTS);
1830                 if (csar & 0x2000)
1831                         return (CPU_MODE_64BIT);        /* CS.L = 1 */
1832                 else
1833                         return (CPU_MODE_COMPATIBILITY);
1834         } else if (vmcs_read(VMCS_GUEST_CR0) & CR0_PE) {
1835                 return (CPU_MODE_PROTECTED);
1836         } else {
1837                 return (CPU_MODE_REAL);
1838         }
1839 }
1840
1841 static enum vm_paging_mode
1842 vmx_paging_mode(void)
1843 {
1844
1845         if (!(vmcs_read(VMCS_GUEST_CR0) & CR0_PG))
1846                 return (PAGING_MODE_FLAT);
1847         if (!(vmcs_read(VMCS_GUEST_CR4) & CR4_PAE))
1848                 return (PAGING_MODE_32);
1849         if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LME)
1850                 return (PAGING_MODE_64);
1851         else
1852                 return (PAGING_MODE_PAE);
1853 }
1854
1855 static uint64_t
1856 inout_str_index(struct vmx *vmx, int vcpuid, int in)
1857 {
1858         uint64_t val;
1859         int error;
1860         enum vm_reg_name reg;
1861
1862         reg = in ? VM_REG_GUEST_RDI : VM_REG_GUEST_RSI;
1863         error = vmx_getreg(vmx, vcpuid, reg, &val);
1864         KASSERT(error == 0, ("%s: vmx_getreg error %d", __func__, error));
1865         return (val);
1866 }
1867
1868 static uint64_t
1869 inout_str_count(struct vmx *vmx, int vcpuid, int rep)
1870 {
1871         uint64_t val;
1872         int error;
1873
1874         if (rep) {
1875                 error = vmx_getreg(vmx, vcpuid, VM_REG_GUEST_RCX, &val);
1876                 KASSERT(!error, ("%s: vmx_getreg error %d", __func__, error));
1877         } else {
1878                 val = 1;
1879         }
1880         return (val);
1881 }
1882
1883 static int
1884 inout_str_addrsize(uint32_t inst_info)
1885 {
1886         uint32_t size;
1887
1888         size = (inst_info >> 7) & 0x7;
1889         switch (size) {
1890         case 0:
1891                 return (2);     /* 16 bit */
1892         case 1:
1893                 return (4);     /* 32 bit */
1894         case 2:
1895                 return (8);     /* 64 bit */
1896         default:
1897                 panic("%s: invalid size encoding %d", __func__, size);
1898         }
1899 }
1900
1901 static void
1902 inout_str_seginfo(struct vmx *vmx, int vcpuid, uint32_t inst_info, int in,
1903     struct vm_inout_str *vis)
1904 {
1905         int error, s;
1906
1907         if (in) {
1908                 vis->seg_name = VM_REG_GUEST_ES;
1909         } else {
1910                 s = (inst_info >> 15) & 0x7;
1911                 vis->seg_name = vm_segment_name(s);
1912         }
1913
1914         error = vmx_getdesc(vmx, vcpuid, vis->seg_name, &vis->seg_desc);
1915         KASSERT(error == 0, ("%s: vmx_getdesc error %d", __func__, error));
1916 }
1917
1918 static void
1919 vmx_paging_info(struct vm_guest_paging *paging)
1920 {
1921         paging->cr3 = vmcs_guest_cr3();
1922         paging->cpl = vmx_cpl();
1923         paging->cpu_mode = vmx_cpu_mode();
1924         paging->paging_mode = vmx_paging_mode();
1925 }
1926
1927 static void
1928 vmexit_inst_emul(struct vm_exit *vmexit, uint64_t gpa, uint64_t gla)
1929 {
1930         struct vm_guest_paging *paging;
1931         uint32_t csar;
1932
1933         paging = &vmexit->u.inst_emul.paging;
1934
1935         vmexit->exitcode = VM_EXITCODE_INST_EMUL;
1936         vmexit->inst_length = 0;
1937         vmexit->u.inst_emul.gpa = gpa;
1938         vmexit->u.inst_emul.gla = gla;
1939         vmx_paging_info(paging);
1940         switch (paging->cpu_mode) {
1941         case CPU_MODE_REAL:
1942                 vmexit->u.inst_emul.cs_base = vmcs_read(VMCS_GUEST_CS_BASE);
1943                 vmexit->u.inst_emul.cs_d = 0;
1944                 break;
1945         case CPU_MODE_PROTECTED:
1946         case CPU_MODE_COMPATIBILITY:
1947                 vmexit->u.inst_emul.cs_base = vmcs_read(VMCS_GUEST_CS_BASE);
1948                 csar = vmcs_read(VMCS_GUEST_CS_ACCESS_RIGHTS);
1949                 vmexit->u.inst_emul.cs_d = SEG_DESC_DEF32(csar);
1950                 break;
1951         default:
1952                 vmexit->u.inst_emul.cs_base = 0;
1953                 vmexit->u.inst_emul.cs_d = 0;
1954                 break;
1955         }
1956         vie_init(&vmexit->u.inst_emul.vie, NULL, 0);
1957 }
1958
1959 static int
1960 ept_fault_type(uint64_t ept_qual)
1961 {
1962         int fault_type;
1963
1964         if (ept_qual & EPT_VIOLATION_DATA_WRITE)
1965                 fault_type = VM_PROT_WRITE;
1966         else if (ept_qual & EPT_VIOLATION_INST_FETCH)
1967                 fault_type = VM_PROT_EXECUTE;
1968         else
1969                 fault_type= VM_PROT_READ;
1970
1971         return (fault_type);
1972 }
1973
1974 static boolean_t
1975 ept_emulation_fault(uint64_t ept_qual)
1976 {
1977         int read, write;
1978
1979         /* EPT fault on an instruction fetch doesn't make sense here */
1980         if (ept_qual & EPT_VIOLATION_INST_FETCH)
1981                 return (FALSE);
1982
1983         /* EPT fault must be a read fault or a write fault */
1984         read = ept_qual & EPT_VIOLATION_DATA_READ ? 1 : 0;
1985         write = ept_qual & EPT_VIOLATION_DATA_WRITE ? 1 : 0;
1986         if ((read | write) == 0)
1987                 return (FALSE);
1988
1989         /*
1990          * The EPT violation must have been caused by accessing a
1991          * guest-physical address that is a translation of a guest-linear
1992          * address.
1993          */
1994         if ((ept_qual & EPT_VIOLATION_GLA_VALID) == 0 ||
1995             (ept_qual & EPT_VIOLATION_XLAT_VALID) == 0) {
1996                 return (FALSE);
1997         }
1998
1999         return (TRUE);
2000 }
2001
2002 static __inline int
2003 apic_access_virtualization(struct vmx *vmx, int vcpuid)
2004 {
2005         uint32_t proc_ctls2;
2006
2007         proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
2008         return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) ? 1 : 0);
2009 }
2010
2011 static __inline int
2012 x2apic_virtualization(struct vmx *vmx, int vcpuid)
2013 {
2014         uint32_t proc_ctls2;
2015
2016         proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
2017         return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_X2APIC_MODE) ? 1 : 0);
2018 }
2019
2020 static int
2021 vmx_handle_apic_write(struct vmx *vmx, int vcpuid, struct vlapic *vlapic,
2022     uint64_t qual)
2023 {
2024         int error, handled, offset;
2025         uint32_t *apic_regs, vector;
2026         bool retu;
2027
2028         handled = HANDLED;
2029         offset = APIC_WRITE_OFFSET(qual);
2030
2031         if (!apic_access_virtualization(vmx, vcpuid)) {
2032                 /*
2033                  * In general there should not be any APIC write VM-exits
2034                  * unless APIC-access virtualization is enabled.
2035                  *
2036                  * However self-IPI virtualization can legitimately trigger
2037                  * an APIC-write VM-exit so treat it specially.
2038                  */
2039                 if (x2apic_virtualization(vmx, vcpuid) &&
2040                     offset == APIC_OFFSET_SELF_IPI) {
2041                         apic_regs = (uint32_t *)(vlapic->apic_page);
2042                         vector = apic_regs[APIC_OFFSET_SELF_IPI / 4];
2043                         vlapic_self_ipi_handler(vlapic, vector);
2044                         return (HANDLED);
2045                 } else
2046                         return (UNHANDLED);
2047         }
2048
2049         switch (offset) {
2050         case APIC_OFFSET_ID:
2051                 vlapic_id_write_handler(vlapic);
2052                 break;
2053         case APIC_OFFSET_LDR:
2054                 vlapic_ldr_write_handler(vlapic);
2055                 break;
2056         case APIC_OFFSET_DFR:
2057                 vlapic_dfr_write_handler(vlapic);
2058                 break;
2059         case APIC_OFFSET_SVR:
2060                 vlapic_svr_write_handler(vlapic);
2061                 break;
2062         case APIC_OFFSET_ESR:
2063                 vlapic_esr_write_handler(vlapic);
2064                 break;
2065         case APIC_OFFSET_ICR_LOW:
2066                 retu = false;
2067                 error = vlapic_icrlo_write_handler(vlapic, &retu);
2068                 if (error != 0 || retu)
2069                         handled = UNHANDLED;
2070                 break;
2071         case APIC_OFFSET_CMCI_LVT:
2072         case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
2073                 vlapic_lvt_write_handler(vlapic, offset);
2074                 break;
2075         case APIC_OFFSET_TIMER_ICR:
2076                 vlapic_icrtmr_write_handler(vlapic);
2077                 break;
2078         case APIC_OFFSET_TIMER_DCR:
2079                 vlapic_dcr_write_handler(vlapic);
2080                 break;
2081         default:
2082                 handled = UNHANDLED;
2083                 break;
2084         }
2085         return (handled);
2086 }
2087
2088 static bool
2089 apic_access_fault(struct vmx *vmx, int vcpuid, uint64_t gpa)
2090 {
2091
2092         if (apic_access_virtualization(vmx, vcpuid) &&
2093             (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE))
2094                 return (true);
2095         else
2096                 return (false);
2097 }
2098
2099 static int
2100 vmx_handle_apic_access(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit)
2101 {
2102         uint64_t qual;
2103         int access_type, offset, allowed;
2104
2105         if (!apic_access_virtualization(vmx, vcpuid))
2106                 return (UNHANDLED);
2107
2108         qual = vmexit->u.vmx.exit_qualification;
2109         access_type = APIC_ACCESS_TYPE(qual);
2110         offset = APIC_ACCESS_OFFSET(qual);
2111
2112         allowed = 0;
2113         if (access_type == 0) {
2114                 /*
2115                  * Read data access to the following registers is expected.
2116                  */
2117                 switch (offset) {
2118                 case APIC_OFFSET_APR:
2119                 case APIC_OFFSET_PPR:
2120                 case APIC_OFFSET_RRR:
2121                 case APIC_OFFSET_CMCI_LVT:
2122                 case APIC_OFFSET_TIMER_CCR:
2123                         allowed = 1;
2124                         break;
2125                 default:
2126                         break;
2127                 }
2128         } else if (access_type == 1) {
2129                 /*
2130                  * Write data access to the following registers is expected.
2131                  */
2132                 switch (offset) {
2133                 case APIC_OFFSET_VER:
2134                 case APIC_OFFSET_APR:
2135                 case APIC_OFFSET_PPR:
2136                 case APIC_OFFSET_RRR:
2137                 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
2138                 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
2139                 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
2140                 case APIC_OFFSET_CMCI_LVT:
2141                 case APIC_OFFSET_TIMER_CCR:
2142                         allowed = 1;
2143                         break;
2144                 default:
2145                         break;
2146                 }
2147         }
2148
2149         if (allowed) {
2150                 vmexit_inst_emul(vmexit, DEFAULT_APIC_BASE + offset,
2151                     VIE_INVALID_GLA);
2152         }
2153
2154         /*
2155          * Regardless of whether the APIC-access is allowed this handler
2156          * always returns UNHANDLED:
2157          * - if the access is allowed then it is handled by emulating the
2158          *   instruction that caused the VM-exit (outside the critical section)
2159          * - if the access is not allowed then it will be converted to an
2160          *   exitcode of VM_EXITCODE_VMX and will be dealt with in userland.
2161          */
2162         return (UNHANDLED);
2163 }
2164
2165 static enum task_switch_reason
2166 vmx_task_switch_reason(uint64_t qual)
2167 {
2168         int reason;
2169
2170         reason = (qual >> 30) & 0x3;
2171         switch (reason) {
2172         case 0:
2173                 return (TSR_CALL);
2174         case 1:
2175                 return (TSR_IRET);
2176         case 2:
2177                 return (TSR_JMP);
2178         case 3:
2179                 return (TSR_IDT_GATE);
2180         default:
2181                 panic("%s: invalid reason %d", __func__, reason);
2182         }
2183 }
2184
2185 static int
2186 emulate_wrmsr(struct vmx *vmx, int vcpuid, u_int num, uint64_t val, bool *retu)
2187 {
2188         int error;
2189
2190         if (lapic_msr(num))
2191                 error = lapic_wrmsr(vmx->vm, vcpuid, num, val, retu);
2192         else
2193                 error = vmx_wrmsr(vmx, vcpuid, num, val, retu);
2194
2195         return (error);
2196 }
2197
2198 static int
2199 emulate_rdmsr(struct vmx *vmx, int vcpuid, u_int num, bool *retu)
2200 {
2201         struct vmxctx *vmxctx;
2202         uint64_t result;
2203         uint32_t eax, edx;
2204         int error;
2205
2206         if (lapic_msr(num))
2207                 error = lapic_rdmsr(vmx->vm, vcpuid, num, &result, retu);
2208         else
2209                 error = vmx_rdmsr(vmx, vcpuid, num, &result, retu);
2210
2211         if (error == 0) {
2212                 eax = result;
2213                 vmxctx = &vmx->ctx[vcpuid];
2214                 error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RAX, eax);
2215                 KASSERT(error == 0, ("vmxctx_setreg(rax) error %d", error));
2216
2217                 edx = result >> 32;
2218                 error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RDX, edx);
2219                 KASSERT(error == 0, ("vmxctx_setreg(rdx) error %d", error));
2220         }
2221
2222         return (error);
2223 }
2224
2225 static int
2226 vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
2227 {
2228         int error, errcode, errcode_valid, handled, in;
2229         struct vmxctx *vmxctx;
2230         struct vlapic *vlapic;
2231         struct vm_inout_str *vis;
2232         struct vm_task_switch *ts;
2233         uint32_t eax, ecx, edx, idtvec_info, idtvec_err, intr_info, inst_info;
2234         uint32_t intr_type, intr_vec, reason;
2235         uint64_t exitintinfo, qual, gpa;
2236         bool retu;
2237
2238         CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_VIRTUAL_NMI) != 0);
2239         CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_NMI_EXITING) != 0);
2240
2241         handled = UNHANDLED;
2242         vmxctx = &vmx->ctx[vcpu];
2243
2244         qual = vmexit->u.vmx.exit_qualification;
2245         reason = vmexit->u.vmx.exit_reason;
2246         vmexit->exitcode = VM_EXITCODE_BOGUS;
2247
2248         vmm_stat_incr(vmx->vm, vcpu, VMEXIT_COUNT, 1);
2249         SDT_PROBE3(vmm, vmx, exit, entry, vmx, vcpu, vmexit);
2250
2251         /*
2252          * VM-entry failures during or after loading guest state.
2253          *
2254          * These VM-exits are uncommon but must be handled specially
2255          * as most VM-exit fields are not populated as usual.
2256          */
2257         if (__predict_false(reason == EXIT_REASON_MCE_DURING_ENTRY)) {
2258                 VCPU_CTR0(vmx->vm, vcpu, "Handling MCE during VM-entry");
2259                 __asm __volatile("int $18");
2260                 return (1);
2261         }
2262
2263         /*
2264          * VM exits that can be triggered during event delivery need to
2265          * be handled specially by re-injecting the event if the IDT
2266          * vectoring information field's valid bit is set.
2267          *
2268          * See "Information for VM Exits During Event Delivery" in Intel SDM
2269          * for details.
2270          */
2271         idtvec_info = vmcs_idt_vectoring_info();
2272         if (idtvec_info & VMCS_IDT_VEC_VALID) {
2273                 idtvec_info &= ~(1 << 12); /* clear undefined bit */
2274                 exitintinfo = idtvec_info;
2275                 if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) {
2276                         idtvec_err = vmcs_idt_vectoring_err();
2277                         exitintinfo |= (uint64_t)idtvec_err << 32;
2278                 }
2279                 error = vm_exit_intinfo(vmx->vm, vcpu, exitintinfo);
2280                 KASSERT(error == 0, ("%s: vm_set_intinfo error %d",
2281                     __func__, error));
2282
2283                 /*
2284                  * If 'virtual NMIs' are being used and the VM-exit
2285                  * happened while injecting an NMI during the previous
2286                  * VM-entry, then clear "blocking by NMI" in the
2287                  * Guest Interruptibility-State so the NMI can be
2288                  * reinjected on the subsequent VM-entry.
2289                  *
2290                  * However, if the NMI was being delivered through a task
2291                  * gate, then the new task must start execution with NMIs
2292                  * blocked so don't clear NMI blocking in this case.
2293                  */
2294                 intr_type = idtvec_info & VMCS_INTR_T_MASK;
2295                 if (intr_type == VMCS_INTR_T_NMI) {
2296                         if (reason != EXIT_REASON_TASK_SWITCH)
2297                                 vmx_clear_nmi_blocking(vmx, vcpu);
2298                         else
2299                                 vmx_assert_nmi_blocking(vmx, vcpu);
2300                 }
2301
2302                 /*
2303                  * Update VM-entry instruction length if the event being
2304                  * delivered was a software interrupt or software exception.
2305                  */
2306                 if (intr_type == VMCS_INTR_T_SWINTR ||
2307                     intr_type == VMCS_INTR_T_PRIV_SWEXCEPTION ||
2308                     intr_type == VMCS_INTR_T_SWEXCEPTION) {
2309                         vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length);
2310                 }
2311         }
2312
2313         switch (reason) {
2314         case EXIT_REASON_TASK_SWITCH:
2315                 ts = &vmexit->u.task_switch;
2316                 ts->tsssel = qual & 0xffff;
2317                 ts->reason = vmx_task_switch_reason(qual);
2318                 ts->ext = 0;
2319                 ts->errcode_valid = 0;
2320                 vmx_paging_info(&ts->paging);
2321                 /*
2322                  * If the task switch was due to a CALL, JMP, IRET, software
2323                  * interrupt (INT n) or software exception (INT3, INTO),
2324                  * then the saved %rip references the instruction that caused
2325                  * the task switch. The instruction length field in the VMCS
2326                  * is valid in this case.
2327                  *
2328                  * In all other cases (e.g., NMI, hardware exception) the
2329                  * saved %rip is one that would have been saved in the old TSS
2330                  * had the task switch completed normally so the instruction
2331                  * length field is not needed in this case and is explicitly
2332                  * set to 0.
2333                  */
2334                 if (ts->reason == TSR_IDT_GATE) {
2335                         KASSERT(idtvec_info & VMCS_IDT_VEC_VALID,
2336                             ("invalid idtvec_info %#x for IDT task switch",
2337                             idtvec_info));
2338                         intr_type = idtvec_info & VMCS_INTR_T_MASK;
2339                         if (intr_type != VMCS_INTR_T_SWINTR &&
2340                             intr_type != VMCS_INTR_T_SWEXCEPTION &&
2341                             intr_type != VMCS_INTR_T_PRIV_SWEXCEPTION) {
2342                                 /* Task switch triggered by external event */
2343                                 ts->ext = 1;
2344                                 vmexit->inst_length = 0;
2345                                 if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) {
2346                                         ts->errcode_valid = 1;
2347                                         ts->errcode = vmcs_idt_vectoring_err();
2348                                 }
2349                         }
2350                 }
2351                 vmexit->exitcode = VM_EXITCODE_TASK_SWITCH;
2352                 SDT_PROBE4(vmm, vmx, exit, taskswitch, vmx, vcpu, vmexit, ts);
2353                 VCPU_CTR4(vmx->vm, vcpu, "task switch reason %d, tss 0x%04x, "
2354                     "%s errcode 0x%016lx", ts->reason, ts->tsssel,
2355                     ts->ext ? "external" : "internal",
2356                     ((uint64_t)ts->errcode << 32) | ts->errcode_valid);
2357                 break;
2358         case EXIT_REASON_CR_ACCESS:
2359                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CR_ACCESS, 1);
2360                 SDT_PROBE4(vmm, vmx, exit, craccess, vmx, vcpu, vmexit, qual);
2361                 switch (qual & 0xf) {
2362                 case 0:
2363                         handled = vmx_emulate_cr0_access(vmx, vcpu, qual);
2364                         break;
2365                 case 4:
2366                         handled = vmx_emulate_cr4_access(vmx, vcpu, qual);
2367                         break;
2368                 case 8:
2369                         handled = vmx_emulate_cr8_access(vmx, vcpu, qual);
2370                         break;
2371                 }
2372                 break;
2373         case EXIT_REASON_RDMSR:
2374                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RDMSR, 1);
2375                 retu = false;
2376                 ecx = vmxctx->guest_rcx;
2377                 VCPU_CTR1(vmx->vm, vcpu, "rdmsr 0x%08x", ecx);
2378                 SDT_PROBE4(vmm, vmx, exit, rdmsr, vmx, vcpu, vmexit, ecx);
2379                 error = emulate_rdmsr(vmx, vcpu, ecx, &retu);
2380                 if (error) {
2381                         vmexit->exitcode = VM_EXITCODE_RDMSR;
2382                         vmexit->u.msr.code = ecx;
2383                 } else if (!retu) {
2384                         handled = HANDLED;
2385                 } else {
2386                         /* Return to userspace with a valid exitcode */
2387                         KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
2388                             ("emulate_rdmsr retu with bogus exitcode"));
2389                 }
2390                 break;
2391         case EXIT_REASON_WRMSR:
2392                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_WRMSR, 1);
2393                 retu = false;
2394                 eax = vmxctx->guest_rax;
2395                 ecx = vmxctx->guest_rcx;
2396                 edx = vmxctx->guest_rdx;
2397                 VCPU_CTR2(vmx->vm, vcpu, "wrmsr 0x%08x value 0x%016lx",
2398                     ecx, (uint64_t)edx << 32 | eax);
2399                 SDT_PROBE5(vmm, vmx, exit, wrmsr, vmx, vmexit, vcpu, ecx,
2400                     (uint64_t)edx << 32 | eax);
2401                 error = emulate_wrmsr(vmx, vcpu, ecx,
2402                     (uint64_t)edx << 32 | eax, &retu);
2403                 if (error) {
2404                         vmexit->exitcode = VM_EXITCODE_WRMSR;
2405                         vmexit->u.msr.code = ecx;
2406                         vmexit->u.msr.wval = (uint64_t)edx << 32 | eax;
2407                 } else if (!retu) {
2408                         handled = HANDLED;
2409                 } else {
2410                         /* Return to userspace with a valid exitcode */
2411                         KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
2412                             ("emulate_wrmsr retu with bogus exitcode"));
2413                 }
2414                 break;
2415         case EXIT_REASON_HLT:
2416                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_HLT, 1);
2417                 SDT_PROBE3(vmm, vmx, exit, halt, vmx, vcpu, vmexit);
2418                 vmexit->exitcode = VM_EXITCODE_HLT;
2419                 vmexit->u.hlt.rflags = vmcs_read(VMCS_GUEST_RFLAGS);
2420                 if (virtual_interrupt_delivery)
2421                         vmexit->u.hlt.intr_status =
2422                             vmcs_read(VMCS_GUEST_INTR_STATUS);
2423                 else
2424                         vmexit->u.hlt.intr_status = 0;
2425                 break;
2426         case EXIT_REASON_MTF:
2427                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_MTRAP, 1);
2428                 SDT_PROBE3(vmm, vmx, exit, mtrap, vmx, vcpu, vmexit);
2429                 vmexit->exitcode = VM_EXITCODE_MTRAP;
2430                 vmexit->inst_length = 0;
2431                 break;
2432         case EXIT_REASON_PAUSE:
2433                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_PAUSE, 1);
2434                 SDT_PROBE3(vmm, vmx, exit, pause, vmx, vcpu, vmexit);
2435                 vmexit->exitcode = VM_EXITCODE_PAUSE;
2436                 break;
2437         case EXIT_REASON_INTR_WINDOW:
2438                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INTR_WINDOW, 1);
2439                 SDT_PROBE3(vmm, vmx, exit, intrwindow, vmx, vcpu, vmexit);
2440                 vmx_clear_int_window_exiting(vmx, vcpu);
2441                 return (1);
2442         case EXIT_REASON_EXT_INTR:
2443                 /*
2444                  * External interrupts serve only to cause VM exits and allow
2445                  * the host interrupt handler to run.
2446                  *
2447                  * If this external interrupt triggers a virtual interrupt
2448                  * to a VM, then that state will be recorded by the
2449                  * host interrupt handler in the VM's softc. We will inject
2450                  * this virtual interrupt during the subsequent VM enter.
2451                  */
2452                 intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
2453                 SDT_PROBE4(vmm, vmx, exit, interrupt,
2454                     vmx, vcpu, vmexit, intr_info);
2455
2456                 /*
2457                  * XXX: Ignore this exit if VMCS_INTR_VALID is not set.
2458                  * This appears to be a bug in VMware Fusion?
2459                  */
2460                 if (!(intr_info & VMCS_INTR_VALID))
2461                         return (1);
2462                 KASSERT((intr_info & VMCS_INTR_VALID) != 0 &&
2463                     (intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_HWINTR,
2464                     ("VM exit interruption info invalid: %#x", intr_info));
2465                 vmx_trigger_hostintr(intr_info & 0xff);
2466
2467                 /*
2468                  * This is special. We want to treat this as an 'handled'
2469                  * VM-exit but not increment the instruction pointer.
2470                  */
2471                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXTINT, 1);
2472                 return (1);
2473         case EXIT_REASON_NMI_WINDOW:
2474                 SDT_PROBE3(vmm, vmx, exit, nmiwindow, vmx, vcpu, vmexit);
2475                 /* Exit to allow the pending virtual NMI to be injected */
2476                 if (vm_nmi_pending(vmx->vm, vcpu))
2477                         vmx_inject_nmi(vmx, vcpu);
2478                 vmx_clear_nmi_window_exiting(vmx, vcpu);
2479                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NMI_WINDOW, 1);
2480                 return (1);
2481         case EXIT_REASON_INOUT:
2482                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INOUT, 1);
2483                 vmexit->exitcode = VM_EXITCODE_INOUT;
2484                 vmexit->u.inout.bytes = (qual & 0x7) + 1;
2485                 vmexit->u.inout.in = in = (qual & 0x8) ? 1 : 0;
2486                 vmexit->u.inout.string = (qual & 0x10) ? 1 : 0;
2487                 vmexit->u.inout.rep = (qual & 0x20) ? 1 : 0;
2488                 vmexit->u.inout.port = (uint16_t)(qual >> 16);
2489                 vmexit->u.inout.eax = (uint32_t)(vmxctx->guest_rax);
2490                 if (vmexit->u.inout.string) {
2491                         inst_info = vmcs_read(VMCS_EXIT_INSTRUCTION_INFO);
2492                         vmexit->exitcode = VM_EXITCODE_INOUT_STR;
2493                         vis = &vmexit->u.inout_str;
2494                         vmx_paging_info(&vis->paging);
2495                         vis->rflags = vmcs_read(VMCS_GUEST_RFLAGS);
2496                         vis->cr0 = vmcs_read(VMCS_GUEST_CR0);
2497                         vis->index = inout_str_index(vmx, vcpu, in);
2498                         vis->count = inout_str_count(vmx, vcpu, vis->inout.rep);
2499                         vis->addrsize = inout_str_addrsize(inst_info);
2500                         inout_str_seginfo(vmx, vcpu, inst_info, in, vis);
2501                 }
2502                 SDT_PROBE3(vmm, vmx, exit, inout, vmx, vcpu, vmexit);
2503                 break;
2504         case EXIT_REASON_CPUID:
2505                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CPUID, 1);
2506                 SDT_PROBE3(vmm, vmx, exit, cpuid, vmx, vcpu, vmexit);
2507                 handled = vmx_handle_cpuid(vmx->vm, vcpu, vmxctx);
2508                 break;
2509         case EXIT_REASON_EXCEPTION:
2510                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXCEPTION, 1);
2511                 intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
2512                 KASSERT((intr_info & VMCS_INTR_VALID) != 0,
2513                     ("VM exit interruption info invalid: %#x", intr_info));
2514
2515                 intr_vec = intr_info & 0xff;
2516                 intr_type = intr_info & VMCS_INTR_T_MASK;
2517
2518                 /*
2519                  * If Virtual NMIs control is 1 and the VM-exit is due to a
2520                  * fault encountered during the execution of IRET then we must
2521                  * restore the state of "virtual-NMI blocking" before resuming
2522                  * the guest.
2523                  *
2524                  * See "Resuming Guest Software after Handling an Exception".
2525                  * See "Information for VM Exits Due to Vectored Events".
2526                  */
2527                 if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
2528                     (intr_vec != IDT_DF) &&
2529                     (intr_info & EXIT_QUAL_NMIUDTI) != 0)
2530                         vmx_restore_nmi_blocking(vmx, vcpu);
2531
2532                 /*
2533                  * The NMI has already been handled in vmx_exit_handle_nmi().
2534                  */
2535                 if (intr_type == VMCS_INTR_T_NMI)
2536                         return (1);
2537
2538                 /*
2539                  * Call the machine check handler by hand. Also don't reflect
2540                  * the machine check back into the guest.
2541                  */
2542                 if (intr_vec == IDT_MC) {
2543                         VCPU_CTR0(vmx->vm, vcpu, "Vectoring to MCE handler");
2544                         __asm __volatile("int $18");
2545                         return (1);
2546                 }
2547
2548                 if (intr_vec == IDT_PF) {
2549                         error = vmxctx_setreg(vmxctx, VM_REG_GUEST_CR2, qual);
2550                         KASSERT(error == 0, ("%s: vmxctx_setreg(cr2) error %d",
2551                             __func__, error));
2552                 }
2553
2554                 /*
2555                  * Software exceptions exhibit trap-like behavior. This in
2556                  * turn requires populating the VM-entry instruction length
2557                  * so that the %rip in the trap frame is past the INT3/INTO
2558                  * instruction.
2559                  */
2560                 if (intr_type == VMCS_INTR_T_SWEXCEPTION)
2561                         vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length);
2562
2563                 /* Reflect all other exceptions back into the guest */
2564                 errcode_valid = errcode = 0;
2565                 if (intr_info & VMCS_INTR_DEL_ERRCODE) {
2566                         errcode_valid = 1;
2567                         errcode = vmcs_read(VMCS_EXIT_INTR_ERRCODE);
2568                 }
2569                 VCPU_CTR2(vmx->vm, vcpu, "Reflecting exception %d/%#x into "
2570                     "the guest", intr_vec, errcode);
2571                 SDT_PROBE5(vmm, vmx, exit, exception,
2572                     vmx, vcpu, vmexit, intr_vec, errcode);
2573                 error = vm_inject_exception(vmx->vm, vcpu, intr_vec,
2574                     errcode_valid, errcode, 0);
2575                 KASSERT(error == 0, ("%s: vm_inject_exception error %d",
2576                     __func__, error));
2577                 return (1);
2578
2579         case EXIT_REASON_EPT_FAULT:
2580                 /*
2581                  * If 'gpa' lies within the address space allocated to
2582                  * memory then this must be a nested page fault otherwise
2583                  * this must be an instruction that accesses MMIO space.
2584                  */
2585                 gpa = vmcs_gpa();
2586                 if (vm_mem_allocated(vmx->vm, vcpu, gpa) ||
2587                     apic_access_fault(vmx, vcpu, gpa)) {
2588                         vmexit->exitcode = VM_EXITCODE_PAGING;
2589                         vmexit->inst_length = 0;
2590                         vmexit->u.paging.gpa = gpa;
2591                         vmexit->u.paging.fault_type = ept_fault_type(qual);
2592                         vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NESTED_FAULT, 1);
2593                         SDT_PROBE5(vmm, vmx, exit, nestedfault,
2594                             vmx, vcpu, vmexit, gpa, qual);
2595                 } else if (ept_emulation_fault(qual)) {
2596                         vmexit_inst_emul(vmexit, gpa, vmcs_gla());
2597                         vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INST_EMUL, 1);
2598                         SDT_PROBE4(vmm, vmx, exit, mmiofault,
2599                             vmx, vcpu, vmexit, gpa);
2600                 }
2601                 /*
2602                  * If Virtual NMIs control is 1 and the VM-exit is due to an
2603                  * EPT fault during the execution of IRET then we must restore
2604                  * the state of "virtual-NMI blocking" before resuming.
2605                  *
2606                  * See description of "NMI unblocking due to IRET" in
2607                  * "Exit Qualification for EPT Violations".
2608                  */
2609                 if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
2610                     (qual & EXIT_QUAL_NMIUDTI) != 0)
2611                         vmx_restore_nmi_blocking(vmx, vcpu);
2612                 break;
2613         case EXIT_REASON_VIRTUALIZED_EOI:
2614                 vmexit->exitcode = VM_EXITCODE_IOAPIC_EOI;
2615                 vmexit->u.ioapic_eoi.vector = qual & 0xFF;
2616                 SDT_PROBE3(vmm, vmx, exit, eoi, vmx, vcpu, vmexit);
2617                 vmexit->inst_length = 0;        /* trap-like */
2618                 break;
2619         case EXIT_REASON_APIC_ACCESS:
2620                 SDT_PROBE3(vmm, vmx, exit, apicaccess, vmx, vcpu, vmexit);
2621                 handled = vmx_handle_apic_access(vmx, vcpu, vmexit);
2622                 break;
2623         case EXIT_REASON_APIC_WRITE:
2624                 /*
2625                  * APIC-write VM exit is trap-like so the %rip is already
2626                  * pointing to the next instruction.
2627                  */
2628                 vmexit->inst_length = 0;
2629                 vlapic = vm_lapic(vmx->vm, vcpu);
2630                 SDT_PROBE4(vmm, vmx, exit, apicwrite,
2631                     vmx, vcpu, vmexit, vlapic);
2632                 handled = vmx_handle_apic_write(vmx, vcpu, vlapic, qual);
2633                 break;
2634         case EXIT_REASON_XSETBV:
2635                 SDT_PROBE3(vmm, vmx, exit, xsetbv, vmx, vcpu, vmexit);
2636                 handled = vmx_emulate_xsetbv(vmx, vcpu, vmexit);
2637                 break;
2638         case EXIT_REASON_MONITOR:
2639                 SDT_PROBE3(vmm, vmx, exit, monitor, vmx, vcpu, vmexit);
2640                 vmexit->exitcode = VM_EXITCODE_MONITOR;
2641                 break;
2642         case EXIT_REASON_MWAIT:
2643                 SDT_PROBE3(vmm, vmx, exit, mwait, vmx, vcpu, vmexit);
2644                 vmexit->exitcode = VM_EXITCODE_MWAIT;
2645                 break;
2646         case EXIT_REASON_VMCALL:
2647         case EXIT_REASON_VMCLEAR:
2648         case EXIT_REASON_VMLAUNCH:
2649         case EXIT_REASON_VMPTRLD:
2650         case EXIT_REASON_VMPTRST:
2651         case EXIT_REASON_VMREAD:
2652         case EXIT_REASON_VMRESUME:
2653         case EXIT_REASON_VMWRITE:
2654         case EXIT_REASON_VMXOFF:
2655         case EXIT_REASON_VMXON:
2656                 SDT_PROBE3(vmm, vmx, exit, vminsn, vmx, vcpu, vmexit);
2657                 vmexit->exitcode = VM_EXITCODE_VMINSN;
2658                 break;
2659         default:
2660                 SDT_PROBE4(vmm, vmx, exit, unknown,
2661                     vmx, vcpu, vmexit, reason);
2662                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_UNKNOWN, 1);
2663                 break;
2664         }
2665
2666         if (handled) {
2667                 /*
2668                  * It is possible that control is returned to userland
2669                  * even though we were able to handle the VM exit in the
2670                  * kernel.
2671                  *
2672                  * In such a case we want to make sure that the userland
2673                  * restarts guest execution at the instruction *after*
2674                  * the one we just processed. Therefore we update the
2675                  * guest rip in the VMCS and in 'vmexit'.
2676                  */
2677                 vmexit->rip += vmexit->inst_length;
2678                 vmexit->inst_length = 0;
2679                 vmcs_write(VMCS_GUEST_RIP, vmexit->rip);
2680         } else {
2681                 if (vmexit->exitcode == VM_EXITCODE_BOGUS) {
2682                         /*
2683                          * If this VM exit was not claimed by anybody then
2684                          * treat it as a generic VMX exit.
2685                          */
2686                         vmexit->exitcode = VM_EXITCODE_VMX;
2687                         vmexit->u.vmx.status = VM_SUCCESS;
2688                         vmexit->u.vmx.inst_type = 0;
2689                         vmexit->u.vmx.inst_error = 0;
2690                 } else {
2691                         /*
2692                          * The exitcode and collateral have been populated.
2693                          * The VM exit will be processed further in userland.
2694                          */
2695                 }
2696         }
2697
2698         SDT_PROBE4(vmm, vmx, exit, return,
2699             vmx, vcpu, vmexit, handled);
2700         return (handled);
2701 }
2702
2703 static __inline void
2704 vmx_exit_inst_error(struct vmxctx *vmxctx, int rc, struct vm_exit *vmexit)
2705 {
2706
2707         KASSERT(vmxctx->inst_fail_status != VM_SUCCESS,
2708             ("vmx_exit_inst_error: invalid inst_fail_status %d",
2709             vmxctx->inst_fail_status));
2710
2711         vmexit->inst_length = 0;
2712         vmexit->exitcode = VM_EXITCODE_VMX;
2713         vmexit->u.vmx.status = vmxctx->inst_fail_status;
2714         vmexit->u.vmx.inst_error = vmcs_instruction_error();
2715         vmexit->u.vmx.exit_reason = ~0;
2716         vmexit->u.vmx.exit_qualification = ~0;
2717
2718         switch (rc) {
2719         case VMX_VMRESUME_ERROR:
2720         case VMX_VMLAUNCH_ERROR:
2721         case VMX_INVEPT_ERROR:
2722                 vmexit->u.vmx.inst_type = rc;
2723                 break;
2724         default:
2725                 panic("vm_exit_inst_error: vmx_enter_guest returned %d", rc);
2726         }
2727 }
2728
2729 /*
2730  * If the NMI-exiting VM execution control is set to '1' then an NMI in
2731  * non-root operation causes a VM-exit. NMI blocking is in effect so it is
2732  * sufficient to simply vector to the NMI handler via a software interrupt.
2733  * However, this must be done before maskable interrupts are enabled
2734  * otherwise the "iret" issued by an interrupt handler will incorrectly
2735  * clear NMI blocking.
2736  */
2737 static __inline void
2738 vmx_exit_handle_nmi(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit)
2739 {
2740         uint32_t intr_info;
2741
2742         KASSERT((read_rflags() & PSL_I) == 0, ("interrupts enabled"));
2743
2744         if (vmexit->u.vmx.exit_reason != EXIT_REASON_EXCEPTION)
2745                 return;
2746
2747         intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
2748         KASSERT((intr_info & VMCS_INTR_VALID) != 0,
2749             ("VM exit interruption info invalid: %#x", intr_info));
2750
2751         if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI) {
2752                 KASSERT((intr_info & 0xff) == IDT_NMI, ("VM exit due "
2753                     "to NMI has invalid vector: %#x", intr_info));
2754                 VCPU_CTR0(vmx->vm, vcpuid, "Vectoring to NMI handler");
2755                 __asm __volatile("int $2");
2756         }
2757 }
2758
2759 static __inline void
2760 vmx_dr_enter_guest(struct vmxctx *vmxctx)
2761 {
2762         register_t rflags;
2763
2764         /* Save host control debug registers. */
2765         vmxctx->host_dr7 = rdr7();
2766         vmxctx->host_debugctl = rdmsr(MSR_DEBUGCTLMSR);
2767
2768         /*
2769          * Disable debugging in DR7 and DEBUGCTL to avoid triggering
2770          * exceptions in the host based on the guest DRx values.  The
2771          * guest DR7 and DEBUGCTL are saved/restored in the VMCS.
2772          */
2773         load_dr7(0);
2774         wrmsr(MSR_DEBUGCTLMSR, 0);
2775
2776         /*
2777          * Disable single stepping the kernel to avoid corrupting the
2778          * guest DR6.  A debugger might still be able to corrupt the
2779          * guest DR6 by setting a breakpoint after this point and then
2780          * single stepping.
2781          */
2782         rflags = read_rflags();
2783         vmxctx->host_tf = rflags & PSL_T;
2784         write_rflags(rflags & ~PSL_T);
2785
2786         /* Save host debug registers. */
2787         vmxctx->host_dr0 = rdr0();
2788         vmxctx->host_dr1 = rdr1();
2789         vmxctx->host_dr2 = rdr2();
2790         vmxctx->host_dr3 = rdr3();
2791         vmxctx->host_dr6 = rdr6();
2792
2793         /* Restore guest debug registers. */
2794         load_dr0(vmxctx->guest_dr0);
2795         load_dr1(vmxctx->guest_dr1);
2796         load_dr2(vmxctx->guest_dr2);
2797         load_dr3(vmxctx->guest_dr3);
2798         load_dr6(vmxctx->guest_dr6);
2799 }
2800
2801 static __inline void
2802 vmx_dr_leave_guest(struct vmxctx *vmxctx)
2803 {
2804
2805         /* Save guest debug registers. */
2806         vmxctx->guest_dr0 = rdr0();
2807         vmxctx->guest_dr1 = rdr1();
2808         vmxctx->guest_dr2 = rdr2();
2809         vmxctx->guest_dr3 = rdr3();
2810         vmxctx->guest_dr6 = rdr6();
2811
2812         /*
2813          * Restore host debug registers.  Restore DR7, DEBUGCTL, and
2814          * PSL_T last.
2815          */
2816         load_dr0(vmxctx->host_dr0);
2817         load_dr1(vmxctx->host_dr1);
2818         load_dr2(vmxctx->host_dr2);
2819         load_dr3(vmxctx->host_dr3);
2820         load_dr6(vmxctx->host_dr6);
2821         wrmsr(MSR_DEBUGCTLMSR, vmxctx->host_debugctl);
2822         load_dr7(vmxctx->host_dr7);
2823         write_rflags(read_rflags() | vmxctx->host_tf);
2824 }
2825
2826 static int
2827 vmx_run(void *arg, int vcpu, register_t rip, pmap_t pmap,
2828     struct vm_eventinfo *evinfo)
2829 {
2830         int rc, handled, launched;
2831         struct vmx *vmx;
2832         struct vm *vm;
2833         struct vmxctx *vmxctx;
2834         struct vmcs *vmcs;
2835         struct vm_exit *vmexit;
2836         struct vlapic *vlapic;
2837         uint32_t exit_reason;
2838         struct region_descriptor gdtr, idtr;
2839         uint16_t ldt_sel;
2840
2841         vmx = arg;
2842         vm = vmx->vm;
2843         vmcs = &vmx->vmcs[vcpu];
2844         vmxctx = &vmx->ctx[vcpu];
2845         vlapic = vm_lapic(vm, vcpu);
2846         vmexit = vm_exitinfo(vm, vcpu);
2847         launched = 0;
2848
2849         KASSERT(vmxctx->pmap == pmap,
2850             ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap));
2851
2852         vmx_msr_guest_enter(vmx, vcpu);
2853
2854         VMPTRLD(vmcs);
2855
2856         /*
2857          * XXX
2858          * We do this every time because we may setup the virtual machine
2859          * from a different process than the one that actually runs it.
2860          *
2861          * If the life of a virtual machine was spent entirely in the context
2862          * of a single process we could do this once in vmx_vminit().
2863          */
2864         vmcs_write(VMCS_HOST_CR3, rcr3());
2865
2866         vmcs_write(VMCS_GUEST_RIP, rip);
2867         vmx_set_pcpu_defaults(vmx, vcpu, pmap);
2868         do {
2869                 KASSERT(vmcs_guest_rip() == rip, ("%s: vmcs guest rip mismatch "
2870                     "%#lx/%#lx", __func__, vmcs_guest_rip(), rip));
2871
2872                 handled = UNHANDLED;
2873                 /*
2874                  * Interrupts are disabled from this point on until the
2875                  * guest starts executing. This is done for the following
2876                  * reasons:
2877                  *
2878                  * If an AST is asserted on this thread after the check below,
2879                  * then the IPI_AST notification will not be lost, because it
2880                  * will cause a VM exit due to external interrupt as soon as
2881                  * the guest state is loaded.
2882                  *
2883                  * A posted interrupt after 'vmx_inject_interrupts()' will
2884                  * not be "lost" because it will be held pending in the host
2885                  * APIC because interrupts are disabled. The pending interrupt
2886                  * will be recognized as soon as the guest state is loaded.
2887                  *
2888                  * The same reasoning applies to the IPI generated by
2889                  * pmap_invalidate_ept().
2890                  */
2891                 disable_intr();
2892                 vmx_inject_interrupts(vmx, vcpu, vlapic, rip);
2893
2894                 /*
2895                  * Check for vcpu suspension after injecting events because
2896                  * vmx_inject_interrupts() can suspend the vcpu due to a
2897                  * triple fault.
2898                  */
2899                 if (vcpu_suspended(evinfo)) {
2900                         enable_intr();
2901                         vm_exit_suspended(vmx->vm, vcpu, rip);
2902                         break;
2903                 }
2904
2905                 if (vcpu_rendezvous_pending(evinfo)) {
2906                         enable_intr();
2907                         vm_exit_rendezvous(vmx->vm, vcpu, rip);
2908                         break;
2909                 }
2910
2911                 if (vcpu_reqidle(evinfo)) {
2912                         enable_intr();
2913                         vm_exit_reqidle(vmx->vm, vcpu, rip);
2914                         break;
2915                 }
2916
2917                 if (vcpu_should_yield(vm, vcpu)) {
2918                         enable_intr();
2919                         vm_exit_astpending(vmx->vm, vcpu, rip);
2920                         vmx_astpending_trace(vmx, vcpu, rip);
2921                         handled = HANDLED;
2922                         break;
2923                 }
2924
2925                 /*
2926                  * VM exits restore the base address but not the
2927                  * limits of GDTR and IDTR.  The VMCS only stores the
2928                  * base address, so VM exits set the limits to 0xffff.
2929                  * Save and restore the full GDTR and IDTR to restore
2930                  * the limits.
2931                  *
2932                  * The VMCS does not save the LDTR at all, and VM
2933                  * exits clear LDTR as if a NULL selector were loaded.
2934                  * The userspace hypervisor probably doesn't use a
2935                  * LDT, but save and restore it to be safe.
2936                  */
2937                 sgdt(&gdtr);
2938                 sidt(&idtr);
2939                 ldt_sel = sldt();
2940
2941                 vmx_run_trace(vmx, vcpu);
2942                 vmx_dr_enter_guest(vmxctx);
2943                 rc = vmx_enter_guest(vmxctx, vmx, launched);
2944                 vmx_dr_leave_guest(vmxctx);
2945
2946                 bare_lgdt(&gdtr);
2947                 lidt(&idtr);
2948                 lldt(ldt_sel);
2949
2950                 /* Collect some information for VM exit processing */
2951                 vmexit->rip = rip = vmcs_guest_rip();
2952                 vmexit->inst_length = vmexit_instruction_length();
2953                 vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason();
2954                 vmexit->u.vmx.exit_qualification = vmcs_exit_qualification();
2955
2956                 /* Update 'nextrip' */
2957                 vmx->state[vcpu].nextrip = rip;
2958
2959                 if (rc == VMX_GUEST_VMEXIT) {
2960                         vmx_exit_handle_nmi(vmx, vcpu, vmexit);
2961                         enable_intr();
2962                         handled = vmx_exit_process(vmx, vcpu, vmexit);
2963                 } else {
2964                         enable_intr();
2965                         vmx_exit_inst_error(vmxctx, rc, vmexit);
2966                 }
2967                 launched = 1;
2968                 vmx_exit_trace(vmx, vcpu, rip, exit_reason, handled);
2969                 rip = vmexit->rip;
2970         } while (handled);
2971
2972         /*
2973          * If a VM exit has been handled then the exitcode must be BOGUS
2974          * If a VM exit is not handled then the exitcode must not be BOGUS
2975          */
2976         if ((handled && vmexit->exitcode != VM_EXITCODE_BOGUS) ||
2977             (!handled && vmexit->exitcode == VM_EXITCODE_BOGUS)) {
2978                 panic("Mismatch between handled (%d) and exitcode (%d)",
2979                       handled, vmexit->exitcode);
2980         }
2981
2982         if (!handled)
2983                 vmm_stat_incr(vm, vcpu, VMEXIT_USERSPACE, 1);
2984
2985         VCPU_CTR1(vm, vcpu, "returning from vmx_run: exitcode %d",
2986             vmexit->exitcode);
2987
2988         VMCLEAR(vmcs);
2989         vmx_msr_guest_exit(vmx, vcpu);
2990
2991         return (0);
2992 }
2993
2994 static void
2995 vmx_vmcleanup(void *arg)
2996 {
2997         int i;
2998         struct vmx *vmx = arg;
2999         uint16_t maxcpus;
3000
3001         if (apic_access_virtualization(vmx, 0))
3002                 vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
3003
3004         maxcpus = vm_get_maxcpus(vmx->vm);
3005         for (i = 0; i < maxcpus; i++)
3006                 vpid_free(vmx->state[i].vpid);
3007
3008         free(vmx, M_VMX);
3009
3010         return;
3011 }
3012
3013 static register_t *
3014 vmxctx_regptr(struct vmxctx *vmxctx, int reg)
3015 {
3016
3017         switch (reg) {
3018         case VM_REG_GUEST_RAX:
3019                 return (&vmxctx->guest_rax);
3020         case VM_REG_GUEST_RBX:
3021                 return (&vmxctx->guest_rbx);
3022         case VM_REG_GUEST_RCX:
3023                 return (&vmxctx->guest_rcx);
3024         case VM_REG_GUEST_RDX:
3025                 return (&vmxctx->guest_rdx);
3026         case VM_REG_GUEST_RSI:
3027                 return (&vmxctx->guest_rsi);
3028         case VM_REG_GUEST_RDI:
3029                 return (&vmxctx->guest_rdi);
3030         case VM_REG_GUEST_RBP:
3031                 return (&vmxctx->guest_rbp);
3032         case VM_REG_GUEST_R8:
3033                 return (&vmxctx->guest_r8);
3034         case VM_REG_GUEST_R9:
3035                 return (&vmxctx->guest_r9);
3036         case VM_REG_GUEST_R10:
3037                 return (&vmxctx->guest_r10);
3038         case VM_REG_GUEST_R11:
3039                 return (&vmxctx->guest_r11);
3040         case VM_REG_GUEST_R12:
3041                 return (&vmxctx->guest_r12);
3042         case VM_REG_GUEST_R13:
3043                 return (&vmxctx->guest_r13);
3044         case VM_REG_GUEST_R14:
3045                 return (&vmxctx->guest_r14);
3046         case VM_REG_GUEST_R15:
3047                 return (&vmxctx->guest_r15);
3048         case VM_REG_GUEST_CR2:
3049                 return (&vmxctx->guest_cr2);
3050         case VM_REG_GUEST_DR0:
3051                 return (&vmxctx->guest_dr0);
3052         case VM_REG_GUEST_DR1:
3053                 return (&vmxctx->guest_dr1);
3054         case VM_REG_GUEST_DR2:
3055                 return (&vmxctx->guest_dr2);
3056         case VM_REG_GUEST_DR3:
3057                 return (&vmxctx->guest_dr3);
3058         case VM_REG_GUEST_DR6:
3059                 return (&vmxctx->guest_dr6);
3060         default:
3061                 break;
3062         }
3063         return (NULL);
3064 }
3065
3066 static int
3067 vmxctx_getreg(struct vmxctx *vmxctx, int reg, uint64_t *retval)
3068 {
3069         register_t *regp;
3070
3071         if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
3072                 *retval = *regp;
3073                 return (0);
3074         } else
3075                 return (EINVAL);
3076 }
3077
3078 static int
3079 vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val)
3080 {
3081         register_t *regp;
3082
3083         if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
3084                 *regp = val;
3085                 return (0);
3086         } else
3087                 return (EINVAL);
3088 }
3089
3090 static int
3091 vmx_get_intr_shadow(struct vmx *vmx, int vcpu, int running, uint64_t *retval)
3092 {
3093         uint64_t gi;
3094         int error;
3095
3096         error = vmcs_getreg(&vmx->vmcs[vcpu], running,
3097             VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY), &gi);
3098         *retval = (gi & HWINTR_BLOCKING) ? 1 : 0;
3099         return (error);
3100 }
3101
3102 static int
3103 vmx_modify_intr_shadow(struct vmx *vmx, int vcpu, int running, uint64_t val)
3104 {
3105         struct vmcs *vmcs;
3106         uint64_t gi;
3107         int error, ident;
3108
3109         /*
3110          * Forcing the vcpu into an interrupt shadow is not supported.
3111          */
3112         if (val) {
3113                 error = EINVAL;
3114                 goto done;
3115         }
3116
3117         vmcs = &vmx->vmcs[vcpu];
3118         ident = VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY);
3119         error = vmcs_getreg(vmcs, running, ident, &gi);
3120         if (error == 0) {
3121                 gi &= ~HWINTR_BLOCKING;
3122                 error = vmcs_setreg(vmcs, running, ident, gi);
3123         }
3124 done:
3125         VCPU_CTR2(vmx->vm, vcpu, "Setting intr_shadow to %#lx %s", val,
3126             error ? "failed" : "succeeded");
3127         return (error);
3128 }
3129
3130 static int
3131 vmx_shadow_reg(int reg)
3132 {
3133         int shreg;
3134
3135         shreg = -1;
3136
3137         switch (reg) {
3138         case VM_REG_GUEST_CR0:
3139                 shreg = VMCS_CR0_SHADOW;
3140                 break;
3141         case VM_REG_GUEST_CR4:
3142                 shreg = VMCS_CR4_SHADOW;
3143                 break;
3144         default:
3145                 break;
3146         }
3147
3148         return (shreg);
3149 }
3150
3151 static int
3152 vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval)
3153 {
3154         int running, hostcpu;
3155         struct vmx *vmx = arg;
3156
3157         running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
3158         if (running && hostcpu != curcpu)
3159                 panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), vcpu);
3160
3161         if (reg == VM_REG_GUEST_INTR_SHADOW)
3162                 return (vmx_get_intr_shadow(vmx, vcpu, running, retval));
3163
3164         if (vmxctx_getreg(&vmx->ctx[vcpu], reg, retval) == 0)
3165                 return (0);
3166
3167         return (vmcs_getreg(&vmx->vmcs[vcpu], running, reg, retval));
3168 }
3169
3170 static int
3171 vmx_setreg(void *arg, int vcpu, int reg, uint64_t val)
3172 {
3173         int error, hostcpu, running, shadow;
3174         uint64_t ctls;
3175         pmap_t pmap;
3176         struct vmx *vmx = arg;
3177
3178         running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
3179         if (running && hostcpu != curcpu)
3180                 panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), vcpu);
3181
3182         if (reg == VM_REG_GUEST_INTR_SHADOW)
3183                 return (vmx_modify_intr_shadow(vmx, vcpu, running, val));
3184
3185         if (vmxctx_setreg(&vmx->ctx[vcpu], reg, val) == 0)
3186                 return (0);
3187
3188         /* Do not permit user write access to VMCS fields by offset. */
3189         if (reg < 0)
3190                 return (EINVAL);
3191
3192         error = vmcs_setreg(&vmx->vmcs[vcpu], running, reg, val);
3193
3194         if (error == 0) {
3195                 /*
3196                  * If the "load EFER" VM-entry control is 1 then the
3197                  * value of EFER.LMA must be identical to "IA-32e mode guest"
3198                  * bit in the VM-entry control.
3199                  */
3200                 if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 &&
3201                     (reg == VM_REG_GUEST_EFER)) {
3202                         vmcs_getreg(&vmx->vmcs[vcpu], running,
3203                                     VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls);
3204                         if (val & EFER_LMA)
3205                                 ctls |= VM_ENTRY_GUEST_LMA;
3206                         else
3207                                 ctls &= ~VM_ENTRY_GUEST_LMA;
3208                         vmcs_setreg(&vmx->vmcs[vcpu], running,
3209                                     VMCS_IDENT(VMCS_ENTRY_CTLS), ctls);
3210                 }
3211
3212                 shadow = vmx_shadow_reg(reg);
3213                 if (shadow > 0) {
3214                         /*
3215                          * Store the unmodified value in the shadow
3216                          */
3217                         error = vmcs_setreg(&vmx->vmcs[vcpu], running,
3218                                     VMCS_IDENT(shadow), val);
3219                 }
3220
3221                 if (reg == VM_REG_GUEST_CR3) {
3222                         /*
3223                          * Invalidate the guest vcpu's TLB mappings to emulate
3224                          * the behavior of updating %cr3.
3225                          *
3226                          * XXX the processor retains global mappings when %cr3
3227                          * is updated but vmx_invvpid() does not.
3228                          */
3229                         pmap = vmx->ctx[vcpu].pmap;
3230                         vmx_invvpid(vmx, vcpu, pmap, running);
3231                 }
3232         }
3233
3234         return (error);
3235 }
3236
3237 static int
3238 vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
3239 {
3240         int hostcpu, running;
3241         struct vmx *vmx = arg;
3242
3243         running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
3244         if (running && hostcpu != curcpu)
3245                 panic("vmx_getdesc: %s%d is running", vm_name(vmx->vm), vcpu);
3246
3247         return (vmcs_getdesc(&vmx->vmcs[vcpu], running, reg, desc));
3248 }
3249
3250 static int
3251 vmx_setdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
3252 {
3253         int hostcpu, running;
3254         struct vmx *vmx = arg;
3255
3256         running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
3257         if (running && hostcpu != curcpu)
3258                 panic("vmx_setdesc: %s%d is running", vm_name(vmx->vm), vcpu);
3259
3260         return (vmcs_setdesc(&vmx->vmcs[vcpu], running, reg, desc));
3261 }
3262
3263 static int
3264 vmx_getcap(void *arg, int vcpu, int type, int *retval)
3265 {
3266         struct vmx *vmx = arg;
3267         int vcap;
3268         int ret;
3269
3270         ret = ENOENT;
3271
3272         vcap = vmx->cap[vcpu].set;
3273
3274         switch (type) {
3275         case VM_CAP_HALT_EXIT:
3276                 if (cap_halt_exit)
3277                         ret = 0;
3278                 break;
3279         case VM_CAP_PAUSE_EXIT:
3280                 if (cap_pause_exit)
3281                         ret = 0;
3282                 break;
3283         case VM_CAP_MTRAP_EXIT:
3284                 if (cap_monitor_trap)
3285                         ret = 0;
3286                 break;
3287         case VM_CAP_UNRESTRICTED_GUEST:
3288                 if (cap_unrestricted_guest)
3289                         ret = 0;
3290                 break;
3291         case VM_CAP_ENABLE_INVPCID:
3292                 if (cap_invpcid)
3293                         ret = 0;
3294                 break;
3295         default:
3296                 break;
3297         }
3298
3299         if (ret == 0)
3300                 *retval = (vcap & (1 << type)) ? 1 : 0;
3301
3302         return (ret);
3303 }
3304
3305 static int
3306 vmx_setcap(void *arg, int vcpu, int type, int val)
3307 {
3308         struct vmx *vmx = arg;
3309         struct vmcs *vmcs = &vmx->vmcs[vcpu];
3310         uint32_t baseval;
3311         uint32_t *pptr;
3312         int error;
3313         int flag;
3314         int reg;
3315         int retval;
3316
3317         retval = ENOENT;
3318         pptr = NULL;
3319
3320         switch (type) {
3321         case VM_CAP_HALT_EXIT:
3322                 if (cap_halt_exit) {
3323                         retval = 0;
3324                         pptr = &vmx->cap[vcpu].proc_ctls;
3325                         baseval = *pptr;
3326                         flag = PROCBASED_HLT_EXITING;
3327                         reg = VMCS_PRI_PROC_BASED_CTLS;
3328                 }
3329                 break;
3330         case VM_CAP_MTRAP_EXIT:
3331                 if (cap_monitor_trap) {
3332                         retval = 0;
3333                         pptr = &vmx->cap[vcpu].proc_ctls;
3334                         baseval = *pptr;
3335                         flag = PROCBASED_MTF;
3336                         reg = VMCS_PRI_PROC_BASED_CTLS;
3337                 }
3338                 break;
3339         case VM_CAP_PAUSE_EXIT:
3340                 if (cap_pause_exit) {
3341                         retval = 0;
3342                         pptr = &vmx->cap[vcpu].proc_ctls;
3343                         baseval = *pptr;
3344                         flag = PROCBASED_PAUSE_EXITING;
3345                         reg = VMCS_PRI_PROC_BASED_CTLS;
3346                 }
3347                 break;
3348         case VM_CAP_UNRESTRICTED_GUEST:
3349                 if (cap_unrestricted_guest) {
3350                         retval = 0;
3351                         pptr = &vmx->cap[vcpu].proc_ctls2;
3352                         baseval = *pptr;
3353                         flag = PROCBASED2_UNRESTRICTED_GUEST;
3354                         reg = VMCS_SEC_PROC_BASED_CTLS;
3355                 }
3356                 break;
3357         case VM_CAP_ENABLE_INVPCID:
3358                 if (cap_invpcid) {
3359                         retval = 0;
3360                         pptr = &vmx->cap[vcpu].proc_ctls2;
3361                         baseval = *pptr;
3362                         flag = PROCBASED2_ENABLE_INVPCID;
3363                         reg = VMCS_SEC_PROC_BASED_CTLS;
3364                 }
3365                 break;
3366         default:
3367                 break;
3368         }
3369
3370         if (retval == 0) {
3371                 if (val) {
3372                         baseval |= flag;
3373                 } else {
3374                         baseval &= ~flag;
3375                 }
3376                 VMPTRLD(vmcs);
3377                 error = vmwrite(reg, baseval);
3378                 VMCLEAR(vmcs);
3379
3380                 if (error) {
3381                         retval = error;
3382                 } else {
3383                         /*
3384                          * Update optional stored flags, and record
3385                          * setting
3386                          */
3387                         if (pptr != NULL) {
3388                                 *pptr = baseval;
3389                         }
3390
3391                         if (val) {
3392                                 vmx->cap[vcpu].set |= (1 << type);
3393                         } else {
3394                                 vmx->cap[vcpu].set &= ~(1 << type);
3395                         }
3396                 }
3397         }
3398
3399         return (retval);
3400 }
3401
3402 struct vlapic_vtx {
3403         struct vlapic   vlapic;
3404         struct pir_desc *pir_desc;
3405         struct vmx      *vmx;
3406         u_int   pending_prio;
3407 };
3408
3409 #define VPR_PRIO_BIT(vpr)       (1 << ((vpr) >> 4))
3410
3411 #define VMX_CTR_PIR(vm, vcpuid, pir_desc, notify, vector, level, msg)   \
3412 do {                                                                    \
3413         VCPU_CTR2(vm, vcpuid, msg " assert %s-triggered vector %d",     \
3414             level ? "level" : "edge", vector);                          \
3415         VCPU_CTR1(vm, vcpuid, msg " pir0 0x%016lx", pir_desc->pir[0]);  \
3416         VCPU_CTR1(vm, vcpuid, msg " pir1 0x%016lx", pir_desc->pir[1]);  \
3417         VCPU_CTR1(vm, vcpuid, msg " pir2 0x%016lx", pir_desc->pir[2]);  \
3418         VCPU_CTR1(vm, vcpuid, msg " pir3 0x%016lx", pir_desc->pir[3]);  \
3419         VCPU_CTR1(vm, vcpuid, msg " notify: %s", notify ? "yes" : "no");\
3420 } while (0)
3421
3422 /*
3423  * vlapic->ops handlers that utilize the APICv hardware assist described in
3424  * Chapter 29 of the Intel SDM.
3425  */
3426 static int
3427 vmx_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
3428 {
3429         struct vlapic_vtx *vlapic_vtx;
3430         struct pir_desc *pir_desc;
3431         uint64_t mask;
3432         int idx, notify = 0;
3433
3434         vlapic_vtx = (struct vlapic_vtx *)vlapic;
3435         pir_desc = vlapic_vtx->pir_desc;
3436
3437         /*
3438          * Keep track of interrupt requests in the PIR descriptor. This is
3439          * because the virtual APIC page pointed to by the VMCS cannot be
3440          * modified if the vcpu is running.
3441          */
3442         idx = vector / 64;
3443         mask = 1UL << (vector % 64);
3444         atomic_set_long(&pir_desc->pir[idx], mask);
3445
3446         /*
3447          * A notification is required whenever the 'pending' bit makes a
3448          * transition from 0->1.
3449          *
3450          * Even if the 'pending' bit is already asserted, notification about
3451          * the incoming interrupt may still be necessary.  For example, if a
3452          * vCPU is HLTed with a high PPR, a low priority interrupt would cause
3453          * the 0->1 'pending' transition with a notification, but the vCPU
3454          * would ignore the interrupt for the time being.  The same vCPU would
3455          * need to then be notified if a high-priority interrupt arrived which
3456          * satisfied the PPR.
3457          *
3458          * The priorities of interrupts injected while 'pending' is asserted
3459          * are tracked in a custom bitfield 'pending_prio'.  Should the
3460          * to-be-injected interrupt exceed the priorities already present, the
3461          * notification is sent.  The priorities recorded in 'pending_prio' are
3462          * cleared whenever the 'pending' bit makes another 0->1 transition.
3463          */
3464         if (atomic_cmpset_long(&pir_desc->pending, 0, 1) != 0) {
3465                 notify = 1;
3466                 vlapic_vtx->pending_prio = 0;
3467         } else {
3468                 const u_int old_prio = vlapic_vtx->pending_prio;
3469                 const u_int prio_bit = VPR_PRIO_BIT(vector & APIC_TPR_INT);
3470
3471                 if ((old_prio & prio_bit) == 0 && prio_bit > old_prio) {
3472                         atomic_set_int(&vlapic_vtx->pending_prio, prio_bit);
3473                         notify = 1;
3474                 }
3475         }
3476
3477         VMX_CTR_PIR(vlapic->vm, vlapic->vcpuid, pir_desc, notify, vector,
3478             level, "vmx_set_intr_ready");
3479         return (notify);
3480 }
3481
3482 static int
3483 vmx_pending_intr(struct vlapic *vlapic, int *vecptr)
3484 {
3485         struct vlapic_vtx *vlapic_vtx;
3486         struct pir_desc *pir_desc;
3487         struct LAPIC *lapic;
3488         uint64_t pending, pirval;
3489         uint32_t ppr, vpr;
3490         int i;
3491
3492         /*
3493          * This function is only expected to be called from the 'HLT' exit
3494          * handler which does not care about the vector that is pending.
3495          */
3496         KASSERT(vecptr == NULL, ("vmx_pending_intr: vecptr must be NULL"));
3497
3498         vlapic_vtx = (struct vlapic_vtx *)vlapic;
3499         pir_desc = vlapic_vtx->pir_desc;
3500
3501         pending = atomic_load_acq_long(&pir_desc->pending);
3502         if (!pending) {
3503                 /*
3504                  * While a virtual interrupt may have already been
3505                  * processed the actual delivery maybe pending the
3506                  * interruptibility of the guest.  Recognize a pending
3507                  * interrupt by reevaluating virtual interrupts
3508                  * following Section 29.2.1 in the Intel SDM Volume 3.
3509                  */
3510                 struct vm_exit *vmexit;
3511                 uint8_t rvi, ppr;
3512
3513                 vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid);
3514                 KASSERT(vmexit->exitcode == VM_EXITCODE_HLT,
3515                     ("vmx_pending_intr: exitcode not 'HLT'"));
3516                 rvi = vmexit->u.hlt.intr_status & APIC_TPR_INT;
3517                 lapic = vlapic->apic_page;
3518                 ppr = lapic->ppr & APIC_TPR_INT;
3519                 if (rvi > ppr) {
3520                         return (1);
3521                 }
3522
3523                 return (0);
3524         }
3525
3526         /*
3527          * If there is an interrupt pending then it will be recognized only
3528          * if its priority is greater than the processor priority.
3529          *
3530          * Special case: if the processor priority is zero then any pending
3531          * interrupt will be recognized.
3532          */
3533         lapic = vlapic->apic_page;
3534         ppr = lapic->ppr & APIC_TPR_INT;
3535         if (ppr == 0)
3536                 return (1);
3537
3538         VCPU_CTR1(vlapic->vm, vlapic->vcpuid, "HLT with non-zero PPR %d",
3539             lapic->ppr);
3540
3541         vpr = 0;
3542         for (i = 3; i >= 0; i--) {
3543                 pirval = pir_desc->pir[i];
3544                 if (pirval != 0) {
3545                         vpr = (i * 64 + flsl(pirval) - 1) & APIC_TPR_INT;
3546                         break;
3547                 }
3548         }
3549
3550         /*
3551          * If the highest-priority pending interrupt falls short of the
3552          * processor priority of this vCPU, ensure that 'pending_prio' does not
3553          * have any stale bits which would preclude a higher-priority interrupt
3554          * from incurring a notification later.
3555          */
3556         if (vpr <= ppr) {
3557                 const u_int prio_bit = VPR_PRIO_BIT(vpr);
3558                 const u_int old = vlapic_vtx->pending_prio;
3559
3560                 if (old > prio_bit && (old & prio_bit) == 0) {
3561                         vlapic_vtx->pending_prio = prio_bit;
3562                 }
3563                 return (0);
3564         }
3565         return (1);
3566 }
3567
3568 static void
3569 vmx_intr_accepted(struct vlapic *vlapic, int vector)
3570 {
3571
3572         panic("vmx_intr_accepted: not expected to be called");
3573 }
3574
3575 static void
3576 vmx_set_tmr(struct vlapic *vlapic, int vector, bool level)
3577 {
3578         struct vlapic_vtx *vlapic_vtx;
3579         struct vmx *vmx;
3580         struct vmcs *vmcs;
3581         uint64_t mask, val;
3582
3583         KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
3584         KASSERT(!vcpu_is_running(vlapic->vm, vlapic->vcpuid, NULL),
3585             ("vmx_set_tmr: vcpu cannot be running"));
3586
3587         vlapic_vtx = (struct vlapic_vtx *)vlapic;
3588         vmx = vlapic_vtx->vmx;
3589         vmcs = &vmx->vmcs[vlapic->vcpuid];
3590         mask = 1UL << (vector % 64);
3591
3592         VMPTRLD(vmcs);
3593         val = vmcs_read(VMCS_EOI_EXIT(vector));
3594         if (level)
3595                 val |= mask;
3596         else
3597                 val &= ~mask;
3598         vmcs_write(VMCS_EOI_EXIT(vector), val);
3599         VMCLEAR(vmcs);
3600 }
3601
3602 static void
3603 vmx_enable_x2apic_mode(struct vlapic *vlapic)
3604 {
3605         struct vmx *vmx;
3606         struct vmcs *vmcs;
3607         uint32_t proc_ctls2;
3608         int vcpuid, error;
3609
3610         vcpuid = vlapic->vcpuid;
3611         vmx = ((struct vlapic_vtx *)vlapic)->vmx;
3612         vmcs = &vmx->vmcs[vcpuid];
3613
3614         proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
3615         KASSERT((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) != 0,
3616             ("%s: invalid proc_ctls2 %#x", __func__, proc_ctls2));
3617
3618         proc_ctls2 &= ~PROCBASED2_VIRTUALIZE_APIC_ACCESSES;
3619         proc_ctls2 |= PROCBASED2_VIRTUALIZE_X2APIC_MODE;
3620         vmx->cap[vcpuid].proc_ctls2 = proc_ctls2;
3621
3622         VMPTRLD(vmcs);
3623         vmcs_write(VMCS_SEC_PROC_BASED_CTLS, proc_ctls2);
3624         VMCLEAR(vmcs);
3625
3626         if (vlapic->vcpuid == 0) {
3627                 /*
3628                  * The nested page table mappings are shared by all vcpus
3629                  * so unmap the APIC access page just once.
3630                  */
3631                 error = vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
3632                 KASSERT(error == 0, ("%s: vm_unmap_mmio error %d",
3633                     __func__, error));
3634
3635                 /*
3636                  * The MSR bitmap is shared by all vcpus so modify it only
3637                  * once in the context of vcpu 0.
3638                  */
3639                 error = vmx_allow_x2apic_msrs(vmx);
3640                 KASSERT(error == 0, ("%s: vmx_allow_x2apic_msrs error %d",
3641                     __func__, error));
3642         }
3643 }
3644
3645 static void
3646 vmx_post_intr(struct vlapic *vlapic, int hostcpu)
3647 {
3648
3649         ipi_cpu(hostcpu, pirvec);
3650 }
3651
3652 /*
3653  * Transfer the pending interrupts in the PIR descriptor to the IRR
3654  * in the virtual APIC page.
3655  */
3656 static void
3657 vmx_inject_pir(struct vlapic *vlapic)
3658 {
3659         struct vlapic_vtx *vlapic_vtx;
3660         struct pir_desc *pir_desc;
3661         struct LAPIC *lapic;
3662         uint64_t val, pirval;
3663         int rvi, pirbase = -1;
3664         uint16_t intr_status_old, intr_status_new;
3665
3666         vlapic_vtx = (struct vlapic_vtx *)vlapic;
3667         pir_desc = vlapic_vtx->pir_desc;
3668         if (atomic_cmpset_long(&pir_desc->pending, 1, 0) == 0) {
3669                 VCPU_CTR0(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: "
3670                     "no posted interrupt pending");
3671                 return;
3672         }
3673
3674         pirval = 0;
3675         pirbase = -1;
3676         lapic = vlapic->apic_page;
3677
3678         val = atomic_readandclear_long(&pir_desc->pir[0]);
3679         if (val != 0) {
3680                 lapic->irr0 |= val;
3681                 lapic->irr1 |= val >> 32;
3682                 pirbase = 0;
3683                 pirval = val;
3684         }
3685
3686         val = atomic_readandclear_long(&pir_desc->pir[1]);
3687         if (val != 0) {
3688                 lapic->irr2 |= val;
3689                 lapic->irr3 |= val >> 32;
3690                 pirbase = 64;
3691                 pirval = val;
3692         }
3693
3694         val = atomic_readandclear_long(&pir_desc->pir[2]);
3695         if (val != 0) {
3696                 lapic->irr4 |= val;
3697                 lapic->irr5 |= val >> 32;
3698                 pirbase = 128;
3699                 pirval = val;
3700         }
3701
3702         val = atomic_readandclear_long(&pir_desc->pir[3]);
3703         if (val != 0) {
3704                 lapic->irr6 |= val;
3705                 lapic->irr7 |= val >> 32;
3706                 pirbase = 192;
3707                 pirval = val;
3708         }
3709
3710         VLAPIC_CTR_IRR(vlapic, "vmx_inject_pir");
3711
3712         /*
3713          * Update RVI so the processor can evaluate pending virtual
3714          * interrupts on VM-entry.
3715          *
3716          * It is possible for pirval to be 0 here, even though the
3717          * pending bit has been set. The scenario is:
3718          * CPU-Y is sending a posted interrupt to CPU-X, which
3719          * is running a guest and processing posted interrupts in h/w.
3720          * CPU-X will eventually exit and the state seen in s/w is
3721          * the pending bit set, but no PIR bits set.
3722          *
3723          *      CPU-X                      CPU-Y
3724          *   (vm running)                (host running)
3725          *   rx posted interrupt
3726          *   CLEAR pending bit
3727          *                               SET PIR bit
3728          *   READ/CLEAR PIR bits
3729          *                               SET pending bit
3730          *   (vm exit)
3731          *   pending bit set, PIR 0
3732          */
3733         if (pirval != 0) {
3734                 rvi = pirbase + flsl(pirval) - 1;
3735                 intr_status_old = vmcs_read(VMCS_GUEST_INTR_STATUS);
3736                 intr_status_new = (intr_status_old & 0xFF00) | rvi;
3737                 if (intr_status_new > intr_status_old) {
3738                         vmcs_write(VMCS_GUEST_INTR_STATUS, intr_status_new);
3739                         VCPU_CTR2(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: "
3740                             "guest_intr_status changed from 0x%04x to 0x%04x",
3741                             intr_status_old, intr_status_new);
3742                 }
3743         }
3744 }
3745
3746 static struct vlapic *
3747 vmx_vlapic_init(void *arg, int vcpuid)
3748 {
3749         struct vmx *vmx;
3750         struct vlapic *vlapic;
3751         struct vlapic_vtx *vlapic_vtx;
3752
3753         vmx = arg;
3754
3755         vlapic = malloc(sizeof(struct vlapic_vtx), M_VLAPIC, M_WAITOK | M_ZERO);
3756         vlapic->vm = vmx->vm;
3757         vlapic->vcpuid = vcpuid;
3758         vlapic->apic_page = (struct LAPIC *)&vmx->apic_page[vcpuid];
3759
3760         vlapic_vtx = (struct vlapic_vtx *)vlapic;
3761         vlapic_vtx->pir_desc = &vmx->pir_desc[vcpuid];
3762         vlapic_vtx->vmx = vmx;
3763
3764         if (virtual_interrupt_delivery) {
3765                 vlapic->ops.set_intr_ready = vmx_set_intr_ready;
3766                 vlapic->ops.pending_intr = vmx_pending_intr;
3767                 vlapic->ops.intr_accepted = vmx_intr_accepted;
3768                 vlapic->ops.set_tmr = vmx_set_tmr;
3769                 vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode;
3770         }
3771
3772         if (posted_interrupts)
3773                 vlapic->ops.post_intr = vmx_post_intr;
3774
3775         vlapic_init(vlapic);
3776
3777         return (vlapic);
3778 }
3779
3780 static void
3781 vmx_vlapic_cleanup(void *arg, struct vlapic *vlapic)
3782 {
3783
3784         vlapic_cleanup(vlapic);
3785         free(vlapic, M_VLAPIC);
3786 }
3787
3788 struct vmm_ops vmm_ops_intel = {
3789         vmx_init,
3790         vmx_cleanup,
3791         vmx_restore,
3792         vmx_vminit,
3793         vmx_run,
3794         vmx_vmcleanup,
3795         vmx_getreg,
3796         vmx_setreg,
3797         vmx_getdesc,
3798         vmx_setdesc,
3799         vmx_getcap,
3800         vmx_setcap,
3801         ept_vmspace_alloc,
3802         ept_vmspace_free,
3803         vmx_vlapic_init,
3804         vmx_vlapic_cleanup,
3805 };