sys/amd64/vmm/intel/vmx.c

   1 /*-
   2  * Copyright (c) 2011 NetApp, Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  *
  26  * $FreeBSD$
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include <sys/param.h>
  33 #include <sys/systm.h>
  34 #include <sys/smp.h>
  35 #include <sys/kernel.h>
  36 #include <sys/malloc.h>
  37 #include <sys/pcpu.h>
  38 #include <sys/proc.h>
  39 #include <sys/sysctl.h>
  40
  41 #include <vm/vm.h>
  42 #include <vm/pmap.h>
  43
  44 #include <machine/psl.h>
  45 #include <machine/cpufunc.h>
  46 #include <machine/md_var.h>
  47 #include <machine/pmap.h>
  48 #include <machine/segments.h>
  49 #include <machine/specialreg.h>
  50 #include <machine/vmparam.h>
  51
  52 #include <machine/vmm.h>
  53 #include "vmm_host.h"
  54 #include "vmm_lapic.h"
  55 #include "vmm_msr.h"
  56 #include "vmm_ktr.h"
  57 #include "vmm_stat.h"
  58
  59 #include "vmx_msr.h"
  60 #include "ept.h"
  61 #include "vmx_cpufunc.h"
  62 #include "vmx.h"
  63 #include "x86.h"
  64 #include "vmx_controls.h"
  65
  66 #define PINBASED_CTLS_ONE_SETTING                                       \
  67         (PINBASED_EXTINT_EXITING        |                               \
  68          PINBASED_NMI_EXITING           |                               \
  69          PINBASED_VIRTUAL_NMI)
  70 #define PINBASED_CTLS_ZERO_SETTING      0
  71
  72 #define PROCBASED_CTLS_WINDOW_SETTING                                   \
  73         (PROCBASED_INT_WINDOW_EXITING   |                               \
  74          PROCBASED_NMI_WINDOW_EXITING)
  75
  76 #define PROCBASED_CTLS_ONE_SETTING                                      \
  77         (PROCBASED_SECONDARY_CONTROLS   |                               \
  78          PROCBASED_IO_EXITING           |                               \
  79          PROCBASED_MSR_BITMAPS          |                               \
  80          PROCBASED_CTLS_WINDOW_SETTING)
  81 #define PROCBASED_CTLS_ZERO_SETTING     \
  82         (PROCBASED_CR3_LOAD_EXITING |   \
  83         PROCBASED_CR3_STORE_EXITING |   \
  84         PROCBASED_IO_BITMAPS)
  85
  86 #define PROCBASED_CTLS2_ONE_SETTING     PROCBASED2_ENABLE_EPT
  87 #define PROCBASED_CTLS2_ZERO_SETTING    0
  88
  89 #define VM_EXIT_CTLS_ONE_SETTING_NO_PAT                                 \
  90         (VM_EXIT_HOST_LMA                       |                       \
  91         VM_EXIT_SAVE_EFER                       |                       \
  92         VM_EXIT_LOAD_EFER)
  93
  94 #define VM_EXIT_CTLS_ONE_SETTING                                        \
  95         (VM_EXIT_CTLS_ONE_SETTING_NO_PAT        |                       \
  96         VM_EXIT_SAVE_PAT                        |                       \
  97         VM_EXIT_LOAD_PAT)
  98 #define VM_EXIT_CTLS_ZERO_SETTING       VM_EXIT_SAVE_DEBUG_CONTROLS
  99
 100 #define VM_ENTRY_CTLS_ONE_SETTING_NO_PAT        VM_ENTRY_LOAD_EFER
 101
 102 #define VM_ENTRY_CTLS_ONE_SETTING                                       \
 103         (VM_ENTRY_CTLS_ONE_SETTING_NO_PAT       |                       \
 104         VM_ENTRY_LOAD_PAT)
 105 #define VM_ENTRY_CTLS_ZERO_SETTING                                      \
 106         (VM_ENTRY_LOAD_DEBUG_CONTROLS           |                       \
 107         VM_ENTRY_INTO_SMM                       |                       \
 108         VM_ENTRY_DEACTIVATE_DUAL_MONITOR)
 109
 110 #define guest_msr_rw(vmx, msr) \
 111         msr_bitmap_change_access((vmx)->msr_bitmap, (msr), MSR_BITMAP_ACCESS_RW)
 112
 113 #define HANDLED         1
 114 #define UNHANDLED       0
 115
 116 MALLOC_DEFINE(M_VMX, "vmx", "vmx");
 117
 118 SYSCTL_DECL(_hw_vmm);
 119 SYSCTL_NODE(_hw_vmm, OID_AUTO, vmx, CTLFLAG_RW, NULL, NULL);
 120
 121 int vmxon_enabled[MAXCPU];
 122 static char vmxon_region[MAXCPU][PAGE_SIZE] __aligned(PAGE_SIZE);
 123
 124 static uint32_t pinbased_ctls, procbased_ctls, procbased_ctls2;
 125 static uint32_t exit_ctls, entry_ctls;
 126
 127 static uint64_t cr0_ones_mask, cr0_zeros_mask;
 128 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_ones_mask, CTLFLAG_RD,
 129              &cr0_ones_mask, 0, NULL);
 130 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_zeros_mask, CTLFLAG_RD,
 131              &cr0_zeros_mask, 0, NULL);
 132
 133 static uint64_t cr4_ones_mask, cr4_zeros_mask;
 134 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_ones_mask, CTLFLAG_RD,
 135              &cr4_ones_mask, 0, NULL);
 136 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_zeros_mask, CTLFLAG_RD,
 137              &cr4_zeros_mask, 0, NULL);
 138
 139 static int vmx_no_patmsr;
 140
 141 static int vmx_initialized;
 142 SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, initialized, CTLFLAG_RD,
 143            &vmx_initialized, 0, "Intel VMX initialized");
 144
 145 /*
 146  * Virtual NMI blocking conditions.
 147  *
 148  * Some processor implementations also require NMI to be blocked if
 149  * the STI_BLOCKING bit is set. It is possible to detect this at runtime
 150  * based on the (exit_reason,exit_qual) tuple being set to
 151  * (EXIT_REASON_INVAL_VMCS, EXIT_QUAL_NMI_WHILE_STI_BLOCKING).
 152  *
 153  * We take the easy way out and also include STI_BLOCKING as one of the
 154  * gating items for vNMI injection.
 155  */
 156 static uint64_t nmi_blocking_bits = VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING |
 157                                     VMCS_INTERRUPTIBILITY_NMI_BLOCKING |
 158                                     VMCS_INTERRUPTIBILITY_STI_BLOCKING;
 159
 160 /*
 161  * Optional capabilities
 162  */
 163 static int cap_halt_exit;
 164 static int cap_pause_exit;
 165 static int cap_unrestricted_guest;
 166 static int cap_monitor_trap;
 167 static int cap_invpcid;
 168
 169 static struct unrhdr *vpid_unr;
 170 static u_int vpid_alloc_failed;
 171 SYSCTL_UINT(_hw_vmm_vmx, OID_AUTO, vpid_alloc_failed, CTLFLAG_RD,
 172             &vpid_alloc_failed, 0, NULL);
 173
 174 #ifdef KTR
 175 static const char *
 176 exit_reason_to_str(int reason)
 177 {
 178         static char reasonbuf[32];
 179
 180         switch (reason) {
 181         case EXIT_REASON_EXCEPTION:
 182                 return "exception";
 183         case EXIT_REASON_EXT_INTR:
 184                 return "extint";
 185         case EXIT_REASON_TRIPLE_FAULT:
 186                 return "triplefault";
 187         case EXIT_REASON_INIT:
 188                 return "init";
 189         case EXIT_REASON_SIPI:
 190                 return "sipi";
 191         case EXIT_REASON_IO_SMI:
 192                 return "iosmi";
 193         case EXIT_REASON_SMI:
 194                 return "smi";
 195         case EXIT_REASON_INTR_WINDOW:
 196                 return "intrwindow";
 197         case EXIT_REASON_NMI_WINDOW:
 198                 return "nmiwindow";
 199         case EXIT_REASON_TASK_SWITCH:
 200                 return "taskswitch";
 201         case EXIT_REASON_CPUID:
 202                 return "cpuid";
 203         case EXIT_REASON_GETSEC:
 204                 return "getsec";
 205         case EXIT_REASON_HLT:
 206                 return "hlt";
 207         case EXIT_REASON_INVD:
 208                 return "invd";
 209         case EXIT_REASON_INVLPG:
 210                 return "invlpg";
 211         case EXIT_REASON_RDPMC:
 212                 return "rdpmc";
 213         case EXIT_REASON_RDTSC:
 214                 return "rdtsc";
 215         case EXIT_REASON_RSM:
 216                 return "rsm";
 217         case EXIT_REASON_VMCALL:
 218                 return "vmcall";
 219         case EXIT_REASON_VMCLEAR:
 220                 return "vmclear";
 221         case EXIT_REASON_VMLAUNCH:
 222                 return "vmlaunch";
 223         case EXIT_REASON_VMPTRLD:
 224                 return "vmptrld";
 225         case EXIT_REASON_VMPTRST:
 226                 return "vmptrst";
 227         case EXIT_REASON_VMREAD:
 228                 return "vmread";
 229         case EXIT_REASON_VMRESUME:
 230                 return "vmresume";
 231         case EXIT_REASON_VMWRITE:
 232                 return "vmwrite";
 233         case EXIT_REASON_VMXOFF:
 234                 return "vmxoff";
 235         case EXIT_REASON_VMXON:
 236                 return "vmxon";
 237         case EXIT_REASON_CR_ACCESS:
 238                 return "craccess";
 239         case EXIT_REASON_DR_ACCESS:
 240                 return "draccess";
 241         case EXIT_REASON_INOUT:
 242                 return "inout";
 243         case EXIT_REASON_RDMSR:
 244                 return "rdmsr";
 245         case EXIT_REASON_WRMSR:
 246                 return "wrmsr";
 247         case EXIT_REASON_INVAL_VMCS:
 248                 return "invalvmcs";
 249         case EXIT_REASON_INVAL_MSR:
 250                 return "invalmsr";
 251         case EXIT_REASON_MWAIT:
 252                 return "mwait";
 253         case EXIT_REASON_MTF:
 254                 return "mtf";
 255         case EXIT_REASON_MONITOR:
 256                 return "monitor";
 257         case EXIT_REASON_PAUSE:
 258                 return "pause";
 259         case EXIT_REASON_MCE:
 260                 return "mce";
 261         case EXIT_REASON_TPR:
 262                 return "tpr";
 263         case EXIT_REASON_APIC:
 264                 return "apic";
 265         case EXIT_REASON_GDTR_IDTR:
 266                 return "gdtridtr";
 267         case EXIT_REASON_LDTR_TR:
 268                 return "ldtrtr";
 269         case EXIT_REASON_EPT_FAULT:
 270                 return "eptfault";
 271         case EXIT_REASON_EPT_MISCONFIG:
 272                 return "eptmisconfig";
 273         case EXIT_REASON_INVEPT:
 274                 return "invept";
 275         case EXIT_REASON_RDTSCP:
 276                 return "rdtscp";
 277         case EXIT_REASON_VMX_PREEMPT:
 278                 return "vmxpreempt";
 279         case EXIT_REASON_INVVPID:
 280                 return "invvpid";
 281         case EXIT_REASON_WBINVD:
 282                 return "wbinvd";
 283         case EXIT_REASON_XSETBV:
 284                 return "xsetbv";
 285         default:
 286                 snprintf(reasonbuf, sizeof(reasonbuf), "%d", reason);
 287                 return (reasonbuf);
 288         }
 289 }
 290
 291 #ifdef SETJMP_TRACE
 292 static const char *
 293 vmx_setjmp_rc2str(int rc)
 294 {
 295         switch (rc) {
 296         case VMX_RETURN_DIRECT:
 297                 return "direct";
 298         case VMX_RETURN_LONGJMP:
 299                 return "longjmp";
 300         case VMX_RETURN_VMRESUME:
 301                 return "vmresume";
 302         case VMX_RETURN_VMLAUNCH:
 303                 return "vmlaunch";
 304         case VMX_RETURN_AST:
 305                 return "ast";
 306         default:
 307                 return "unknown";
 308         }
 309 }
 310
 311 #define SETJMP_TRACE(vmx, vcpu, vmxctx, regname)                          \
 312         VMM_CTR1((vmx)->vm, (vcpu), "setjmp trace " #regname " 0x%016lx", \
 313                  (vmxctx)->regname)
 314
 315 static void
 316 vmx_setjmp_trace(struct vmx *vmx, int vcpu, struct vmxctx *vmxctx, int rc)
 317 {
 318         uint64_t host_rip, host_rsp;
 319
 320         if (vmxctx != &vmx->ctx[vcpu])
 321                 panic("vmx_setjmp_trace: invalid vmxctx %p; should be %p",
 322                         vmxctx, &vmx->ctx[vcpu]);
 323
 324         VMM_CTR1((vmx)->vm, (vcpu), "vmxctx = %p", vmxctx);
 325         VMM_CTR2((vmx)->vm, (vcpu), "setjmp return code %s(%d)",
 326                  vmx_setjmp_rc2str(rc), rc);
 327
 328         host_rsp = host_rip = ~0;
 329         vmread(VMCS_HOST_RIP, &host_rip);
 330         vmread(VMCS_HOST_RSP, &host_rsp);
 331         VMM_CTR2((vmx)->vm, (vcpu), "vmcs host_rip 0x%016lx, host_rsp 0x%016lx",
 332                  host_rip, host_rsp);
 333
 334         SETJMP_TRACE(vmx, vcpu, vmxctx, host_r15);
 335         SETJMP_TRACE(vmx, vcpu, vmxctx, host_r14);
 336         SETJMP_TRACE(vmx, vcpu, vmxctx, host_r13);
 337         SETJMP_TRACE(vmx, vcpu, vmxctx, host_r12);
 338         SETJMP_TRACE(vmx, vcpu, vmxctx, host_rbp);
 339         SETJMP_TRACE(vmx, vcpu, vmxctx, host_rsp);
 340         SETJMP_TRACE(vmx, vcpu, vmxctx, host_rbx);
 341         SETJMP_TRACE(vmx, vcpu, vmxctx, host_rip);
 342
 343         SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rdi);
 344         SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rsi);
 345         SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rdx);
 346         SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rcx);
 347         SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r8);
 348         SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r9);
 349         SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rax);
 350         SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rbx);
 351         SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rbp);
 352         SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r10);
 353         SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r11);
 354         SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r12);
 355         SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r13);
 356         SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r14);
 357         SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r15);
 358         SETJMP_TRACE(vmx, vcpu, vmxctx, guest_cr2);
 359 }
 360 #endif
 361 #else
 362 static void __inline
 363 vmx_setjmp_trace(struct vmx *vmx, int vcpu, struct vmxctx *vmxctx, int rc)
 364 {
 365         return;
 366 }
 367 #endif  /* KTR */
 368
 369 u_long
 370 vmx_fix_cr0(u_long cr0)
 371 {
 372
 373         return ((cr0 | cr0_ones_mask) & ~cr0_zeros_mask);
 374 }
 375
 376 u_long
 377 vmx_fix_cr4(u_long cr4)
 378 {
 379
 380         return ((cr4 | cr4_ones_mask) & ~cr4_zeros_mask);
 381 }
 382
 383 static void
 384 vpid_free(int vpid)
 385 {
 386         if (vpid < 0 || vpid > 0xffff)
 387                 panic("vpid_free: invalid vpid %d", vpid);
 388
 389         /*
 390          * VPIDs [0,VM_MAXCPU] are special and are not allocated from
 391          * the unit number allocator.
 392          */
 393
 394         if (vpid > VM_MAXCPU)
 395                 free_unr(vpid_unr, vpid);
 396 }
 397
 398 static void
 399 vpid_alloc(uint16_t *vpid, int num)
 400 {
 401         int i, x;
 402
 403         if (num <= 0 || num > VM_MAXCPU)
 404                 panic("invalid number of vpids requested: %d", num);
 405
 406         /*
 407          * If the "enable vpid" execution control is not enabled then the
 408          * VPID is required to be 0 for all vcpus.
 409          */
 410         if ((procbased_ctls2 & PROCBASED2_ENABLE_VPID) == 0) {
 411                 for (i = 0; i < num; i++)
 412                         vpid[i] = 0;
 413                 return;
 414         }
 415
 416         /*
 417          * Allocate a unique VPID for each vcpu from the unit number allocator.
 418          */
 419         for (i = 0; i < num; i++) {
 420                 x = alloc_unr(vpid_unr);
 421                 if (x == -1)
 422                         break;
 423                 else
 424                         vpid[i] = x;
 425         }
 426
 427         if (i < num) {
 428                 atomic_add_int(&vpid_alloc_failed, 1);
 429
 430                 /*
 431                  * If the unit number allocator does not have enough unique
 432                  * VPIDs then we need to allocate from the [1,VM_MAXCPU] range.
 433                  *
 434                  * These VPIDs are not be unique across VMs but this does not
 435                  * affect correctness because the combined mappings are also
 436                  * tagged with the EP4TA which is unique for each VM.
 437                  *
 438                  * It is still sub-optimal because the invvpid will invalidate
 439                  * combined mappings for a particular VPID across all EP4TAs.
 440                  */
 441                 while (i-- > 0)
 442                         vpid_free(vpid[i]);
 443
 444                 for (i = 0; i < num; i++)
 445                         vpid[i] = i + 1;
 446         }
 447 }
 448
 449 static void
 450 vpid_init(void)
 451 {
 452         /*
 453          * VPID 0 is required when the "enable VPID" execution control is
 454          * disabled.
 455          *
 456          * VPIDs [1,VM_MAXCPU] are used as the "overflow namespace" when the
 457          * unit number allocator does not have sufficient unique VPIDs to
 458          * satisfy the allocation.
 459          *
 460          * The remaining VPIDs are managed by the unit number allocator.
 461          */
 462         vpid_unr = new_unrhdr(VM_MAXCPU + 1, 0xffff, NULL);
 463 }
 464
 465 static void
 466 msr_save_area_init(struct msr_entry *g_area, int *g_count)
 467 {
 468         int cnt;
 469
 470         static struct msr_entry guest_msrs[] = {
 471                 { MSR_KGSBASE, 0, 0 },
 472         };
 473
 474         cnt = sizeof(guest_msrs) / sizeof(guest_msrs[0]);
 475         if (cnt > GUEST_MSR_MAX_ENTRIES)
 476                 panic("guest msr save area overrun");
 477         bcopy(guest_msrs, g_area, sizeof(guest_msrs));
 478         *g_count = cnt;
 479 }
 480
 481 static void
 482 vmx_disable(void *arg __unused)
 483 {
 484         struct invvpid_desc invvpid_desc = { 0 };
 485         struct invept_desc invept_desc = { 0 };
 486
 487         if (vmxon_enabled[curcpu]) {
 488                 /*
 489                  * See sections 25.3.3.3 and 25.3.3.4 in Intel Vol 3b.
 490                  *
 491                  * VMXON or VMXOFF are not required to invalidate any TLB
 492                  * caching structures. This prevents potential retention of
 493                  * cached information in the TLB between distinct VMX episodes.
 494                  */
 495                 invvpid(INVVPID_TYPE_ALL_CONTEXTS, invvpid_desc);
 496                 invept(INVEPT_TYPE_ALL_CONTEXTS, invept_desc);
 497                 vmxoff();
 498         }
 499         load_cr4(rcr4() & ~CR4_VMXE);
 500 }
 501
 502 static int
 503 vmx_cleanup(void)
 504 {
 505
 506         if (vpid_unr != NULL) {
 507                 delete_unrhdr(vpid_unr);
 508                 vpid_unr = NULL;
 509         }
 510
 511         smp_rendezvous(NULL, vmx_disable, NULL, NULL);
 512
 513         return (0);
 514 }
 515
 516 static void
 517 vmx_enable(void *arg __unused)
 518 {
 519         int error;
 520
 521         load_cr4(rcr4() | CR4_VMXE);
 522
 523         *(uint32_t *)vmxon_region[curcpu] = vmx_revision();
 524         error = vmxon(vmxon_region[curcpu]);
 525         if (error == 0)
 526                 vmxon_enabled[curcpu] = 1;
 527 }
 528
 529 static int
 530 vmx_init(void)
 531 {
 532         int error;
 533         uint64_t fixed0, fixed1, feature_control;
 534         uint32_t tmp;
 535
 536         /* CPUID.1:ECX[bit 5] must be 1 for processor to support VMX */
 537         if (!(cpu_feature2 & CPUID2_VMX)) {
 538                 printf("vmx_init: processor does not support VMX operation\n");
 539                 return (ENXIO);
 540         }
 541
 542         /*
 543          * Verify that MSR_IA32_FEATURE_CONTROL lock and VMXON enable bits
 544          * are set (bits 0 and 2 respectively).
 545          */
 546         feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
 547         if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 0 ||
 548             (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) {
 549                 printf("vmx_init: VMX operation disabled by BIOS\n");
 550                 return (ENXIO);
 551         }
 552
 553         /* Check support for primary processor-based VM-execution controls */
 554         error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 555                                MSR_VMX_TRUE_PROCBASED_CTLS,
 556                                PROCBASED_CTLS_ONE_SETTING,
 557                                PROCBASED_CTLS_ZERO_SETTING, &procbased_ctls);
 558         if (error) {
 559                 printf("vmx_init: processor does not support desired primary "
 560                        "processor-based controls\n");
 561                 return (error);
 562         }
 563
 564         /* Clear the processor-based ctl bits that are set on demand */
 565         procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING;
 566
 567         /* Check support for secondary processor-based VM-execution controls */
 568         error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
 569                                MSR_VMX_PROCBASED_CTLS2,
 570                                PROCBASED_CTLS2_ONE_SETTING,
 571                                PROCBASED_CTLS2_ZERO_SETTING, &procbased_ctls2);
 572         if (error) {
 573                 printf("vmx_init: processor does not support desired secondary "
 574                        "processor-based controls\n");
 575                 return (error);
 576         }
 577
 578         /* Check support for VPID */
 579         error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
 580                                PROCBASED2_ENABLE_VPID, 0, &tmp);
 581         if (error == 0)
 582                 procbased_ctls2 |= PROCBASED2_ENABLE_VPID;
 583
 584         /* Check support for pin-based VM-execution controls */
 585         error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
 586                                MSR_VMX_TRUE_PINBASED_CTLS,
 587                                PINBASED_CTLS_ONE_SETTING,
 588                                PINBASED_CTLS_ZERO_SETTING, &pinbased_ctls);
 589         if (error) {
 590                 printf("vmx_init: processor does not support desired "
 591                        "pin-based controls\n");
 592                 return (error);
 593         }
 594
 595         /* Check support for VM-exit controls */
 596         error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, MSR_VMX_TRUE_EXIT_CTLS,
 597                                VM_EXIT_CTLS_ONE_SETTING,
 598                                VM_EXIT_CTLS_ZERO_SETTING,
 599                                &exit_ctls);
 600         if (error) {
 601                 /* Try again without the PAT MSR bits */
 602                 error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS,
 603                                        MSR_VMX_TRUE_EXIT_CTLS,
 604                                        VM_EXIT_CTLS_ONE_SETTING_NO_PAT,
 605                                        VM_EXIT_CTLS_ZERO_SETTING,
 606                                        &exit_ctls);
 607                 if (error) {
 608                         printf("vmx_init: processor does not support desired "
 609                                "exit controls\n");
 610                         return (error);
 611                 } else {
 612                         if (bootverbose)
 613                                 printf("vmm: PAT MSR access not supported\n");
 614                         guest_msr_valid(MSR_PAT);
 615                         vmx_no_patmsr = 1;
 616                 }
 617         }
 618
 619         /* Check support for VM-entry controls */
 620         if (!vmx_no_patmsr) {
 621                 error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS,
 622                                        MSR_VMX_TRUE_ENTRY_CTLS,
 623                                        VM_ENTRY_CTLS_ONE_SETTING,
 624                                        VM_ENTRY_CTLS_ZERO_SETTING,
 625                                        &entry_ctls);
 626         } else {
 627                 error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS,
 628                                        MSR_VMX_TRUE_ENTRY_CTLS,
 629                                        VM_ENTRY_CTLS_ONE_SETTING_NO_PAT,
 630                                        VM_ENTRY_CTLS_ZERO_SETTING,
 631                                        &entry_ctls);
 632         }
 633
 634         if (error) {
 635                 printf("vmx_init: processor does not support desired "
 636                        "entry controls\n");
 637                        return (error);
 638         }
 639
 640         /*
 641          * Check support for optional features by testing them
 642          * as individual bits
 643          */
 644         cap_halt_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 645                                         MSR_VMX_TRUE_PROCBASED_CTLS,
 646                                         PROCBASED_HLT_EXITING, 0,
 647                                         &tmp) == 0);
 648
 649         cap_monitor_trap = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 650                                         MSR_VMX_PROCBASED_CTLS,
 651                                         PROCBASED_MTF, 0,
 652                                         &tmp) == 0);
 653
 654         cap_pause_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 655                                          MSR_VMX_TRUE_PROCBASED_CTLS,
 656                                          PROCBASED_PAUSE_EXITING, 0,
 657                                          &tmp) == 0);
 658
 659         cap_unrestricted_guest = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
 660                                         MSR_VMX_PROCBASED_CTLS2,
 661                                         PROCBASED2_UNRESTRICTED_GUEST, 0,
 662                                         &tmp) == 0);
 663
 664         cap_invpcid = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
 665             MSR_VMX_PROCBASED_CTLS2, PROCBASED2_ENABLE_INVPCID, 0,
 666             &tmp) == 0);
 667
 668
 669         /* Initialize EPT */
 670         error = ept_init();
 671         if (error) {
 672                 printf("vmx_init: ept initialization failed (%d)\n", error);
 673                 return (error);
 674         }
 675
 676         /*
 677          * Stash the cr0 and cr4 bits that must be fixed to 0 or 1
 678          */
 679         fixed0 = rdmsr(MSR_VMX_CR0_FIXED0);
 680         fixed1 = rdmsr(MSR_VMX_CR0_FIXED1);
 681         cr0_ones_mask = fixed0 & fixed1;
 682         cr0_zeros_mask = ~fixed0 & ~fixed1;
 683
 684         /*
 685          * CR0_PE and CR0_PG can be set to zero in VMX non-root operation
 686          * if unrestricted guest execution is allowed.
 687          */
 688         if (cap_unrestricted_guest)
 689                 cr0_ones_mask &= ~(CR0_PG | CR0_PE);
 690
 691         /*
 692          * Do not allow the guest to set CR0_NW or CR0_CD.
 693          */
 694         cr0_zeros_mask |= (CR0_NW | CR0_CD);
 695
 696         fixed0 = rdmsr(MSR_VMX_CR4_FIXED0);
 697         fixed1 = rdmsr(MSR_VMX_CR4_FIXED1);
 698         cr4_ones_mask = fixed0 & fixed1;
 699         cr4_zeros_mask = ~fixed0 & ~fixed1;
 700
 701         vpid_init();
 702
 703         /* enable VMX operation */
 704         smp_rendezvous(NULL, vmx_enable, NULL, NULL);
 705
 706         vmx_initialized = 1;
 707
 708         return (0);
 709 }
 710
 711 static int
 712 vmx_setup_cr_shadow(int which, struct vmcs *vmcs, uint32_t initial)
 713 {
 714         int error, mask_ident, shadow_ident;
 715         uint64_t mask_value;
 716
 717         if (which != 0 && which != 4)
 718                 panic("vmx_setup_cr_shadow: unknown cr%d", which);
 719
 720         if (which == 0) {
 721                 mask_ident = VMCS_CR0_MASK;
 722                 mask_value = cr0_ones_mask | cr0_zeros_mask;
 723                 shadow_ident = VMCS_CR0_SHADOW;
 724         } else {
 725                 mask_ident = VMCS_CR4_MASK;
 726                 mask_value = cr4_ones_mask | cr4_zeros_mask;
 727                 shadow_ident = VMCS_CR4_SHADOW;
 728         }
 729
 730         error = vmcs_setreg(vmcs, 0, VMCS_IDENT(mask_ident), mask_value);
 731         if (error)
 732                 return (error);
 733
 734         error = vmcs_setreg(vmcs, 0, VMCS_IDENT(shadow_ident), initial);
 735         if (error)
 736                 return (error);
 737
 738         return (0);
 739 }
 740 #define vmx_setup_cr0_shadow(vmcs,init) vmx_setup_cr_shadow(0, (vmcs), (init))
 741 #define vmx_setup_cr4_shadow(vmcs,init) vmx_setup_cr_shadow(4, (vmcs), (init))
 742
 743 static void *
 744 vmx_vminit(struct vm *vm, pmap_t pmap)
 745 {
 746         uint16_t vpid[VM_MAXCPU];
 747         int i, error, guest_msr_count;
 748         struct vmx *vmx;
 749
 750         vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO);
 751         if ((uintptr_t)vmx & PAGE_MASK) {
 752                 panic("malloc of struct vmx not aligned on %d byte boundary",
 753                       PAGE_SIZE);
 754         }
 755         vmx->vm = vm;
 756
 757         vmx->eptp = eptp(vtophys((vm_offset_t)pmap->pm_pml4));
 758
 759         /*
 760          * Clean up EPTP-tagged guest physical and combined mappings
 761          *
 762          * VMX transitions are not required to invalidate any guest physical
 763          * mappings. So, it may be possible for stale guest physical mappings
 764          * to be present in the processor TLBs.
 765          *
 766          * Combined mappings for this EP4TA are also invalidated for all VPIDs.
 767          */
 768         ept_invalidate_mappings(vmx->eptp);
 769
 770         msr_bitmap_initialize(vmx->msr_bitmap);
 771
 772         /*
 773          * It is safe to allow direct access to MSR_GSBASE and MSR_FSBASE.
 774          * The guest FSBASE and GSBASE are saved and restored during
 775          * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are
 776          * always restored from the vmcs host state area on vm-exit.
 777          *
 778          * The SYSENTER_CS/ESP/EIP MSRs are identical to FS/GSBASE in
 779          * how they are saved/restored so can be directly accessed by the
 780          * guest.
 781          *
 782          * Guest KGSBASE is saved and restored in the guest MSR save area.
 783          * Host KGSBASE is restored before returning to userland from the pcb.
 784          * There will be a window of time when we are executing in the host
 785          * kernel context with a value of KGSBASE from the guest. This is ok
 786          * because the value of KGSBASE is inconsequential in kernel context.
 787          *
 788          * MSR_EFER is saved and restored in the guest VMCS area on a
 789          * VM exit and entry respectively. It is also restored from the
 790          * host VMCS area on a VM exit.
 791          */
 792         if (guest_msr_rw(vmx, MSR_GSBASE) ||
 793             guest_msr_rw(vmx, MSR_FSBASE) ||
 794             guest_msr_rw(vmx, MSR_SYSENTER_CS_MSR) ||
 795             guest_msr_rw(vmx, MSR_SYSENTER_ESP_MSR) ||
 796             guest_msr_rw(vmx, MSR_SYSENTER_EIP_MSR) ||
 797             guest_msr_rw(vmx, MSR_KGSBASE) ||
 798             guest_msr_rw(vmx, MSR_EFER))
 799                 panic("vmx_vminit: error setting guest msr access");
 800
 801         /*
 802          * MSR_PAT is saved and restored in the guest VMCS are on a VM exit
 803          * and entry respectively. It is also restored from the host VMCS
 804          * area on a VM exit. However, if running on a system with no
 805          * MSR_PAT save/restore support, leave access disabled so accesses
 806          * will be trapped.
 807          */
 808         if (!vmx_no_patmsr && guest_msr_rw(vmx, MSR_PAT))
 809                 panic("vmx_vminit: error setting guest pat msr access");
 810
 811         vpid_alloc(vpid, VM_MAXCPU);
 812
 813         for (i = 0; i < VM_MAXCPU; i++) {
 814                 vmx->vmcs[i].identifier = vmx_revision();
 815                 error = vmclear(&vmx->vmcs[i]);
 816                 if (error != 0) {
 817                         panic("vmx_vminit: vmclear error %d on vcpu %d\n",
 818                               error, i);
 819                 }
 820
 821                 error = vmcs_set_defaults(&vmx->vmcs[i],
 822                                           (u_long)vmx_longjmp,
 823                                           (u_long)&vmx->ctx[i],
 824                                           vmx->eptp,
 825                                           pinbased_ctls,
 826                                           procbased_ctls,
 827                                           procbased_ctls2,
 828                                           exit_ctls, entry_ctls,
 829                                           vtophys(vmx->msr_bitmap),
 830                                           vpid[i]);
 831
 832                 if (error != 0)
 833                         panic("vmx_vminit: vmcs_set_defaults error %d", error);
 834
 835                 vmx->cap[i].set = 0;
 836                 vmx->cap[i].proc_ctls = procbased_ctls;
 837                 vmx->cap[i].proc_ctls2 = procbased_ctls2;
 838
 839                 vmx->state[i].lastcpu = -1;
 840                 vmx->state[i].vpid = vpid[i];
 841
 842                 msr_save_area_init(vmx->guest_msrs[i], &guest_msr_count);
 843
 844                 error = vmcs_set_msr_save(&vmx->vmcs[i],
 845                                           vtophys(vmx->guest_msrs[i]),
 846                                           guest_msr_count);
 847                 if (error != 0)
 848                         panic("vmcs_set_msr_save error %d", error);
 849
 850                 /*
 851                  * Set up the CR0/4 shadows, and init the read shadow
 852                  * to the power-on register value from the Intel Sys Arch.
 853                  *  CR0 - 0x60000010
 854                  *  CR4 - 0
 855                  */
 856                 error = vmx_setup_cr0_shadow(&vmx->vmcs[i], 0x60000010);
 857                 if (error != 0)
 858                         panic("vmx_setup_cr0_shadow %d", error);
 859
 860                 error = vmx_setup_cr4_shadow(&vmx->vmcs[i], 0);
 861                 if (error != 0)
 862                         panic("vmx_setup_cr4_shadow %d", error);
 863
 864                 vmx->ctx[i].pmap = pmap;
 865                 vmx->ctx[i].eptp = vmx->eptp;
 866         }
 867
 868         return (vmx);
 869 }
 870
 871 static int
 872 vmx_handle_cpuid(struct vm *vm, int vcpu, struct vmxctx *vmxctx)
 873 {
 874         int handled, func;
 875
 876         func = vmxctx->guest_rax;
 877
 878         handled = x86_emulate_cpuid(vm, vcpu,
 879                                     (uint32_t*)(&vmxctx->guest_rax),
 880                                     (uint32_t*)(&vmxctx->guest_rbx),
 881                                     (uint32_t*)(&vmxctx->guest_rcx),
 882                                     (uint32_t*)(&vmxctx->guest_rdx));
 883         return (handled);
 884 }
 885
 886 static __inline void
 887 vmx_run_trace(struct vmx *vmx, int vcpu)
 888 {
 889 #ifdef KTR
 890         VMM_CTR1(vmx->vm, vcpu, "Resume execution at 0x%0lx", vmcs_guest_rip());
 891 #endif
 892 }
 893
 894 static __inline void
 895 vmx_exit_trace(struct vmx *vmx, int vcpu, uint64_t rip, uint32_t exit_reason,
 896                int handled)
 897 {
 898 #ifdef KTR
 899         VMM_CTR3(vmx->vm, vcpu, "%s %s vmexit at 0x%0lx",
 900                  handled ? "handled" : "unhandled",
 901                  exit_reason_to_str(exit_reason), rip);
 902 #endif
 903 }
 904
 905 static __inline void
 906 vmx_astpending_trace(struct vmx *vmx, int vcpu, uint64_t rip)
 907 {
 908 #ifdef KTR
 909         VMM_CTR1(vmx->vm, vcpu, "astpending vmexit at 0x%0lx", rip);
 910 #endif
 911 }
 912
 913 static int
 914 vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu)
 915 {
 916         int error, lastcpu;
 917         struct vmxstate *vmxstate;
 918         struct invvpid_desc invvpid_desc = { 0 };
 919
 920         vmxstate = &vmx->state[vcpu];
 921         lastcpu = vmxstate->lastcpu;
 922         vmxstate->lastcpu = curcpu;
 923
 924         if (lastcpu == curcpu) {
 925                 error = 0;
 926                 goto done;
 927         }
 928
 929         vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1);
 930
 931         error = vmwrite(VMCS_HOST_TR_BASE, vmm_get_host_trbase());
 932         if (error != 0)
 933                 goto done;
 934
 935         error = vmwrite(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase());
 936         if (error != 0)
 937                 goto done;
 938
 939         error = vmwrite(VMCS_HOST_GS_BASE, vmm_get_host_gsbase());
 940         if (error != 0)
 941                 goto done;
 942
 943         /*
 944          * If we are using VPIDs then invalidate all mappings tagged with 'vpid'
 945          *
 946          * We do this because this vcpu was executing on a different host
 947          * cpu when it last ran. We do not track whether it invalidated
 948          * mappings associated with its 'vpid' during that run. So we must
 949          * assume that the mappings associated with 'vpid' on 'curcpu' are
 950          * stale and invalidate them.
 951          *
 952          * Note that we incur this penalty only when the scheduler chooses to
 953          * move the thread associated with this vcpu between host cpus.
 954          *
 955          * Note also that this will invalidate mappings tagged with 'vpid'
 956          * for "all" EP4TAs.
 957          */
 958         if (vmxstate->vpid != 0) {
 959                 invvpid_desc.vpid = vmxstate->vpid;
 960                 invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc);
 961         }
 962 done:
 963         return (error);
 964 }
 965
 966 static void
 967 vm_exit_update_rip(struct vm_exit *vmexit)
 968 {
 969         int error;
 970
 971         error = vmwrite(VMCS_GUEST_RIP, vmexit->rip + vmexit->inst_length);
 972         if (error)
 973                 panic("vmx_run: error %d writing to VMCS_GUEST_RIP", error);
 974 }
 975
 976 /*
 977  * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set.
 978  */
 979 CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0);
 980
 981 static void __inline
 982 vmx_set_int_window_exiting(struct vmx *vmx, int vcpu)
 983 {
 984         int error;
 985
 986         vmx->cap[vcpu].proc_ctls |= PROCBASED_INT_WINDOW_EXITING;
 987
 988         error = vmwrite(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
 989         if (error)
 990                 panic("vmx_set_int_window_exiting: vmwrite error %d", error);
 991 }
 992
 993 static void __inline
 994 vmx_clear_int_window_exiting(struct vmx *vmx, int vcpu)
 995 {
 996         int error;
 997
 998         vmx->cap[vcpu].proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING;
 999
1000         error = vmwrite(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1001         if (error)
1002                 panic("vmx_clear_int_window_exiting: vmwrite error %d", error);
1003 }
1004
1005 static void __inline
1006 vmx_set_nmi_window_exiting(struct vmx *vmx, int vcpu)
1007 {
1008         int error;
1009
1010         vmx->cap[vcpu].proc_ctls |= PROCBASED_NMI_WINDOW_EXITING;
1011
1012         error = vmwrite(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1013         if (error)
1014                 panic("vmx_set_nmi_window_exiting: vmwrite error %d", error);
1015 }
1016
1017 static void __inline
1018 vmx_clear_nmi_window_exiting(struct vmx *vmx, int vcpu)
1019 {
1020         int error;
1021
1022         vmx->cap[vcpu].proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING;
1023
1024         error = vmwrite(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1025         if (error)
1026                 panic("vmx_clear_nmi_window_exiting: vmwrite error %d", error);
1027 }
1028
1029 static int
1030 vmx_inject_nmi(struct vmx *vmx, int vcpu)
1031 {
1032         int error;
1033         uint64_t info, interruptibility;
1034
1035         /* Bail out if no NMI requested */
1036         if (!vm_nmi_pending(vmx->vm, vcpu))
1037                 return (0);
1038
1039         error = vmread(VMCS_GUEST_INTERRUPTIBILITY, &interruptibility);
1040         if (error) {
1041                 panic("vmx_inject_nmi: vmread(interruptibility) %d",
1042                         error);
1043         }
1044         if (interruptibility & nmi_blocking_bits)
1045                 goto nmiblocked;
1046
1047         /*
1048          * Inject the virtual NMI. The vector must be the NMI IDT entry
1049          * or the VMCS entry check will fail.
1050          */
1051         info = VMCS_INTERRUPTION_INFO_NMI | VMCS_INTERRUPTION_INFO_VALID;
1052         info |= IDT_NMI;
1053
1054         error = vmwrite(VMCS_ENTRY_INTR_INFO, info);
1055         if (error)
1056                 panic("vmx_inject_nmi: vmwrite(intrinfo) %d", error);
1057
1058         VMM_CTR0(vmx->vm, vcpu, "Injecting vNMI");
1059
1060         /* Clear the request */
1061         vm_nmi_clear(vmx->vm, vcpu);
1062         return (1);
1063
1064 nmiblocked:
1065         /*
1066          * Set the NMI Window Exiting execution control so we can inject
1067          * the virtual NMI as soon as blocking condition goes away.
1068          */
1069         vmx_set_nmi_window_exiting(vmx, vcpu);
1070
1071         VMM_CTR0(vmx->vm, vcpu, "Enabling NMI window exiting");
1072         return (1);
1073 }
1074
1075 static void
1076 vmx_inject_interrupts(struct vmx *vmx, int vcpu)
1077 {
1078         int error, vector;
1079         uint64_t info, rflags, interruptibility;
1080
1081         const int HWINTR_BLOCKED = VMCS_INTERRUPTIBILITY_STI_BLOCKING |
1082                                    VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING;
1083
1084         /*
1085          * If there is already an interrupt pending then just return.
1086          *
1087          * This could happen if an interrupt was injected on a prior
1088          * VM entry but the actual entry into guest mode was aborted
1089          * because of a pending AST.
1090          */
1091         error = vmread(VMCS_ENTRY_INTR_INFO, &info);
1092         if (error)
1093                 panic("vmx_inject_interrupts: vmread(intrinfo) %d", error);
1094         if (info & VMCS_INTERRUPTION_INFO_VALID)
1095                 return;
1096
1097         /*
1098          * NMI injection has priority so deal with those first
1099          */
1100         if (vmx_inject_nmi(vmx, vcpu))
1101                 return;
1102
1103         /* Ask the local apic for a vector to inject */
1104         vector = lapic_pending_intr(vmx->vm, vcpu);
1105         if (vector < 0)
1106                 return;
1107
1108         if (vector < 32 || vector > 255)
1109                 panic("vmx_inject_interrupts: invalid vector %d\n", vector);
1110
1111         /* Check RFLAGS.IF and the interruptibility state of the guest */
1112         error = vmread(VMCS_GUEST_RFLAGS, &rflags);
1113         if (error)
1114                 panic("vmx_inject_interrupts: vmread(rflags) %d", error);
1115
1116         if ((rflags & PSL_I) == 0)
1117                 goto cantinject;
1118
1119         error = vmread(VMCS_GUEST_INTERRUPTIBILITY, &interruptibility);
1120         if (error) {
1121                 panic("vmx_inject_interrupts: vmread(interruptibility) %d",
1122                         error);
1123         }
1124         if (interruptibility & HWINTR_BLOCKED)
1125                 goto cantinject;
1126
1127         /* Inject the interrupt */
1128         info = VMCS_INTERRUPTION_INFO_HW_INTR | VMCS_INTERRUPTION_INFO_VALID;
1129         info |= vector;
1130         error = vmwrite(VMCS_ENTRY_INTR_INFO, info);
1131         if (error)
1132                 panic("vmx_inject_interrupts: vmwrite(intrinfo) %d", error);
1133
1134         /* Update the Local APIC ISR */
1135         lapic_intr_accepted(vmx->vm, vcpu, vector);
1136
1137         VMM_CTR1(vmx->vm, vcpu, "Injecting hwintr at vector %d", vector);
1138
1139         return;
1140
1141 cantinject:
1142         /*
1143          * Set the Interrupt Window Exiting execution control so we can inject
1144          * the interrupt as soon as blocking condition goes away.
1145          */
1146         vmx_set_int_window_exiting(vmx, vcpu);
1147
1148         VMM_CTR0(vmx->vm, vcpu, "Enabling interrupt window exiting");
1149 }
1150
1151 static int
1152 vmx_emulate_cr_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
1153 {
1154         int error, cr, vmcs_guest_cr, vmcs_shadow_cr;
1155         uint64_t crval, regval, ones_mask, zeros_mask;
1156         const struct vmxctx *vmxctx;
1157
1158         /* We only handle mov to %cr0 or %cr4 at this time */
1159         if ((exitqual & 0xf0) != 0x00)
1160                 return (UNHANDLED);
1161
1162         cr = exitqual & 0xf;
1163         if (cr != 0 && cr != 4)
1164                 return (UNHANDLED);
1165
1166         vmxctx = &vmx->ctx[vcpu];
1167
1168         /*
1169          * We must use vmwrite() directly here because vmcs_setreg() will
1170          * call vmclear(vmcs) as a side-effect which we certainly don't want.
1171          */
1172         switch ((exitqual >> 8) & 0xf) {
1173         case 0:
1174                 regval = vmxctx->guest_rax;
1175                 break;
1176         case 1:
1177                 regval = vmxctx->guest_rcx;
1178                 break;
1179         case 2:
1180                 regval = vmxctx->guest_rdx;
1181                 break;
1182         case 3:
1183                 regval = vmxctx->guest_rbx;
1184                 break;
1185         case 4:
1186                 error = vmread(VMCS_GUEST_RSP, &regval);
1187                 if (error) {
1188                         panic("vmx_emulate_cr_access: "
1189                               "error %d reading guest rsp", error);
1190                 }
1191                 break;
1192         case 5:
1193                 regval = vmxctx->guest_rbp;
1194                 break;
1195         case 6:
1196                 regval = vmxctx->guest_rsi;
1197                 break;
1198         case 7:
1199                 regval = vmxctx->guest_rdi;
1200                 break;
1201         case 8:
1202                 regval = vmxctx->guest_r8;
1203                 break;
1204         case 9:
1205                 regval = vmxctx->guest_r9;
1206                 break;
1207         case 10:
1208                 regval = vmxctx->guest_r10;
1209                 break;
1210         case 11:
1211                 regval = vmxctx->guest_r11;
1212                 break;
1213         case 12:
1214                 regval = vmxctx->guest_r12;
1215                 break;
1216         case 13:
1217                 regval = vmxctx->guest_r13;
1218                 break;
1219         case 14:
1220                 regval = vmxctx->guest_r14;
1221                 break;
1222         case 15:
1223                 regval = vmxctx->guest_r15;
1224                 break;
1225         }
1226
1227         if (cr == 0) {
1228                 ones_mask = cr0_ones_mask;
1229                 zeros_mask = cr0_zeros_mask;
1230                 vmcs_guest_cr = VMCS_GUEST_CR0;
1231                 vmcs_shadow_cr = VMCS_CR0_SHADOW;
1232         } else {
1233                 ones_mask = cr4_ones_mask;
1234                 zeros_mask = cr4_zeros_mask;
1235                 vmcs_guest_cr = VMCS_GUEST_CR4;
1236                 vmcs_shadow_cr = VMCS_CR4_SHADOW;
1237         }
1238
1239         error = vmwrite(vmcs_shadow_cr, regval);
1240         if (error) {
1241                 panic("vmx_emulate_cr_access: error %d writing cr%d shadow",
1242                       error, cr);
1243         }
1244
1245         crval = regval | ones_mask;
1246         crval &= ~zeros_mask;
1247         error = vmwrite(vmcs_guest_cr, crval);
1248         if (error) {
1249                 panic("vmx_emulate_cr_access: error %d writing cr%d",
1250                       error, cr);
1251         }
1252
1253         if (cr == 0 && regval & CR0_PG) {
1254                 uint64_t efer, entry_ctls;
1255
1256                 /*
1257                  * If CR0.PG is 1 and EFER.LME is 1 then EFER.LMA and
1258                  * the "IA-32e mode guest" bit in VM-entry control must be
1259                  * equal.
1260                  */
1261                 error = vmread(VMCS_GUEST_IA32_EFER, &efer);
1262                 if (error) {
1263                   panic("vmx_emulate_cr_access: error %d efer read",
1264                         error);
1265                 }
1266                 if (efer & EFER_LME) {
1267                         efer |= EFER_LMA;
1268                         error = vmwrite(VMCS_GUEST_IA32_EFER, efer);
1269                         if (error) {
1270                                 panic("vmx_emulate_cr_access: error %d"
1271                                       " efer write", error);
1272                         }
1273                         error = vmread(VMCS_ENTRY_CTLS, &entry_ctls);
1274                         if (error) {
1275                                 panic("vmx_emulate_cr_access: error %d"
1276                                       " entry ctls read", error);
1277                         }
1278                         entry_ctls |= VM_ENTRY_GUEST_LMA;
1279                         error = vmwrite(VMCS_ENTRY_CTLS, entry_ctls);
1280                         if (error) {
1281                                 panic("vmx_emulate_cr_access: error %d"
1282                                       " entry ctls write", error);
1283                         }
1284                 }
1285         }
1286
1287         return (HANDLED);
1288 }
1289
1290 static int
1291 ept_fault_type(uint64_t ept_qual)
1292 {
1293         int fault_type;
1294
1295         if (ept_qual & EPT_VIOLATION_DATA_WRITE)
1296                 fault_type = VM_PROT_WRITE;
1297         else if (ept_qual & EPT_VIOLATION_INST_FETCH)
1298                 fault_type = VM_PROT_EXECUTE;
1299         else
1300                 fault_type= VM_PROT_READ;
1301
1302         return (fault_type);
1303 }
1304
1305 static int
1306 ept_protection(uint64_t ept_qual)
1307 {
1308         int prot = 0;
1309
1310         if (ept_qual & EPT_VIOLATION_GPA_READABLE)
1311                 prot |= VM_PROT_READ;
1312         if (ept_qual & EPT_VIOLATION_GPA_WRITEABLE)
1313                 prot |= VM_PROT_WRITE;
1314         if (ept_qual & EPT_VIOLATION_GPA_EXECUTABLE)
1315                 prot |= VM_PROT_EXECUTE;
1316
1317         return (prot);
1318 }
1319
1320 static boolean_t
1321 ept_emulation_fault(uint64_t ept_qual)
1322 {
1323         int read, write;
1324
1325         /* EPT fault on an instruction fetch doesn't make sense here */
1326         if (ept_qual & EPT_VIOLATION_INST_FETCH)
1327                 return (FALSE);
1328
1329         /* EPT fault must be a read fault or a write fault */
1330         read = ept_qual & EPT_VIOLATION_DATA_READ ? 1 : 0;
1331         write = ept_qual & EPT_VIOLATION_DATA_WRITE ? 1 : 0;
1332         if ((read | write) == 0)
1333                 return (FALSE);
1334
1335         /*
1336          * The EPT violation must have been caused by accessing a
1337          * guest-physical address that is a translation of a guest-linear
1338          * address.
1339          */
1340         if ((ept_qual & EPT_VIOLATION_GLA_VALID) == 0 ||
1341             (ept_qual & EPT_VIOLATION_XLAT_VALID) == 0) {
1342                 return (FALSE);
1343         }
1344
1345         return (TRUE);
1346 }
1347
1348 static int
1349 vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
1350 {
1351         int error, handled;
1352         struct vmcs *vmcs;
1353         struct vmxctx *vmxctx;
1354         uint32_t eax, ecx, edx, idtvec_info, idtvec_err, reason;
1355         uint64_t qual, gpa;
1356
1357         handled = 0;
1358         vmcs = &vmx->vmcs[vcpu];
1359         vmxctx = &vmx->ctx[vcpu];
1360         qual = vmexit->u.vmx.exit_qualification;
1361         reason = vmexit->u.vmx.exit_reason;
1362         vmexit->exitcode = VM_EXITCODE_BOGUS;
1363
1364         vmm_stat_incr(vmx->vm, vcpu, VMEXIT_COUNT, 1);
1365
1366         /*
1367          * VM exits that could be triggered during event injection on the
1368          * previous VM entry need to be handled specially by re-injecting
1369          * the event.
1370          *
1371          * See "Information for VM Exits During Event Delivery" in Intel SDM
1372          * for details.
1373          */
1374         switch (reason) {
1375         case EXIT_REASON_EPT_FAULT:
1376         case EXIT_REASON_EPT_MISCONFIG:
1377         case EXIT_REASON_APIC:
1378         case EXIT_REASON_TASK_SWITCH:
1379         case EXIT_REASON_EXCEPTION:
1380                 idtvec_info = vmcs_idt_vectoring_info();
1381                 if (idtvec_info & VMCS_IDT_VEC_VALID) {
1382                         idtvec_info &= ~(1 << 12); /* clear undefined bit */
1383                         vmwrite(VMCS_ENTRY_INTR_INFO, idtvec_info);
1384                         if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) {
1385                                 idtvec_err = vmcs_idt_vectoring_err();
1386                                 vmwrite(VMCS_ENTRY_EXCEPTION_ERROR, idtvec_err);
1387                         }
1388                         vmwrite(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length);
1389                 }
1390         default:
1391                 break;
1392         }
1393
1394         switch (reason) {
1395         case EXIT_REASON_CR_ACCESS:
1396                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CR_ACCESS, 1);
1397                 handled = vmx_emulate_cr_access(vmx, vcpu, qual);
1398                 break;
1399         case EXIT_REASON_RDMSR:
1400                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RDMSR, 1);
1401                 ecx = vmxctx->guest_rcx;
1402                 error = emulate_rdmsr(vmx->vm, vcpu, ecx);
1403                 if (error) {
1404                         vmexit->exitcode = VM_EXITCODE_RDMSR;
1405                         vmexit->u.msr.code = ecx;
1406                 } else
1407                         handled = 1;
1408                 break;
1409         case EXIT_REASON_WRMSR:
1410                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_WRMSR, 1);
1411                 eax = vmxctx->guest_rax;
1412                 ecx = vmxctx->guest_rcx;
1413                 edx = vmxctx->guest_rdx;
1414                 error = emulate_wrmsr(vmx->vm, vcpu, ecx,
1415                                         (uint64_t)edx << 32 | eax);
1416                 if (error) {
1417                         vmexit->exitcode = VM_EXITCODE_WRMSR;
1418                         vmexit->u.msr.code = ecx;
1419                         vmexit->u.msr.wval = (uint64_t)edx << 32 | eax;
1420                 } else
1421                         handled = 1;
1422                 break;
1423         case EXIT_REASON_HLT:
1424                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_HLT, 1);
1425                 vmexit->exitcode = VM_EXITCODE_HLT;
1426                 break;
1427         case EXIT_REASON_MTF:
1428                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_MTRAP, 1);
1429                 vmexit->exitcode = VM_EXITCODE_MTRAP;
1430                 break;
1431         case EXIT_REASON_PAUSE:
1432                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_PAUSE, 1);
1433                 vmexit->exitcode = VM_EXITCODE_PAUSE;
1434                 break;
1435         case EXIT_REASON_INTR_WINDOW:
1436                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INTR_WINDOW, 1);
1437                 vmx_clear_int_window_exiting(vmx, vcpu);
1438                 VMM_CTR0(vmx->vm, vcpu, "Disabling interrupt window exiting");
1439                 return (1);
1440         case EXIT_REASON_EXT_INTR:
1441                 /*
1442                  * External interrupts serve only to cause VM exits and allow
1443                  * the host interrupt handler to run.
1444                  *
1445                  * If this external interrupt triggers a virtual interrupt
1446                  * to a VM, then that state will be recorded by the
1447                  * host interrupt handler in the VM's softc. We will inject
1448                  * this virtual interrupt during the subsequent VM enter.
1449                  */
1450
1451                 /*
1452                  * This is special. We want to treat this as an 'handled'
1453                  * VM-exit but not increment the instruction pointer.
1454                  */
1455                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXTINT, 1);
1456                 return (1);
1457         case EXIT_REASON_NMI_WINDOW:
1458                 /* Exit to allow the pending virtual NMI to be injected */
1459                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NMI_WINDOW, 1);
1460                 vmx_clear_nmi_window_exiting(vmx, vcpu);
1461                 VMM_CTR0(vmx->vm, vcpu, "Disabling NMI window exiting");
1462                 return (1);
1463         case EXIT_REASON_INOUT:
1464                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INOUT, 1);
1465                 vmexit->exitcode = VM_EXITCODE_INOUT;
1466                 vmexit->u.inout.bytes = (qual & 0x7) + 1;
1467                 vmexit->u.inout.in = (qual & 0x8) ? 1 : 0;
1468                 vmexit->u.inout.string = (qual & 0x10) ? 1 : 0;
1469                 vmexit->u.inout.rep = (qual & 0x20) ? 1 : 0;
1470                 vmexit->u.inout.port = (uint16_t)(qual >> 16);
1471                 vmexit->u.inout.eax = (uint32_t)(vmxctx->guest_rax);
1472                 break;
1473         case EXIT_REASON_CPUID:
1474                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CPUID, 1);
1475                 handled = vmx_handle_cpuid(vmx->vm, vcpu, vmxctx);
1476                 break;
1477         case EXIT_REASON_EPT_FAULT:
1478                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EPT_FAULT, 1);
1479                 /*
1480                  * If 'gpa' lies within the address space allocated to
1481                  * memory then this must be a nested page fault otherwise
1482                  * this must be an instruction that accesses MMIO space.
1483                  */
1484                 gpa = vmcs_gpa();
1485                 if (vm_mem_allocated(vmx->vm, gpa)) {
1486                         vmexit->exitcode = VM_EXITCODE_PAGING;
1487                         vmexit->u.paging.gpa = gpa;
1488                         vmexit->u.paging.fault_type = ept_fault_type(qual);
1489                         vmexit->u.paging.protection = ept_protection(qual);
1490                 } else if (ept_emulation_fault(qual)) {
1491                         vmexit->exitcode = VM_EXITCODE_INST_EMUL;
1492                         vmexit->u.inst_emul.gpa = gpa;
1493                         vmexit->u.inst_emul.gla = vmcs_gla();
1494                         vmexit->u.inst_emul.cr3 = vmcs_guest_cr3();
1495                 }
1496                 break;
1497         default:
1498                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_UNKNOWN, 1);
1499                 break;
1500         }
1501
1502         if (handled) {
1503                 /*
1504                  * It is possible that control is returned to userland
1505                  * even though we were able to handle the VM exit in the
1506                  * kernel.
1507                  *
1508                  * In such a case we want to make sure that the userland
1509                  * restarts guest execution at the instruction *after*
1510                  * the one we just processed. Therefore we update the
1511                  * guest rip in the VMCS and in 'vmexit'.
1512                  */
1513                 vm_exit_update_rip(vmexit);
1514                 vmexit->rip += vmexit->inst_length;
1515                 vmexit->inst_length = 0;
1516         } else {
1517                 if (vmexit->exitcode == VM_EXITCODE_BOGUS) {
1518                         /*
1519                          * If this VM exit was not claimed by anybody then
1520                          * treat it as a generic VMX exit.
1521                          */
1522                         vmexit->exitcode = VM_EXITCODE_VMX;
1523                         vmexit->u.vmx.error = 0;
1524                 } else {
1525                         /*
1526                          * The exitcode and collateral have been populated.
1527                          * The VM exit will be processed further in userland.
1528                          */
1529                 }
1530         }
1531         return (handled);
1532 }
1533
1534 static int
1535 vmx_run(void *arg, int vcpu, register_t rip, pmap_t pmap)
1536 {
1537         int error, vie, rc, handled, astpending;
1538         uint32_t exit_reason;
1539         struct vmx *vmx;
1540         struct vmxctx *vmxctx;
1541         struct vmcs *vmcs;
1542         struct vm_exit *vmexit;
1543
1544         vmx = arg;
1545         vmcs = &vmx->vmcs[vcpu];
1546         vmxctx = &vmx->ctx[vcpu];
1547         vmxctx->launched = 0;
1548
1549         astpending = 0;
1550         vmexit = vm_exitinfo(vmx->vm, vcpu);
1551
1552         KASSERT(vmxctx->pmap == pmap,
1553             ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap));
1554         KASSERT(vmxctx->eptp == vmx->eptp,
1555             ("eptp %p different than ctx eptp %#lx", eptp, vmxctx->eptp));
1556
1557         /*
1558          * XXX Can we avoid doing this every time we do a vm run?
1559          */
1560         VMPTRLD(vmcs);
1561
1562         /*
1563          * XXX
1564          * We do this every time because we may setup the virtual machine
1565          * from a different process than the one that actually runs it.
1566          *
1567          * If the life of a virtual machine was spent entirely in the context
1568          * of a single process we could do this once in vmcs_set_defaults().
1569          */
1570         if ((error = vmwrite(VMCS_HOST_CR3, rcr3())) != 0)
1571                 panic("vmx_run: error %d writing to VMCS_HOST_CR3", error);
1572
1573         if ((error = vmwrite(VMCS_GUEST_RIP, rip)) != 0)
1574                 panic("vmx_run: error %d writing to VMCS_GUEST_RIP", error);
1575
1576         if ((error = vmx_set_pcpu_defaults(vmx, vcpu)) != 0)
1577                 panic("vmx_run: error %d setting up pcpu defaults", error);
1578
1579         do {
1580                 lapic_timer_tick(vmx->vm, vcpu);
1581                 vmx_inject_interrupts(vmx, vcpu);
1582                 vmx_run_trace(vmx, vcpu);
1583                 rc = vmx_setjmp(vmxctx);
1584 #ifdef SETJMP_TRACE
1585                 vmx_setjmp_trace(vmx, vcpu, vmxctx, rc);
1586 #endif
1587                 switch (rc) {
1588                 case VMX_RETURN_DIRECT:
1589                         if (vmxctx->launched == 0) {
1590                                 vmxctx->launched = 1;
1591                                 vmx_launch(vmxctx);
1592                         } else
1593                                 vmx_resume(vmxctx);
1594                         panic("vmx_launch/resume should not return");
1595                         break;
1596                 case VMX_RETURN_LONGJMP:
1597                         break;                  /* vm exit */
1598                 case VMX_RETURN_AST:
1599                         astpending = 1;
1600                         break;
1601                 case VMX_RETURN_VMRESUME:
1602                         vie = vmcs_instruction_error();
1603                         if (vmxctx->launch_error == VM_FAIL_INVALID ||
1604                             vie != VMRESUME_WITH_NON_LAUNCHED_VMCS) {
1605                                 printf("vmresume error %d vmcs inst error %d\n",
1606                                         vmxctx->launch_error, vie);
1607                                 goto err_exit;
1608                         }
1609                         vmx_launch(vmxctx);     /* try to launch the guest */
1610                         panic("vmx_launch should not return");
1611                         break;
1612                 case VMX_RETURN_VMLAUNCH:
1613                         vie = vmcs_instruction_error();
1614 #if 1
1615                         printf("vmlaunch error %d vmcs inst error %d\n",
1616                                 vmxctx->launch_error, vie);
1617 #endif
1618                         goto err_exit;
1619                 case VMX_RETURN_INVEPT:
1620                         panic("vm %s:%d invept error %d",
1621                               vm_name(vmx->vm), vcpu, vmxctx->launch_error);
1622                 default:
1623                         panic("vmx_setjmp returned %d", rc);
1624                 }
1625
1626                 /* enable interrupts */
1627                 enable_intr();
1628
1629                 /* collect some basic information for VM exit processing */
1630                 vmexit->rip = rip = vmcs_guest_rip();
1631                 vmexit->inst_length = vmexit_instruction_length();
1632                 vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason();
1633                 vmexit->u.vmx.exit_qualification = vmcs_exit_qualification();
1634
1635                 if (astpending) {
1636                         handled = 1;
1637                         vmexit->inst_length = 0;
1638                         vmexit->exitcode = VM_EXITCODE_BOGUS;
1639                         vmx_astpending_trace(vmx, vcpu, rip);
1640                         vmm_stat_incr(vmx->vm, vcpu, VMEXIT_ASTPENDING, 1);
1641                         break;
1642                 }
1643
1644                 handled = vmx_exit_process(vmx, vcpu, vmexit);
1645                 vmx_exit_trace(vmx, vcpu, rip, exit_reason, handled);
1646
1647         } while (handled);
1648
1649         /*
1650          * If a VM exit has been handled then the exitcode must be BOGUS
1651          * If a VM exit is not handled then the exitcode must not be BOGUS
1652          */
1653         if ((handled && vmexit->exitcode != VM_EXITCODE_BOGUS) ||
1654             (!handled && vmexit->exitcode == VM_EXITCODE_BOGUS)) {
1655                 panic("Mismatch between handled (%d) and exitcode (%d)",
1656                       handled, vmexit->exitcode);
1657         }
1658
1659         if (!handled)
1660                 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_USERSPACE, 1);
1661
1662         VMM_CTR1(vmx->vm, vcpu, "goto userland: exitcode %d",vmexit->exitcode);
1663
1664         /*
1665          * XXX
1666          * We need to do this to ensure that any VMCS state cached by the
1667          * processor is flushed to memory. We need to do this in case the
1668          * VM moves to a different cpu the next time it runs.
1669          *
1670          * Can we avoid doing this?
1671          */
1672         VMCLEAR(vmcs);
1673         return (0);
1674
1675 err_exit:
1676         vmexit->exitcode = VM_EXITCODE_VMX;
1677         vmexit->u.vmx.exit_reason = (uint32_t)-1;
1678         vmexit->u.vmx.exit_qualification = (uint32_t)-1;
1679         vmexit->u.vmx.error = vie;
1680         VMCLEAR(vmcs);
1681         return (ENOEXEC);
1682 }
1683
1684 static void
1685 vmx_vmcleanup(void *arg)
1686 {
1687         int i, error;
1688         struct vmx *vmx = arg;
1689
1690         for (i = 0; i < VM_MAXCPU; i++)
1691                 vpid_free(vmx->state[i].vpid);
1692
1693         /*
1694          * XXXSMP we also need to clear the VMCS active on the other vcpus.
1695          */
1696         error = vmclear(&vmx->vmcs[0]);
1697         if (error != 0)
1698                 panic("vmx_vmcleanup: vmclear error %d on vcpu 0", error);
1699
1700         free(vmx, M_VMX);
1701
1702         return;
1703 }
1704
1705 static register_t *
1706 vmxctx_regptr(struct vmxctx *vmxctx, int reg)
1707 {
1708
1709         switch (reg) {
1710         case VM_REG_GUEST_RAX:
1711                 return (&vmxctx->guest_rax);
1712         case VM_REG_GUEST_RBX:
1713                 return (&vmxctx->guest_rbx);
1714         case VM_REG_GUEST_RCX:
1715                 return (&vmxctx->guest_rcx);
1716         case VM_REG_GUEST_RDX:
1717                 return (&vmxctx->guest_rdx);
1718         case VM_REG_GUEST_RSI:
1719                 return (&vmxctx->guest_rsi);
1720         case VM_REG_GUEST_RDI:
1721                 return (&vmxctx->guest_rdi);
1722         case VM_REG_GUEST_RBP:
1723                 return (&vmxctx->guest_rbp);
1724         case VM_REG_GUEST_R8:
1725                 return (&vmxctx->guest_r8);
1726         case VM_REG_GUEST_R9:
1727                 return (&vmxctx->guest_r9);
1728         case VM_REG_GUEST_R10:
1729                 return (&vmxctx->guest_r10);
1730         case VM_REG_GUEST_R11:
1731                 return (&vmxctx->guest_r11);
1732         case VM_REG_GUEST_R12:
1733                 return (&vmxctx->guest_r12);
1734         case VM_REG_GUEST_R13:
1735                 return (&vmxctx->guest_r13);
1736         case VM_REG_GUEST_R14:
1737                 return (&vmxctx->guest_r14);
1738         case VM_REG_GUEST_R15:
1739                 return (&vmxctx->guest_r15);
1740         default:
1741                 break;
1742         }
1743         return (NULL);
1744 }
1745
1746 static int
1747 vmxctx_getreg(struct vmxctx *vmxctx, int reg, uint64_t *retval)
1748 {
1749         register_t *regp;
1750
1751         if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
1752                 *retval = *regp;
1753                 return (0);
1754         } else
1755                 return (EINVAL);
1756 }
1757
1758 static int
1759 vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val)
1760 {
1761         register_t *regp;
1762
1763         if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
1764                 *regp = val;
1765                 return (0);
1766         } else
1767                 return (EINVAL);
1768 }
1769
1770 static int
1771 vmx_shadow_reg(int reg)
1772 {
1773         int shreg;
1774
1775         shreg = -1;
1776
1777         switch (reg) {
1778         case VM_REG_GUEST_CR0:
1779                 shreg = VMCS_CR0_SHADOW;
1780                 break;
1781         case VM_REG_GUEST_CR4:
1782                 shreg = VMCS_CR4_SHADOW;
1783                 break;
1784         default:
1785                 break;
1786         }
1787
1788         return (shreg);
1789 }
1790
1791 static int
1792 vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval)
1793 {
1794         int running, hostcpu;
1795         struct vmx *vmx = arg;
1796
1797         running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
1798         if (running && hostcpu != curcpu)
1799                 panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), vcpu);
1800
1801         if (vmxctx_getreg(&vmx->ctx[vcpu], reg, retval) == 0)
1802                 return (0);
1803
1804         return (vmcs_getreg(&vmx->vmcs[vcpu], running, reg, retval));
1805 }
1806
1807 static int
1808 vmx_setreg(void *arg, int vcpu, int reg, uint64_t val)
1809 {
1810         int error, hostcpu, running, shadow;
1811         uint64_t ctls;
1812         struct vmx *vmx = arg;
1813
1814         running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
1815         if (running && hostcpu != curcpu)
1816                 panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), vcpu);
1817
1818         if (vmxctx_setreg(&vmx->ctx[vcpu], reg, val) == 0)
1819                 return (0);
1820
1821         error = vmcs_setreg(&vmx->vmcs[vcpu], running, reg, val);
1822
1823         if (error == 0) {
1824                 /*
1825                  * If the "load EFER" VM-entry control is 1 then the
1826                  * value of EFER.LMA must be identical to "IA-32e mode guest"
1827                  * bit in the VM-entry control.
1828                  */
1829                 if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 &&
1830                     (reg == VM_REG_GUEST_EFER)) {
1831                         vmcs_getreg(&vmx->vmcs[vcpu], running,
1832                                     VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls);
1833                         if (val & EFER_LMA)
1834                                 ctls |= VM_ENTRY_GUEST_LMA;
1835                         else
1836                                 ctls &= ~VM_ENTRY_GUEST_LMA;
1837                         vmcs_setreg(&vmx->vmcs[vcpu], running,
1838                                     VMCS_IDENT(VMCS_ENTRY_CTLS), ctls);
1839                 }
1840
1841                 shadow = vmx_shadow_reg(reg);
1842                 if (shadow > 0) {
1843                         /*
1844                          * Store the unmodified value in the shadow
1845                          */
1846                         error = vmcs_setreg(&vmx->vmcs[vcpu], running,
1847                                     VMCS_IDENT(shadow), val);
1848                 }
1849         }
1850
1851         return (error);
1852 }
1853
1854 static int
1855 vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
1856 {
1857         struct vmx *vmx = arg;
1858
1859         return (vmcs_getdesc(&vmx->vmcs[vcpu], reg, desc));
1860 }
1861
1862 static int
1863 vmx_setdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
1864 {
1865         struct vmx *vmx = arg;
1866
1867         return (vmcs_setdesc(&vmx->vmcs[vcpu], reg, desc));
1868 }
1869
1870 static int
1871 vmx_inject(void *arg, int vcpu, int type, int vector, uint32_t code,
1872            int code_valid)
1873 {
1874         int error;
1875         uint64_t info;
1876         struct vmx *vmx = arg;
1877         struct vmcs *vmcs = &vmx->vmcs[vcpu];
1878
1879         static uint32_t type_map[VM_EVENT_MAX] = {
1880                 0x1,            /* VM_EVENT_NONE */
1881                 0x0,            /* VM_HW_INTR */
1882                 0x2,            /* VM_NMI */
1883                 0x3,            /* VM_HW_EXCEPTION */
1884                 0x4,            /* VM_SW_INTR */
1885                 0x5,            /* VM_PRIV_SW_EXCEPTION */
1886                 0x6,            /* VM_SW_EXCEPTION */
1887         };
1888
1889         /*
1890          * If there is already an exception pending to be delivered to the
1891          * vcpu then just return.
1892          */
1893         error = vmcs_getreg(vmcs, 0, VMCS_IDENT(VMCS_ENTRY_INTR_INFO), &info);
1894         if (error)
1895                 return (error);
1896
1897         if (info & VMCS_INTERRUPTION_INFO_VALID)
1898                 return (EAGAIN);
1899
1900         info = vector | (type_map[type] << 8) | (code_valid ? 1 << 11 : 0);
1901         info |= VMCS_INTERRUPTION_INFO_VALID;
1902         error = vmcs_setreg(vmcs, 0, VMCS_IDENT(VMCS_ENTRY_INTR_INFO), info);
1903         if (error != 0)
1904                 return (error);
1905
1906         if (code_valid) {
1907                 error = vmcs_setreg(vmcs, 0,
1908                                     VMCS_IDENT(VMCS_ENTRY_EXCEPTION_ERROR),
1909                                     code);
1910         }
1911         return (error);
1912 }
1913
1914 static int
1915 vmx_getcap(void *arg, int vcpu, int type, int *retval)
1916 {
1917         struct vmx *vmx = arg;
1918         int vcap;
1919         int ret;
1920
1921         ret = ENOENT;
1922
1923         vcap = vmx->cap[vcpu].set;
1924
1925         switch (type) {
1926         case VM_CAP_HALT_EXIT:
1927                 if (cap_halt_exit)
1928                         ret = 0;
1929                 break;
1930         case VM_CAP_PAUSE_EXIT:
1931                 if (cap_pause_exit)
1932                         ret = 0;
1933                 break;
1934         case VM_CAP_MTRAP_EXIT:
1935                 if (cap_monitor_trap)
1936                         ret = 0;
1937                 break;
1938         case VM_CAP_UNRESTRICTED_GUEST:
1939                 if (cap_unrestricted_guest)
1940                         ret = 0;
1941                 break;
1942         case VM_CAP_ENABLE_INVPCID:
1943                 if (cap_invpcid)
1944                         ret = 0;
1945                 break;
1946         default:
1947                 break;
1948         }
1949
1950         if (ret == 0)
1951                 *retval = (vcap & (1 << type)) ? 1 : 0;
1952
1953         return (ret);
1954 }
1955
1956 static int
1957 vmx_setcap(void *arg, int vcpu, int type, int val)
1958 {
1959         struct vmx *vmx = arg;
1960         struct vmcs *vmcs = &vmx->vmcs[vcpu];
1961         uint32_t baseval;
1962         uint32_t *pptr;
1963         int error;
1964         int flag;
1965         int reg;
1966         int retval;
1967
1968         retval = ENOENT;
1969         pptr = NULL;
1970
1971         switch (type) {
1972         case VM_CAP_HALT_EXIT:
1973                 if (cap_halt_exit) {
1974                         retval = 0;
1975                         pptr = &vmx->cap[vcpu].proc_ctls;
1976                         baseval = *pptr;
1977                         flag = PROCBASED_HLT_EXITING;
1978                         reg = VMCS_PRI_PROC_BASED_CTLS;
1979                 }
1980                 break;
1981         case VM_CAP_MTRAP_EXIT:
1982                 if (cap_monitor_trap) {
1983                         retval = 0;
1984                         pptr = &vmx->cap[vcpu].proc_ctls;
1985                         baseval = *pptr;
1986                         flag = PROCBASED_MTF;
1987                         reg = VMCS_PRI_PROC_BASED_CTLS;
1988                 }
1989                 break;
1990         case VM_CAP_PAUSE_EXIT:
1991                 if (cap_pause_exit) {
1992                         retval = 0;
1993                         pptr = &vmx->cap[vcpu].proc_ctls;
1994                         baseval = *pptr;
1995                         flag = PROCBASED_PAUSE_EXITING;
1996                         reg = VMCS_PRI_PROC_BASED_CTLS;
1997                 }
1998                 break;
1999         case VM_CAP_UNRESTRICTED_GUEST:
2000                 if (cap_unrestricted_guest) {
2001                         retval = 0;
2002                         pptr = &vmx->cap[vcpu].proc_ctls2;
2003                         baseval = *pptr;
2004                         flag = PROCBASED2_UNRESTRICTED_GUEST;
2005                         reg = VMCS_SEC_PROC_BASED_CTLS;
2006                 }
2007                 break;
2008         case VM_CAP_ENABLE_INVPCID:
2009                 if (cap_invpcid) {
2010                         retval = 0;
2011                         pptr = &vmx->cap[vcpu].proc_ctls2;
2012                         baseval = *pptr;
2013                         flag = PROCBASED2_ENABLE_INVPCID;
2014                         reg = VMCS_SEC_PROC_BASED_CTLS;
2015                 }
2016                 break;
2017         default:
2018                 break;
2019         }
2020
2021         if (retval == 0) {
2022                 if (val) {
2023                         baseval |= flag;
2024                 } else {
2025                         baseval &= ~flag;
2026                 }
2027                 VMPTRLD(vmcs);
2028                 error = vmwrite(reg, baseval);
2029                 VMCLEAR(vmcs);
2030
2031                 if (error) {
2032                         retval = error;
2033                 } else {
2034                         /*
2035                          * Update optional stored flags, and record
2036                          * setting
2037                          */
2038                         if (pptr != NULL) {
2039                                 *pptr = baseval;
2040                         }
2041
2042                         if (val) {
2043                                 vmx->cap[vcpu].set |= (1 << type);
2044                         } else {
2045                                 vmx->cap[vcpu].set &= ~(1 << type);
2046                         }
2047                 }
2048         }
2049
2050         return (retval);
2051 }
2052
2053 struct vmm_ops vmm_ops_intel = {
2054         vmx_init,
2055         vmx_cleanup,
2056         vmx_vminit,
2057         vmx_run,
2058         vmx_vmcleanup,
2059         vmx_getreg,
2060         vmx_setreg,
2061         vmx_getdesc,
2062         vmx_setdesc,
2063         vmx_inject,
2064         vmx_getcap,
2065         vmx_setcap,
2066         ept_vmspace_alloc,
2067         ept_vmspace_free,
2068 };