usr.sbin/bhyvectl/bhyvectl.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause
   3  *
   4  * Copyright (c) 2011 NetApp, Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 #include <sys/cdefs.h>
  30 #include <sys/param.h>
  31 #include <sys/types.h>
  32 #include <sys/cpuset.h>
  33 #include <sys/errno.h>
  34 #include <sys/mman.h>
  35 #include <sys/nv.h>
  36 #include <sys/socket.h>
  37 #include <sys/sysctl.h>
  38 #include <sys/un.h>
  39
  40 #include <stdio.h>
  41 #include <stdlib.h>
  42 #include <stdbool.h>
  43 #include <string.h>
  44 #include <unistd.h>
  45 #include <libgen.h>
  46 #include <libutil.h>
  47 #include <fcntl.h>
  48 #include <getopt.h>
  49 #include <time.h>
  50 #include <assert.h>
  51 #include <libutil.h>
  52
  53 #include <machine/cpufunc.h>
  54 #include <machine/specialreg.h>
  55 #include <machine/vmm.h>
  56 #include <machine/vmm_dev.h>
  57 #include <vmmapi.h>
  58
  59 #include "amd/vmcb.h"
  60 #include "intel/vmcs.h"
  61
  62 #ifdef BHYVE_SNAPSHOT
  63 #include "snapshot.h"
  64 #endif
  65
  66 #define MB      (1UL << 20)
  67 #define GB      (1UL << 30)
  68
  69 #define REQ_ARG         required_argument
  70 #define NO_ARG          no_argument
  71 #define OPT_ARG         optional_argument
  72
  73 static const char *progname;
  74
  75 static void
  76 usage(bool cpu_intel)
  77 {
  78
  79         (void)fprintf(stderr,
  80         "Usage: %s --vm=<vmname>\n"
  81         "       [--cpu=<vcpu_number>]\n"
  82         "       [--create]\n"
  83         "       [--destroy]\n"
  84 #ifdef BHYVE_SNAPSHOT
  85         "       [--checkpoint=<filename> | --suspend=<filename>]\n"
  86 #endif
  87         "       [--get-all]\n"
  88         "       [--get-stats]\n"
  89         "       [--set-desc-ds]\n"
  90         "       [--get-desc-ds]\n"
  91         "       [--set-desc-es]\n"
  92         "       [--get-desc-es]\n"
  93         "       [--set-desc-gs]\n"
  94         "       [--get-desc-gs]\n"
  95         "       [--set-desc-fs]\n"
  96         "       [--get-desc-fs]\n"
  97         "       [--set-desc-cs]\n"
  98         "       [--get-desc-cs]\n"
  99         "       [--set-desc-ss]\n"
 100         "       [--get-desc-ss]\n"
 101         "       [--set-desc-tr]\n"
 102         "       [--get-desc-tr]\n"
 103         "       [--set-desc-ldtr]\n"
 104         "       [--get-desc-ldtr]\n"
 105         "       [--set-desc-gdtr]\n"
 106         "       [--get-desc-gdtr]\n"
 107         "       [--set-desc-idtr]\n"
 108         "       [--get-desc-idtr]\n"
 109         "       [--run]\n"
 110         "       [--capname=<capname>]\n"
 111         "       [--getcap]\n"
 112         "       [--setcap=<0|1>]\n"
 113         "       [--desc-base=<BASE>]\n"
 114         "       [--desc-limit=<LIMIT>]\n"
 115         "       [--desc-access=<ACCESS>]\n"
 116         "       [--set-cr0=<CR0>]\n"
 117         "       [--get-cr0]\n"
 118         "       [--set-cr2=<CR2>]\n"
 119         "       [--get-cr2]\n"
 120         "       [--set-cr3=<CR3>]\n"
 121         "       [--get-cr3]\n"
 122         "       [--set-cr4=<CR4>]\n"
 123         "       [--get-cr4]\n"
 124         "       [--set-dr0=<DR0>]\n"
 125         "       [--get-dr0]\n"
 126         "       [--set-dr1=<DR1>]\n"
 127         "       [--get-dr1]\n"
 128         "       [--set-dr2=<DR2>]\n"
 129         "       [--get-dr2]\n"
 130         "       [--set-dr3=<DR3>]\n"
 131         "       [--get-dr3]\n"
 132         "       [--set-dr6=<DR6>]\n"
 133         "       [--get-dr6]\n"
 134         "       [--set-dr7=<DR7>]\n"
 135         "       [--get-dr7]\n"
 136         "       [--set-rsp=<RSP>]\n"
 137         "       [--get-rsp]\n"
 138         "       [--set-rip=<RIP>]\n"
 139         "       [--get-rip]\n"
 140         "       [--get-rax]\n"
 141         "       [--set-rax=<RAX>]\n"
 142         "       [--get-rbx]\n"
 143         "       [--get-rcx]\n"
 144         "       [--get-rdx]\n"
 145         "       [--get-rsi]\n"
 146         "       [--get-rdi]\n"
 147         "       [--get-rbp]\n"
 148         "       [--get-r8]\n"
 149         "       [--get-r9]\n"
 150         "       [--get-r10]\n"
 151         "       [--get-r11]\n"
 152         "       [--get-r12]\n"
 153         "       [--get-r13]\n"
 154         "       [--get-r14]\n"
 155         "       [--get-r15]\n"
 156         "       [--set-rflags=<RFLAGS>]\n"
 157         "       [--get-rflags]\n"
 158         "       [--set-cs]\n"
 159         "       [--get-cs]\n"
 160         "       [--set-ds]\n"
 161         "       [--get-ds]\n"
 162         "       [--set-es]\n"
 163         "       [--get-es]\n"
 164         "       [--set-fs]\n"
 165         "       [--get-fs]\n"
 166         "       [--set-gs]\n"
 167         "       [--get-gs]\n"
 168         "       [--set-ss]\n"
 169         "       [--get-ss]\n"
 170         "       [--get-tr]\n"
 171         "       [--get-ldtr]\n"
 172         "       [--set-x2apic-state=<state>]\n"
 173         "       [--get-x2apic-state]\n"
 174         "       [--unassign-pptdev=<bus/slot/func>]\n"
 175         "       [--set-mem=<memory in units of MB>]\n"
 176         "       [--get-lowmem]\n"
 177         "       [--get-highmem]\n"
 178         "       [--get-gpa-pmap]\n"
 179         "       [--assert-lapic-lvt=<pin>]\n"
 180         "       [--inject-nmi]\n"
 181         "       [--force-reset]\n"
 182         "       [--force-poweroff]\n"
 183         "       [--get-rtc-time]\n"
 184         "       [--set-rtc-time=<secs>]\n"
 185         "       [--get-rtc-nvram]\n"
 186         "       [--set-rtc-nvram=<val>]\n"
 187         "       [--rtc-nvram-offset=<offset>]\n"
 188         "       [--get-active-cpus]\n"
 189         "       [--get-debug-cpus]\n"
 190         "       [--get-suspended-cpus]\n"
 191         "       [--get-intinfo]\n"
 192         "       [--get-eptp]\n"
 193         "       [--set-exception-bitmap]\n"
 194         "       [--get-exception-bitmap]\n"
 195         "       [--get-tsc-offset]\n"
 196         "       [--get-guest-pat]\n"
 197         "       [--get-io-bitmap-address]\n"
 198         "       [--get-msr-bitmap]\n"
 199         "       [--get-msr-bitmap-address]\n"
 200         "       [--get-guest-sysenter]\n"
 201         "       [--get-exit-reason]\n"
 202         "       [--get-cpu-topology]\n",
 203         progname);
 204
 205         if (cpu_intel) {
 206                 (void)fprintf(stderr,
 207                 "       [--get-vmcs-pinbased-ctls]\n"
 208                 "       [--get-vmcs-procbased-ctls]\n"
 209                 "       [--get-vmcs-procbased-ctls2]\n"
 210                 "       [--get-vmcs-entry-interruption-info]\n"
 211                 "       [--set-vmcs-entry-interruption-info=<info>]\n"
 212                 "       [--get-vmcs-guest-physical-address\n"
 213                 "       [--get-vmcs-guest-linear-address\n"
 214                 "       [--get-vmcs-host-pat]\n"
 215                 "       [--get-vmcs-host-cr0]\n"
 216                 "       [--get-vmcs-host-cr3]\n"
 217                 "       [--get-vmcs-host-cr4]\n"
 218                 "       [--get-vmcs-host-rip]\n"
 219                 "       [--get-vmcs-host-rsp]\n"
 220                 "       [--get-vmcs-cr0-mask]\n"
 221                 "       [--get-vmcs-cr0-shadow]\n"
 222                 "       [--get-vmcs-cr4-mask]\n"
 223                 "       [--get-vmcs-cr4-shadow]\n"
 224                 "       [--get-vmcs-cr3-targets]\n"
 225                 "       [--get-vmcs-apic-access-address]\n"
 226                 "       [--get-vmcs-virtual-apic-address]\n"
 227                 "       [--get-vmcs-tpr-threshold]\n"
 228                 "       [--get-vmcs-vpid]\n"
 229                 "       [--get-vmcs-instruction-error]\n"
 230                 "       [--get-vmcs-exit-ctls]\n"
 231                 "       [--get-vmcs-entry-ctls]\n"
 232                 "       [--get-vmcs-link]\n"
 233                 "       [--get-vmcs-exit-qualification]\n"
 234                 "       [--get-vmcs-exit-interruption-info]\n"
 235                 "       [--get-vmcs-exit-interruption-error]\n"
 236                 "       [--get-vmcs-interruptibility]\n"
 237                 );
 238         } else {
 239                 (void)fprintf(stderr,
 240                 "       [--get-vmcb-intercepts]\n"
 241                 "       [--get-vmcb-asid]\n"
 242                 "       [--get-vmcb-exit-details]\n"
 243                 "       [--get-vmcb-tlb-ctrl]\n"
 244                 "       [--get-vmcb-virq]\n"
 245                 "       [--get-avic-apic-bar]\n"
 246                 "       [--get-avic-backing-page]\n"
 247                 "       [--get-avic-table]\n"
 248                 );
 249         }
 250         exit(1);
 251 }
 252
 253 static int get_rtc_time, set_rtc_time;
 254 static int get_rtc_nvram, set_rtc_nvram;
 255 static int rtc_nvram_offset;
 256 static uint8_t rtc_nvram_value;
 257 static time_t rtc_secs;
 258
 259 static int get_stats, getcap, setcap, capval, get_gpa_pmap;
 260 static int inject_nmi, assert_lapic_lvt;
 261 static int force_reset, force_poweroff;
 262 static const char *capname;
 263 static int create, destroy, get_memmap, get_memseg;
 264 static int get_intinfo;
 265 static int get_active_cpus, get_debug_cpus, get_suspended_cpus;
 266 static uint64_t memsize;
 267 static int set_cr0, get_cr0, set_cr2, get_cr2, set_cr3, get_cr3;
 268 static int set_cr4, get_cr4;
 269 static int set_efer, get_efer;
 270 static int set_dr0, get_dr0;
 271 static int set_dr1, get_dr1;
 272 static int set_dr2, get_dr2;
 273 static int set_dr3, get_dr3;
 274 static int set_dr6, get_dr6;
 275 static int set_dr7, get_dr7;
 276 static int set_rsp, get_rsp, set_rip, get_rip, set_rflags, get_rflags;
 277 static int set_rax, get_rax;
 278 static int get_rbx, get_rcx, get_rdx, get_rsi, get_rdi, get_rbp;
 279 static int get_r8, get_r9, get_r10, get_r11, get_r12, get_r13, get_r14, get_r15;
 280 static int set_desc_ds, get_desc_ds;
 281 static int set_desc_es, get_desc_es;
 282 static int set_desc_fs, get_desc_fs;
 283 static int set_desc_gs, get_desc_gs;
 284 static int set_desc_cs, get_desc_cs;
 285 static int set_desc_ss, get_desc_ss;
 286 static int set_desc_gdtr, get_desc_gdtr;
 287 static int set_desc_idtr, get_desc_idtr;
 288 static int set_desc_tr, get_desc_tr;
 289 static int set_desc_ldtr, get_desc_ldtr;
 290 static int set_cs, set_ds, set_es, set_fs, set_gs, set_ss, set_tr, set_ldtr;
 291 static int get_cs, get_ds, get_es, get_fs, get_gs, get_ss, get_tr, get_ldtr;
 292 static int set_x2apic_state, get_x2apic_state;
 293 static enum x2apic_state x2apic_state;
 294 static int unassign_pptdev, bus, slot, func;
 295 static int run;
 296 static int get_cpu_topology;
 297 #ifdef BHYVE_SNAPSHOT
 298 static int vm_suspend_opt;
 299 #endif
 300
 301 /*
 302  * VMCB specific.
 303  */
 304 static int get_vmcb_intercept, get_vmcb_exit_details, get_vmcb_tlb_ctrl;
 305 static int get_vmcb_virq, get_avic_table;
 306
 307 /*
 308  * VMCS-specific fields
 309  */
 310 static int get_pinbased_ctls, get_procbased_ctls, get_procbased_ctls2;
 311 static int get_eptp, get_io_bitmap, get_tsc_offset;
 312 static int get_vmcs_entry_interruption_info;
 313 static int get_vmcs_interruptibility;
 314 static int get_vmcs_gpa, get_vmcs_gla;
 315 static int get_exception_bitmap;
 316 static int get_cr0_mask, get_cr0_shadow;
 317 static int get_cr4_mask, get_cr4_shadow;
 318 static int get_cr3_targets;
 319 static int get_apic_access_addr, get_virtual_apic_addr, get_tpr_threshold;
 320 static int get_msr_bitmap, get_msr_bitmap_address;
 321 static int get_vpid_asid;
 322 static int get_inst_err, get_exit_ctls, get_entry_ctls;
 323 static int get_host_cr0, get_host_cr3, get_host_cr4;
 324 static int get_host_rip, get_host_rsp;
 325 static int get_guest_pat, get_host_pat;
 326 static int get_guest_sysenter, get_vmcs_link;
 327 static int get_exit_reason, get_vmcs_exit_qualification;
 328 static int get_vmcs_exit_interruption_info, get_vmcs_exit_interruption_error;
 329 static int get_vmcs_exit_inst_length;
 330
 331 static uint64_t desc_base;
 332 static uint32_t desc_limit, desc_access;
 333
 334 static int get_all;
 335
 336 static void
 337 dump_vm_run_exitcode(struct vm_exit *vmexit, int vcpu)
 338 {
 339         printf("vm exit[%d]\n", vcpu);
 340         printf("\trip\t\t0x%016lx\n", vmexit->rip);
 341         printf("\tinst_length\t%d\n", vmexit->inst_length);
 342         switch (vmexit->exitcode) {
 343         case VM_EXITCODE_INOUT:
 344                 printf("\treason\t\tINOUT\n");
 345                 printf("\tdirection\t%s\n", vmexit->u.inout.in ? "IN" : "OUT");
 346                 printf("\tbytes\t\t%d\n", vmexit->u.inout.bytes);
 347                 printf("\tflags\t\t%s%s\n",
 348                         vmexit->u.inout.string ? "STRING " : "",
 349                         vmexit->u.inout.rep ? "REP " : "");
 350                 printf("\tport\t\t0x%04x\n", vmexit->u.inout.port);
 351                 printf("\teax\t\t0x%08x\n", vmexit->u.inout.eax);
 352                 break;
 353         case VM_EXITCODE_VMX:
 354                 printf("\treason\t\tVMX\n");
 355                 printf("\tstatus\t\t%d\n", vmexit->u.vmx.status);
 356                 printf("\texit_reason\t0x%08x (%u)\n",
 357                     vmexit->u.vmx.exit_reason, vmexit->u.vmx.exit_reason);
 358                 printf("\tqualification\t0x%016lx\n",
 359                         vmexit->u.vmx.exit_qualification);
 360                 printf("\tinst_type\t\t%d\n", vmexit->u.vmx.inst_type);
 361                 printf("\tinst_error\t\t%d\n", vmexit->u.vmx.inst_error);
 362                 break;
 363         case VM_EXITCODE_SVM:
 364                 printf("\treason\t\tSVM\n");
 365                 printf("\texit_reason\t\t%#lx\n", vmexit->u.svm.exitcode);
 366                 printf("\texitinfo1\t\t%#lx\n", vmexit->u.svm.exitinfo1);
 367                 printf("\texitinfo2\t\t%#lx\n", vmexit->u.svm.exitinfo2);
 368                 break;
 369         default:
 370                 printf("*** unknown vm run exitcode %d\n", vmexit->exitcode);
 371                 break;
 372         }
 373 }
 374
 375 /* AMD 6th generation and Intel compatible MSRs */
 376 #define MSR_AMD6TH_START        0xC0000000
 377 #define MSR_AMD6TH_END          0xC0001FFF
 378 /* AMD 7th and 8th generation compatible MSRs */
 379 #define MSR_AMD7TH_START        0xC0010000
 380 #define MSR_AMD7TH_END          0xC0011FFF
 381
 382 static const char *
 383 msr_name(uint32_t msr)
 384 {
 385         static char buf[32];
 386
 387         switch(msr) {
 388         case MSR_TSC:
 389                 return ("MSR_TSC");
 390         case MSR_EFER:
 391                 return ("MSR_EFER");
 392         case MSR_STAR:
 393                 return ("MSR_STAR");
 394         case MSR_LSTAR:
 395                 return ("MSR_LSTAR");
 396         case MSR_CSTAR:
 397                 return ("MSR_CSTAR");
 398         case MSR_SF_MASK:
 399                 return ("MSR_SF_MASK");
 400         case MSR_FSBASE:
 401                 return ("MSR_FSBASE");
 402         case MSR_GSBASE:
 403                 return ("MSR_GSBASE");
 404         case MSR_KGSBASE:
 405                 return ("MSR_KGSBASE");
 406         case MSR_SYSENTER_CS_MSR:
 407                 return ("MSR_SYSENTER_CS_MSR");
 408         case MSR_SYSENTER_ESP_MSR:
 409                 return ("MSR_SYSENTER_ESP_MSR");
 410         case MSR_SYSENTER_EIP_MSR:
 411                 return ("MSR_SYSENTER_EIP_MSR");
 412         case MSR_PAT:
 413                 return ("MSR_PAT");
 414         }
 415         snprintf(buf, sizeof(buf), "MSR       %#08x", msr);
 416
 417         return (buf);
 418 }
 419
 420 static inline void
 421 print_msr_pm(uint64_t msr, int vcpu, int readable, int writeable)
 422 {
 423
 424         if (readable || writeable) {
 425                 printf("%-20s[%d]\t\t%c%c\n", msr_name(msr), vcpu,
 426                         readable ? 'R' : '-', writeable ? 'W' : '-');
 427         }
 428 }
 429
 430 /*
 431  * Reference APM vol2, section 15.11 MSR Intercepts.
 432  */
 433 static void
 434 dump_amd_msr_pm(const char *bitmap, int vcpu)
 435 {
 436         int byte, bit, readable, writeable;
 437         uint32_t msr;
 438
 439         for (msr = 0; msr < 0x2000; msr++) {
 440                 byte = msr / 4;
 441                 bit = (msr % 4) * 2;
 442
 443                 /* Look at MSRs in the range 0x00000000 to 0x00001FFF */
 444                 readable = (bitmap[byte] & (1 << bit)) ? 0 : 1;
 445                 writeable = (bitmap[byte] & (2 << bit)) ?  0 : 1;
 446                 print_msr_pm(msr, vcpu, readable, writeable);
 447
 448                 /* Look at MSRs in the range 0xC0000000 to 0xC0001FFF */
 449                 byte += 2048;
 450                 readable = (bitmap[byte] & (1 << bit)) ? 0 : 1;
 451                 writeable = (bitmap[byte] & (2 << bit)) ?  0 : 1;
 452                 print_msr_pm(msr + MSR_AMD6TH_START, vcpu, readable,
 453                                 writeable);
 454
 455                 /* MSR 0xC0010000 to 0xC0011FF is only for AMD */
 456                 byte += 4096;
 457                 readable = (bitmap[byte] & (1 << bit)) ? 0 : 1;
 458                 writeable = (bitmap[byte] & (2 << bit)) ?  0 : 1;
 459                 print_msr_pm(msr + MSR_AMD7TH_START, vcpu, readable,
 460                                 writeable);
 461         }
 462 }
 463
 464 /*
 465  * Reference Intel SDM Vol3 Section 24.6.9 MSR-Bitmap Address
 466  */
 467 static void
 468 dump_intel_msr_pm(const char *bitmap, int vcpu)
 469 {
 470         int byte, bit, readable, writeable;
 471         uint32_t msr;
 472
 473         for (msr = 0; msr < 0x2000; msr++) {
 474                 byte = msr / 8;
 475                 bit = msr & 0x7;
 476
 477                 /* Look at MSRs in the range 0x00000000 to 0x00001FFF */
 478                 readable = (bitmap[byte] & (1 << bit)) ? 0 : 1;
 479                 writeable = (bitmap[2048 + byte] & (1 << bit)) ?  0 : 1;
 480                 print_msr_pm(msr, vcpu, readable, writeable);
 481
 482                 /* Look at MSRs in the range 0xC0000000 to 0xC0001FFF */
 483                 byte += 1024;
 484                 readable = (bitmap[byte] & (1 << bit)) ? 0 : 1;
 485                 writeable = (bitmap[2048 + byte] & (1 << bit)) ?  0 : 1;
 486                 print_msr_pm(msr + MSR_AMD6TH_START, vcpu, readable,
 487                                 writeable);
 488         }
 489 }
 490
 491 static int
 492 dump_msr_bitmap(int vcpu, uint64_t addr, bool cpu_intel)
 493 {
 494         char *bitmap;
 495         int error, fd, map_size;
 496
 497         error = -1;
 498         bitmap = MAP_FAILED;
 499
 500         fd = open("/dev/mem", O_RDONLY, 0);
 501         if (fd < 0) {
 502                 perror("Couldn't open /dev/mem");
 503                 goto done;
 504         }
 505
 506         if (cpu_intel)
 507                 map_size = PAGE_SIZE;
 508         else
 509                 map_size = 2 * PAGE_SIZE;
 510
 511         bitmap = mmap(NULL, map_size, PROT_READ, MAP_SHARED, fd, addr);
 512         if (bitmap == MAP_FAILED) {
 513                 perror("mmap failed");
 514                 goto done;
 515         }
 516
 517         if (cpu_intel)
 518                 dump_intel_msr_pm(bitmap, vcpu);
 519         else
 520                 dump_amd_msr_pm(bitmap, vcpu);
 521
 522         error = 0;
 523 done:
 524         if (bitmap != MAP_FAILED)
 525                 munmap((void *)bitmap, map_size);
 526         if (fd >= 0)
 527                 close(fd);
 528
 529         return (error);
 530 }
 531
 532 static int
 533 vm_get_vmcs_field(struct vcpu *vcpu, int field, uint64_t *ret_val)
 534 {
 535
 536         return (vm_get_register(vcpu, VMCS_IDENT(field), ret_val));
 537 }
 538
 539 static int
 540 vm_get_vmcb_field(struct vcpu *vcpu, int off, int bytes,
 541         uint64_t *ret_val)
 542 {
 543
 544         return (vm_get_register(vcpu, VMCB_ACCESS(off, bytes), ret_val));
 545 }
 546
 547 enum {
 548         VMNAME = 1000,  /* avoid collision with return values from getopt */
 549         VCPU,
 550         SET_MEM,
 551         SET_EFER,
 552         SET_CR0,
 553         SET_CR2,
 554         SET_CR3,
 555         SET_CR4,
 556         SET_DR0,
 557         SET_DR1,
 558         SET_DR2,
 559         SET_DR3,
 560         SET_DR6,
 561         SET_DR7,
 562         SET_RSP,
 563         SET_RIP,
 564         SET_RAX,
 565         SET_RFLAGS,
 566         DESC_BASE,
 567         DESC_LIMIT,
 568         DESC_ACCESS,
 569         SET_CS,
 570         SET_DS,
 571         SET_ES,
 572         SET_FS,
 573         SET_GS,
 574         SET_SS,
 575         SET_TR,
 576         SET_LDTR,
 577         SET_X2APIC_STATE,
 578         SET_CAP,
 579         CAPNAME,
 580         UNASSIGN_PPTDEV,
 581         GET_GPA_PMAP,
 582         ASSERT_LAPIC_LVT,
 583         SET_RTC_TIME,
 584         SET_RTC_NVRAM,
 585         RTC_NVRAM_OFFSET,
 586 #ifdef BHYVE_SNAPSHOT
 587         SET_CHECKPOINT_FILE,
 588         SET_SUSPEND_FILE,
 589 #endif
 590 };
 591
 592 static void
 593 print_cpus(const char *banner, const cpuset_t *cpus)
 594 {
 595         int i, first;
 596
 597         first = 1;
 598         printf("%s:\t", banner);
 599         if (!CPU_EMPTY(cpus)) {
 600                 for (i = 0; i < CPU_SETSIZE; i++) {
 601                         if (CPU_ISSET(i, cpus)) {
 602                                 printf("%s%d", first ? " " : ", ", i);
 603                                 first = 0;
 604                         }
 605                 }
 606         } else
 607                 printf(" (none)");
 608         printf("\n");
 609 }
 610
 611 static void
 612 print_intinfo(const char *banner, uint64_t info)
 613 {
 614         int type;
 615
 616         printf("%s:\t", banner);
 617         if (info & VM_INTINFO_VALID) {
 618                 type = info & VM_INTINFO_TYPE;
 619                 switch (type) {
 620                 case VM_INTINFO_HWINTR:
 621                         printf("extint");
 622                         break;
 623                 case VM_INTINFO_NMI:
 624                         printf("nmi");
 625                         break;
 626                 case VM_INTINFO_SWINTR:
 627                         printf("swint");
 628                         break;
 629                 default:
 630                         printf("exception");
 631                         break;
 632                 }
 633                 printf(" vector %d", (int)VM_INTINFO_VECTOR(info));
 634                 if (info & VM_INTINFO_DEL_ERRCODE)
 635                         printf(" errcode %#x", (u_int)(info >> 32));
 636         } else {
 637                 printf("n/a");
 638         }
 639         printf("\n");
 640 }
 641
 642 static bool
 643 cpu_vendor_intel(void)
 644 {
 645         u_int regs[4], v[3];
 646
 647         do_cpuid(0, regs);
 648         v[0] = regs[1];
 649         v[1] = regs[3];
 650         v[2] = regs[2];
 651
 652         if (memcmp(v, "GenuineIntel", sizeof(v)) == 0)
 653                 return (true);
 654         if (memcmp(v, "AuthenticAMD", sizeof(v)) == 0 ||
 655             memcmp(v, "HygonGenuine", sizeof(v)) == 0)
 656                 return (false);
 657         fprintf(stderr, "Unknown cpu vendor \"%s\"\n", (const char *)v);
 658         exit(1);
 659 }
 660
 661 static int
 662 get_all_registers(struct vcpu *vcpu, int vcpuid)
 663 {
 664         uint64_t cr0, cr2, cr3, cr4, dr0, dr1, dr2, dr3, dr6, dr7;
 665         uint64_t rsp, rip, rflags, efer;
 666         uint64_t rax, rbx, rcx, rdx, rsi, rdi, rbp;
 667         uint64_t r8, r9, r10, r11, r12, r13, r14, r15;
 668         int error = 0;
 669
 670         if (!error && (get_efer || get_all)) {
 671                 error = vm_get_register(vcpu, VM_REG_GUEST_EFER, &efer);
 672                 if (error == 0)
 673                         printf("efer[%d]\t\t0x%016lx\n", vcpuid, efer);
 674         }
 675
 676         if (!error && (get_cr0 || get_all)) {
 677                 error = vm_get_register(vcpu, VM_REG_GUEST_CR0, &cr0);
 678                 if (error == 0)
 679                         printf("cr0[%d]\t\t0x%016lx\n", vcpuid, cr0);
 680         }
 681
 682         if (!error && (get_cr2 || get_all)) {
 683                 error = vm_get_register(vcpu, VM_REG_GUEST_CR2, &cr2);
 684                 if (error == 0)
 685                         printf("cr2[%d]\t\t0x%016lx\n", vcpuid, cr2);
 686         }
 687
 688         if (!error && (get_cr3 || get_all)) {
 689                 error = vm_get_register(vcpu, VM_REG_GUEST_CR3, &cr3);
 690                 if (error == 0)
 691                         printf("cr3[%d]\t\t0x%016lx\n", vcpuid, cr3);
 692         }
 693
 694         if (!error && (get_cr4 || get_all)) {
 695                 error = vm_get_register(vcpu, VM_REG_GUEST_CR4, &cr4);
 696                 if (error == 0)
 697                         printf("cr4[%d]\t\t0x%016lx\n", vcpuid, cr4);
 698         }
 699
 700         if (!error && (get_dr0 || get_all)) {
 701                 error = vm_get_register(vcpu, VM_REG_GUEST_DR0, &dr0);
 702                 if (error == 0)
 703                         printf("dr0[%d]\t\t0x%016lx\n", vcpuid, dr0);
 704         }
 705
 706         if (!error && (get_dr1 || get_all)) {
 707                 error = vm_get_register(vcpu, VM_REG_GUEST_DR1, &dr1);
 708                 if (error == 0)
 709                         printf("dr1[%d]\t\t0x%016lx\n", vcpuid, dr1);
 710         }
 711
 712         if (!error && (get_dr2 || get_all)) {
 713                 error = vm_get_register(vcpu, VM_REG_GUEST_DR2, &dr2);
 714                 if (error == 0)
 715                         printf("dr2[%d]\t\t0x%016lx\n", vcpuid, dr2);
 716         }
 717
 718         if (!error && (get_dr3 || get_all)) {
 719                 error = vm_get_register(vcpu, VM_REG_GUEST_DR3, &dr3);
 720                 if (error == 0)
 721                         printf("dr3[%d]\t\t0x%016lx\n", vcpuid, dr3);
 722         }
 723
 724         if (!error && (get_dr6 || get_all)) {
 725                 error = vm_get_register(vcpu, VM_REG_GUEST_DR6, &dr6);
 726                 if (error == 0)
 727                         printf("dr6[%d]\t\t0x%016lx\n", vcpuid, dr6);
 728         }
 729
 730         if (!error && (get_dr7 || get_all)) {
 731                 error = vm_get_register(vcpu, VM_REG_GUEST_DR7, &dr7);
 732                 if (error == 0)
 733                         printf("dr7[%d]\t\t0x%016lx\n", vcpuid, dr7);
 734         }
 735
 736         if (!error && (get_rsp || get_all)) {
 737                 error = vm_get_register(vcpu, VM_REG_GUEST_RSP, &rsp);
 738                 if (error == 0)
 739                         printf("rsp[%d]\t\t0x%016lx\n", vcpuid, rsp);
 740         }
 741
 742         if (!error && (get_rip || get_all)) {
 743                 error = vm_get_register(vcpu, VM_REG_GUEST_RIP, &rip);
 744                 if (error == 0)
 745                         printf("rip[%d]\t\t0x%016lx\n", vcpuid, rip);
 746         }
 747
 748         if (!error && (get_rax || get_all)) {
 749                 error = vm_get_register(vcpu, VM_REG_GUEST_RAX, &rax);
 750                 if (error == 0)
 751                         printf("rax[%d]\t\t0x%016lx\n", vcpuid, rax);
 752         }
 753
 754         if (!error && (get_rbx || get_all)) {
 755                 error = vm_get_register(vcpu, VM_REG_GUEST_RBX, &rbx);
 756                 if (error == 0)
 757                         printf("rbx[%d]\t\t0x%016lx\n", vcpuid, rbx);
 758         }
 759
 760         if (!error && (get_rcx || get_all)) {
 761                 error = vm_get_register(vcpu, VM_REG_GUEST_RCX, &rcx);
 762                 if (error == 0)
 763                         printf("rcx[%d]\t\t0x%016lx\n", vcpuid, rcx);
 764         }
 765
 766         if (!error && (get_rdx || get_all)) {
 767                 error = vm_get_register(vcpu, VM_REG_GUEST_RDX, &rdx);
 768                 if (error == 0)
 769                         printf("rdx[%d]\t\t0x%016lx\n", vcpuid, rdx);
 770         }
 771
 772         if (!error && (get_rsi || get_all)) {
 773                 error = vm_get_register(vcpu, VM_REG_GUEST_RSI, &rsi);
 774                 if (error == 0)
 775                         printf("rsi[%d]\t\t0x%016lx\n", vcpuid, rsi);
 776         }
 777
 778         if (!error && (get_rdi || get_all)) {
 779                 error = vm_get_register(vcpu, VM_REG_GUEST_RDI, &rdi);
 780                 if (error == 0)
 781                         printf("rdi[%d]\t\t0x%016lx\n", vcpuid, rdi);
 782         }
 783
 784         if (!error && (get_rbp || get_all)) {
 785                 error = vm_get_register(vcpu, VM_REG_GUEST_RBP, &rbp);
 786                 if (error == 0)
 787                         printf("rbp[%d]\t\t0x%016lx\n", vcpuid, rbp);
 788         }
 789
 790         if (!error && (get_r8 || get_all)) {
 791                 error = vm_get_register(vcpu, VM_REG_GUEST_R8, &r8);
 792                 if (error == 0)
 793                         printf("r8[%d]\t\t0x%016lx\n", vcpuid, r8);
 794         }
 795
 796         if (!error && (get_r9 || get_all)) {
 797                 error = vm_get_register(vcpu, VM_REG_GUEST_R9, &r9);
 798                 if (error == 0)
 799                         printf("r9[%d]\t\t0x%016lx\n", vcpuid, r9);
 800         }
 801
 802         if (!error && (get_r10 || get_all)) {
 803                 error = vm_get_register(vcpu, VM_REG_GUEST_R10, &r10);
 804                 if (error == 0)
 805                         printf("r10[%d]\t\t0x%016lx\n", vcpuid, r10);
 806         }
 807
 808         if (!error && (get_r11 || get_all)) {
 809                 error = vm_get_register(vcpu, VM_REG_GUEST_R11, &r11);
 810                 if (error == 0)
 811                         printf("r11[%d]\t\t0x%016lx\n", vcpuid, r11);
 812         }
 813
 814         if (!error && (get_r12 || get_all)) {
 815                 error = vm_get_register(vcpu, VM_REG_GUEST_R12, &r12);
 816                 if (error == 0)
 817                         printf("r12[%d]\t\t0x%016lx\n", vcpuid, r12);
 818         }
 819
 820         if (!error && (get_r13 || get_all)) {
 821                 error = vm_get_register(vcpu, VM_REG_GUEST_R13, &r13);
 822                 if (error == 0)
 823                         printf("r13[%d]\t\t0x%016lx\n", vcpuid, r13);
 824         }
 825
 826         if (!error && (get_r14 || get_all)) {
 827                 error = vm_get_register(vcpu, VM_REG_GUEST_R14, &r14);
 828                 if (error == 0)
 829                         printf("r14[%d]\t\t0x%016lx\n", vcpuid, r14);
 830         }
 831
 832         if (!error && (get_r15 || get_all)) {
 833                 error = vm_get_register(vcpu, VM_REG_GUEST_R15, &r15);
 834                 if (error == 0)
 835                         printf("r15[%d]\t\t0x%016lx\n", vcpuid, r15);
 836         }
 837
 838         if (!error && (get_rflags || get_all)) {
 839                 error = vm_get_register(vcpu, VM_REG_GUEST_RFLAGS,
 840                                         &rflags);
 841                 if (error == 0)
 842                         printf("rflags[%d]\t0x%016lx\n", vcpuid, rflags);
 843         }
 844
 845         return (error);
 846 }
 847
 848 static int
 849 get_all_segments(struct vcpu *vcpu, int vcpuid)
 850 {
 851         uint64_t cs, ds, es, fs, gs, ss, tr, ldtr;
 852         int error = 0;
 853
 854         if (!error && (get_desc_ds || get_all)) {
 855                 error = vm_get_desc(vcpu, VM_REG_GUEST_DS,
 856                                    &desc_base, &desc_limit, &desc_access);
 857                 if (error == 0) {
 858                         printf("ds desc[%d]\t0x%016lx/0x%08x/0x%08x\n",
 859                               vcpuid, desc_base, desc_limit, desc_access);
 860                 }
 861         }
 862
 863         if (!error && (get_desc_es || get_all)) {
 864                 error = vm_get_desc(vcpu, VM_REG_GUEST_ES,
 865                                     &desc_base, &desc_limit, &desc_access);
 866                 if (error == 0) {
 867                         printf("es desc[%d]\t0x%016lx/0x%08x/0x%08x\n",
 868                                vcpuid, desc_base, desc_limit, desc_access);
 869                 }
 870         }
 871
 872         if (!error && (get_desc_fs || get_all)) {
 873                 error = vm_get_desc(vcpu, VM_REG_GUEST_FS,
 874                                     &desc_base, &desc_limit, &desc_access);
 875                 if (error == 0) {
 876                         printf("fs desc[%d]\t0x%016lx/0x%08x/0x%08x\n",
 877                                vcpuid, desc_base, desc_limit, desc_access);
 878                 }
 879         }
 880
 881         if (!error && (get_desc_gs || get_all)) {
 882                 error = vm_get_desc(vcpu, VM_REG_GUEST_GS,
 883                                     &desc_base, &desc_limit, &desc_access);
 884                 if (error == 0) {
 885                         printf("gs desc[%d]\t0x%016lx/0x%08x/0x%08x\n",
 886                                vcpuid, desc_base, desc_limit, desc_access);
 887                 }
 888         }
 889
 890         if (!error && (get_desc_ss || get_all)) {
 891                 error = vm_get_desc(vcpu, VM_REG_GUEST_SS,
 892                                     &desc_base, &desc_limit, &desc_access);
 893                 if (error == 0) {
 894                         printf("ss desc[%d]\t0x%016lx/0x%08x/0x%08x\n",
 895                                vcpuid, desc_base, desc_limit, desc_access);
 896                 }
 897         }
 898
 899         if (!error && (get_desc_cs || get_all)) {
 900                 error = vm_get_desc(vcpu, VM_REG_GUEST_CS,
 901                                     &desc_base, &desc_limit, &desc_access);
 902                 if (error == 0) {
 903                         printf("cs desc[%d]\t0x%016lx/0x%08x/0x%08x\n",
 904                                vcpuid, desc_base, desc_limit, desc_access);
 905                 }
 906         }
 907
 908         if (!error && (get_desc_tr || get_all)) {
 909                 error = vm_get_desc(vcpu, VM_REG_GUEST_TR,
 910                                     &desc_base, &desc_limit, &desc_access);
 911                 if (error == 0) {
 912                         printf("tr desc[%d]\t0x%016lx/0x%08x/0x%08x\n",
 913                                vcpuid, desc_base, desc_limit, desc_access);
 914                 }
 915         }
 916
 917         if (!error && (get_desc_ldtr || get_all)) {
 918                 error = vm_get_desc(vcpu, VM_REG_GUEST_LDTR,
 919                                     &desc_base, &desc_limit, &desc_access);
 920                 if (error == 0) {
 921                         printf("ldtr desc[%d]\t0x%016lx/0x%08x/0x%08x\n",
 922                                vcpuid, desc_base, desc_limit, desc_access);
 923                 }
 924         }
 925
 926         if (!error && (get_desc_gdtr || get_all)) {
 927                 error = vm_get_desc(vcpu, VM_REG_GUEST_GDTR,
 928                                     &desc_base, &desc_limit, &desc_access);
 929                 if (error == 0) {
 930                         printf("gdtr[%d]\t\t0x%016lx/0x%08x\n",
 931                                vcpuid, desc_base, desc_limit);
 932                 }
 933         }
 934
 935         if (!error && (get_desc_idtr || get_all)) {
 936                 error = vm_get_desc(vcpu, VM_REG_GUEST_IDTR,
 937                                     &desc_base, &desc_limit, &desc_access);
 938                 if (error == 0) {
 939                         printf("idtr[%d]\t\t0x%016lx/0x%08x\n",
 940                                vcpuid, desc_base, desc_limit);
 941                 }
 942         }
 943
 944         if (!error && (get_cs || get_all)) {
 945                 error = vm_get_register(vcpu, VM_REG_GUEST_CS, &cs);
 946                 if (error == 0)
 947                         printf("cs[%d]\t\t0x%04lx\n", vcpuid, cs);
 948         }
 949
 950         if (!error && (get_ds || get_all)) {
 951                 error = vm_get_register(vcpu, VM_REG_GUEST_DS, &ds);
 952                 if (error == 0)
 953                         printf("ds[%d]\t\t0x%04lx\n", vcpuid, ds);
 954         }
 955
 956         if (!error && (get_es || get_all)) {
 957                 error = vm_get_register(vcpu, VM_REG_GUEST_ES, &es);
 958                 if (error == 0)
 959                         printf("es[%d]\t\t0x%04lx\n", vcpuid, es);
 960         }
 961
 962         if (!error && (get_fs || get_all)) {
 963                 error = vm_get_register(vcpu, VM_REG_GUEST_FS, &fs);
 964                 if (error == 0)
 965                         printf("fs[%d]\t\t0x%04lx\n", vcpuid, fs);
 966         }
 967
 968         if (!error && (get_gs || get_all)) {
 969                 error = vm_get_register(vcpu, VM_REG_GUEST_GS, &gs);
 970                 if (error == 0)
 971                         printf("gs[%d]\t\t0x%04lx\n", vcpuid, gs);
 972         }
 973
 974         if (!error && (get_ss || get_all)) {
 975                 error = vm_get_register(vcpu, VM_REG_GUEST_SS, &ss);
 976                 if (error == 0)
 977                         printf("ss[%d]\t\t0x%04lx\n", vcpuid, ss);
 978         }
 979
 980         if (!error && (get_tr || get_all)) {
 981                 error = vm_get_register(vcpu, VM_REG_GUEST_TR, &tr);
 982                 if (error == 0)
 983                         printf("tr[%d]\t\t0x%04lx\n", vcpuid, tr);
 984         }
 985
 986         if (!error && (get_ldtr || get_all)) {
 987                 error = vm_get_register(vcpu, VM_REG_GUEST_LDTR, &ldtr);
 988                 if (error == 0)
 989                         printf("ldtr[%d]\t\t0x%04lx\n", vcpuid, ldtr);
 990         }
 991
 992         return (error);
 993 }
 994
 995 static int
 996 get_misc_vmcs(struct vcpu *vcpu, int vcpuid)
 997 {
 998         uint64_t ctl, cr0, cr3, cr4, rsp, rip, pat, addr, u64;
 999         int error = 0;
1000
1001         if (!error && (get_cr0_mask || get_all)) {
1002                 uint64_t cr0mask;
1003                 error = vm_get_vmcs_field(vcpu, VMCS_CR0_MASK, &cr0mask);
1004                 if (error == 0)
1005                         printf("cr0_mask[%d]\t\t0x%016lx\n", vcpuid, cr0mask);
1006         }
1007
1008         if (!error && (get_cr0_shadow || get_all)) {
1009                 uint64_t cr0shadow;
1010                 error = vm_get_vmcs_field(vcpu, VMCS_CR0_SHADOW,
1011                                           &cr0shadow);
1012                 if (error == 0)
1013                         printf("cr0_shadow[%d]\t\t0x%016lx\n", vcpuid, cr0shadow);
1014         }
1015
1016         if (!error && (get_cr4_mask || get_all)) {
1017                 uint64_t cr4mask;
1018                 error = vm_get_vmcs_field(vcpu, VMCS_CR4_MASK, &cr4mask);
1019                 if (error == 0)
1020                         printf("cr4_mask[%d]\t\t0x%016lx\n", vcpuid, cr4mask);
1021         }
1022
1023         if (!error && (get_cr4_shadow || get_all)) {
1024                 uint64_t cr4shadow;
1025                 error = vm_get_vmcs_field(vcpu, VMCS_CR4_SHADOW,
1026                                           &cr4shadow);
1027                 if (error == 0)
1028                         printf("cr4_shadow[%d]\t\t0x%016lx\n", vcpuid, cr4shadow);
1029         }
1030
1031         if (!error && (get_cr3_targets || get_all)) {
1032                 uint64_t target_count, target_addr;
1033                 error = vm_get_vmcs_field(vcpu, VMCS_CR3_TARGET_COUNT,
1034                                           &target_count);
1035                 if (error == 0) {
1036                         printf("cr3_target_count[%d]\t0x%016lx\n",
1037                                 vcpuid, target_count);
1038                 }
1039
1040                 error = vm_get_vmcs_field(vcpu, VMCS_CR3_TARGET0,
1041                                           &target_addr);
1042                 if (error == 0) {
1043                         printf("cr3_target0[%d]\t\t0x%016lx\n",
1044                                 vcpuid, target_addr);
1045                 }
1046
1047                 error = vm_get_vmcs_field(vcpu, VMCS_CR3_TARGET1,
1048                                           &target_addr);
1049                 if (error == 0) {
1050                         printf("cr3_target1[%d]\t\t0x%016lx\n",
1051                                 vcpuid, target_addr);
1052                 }
1053
1054                 error = vm_get_vmcs_field(vcpu, VMCS_CR3_TARGET2,
1055                                           &target_addr);
1056                 if (error == 0) {
1057                         printf("cr3_target2[%d]\t\t0x%016lx\n",
1058                                 vcpuid, target_addr);
1059                 }
1060
1061                 error = vm_get_vmcs_field(vcpu, VMCS_CR3_TARGET3,
1062                                           &target_addr);
1063                 if (error == 0) {
1064                         printf("cr3_target3[%d]\t\t0x%016lx\n",
1065                                 vcpuid, target_addr);
1066                 }
1067         }
1068
1069         if (!error && (get_pinbased_ctls || get_all)) {
1070                 error = vm_get_vmcs_field(vcpu, VMCS_PIN_BASED_CTLS, &ctl);
1071                 if (error == 0)
1072                         printf("pinbased_ctls[%d]\t0x%016lx\n", vcpuid, ctl);
1073         }
1074
1075         if (!error && (get_procbased_ctls || get_all)) {
1076                 error = vm_get_vmcs_field(vcpu,
1077                                           VMCS_PRI_PROC_BASED_CTLS, &ctl);
1078                 if (error == 0)
1079                         printf("procbased_ctls[%d]\t0x%016lx\n", vcpuid, ctl);
1080         }
1081
1082         if (!error && (get_procbased_ctls2 || get_all)) {
1083                 error = vm_get_vmcs_field(vcpu,
1084                                           VMCS_SEC_PROC_BASED_CTLS, &ctl);
1085                 if (error == 0)
1086                         printf("procbased_ctls2[%d]\t0x%016lx\n", vcpuid, ctl);
1087         }
1088
1089         if (!error && (get_vmcs_gla || get_all)) {
1090                 error = vm_get_vmcs_field(vcpu,
1091                                           VMCS_GUEST_LINEAR_ADDRESS, &u64);
1092                 if (error == 0)
1093                         printf("gla[%d]\t\t0x%016lx\n", vcpuid, u64);
1094         }
1095
1096         if (!error && (get_vmcs_gpa || get_all)) {
1097                 error = vm_get_vmcs_field(vcpu,
1098                                           VMCS_GUEST_PHYSICAL_ADDRESS, &u64);
1099                 if (error == 0)
1100                         printf("gpa[%d]\t\t0x%016lx\n", vcpuid, u64);
1101         }
1102
1103         if (!error && (get_vmcs_entry_interruption_info ||
1104                 get_all)) {
1105                 error = vm_get_vmcs_field(vcpu, VMCS_ENTRY_INTR_INFO,&u64);
1106                 if (error == 0) {
1107                         printf("entry_interruption_info[%d]\t0x%016lx\n",
1108                                 vcpuid, u64);
1109                 }
1110         }
1111
1112         if (!error && (get_tpr_threshold || get_all)) {
1113                 uint64_t threshold;
1114                 error = vm_get_vmcs_field(vcpu, VMCS_TPR_THRESHOLD,
1115                                           &threshold);
1116                 if (error == 0)
1117                         printf("tpr_threshold[%d]\t0x%016lx\n", vcpuid, threshold);
1118         }
1119
1120         if (!error && (get_inst_err || get_all)) {
1121                 uint64_t insterr;
1122                 error = vm_get_vmcs_field(vcpu, VMCS_INSTRUCTION_ERROR,
1123                                           &insterr);
1124                 if (error == 0) {
1125                         printf("instruction_error[%d]\t0x%016lx\n",
1126                                 vcpuid, insterr);
1127                 }
1128         }
1129
1130         if (!error && (get_exit_ctls || get_all)) {
1131                 error = vm_get_vmcs_field(vcpu, VMCS_EXIT_CTLS, &ctl);
1132                 if (error == 0)
1133                         printf("exit_ctls[%d]\t\t0x%016lx\n", vcpuid, ctl);
1134         }
1135
1136         if (!error && (get_entry_ctls || get_all)) {
1137                 error = vm_get_vmcs_field(vcpu, VMCS_ENTRY_CTLS, &ctl);
1138                 if (error == 0)
1139                         printf("entry_ctls[%d]\t\t0x%016lx\n", vcpuid, ctl);
1140         }
1141
1142         if (!error && (get_host_pat || get_all)) {
1143                 error = vm_get_vmcs_field(vcpu, VMCS_HOST_IA32_PAT, &pat);
1144                 if (error == 0)
1145                         printf("host_pat[%d]\t\t0x%016lx\n", vcpuid, pat);
1146         }
1147
1148         if (!error && (get_host_cr0 || get_all)) {
1149                 error = vm_get_vmcs_field(vcpu, VMCS_HOST_CR0, &cr0);
1150                 if (error == 0)
1151                         printf("host_cr0[%d]\t\t0x%016lx\n", vcpuid, cr0);
1152         }
1153
1154         if (!error && (get_host_cr3 || get_all)) {
1155                 error = vm_get_vmcs_field(vcpu, VMCS_HOST_CR3, &cr3);
1156                 if (error == 0)
1157                         printf("host_cr3[%d]\t\t0x%016lx\n", vcpuid, cr3);
1158         }
1159
1160         if (!error && (get_host_cr4 || get_all)) {
1161                 error = vm_get_vmcs_field(vcpu, VMCS_HOST_CR4, &cr4);
1162                 if (error == 0)
1163                         printf("host_cr4[%d]\t\t0x%016lx\n", vcpuid, cr4);
1164         }
1165
1166         if (!error && (get_host_rip || get_all)) {
1167                 error = vm_get_vmcs_field(vcpu, VMCS_HOST_RIP, &rip);
1168                 if (error == 0)
1169                         printf("host_rip[%d]\t\t0x%016lx\n", vcpuid, rip);
1170         }
1171
1172         if (!error && (get_host_rsp || get_all)) {
1173                 error = vm_get_vmcs_field(vcpu, VMCS_HOST_RSP, &rsp);
1174                 if (error == 0)
1175                         printf("host_rsp[%d]\t\t0x%016lx\n", vcpuid, rsp);
1176         }
1177
1178         if (!error && (get_vmcs_link || get_all)) {
1179                 error = vm_get_vmcs_field(vcpu, VMCS_LINK_POINTER, &addr);
1180                 if (error == 0)
1181                         printf("vmcs_pointer[%d]\t0x%016lx\n", vcpuid, addr);
1182         }
1183
1184         if (!error && (get_vmcs_exit_interruption_info || get_all)) {
1185                 error = vm_get_vmcs_field(vcpu, VMCS_EXIT_INTR_INFO, &u64);
1186                 if (error == 0) {
1187                         printf("vmcs_exit_interruption_info[%d]\t0x%016lx\n",
1188                                 vcpuid, u64);
1189                 }
1190         }
1191
1192         if (!error && (get_vmcs_exit_interruption_error || get_all)) {
1193                 error = vm_get_vmcs_field(vcpu, VMCS_EXIT_INTR_ERRCODE,
1194                                           &u64);
1195                 if (error == 0) {
1196                         printf("vmcs_exit_interruption_error[%d]\t0x%016lx\n",
1197                                 vcpuid, u64);
1198                 }
1199         }
1200
1201         if (!error && (get_vmcs_interruptibility || get_all)) {
1202                 error = vm_get_vmcs_field(vcpu,
1203                                           VMCS_GUEST_INTERRUPTIBILITY, &u64);
1204                 if (error == 0) {
1205                         printf("vmcs_guest_interruptibility[%d]\t0x%016lx\n",
1206                                 vcpuid, u64);
1207                 }
1208         }
1209
1210         if (!error && (get_vmcs_exit_inst_length || get_all)) {
1211                 error = vm_get_vmcs_field(vcpu,
1212                     VMCS_EXIT_INSTRUCTION_LENGTH, &u64);
1213                 if (error == 0)
1214                         printf("vmcs_exit_inst_length[%d]\t0x%08x\n", vcpuid,
1215                             (uint32_t)u64);
1216         }
1217
1218         if (!error && (get_vmcs_exit_qualification || get_all)) {
1219                 error = vm_get_vmcs_field(vcpu, VMCS_EXIT_QUALIFICATION,
1220                                           &u64);
1221                 if (error == 0)
1222                         printf("vmcs_exit_qualification[%d]\t0x%016lx\n",
1223                                 vcpuid, u64);
1224         }
1225
1226         return (error);
1227 }
1228
1229 static int
1230 get_misc_vmcb(struct vcpu *vcpu, int vcpuid)
1231 {
1232         uint64_t ctl, addr;
1233         int error = 0;
1234
1235         if (!error && (get_vmcb_intercept || get_all)) {
1236                 error = vm_get_vmcb_field(vcpu, VMCB_OFF_CR_INTERCEPT, 4,
1237                     &ctl);
1238                 if (error == 0)
1239                         printf("cr_intercept[%d]\t0x%08x\n", vcpuid, (int)ctl);
1240
1241                 error = vm_get_vmcb_field(vcpu, VMCB_OFF_DR_INTERCEPT, 4,
1242                     &ctl);
1243                 if (error == 0)
1244                         printf("dr_intercept[%d]\t0x%08x\n", vcpuid, (int)ctl);
1245
1246                 error = vm_get_vmcb_field(vcpu, VMCB_OFF_EXC_INTERCEPT, 4,
1247                     &ctl);
1248                 if (error == 0)
1249                         printf("exc_intercept[%d]\t0x%08x\n", vcpuid, (int)ctl);
1250
1251                 error = vm_get_vmcb_field(vcpu, VMCB_OFF_INST1_INTERCEPT,
1252                     4, &ctl);
1253                 if (error == 0)
1254                         printf("inst1_intercept[%d]\t0x%08x\n", vcpuid, (int)ctl);
1255
1256                 error = vm_get_vmcb_field(vcpu, VMCB_OFF_INST2_INTERCEPT,
1257                     4, &ctl);
1258                 if (error == 0)
1259                         printf("inst2_intercept[%d]\t0x%08x\n", vcpuid, (int)ctl);
1260         }
1261
1262         if (!error && (get_vmcb_tlb_ctrl || get_all)) {
1263                 error = vm_get_vmcb_field(vcpu, VMCB_OFF_TLB_CTRL,
1264                                           4, &ctl);
1265                 if (error == 0)
1266                         printf("TLB ctrl[%d]\t0x%016lx\n", vcpuid, ctl);
1267         }
1268
1269         if (!error && (get_vmcb_exit_details || get_all)) {
1270                 error = vm_get_vmcb_field(vcpu, VMCB_OFF_EXITINFO1,
1271                                           8, &ctl);
1272                 if (error == 0)
1273                         printf("exitinfo1[%d]\t0x%016lx\n", vcpuid, ctl);
1274                 error = vm_get_vmcb_field(vcpu, VMCB_OFF_EXITINFO2,
1275                                           8, &ctl);
1276                 if (error == 0)
1277                         printf("exitinfo2[%d]\t0x%016lx\n", vcpuid, ctl);
1278                 error = vm_get_vmcb_field(vcpu, VMCB_OFF_EXITINTINFO,
1279                                           8, &ctl);
1280                 if (error == 0)
1281                         printf("exitintinfo[%d]\t0x%016lx\n", vcpuid, ctl);
1282         }
1283
1284         if (!error && (get_vmcb_virq || get_all)) {
1285                 error = vm_get_vmcb_field(vcpu, VMCB_OFF_VIRQ,
1286                                           8, &ctl);
1287                 if (error == 0)
1288                         printf("v_irq/tpr[%d]\t0x%016lx\n", vcpuid, ctl);
1289         }
1290
1291         if (!error && (get_apic_access_addr || get_all)) {
1292                 error = vm_get_vmcb_field(vcpu, VMCB_OFF_AVIC_BAR, 8,
1293                                           &addr);
1294                 if (error == 0)
1295                         printf("AVIC apic_bar[%d]\t0x%016lx\n", vcpuid, addr);
1296         }
1297
1298         if (!error && (get_virtual_apic_addr || get_all)) {
1299                 error = vm_get_vmcb_field(vcpu, VMCB_OFF_AVIC_PAGE, 8,
1300                                           &addr);
1301                 if (error == 0)
1302                         printf("AVIC backing page[%d]\t0x%016lx\n", vcpuid, addr);
1303         }
1304
1305         if (!error && (get_avic_table || get_all)) {
1306                 error = vm_get_vmcb_field(vcpu, VMCB_OFF_AVIC_LT, 8,
1307                                           &addr);
1308                 if (error == 0)
1309                         printf("AVIC logical table[%d]\t0x%016lx\n",
1310                                 vcpuid, addr);
1311                 error = vm_get_vmcb_field(vcpu, VMCB_OFF_AVIC_PT, 8,
1312                                           &addr);
1313                 if (error == 0)
1314                         printf("AVIC physical table[%d]\t0x%016lx\n",
1315                                 vcpuid, addr);
1316         }
1317
1318         return (error);
1319 }
1320
1321 static struct option *
1322 setup_options(bool cpu_intel)
1323 {
1324         const struct option common_opts[] = {
1325                 { "vm",         REQ_ARG,        0,      VMNAME },
1326                 { "cpu",        REQ_ARG,        0,      VCPU },
1327                 { "set-mem",    REQ_ARG,        0,      SET_MEM },
1328                 { "set-efer",   REQ_ARG,        0,      SET_EFER },
1329                 { "set-cr0",    REQ_ARG,        0,      SET_CR0 },
1330                 { "set-cr2",    REQ_ARG,        0,      SET_CR2 },
1331                 { "set-cr3",    REQ_ARG,        0,      SET_CR3 },
1332                 { "set-cr4",    REQ_ARG,        0,      SET_CR4 },
1333                 { "set-dr0",    REQ_ARG,        0,      SET_DR0 },
1334                 { "set-dr1",    REQ_ARG,        0,      SET_DR1 },
1335                 { "set-dr2",    REQ_ARG,        0,      SET_DR2 },
1336                 { "set-dr3",    REQ_ARG,        0,      SET_DR3 },
1337                 { "set-dr6",    REQ_ARG,        0,      SET_DR6 },
1338                 { "set-dr7",    REQ_ARG,        0,      SET_DR7 },
1339                 { "set-rsp",    REQ_ARG,        0,      SET_RSP },
1340                 { "set-rip",    REQ_ARG,        0,      SET_RIP },
1341                 { "set-rax",    REQ_ARG,        0,      SET_RAX },
1342                 { "set-rflags", REQ_ARG,        0,      SET_RFLAGS },
1343                 { "desc-base",  REQ_ARG,        0,      DESC_BASE },
1344                 { "desc-limit", REQ_ARG,        0,      DESC_LIMIT },
1345                 { "desc-access",REQ_ARG,        0,      DESC_ACCESS },
1346                 { "set-cs",     REQ_ARG,        0,      SET_CS },
1347                 { "set-ds",     REQ_ARG,        0,      SET_DS },
1348                 { "set-es",     REQ_ARG,        0,      SET_ES },
1349                 { "set-fs",     REQ_ARG,        0,      SET_FS },
1350                 { "set-gs",     REQ_ARG,        0,      SET_GS },
1351                 { "set-ss",     REQ_ARG,        0,      SET_SS },
1352                 { "set-tr",     REQ_ARG,        0,      SET_TR },
1353                 { "set-ldtr",   REQ_ARG,        0,      SET_LDTR },
1354                 { "set-x2apic-state",REQ_ARG,   0,      SET_X2APIC_STATE },
1355                 { "capname",    REQ_ARG,        0,      CAPNAME },
1356                 { "unassign-pptdev", REQ_ARG,   0,      UNASSIGN_PPTDEV },
1357                 { "setcap",     REQ_ARG,        0,      SET_CAP },
1358                 { "get-gpa-pmap", REQ_ARG,      0,      GET_GPA_PMAP },
1359                 { "assert-lapic-lvt", REQ_ARG,  0,      ASSERT_LAPIC_LVT },
1360                 { "get-rtc-time", NO_ARG,       &get_rtc_time,  1 },
1361                 { "set-rtc-time", REQ_ARG,      0,      SET_RTC_TIME },
1362                 { "rtc-nvram-offset", REQ_ARG,  0,      RTC_NVRAM_OFFSET },
1363                 { "get-rtc-nvram", NO_ARG,      &get_rtc_nvram, 1 },
1364                 { "set-rtc-nvram", REQ_ARG,     0,      SET_RTC_NVRAM },
1365                 { "getcap",     NO_ARG,         &getcap,        1 },
1366                 { "get-stats",  NO_ARG,         &get_stats,     1 },
1367                 { "get-desc-ds",NO_ARG,         &get_desc_ds,   1 },
1368                 { "set-desc-ds",NO_ARG,         &set_desc_ds,   1 },
1369                 { "get-desc-es",NO_ARG,         &get_desc_es,   1 },
1370                 { "set-desc-es",NO_ARG,         &set_desc_es,   1 },
1371                 { "get-desc-ss",NO_ARG,         &get_desc_ss,   1 },
1372                 { "set-desc-ss",NO_ARG,         &set_desc_ss,   1 },
1373                 { "get-desc-cs",NO_ARG,         &get_desc_cs,   1 },
1374                 { "set-desc-cs",NO_ARG,         &set_desc_cs,   1 },
1375                 { "get-desc-fs",NO_ARG,         &get_desc_fs,   1 },
1376                 { "set-desc-fs",NO_ARG,         &set_desc_fs,   1 },
1377                 { "get-desc-gs",NO_ARG,         &get_desc_gs,   1 },
1378                 { "set-desc-gs",NO_ARG,         &set_desc_gs,   1 },
1379                 { "get-desc-tr",NO_ARG,         &get_desc_tr,   1 },
1380                 { "set-desc-tr",NO_ARG,         &set_desc_tr,   1 },
1381                 { "set-desc-ldtr", NO_ARG,      &set_desc_ldtr, 1 },
1382                 { "get-desc-ldtr", NO_ARG,      &get_desc_ldtr, 1 },
1383                 { "set-desc-gdtr", NO_ARG,      &set_desc_gdtr, 1 },
1384                 { "get-desc-gdtr", NO_ARG,      &get_desc_gdtr, 1 },
1385                 { "set-desc-idtr", NO_ARG,      &set_desc_idtr, 1 },
1386                 { "get-desc-idtr", NO_ARG,      &get_desc_idtr, 1 },
1387                 { "get-memmap", NO_ARG,         &get_memmap,    1 },
1388                 { "get-memseg", NO_ARG,         &get_memseg,    1 },
1389                 { "get-efer",   NO_ARG,         &get_efer,      1 },
1390                 { "get-cr0",    NO_ARG,         &get_cr0,       1 },
1391                 { "get-cr2",    NO_ARG,         &get_cr2,       1 },
1392                 { "get-cr3",    NO_ARG,         &get_cr3,       1 },
1393                 { "get-cr4",    NO_ARG,         &get_cr4,       1 },
1394                 { "get-dr0",    NO_ARG,         &get_dr0,       1 },
1395                 { "get-dr1",    NO_ARG,         &get_dr1,       1 },
1396                 { "get-dr2",    NO_ARG,         &get_dr2,       1 },
1397                 { "get-dr3",    NO_ARG,         &get_dr3,       1 },
1398                 { "get-dr6",    NO_ARG,         &get_dr6,       1 },
1399                 { "get-dr7",    NO_ARG,         &get_dr7,       1 },
1400                 { "get-rsp",    NO_ARG,         &get_rsp,       1 },
1401                 { "get-rip",    NO_ARG,         &get_rip,       1 },
1402                 { "get-rax",    NO_ARG,         &get_rax,       1 },
1403                 { "get-rbx",    NO_ARG,         &get_rbx,       1 },
1404                 { "get-rcx",    NO_ARG,         &get_rcx,       1 },
1405                 { "get-rdx",    NO_ARG,         &get_rdx,       1 },
1406                 { "get-rsi",    NO_ARG,         &get_rsi,       1 },
1407                 { "get-rdi",    NO_ARG,         &get_rdi,       1 },
1408                 { "get-rbp",    NO_ARG,         &get_rbp,       1 },
1409                 { "get-r8",     NO_ARG,         &get_r8,        1 },
1410                 { "get-r9",     NO_ARG,         &get_r9,        1 },
1411                 { "get-r10",    NO_ARG,         &get_r10,       1 },
1412                 { "get-r11",    NO_ARG,         &get_r11,       1 },
1413                 { "get-r12",    NO_ARG,         &get_r12,       1 },
1414                 { "get-r13",    NO_ARG,         &get_r13,       1 },
1415                 { "get-r14",    NO_ARG,         &get_r14,       1 },
1416                 { "get-r15",    NO_ARG,         &get_r15,       1 },
1417                 { "get-rflags", NO_ARG,         &get_rflags,    1 },
1418                 { "get-cs",     NO_ARG,         &get_cs,        1 },
1419                 { "get-ds",     NO_ARG,         &get_ds,        1 },
1420                 { "get-es",     NO_ARG,         &get_es,        1 },
1421                 { "get-fs",     NO_ARG,         &get_fs,        1 },
1422                 { "get-gs",     NO_ARG,         &get_gs,        1 },
1423                 { "get-ss",     NO_ARG,         &get_ss,        1 },
1424                 { "get-tr",     NO_ARG,         &get_tr,        1 },
1425                 { "get-ldtr",   NO_ARG,         &get_ldtr,      1 },
1426                 { "get-eptp",   NO_ARG,         &get_eptp,      1 },
1427                 { "get-exception-bitmap",
1428                                         NO_ARG, &get_exception_bitmap,  1 },
1429                 { "get-io-bitmap-address",
1430                                         NO_ARG, &get_io_bitmap,         1 },
1431                 { "get-tsc-offset",     NO_ARG, &get_tsc_offset,        1 },
1432                 { "get-msr-bitmap",
1433                                         NO_ARG, &get_msr_bitmap,        1 },
1434                 { "get-msr-bitmap-address",
1435                                         NO_ARG, &get_msr_bitmap_address, 1 },
1436                 { "get-guest-pat",      NO_ARG, &get_guest_pat,         1 },
1437                 { "get-guest-sysenter",
1438                                         NO_ARG, &get_guest_sysenter,    1 },
1439                 { "get-exit-reason",
1440                                         NO_ARG, &get_exit_reason,       1 },
1441                 { "get-x2apic-state",   NO_ARG, &get_x2apic_state,      1 },
1442                 { "get-all",            NO_ARG, &get_all,               1 },
1443                 { "run",                NO_ARG, &run,                   1 },
1444                 { "create",             NO_ARG, &create,                1 },
1445                 { "destroy",            NO_ARG, &destroy,               1 },
1446                 { "inject-nmi",         NO_ARG, &inject_nmi,            1 },
1447                 { "force-reset",        NO_ARG, &force_reset,           1 },
1448                 { "force-poweroff",     NO_ARG, &force_poweroff,        1 },
1449                 { "get-active-cpus",    NO_ARG, &get_active_cpus,       1 },
1450                 { "get-debug-cpus",     NO_ARG, &get_debug_cpus,        1 },
1451                 { "get-suspended-cpus", NO_ARG, &get_suspended_cpus,    1 },
1452                 { "get-intinfo",        NO_ARG, &get_intinfo,           1 },
1453                 { "get-cpu-topology",   NO_ARG, &get_cpu_topology,      1 },
1454 #ifdef BHYVE_SNAPSHOT
1455                 { "checkpoint",         REQ_ARG, 0,     SET_CHECKPOINT_FILE},
1456                 { "suspend",            REQ_ARG, 0,     SET_SUSPEND_FILE},
1457 #endif
1458         };
1459
1460         const struct option intel_opts[] = {
1461                 { "get-vmcs-pinbased-ctls",
1462                                 NO_ARG,         &get_pinbased_ctls, 1 },
1463                 { "get-vmcs-procbased-ctls",
1464                                 NO_ARG,         &get_procbased_ctls, 1 },
1465                 { "get-vmcs-procbased-ctls2",
1466                                 NO_ARG,         &get_procbased_ctls2, 1 },
1467                 { "get-vmcs-guest-linear-address",
1468                                 NO_ARG,         &get_vmcs_gla,  1 },
1469                 { "get-vmcs-guest-physical-address",
1470                                 NO_ARG,         &get_vmcs_gpa,  1 },
1471                 { "get-vmcs-entry-interruption-info",
1472                                 NO_ARG, &get_vmcs_entry_interruption_info, 1},
1473                 { "get-vmcs-cr0-mask", NO_ARG,  &get_cr0_mask,  1 },
1474                 { "get-vmcs-cr0-shadow", NO_ARG,&get_cr0_shadow, 1 },
1475                 { "get-vmcs-cr4-mask",          NO_ARG, &get_cr4_mask,    1 },
1476                 { "get-vmcs-cr4-shadow",        NO_ARG, &get_cr4_shadow,  1 },
1477                 { "get-vmcs-cr3-targets",       NO_ARG, &get_cr3_targets, 1 },
1478                 { "get-vmcs-tpr-threshold",
1479                                         NO_ARG, &get_tpr_threshold, 1 },
1480                 { "get-vmcs-vpid",      NO_ARG, &get_vpid_asid,     1 },
1481                 { "get-vmcs-exit-ctls", NO_ARG, &get_exit_ctls,     1 },
1482                 { "get-vmcs-entry-ctls",
1483                                         NO_ARG, &get_entry_ctls, 1 },
1484                 { "get-vmcs-instruction-error",
1485                                         NO_ARG, &get_inst_err,  1 },
1486                 { "get-vmcs-host-pat",  NO_ARG, &get_host_pat,  1 },
1487                 { "get-vmcs-host-cr0",
1488                                         NO_ARG, &get_host_cr0,  1 },
1489                 { "get-vmcs-exit-qualification",
1490                                 NO_ARG, &get_vmcs_exit_qualification, 1 },
1491                 { "get-vmcs-exit-inst-length",
1492                                 NO_ARG, &get_vmcs_exit_inst_length, 1 },
1493                 { "get-vmcs-interruptibility",
1494                                 NO_ARG, &get_vmcs_interruptibility, 1 },
1495                 { "get-vmcs-exit-interruption-error",
1496                                 NO_ARG, &get_vmcs_exit_interruption_error, 1 },
1497                 { "get-vmcs-exit-interruption-info",
1498                                 NO_ARG, &get_vmcs_exit_interruption_info, 1 },
1499                 { "get-vmcs-link",      NO_ARG,         &get_vmcs_link, 1 },
1500                 { "get-vmcs-host-cr3",
1501                                         NO_ARG,         &get_host_cr3,  1 },
1502                 { "get-vmcs-host-cr4",
1503                                 NO_ARG,         &get_host_cr4,  1 },
1504                 { "get-vmcs-host-rip",
1505                                 NO_ARG,         &get_host_rip,  1 },
1506                 { "get-vmcs-host-rsp",
1507                                 NO_ARG,         &get_host_rsp,  1 },
1508                 { "get-apic-access-address",
1509                                 NO_ARG,         &get_apic_access_addr, 1},
1510                 { "get-virtual-apic-address",
1511                                 NO_ARG,         &get_virtual_apic_addr, 1}
1512         };
1513
1514         const struct option amd_opts[] = {
1515                 { "get-vmcb-intercepts",
1516                                 NO_ARG, &get_vmcb_intercept,    1 },
1517                 { "get-vmcb-asid",
1518                                 NO_ARG, &get_vpid_asid,         1 },
1519                 { "get-vmcb-exit-details",
1520                                 NO_ARG, &get_vmcb_exit_details, 1 },
1521                 { "get-vmcb-tlb-ctrl",
1522                                 NO_ARG, &get_vmcb_tlb_ctrl,     1 },
1523                 { "get-vmcb-virq",
1524                                 NO_ARG, &get_vmcb_virq,         1 },
1525                 { "get-avic-apic-bar",
1526                                 NO_ARG, &get_apic_access_addr,  1 },
1527                 { "get-avic-backing-page",
1528                                 NO_ARG, &get_virtual_apic_addr, 1 },
1529                 { "get-avic-table",
1530                                 NO_ARG, &get_avic_table,        1 }
1531         };
1532
1533         const struct option null_opt = {
1534                 NULL, 0, NULL, 0
1535         };
1536
1537         struct option *all_opts;
1538         char *cp;
1539         int optlen;
1540
1541         optlen = sizeof(common_opts);
1542
1543         if (cpu_intel)
1544                 optlen += sizeof(intel_opts);
1545         else
1546                 optlen += sizeof(amd_opts);
1547
1548         optlen += sizeof(null_opt);
1549
1550         all_opts = malloc(optlen);
1551
1552         cp = (char *)all_opts;
1553         memcpy(cp, common_opts, sizeof(common_opts));
1554         cp += sizeof(common_opts);
1555
1556         if (cpu_intel) {
1557                 memcpy(cp, intel_opts, sizeof(intel_opts));
1558                 cp += sizeof(intel_opts);
1559         } else {
1560                 memcpy(cp, amd_opts, sizeof(amd_opts));
1561                 cp += sizeof(amd_opts);
1562         }
1563
1564         memcpy(cp, &null_opt, sizeof(null_opt));
1565         cp += sizeof(null_opt);
1566
1567         return (all_opts);
1568 }
1569
1570 static const char *
1571 wday_str(int idx)
1572 {
1573         static const char *weekdays[] = {
1574                 "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"
1575         };
1576
1577         if (idx >= 0 && idx < 7)
1578                 return (weekdays[idx]);
1579         else
1580                 return ("UNK");
1581 }
1582
1583 static const char *
1584 mon_str(int idx)
1585 {
1586         static const char *months[] = {
1587                 "Jan", "Feb", "Mar", "Apr", "May", "Jun",
1588                 "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
1589         };
1590
1591         if (idx >= 0 && idx < 12)
1592                 return (months[idx]);
1593         else
1594                 return ("UNK");
1595 }
1596
1597 static int
1598 show_memmap(struct vmctx *ctx)
1599 {
1600         char name[SPECNAMELEN + 1], numbuf[8];
1601         vm_ooffset_t segoff;
1602         vm_paddr_t gpa;
1603         size_t maplen, seglen;
1604         int error, flags, prot, segid, delim;
1605
1606         printf("Address     Length      Segment     Offset      ");
1607         printf("Prot  Flags\n");
1608
1609         gpa = 0;
1610         while (1) {
1611                 error = vm_mmap_getnext(ctx, &gpa, &segid, &segoff, &maplen,
1612                     &prot, &flags);
1613                 if (error)
1614                         return (errno == ENOENT ? 0 : error);
1615
1616                 error = vm_get_memseg(ctx, segid, &seglen, name, sizeof(name));
1617                 if (error)
1618                         return (error);
1619
1620                 printf("%-12lX", gpa);
1621                 humanize_number(numbuf, sizeof(numbuf), maplen, "B",
1622                     HN_AUTOSCALE, HN_NOSPACE);
1623                 printf("%-12s", numbuf);
1624
1625                 printf("%-12s", name[0] ? name : "sysmem");
1626                 printf("%-12lX", segoff);
1627                 printf("%c%c%c   ", prot & PROT_READ ? 'R' : '-',
1628                     prot & PROT_WRITE ? 'W' : '-',
1629                     prot & PROT_EXEC ? 'X' : '-');
1630
1631                 delim = '\0';
1632                 if (flags & VM_MEMMAP_F_WIRED) {
1633                         printf("%cwired", delim);
1634                         delim = '/';
1635                 }
1636                 if (flags & VM_MEMMAP_F_IOMMU) {
1637                         printf("%ciommu", delim);
1638                         delim = '/';
1639                 }
1640                 printf("\n");
1641
1642                 gpa += maplen;
1643         }
1644 }
1645
1646 static int
1647 show_memseg(struct vmctx *ctx)
1648 {
1649         char name[SPECNAMELEN + 1], numbuf[8];
1650         size_t seglen;
1651         int error, segid;
1652
1653         printf("ID  Length      Name\n");
1654
1655         segid = 0;
1656         while (1) {
1657                 error = vm_get_memseg(ctx, segid, &seglen, name, sizeof(name));
1658                 if (error)
1659                         return (errno == EINVAL ? 0 : error);
1660
1661                 if (seglen) {
1662                         printf("%-4d", segid);
1663                         humanize_number(numbuf, sizeof(numbuf), seglen, "B",
1664                             HN_AUTOSCALE, HN_NOSPACE);
1665                         printf("%-12s", numbuf);
1666                         printf("%s", name[0] ? name : "sysmem");
1667                         printf("\n");
1668                 }
1669                 segid++;
1670         }
1671 }
1672
1673 #ifdef BHYVE_SNAPSHOT
1674 static int
1675 send_message(const char *vmname, nvlist_t *nvl)
1676 {
1677         struct sockaddr_un addr;
1678         int err = 0, socket_fd;
1679
1680         socket_fd = socket(PF_UNIX, SOCK_STREAM, 0);
1681         if (socket_fd < 0) {
1682                 perror("Error creating bhyvectl socket");
1683                 err = errno;
1684                 goto done;
1685         }
1686
1687         memset(&addr, 0, sizeof(struct sockaddr_un));
1688         snprintf(addr.sun_path, sizeof(addr.sun_path), "%s%s",
1689             BHYVE_RUN_DIR, vmname);
1690         addr.sun_family = AF_UNIX;
1691         addr.sun_len = SUN_LEN(&addr);
1692
1693         if (connect(socket_fd, (struct sockaddr *)&addr, addr.sun_len) != 0) {
1694                 perror("connect() failed");
1695                 err = errno;
1696                 goto done;
1697         }
1698
1699         if (nvlist_send(socket_fd, nvl) < 0) {
1700                 perror("nvlist_send() failed");
1701                 err = errno;
1702         }
1703 done:
1704         nvlist_destroy(nvl);
1705
1706         if (socket_fd >= 0)
1707                 close(socket_fd);
1708         return (err);
1709 }
1710
1711 static int
1712 open_directory(const char *file)
1713 {
1714         char *path;
1715         int fd;
1716
1717         if ((path = strdup(file)) == NULL)
1718                 return (-1);
1719
1720         dirname(path);
1721         fd = open(path, O_DIRECTORY);
1722         free(path);
1723
1724         return (fd);
1725 }
1726
1727 static int
1728 snapshot_request(const char *vmname, char *file, bool suspend)
1729 {
1730         nvlist_t *nvl;
1731         int fd;
1732
1733         if ((fd = open_directory(file)) < 0)
1734                 return (errno);
1735
1736         nvl = nvlist_create(0);
1737         nvlist_add_string(nvl, "cmd", "checkpoint");
1738         nvlist_add_string(nvl, "filename", basename(file));
1739         nvlist_add_bool(nvl, "suspend", suspend);
1740         nvlist_move_descriptor(nvl, "fddir", fd);
1741
1742         return (send_message(vmname, nvl));
1743 }
1744 #endif
1745
1746 int
1747 main(int argc, char *argv[])
1748 {
1749         char *vmname;
1750         int error, ch, vcpuid, ptenum;
1751         vm_paddr_t gpa_pmap;
1752         struct vm_run vmrun;
1753         uint64_t rax, cr0, cr2, cr3, cr4, dr0, dr1, dr2, dr3, dr6, dr7;
1754         uint64_t rsp, rip, rflags, efer, pat;
1755         uint64_t eptp, bm, addr, u64, pteval[4], *pte, info[2];
1756         struct vmctx *ctx;
1757         struct vcpu *vcpu;
1758         cpuset_t cpus;
1759         bool cpu_intel;
1760         uint64_t cs, ds, es, fs, gs, ss, tr, ldtr;
1761         struct tm tm;
1762         struct option *opts;
1763 #ifdef BHYVE_SNAPSHOT
1764         char *checkpoint_file = NULL;
1765 #endif
1766
1767         cpu_intel = cpu_vendor_intel();
1768         opts = setup_options(cpu_intel);
1769
1770         vcpuid = 0;
1771         vmname = NULL;
1772         assert_lapic_lvt = -1;
1773         progname = basename(argv[0]);
1774
1775         while ((ch = getopt_long(argc, argv, "", opts, NULL)) != -1) {
1776                 switch (ch) {
1777                 case 0:
1778                         break;
1779                 case VMNAME:
1780                         vmname = optarg;
1781                         break;
1782                 case VCPU:
1783                         vcpuid = atoi(optarg);
1784                         break;
1785                 case SET_MEM:
1786                         memsize = atoi(optarg) * MB;
1787                         memsize = roundup(memsize, 2 * MB);
1788                         break;
1789                 case SET_EFER:
1790                         efer = strtoul(optarg, NULL, 0);
1791                         set_efer = 1;
1792                         break;
1793                 case SET_CR0:
1794                         cr0 = strtoul(optarg, NULL, 0);
1795                         set_cr0 = 1;
1796                         break;
1797                 case SET_CR2:
1798                         cr2 = strtoul(optarg, NULL, 0);
1799                         set_cr2 = 1;
1800                         break;
1801                 case SET_CR3:
1802                         cr3 = strtoul(optarg, NULL, 0);
1803                         set_cr3 = 1;
1804                         break;
1805                 case SET_CR4:
1806                         cr4 = strtoul(optarg, NULL, 0);
1807                         set_cr4 = 1;
1808                         break;
1809                 case SET_DR0:
1810                         dr0 = strtoul(optarg, NULL, 0);
1811                         set_dr0 = 1;
1812                         break;
1813                 case SET_DR1:
1814                         dr1 = strtoul(optarg, NULL, 0);
1815                         set_dr1 = 1;
1816                         break;
1817                 case SET_DR2:
1818                         dr2 = strtoul(optarg, NULL, 0);
1819                         set_dr2 = 1;
1820                         break;
1821                 case SET_DR3:
1822                         dr3 = strtoul(optarg, NULL, 0);
1823                         set_dr3 = 1;
1824                         break;
1825                 case SET_DR6:
1826                         dr6 = strtoul(optarg, NULL, 0);
1827                         set_dr6 = 1;
1828                         break;
1829                 case SET_DR7:
1830                         dr7 = strtoul(optarg, NULL, 0);
1831                         set_dr7 = 1;
1832                         break;
1833                 case SET_RSP:
1834                         rsp = strtoul(optarg, NULL, 0);
1835                         set_rsp = 1;
1836                         break;
1837                 case SET_RIP:
1838                         rip = strtoul(optarg, NULL, 0);
1839                         set_rip = 1;
1840                         break;
1841                 case SET_RAX:
1842                         rax = strtoul(optarg, NULL, 0);
1843                         set_rax = 1;
1844                         break;
1845                 case SET_RFLAGS:
1846                         rflags = strtoul(optarg, NULL, 0);
1847                         set_rflags = 1;
1848                         break;
1849                 case DESC_BASE:
1850                         desc_base = strtoul(optarg, NULL, 0);
1851                         break;
1852                 case DESC_LIMIT:
1853                         desc_limit = strtoul(optarg, NULL, 0);
1854                         break;
1855                 case DESC_ACCESS:
1856                         desc_access = strtoul(optarg, NULL, 0);
1857                         break;
1858                 case SET_CS:
1859                         cs = strtoul(optarg, NULL, 0);
1860                         set_cs = 1;
1861                         break;
1862                 case SET_DS:
1863                         ds = strtoul(optarg, NULL, 0);
1864                         set_ds = 1;
1865                         break;
1866                 case SET_ES:
1867                         es = strtoul(optarg, NULL, 0);
1868                         set_es = 1;
1869                         break;
1870                 case SET_FS:
1871                         fs = strtoul(optarg, NULL, 0);
1872                         set_fs = 1;
1873                         break;
1874                 case SET_GS:
1875                         gs = strtoul(optarg, NULL, 0);
1876                         set_gs = 1;
1877                         break;
1878                 case SET_SS:
1879                         ss = strtoul(optarg, NULL, 0);
1880                         set_ss = 1;
1881                         break;
1882                 case SET_TR:
1883                         tr = strtoul(optarg, NULL, 0);
1884                         set_tr = 1;
1885                         break;
1886                 case SET_LDTR:
1887                         ldtr = strtoul(optarg, NULL, 0);
1888                         set_ldtr = 1;
1889                         break;
1890                 case SET_X2APIC_STATE:
1891                         x2apic_state = strtol(optarg, NULL, 0);
1892                         set_x2apic_state = 1;
1893                         break;
1894                 case SET_CAP:
1895                         capval = strtoul(optarg, NULL, 0);
1896                         setcap = 1;
1897                         break;
1898                 case SET_RTC_TIME:
1899                         rtc_secs = strtoul(optarg, NULL, 0);
1900                         set_rtc_time = 1;
1901                         break;
1902                 case SET_RTC_NVRAM:
1903                         rtc_nvram_value = (uint8_t)strtoul(optarg, NULL, 0);
1904                         set_rtc_nvram = 1;
1905                         break;
1906                 case RTC_NVRAM_OFFSET:
1907                         rtc_nvram_offset = strtoul(optarg, NULL, 0);
1908                         break;
1909                 case GET_GPA_PMAP:
1910                         gpa_pmap = strtoul(optarg, NULL, 0);
1911                         get_gpa_pmap = 1;
1912                         break;
1913                 case CAPNAME:
1914                         capname = optarg;
1915                         break;
1916                 case UNASSIGN_PPTDEV:
1917                         unassign_pptdev = 1;
1918                         if (sscanf(optarg, "%d/%d/%d", &bus, &slot, &func) != 3)
1919                                 usage(cpu_intel);
1920                         break;
1921                 case ASSERT_LAPIC_LVT:
1922                         assert_lapic_lvt = atoi(optarg);
1923                         break;
1924 #ifdef BHYVE_SNAPSHOT
1925                 case SET_CHECKPOINT_FILE:
1926                 case SET_SUSPEND_FILE:
1927                         if (checkpoint_file != NULL)
1928                                 usage(cpu_intel);
1929
1930                         checkpoint_file = optarg;
1931                         vm_suspend_opt = (ch == SET_SUSPEND_FILE);
1932                         break;
1933 #endif
1934                 default:
1935                         usage(cpu_intel);
1936                 }
1937         }
1938         argc -= optind;
1939         argv += optind;
1940
1941         if (vmname == NULL)
1942                 usage(cpu_intel);
1943
1944         error = 0;
1945
1946         if (!error && create)
1947                 error = vm_create(vmname);
1948
1949         if (!error) {
1950                 ctx = vm_open(vmname);
1951                 if (ctx == NULL) {
1952                         fprintf(stderr,
1953                             "vm_open: %s could not be opened: %s\n",
1954                             vmname, strerror(errno));
1955                         exit (1);
1956                 }
1957                 vcpu = vm_vcpu_open(ctx, vcpuid);
1958         }
1959
1960         if (!error && memsize)
1961                 error = vm_setup_memory(ctx, memsize, VM_MMAP_ALL);
1962
1963         if (!error && set_efer)
1964                 error = vm_set_register(vcpu, VM_REG_GUEST_EFER, efer);
1965
1966         if (!error && set_cr0)
1967                 error = vm_set_register(vcpu, VM_REG_GUEST_CR0, cr0);
1968
1969         if (!error && set_cr2)
1970                 error = vm_set_register(vcpu, VM_REG_GUEST_CR2, cr2);
1971
1972         if (!error && set_cr3)
1973                 error = vm_set_register(vcpu, VM_REG_GUEST_CR3, cr3);
1974
1975         if (!error && set_cr4)
1976                 error = vm_set_register(vcpu, VM_REG_GUEST_CR4, cr4);
1977
1978         if (!error && set_dr0)
1979                 error = vm_set_register(vcpu, VM_REG_GUEST_DR0, dr0);
1980
1981         if (!error && set_dr1)
1982                 error = vm_set_register(vcpu, VM_REG_GUEST_DR1, dr1);
1983
1984         if (!error && set_dr2)
1985                 error = vm_set_register(vcpu, VM_REG_GUEST_DR2, dr2);
1986
1987         if (!error && set_dr3)
1988                 error = vm_set_register(vcpu, VM_REG_GUEST_DR3, dr3);
1989
1990         if (!error && set_dr6)
1991                 error = vm_set_register(vcpu, VM_REG_GUEST_DR6, dr6);
1992
1993         if (!error && set_dr7)
1994                 error = vm_set_register(vcpu, VM_REG_GUEST_DR7, dr7);
1995
1996         if (!error && set_rsp)
1997                 error = vm_set_register(vcpu, VM_REG_GUEST_RSP, rsp);
1998
1999         if (!error && set_rip)
2000                 error = vm_set_register(vcpu, VM_REG_GUEST_RIP, rip);
2001
2002         if (!error && set_rax)
2003                 error = vm_set_register(vcpu, VM_REG_GUEST_RAX, rax);
2004
2005         if (!error && set_rflags) {
2006                 error = vm_set_register(vcpu, VM_REG_GUEST_RFLAGS,
2007                                         rflags);
2008         }
2009
2010         if (!error && set_desc_ds) {
2011                 error = vm_set_desc(vcpu, VM_REG_GUEST_DS,
2012                                     desc_base, desc_limit, desc_access);
2013         }
2014
2015         if (!error && set_desc_es) {
2016                 error = vm_set_desc(vcpu, VM_REG_GUEST_ES,
2017                                     desc_base, desc_limit, desc_access);
2018         }
2019
2020         if (!error && set_desc_ss) {
2021                 error = vm_set_desc(vcpu, VM_REG_GUEST_SS,
2022                                     desc_base, desc_limit, desc_access);
2023         }
2024
2025         if (!error && set_desc_cs) {
2026                 error = vm_set_desc(vcpu, VM_REG_GUEST_CS,
2027                                     desc_base, desc_limit, desc_access);
2028         }
2029
2030         if (!error && set_desc_fs) {
2031                 error = vm_set_desc(vcpu, VM_REG_GUEST_FS,
2032                                     desc_base, desc_limit, desc_access);
2033         }
2034
2035         if (!error && set_desc_gs) {
2036                 error = vm_set_desc(vcpu, VM_REG_GUEST_GS,
2037                                     desc_base, desc_limit, desc_access);
2038         }
2039
2040         if (!error && set_desc_tr) {
2041                 error = vm_set_desc(vcpu, VM_REG_GUEST_TR,
2042                                     desc_base, desc_limit, desc_access);
2043         }
2044
2045         if (!error && set_desc_ldtr) {
2046                 error = vm_set_desc(vcpu, VM_REG_GUEST_LDTR,
2047                                     desc_base, desc_limit, desc_access);
2048         }
2049
2050         if (!error && set_desc_gdtr) {
2051                 error = vm_set_desc(vcpu, VM_REG_GUEST_GDTR,
2052                                     desc_base, desc_limit, 0);
2053         }
2054
2055         if (!error && set_desc_idtr) {
2056                 error = vm_set_desc(vcpu, VM_REG_GUEST_IDTR,
2057                                     desc_base, desc_limit, 0);
2058         }
2059
2060         if (!error && set_cs)
2061                 error = vm_set_register(vcpu, VM_REG_GUEST_CS, cs);
2062
2063         if (!error && set_ds)
2064                 error = vm_set_register(vcpu, VM_REG_GUEST_DS, ds);
2065
2066         if (!error && set_es)
2067                 error = vm_set_register(vcpu, VM_REG_GUEST_ES, es);
2068
2069         if (!error && set_fs)
2070                 error = vm_set_register(vcpu, VM_REG_GUEST_FS, fs);
2071
2072         if (!error && set_gs)
2073                 error = vm_set_register(vcpu, VM_REG_GUEST_GS, gs);
2074
2075         if (!error && set_ss)
2076                 error = vm_set_register(vcpu, VM_REG_GUEST_SS, ss);
2077
2078         if (!error && set_tr)
2079                 error = vm_set_register(vcpu, VM_REG_GUEST_TR, tr);
2080
2081         if (!error && set_ldtr)
2082                 error = vm_set_register(vcpu, VM_REG_GUEST_LDTR, ldtr);
2083
2084         if (!error && set_x2apic_state)
2085                 error = vm_set_x2apic_state(vcpu, x2apic_state);
2086
2087         if (!error && unassign_pptdev)
2088                 error = vm_unassign_pptdev(ctx, bus, slot, func);
2089
2090         if (!error && inject_nmi) {
2091                 error = vm_inject_nmi(vcpu);
2092         }
2093
2094         if (!error && assert_lapic_lvt != -1) {
2095                 error = vm_lapic_local_irq(vcpu, assert_lapic_lvt);
2096         }
2097
2098         if (!error && (get_memseg || get_all))
2099                 error = show_memseg(ctx);
2100
2101         if (!error && (get_memmap || get_all))
2102                 error = show_memmap(ctx);
2103
2104         if (!error)
2105                 error = get_all_registers(vcpu, vcpuid);
2106
2107         if (!error)
2108                 error = get_all_segments(vcpu, vcpuid);
2109
2110         if (!error) {
2111                 if (cpu_intel)
2112                         error = get_misc_vmcs(vcpu, vcpuid);
2113                 else
2114                         error = get_misc_vmcb(vcpu, vcpuid);
2115         }
2116
2117         if (!error && (get_x2apic_state || get_all)) {
2118                 error = vm_get_x2apic_state(vcpu, &x2apic_state);
2119                 if (error == 0)
2120                         printf("x2apic_state[%d]\t%d\n", vcpuid, x2apic_state);
2121         }
2122
2123         if (!error && (get_eptp || get_all)) {
2124                 if (cpu_intel)
2125                         error = vm_get_vmcs_field(vcpu, VMCS_EPTP, &eptp);
2126                 else
2127                         error = vm_get_vmcb_field(vcpu, VMCB_OFF_NPT_BASE,
2128                                                    8, &eptp);
2129                 if (error == 0)
2130                         printf("%s[%d]\t\t0x%016lx\n",
2131                                 cpu_intel ? "eptp" : "rvi/npt", vcpuid, eptp);
2132         }
2133
2134         if (!error && (get_exception_bitmap || get_all)) {
2135                 if(cpu_intel)
2136                         error = vm_get_vmcs_field(vcpu,
2137                                                 VMCS_EXCEPTION_BITMAP, &bm);
2138                 else
2139                         error = vm_get_vmcb_field(vcpu,
2140                                                   VMCB_OFF_EXC_INTERCEPT,
2141                                                   4, &bm);
2142                 if (error == 0)
2143                         printf("exception_bitmap[%d]\t%#lx\n", vcpuid, bm);
2144         }
2145
2146         if (!error && (get_io_bitmap || get_all)) {
2147                 if (cpu_intel) {
2148                         error = vm_get_vmcs_field(vcpu, VMCS_IO_BITMAP_A,
2149                                                   &bm);
2150                         if (error == 0)
2151                                 printf("io_bitmap_a[%d]\t%#lx\n", vcpuid, bm);
2152                         error = vm_get_vmcs_field(vcpu, VMCS_IO_BITMAP_B,
2153                                                   &bm);
2154                         if (error == 0)
2155                                 printf("io_bitmap_b[%d]\t%#lx\n", vcpuid, bm);
2156                 } else {
2157                         error = vm_get_vmcb_field(vcpu,
2158                                                   VMCB_OFF_IO_PERM, 8, &bm);
2159                         if (error == 0)
2160                                 printf("io_bitmap[%d]\t%#lx\n", vcpuid, bm);
2161                 }
2162         }
2163
2164         if (!error && (get_tsc_offset || get_all)) {
2165                 uint64_t tscoff;
2166                 if (cpu_intel)
2167                         error = vm_get_vmcs_field(vcpu, VMCS_TSC_OFFSET,
2168                                                   &tscoff);
2169                 else
2170                         error = vm_get_vmcb_field(vcpu,
2171                                                   VMCB_OFF_TSC_OFFSET,
2172                                                   8, &tscoff);
2173                 if (error == 0)
2174                         printf("tsc_offset[%d]\t0x%016lx\n", vcpuid, tscoff);
2175         }
2176
2177         if (!error && (get_msr_bitmap_address || get_all)) {
2178                 if (cpu_intel)
2179                         error = vm_get_vmcs_field(vcpu, VMCS_MSR_BITMAP,
2180                                                   &addr);
2181                 else
2182                         error = vm_get_vmcb_field(vcpu,
2183                                                   VMCB_OFF_MSR_PERM, 8, &addr);
2184                 if (error == 0)
2185                         printf("msr_bitmap[%d]\t\t%#lx\n", vcpuid, addr);
2186         }
2187
2188         if (!error && (get_msr_bitmap || get_all)) {
2189                 if (cpu_intel) {
2190                         error = vm_get_vmcs_field(vcpu,
2191                                                   VMCS_MSR_BITMAP, &addr);
2192                 } else {
2193                         error = vm_get_vmcb_field(vcpu,
2194                                                   VMCB_OFF_MSR_PERM, 8,
2195                                                   &addr);
2196                 }
2197
2198                 if (error == 0)
2199                         error = dump_msr_bitmap(vcpuid, addr, cpu_intel);
2200         }
2201
2202         if (!error && (get_vpid_asid || get_all)) {
2203                 uint64_t vpid;
2204                 if (cpu_intel)
2205                         error = vm_get_vmcs_field(vcpu, VMCS_VPID, &vpid);
2206                 else
2207                         error = vm_get_vmcb_field(vcpu, VMCB_OFF_ASID,
2208                                                   4, &vpid);
2209                 if (error == 0)
2210                         printf("%s[%d]\t\t0x%04lx\n",
2211                                 cpu_intel ? "vpid" : "asid", vcpuid, vpid);
2212         }
2213
2214         if (!error && (get_guest_pat || get_all)) {
2215                 if (cpu_intel)
2216                         error = vm_get_vmcs_field(vcpu,
2217                                                   VMCS_GUEST_IA32_PAT, &pat);
2218                 else
2219                         error = vm_get_vmcb_field(vcpu,
2220                                                   VMCB_OFF_GUEST_PAT, 8, &pat);
2221                 if (error == 0)
2222                         printf("guest_pat[%d]\t\t0x%016lx\n", vcpuid, pat);
2223         }
2224
2225         if (!error && (get_guest_sysenter || get_all)) {
2226                 if (cpu_intel)
2227                         error = vm_get_vmcs_field(vcpu,
2228                                                   VMCS_GUEST_IA32_SYSENTER_CS,
2229                                                   &cs);
2230                 else
2231                         error = vm_get_vmcb_field(vcpu,
2232                                                   VMCB_OFF_SYSENTER_CS, 8,
2233                                                   &cs);
2234
2235                 if (error == 0)
2236                         printf("guest_sysenter_cs[%d]\t%#lx\n", vcpuid, cs);
2237                 if (cpu_intel)
2238                         error = vm_get_vmcs_field(vcpu,
2239                                                   VMCS_GUEST_IA32_SYSENTER_ESP,
2240                                                   &rsp);
2241                 else
2242                         error = vm_get_vmcb_field(vcpu,
2243                                                   VMCB_OFF_SYSENTER_ESP, 8,
2244                                                   &rsp);
2245
2246                 if (error == 0)
2247                         printf("guest_sysenter_sp[%d]\t%#lx\n", vcpuid, rsp);
2248                 if (cpu_intel)
2249                         error = vm_get_vmcs_field(vcpu,
2250                                                   VMCS_GUEST_IA32_SYSENTER_EIP,
2251                                                   &rip);
2252                 else
2253                         error = vm_get_vmcb_field(vcpu,
2254                                                   VMCB_OFF_SYSENTER_EIP, 8,
2255                                                   &rip);
2256                 if (error == 0)
2257                         printf("guest_sysenter_ip[%d]\t%#lx\n", vcpuid, rip);
2258         }
2259
2260         if (!error && (get_exit_reason || get_all)) {
2261                 if (cpu_intel)
2262                         error = vm_get_vmcs_field(vcpu, VMCS_EXIT_REASON,
2263                                                   &u64);
2264                 else
2265                         error = vm_get_vmcb_field(vcpu,
2266                                                   VMCB_OFF_EXIT_REASON, 8,
2267                                                   &u64);
2268                 if (error == 0)
2269                         printf("exit_reason[%d]\t%#lx\n", vcpuid, u64);
2270         }
2271
2272         if (!error && setcap) {
2273                 int captype;
2274                 captype = vm_capability_name2type(capname);
2275                 error = vm_set_capability(vcpu, captype, capval);
2276                 if (error != 0 && errno == ENOENT)
2277                         printf("Capability \"%s\" is not available\n", capname);
2278         }
2279
2280         if (!error && get_gpa_pmap) {
2281                 error = vm_get_gpa_pmap(ctx, gpa_pmap, pteval, &ptenum);
2282                 if (error == 0) {
2283                         printf("gpa %#lx:", gpa_pmap);
2284                         pte = &pteval[0];
2285                         while (ptenum-- > 0)
2286                                 printf(" %#lx", *pte++);
2287                         printf("\n");
2288                 }
2289         }
2290
2291         if (!error && set_rtc_nvram)
2292                 error = vm_rtc_write(ctx, rtc_nvram_offset, rtc_nvram_value);
2293
2294         if (!error && (get_rtc_nvram || get_all)) {
2295                 error = vm_rtc_read(ctx, rtc_nvram_offset, &rtc_nvram_value);
2296                 if (error == 0) {
2297                         printf("rtc nvram[%03d]: 0x%02x\n", rtc_nvram_offset,
2298                             rtc_nvram_value);
2299                 }
2300         }
2301
2302         if (!error && set_rtc_time)
2303                 error = vm_rtc_settime(ctx, rtc_secs);
2304
2305         if (!error && (get_rtc_time || get_all)) {
2306                 error = vm_rtc_gettime(ctx, &rtc_secs);
2307                 if (error == 0) {
2308                         gmtime_r(&rtc_secs, &tm);
2309                         printf("rtc time %#lx: %s %s %02d %02d:%02d:%02d %d\n",
2310                             rtc_secs, wday_str(tm.tm_wday), mon_str(tm.tm_mon),
2311                             tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec,
2312                             1900 + tm.tm_year);
2313                 }
2314         }
2315
2316         if (!error && (getcap || get_all)) {
2317                 int captype, val, getcaptype;
2318
2319                 if (getcap && capname)
2320                         getcaptype = vm_capability_name2type(capname);
2321                 else
2322                         getcaptype = -1;
2323
2324                 for (captype = 0; captype < VM_CAP_MAX; captype++) {
2325                         if (getcaptype >= 0 && captype != getcaptype)
2326                                 continue;
2327                         error = vm_get_capability(vcpu, captype, &val);
2328                         if (error == 0) {
2329                                 printf("Capability \"%s\" is %s on vcpu %d\n",
2330                                         vm_capability_type2name(captype),
2331                                         val ? "set" : "not set", vcpuid);
2332                         } else if (errno == ENOENT) {
2333                                 error = 0;
2334                                 printf("Capability \"%s\" is not available\n",
2335                                         vm_capability_type2name(captype));
2336                         } else {
2337                                 break;
2338                         }
2339                 }
2340         }
2341
2342         if (!error && (get_active_cpus || get_all)) {
2343                 error = vm_active_cpus(ctx, &cpus);
2344                 if (!error)
2345                         print_cpus("active cpus", &cpus);
2346         }
2347
2348         if (!error && (get_debug_cpus || get_all)) {
2349                 error = vm_debug_cpus(ctx, &cpus);
2350                 if (!error)
2351                         print_cpus("debug cpus", &cpus);
2352         }
2353
2354         if (!error && (get_suspended_cpus || get_all)) {
2355                 error = vm_suspended_cpus(ctx, &cpus);
2356                 if (!error)
2357                         print_cpus("suspended cpus", &cpus);
2358         }
2359
2360         if (!error && (get_intinfo || get_all)) {
2361                 error = vm_get_intinfo(vcpu, &info[0], &info[1]);
2362                 if (!error) {
2363                         print_intinfo("pending", info[0]);
2364                         print_intinfo("current", info[1]);
2365                 }
2366         }
2367
2368         if (!error && (get_stats || get_all)) {
2369                 int i, num_stats;
2370                 uint64_t *stats;
2371                 struct timeval tv;
2372                 const char *desc;
2373
2374                 stats = vm_get_stats(vcpu, &tv, &num_stats);
2375                 if (stats != NULL) {
2376                         printf("vcpu%d stats:\n", vcpuid);
2377                         for (i = 0; i < num_stats; i++) {
2378                                 desc = vm_get_stat_desc(ctx, i);
2379                                 printf("%-40s\t%ld\n", desc, stats[i]);
2380                         }
2381                 }
2382         }
2383
2384         if (!error && (get_cpu_topology || get_all)) {
2385                 uint16_t sockets, cores, threads, maxcpus;
2386
2387                 vm_get_topology(ctx, &sockets, &cores, &threads, &maxcpus);
2388                 printf("cpu_topology:\tsockets=%hu, cores=%hu, threads=%hu, "
2389                     "maxcpus=%hu\n", sockets, cores, threads, maxcpus);
2390         }
2391
2392         if (!error && run) {
2393                 struct vm_exit vmexit;
2394                 cpuset_t cpuset;
2395
2396                 vmrun.vm_exit = &vmexit;
2397                 vmrun.cpuset = &cpuset;
2398                 vmrun.cpusetsize = sizeof(cpuset);
2399                 error = vm_run(vcpu, &vmrun);
2400                 if (error == 0)
2401                         dump_vm_run_exitcode(&vmexit, vcpuid);
2402                 else
2403                         printf("vm_run error %d\n", error);
2404         }
2405
2406         if (!error && force_reset)
2407                 error = vm_suspend(ctx, VM_SUSPEND_RESET);
2408
2409         if (!error && force_poweroff)
2410                 error = vm_suspend(ctx, VM_SUSPEND_POWEROFF);
2411
2412         if (error)
2413                 printf("errno = %d\n", errno);
2414
2415         if (!error && destroy)
2416                 vm_destroy(ctx);
2417
2418 #ifdef BHYVE_SNAPSHOT
2419         if (!error && checkpoint_file)
2420                 error = snapshot_request(vmname, checkpoint_file, vm_suspend_opt);
2421 #endif
2422
2423         free (opts);
2424         exit(error);
2425 }