usr.sbin/bhyvectl/bhyvectl.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2011 NetApp, Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  *
  28  * $FreeBSD$
  29  */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33
  34 #include <sys/param.h>
  35 #include <sys/types.h>
  36 #include <sys/sysctl.h>
  37 #include <sys/errno.h>
  38 #include <sys/mman.h>
  39 #include <sys/cpuset.h>
  40
  41 #include <stdio.h>
  42 #include <stdlib.h>
  43 #include <stdbool.h>
  44 #include <string.h>
  45 #include <unistd.h>
  46 #include <libgen.h>
  47 #include <libutil.h>
  48 #include <fcntl.h>
  49 #include <getopt.h>
  50 #include <time.h>
  51 #include <assert.h>
  52 #include <libutil.h>
  53
  54 #include <machine/cpufunc.h>
  55 #include <machine/specialreg.h>
  56 #include <machine/vmm.h>
  57 #include <machine/vmm_dev.h>
  58 #include <vmmapi.h>
  59
  60 #include <sys/socket.h>
  61 #include <sys/un.h>
  62
  63 #include "amd/vmcb.h"
  64 #include "intel/vmcs.h"
  65
  66 #define MB      (1UL << 20)
  67 #define GB      (1UL << 30)
  68
  69 #define REQ_ARG         required_argument
  70 #define NO_ARG          no_argument
  71 #define OPT_ARG         optional_argument
  72
  73 #define CHECKPOINT_RUN_DIR "/var/run/bhyve/checkpoint"
  74 #define MAX_VMNAME 100
  75
  76 static const char *progname;
  77
  78 static void
  79 usage(bool cpu_intel)
  80 {
  81
  82         (void)fprintf(stderr,
  83         "Usage: %s --vm=<vmname>\n"
  84         "       [--cpu=<vcpu_number>]\n"
  85         "       [--create]\n"
  86         "       [--destroy]\n"
  87 #ifdef BHYVE_SNAPSHOT
  88         "       [--checkpoint=<filename>]\n"
  89         "       [--suspend=<filename>]\n"
  90 #endif
  91         "       [--get-all]\n"
  92         "       [--get-stats]\n"
  93         "       [--set-desc-ds]\n"
  94         "       [--get-desc-ds]\n"
  95         "       [--set-desc-es]\n"
  96         "       [--get-desc-es]\n"
  97         "       [--set-desc-gs]\n"
  98         "       [--get-desc-gs]\n"
  99         "       [--set-desc-fs]\n"
 100         "       [--get-desc-fs]\n"
 101         "       [--set-desc-cs]\n"
 102         "       [--get-desc-cs]\n"
 103         "       [--set-desc-ss]\n"
 104         "       [--get-desc-ss]\n"
 105         "       [--set-desc-tr]\n"
 106         "       [--get-desc-tr]\n"
 107         "       [--set-desc-ldtr]\n"
 108         "       [--get-desc-ldtr]\n"
 109         "       [--set-desc-gdtr]\n"
 110         "       [--get-desc-gdtr]\n"
 111         "       [--set-desc-idtr]\n"
 112         "       [--get-desc-idtr]\n"
 113         "       [--run]\n"
 114         "       [--capname=<capname>]\n"
 115         "       [--getcap]\n"
 116         "       [--setcap=<0|1>]\n"
 117         "       [--desc-base=<BASE>]\n"
 118         "       [--desc-limit=<LIMIT>]\n"
 119         "       [--desc-access=<ACCESS>]\n"
 120         "       [--set-cr0=<CR0>]\n"
 121         "       [--get-cr0]\n"
 122         "       [--set-cr2=<CR2>]\n"
 123         "       [--get-cr2]\n"
 124         "       [--set-cr3=<CR3>]\n"
 125         "       [--get-cr3]\n"
 126         "       [--set-cr4=<CR4>]\n"
 127         "       [--get-cr4]\n"
 128         "       [--set-dr0=<DR0>]\n"
 129         "       [--get-dr0]\n"
 130         "       [--set-dr1=<DR1>]\n"
 131         "       [--get-dr1]\n"
 132         "       [--set-dr2=<DR2>]\n"
 133         "       [--get-dr2]\n"
 134         "       [--set-dr3=<DR3>]\n"
 135         "       [--get-dr3]\n"
 136         "       [--set-dr6=<DR6>]\n"
 137         "       [--get-dr6]\n"
 138         "       [--set-dr7=<DR7>]\n"
 139         "       [--get-dr7]\n"
 140         "       [--set-rsp=<RSP>]\n"
 141         "       [--get-rsp]\n"
 142         "       [--set-rip=<RIP>]\n"
 143         "       [--get-rip]\n"
 144         "       [--get-rax]\n"
 145         "       [--set-rax=<RAX>]\n"
 146         "       [--get-rbx]\n"
 147         "       [--get-rcx]\n"
 148         "       [--get-rdx]\n"
 149         "       [--get-rsi]\n"
 150         "       [--get-rdi]\n"
 151         "       [--get-rbp]\n"
 152         "       [--get-r8]\n"
 153         "       [--get-r9]\n"
 154         "       [--get-r10]\n"
 155         "       [--get-r11]\n"
 156         "       [--get-r12]\n"
 157         "       [--get-r13]\n"
 158         "       [--get-r14]\n"
 159         "       [--get-r15]\n"
 160         "       [--set-rflags=<RFLAGS>]\n"
 161         "       [--get-rflags]\n"
 162         "       [--set-cs]\n"
 163         "       [--get-cs]\n"
 164         "       [--set-ds]\n"
 165         "       [--get-ds]\n"
 166         "       [--set-es]\n"
 167         "       [--get-es]\n"
 168         "       [--set-fs]\n"
 169         "       [--get-fs]\n"
 170         "       [--set-gs]\n"
 171         "       [--get-gs]\n"
 172         "       [--set-ss]\n"
 173         "       [--get-ss]\n"
 174         "       [--get-tr]\n"
 175         "       [--get-ldtr]\n"
 176         "       [--set-x2apic-state=<state>]\n"
 177         "       [--get-x2apic-state]\n"
 178         "       [--unassign-pptdev=<bus/slot/func>]\n"
 179         "       [--set-mem=<memory in units of MB>]\n"
 180         "       [--get-lowmem]\n"
 181         "       [--get-highmem]\n"
 182         "       [--get-gpa-pmap]\n"
 183         "       [--assert-lapic-lvt=<pin>]\n"
 184         "       [--inject-nmi]\n"
 185         "       [--force-reset]\n"
 186         "       [--force-poweroff]\n"
 187         "       [--get-rtc-time]\n"
 188         "       [--set-rtc-time=<secs>]\n"
 189         "       [--get-rtc-nvram]\n"
 190         "       [--set-rtc-nvram=<val>]\n"
 191         "       [--rtc-nvram-offset=<offset>]\n"
 192         "       [--get-active-cpus]\n"
 193         "       [--get-suspended-cpus]\n"
 194         "       [--get-intinfo]\n"
 195         "       [--get-eptp]\n"
 196         "       [--set-exception-bitmap]\n"
 197         "       [--get-exception-bitmap]\n"
 198         "       [--get-tsc-offset]\n"
 199         "       [--get-guest-pat]\n"
 200         "       [--get-io-bitmap-address]\n"
 201         "       [--get-msr-bitmap]\n"
 202         "       [--get-msr-bitmap-address]\n"
 203         "       [--get-guest-sysenter]\n"
 204         "       [--get-exit-reason]\n"
 205         "       [--get-cpu-topology]\n",
 206         progname);
 207
 208         if (cpu_intel) {
 209                 (void)fprintf(stderr,
 210                 "       [--get-vmcs-pinbased-ctls]\n"
 211                 "       [--get-vmcs-procbased-ctls]\n"
 212                 "       [--get-vmcs-procbased-ctls2]\n"
 213                 "       [--get-vmcs-entry-interruption-info]\n"
 214                 "       [--set-vmcs-entry-interruption-info=<info>]\n"
 215                 "       [--get-vmcs-guest-physical-address\n"
 216                 "       [--get-vmcs-guest-linear-address\n"
 217                 "       [--get-vmcs-host-pat]\n"
 218                 "       [--get-vmcs-host-cr0]\n"
 219                 "       [--get-vmcs-host-cr3]\n"
 220                 "       [--get-vmcs-host-cr4]\n"
 221                 "       [--get-vmcs-host-rip]\n"
 222                 "       [--get-vmcs-host-rsp]\n"
 223                 "       [--get-vmcs-cr0-mask]\n"
 224                 "       [--get-vmcs-cr0-shadow]\n"
 225                 "       [--get-vmcs-cr4-mask]\n"
 226                 "       [--get-vmcs-cr4-shadow]\n"
 227                 "       [--get-vmcs-cr3-targets]\n"
 228                 "       [--get-vmcs-apic-access-address]\n"
 229                 "       [--get-vmcs-virtual-apic-address]\n"
 230                 "       [--get-vmcs-tpr-threshold]\n"
 231                 "       [--get-vmcs-vpid]\n"
 232                 "       [--get-vmcs-instruction-error]\n"
 233                 "       [--get-vmcs-exit-ctls]\n"
 234                 "       [--get-vmcs-entry-ctls]\n"
 235                 "       [--get-vmcs-link]\n"
 236                 "       [--get-vmcs-exit-qualification]\n"
 237                 "       [--get-vmcs-exit-interruption-info]\n"
 238                 "       [--get-vmcs-exit-interruption-error]\n"
 239                 "       [--get-vmcs-interruptibility]\n"
 240                 );
 241         } else {
 242                 (void)fprintf(stderr,
 243                 "       [--get-vmcb-intercepts]\n"
 244                 "       [--get-vmcb-asid]\n"
 245                 "       [--get-vmcb-exit-details]\n"
 246                 "       [--get-vmcb-tlb-ctrl]\n"
 247                 "       [--get-vmcb-virq]\n"
 248                 "       [--get-avic-apic-bar]\n"
 249                 "       [--get-avic-backing-page]\n"
 250                 "       [--get-avic-table]\n"
 251                 );
 252         }
 253         exit(1);
 254 }
 255
 256 static int get_rtc_time, set_rtc_time;
 257 static int get_rtc_nvram, set_rtc_nvram;
 258 static int rtc_nvram_offset;
 259 static uint8_t rtc_nvram_value;
 260 static time_t rtc_secs;
 261
 262 static int get_stats, getcap, setcap, capval, get_gpa_pmap;
 263 static int inject_nmi, assert_lapic_lvt;
 264 static int force_reset, force_poweroff;
 265 static const char *capname;
 266 static int create, destroy, get_memmap, get_memseg;
 267 static int get_intinfo;
 268 static int get_active_cpus, get_suspended_cpus;
 269 static uint64_t memsize;
 270 static int set_cr0, get_cr0, set_cr2, get_cr2, set_cr3, get_cr3;
 271 static int set_cr4, get_cr4;
 272 static int set_efer, get_efer;
 273 static int set_dr0, get_dr0;
 274 static int set_dr1, get_dr1;
 275 static int set_dr2, get_dr2;
 276 static int set_dr3, get_dr3;
 277 static int set_dr6, get_dr6;
 278 static int set_dr7, get_dr7;
 279 static int set_rsp, get_rsp, set_rip, get_rip, set_rflags, get_rflags;
 280 static int set_rax, get_rax;
 281 static int get_rbx, get_rcx, get_rdx, get_rsi, get_rdi, get_rbp;
 282 static int get_r8, get_r9, get_r10, get_r11, get_r12, get_r13, get_r14, get_r15;
 283 static int set_desc_ds, get_desc_ds;
 284 static int set_desc_es, get_desc_es;
 285 static int set_desc_fs, get_desc_fs;
 286 static int set_desc_gs, get_desc_gs;
 287 static int set_desc_cs, get_desc_cs;
 288 static int set_desc_ss, get_desc_ss;
 289 static int set_desc_gdtr, get_desc_gdtr;
 290 static int set_desc_idtr, get_desc_idtr;
 291 static int set_desc_tr, get_desc_tr;
 292 static int set_desc_ldtr, get_desc_ldtr;
 293 static int set_cs, set_ds, set_es, set_fs, set_gs, set_ss, set_tr, set_ldtr;
 294 static int get_cs, get_ds, get_es, get_fs, get_gs, get_ss, get_tr, get_ldtr;
 295 static int set_x2apic_state, get_x2apic_state;
 296 enum x2apic_state x2apic_state;
 297 static int unassign_pptdev, bus, slot, func;
 298 static int run;
 299 static int get_cpu_topology;
 300 #ifdef BHYVE_SNAPSHOT
 301 static int vm_checkpoint_opt;
 302 static int vm_suspend_opt;
 303 #endif
 304
 305 /*
 306  * VMCB specific.
 307  */
 308 static int get_vmcb_intercept, get_vmcb_exit_details, get_vmcb_tlb_ctrl;
 309 static int get_vmcb_virq, get_avic_table;
 310
 311 /*
 312  * VMCS-specific fields
 313  */
 314 static int get_pinbased_ctls, get_procbased_ctls, get_procbased_ctls2;
 315 static int get_eptp, get_io_bitmap, get_tsc_offset;
 316 static int get_vmcs_entry_interruption_info;
 317 static int get_vmcs_interruptibility;
 318 uint32_t vmcs_entry_interruption_info;
 319 static int get_vmcs_gpa, get_vmcs_gla;
 320 static int get_exception_bitmap;
 321 static int get_cr0_mask, get_cr0_shadow;
 322 static int get_cr4_mask, get_cr4_shadow;
 323 static int get_cr3_targets;
 324 static int get_apic_access_addr, get_virtual_apic_addr, get_tpr_threshold;
 325 static int get_msr_bitmap, get_msr_bitmap_address;
 326 static int get_vpid_asid;
 327 static int get_inst_err, get_exit_ctls, get_entry_ctls;
 328 static int get_host_cr0, get_host_cr3, get_host_cr4;
 329 static int get_host_rip, get_host_rsp;
 330 static int get_guest_pat, get_host_pat;
 331 static int get_guest_sysenter, get_vmcs_link;
 332 static int get_exit_reason, get_vmcs_exit_qualification;
 333 static int get_vmcs_exit_interruption_info, get_vmcs_exit_interruption_error;
 334 static int get_vmcs_exit_inst_length;
 335
 336 static uint64_t desc_base;
 337 static uint32_t desc_limit, desc_access;
 338
 339 static int get_all;
 340
 341 static void
 342 dump_vm_run_exitcode(struct vm_exit *vmexit, int vcpu)
 343 {
 344         printf("vm exit[%d]\n", vcpu);
 345         printf("\trip\t\t0x%016lx\n", vmexit->rip);
 346         printf("\tinst_length\t%d\n", vmexit->inst_length);
 347         switch (vmexit->exitcode) {
 348         case VM_EXITCODE_INOUT:
 349                 printf("\treason\t\tINOUT\n");
 350                 printf("\tdirection\t%s\n", vmexit->u.inout.in ? "IN" : "OUT");
 351                 printf("\tbytes\t\t%d\n", vmexit->u.inout.bytes);
 352                 printf("\tflags\t\t%s%s\n",
 353                         vmexit->u.inout.string ? "STRING " : "",
 354                         vmexit->u.inout.rep ? "REP " : "");
 355                 printf("\tport\t\t0x%04x\n", vmexit->u.inout.port);
 356                 printf("\teax\t\t0x%08x\n", vmexit->u.inout.eax);
 357                 break;
 358         case VM_EXITCODE_VMX:
 359                 printf("\treason\t\tVMX\n");
 360                 printf("\tstatus\t\t%d\n", vmexit->u.vmx.status);
 361                 printf("\texit_reason\t0x%08x (%u)\n",
 362                     vmexit->u.vmx.exit_reason, vmexit->u.vmx.exit_reason);
 363                 printf("\tqualification\t0x%016lx\n",
 364                         vmexit->u.vmx.exit_qualification);
 365                 printf("\tinst_type\t\t%d\n", vmexit->u.vmx.inst_type);
 366                 printf("\tinst_error\t\t%d\n", vmexit->u.vmx.inst_error);
 367                 break;
 368         case VM_EXITCODE_SVM:
 369                 printf("\treason\t\tSVM\n");
 370                 printf("\texit_reason\t\t%#lx\n", vmexit->u.svm.exitcode);
 371                 printf("\texitinfo1\t\t%#lx\n", vmexit->u.svm.exitinfo1);
 372                 printf("\texitinfo2\t\t%#lx\n", vmexit->u.svm.exitinfo2);
 373                 break;
 374         default:
 375                 printf("*** unknown vm run exitcode %d\n", vmexit->exitcode);
 376                 break;
 377         }
 378 }
 379
 380 /* AMD 6th generation and Intel compatible MSRs */
 381 #define MSR_AMD6TH_START        0xC0000000
 382 #define MSR_AMD6TH_END          0xC0001FFF
 383 /* AMD 7th and 8th generation compatible MSRs */
 384 #define MSR_AMD7TH_START        0xC0010000
 385 #define MSR_AMD7TH_END          0xC0011FFF
 386
 387 static const char *
 388 msr_name(uint32_t msr)
 389 {
 390         static char buf[32];
 391
 392         switch(msr) {
 393         case MSR_TSC:
 394                 return ("MSR_TSC");
 395         case MSR_EFER:
 396                 return ("MSR_EFER");
 397         case MSR_STAR:
 398                 return ("MSR_STAR");
 399         case MSR_LSTAR:
 400                 return ("MSR_LSTAR");
 401         case MSR_CSTAR:
 402                 return ("MSR_CSTAR");
 403         case MSR_SF_MASK:
 404                 return ("MSR_SF_MASK");
 405         case MSR_FSBASE:
 406                 return ("MSR_FSBASE");
 407         case MSR_GSBASE:
 408                 return ("MSR_GSBASE");
 409         case MSR_KGSBASE:
 410                 return ("MSR_KGSBASE");
 411         case MSR_SYSENTER_CS_MSR:
 412                 return ("MSR_SYSENTER_CS_MSR");
 413         case MSR_SYSENTER_ESP_MSR:
 414                 return ("MSR_SYSENTER_ESP_MSR");
 415         case MSR_SYSENTER_EIP_MSR:
 416                 return ("MSR_SYSENTER_EIP_MSR");
 417         case MSR_PAT:
 418                 return ("MSR_PAT");
 419         }
 420         snprintf(buf, sizeof(buf), "MSR       %#08x", msr);
 421
 422         return (buf);
 423 }
 424
 425 static inline void
 426 print_msr_pm(uint64_t msr, int vcpu, int readable, int writeable)
 427 {
 428
 429         if (readable || writeable) {
 430                 printf("%-20s[%d]\t\t%c%c\n", msr_name(msr), vcpu,
 431                         readable ? 'R' : '-', writeable ? 'W' : '-');
 432         }
 433 }
 434
 435 /*
 436  * Reference APM vol2, section 15.11 MSR Intercepts.
 437  */
 438 static void
 439 dump_amd_msr_pm(const char *bitmap, int vcpu)
 440 {
 441         int byte, bit, readable, writeable;
 442         uint32_t msr;
 443
 444         for (msr = 0; msr < 0x2000; msr++) {
 445                 byte = msr / 4;
 446                 bit = (msr % 4) * 2;
 447
 448                 /* Look at MSRs in the range 0x00000000 to 0x00001FFF */
 449                 readable = (bitmap[byte] & (1 << bit)) ? 0 : 1;
 450                 writeable = (bitmap[byte] & (2 << bit)) ?  0 : 1;
 451                 print_msr_pm(msr, vcpu, readable, writeable);
 452
 453                 /* Look at MSRs in the range 0xC0000000 to 0xC0001FFF */
 454                 byte += 2048;
 455                 readable = (bitmap[byte] & (1 << bit)) ? 0 : 1;
 456                 writeable = (bitmap[byte] & (2 << bit)) ?  0 : 1;
 457                 print_msr_pm(msr + MSR_AMD6TH_START, vcpu, readable,
 458                                 writeable);
 459
 460                 /* MSR 0xC0010000 to 0xC0011FF is only for AMD */
 461                 byte += 4096;
 462                 readable = (bitmap[byte] & (1 << bit)) ? 0 : 1;
 463                 writeable = (bitmap[byte] & (2 << bit)) ?  0 : 1;
 464                 print_msr_pm(msr + MSR_AMD7TH_START, vcpu, readable,
 465                                 writeable);
 466         }
 467 }
 468
 469 /*
 470  * Reference Intel SDM Vol3 Section 24.6.9 MSR-Bitmap Address
 471  */
 472 static void
 473 dump_intel_msr_pm(const char *bitmap, int vcpu)
 474 {
 475         int byte, bit, readable, writeable;
 476         uint32_t msr;
 477
 478         for (msr = 0; msr < 0x2000; msr++) {
 479                 byte = msr / 8;
 480                 bit = msr & 0x7;
 481
 482                 /* Look at MSRs in the range 0x00000000 to 0x00001FFF */
 483                 readable = (bitmap[byte] & (1 << bit)) ? 0 : 1;
 484                 writeable = (bitmap[2048 + byte] & (1 << bit)) ?  0 : 1;
 485                 print_msr_pm(msr, vcpu, readable, writeable);
 486
 487                 /* Look at MSRs in the range 0xC0000000 to 0xC0001FFF */
 488                 byte += 1024;
 489                 readable = (bitmap[byte] & (1 << bit)) ? 0 : 1;
 490                 writeable = (bitmap[2048 + byte] & (1 << bit)) ?  0 : 1;
 491                 print_msr_pm(msr + MSR_AMD6TH_START, vcpu, readable,
 492                                 writeable);
 493         }
 494 }
 495
 496 static int
 497 dump_msr_bitmap(int vcpu, uint64_t addr, bool cpu_intel)
 498 {
 499         int error, fd, map_size;
 500         const char *bitmap;
 501
 502         error = -1;
 503         bitmap = MAP_FAILED;
 504
 505         fd = open("/dev/mem", O_RDONLY, 0);
 506         if (fd < 0) {
 507                 perror("Couldn't open /dev/mem");
 508                 goto done;
 509         }
 510
 511         if (cpu_intel)
 512                 map_size = PAGE_SIZE;
 513         else
 514                 map_size = 2 * PAGE_SIZE;
 515
 516         bitmap = mmap(NULL, map_size, PROT_READ, MAP_SHARED, fd, addr);
 517         if (bitmap == MAP_FAILED) {
 518                 perror("mmap failed");
 519                 goto done;
 520         }
 521
 522         if (cpu_intel)
 523                 dump_intel_msr_pm(bitmap, vcpu);
 524         else
 525                 dump_amd_msr_pm(bitmap, vcpu);
 526
 527         error = 0;
 528 done:
 529         if (bitmap != MAP_FAILED)
 530                 munmap((void *)bitmap, map_size);
 531         if (fd >= 0)
 532                 close(fd);
 533
 534         return (error);
 535 }
 536
 537 static int
 538 vm_get_vmcs_field(struct vmctx *ctx, int vcpu, int field, uint64_t *ret_val)
 539 {
 540
 541         return (vm_get_register(ctx, vcpu, VMCS_IDENT(field), ret_val));
 542 }
 543
 544 static int
 545 vm_get_vmcb_field(struct vmctx *ctx, int vcpu, int off, int bytes,
 546         uint64_t *ret_val)
 547 {
 548
 549         return (vm_get_register(ctx, vcpu, VMCB_ACCESS(off, bytes), ret_val));
 550 }
 551
 552 enum {
 553         VMNAME = 1000,  /* avoid collision with return values from getopt */
 554         VCPU,
 555         SET_MEM,
 556         SET_EFER,
 557         SET_CR0,
 558         SET_CR2,
 559         SET_CR3,
 560         SET_CR4,
 561         SET_DR0,
 562         SET_DR1,
 563         SET_DR2,
 564         SET_DR3,
 565         SET_DR6,
 566         SET_DR7,
 567         SET_RSP,
 568         SET_RIP,
 569         SET_RAX,
 570         SET_RFLAGS,
 571         DESC_BASE,
 572         DESC_LIMIT,
 573         DESC_ACCESS,
 574         SET_CS,
 575         SET_DS,
 576         SET_ES,
 577         SET_FS,
 578         SET_GS,
 579         SET_SS,
 580         SET_TR,
 581         SET_LDTR,
 582         SET_X2APIC_STATE,
 583         SET_CAP,
 584         CAPNAME,
 585         UNASSIGN_PPTDEV,
 586         GET_GPA_PMAP,
 587         ASSERT_LAPIC_LVT,
 588         SET_RTC_TIME,
 589         SET_RTC_NVRAM,
 590         RTC_NVRAM_OFFSET,
 591 #ifdef BHYVE_SNAPSHOT
 592         SET_CHECKPOINT_FILE,
 593         SET_SUSPEND_FILE,
 594 #endif
 595 };
 596
 597 static void
 598 print_cpus(const char *banner, const cpuset_t *cpus)
 599 {
 600         int i, first;
 601
 602         first = 1;
 603         printf("%s:\t", banner);
 604         if (!CPU_EMPTY(cpus)) {
 605                 for (i = 0; i < CPU_SETSIZE; i++) {
 606                         if (CPU_ISSET(i, cpus)) {
 607                                 printf("%s%d", first ? " " : ", ", i);
 608                                 first = 0;
 609                         }
 610                 }
 611         } else
 612                 printf(" (none)");
 613         printf("\n");
 614 }
 615
 616 static void
 617 print_intinfo(const char *banner, uint64_t info)
 618 {
 619         int type;
 620
 621         printf("%s:\t", banner);
 622         if (info & VM_INTINFO_VALID) {
 623                 type = info & VM_INTINFO_TYPE;
 624                 switch (type) {
 625                 case VM_INTINFO_HWINTR:
 626                         printf("extint");
 627                         break;
 628                 case VM_INTINFO_NMI:
 629                         printf("nmi");
 630                         break;
 631                 case VM_INTINFO_SWINTR:
 632                         printf("swint");
 633                         break;
 634                 default:
 635                         printf("exception");
 636                         break;
 637                 }
 638                 printf(" vector %d", (int)VM_INTINFO_VECTOR(info));
 639                 if (info & VM_INTINFO_DEL_ERRCODE)
 640                         printf(" errcode %#x", (u_int)(info >> 32));
 641         } else {
 642                 printf("n/a");
 643         }
 644         printf("\n");
 645 }
 646
 647 static bool
 648 cpu_vendor_intel(void)
 649 {
 650         u_int regs[4];
 651         char cpu_vendor[13];
 652
 653         do_cpuid(0, regs);
 654         ((u_int *)&cpu_vendor)[0] = regs[1];
 655         ((u_int *)&cpu_vendor)[1] = regs[3];
 656         ((u_int *)&cpu_vendor)[2] = regs[2];
 657         cpu_vendor[12] = '\0';
 658
 659         if (strcmp(cpu_vendor, "AuthenticAMD") == 0) {
 660                 return (false);
 661         } else if (strcmp(cpu_vendor, "HygonGenuine") == 0) {
 662                 return (false);
 663         } else if (strcmp(cpu_vendor, "GenuineIntel") == 0) {
 664                 return (true);
 665         } else {
 666                 fprintf(stderr, "Unknown cpu vendor \"%s\"\n", cpu_vendor);
 667                 exit(1);
 668         }
 669 }
 670
 671 static int
 672 get_all_registers(struct vmctx *ctx, int vcpu)
 673 {
 674         uint64_t cr0, cr2, cr3, cr4, dr0, dr1, dr2, dr3, dr6, dr7;
 675         uint64_t rsp, rip, rflags, efer;
 676         uint64_t rax, rbx, rcx, rdx, rsi, rdi, rbp;
 677         uint64_t r8, r9, r10, r11, r12, r13, r14, r15;
 678         int error = 0;
 679
 680         if (!error && (get_efer || get_all)) {
 681                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_EFER, &efer);
 682                 if (error == 0)
 683                         printf("efer[%d]\t\t0x%016lx\n", vcpu, efer);
 684         }
 685
 686         if (!error && (get_cr0 || get_all)) {
 687                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_CR0, &cr0);
 688                 if (error == 0)
 689                         printf("cr0[%d]\t\t0x%016lx\n", vcpu, cr0);
 690         }
 691
 692         if (!error && (get_cr2 || get_all)) {
 693                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_CR2, &cr2);
 694                 if (error == 0)
 695                         printf("cr2[%d]\t\t0x%016lx\n", vcpu, cr2);
 696         }
 697
 698         if (!error && (get_cr3 || get_all)) {
 699                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_CR3, &cr3);
 700                 if (error == 0)
 701                         printf("cr3[%d]\t\t0x%016lx\n", vcpu, cr3);
 702         }
 703
 704         if (!error && (get_cr4 || get_all)) {
 705                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_CR4, &cr4);
 706                 if (error == 0)
 707                         printf("cr4[%d]\t\t0x%016lx\n", vcpu, cr4);
 708         }
 709
 710         if (!error && (get_dr0 || get_all)) {
 711                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_DR0, &dr0);
 712                 if (error == 0)
 713                         printf("dr0[%d]\t\t0x%016lx\n", vcpu, dr0);
 714         }
 715
 716         if (!error && (get_dr1 || get_all)) {
 717                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_DR1, &dr1);
 718                 if (error == 0)
 719                         printf("dr1[%d]\t\t0x%016lx\n", vcpu, dr1);
 720         }
 721
 722         if (!error && (get_dr2 || get_all)) {
 723                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_DR2, &dr2);
 724                 if (error == 0)
 725                         printf("dr2[%d]\t\t0x%016lx\n", vcpu, dr2);
 726         }
 727
 728         if (!error && (get_dr3 || get_all)) {
 729                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_DR3, &dr3);
 730                 if (error == 0)
 731                         printf("dr3[%d]\t\t0x%016lx\n", vcpu, dr3);
 732         }
 733
 734         if (!error && (get_dr6 || get_all)) {
 735                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_DR6, &dr6);
 736                 if (error == 0)
 737                         printf("dr6[%d]\t\t0x%016lx\n", vcpu, dr6);
 738         }
 739
 740         if (!error && (get_dr7 || get_all)) {
 741                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_DR7, &dr7);
 742                 if (error == 0)
 743                         printf("dr7[%d]\t\t0x%016lx\n", vcpu, dr7);
 744         }
 745
 746         if (!error && (get_rsp || get_all)) {
 747                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_RSP, &rsp);
 748                 if (error == 0)
 749                         printf("rsp[%d]\t\t0x%016lx\n", vcpu, rsp);
 750         }
 751
 752         if (!error && (get_rip || get_all)) {
 753                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_RIP, &rip);
 754                 if (error == 0)
 755                         printf("rip[%d]\t\t0x%016lx\n", vcpu, rip);
 756         }
 757
 758         if (!error && (get_rax || get_all)) {
 759                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_RAX, &rax);
 760                 if (error == 0)
 761                         printf("rax[%d]\t\t0x%016lx\n", vcpu, rax);
 762         }
 763
 764         if (!error && (get_rbx || get_all)) {
 765                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_RBX, &rbx);
 766                 if (error == 0)
 767                         printf("rbx[%d]\t\t0x%016lx\n", vcpu, rbx);
 768         }
 769
 770         if (!error && (get_rcx || get_all)) {
 771                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_RCX, &rcx);
 772                 if (error == 0)
 773                         printf("rcx[%d]\t\t0x%016lx\n", vcpu, rcx);
 774         }
 775
 776         if (!error && (get_rdx || get_all)) {
 777                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_RDX, &rdx);
 778                 if (error == 0)
 779                         printf("rdx[%d]\t\t0x%016lx\n", vcpu, rdx);
 780         }
 781
 782         if (!error && (get_rsi || get_all)) {
 783                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_RSI, &rsi);
 784                 if (error == 0)
 785                         printf("rsi[%d]\t\t0x%016lx\n", vcpu, rsi);
 786         }
 787
 788         if (!error && (get_rdi || get_all)) {
 789                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_RDI, &rdi);
 790                 if (error == 0)
 791                         printf("rdi[%d]\t\t0x%016lx\n", vcpu, rdi);
 792         }
 793
 794         if (!error && (get_rbp || get_all)) {
 795                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_RBP, &rbp);
 796                 if (error == 0)
 797                         printf("rbp[%d]\t\t0x%016lx\n", vcpu, rbp);
 798         }
 799
 800         if (!error && (get_r8 || get_all)) {
 801                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_R8, &r8);
 802                 if (error == 0)
 803                         printf("r8[%d]\t\t0x%016lx\n", vcpu, r8);
 804         }
 805
 806         if (!error && (get_r9 || get_all)) {
 807                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_R9, &r9);
 808                 if (error == 0)
 809                         printf("r9[%d]\t\t0x%016lx\n", vcpu, r9);
 810         }
 811
 812         if (!error && (get_r10 || get_all)) {
 813                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_R10, &r10);
 814                 if (error == 0)
 815                         printf("r10[%d]\t\t0x%016lx\n", vcpu, r10);
 816         }
 817
 818         if (!error && (get_r11 || get_all)) {
 819                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_R11, &r11);
 820                 if (error == 0)
 821                         printf("r11[%d]\t\t0x%016lx\n", vcpu, r11);
 822         }
 823
 824         if (!error && (get_r12 || get_all)) {
 825                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_R12, &r12);
 826                 if (error == 0)
 827                         printf("r12[%d]\t\t0x%016lx\n", vcpu, r12);
 828         }
 829
 830         if (!error && (get_r13 || get_all)) {
 831                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_R13, &r13);
 832                 if (error == 0)
 833                         printf("r13[%d]\t\t0x%016lx\n", vcpu, r13);
 834         }
 835
 836         if (!error && (get_r14 || get_all)) {
 837                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_R14, &r14);
 838                 if (error == 0)
 839                         printf("r14[%d]\t\t0x%016lx\n", vcpu, r14);
 840         }
 841
 842         if (!error && (get_r15 || get_all)) {
 843                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_R15, &r15);
 844                 if (error == 0)
 845                         printf("r15[%d]\t\t0x%016lx\n", vcpu, r15);
 846         }
 847
 848         if (!error && (get_rflags || get_all)) {
 849                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_RFLAGS,
 850                                         &rflags);
 851                 if (error == 0)
 852                         printf("rflags[%d]\t0x%016lx\n", vcpu, rflags);
 853         }
 854
 855         return (error);
 856 }
 857
 858 static int
 859 get_all_segments(struct vmctx *ctx, int vcpu)
 860 {
 861         uint64_t cs, ds, es, fs, gs, ss, tr, ldtr;
 862         int error = 0;
 863
 864         if (!error && (get_desc_ds || get_all)) {
 865                 error = vm_get_desc(ctx, vcpu, VM_REG_GUEST_DS,
 866                                    &desc_base, &desc_limit, &desc_access);
 867                 if (error == 0) {
 868                         printf("ds desc[%d]\t0x%016lx/0x%08x/0x%08x\n",
 869                               vcpu, desc_base, desc_limit, desc_access);
 870                 }
 871         }
 872
 873         if (!error && (get_desc_es || get_all)) {
 874                 error = vm_get_desc(ctx, vcpu, VM_REG_GUEST_ES,
 875                                     &desc_base, &desc_limit, &desc_access);
 876                 if (error == 0) {
 877                         printf("es desc[%d]\t0x%016lx/0x%08x/0x%08x\n",
 878                                vcpu, desc_base, desc_limit, desc_access);
 879                 }
 880         }
 881
 882         if (!error && (get_desc_fs || get_all)) {
 883                 error = vm_get_desc(ctx, vcpu, VM_REG_GUEST_FS,
 884                                     &desc_base, &desc_limit, &desc_access);
 885                 if (error == 0) {
 886                         printf("fs desc[%d]\t0x%016lx/0x%08x/0x%08x\n",
 887                                vcpu, desc_base, desc_limit, desc_access);
 888                 }
 889         }
 890
 891         if (!error && (get_desc_gs || get_all)) {
 892                 error = vm_get_desc(ctx, vcpu, VM_REG_GUEST_GS,
 893                                     &desc_base, &desc_limit, &desc_access);
 894                 if (error == 0) {
 895                         printf("gs desc[%d]\t0x%016lx/0x%08x/0x%08x\n",
 896                                vcpu, desc_base, desc_limit, desc_access);
 897                 }
 898         }
 899
 900         if (!error && (get_desc_ss || get_all)) {
 901                 error = vm_get_desc(ctx, vcpu, VM_REG_GUEST_SS,
 902                                     &desc_base, &desc_limit, &desc_access);
 903                 if (error == 0) {
 904                         printf("ss desc[%d]\t0x%016lx/0x%08x/0x%08x\n",
 905                                vcpu, desc_base, desc_limit, desc_access);
 906                 }
 907         }
 908
 909         if (!error && (get_desc_cs || get_all)) {
 910                 error = vm_get_desc(ctx, vcpu, VM_REG_GUEST_CS,
 911                                     &desc_base, &desc_limit, &desc_access);
 912                 if (error == 0) {
 913                         printf("cs desc[%d]\t0x%016lx/0x%08x/0x%08x\n",
 914                                vcpu, desc_base, desc_limit, desc_access);
 915                 }
 916         }
 917
 918         if (!error && (get_desc_tr || get_all)) {
 919                 error = vm_get_desc(ctx, vcpu, VM_REG_GUEST_TR,
 920                                     &desc_base, &desc_limit, &desc_access);
 921                 if (error == 0) {
 922                         printf("tr desc[%d]\t0x%016lx/0x%08x/0x%08x\n",
 923                                vcpu, desc_base, desc_limit, desc_access);
 924                 }
 925         }
 926
 927         if (!error && (get_desc_ldtr || get_all)) {
 928                 error = vm_get_desc(ctx, vcpu, VM_REG_GUEST_LDTR,
 929                                     &desc_base, &desc_limit, &desc_access);
 930                 if (error == 0) {
 931                         printf("ldtr desc[%d]\t0x%016lx/0x%08x/0x%08x\n",
 932                                vcpu, desc_base, desc_limit, desc_access);
 933                 }
 934         }
 935
 936         if (!error && (get_desc_gdtr || get_all)) {
 937                 error = vm_get_desc(ctx, vcpu, VM_REG_GUEST_GDTR,
 938                                     &desc_base, &desc_limit, &desc_access);
 939                 if (error == 0) {
 940                         printf("gdtr[%d]\t\t0x%016lx/0x%08x\n",
 941                                vcpu, desc_base, desc_limit);
 942                 }
 943         }
 944
 945         if (!error && (get_desc_idtr || get_all)) {
 946                 error = vm_get_desc(ctx, vcpu, VM_REG_GUEST_IDTR,
 947                                     &desc_base, &desc_limit, &desc_access);
 948                 if (error == 0) {
 949                         printf("idtr[%d]\t\t0x%016lx/0x%08x\n",
 950                                vcpu, desc_base, desc_limit);
 951                 }
 952         }
 953
 954         if (!error && (get_cs || get_all)) {
 955                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_CS, &cs);
 956                 if (error == 0)
 957                         printf("cs[%d]\t\t0x%04lx\n", vcpu, cs);
 958         }
 959
 960         if (!error && (get_ds || get_all)) {
 961                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_DS, &ds);
 962                 if (error == 0)
 963                         printf("ds[%d]\t\t0x%04lx\n", vcpu, ds);
 964         }
 965
 966         if (!error && (get_es || get_all)) {
 967                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_ES, &es);
 968                 if (error == 0)
 969                         printf("es[%d]\t\t0x%04lx\n", vcpu, es);
 970         }
 971
 972         if (!error && (get_fs || get_all)) {
 973                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_FS, &fs);
 974                 if (error == 0)
 975                         printf("fs[%d]\t\t0x%04lx\n", vcpu, fs);
 976         }
 977
 978         if (!error && (get_gs || get_all)) {
 979                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_GS, &gs);
 980                 if (error == 0)
 981                         printf("gs[%d]\t\t0x%04lx\n", vcpu, gs);
 982         }
 983
 984         if (!error && (get_ss || get_all)) {
 985                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_SS, &ss);
 986                 if (error == 0)
 987                         printf("ss[%d]\t\t0x%04lx\n", vcpu, ss);
 988         }
 989
 990         if (!error && (get_tr || get_all)) {
 991                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_TR, &tr);
 992                 if (error == 0)
 993                         printf("tr[%d]\t\t0x%04lx\n", vcpu, tr);
 994         }
 995
 996         if (!error && (get_ldtr || get_all)) {
 997                 error = vm_get_register(ctx, vcpu, VM_REG_GUEST_LDTR, &ldtr);
 998                 if (error == 0)
 999                         printf("ldtr[%d]\t\t0x%04lx\n", vcpu, ldtr);
1000         }
1001
1002         return (error);
1003 }
1004
1005 static int
1006 get_misc_vmcs(struct vmctx *ctx, int vcpu)
1007 {
1008         uint64_t ctl, cr0, cr3, cr4, rsp, rip, pat, addr, u64;
1009         int error = 0;
1010
1011         if (!error && (get_cr0_mask || get_all)) {
1012                 uint64_t cr0mask;
1013                 error = vm_get_vmcs_field(ctx, vcpu, VMCS_CR0_MASK, &cr0mask);
1014                 if (error == 0)
1015                         printf("cr0_mask[%d]\t\t0x%016lx\n", vcpu, cr0mask);
1016         }
1017
1018         if (!error && (get_cr0_shadow || get_all)) {
1019                 uint64_t cr0shadow;
1020                 error = vm_get_vmcs_field(ctx, vcpu, VMCS_CR0_SHADOW,
1021                                           &cr0shadow);
1022                 if (error == 0)
1023                         printf("cr0_shadow[%d]\t\t0x%016lx\n", vcpu, cr0shadow);
1024         }
1025
1026         if (!error && (get_cr4_mask || get_all)) {
1027                 uint64_t cr4mask;
1028                 error = vm_get_vmcs_field(ctx, vcpu, VMCS_CR4_MASK, &cr4mask);
1029                 if (error == 0)
1030                         printf("cr4_mask[%d]\t\t0x%016lx\n", vcpu, cr4mask);
1031         }
1032
1033         if (!error && (get_cr4_shadow || get_all)) {
1034                 uint64_t cr4shadow;
1035                 error = vm_get_vmcs_field(ctx, vcpu, VMCS_CR4_SHADOW,
1036                                           &cr4shadow);
1037                 if (error == 0)
1038                         printf("cr4_shadow[%d]\t\t0x%016lx\n", vcpu, cr4shadow);
1039         }
1040
1041         if (!error && (get_cr3_targets || get_all)) {
1042                 uint64_t target_count, target_addr;
1043                 error = vm_get_vmcs_field(ctx, vcpu, VMCS_CR3_TARGET_COUNT,
1044                                           &target_count);
1045                 if (error == 0) {
1046                         printf("cr3_target_count[%d]\t0x%016lx\n",
1047                                 vcpu, target_count);
1048                 }
1049
1050                 error = vm_get_vmcs_field(ctx, vcpu, VMCS_CR3_TARGET0,
1051                                           &target_addr);
1052                 if (error == 0) {
1053                         printf("cr3_target0[%d]\t\t0x%016lx\n",
1054                                 vcpu, target_addr);
1055                 }
1056
1057                 error = vm_get_vmcs_field(ctx, vcpu, VMCS_CR3_TARGET1,
1058                                           &target_addr);
1059                 if (error == 0) {
1060                         printf("cr3_target1[%d]\t\t0x%016lx\n",
1061                                 vcpu, target_addr);
1062                 }
1063
1064                 error = vm_get_vmcs_field(ctx, vcpu, VMCS_CR3_TARGET2,
1065                                           &target_addr);
1066                 if (error == 0) {
1067                         printf("cr3_target2[%d]\t\t0x%016lx\n",
1068                                 vcpu, target_addr);
1069                 }
1070
1071                 error = vm_get_vmcs_field(ctx, vcpu, VMCS_CR3_TARGET3,
1072                                           &target_addr);
1073                 if (error == 0) {
1074                         printf("cr3_target3[%d]\t\t0x%016lx\n",
1075                                 vcpu, target_addr);
1076                 }
1077         }
1078
1079         if (!error && (get_pinbased_ctls || get_all)) {
1080                 error = vm_get_vmcs_field(ctx, vcpu, VMCS_PIN_BASED_CTLS, &ctl);
1081                 if (error == 0)
1082                         printf("pinbased_ctls[%d]\t0x%016lx\n", vcpu, ctl);
1083         }
1084
1085         if (!error && (get_procbased_ctls || get_all)) {
1086                 error = vm_get_vmcs_field(ctx, vcpu,
1087                                           VMCS_PRI_PROC_BASED_CTLS, &ctl);
1088                 if (error == 0)
1089                         printf("procbased_ctls[%d]\t0x%016lx\n", vcpu, ctl);
1090         }
1091
1092         if (!error && (get_procbased_ctls2 || get_all)) {
1093                 error = vm_get_vmcs_field(ctx, vcpu,
1094                                           VMCS_SEC_PROC_BASED_CTLS, &ctl);
1095                 if (error == 0)
1096                         printf("procbased_ctls2[%d]\t0x%016lx\n", vcpu, ctl);
1097         }
1098
1099         if (!error && (get_vmcs_gla || get_all)) {
1100                 error = vm_get_vmcs_field(ctx, vcpu,
1101                                           VMCS_GUEST_LINEAR_ADDRESS, &u64);
1102                 if (error == 0)
1103                         printf("gla[%d]\t\t0x%016lx\n", vcpu, u64);
1104         }
1105
1106         if (!error && (get_vmcs_gpa || get_all)) {
1107                 error = vm_get_vmcs_field(ctx, vcpu,
1108                                           VMCS_GUEST_PHYSICAL_ADDRESS, &u64);
1109                 if (error == 0)
1110                         printf("gpa[%d]\t\t0x%016lx\n", vcpu, u64);
1111         }
1112
1113         if (!error && (get_vmcs_entry_interruption_info ||
1114                 get_all)) {
1115                 error = vm_get_vmcs_field(ctx, vcpu, VMCS_ENTRY_INTR_INFO,&u64);
1116                 if (error == 0) {
1117                         printf("entry_interruption_info[%d]\t0x%016lx\n",
1118                                 vcpu, u64);
1119                 }
1120         }
1121
1122         if (!error && (get_tpr_threshold || get_all)) {
1123                 uint64_t threshold;
1124                 error = vm_get_vmcs_field(ctx, vcpu, VMCS_TPR_THRESHOLD,
1125                                           &threshold);
1126                 if (error == 0)
1127                         printf("tpr_threshold[%d]\t0x%016lx\n", vcpu, threshold);
1128         }
1129
1130         if (!error && (get_inst_err || get_all)) {
1131                 uint64_t insterr;
1132                 error = vm_get_vmcs_field(ctx, vcpu, VMCS_INSTRUCTION_ERROR,
1133                                           &insterr);
1134                 if (error == 0) {
1135                         printf("instruction_error[%d]\t0x%016lx\n",
1136                                 vcpu, insterr);
1137                 }
1138         }
1139
1140         if (!error && (get_exit_ctls || get_all)) {
1141                 error = vm_get_vmcs_field(ctx, vcpu, VMCS_EXIT_CTLS, &ctl);
1142                 if (error == 0)
1143                         printf("exit_ctls[%d]\t\t0x%016lx\n", vcpu, ctl);
1144         }
1145
1146         if (!error && (get_entry_ctls || get_all)) {
1147                 error = vm_get_vmcs_field(ctx, vcpu, VMCS_ENTRY_CTLS, &ctl);
1148                 if (error == 0)
1149                         printf("entry_ctls[%d]\t\t0x%016lx\n", vcpu, ctl);
1150         }
1151
1152         if (!error && (get_host_pat || get_all)) {
1153                 error = vm_get_vmcs_field(ctx, vcpu, VMCS_HOST_IA32_PAT, &pat);
1154                 if (error == 0)
1155                         printf("host_pat[%d]\t\t0x%016lx\n", vcpu, pat);
1156         }
1157
1158         if (!error && (get_host_cr0 || get_all)) {
1159                 error = vm_get_vmcs_field(ctx, vcpu, VMCS_HOST_CR0, &cr0);
1160                 if (error == 0)
1161                         printf("host_cr0[%d]\t\t0x%016lx\n", vcpu, cr0);
1162         }
1163
1164         if (!error && (get_host_cr3 || get_all)) {
1165                 error = vm_get_vmcs_field(ctx, vcpu, VMCS_HOST_CR3, &cr3);
1166                 if (error == 0)
1167                         printf("host_cr3[%d]\t\t0x%016lx\n", vcpu, cr3);
1168         }
1169
1170         if (!error && (get_host_cr4 || get_all)) {
1171                 error = vm_get_vmcs_field(ctx, vcpu, VMCS_HOST_CR4, &cr4);
1172                 if (error == 0)
1173                         printf("host_cr4[%d]\t\t0x%016lx\n", vcpu, cr4);
1174         }
1175
1176         if (!error && (get_host_rip || get_all)) {
1177                 error = vm_get_vmcs_field(ctx, vcpu, VMCS_HOST_RIP, &rip);
1178                 if (error == 0)
1179                         printf("host_rip[%d]\t\t0x%016lx\n", vcpu, rip);
1180         }
1181
1182         if (!error && (get_host_rsp || get_all)) {
1183                 error = vm_get_vmcs_field(ctx, vcpu, VMCS_HOST_RSP, &rsp);
1184                 if (error == 0)
1185                         printf("host_rsp[%d]\t\t0x%016lx\n", vcpu, rsp);
1186         }
1187
1188         if (!error && (get_vmcs_link || get_all)) {
1189                 error = vm_get_vmcs_field(ctx, vcpu, VMCS_LINK_POINTER, &addr);
1190                 if (error == 0)
1191                         printf("vmcs_pointer[%d]\t0x%016lx\n", vcpu, addr);
1192         }
1193
1194         if (!error && (get_vmcs_exit_interruption_info || get_all)) {
1195                 error = vm_get_vmcs_field(ctx, vcpu, VMCS_EXIT_INTR_INFO, &u64);
1196                 if (error == 0) {
1197                         printf("vmcs_exit_interruption_info[%d]\t0x%016lx\n",
1198                                 vcpu, u64);
1199                 }
1200         }
1201
1202         if (!error && (get_vmcs_exit_interruption_error || get_all)) {
1203                 error = vm_get_vmcs_field(ctx, vcpu, VMCS_EXIT_INTR_ERRCODE,
1204                                           &u64);
1205                 if (error == 0) {
1206                         printf("vmcs_exit_interruption_error[%d]\t0x%016lx\n",
1207                                 vcpu, u64);
1208                 }
1209         }
1210
1211         if (!error && (get_vmcs_interruptibility || get_all)) {
1212                 error = vm_get_vmcs_field(ctx, vcpu,
1213                                           VMCS_GUEST_INTERRUPTIBILITY, &u64);
1214                 if (error == 0) {
1215                         printf("vmcs_guest_interruptibility[%d]\t0x%016lx\n",
1216                                 vcpu, u64);
1217                 }
1218         }
1219
1220         if (!error && (get_vmcs_exit_inst_length || get_all)) {
1221                 error = vm_get_vmcs_field(ctx, vcpu,
1222                     VMCS_EXIT_INSTRUCTION_LENGTH, &u64);
1223                 if (error == 0)
1224                         printf("vmcs_exit_inst_length[%d]\t0x%08x\n", vcpu,
1225                             (uint32_t)u64);
1226         }
1227
1228         if (!error && (get_vmcs_exit_qualification || get_all)) {
1229                 error = vm_get_vmcs_field(ctx, vcpu, VMCS_EXIT_QUALIFICATION,
1230                                           &u64);
1231                 if (error == 0)
1232                         printf("vmcs_exit_qualification[%d]\t0x%016lx\n",
1233                                 vcpu, u64);
1234         }
1235
1236         return (error);
1237 }
1238
1239 static int
1240 get_misc_vmcb(struct vmctx *ctx, int vcpu)
1241 {
1242         uint64_t ctl, addr;
1243         int error = 0;
1244
1245         if (!error && (get_vmcb_intercept || get_all)) {
1246                 error = vm_get_vmcb_field(ctx, vcpu, VMCB_OFF_CR_INTERCEPT, 4,
1247                     &ctl);
1248                 if (error == 0)
1249                         printf("cr_intercept[%d]\t0x%08x\n", vcpu, (int)ctl);
1250
1251                 error = vm_get_vmcb_field(ctx, vcpu, VMCB_OFF_DR_INTERCEPT, 4,
1252                     &ctl);
1253                 if (error == 0)
1254                         printf("dr_intercept[%d]\t0x%08x\n", vcpu, (int)ctl);
1255
1256                 error = vm_get_vmcb_field(ctx, vcpu, VMCB_OFF_EXC_INTERCEPT, 4,
1257                     &ctl);
1258                 if (error == 0)
1259                         printf("exc_intercept[%d]\t0x%08x\n", vcpu, (int)ctl);
1260
1261                 error = vm_get_vmcb_field(ctx, vcpu, VMCB_OFF_INST1_INTERCEPT,
1262                     4, &ctl);
1263                 if (error == 0)
1264                         printf("inst1_intercept[%d]\t0x%08x\n", vcpu, (int)ctl);
1265
1266                 error = vm_get_vmcb_field(ctx, vcpu, VMCB_OFF_INST2_INTERCEPT,
1267                     4, &ctl);
1268                 if (error == 0)
1269                         printf("inst2_intercept[%d]\t0x%08x\n", vcpu, (int)ctl);
1270         }
1271
1272         if (!error && (get_vmcb_tlb_ctrl || get_all)) {
1273                 error = vm_get_vmcb_field(ctx, vcpu, VMCB_OFF_TLB_CTRL,
1274                                           4, &ctl);
1275                 if (error == 0)
1276                         printf("TLB ctrl[%d]\t0x%016lx\n", vcpu, ctl);
1277         }
1278
1279         if (!error && (get_vmcb_exit_details || get_all)) {
1280                 error = vm_get_vmcb_field(ctx, vcpu, VMCB_OFF_EXITINFO1,
1281                                           8, &ctl);
1282                 if (error == 0)
1283                         printf("exitinfo1[%d]\t0x%016lx\n", vcpu, ctl);
1284                 error = vm_get_vmcb_field(ctx, vcpu, VMCB_OFF_EXITINFO2,
1285                                           8, &ctl);
1286                 if (error == 0)
1287                         printf("exitinfo2[%d]\t0x%016lx\n", vcpu, ctl);
1288                 error = vm_get_vmcb_field(ctx, vcpu, VMCB_OFF_EXITINTINFO,
1289                                           8, &ctl);
1290                 if (error == 0)
1291                         printf("exitintinfo[%d]\t0x%016lx\n", vcpu, ctl);
1292         }
1293
1294         if (!error && (get_vmcb_virq || get_all)) {
1295                 error = vm_get_vmcb_field(ctx, vcpu, VMCB_OFF_VIRQ,
1296                                           8, &ctl);
1297                 if (error == 0)
1298                         printf("v_irq/tpr[%d]\t0x%016lx\n", vcpu, ctl);
1299         }
1300
1301         if (!error && (get_apic_access_addr || get_all)) {
1302                 error = vm_get_vmcb_field(ctx, vcpu, VMCB_OFF_AVIC_BAR, 8,
1303                                           &addr);
1304                 if (error == 0)
1305                         printf("AVIC apic_bar[%d]\t0x%016lx\n", vcpu, addr);
1306         }
1307
1308         if (!error && (get_virtual_apic_addr || get_all)) {
1309                 error = vm_get_vmcb_field(ctx, vcpu, VMCB_OFF_AVIC_PAGE, 8,
1310                                           &addr);
1311                 if (error == 0)
1312                         printf("AVIC backing page[%d]\t0x%016lx\n", vcpu, addr);
1313         }
1314
1315         if (!error && (get_avic_table || get_all)) {
1316                 error = vm_get_vmcb_field(ctx, vcpu, VMCB_OFF_AVIC_LT, 8,
1317                                           &addr);
1318                 if (error == 0)
1319                         printf("AVIC logical table[%d]\t0x%016lx\n",
1320                                 vcpu, addr);
1321                 error = vm_get_vmcb_field(ctx, vcpu, VMCB_OFF_AVIC_PT, 8,
1322                                           &addr);
1323                 if (error == 0)
1324                         printf("AVIC physical table[%d]\t0x%016lx\n",
1325                                 vcpu, addr);
1326         }
1327
1328         return (error);
1329 }
1330
1331 static struct option *
1332 setup_options(bool cpu_intel)
1333 {
1334         const struct option common_opts[] = {
1335                 { "vm",         REQ_ARG,        0,      VMNAME },
1336                 { "cpu",        REQ_ARG,        0,      VCPU },
1337                 { "set-mem",    REQ_ARG,        0,      SET_MEM },
1338                 { "set-efer",   REQ_ARG,        0,      SET_EFER },
1339                 { "set-cr0",    REQ_ARG,        0,      SET_CR0 },
1340                 { "set-cr2",    REQ_ARG,        0,      SET_CR2 },
1341                 { "set-cr3",    REQ_ARG,        0,      SET_CR3 },
1342                 { "set-cr4",    REQ_ARG,        0,      SET_CR4 },
1343                 { "set-dr0",    REQ_ARG,        0,      SET_DR0 },
1344                 { "set-dr1",    REQ_ARG,        0,      SET_DR1 },
1345                 { "set-dr2",    REQ_ARG,        0,      SET_DR2 },
1346                 { "set-dr3",    REQ_ARG,        0,      SET_DR3 },
1347                 { "set-dr6",    REQ_ARG,        0,      SET_DR6 },
1348                 { "set-dr7",    REQ_ARG,        0,      SET_DR7 },
1349                 { "set-rsp",    REQ_ARG,        0,      SET_RSP },
1350                 { "set-rip",    REQ_ARG,        0,      SET_RIP },
1351                 { "set-rax",    REQ_ARG,        0,      SET_RAX },
1352                 { "set-rflags", REQ_ARG,        0,      SET_RFLAGS },
1353                 { "desc-base",  REQ_ARG,        0,      DESC_BASE },
1354                 { "desc-limit", REQ_ARG,        0,      DESC_LIMIT },
1355                 { "desc-access",REQ_ARG,        0,      DESC_ACCESS },
1356                 { "set-cs",     REQ_ARG,        0,      SET_CS },
1357                 { "set-ds",     REQ_ARG,        0,      SET_DS },
1358                 { "set-es",     REQ_ARG,        0,      SET_ES },
1359                 { "set-fs",     REQ_ARG,        0,      SET_FS },
1360                 { "set-gs",     REQ_ARG,        0,      SET_GS },
1361                 { "set-ss",     REQ_ARG,        0,      SET_SS },
1362                 { "set-tr",     REQ_ARG,        0,      SET_TR },
1363                 { "set-ldtr",   REQ_ARG,        0,      SET_LDTR },
1364                 { "set-x2apic-state",REQ_ARG,   0,      SET_X2APIC_STATE },
1365                 { "capname",    REQ_ARG,        0,      CAPNAME },
1366                 { "unassign-pptdev", REQ_ARG,   0,      UNASSIGN_PPTDEV },
1367                 { "setcap",     REQ_ARG,        0,      SET_CAP },
1368                 { "get-gpa-pmap", REQ_ARG,      0,      GET_GPA_PMAP },
1369                 { "assert-lapic-lvt", REQ_ARG,  0,      ASSERT_LAPIC_LVT },
1370                 { "get-rtc-time", NO_ARG,       &get_rtc_time,  1 },
1371                 { "set-rtc-time", REQ_ARG,      0,      SET_RTC_TIME },
1372                 { "rtc-nvram-offset", REQ_ARG,  0,      RTC_NVRAM_OFFSET },
1373                 { "get-rtc-nvram", NO_ARG,      &get_rtc_nvram, 1 },
1374                 { "set-rtc-nvram", REQ_ARG,     0,      SET_RTC_NVRAM },
1375                 { "getcap",     NO_ARG,         &getcap,        1 },
1376                 { "get-stats",  NO_ARG,         &get_stats,     1 },
1377                 { "get-desc-ds",NO_ARG,         &get_desc_ds,   1 },
1378                 { "set-desc-ds",NO_ARG,         &set_desc_ds,   1 },
1379                 { "get-desc-es",NO_ARG,         &get_desc_es,   1 },
1380                 { "set-desc-es",NO_ARG,         &set_desc_es,   1 },
1381                 { "get-desc-ss",NO_ARG,         &get_desc_ss,   1 },
1382                 { "set-desc-ss",NO_ARG,         &set_desc_ss,   1 },
1383                 { "get-desc-cs",NO_ARG,         &get_desc_cs,   1 },
1384                 { "set-desc-cs",NO_ARG,         &set_desc_cs,   1 },
1385                 { "get-desc-fs",NO_ARG,         &get_desc_fs,   1 },
1386                 { "set-desc-fs",NO_ARG,         &set_desc_fs,   1 },
1387                 { "get-desc-gs",NO_ARG,         &get_desc_gs,   1 },
1388                 { "set-desc-gs",NO_ARG,         &set_desc_gs,   1 },
1389                 { "get-desc-tr",NO_ARG,         &get_desc_tr,   1 },
1390                 { "set-desc-tr",NO_ARG,         &set_desc_tr,   1 },
1391                 { "set-desc-ldtr", NO_ARG,      &set_desc_ldtr, 1 },
1392                 { "get-desc-ldtr", NO_ARG,      &get_desc_ldtr, 1 },
1393                 { "set-desc-gdtr", NO_ARG,      &set_desc_gdtr, 1 },
1394                 { "get-desc-gdtr", NO_ARG,      &get_desc_gdtr, 1 },
1395                 { "set-desc-idtr", NO_ARG,      &set_desc_idtr, 1 },
1396                 { "get-desc-idtr", NO_ARG,      &get_desc_idtr, 1 },
1397                 { "get-memmap", NO_ARG,         &get_memmap,    1 },
1398                 { "get-memseg", NO_ARG,         &get_memseg,    1 },
1399                 { "get-efer",   NO_ARG,         &get_efer,      1 },
1400                 { "get-cr0",    NO_ARG,         &get_cr0,       1 },
1401                 { "get-cr2",    NO_ARG,         &get_cr2,       1 },
1402                 { "get-cr3",    NO_ARG,         &get_cr3,       1 },
1403                 { "get-cr4",    NO_ARG,         &get_cr4,       1 },
1404                 { "get-dr0",    NO_ARG,         &get_dr0,       1 },
1405                 { "get-dr1",    NO_ARG,         &get_dr1,       1 },
1406                 { "get-dr2",    NO_ARG,         &get_dr2,       1 },
1407                 { "get-dr3",    NO_ARG,         &get_dr3,       1 },
1408                 { "get-dr6",    NO_ARG,         &get_dr6,       1 },
1409                 { "get-dr7",    NO_ARG,         &get_dr7,       1 },
1410                 { "get-rsp",    NO_ARG,         &get_rsp,       1 },
1411                 { "get-rip",    NO_ARG,         &get_rip,       1 },
1412                 { "get-rax",    NO_ARG,         &get_rax,       1 },
1413                 { "get-rbx",    NO_ARG,         &get_rbx,       1 },
1414                 { "get-rcx",    NO_ARG,         &get_rcx,       1 },
1415                 { "get-rdx",    NO_ARG,         &get_rdx,       1 },
1416                 { "get-rsi",    NO_ARG,         &get_rsi,       1 },
1417                 { "get-rdi",    NO_ARG,         &get_rdi,       1 },
1418                 { "get-rbp",    NO_ARG,         &get_rbp,       1 },
1419                 { "get-r8",     NO_ARG,         &get_r8,        1 },
1420                 { "get-r9",     NO_ARG,         &get_r9,        1 },
1421                 { "get-r10",    NO_ARG,         &get_r10,       1 },
1422                 { "get-r11",    NO_ARG,         &get_r11,       1 },
1423                 { "get-r12",    NO_ARG,         &get_r12,       1 },
1424                 { "get-r13",    NO_ARG,         &get_r13,       1 },
1425                 { "get-r14",    NO_ARG,         &get_r14,       1 },
1426                 { "get-r15",    NO_ARG,         &get_r15,       1 },
1427                 { "get-rflags", NO_ARG,         &get_rflags,    1 },
1428                 { "get-cs",     NO_ARG,         &get_cs,        1 },
1429                 { "get-ds",     NO_ARG,         &get_ds,        1 },
1430                 { "get-es",     NO_ARG,         &get_es,        1 },
1431                 { "get-fs",     NO_ARG,         &get_fs,        1 },
1432                 { "get-gs",     NO_ARG,         &get_gs,        1 },
1433                 { "get-ss",     NO_ARG,         &get_ss,        1 },
1434                 { "get-tr",     NO_ARG,         &get_tr,        1 },
1435                 { "get-ldtr",   NO_ARG,         &get_ldtr,      1 },
1436                 { "get-eptp",   NO_ARG,         &get_eptp,      1 },
1437                 { "get-exception-bitmap",
1438                                         NO_ARG, &get_exception_bitmap,  1 },
1439                 { "get-io-bitmap-address",
1440                                         NO_ARG, &get_io_bitmap,         1 },
1441                 { "get-tsc-offset",     NO_ARG, &get_tsc_offset,        1 },
1442                 { "get-msr-bitmap",
1443                                         NO_ARG, &get_msr_bitmap,        1 },
1444                 { "get-msr-bitmap-address",
1445                                         NO_ARG, &get_msr_bitmap_address, 1 },
1446                 { "get-guest-pat",      NO_ARG, &get_guest_pat,         1 },
1447                 { "get-guest-sysenter",
1448                                         NO_ARG, &get_guest_sysenter,    1 },
1449                 { "get-exit-reason",
1450                                         NO_ARG, &get_exit_reason,       1 },
1451                 { "get-x2apic-state",   NO_ARG, &get_x2apic_state,      1 },
1452                 { "get-all",            NO_ARG, &get_all,               1 },
1453                 { "run",                NO_ARG, &run,                   1 },
1454                 { "create",             NO_ARG, &create,                1 },
1455                 { "destroy",            NO_ARG, &destroy,               1 },
1456                 { "inject-nmi",         NO_ARG, &inject_nmi,            1 },
1457                 { "force-reset",        NO_ARG, &force_reset,           1 },
1458                 { "force-poweroff",     NO_ARG, &force_poweroff,        1 },
1459                 { "get-active-cpus",    NO_ARG, &get_active_cpus,       1 },
1460                 { "get-suspended-cpus", NO_ARG, &get_suspended_cpus,    1 },
1461                 { "get-intinfo",        NO_ARG, &get_intinfo,           1 },
1462                 { "get-cpu-topology",   NO_ARG, &get_cpu_topology,      1 },
1463 #ifdef BHYVE_SNAPSHOT
1464                 { "checkpoint",         REQ_ARG, 0,     SET_CHECKPOINT_FILE},
1465                 { "suspend",            REQ_ARG, 0,     SET_SUSPEND_FILE},
1466 #endif
1467         };
1468
1469         const struct option intel_opts[] = {
1470                 { "get-vmcs-pinbased-ctls",
1471                                 NO_ARG,         &get_pinbased_ctls, 1 },
1472                 { "get-vmcs-procbased-ctls",
1473                                 NO_ARG,         &get_procbased_ctls, 1 },
1474                 { "get-vmcs-procbased-ctls2",
1475                                 NO_ARG,         &get_procbased_ctls2, 1 },
1476                 { "get-vmcs-guest-linear-address",
1477                                 NO_ARG,         &get_vmcs_gla,  1 },
1478                 { "get-vmcs-guest-physical-address",
1479                                 NO_ARG,         &get_vmcs_gpa,  1 },
1480                 { "get-vmcs-entry-interruption-info",
1481                                 NO_ARG, &get_vmcs_entry_interruption_info, 1},
1482                 { "get-vmcs-cr0-mask", NO_ARG,  &get_cr0_mask,  1 },
1483                 { "get-vmcs-cr0-shadow", NO_ARG,&get_cr0_shadow, 1 },
1484                 { "get-vmcs-cr4-mask",          NO_ARG, &get_cr4_mask,    1 },
1485                 { "get-vmcs-cr4-shadow",        NO_ARG, &get_cr4_shadow,  1 },
1486                 { "get-vmcs-cr3-targets",       NO_ARG, &get_cr3_targets, 1 },
1487                 { "get-vmcs-tpr-threshold",
1488                                         NO_ARG, &get_tpr_threshold, 1 },
1489                 { "get-vmcs-vpid",      NO_ARG, &get_vpid_asid,     1 },
1490                 { "get-vmcs-exit-ctls", NO_ARG, &get_exit_ctls,     1 },
1491                 { "get-vmcs-entry-ctls",
1492                                         NO_ARG, &get_entry_ctls, 1 },
1493                 { "get-vmcs-instruction-error",
1494                                         NO_ARG, &get_inst_err,  1 },
1495                 { "get-vmcs-host-pat",  NO_ARG, &get_host_pat,  1 },
1496                 { "get-vmcs-host-cr0",
1497                                         NO_ARG, &get_host_cr0,  1 },
1498                 { "get-vmcs-exit-qualification",
1499                                 NO_ARG, &get_vmcs_exit_qualification, 1 },
1500                 { "get-vmcs-exit-inst-length",
1501                                 NO_ARG, &get_vmcs_exit_inst_length, 1 },
1502                 { "get-vmcs-interruptibility",
1503                                 NO_ARG, &get_vmcs_interruptibility, 1 },
1504                 { "get-vmcs-exit-interruption-error",
1505                                 NO_ARG, &get_vmcs_exit_interruption_error, 1 },
1506                 { "get-vmcs-exit-interruption-info",
1507                                 NO_ARG, &get_vmcs_exit_interruption_info, 1 },
1508                 { "get-vmcs-link",      NO_ARG,         &get_vmcs_link, 1 },
1509                 { "get-vmcs-host-cr3",
1510                                         NO_ARG,         &get_host_cr3,  1 },
1511                 { "get-vmcs-host-cr4",
1512                                 NO_ARG,         &get_host_cr4,  1 },
1513                 { "get-vmcs-host-rip",
1514                                 NO_ARG,         &get_host_rip,  1 },
1515                 { "get-vmcs-host-rsp",
1516                                 NO_ARG,         &get_host_rsp,  1 },
1517                 { "get-apic-access-address",
1518                                 NO_ARG,         &get_apic_access_addr, 1},
1519                 { "get-virtual-apic-address",
1520                                 NO_ARG,         &get_virtual_apic_addr, 1}
1521         };
1522
1523         const struct option amd_opts[] = {
1524                 { "get-vmcb-intercepts",
1525                                 NO_ARG, &get_vmcb_intercept,    1 },
1526                 { "get-vmcb-asid",
1527                                 NO_ARG, &get_vpid_asid,         1 },
1528                 { "get-vmcb-exit-details",
1529                                 NO_ARG, &get_vmcb_exit_details, 1 },
1530                 { "get-vmcb-tlb-ctrl",
1531                                 NO_ARG, &get_vmcb_tlb_ctrl,     1 },
1532                 { "get-vmcb-virq",
1533                                 NO_ARG, &get_vmcb_virq,         1 },
1534                 { "get-avic-apic-bar",
1535                                 NO_ARG, &get_apic_access_addr,  1 },
1536                 { "get-avic-backing-page",
1537                                 NO_ARG, &get_virtual_apic_addr, 1 },
1538                 { "get-avic-table",
1539                                 NO_ARG, &get_avic_table,        1 }
1540         };
1541
1542         const struct option null_opt = {
1543                 NULL, 0, NULL, 0
1544         };
1545
1546         struct option *all_opts;
1547         char *cp;
1548         int optlen;
1549
1550         optlen = sizeof(common_opts);
1551
1552         if (cpu_intel)
1553                 optlen += sizeof(intel_opts);
1554         else
1555                 optlen += sizeof(amd_opts);
1556
1557         optlen += sizeof(null_opt);
1558
1559         all_opts = malloc(optlen);
1560
1561         cp = (char *)all_opts;
1562         memcpy(cp, common_opts, sizeof(common_opts));
1563         cp += sizeof(common_opts);
1564
1565         if (cpu_intel) {
1566                 memcpy(cp, intel_opts, sizeof(intel_opts));
1567                 cp += sizeof(intel_opts);
1568         } else {
1569                 memcpy(cp, amd_opts, sizeof(amd_opts));
1570                 cp += sizeof(amd_opts);
1571         }
1572
1573         memcpy(cp, &null_opt, sizeof(null_opt));
1574         cp += sizeof(null_opt);
1575
1576         return (all_opts);
1577 }
1578
1579 static const char *
1580 wday_str(int idx)
1581 {
1582         static const char *weekdays[] = {
1583                 "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"
1584         };
1585
1586         if (idx >= 0 && idx < 7)
1587                 return (weekdays[idx]);
1588         else
1589                 return ("UNK");
1590 }
1591
1592 static const char *
1593 mon_str(int idx)
1594 {
1595         static const char *months[] = {
1596                 "Jan", "Feb", "Mar", "Apr", "May", "Jun",
1597                 "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
1598         };
1599
1600         if (idx >= 0 && idx < 12)
1601                 return (months[idx]);
1602         else
1603                 return ("UNK");
1604 }
1605
1606 static int
1607 show_memmap(struct vmctx *ctx)
1608 {
1609         char name[SPECNAMELEN + 1], numbuf[8];
1610         vm_ooffset_t segoff;
1611         vm_paddr_t gpa;
1612         size_t maplen, seglen;
1613         int error, flags, prot, segid, delim;
1614
1615         printf("Address     Length      Segment     Offset      ");
1616         printf("Prot  Flags\n");
1617
1618         gpa = 0;
1619         while (1) {
1620                 error = vm_mmap_getnext(ctx, &gpa, &segid, &segoff, &maplen,
1621                     &prot, &flags);
1622                 if (error)
1623                         return (errno == ENOENT ? 0 : error);
1624
1625                 error = vm_get_memseg(ctx, segid, &seglen, name, sizeof(name));
1626                 if (error)
1627                         return (error);
1628
1629                 printf("%-12lX", gpa);
1630                 humanize_number(numbuf, sizeof(numbuf), maplen, "B",
1631                     HN_AUTOSCALE, HN_NOSPACE);
1632                 printf("%-12s", numbuf);
1633
1634                 printf("%-12s", name[0] ? name : "sysmem");
1635                 printf("%-12lX", segoff);
1636                 printf("%c%c%c   ", prot & PROT_READ ? 'R' : '-',
1637                     prot & PROT_WRITE ? 'W' : '-',
1638                     prot & PROT_EXEC ? 'X' : '-');
1639
1640                 delim = '\0';
1641                 if (flags & VM_MEMMAP_F_WIRED) {
1642                         printf("%cwired", delim);
1643                         delim = '/';
1644                 }
1645                 if (flags & VM_MEMMAP_F_IOMMU) {
1646                         printf("%ciommu", delim);
1647                         delim = '/';
1648                 }
1649                 printf("\n");
1650
1651                 gpa += maplen;
1652         }
1653 }
1654
1655 static int
1656 show_memseg(struct vmctx *ctx)
1657 {
1658         char name[SPECNAMELEN + 1], numbuf[8];
1659         size_t seglen;
1660         int error, segid;
1661
1662         printf("ID  Length      Name\n");
1663
1664         segid = 0;
1665         while (1) {
1666                 error = vm_get_memseg(ctx, segid, &seglen, name, sizeof(name));
1667                 if (error)
1668                         return (errno == EINVAL ? 0 : error);
1669
1670                 if (seglen) {
1671                         printf("%-4d", segid);
1672                         humanize_number(numbuf, sizeof(numbuf), seglen, "B",
1673                             HN_AUTOSCALE, HN_NOSPACE);
1674                         printf("%-12s", numbuf);
1675                         printf("%s", name[0] ? name : "sysmem");
1676                         printf("\n");
1677                 }
1678                 segid++;
1679         }
1680 }
1681
1682 #ifdef BHYVE_SNAPSHOT
1683 static int
1684 send_checkpoint_op_req(struct vmctx *ctx, struct checkpoint_op *op)
1685 {
1686         struct sockaddr_un addr;
1687         int socket_fd, len, len_sent, total_sent;
1688         int err = 0;
1689         char vmname_buf[MAX_VMNAME];
1690
1691         socket_fd = socket(PF_UNIX, SOCK_STREAM, 0);
1692         if (socket_fd < 0) {
1693                 perror("Error creating bhyvectl socket");
1694                 err = -1;
1695                 goto done;
1696         }
1697
1698         memset(&addr, 0, sizeof(struct sockaddr_un));
1699         addr.sun_family = AF_UNIX;
1700
1701         err = vm_get_name(ctx, vmname_buf, MAX_VMNAME - 1);
1702         if (err != 0) {
1703                 perror("Failed to get VM name");
1704                 goto done;
1705         }
1706
1707         snprintf(addr.sun_path, sizeof(addr.sun_path), "%s/%s", CHECKPOINT_RUN_DIR, vmname_buf);
1708
1709         if (connect(socket_fd, (struct sockaddr *)&addr,
1710                         sizeof(struct sockaddr_un)) != 0) {
1711                 perror("Connect to VM socket failed");
1712                 err = -1;
1713                 goto done;
1714         }
1715
1716         len = sizeof(*op);
1717         total_sent = 0;
1718         while ((len_sent = send(socket_fd, (char *)op + total_sent, len - total_sent, 0)) > 0) {
1719                 total_sent += len_sent;
1720         }
1721
1722         if (len_sent < 0) {
1723                 perror("Failed to send checkpoint operation request");
1724                 err = -1;
1725         }
1726
1727 done:
1728         if (socket_fd > 0)
1729                 close(socket_fd);
1730         return (err);
1731 }
1732
1733 static int
1734 send_start_checkpoint(struct vmctx *ctx, const char *checkpoint_file)
1735 {
1736         struct checkpoint_op op;
1737
1738         op.op = START_CHECKPOINT;
1739         strncpy(op.snapshot_filename, checkpoint_file, MAX_SNAPSHOT_VMNAME);
1740         op.snapshot_filename[MAX_SNAPSHOT_VMNAME - 1] = 0;
1741
1742         return (send_checkpoint_op_req(ctx, &op));
1743 }
1744
1745 static int
1746 send_start_suspend(struct vmctx *ctx, const char *suspend_file)
1747 {
1748         struct checkpoint_op op;
1749
1750         op.op = START_SUSPEND;
1751         strncpy(op.snapshot_filename, suspend_file, MAX_SNAPSHOT_VMNAME);
1752         op.snapshot_filename[MAX_SNAPSHOT_VMNAME - 1] = 0;
1753
1754         return (send_checkpoint_op_req(ctx, &op));
1755 }
1756 #endif
1757
1758 int
1759 main(int argc, char *argv[])
1760 {
1761         char *vmname;
1762         int error, ch, vcpu, ptenum;
1763         vm_paddr_t gpa_pmap;
1764         struct vm_exit vmexit;
1765         uint64_t rax, cr0, cr2, cr3, cr4, dr0, dr1, dr2, dr3, dr6, dr7;
1766         uint64_t rsp, rip, rflags, efer, pat;
1767         uint64_t eptp, bm, addr, u64, pteval[4], *pte, info[2];
1768         struct vmctx *ctx;
1769         cpuset_t cpus;
1770         bool cpu_intel;
1771         uint64_t cs, ds, es, fs, gs, ss, tr, ldtr;
1772         struct tm tm;
1773         struct option *opts;
1774 #ifdef BHYVE_SNAPSHOT
1775         char *checkpoint_file, *suspend_file;
1776 #endif
1777
1778         cpu_intel = cpu_vendor_intel();
1779         opts = setup_options(cpu_intel);
1780
1781         vcpu = 0;
1782         vmname = NULL;
1783         assert_lapic_lvt = -1;
1784         progname = basename(argv[0]);
1785
1786         while ((ch = getopt_long(argc, argv, "", opts, NULL)) != -1) {
1787                 switch (ch) {
1788                 case 0:
1789                         break;
1790                 case VMNAME:
1791                         vmname = optarg;
1792                         break;
1793                 case VCPU:
1794                         vcpu = atoi(optarg);
1795                         break;
1796                 case SET_MEM:
1797                         memsize = atoi(optarg) * MB;
1798                         memsize = roundup(memsize, 2 * MB);
1799                         break;
1800                 case SET_EFER:
1801                         efer = strtoul(optarg, NULL, 0);
1802                         set_efer = 1;
1803                         break;
1804                 case SET_CR0:
1805                         cr0 = strtoul(optarg, NULL, 0);
1806                         set_cr0 = 1;
1807                         break;
1808                 case SET_CR2:
1809                         cr2 = strtoul(optarg, NULL, 0);
1810                         set_cr2 = 1;
1811                         break;
1812                 case SET_CR3:
1813                         cr3 = strtoul(optarg, NULL, 0);
1814                         set_cr3 = 1;
1815                         break;
1816                 case SET_CR4:
1817                         cr4 = strtoul(optarg, NULL, 0);
1818                         set_cr4 = 1;
1819                         break;
1820                 case SET_DR0:
1821                         dr0 = strtoul(optarg, NULL, 0);
1822                         set_dr0 = 1;
1823                         break;
1824                 case SET_DR1:
1825                         dr1 = strtoul(optarg, NULL, 0);
1826                         set_dr1 = 1;
1827                         break;
1828                 case SET_DR2:
1829                         dr2 = strtoul(optarg, NULL, 0);
1830                         set_dr2 = 1;
1831                         break;
1832                 case SET_DR3:
1833                         dr3 = strtoul(optarg, NULL, 0);
1834                         set_dr3 = 1;
1835                         break;
1836                 case SET_DR6:
1837                         dr6 = strtoul(optarg, NULL, 0);
1838                         set_dr6 = 1;
1839                         break;
1840                 case SET_DR7:
1841                         dr7 = strtoul(optarg, NULL, 0);
1842                         set_dr7 = 1;
1843                         break;
1844                 case SET_RSP:
1845                         rsp = strtoul(optarg, NULL, 0);
1846                         set_rsp = 1;
1847                         break;
1848                 case SET_RIP:
1849                         rip = strtoul(optarg, NULL, 0);
1850                         set_rip = 1;
1851                         break;
1852                 case SET_RAX:
1853                         rax = strtoul(optarg, NULL, 0);
1854                         set_rax = 1;
1855                         break;
1856                 case SET_RFLAGS:
1857                         rflags = strtoul(optarg, NULL, 0);
1858                         set_rflags = 1;
1859                         break;
1860                 case DESC_BASE:
1861                         desc_base = strtoul(optarg, NULL, 0);
1862                         break;
1863                 case DESC_LIMIT:
1864                         desc_limit = strtoul(optarg, NULL, 0);
1865                         break;
1866                 case DESC_ACCESS:
1867                         desc_access = strtoul(optarg, NULL, 0);
1868                         break;
1869                 case SET_CS:
1870                         cs = strtoul(optarg, NULL, 0);
1871                         set_cs = 1;
1872                         break;
1873                 case SET_DS:
1874                         ds = strtoul(optarg, NULL, 0);
1875                         set_ds = 1;
1876                         break;
1877                 case SET_ES:
1878                         es = strtoul(optarg, NULL, 0);
1879                         set_es = 1;
1880                         break;
1881                 case SET_FS:
1882                         fs = strtoul(optarg, NULL, 0);
1883                         set_fs = 1;
1884                         break;
1885                 case SET_GS:
1886                         gs = strtoul(optarg, NULL, 0);
1887                         set_gs = 1;
1888                         break;
1889                 case SET_SS:
1890                         ss = strtoul(optarg, NULL, 0);
1891                         set_ss = 1;
1892                         break;
1893                 case SET_TR:
1894                         tr = strtoul(optarg, NULL, 0);
1895                         set_tr = 1;
1896                         break;
1897                 case SET_LDTR:
1898                         ldtr = strtoul(optarg, NULL, 0);
1899                         set_ldtr = 1;
1900                         break;
1901                 case SET_X2APIC_STATE:
1902                         x2apic_state = strtol(optarg, NULL, 0);
1903                         set_x2apic_state = 1;
1904                         break;
1905                 case SET_CAP:
1906                         capval = strtoul(optarg, NULL, 0);
1907                         setcap = 1;
1908                         break;
1909                 case SET_RTC_TIME:
1910                         rtc_secs = strtoul(optarg, NULL, 0);
1911                         set_rtc_time = 1;
1912                         break;
1913                 case SET_RTC_NVRAM:
1914                         rtc_nvram_value = (uint8_t)strtoul(optarg, NULL, 0);
1915                         set_rtc_nvram = 1;
1916                         break;
1917                 case RTC_NVRAM_OFFSET:
1918                         rtc_nvram_offset = strtoul(optarg, NULL, 0);
1919                         break;
1920                 case GET_GPA_PMAP:
1921                         gpa_pmap = strtoul(optarg, NULL, 0);
1922                         get_gpa_pmap = 1;
1923                         break;
1924                 case CAPNAME:
1925                         capname = optarg;
1926                         break;
1927                 case UNASSIGN_PPTDEV:
1928                         unassign_pptdev = 1;
1929                         if (sscanf(optarg, "%d/%d/%d", &bus, &slot, &func) != 3)
1930                                 usage(cpu_intel);
1931                         break;
1932                 case ASSERT_LAPIC_LVT:
1933                         assert_lapic_lvt = atoi(optarg);
1934                         break;
1935 #ifdef BHYVE_SNAPSHOT
1936                 case SET_CHECKPOINT_FILE:
1937                         vm_checkpoint_opt = 1;
1938                         checkpoint_file = optarg;
1939                         break;
1940                 case SET_SUSPEND_FILE:
1941                         vm_suspend_opt = 1;
1942                         suspend_file = optarg;
1943                         break;
1944 #endif
1945                 default:
1946                         usage(cpu_intel);
1947                 }
1948         }
1949         argc -= optind;
1950         argv += optind;
1951
1952         if (vmname == NULL)
1953                 usage(cpu_intel);
1954
1955         error = 0;
1956
1957         if (!error && create)
1958                 error = vm_create(vmname);
1959
1960         if (!error) {
1961                 ctx = vm_open(vmname);
1962                 if (ctx == NULL) {
1963                         printf("VM:%s is not created.\n", vmname);
1964                         exit (1);
1965                 }
1966         }
1967
1968         if (!error && memsize)
1969                 error = vm_setup_memory(ctx, memsize, VM_MMAP_ALL);
1970
1971         if (!error && set_efer)
1972                 error = vm_set_register(ctx, vcpu, VM_REG_GUEST_EFER, efer);
1973
1974         if (!error && set_cr0)
1975                 error = vm_set_register(ctx, vcpu, VM_REG_GUEST_CR0, cr0);
1976
1977         if (!error && set_cr2)
1978                 error = vm_set_register(ctx, vcpu, VM_REG_GUEST_CR2, cr2);
1979
1980         if (!error && set_cr3)
1981                 error = vm_set_register(ctx, vcpu, VM_REG_GUEST_CR3, cr3);
1982
1983         if (!error && set_cr4)
1984                 error = vm_set_register(ctx, vcpu, VM_REG_GUEST_CR4, cr4);
1985
1986         if (!error && set_dr0)
1987                 error = vm_set_register(ctx, vcpu, VM_REG_GUEST_DR0, dr0);
1988
1989         if (!error && set_dr1)
1990                 error = vm_set_register(ctx, vcpu, VM_REG_GUEST_DR1, dr1);
1991
1992         if (!error && set_dr2)
1993                 error = vm_set_register(ctx, vcpu, VM_REG_GUEST_DR2, dr2);
1994
1995         if (!error && set_dr3)
1996                 error = vm_set_register(ctx, vcpu, VM_REG_GUEST_DR3, dr3);
1997
1998         if (!error && set_dr6)
1999                 error = vm_set_register(ctx, vcpu, VM_REG_GUEST_DR6, dr6);
2000
2001         if (!error && set_dr7)
2002                 error = vm_set_register(ctx, vcpu, VM_REG_GUEST_DR7, dr7);
2003
2004         if (!error && set_rsp)
2005                 error = vm_set_register(ctx, vcpu, VM_REG_GUEST_RSP, rsp);
2006
2007         if (!error && set_rip)
2008                 error = vm_set_register(ctx, vcpu, VM_REG_GUEST_RIP, rip);
2009
2010         if (!error && set_rax)
2011                 error = vm_set_register(ctx, vcpu, VM_REG_GUEST_RAX, rax);
2012
2013         if (!error && set_rflags) {
2014                 error = vm_set_register(ctx, vcpu, VM_REG_GUEST_RFLAGS,
2015                                         rflags);
2016         }
2017
2018         if (!error && set_desc_ds) {
2019                 error = vm_set_desc(ctx, vcpu, VM_REG_GUEST_DS,
2020                                     desc_base, desc_limit, desc_access);
2021         }
2022
2023         if (!error && set_desc_es) {
2024                 error = vm_set_desc(ctx, vcpu, VM_REG_GUEST_ES,
2025                                     desc_base, desc_limit, desc_access);
2026         }
2027
2028         if (!error && set_desc_ss) {
2029                 error = vm_set_desc(ctx, vcpu, VM_REG_GUEST_SS,
2030                                     desc_base, desc_limit, desc_access);
2031         }
2032
2033         if (!error && set_desc_cs) {
2034                 error = vm_set_desc(ctx, vcpu, VM_REG_GUEST_CS,
2035                                     desc_base, desc_limit, desc_access);
2036         }
2037
2038         if (!error && set_desc_fs) {
2039                 error = vm_set_desc(ctx, vcpu, VM_REG_GUEST_FS,
2040                                     desc_base, desc_limit, desc_access);
2041         }
2042
2043         if (!error && set_desc_gs) {
2044                 error = vm_set_desc(ctx, vcpu, VM_REG_GUEST_GS,
2045                                     desc_base, desc_limit, desc_access);
2046         }
2047
2048         if (!error && set_desc_tr) {
2049                 error = vm_set_desc(ctx, vcpu, VM_REG_GUEST_TR,
2050                                     desc_base, desc_limit, desc_access);
2051         }
2052
2053         if (!error && set_desc_ldtr) {
2054                 error = vm_set_desc(ctx, vcpu, VM_REG_GUEST_LDTR,
2055                                     desc_base, desc_limit, desc_access);
2056         }
2057
2058         if (!error && set_desc_gdtr) {
2059                 error = vm_set_desc(ctx, vcpu, VM_REG_GUEST_GDTR,
2060                                     desc_base, desc_limit, 0);
2061         }
2062
2063         if (!error && set_desc_idtr) {
2064                 error = vm_set_desc(ctx, vcpu, VM_REG_GUEST_IDTR,
2065                                     desc_base, desc_limit, 0);
2066         }
2067
2068         if (!error && set_cs)
2069                 error = vm_set_register(ctx, vcpu, VM_REG_GUEST_CS, cs);
2070
2071         if (!error && set_ds)
2072                 error = vm_set_register(ctx, vcpu, VM_REG_GUEST_DS, ds);
2073
2074         if (!error && set_es)
2075                 error = vm_set_register(ctx, vcpu, VM_REG_GUEST_ES, es);
2076
2077         if (!error && set_fs)
2078                 error = vm_set_register(ctx, vcpu, VM_REG_GUEST_FS, fs);
2079
2080         if (!error && set_gs)
2081                 error = vm_set_register(ctx, vcpu, VM_REG_GUEST_GS, gs);
2082
2083         if (!error && set_ss)
2084                 error = vm_set_register(ctx, vcpu, VM_REG_GUEST_SS, ss);
2085
2086         if (!error && set_tr)
2087                 error = vm_set_register(ctx, vcpu, VM_REG_GUEST_TR, tr);
2088
2089         if (!error && set_ldtr)
2090                 error = vm_set_register(ctx, vcpu, VM_REG_GUEST_LDTR, ldtr);
2091
2092         if (!error && set_x2apic_state)
2093                 error = vm_set_x2apic_state(ctx, vcpu, x2apic_state);
2094
2095         if (!error && unassign_pptdev)
2096                 error = vm_unassign_pptdev(ctx, bus, slot, func);
2097
2098         if (!error && inject_nmi) {
2099                 error = vm_inject_nmi(ctx, vcpu);
2100         }
2101
2102         if (!error && assert_lapic_lvt != -1) {
2103                 error = vm_lapic_local_irq(ctx, vcpu, assert_lapic_lvt);
2104         }
2105
2106         if (!error && (get_memseg || get_all))
2107                 error = show_memseg(ctx);
2108
2109         if (!error && (get_memmap || get_all))
2110                 error = show_memmap(ctx);
2111
2112         if (!error)
2113                 error = get_all_registers(ctx, vcpu);
2114
2115         if (!error)
2116                 error = get_all_segments(ctx, vcpu);
2117
2118         if (!error) {
2119                 if (cpu_intel)
2120                         error = get_misc_vmcs(ctx, vcpu);
2121                 else
2122                         error = get_misc_vmcb(ctx, vcpu);
2123         }
2124
2125         if (!error && (get_x2apic_state || get_all)) {
2126                 error = vm_get_x2apic_state(ctx, vcpu, &x2apic_state);
2127                 if (error == 0)
2128                         printf("x2apic_state[%d]\t%d\n", vcpu, x2apic_state);
2129         }
2130
2131         if (!error && (get_eptp || get_all)) {
2132                 if (cpu_intel)
2133                         error = vm_get_vmcs_field(ctx, vcpu, VMCS_EPTP, &eptp);
2134                 else
2135                         error = vm_get_vmcb_field(ctx, vcpu, VMCB_OFF_NPT_BASE,
2136                                                    8, &eptp);
2137                 if (error == 0)
2138                         printf("%s[%d]\t\t0x%016lx\n",
2139                                 cpu_intel ? "eptp" : "rvi/npt", vcpu, eptp);
2140         }
2141
2142         if (!error && (get_exception_bitmap || get_all)) {
2143                 if(cpu_intel)
2144                         error = vm_get_vmcs_field(ctx, vcpu,
2145                                                 VMCS_EXCEPTION_BITMAP, &bm);
2146                 else
2147                         error = vm_get_vmcb_field(ctx, vcpu,
2148                                                   VMCB_OFF_EXC_INTERCEPT,
2149                                                   4, &bm);
2150                 if (error == 0)
2151                         printf("exception_bitmap[%d]\t%#lx\n", vcpu, bm);
2152         }
2153
2154         if (!error && (get_io_bitmap || get_all)) {
2155                 if (cpu_intel) {
2156                         error = vm_get_vmcs_field(ctx, vcpu, VMCS_IO_BITMAP_A,
2157                                                   &bm);
2158                         if (error == 0)
2159                                 printf("io_bitmap_a[%d]\t%#lx\n", vcpu, bm);
2160                         error = vm_get_vmcs_field(ctx, vcpu, VMCS_IO_BITMAP_B,
2161                                                   &bm);
2162                         if (error == 0)
2163                                 printf("io_bitmap_b[%d]\t%#lx\n", vcpu, bm);
2164                 } else {
2165                         error = vm_get_vmcb_field(ctx, vcpu,
2166                                                   VMCB_OFF_IO_PERM, 8, &bm);
2167                         if (error == 0)
2168                                 printf("io_bitmap[%d]\t%#lx\n", vcpu, bm);
2169                 }
2170         }
2171
2172         if (!error && (get_tsc_offset || get_all)) {
2173                 uint64_t tscoff;
2174                 if (cpu_intel)
2175                         error = vm_get_vmcs_field(ctx, vcpu, VMCS_TSC_OFFSET,
2176                                                   &tscoff);
2177                 else
2178                         error = vm_get_vmcb_field(ctx, vcpu,
2179                                                   VMCB_OFF_TSC_OFFSET,
2180                                                   8, &tscoff);
2181                 if (error == 0)
2182                         printf("tsc_offset[%d]\t0x%016lx\n", vcpu, tscoff);
2183         }
2184
2185         if (!error && (get_msr_bitmap_address || get_all)) {
2186                 if (cpu_intel)
2187                         error = vm_get_vmcs_field(ctx, vcpu, VMCS_MSR_BITMAP,
2188                                                   &addr);
2189                 else
2190                         error = vm_get_vmcb_field(ctx, vcpu,
2191                                                   VMCB_OFF_MSR_PERM, 8, &addr);
2192                 if (error == 0)
2193                         printf("msr_bitmap[%d]\t\t%#lx\n", vcpu, addr);
2194         }
2195
2196         if (!error && (get_msr_bitmap || get_all)) {
2197                 if (cpu_intel) {
2198                         error = vm_get_vmcs_field(ctx, vcpu,
2199                                                   VMCS_MSR_BITMAP, &addr);
2200                 } else {
2201                         error = vm_get_vmcb_field(ctx, vcpu,
2202                                                   VMCB_OFF_MSR_PERM, 8,
2203                                                   &addr);
2204                 }
2205
2206                 if (error == 0)
2207                         error = dump_msr_bitmap(vcpu, addr, cpu_intel);
2208         }
2209
2210         if (!error && (get_vpid_asid || get_all)) {
2211                 uint64_t vpid;
2212                 if (cpu_intel)
2213                         error = vm_get_vmcs_field(ctx, vcpu, VMCS_VPID, &vpid);
2214                 else
2215                         error = vm_get_vmcb_field(ctx, vcpu, VMCB_OFF_ASID,
2216                                                   4, &vpid);
2217                 if (error == 0)
2218                         printf("%s[%d]\t\t0x%04lx\n",
2219                                 cpu_intel ? "vpid" : "asid", vcpu, vpid);
2220         }
2221
2222         if (!error && (get_guest_pat || get_all)) {
2223                 if (cpu_intel)
2224                         error = vm_get_vmcs_field(ctx, vcpu,
2225                                                   VMCS_GUEST_IA32_PAT, &pat);
2226                 else
2227                         error = vm_get_vmcb_field(ctx, vcpu,
2228                                                   VMCB_OFF_GUEST_PAT, 8, &pat);
2229                 if (error == 0)
2230                         printf("guest_pat[%d]\t\t0x%016lx\n", vcpu, pat);
2231         }
2232
2233         if (!error && (get_guest_sysenter || get_all)) {
2234                 if (cpu_intel)
2235                         error = vm_get_vmcs_field(ctx, vcpu,
2236                                                   VMCS_GUEST_IA32_SYSENTER_CS,
2237                                                   &cs);
2238                 else
2239                         error = vm_get_vmcb_field(ctx, vcpu,
2240                                                   VMCB_OFF_SYSENTER_CS, 8,
2241                                                   &cs);
2242
2243                 if (error == 0)
2244                         printf("guest_sysenter_cs[%d]\t%#lx\n", vcpu, cs);
2245                 if (cpu_intel)
2246                         error = vm_get_vmcs_field(ctx, vcpu,
2247                                                   VMCS_GUEST_IA32_SYSENTER_ESP,
2248                                                   &rsp);
2249                 else
2250                         error = vm_get_vmcb_field(ctx, vcpu,
2251                                                   VMCB_OFF_SYSENTER_ESP, 8,
2252                                                   &rsp);
2253
2254                 if (error == 0)
2255                         printf("guest_sysenter_sp[%d]\t%#lx\n", vcpu, rsp);
2256                 if (cpu_intel)
2257                         error = vm_get_vmcs_field(ctx, vcpu,
2258                                                   VMCS_GUEST_IA32_SYSENTER_EIP,
2259                                                   &rip);
2260                 else
2261                         error = vm_get_vmcb_field(ctx, vcpu,
2262                                                   VMCB_OFF_SYSENTER_EIP, 8,
2263                                                   &rip);
2264                 if (error == 0)
2265                         printf("guest_sysenter_ip[%d]\t%#lx\n", vcpu, rip);
2266         }
2267
2268         if (!error && (get_exit_reason || get_all)) {
2269                 if (cpu_intel)
2270                         error = vm_get_vmcs_field(ctx, vcpu, VMCS_EXIT_REASON,
2271                                                   &u64);
2272                 else
2273                         error = vm_get_vmcb_field(ctx, vcpu,
2274                                                   VMCB_OFF_EXIT_REASON, 8,
2275                                                   &u64);
2276                 if (error == 0)
2277                         printf("exit_reason[%d]\t%#lx\n", vcpu, u64);
2278         }
2279
2280         if (!error && setcap) {
2281                 int captype;
2282                 captype = vm_capability_name2type(capname);
2283                 error = vm_set_capability(ctx, vcpu, captype, capval);
2284                 if (error != 0 && errno == ENOENT)
2285                         printf("Capability \"%s\" is not available\n", capname);
2286         }
2287
2288         if (!error && get_gpa_pmap) {
2289                 error = vm_get_gpa_pmap(ctx, gpa_pmap, pteval, &ptenum);
2290                 if (error == 0) {
2291                         printf("gpa %#lx:", gpa_pmap);
2292                         pte = &pteval[0];
2293                         while (ptenum-- > 0)
2294                                 printf(" %#lx", *pte++);
2295                         printf("\n");
2296                 }
2297         }
2298
2299         if (!error && set_rtc_nvram)
2300                 error = vm_rtc_write(ctx, rtc_nvram_offset, rtc_nvram_value);
2301
2302         if (!error && (get_rtc_nvram || get_all)) {
2303                 error = vm_rtc_read(ctx, rtc_nvram_offset, &rtc_nvram_value);
2304                 if (error == 0) {
2305                         printf("rtc nvram[%03d]: 0x%02x\n", rtc_nvram_offset,
2306                             rtc_nvram_value);
2307                 }
2308         }
2309
2310         if (!error && set_rtc_time)
2311                 error = vm_rtc_settime(ctx, rtc_secs);
2312
2313         if (!error && (get_rtc_time || get_all)) {
2314                 error = vm_rtc_gettime(ctx, &rtc_secs);
2315                 if (error == 0) {
2316                         gmtime_r(&rtc_secs, &tm);
2317                         printf("rtc time %#lx: %s %s %02d %02d:%02d:%02d %d\n",
2318                             rtc_secs, wday_str(tm.tm_wday), mon_str(tm.tm_mon),
2319                             tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec,
2320                             1900 + tm.tm_year);
2321                 }
2322         }
2323
2324         if (!error && (getcap || get_all)) {
2325                 int captype, val, getcaptype;
2326
2327                 if (getcap && capname)
2328                         getcaptype = vm_capability_name2type(capname);
2329                 else
2330                         getcaptype = -1;
2331
2332                 for (captype = 0; captype < VM_CAP_MAX; captype++) {
2333                         if (getcaptype >= 0 && captype != getcaptype)
2334                                 continue;
2335                         error = vm_get_capability(ctx, vcpu, captype, &val);
2336                         if (error == 0) {
2337                                 printf("Capability \"%s\" is %s on vcpu %d\n",
2338                                         vm_capability_type2name(captype),
2339                                         val ? "set" : "not set", vcpu);
2340                         } else if (errno == ENOENT) {
2341                                 error = 0;
2342                                 printf("Capability \"%s\" is not available\n",
2343                                         vm_capability_type2name(captype));
2344                         } else {
2345                                 break;
2346                         }
2347                 }
2348         }
2349
2350         if (!error && (get_active_cpus || get_all)) {
2351                 error = vm_active_cpus(ctx, &cpus);
2352                 if (!error)
2353                         print_cpus("active cpus", &cpus);
2354         }
2355
2356         if (!error && (get_suspended_cpus || get_all)) {
2357                 error = vm_suspended_cpus(ctx, &cpus);
2358                 if (!error)
2359                         print_cpus("suspended cpus", &cpus);
2360         }
2361
2362         if (!error && (get_intinfo || get_all)) {
2363                 error = vm_get_intinfo(ctx, vcpu, &info[0], &info[1]);
2364                 if (!error) {
2365                         print_intinfo("pending", info[0]);
2366                         print_intinfo("current", info[1]);
2367                 }
2368         }
2369
2370         if (!error && (get_stats || get_all)) {
2371                 int i, num_stats;
2372                 uint64_t *stats;
2373                 struct timeval tv;
2374                 const char *desc;
2375
2376                 stats = vm_get_stats(ctx, vcpu, &tv, &num_stats);
2377                 if (stats != NULL) {
2378                         printf("vcpu%d stats:\n", vcpu);
2379                         for (i = 0; i < num_stats; i++) {
2380                                 desc = vm_get_stat_desc(ctx, i);
2381                                 printf("%-40s\t%ld\n", desc, stats[i]);
2382                         }
2383                 }
2384         }
2385
2386         if (!error && (get_cpu_topology || get_all)) {
2387                 uint16_t sockets, cores, threads, maxcpus;
2388
2389                 vm_get_topology(ctx, &sockets, &cores, &threads, &maxcpus);
2390                 printf("cpu_topology:\tsockets=%hu, cores=%hu, threads=%hu, "
2391                     "maxcpus=%hu\n", sockets, cores, threads, maxcpus);
2392         }
2393
2394         if (!error && run) {
2395                 error = vm_run(ctx, vcpu, &vmexit);
2396                 if (error == 0)
2397                         dump_vm_run_exitcode(&vmexit, vcpu);
2398                 else
2399                         printf("vm_run error %d\n", error);
2400         }
2401
2402         if (!error && force_reset)
2403                 error = vm_suspend(ctx, VM_SUSPEND_RESET);
2404
2405         if (!error && force_poweroff)
2406                 error = vm_suspend(ctx, VM_SUSPEND_POWEROFF);
2407
2408         if (error)
2409                 printf("errno = %d\n", errno);
2410
2411         if (!error && destroy)
2412                 vm_destroy(ctx);
2413
2414 #ifdef BHYVE_SNAPSHOT
2415         if (!error && vm_checkpoint_opt)
2416                 error = send_start_checkpoint(ctx, checkpoint_file);
2417
2418         if (!error && vm_suspend_opt)
2419                 error = send_start_suspend(ctx, suspend_file);
2420 #endif
2421
2422         free (opts);
2423         exit(error);
2424 }