usr.sbin/bhyve/bhyverun.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2011 NetApp, Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  *
  28  * $FreeBSD$
  29  */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33
  34 #include <sys/types.h>
  35 #ifndef WITHOUT_CAPSICUM
  36 #include <sys/capsicum.h>
  37 #endif
  38 #include <sys/mman.h>
  39 #ifdef BHYVE_SNAPSHOT
  40 #include <sys/socket.h>
  41 #include <sys/stat.h>
  42 #endif
  43 #include <sys/time.h>
  44 #ifdef BHYVE_SNAPSHOT
  45 #include <sys/un.h>
  46 #endif
  47
  48 #include <amd64/vmm/intel/vmcs.h>
  49 #include <x86/apicreg.h>
  50
  51 #include <machine/atomic.h>
  52 #include <machine/segments.h>
  53
  54 #ifndef WITHOUT_CAPSICUM
  55 #include <capsicum_helpers.h>
  56 #endif
  57 #include <stdio.h>
  58 #include <stdlib.h>
  59 #include <string.h>
  60 #include <err.h>
  61 #include <errno.h>
  62 #ifdef BHYVE_SNAPSHOT
  63 #include <fcntl.h>
  64 #endif
  65 #include <libgen.h>
  66 #include <unistd.h>
  67 #include <assert.h>
  68 #include <pthread.h>
  69 #include <pthread_np.h>
  70 #include <sysexits.h>
  71 #include <stdbool.h>
  72 #include <stdint.h>
  73 #ifdef BHYVE_SNAPSHOT
  74 #include <ucl.h>
  75 #include <unistd.h>
  76
  77 #include <libxo/xo.h>
  78 #endif
  79
  80 #include <machine/vmm.h>
  81 #ifndef WITHOUT_CAPSICUM
  82 #include <machine/vmm_dev.h>
  83 #endif
  84 #include <machine/vmm_instruction_emul.h>
  85 #include <vmmapi.h>
  86
  87 #include "bhyverun.h"
  88 #include "acpi.h"
  89 #include "atkbdc.h"
  90 #include "bootrom.h"
  91 #include "config.h"
  92 #include "inout.h"
  93 #include "debug.h"
  94 #include "fwctl.h"
  95 #include "gdb.h"
  96 #include "ioapic.h"
  97 #include "kernemu_dev.h"
  98 #include "mem.h"
  99 #include "mevent.h"
 100 #include "mptbl.h"
 101 #include "pci_emul.h"
 102 #include "pci_irq.h"
 103 #include "pci_lpc.h"
 104 #include "qemu_fwcfg.h"
 105 #include "smbiostbl.h"
 106 #ifdef BHYVE_SNAPSHOT
 107 #include "snapshot.h"
 108 #endif
 109 #include "xmsr.h"
 110 #include "spinup_ap.h"
 111 #include "rtc.h"
 112 #include "vmgenc.h"
 113
 114 #define GUEST_NIO_PORT          0x488   /* guest upcalls via i/o port */
 115
 116 #define MB              (1024UL * 1024)
 117 #define GB              (1024UL * MB)
 118
 119 static const char * const vmx_exit_reason_desc[] = {
 120         [EXIT_REASON_EXCEPTION] = "Exception or non-maskable interrupt (NMI)",
 121         [EXIT_REASON_EXT_INTR] = "External interrupt",
 122         [EXIT_REASON_TRIPLE_FAULT] = "Triple fault",
 123         [EXIT_REASON_INIT] = "INIT signal",
 124         [EXIT_REASON_SIPI] = "Start-up IPI (SIPI)",
 125         [EXIT_REASON_IO_SMI] = "I/O system-management interrupt (SMI)",
 126         [EXIT_REASON_SMI] = "Other SMI",
 127         [EXIT_REASON_INTR_WINDOW] = "Interrupt window",
 128         [EXIT_REASON_NMI_WINDOW] = "NMI window",
 129         [EXIT_REASON_TASK_SWITCH] = "Task switch",
 130         [EXIT_REASON_CPUID] = "CPUID",
 131         [EXIT_REASON_GETSEC] = "GETSEC",
 132         [EXIT_REASON_HLT] = "HLT",
 133         [EXIT_REASON_INVD] = "INVD",
 134         [EXIT_REASON_INVLPG] = "INVLPG",
 135         [EXIT_REASON_RDPMC] = "RDPMC",
 136         [EXIT_REASON_RDTSC] = "RDTSC",
 137         [EXIT_REASON_RSM] = "RSM",
 138         [EXIT_REASON_VMCALL] = "VMCALL",
 139         [EXIT_REASON_VMCLEAR] = "VMCLEAR",
 140         [EXIT_REASON_VMLAUNCH] = "VMLAUNCH",
 141         [EXIT_REASON_VMPTRLD] = "VMPTRLD",
 142         [EXIT_REASON_VMPTRST] = "VMPTRST",
 143         [EXIT_REASON_VMREAD] = "VMREAD",
 144         [EXIT_REASON_VMRESUME] = "VMRESUME",
 145         [EXIT_REASON_VMWRITE] = "VMWRITE",
 146         [EXIT_REASON_VMXOFF] = "VMXOFF",
 147         [EXIT_REASON_VMXON] = "VMXON",
 148         [EXIT_REASON_CR_ACCESS] = "Control-register accesses",
 149         [EXIT_REASON_DR_ACCESS] = "MOV DR",
 150         [EXIT_REASON_INOUT] = "I/O instruction",
 151         [EXIT_REASON_RDMSR] = "RDMSR",
 152         [EXIT_REASON_WRMSR] = "WRMSR",
 153         [EXIT_REASON_INVAL_VMCS] =
 154             "VM-entry failure due to invalid guest state",
 155         [EXIT_REASON_INVAL_MSR] = "VM-entry failure due to MSR loading",
 156         [EXIT_REASON_MWAIT] = "MWAIT",
 157         [EXIT_REASON_MTF] = "Monitor trap flag",
 158         [EXIT_REASON_MONITOR] = "MONITOR",
 159         [EXIT_REASON_PAUSE] = "PAUSE",
 160         [EXIT_REASON_MCE_DURING_ENTRY] =
 161             "VM-entry failure due to machine-check event",
 162         [EXIT_REASON_TPR] = "TPR below threshold",
 163         [EXIT_REASON_APIC_ACCESS] = "APIC access",
 164         [EXIT_REASON_VIRTUALIZED_EOI] = "Virtualized EOI",
 165         [EXIT_REASON_GDTR_IDTR] = "Access to GDTR or IDTR",
 166         [EXIT_REASON_LDTR_TR] = "Access to LDTR or TR",
 167         [EXIT_REASON_EPT_FAULT] = "EPT violation",
 168         [EXIT_REASON_EPT_MISCONFIG] = "EPT misconfiguration",
 169         [EXIT_REASON_INVEPT] = "INVEPT",
 170         [EXIT_REASON_RDTSCP] = "RDTSCP",
 171         [EXIT_REASON_VMX_PREEMPT] = "VMX-preemption timer expired",
 172         [EXIT_REASON_INVVPID] = "INVVPID",
 173         [EXIT_REASON_WBINVD] = "WBINVD",
 174         [EXIT_REASON_XSETBV] = "XSETBV",
 175         [EXIT_REASON_APIC_WRITE] = "APIC write",
 176         [EXIT_REASON_RDRAND] = "RDRAND",
 177         [EXIT_REASON_INVPCID] = "INVPCID",
 178         [EXIT_REASON_VMFUNC] = "VMFUNC",
 179         [EXIT_REASON_ENCLS] = "ENCLS",
 180         [EXIT_REASON_RDSEED] = "RDSEED",
 181         [EXIT_REASON_PM_LOG_FULL] = "Page-modification log full",
 182         [EXIT_REASON_XSAVES] = "XSAVES",
 183         [EXIT_REASON_XRSTORS] = "XRSTORS"
 184 };
 185
 186 typedef int (*vmexit_handler_t)(struct vmctx *, struct vcpu *, struct vm_exit *);
 187
 188 int guest_ncpus;
 189 uint16_t cpu_cores, cpu_sockets, cpu_threads;
 190
 191 int raw_stdio = 0;
 192
 193 static char *progname;
 194 static const int BSP = 0;
 195
 196 static cpuset_t cpumask;
 197
 198 static void vm_loop(struct vmctx *ctx, struct vcpu *vcpu);
 199
 200 static struct bhyvestats {
 201         uint64_t        vmexit_bogus;
 202         uint64_t        vmexit_reqidle;
 203         uint64_t        vmexit_hlt;
 204         uint64_t        vmexit_pause;
 205         uint64_t        vmexit_mtrap;
 206         uint64_t        vmexit_inst_emul;
 207         uint64_t        cpu_switch_rotate;
 208         uint64_t        cpu_switch_direct;
 209 } stats;
 210
 211 static struct vcpu_info {
 212         struct vmctx    *ctx;
 213         struct vcpu     *vcpu;
 214         int             vcpuid;
 215 } *vcpu_info;
 216
 217 static cpuset_t **vcpumap;
 218
 219 static void
 220 usage(int code)
 221 {
 222
 223         fprintf(stderr,
 224                 "Usage: %s [-AaCDeHhPSuWwxY]\n"
 225                 "       %*s [-c [[cpus=]numcpus][,sockets=n][,cores=n][,threads=n]]\n"
 226                 "       %*s [-G port] [-k config_file] [-l lpc] [-m mem] [-o var=value]\n"
 227                 "       %*s [-p vcpu:hostcpu] [-r file] [-s pci] [-U uuid] vmname\n"
 228                 "       -A: create ACPI tables\n"
 229                 "       -a: local apic is in xAPIC mode (deprecated)\n"
 230                 "       -C: include guest memory in core file\n"
 231                 "       -c: number of CPUs and/or topology specification\n"
 232                 "       -D: destroy on power-off\n"
 233                 "       -e: exit on unhandled I/O access\n"
 234                 "       -G: start a debug server\n"
 235                 "       -H: vmexit from the guest on HLT\n"
 236                 "       -h: help\n"
 237                 "       -k: key=value flat config file\n"
 238                 "       -K: PS2 keyboard layout\n"
 239                 "       -l: LPC device configuration\n"
 240                 "       -m: memory size\n"
 241                 "       -o: set config 'var' to 'value'\n"
 242                 "       -P: vmexit from the guest on pause\n"
 243                 "       -p: pin 'vcpu' to 'hostcpu'\n"
 244 #ifdef BHYVE_SNAPSHOT
 245                 "       -r: path to checkpoint file\n"
 246 #endif
 247                 "       -S: guest memory cannot be swapped\n"
 248                 "       -s: <slot,driver,configinfo> PCI slot config\n"
 249                 "       -U: UUID\n"
 250                 "       -u: RTC keeps UTC time\n"
 251                 "       -W: force virtio to use single-vector MSI\n"
 252                 "       -w: ignore unimplemented MSRs\n"
 253                 "       -x: local APIC is in x2APIC mode\n"
 254                 "       -Y: disable MPtable generation\n",
 255                 progname, (int)strlen(progname), "", (int)strlen(progname), "",
 256                 (int)strlen(progname), "");
 257
 258         exit(code);
 259 }
 260
 261 /*
 262  * XXX This parser is known to have the following issues:
 263  * 1.  It accepts null key=value tokens ",," as setting "cpus" to an
 264  *     empty string.
 265  *
 266  * The acceptance of a null specification ('-c ""') is by design to match the
 267  * manual page syntax specification, this results in a topology of 1 vCPU.
 268  */
 269 static int
 270 topology_parse(const char *opt)
 271 {
 272         char *cp, *str, *tofree;
 273
 274         if (*opt == '\0') {
 275                 set_config_value("sockets", "1");
 276                 set_config_value("cores", "1");
 277                 set_config_value("threads", "1");
 278                 set_config_value("cpus", "1");
 279                 return (0);
 280         }
 281
 282         tofree = str = strdup(opt);
 283         if (str == NULL)
 284                 errx(4, "Failed to allocate memory");
 285
 286         while ((cp = strsep(&str, ",")) != NULL) {
 287                 if (strncmp(cp, "cpus=", strlen("cpus=")) == 0)
 288                         set_config_value("cpus", cp + strlen("cpus="));
 289                 else if (strncmp(cp, "sockets=", strlen("sockets=")) == 0)
 290                         set_config_value("sockets", cp + strlen("sockets="));
 291                 else if (strncmp(cp, "cores=", strlen("cores=")) == 0)
 292                         set_config_value("cores", cp + strlen("cores="));
 293                 else if (strncmp(cp, "threads=", strlen("threads=")) == 0)
 294                         set_config_value("threads", cp + strlen("threads="));
 295 #ifdef notyet  /* Do not expose this until vmm.ko implements it */
 296                 else if (strncmp(cp, "maxcpus=", strlen("maxcpus=")) == 0)
 297                         set_config_value("maxcpus", cp + strlen("maxcpus="));
 298 #endif
 299                 else if (strchr(cp, '=') != NULL)
 300                         goto out;
 301                 else
 302                         set_config_value("cpus", cp);
 303         }
 304         free(tofree);
 305         return (0);
 306
 307 out:
 308         free(tofree);
 309         return (-1);
 310 }
 311
 312 static int
 313 parse_int_value(const char *key, const char *value, int minval, int maxval)
 314 {
 315         char *cp;
 316         long lval;
 317
 318         errno = 0;
 319         lval = strtol(value, &cp, 0);
 320         if (errno != 0 || *cp != '\0' || cp == value || lval < minval ||
 321             lval > maxval)
 322                 errx(4, "Invalid value for %s: '%s'", key, value);
 323         return (lval);
 324 }
 325
 326 /*
 327  * Set the sockets, cores, threads, and guest_cpus variables based on
 328  * the configured topology.
 329  *
 330  * The limits of UINT16_MAX are due to the types passed to
 331  * vm_set_topology().  vmm.ko may enforce tighter limits.
 332  */
 333 static void
 334 calc_topology(void)
 335 {
 336         const char *value;
 337         bool explicit_cpus;
 338         uint64_t ncpus;
 339
 340         value = get_config_value("cpus");
 341         if (value != NULL) {
 342                 guest_ncpus = parse_int_value("cpus", value, 1, UINT16_MAX);
 343                 explicit_cpus = true;
 344         } else {
 345                 guest_ncpus = 1;
 346                 explicit_cpus = false;
 347         }
 348         value = get_config_value("cores");
 349         if (value != NULL)
 350                 cpu_cores = parse_int_value("cores", value, 1, UINT16_MAX);
 351         else
 352                 cpu_cores = 1;
 353         value = get_config_value("threads");
 354         if (value != NULL)
 355                 cpu_threads = parse_int_value("threads", value, 1, UINT16_MAX);
 356         else
 357                 cpu_threads = 1;
 358         value = get_config_value("sockets");
 359         if (value != NULL)
 360                 cpu_sockets = parse_int_value("sockets", value, 1, UINT16_MAX);
 361         else
 362                 cpu_sockets = guest_ncpus;
 363
 364         /*
 365          * Compute sockets * cores * threads avoiding overflow.  The
 366          * range check above insures these are 16 bit values.
 367          */
 368         ncpus = (uint64_t)cpu_sockets * cpu_cores * cpu_threads;
 369         if (ncpus > UINT16_MAX)
 370                 errx(4, "Computed number of vCPUs too high: %ju",
 371                     (uintmax_t)ncpus);
 372
 373         if (explicit_cpus) {
 374                 if (guest_ncpus != (int)ncpus)
 375                         errx(4, "Topology (%d sockets, %d cores, %d threads) "
 376                             "does not match %d vCPUs",
 377                             cpu_sockets, cpu_cores, cpu_threads,
 378                             guest_ncpus);
 379         } else
 380                 guest_ncpus = ncpus;
 381 }
 382
 383 static int
 384 pincpu_parse(const char *opt)
 385 {
 386         const char *value;
 387         char *newval;
 388         char key[16];
 389         int vcpu, pcpu;
 390
 391         if (sscanf(opt, "%d:%d", &vcpu, &pcpu) != 2) {
 392                 fprintf(stderr, "invalid format: %s\n", opt);
 393                 return (-1);
 394         }
 395
 396         if (vcpu < 0) {
 397                 fprintf(stderr, "invalid vcpu '%d'\n", vcpu);
 398                 return (-1);
 399         }
 400
 401         if (pcpu < 0 || pcpu >= CPU_SETSIZE) {
 402                 fprintf(stderr, "hostcpu '%d' outside valid range from "
 403                     "0 to %d\n", pcpu, CPU_SETSIZE - 1);
 404                 return (-1);
 405         }
 406
 407         snprintf(key, sizeof(key), "vcpu.%d.cpuset", vcpu);
 408         value = get_config_value(key);
 409
 410         if (asprintf(&newval, "%s%s%d", value != NULL ? value : "",
 411             value != NULL ? "," : "", pcpu) == -1) {
 412                 perror("failed to build new cpuset string");
 413                 return (-1);
 414         }
 415
 416         set_config_value(key, newval);
 417         free(newval);
 418         return (0);
 419 }
 420
 421 static void
 422 parse_cpuset(int vcpu, const char *list, cpuset_t *set)
 423 {
 424         char *cp, *token;
 425         int pcpu, start;
 426
 427         CPU_ZERO(set);
 428         start = -1;
 429         token = __DECONST(char *, list);
 430         for (;;) {
 431                 pcpu = strtoul(token, &cp, 0);
 432                 if (cp == token)
 433                         errx(4, "invalid cpuset for vcpu %d: '%s'", vcpu, list);
 434                 if (pcpu < 0 || pcpu >= CPU_SETSIZE)
 435                         errx(4, "hostcpu '%d' outside valid range from 0 to %d",
 436                             pcpu, CPU_SETSIZE - 1);
 437                 switch (*cp) {
 438                 case ',':
 439                 case '\0':
 440                         if (start >= 0) {
 441                                 if (start > pcpu)
 442                                         errx(4, "Invalid hostcpu range %d-%d",
 443                                             start, pcpu);
 444                                 while (start < pcpu) {
 445                                         CPU_SET(start, set);
 446                                         start++;
 447                                 }
 448                                 start = -1;
 449                         }
 450                         CPU_SET(pcpu, set);
 451                         break;
 452                 case '-':
 453                         if (start >= 0)
 454                                 errx(4, "invalid cpuset for vcpu %d: '%s'",
 455                                     vcpu, list);
 456                         start = pcpu;
 457                         break;
 458                 default:
 459                         errx(4, "invalid cpuset for vcpu %d: '%s'", vcpu, list);
 460                 }
 461                 if (*cp == '\0')
 462                         break;
 463                 token = cp + 1;
 464         }
 465 }
 466
 467 static void
 468 build_vcpumaps(void)
 469 {
 470         char key[16];
 471         const char *value;
 472         int vcpu;
 473
 474         vcpumap = calloc(guest_ncpus, sizeof(*vcpumap));
 475         for (vcpu = 0; vcpu < guest_ncpus; vcpu++) {
 476                 snprintf(key, sizeof(key), "vcpu.%d.cpuset", vcpu);
 477                 value = get_config_value(key);
 478                 if (value == NULL)
 479                         continue;
 480                 vcpumap[vcpu] = malloc(sizeof(cpuset_t));
 481                 if (vcpumap[vcpu] == NULL)
 482                         err(4, "Failed to allocate cpuset for vcpu %d", vcpu);
 483                 parse_cpuset(vcpu, value, vcpumap[vcpu]);
 484         }
 485 }
 486
 487 void
 488 vm_inject_fault(struct vcpu *vcpu, int vector, int errcode_valid,
 489     int errcode)
 490 {
 491         int error, restart_instruction;
 492
 493         restart_instruction = 1;
 494
 495         error = vm_inject_exception(vcpu, vector, errcode_valid, errcode,
 496             restart_instruction);
 497         assert(error == 0);
 498 }
 499
 500 void *
 501 paddr_guest2host(struct vmctx *ctx, uintptr_t gaddr, size_t len)
 502 {
 503
 504         return (vm_map_gpa(ctx, gaddr, len));
 505 }
 506
 507 #ifdef BHYVE_SNAPSHOT
 508 uintptr_t
 509 paddr_host2guest(struct vmctx *ctx, void *addr)
 510 {
 511         return (vm_rev_map_gpa(ctx, addr));
 512 }
 513 #endif
 514
 515 int
 516 fbsdrun_virtio_msix(void)
 517 {
 518
 519         return (get_config_bool_default("virtio_msix", true));
 520 }
 521
 522 static void *
 523 fbsdrun_start_thread(void *param)
 524 {
 525         char tname[MAXCOMLEN + 1];
 526         struct vcpu_info *vi = param;
 527         int error;
 528
 529         snprintf(tname, sizeof(tname), "vcpu %d", vi->vcpuid);
 530         pthread_set_name_np(pthread_self(), tname);
 531
 532         if (vcpumap[vi->vcpuid] != NULL) {
 533                 error = pthread_setaffinity_np(pthread_self(),
 534                     sizeof(cpuset_t), vcpumap[vi->vcpuid]);
 535                 assert(error == 0);
 536         }
 537
 538 #ifdef BHYVE_SNAPSHOT
 539         checkpoint_cpu_add(vi->vcpuid);
 540 #endif
 541         gdb_cpu_add(vi->vcpu);
 542
 543         vm_loop(vi->ctx, vi->vcpu);
 544
 545         /* not reached */
 546         exit(1);
 547         return (NULL);
 548 }
 549
 550 static void
 551 fbsdrun_addcpu(struct vcpu_info *vi)
 552 {
 553         pthread_t thr;
 554         int error;
 555
 556         error = vm_activate_cpu(vi->vcpu);
 557         if (error != 0)
 558                 err(EX_OSERR, "could not activate CPU %d", vi->vcpuid);
 559
 560         CPU_SET_ATOMIC(vi->vcpuid, &cpumask);
 561
 562         vm_suspend_cpu(vi->vcpu);
 563
 564         error = pthread_create(&thr, NULL, fbsdrun_start_thread, vi);
 565         assert(error == 0);
 566 }
 567
 568 static int
 569 fbsdrun_deletecpu(int vcpu)
 570 {
 571
 572         if (!CPU_ISSET(vcpu, &cpumask)) {
 573                 fprintf(stderr, "Attempting to delete unknown cpu %d\n", vcpu);
 574                 exit(4);
 575         }
 576
 577         CPU_CLR_ATOMIC(vcpu, &cpumask);
 578         return (CPU_EMPTY(&cpumask));
 579 }
 580
 581 static int
 582 vmexit_handle_notify(struct vmctx *ctx __unused, struct vcpu *vcpu __unused,
 583     struct vm_exit *vme __unused, uint32_t eax __unused)
 584 {
 585 #if BHYVE_DEBUG
 586         /*
 587          * put guest-driven debug here
 588          */
 589 #endif
 590         return (VMEXIT_CONTINUE);
 591 }
 592
 593 static int
 594 vmexit_inout(struct vmctx *ctx, struct vcpu *vcpu, struct vm_exit *vme)
 595 {
 596         int error;
 597         int bytes, port, in, out;
 598
 599         port = vme->u.inout.port;
 600         bytes = vme->u.inout.bytes;
 601         in = vme->u.inout.in;
 602         out = !in;
 603
 604         /* Extra-special case of host notifications */
 605         if (out && port == GUEST_NIO_PORT) {
 606                 error = vmexit_handle_notify(ctx, vcpu, vme, vme->u.inout.eax);
 607                 return (error);
 608         }
 609
 610         error = emulate_inout(ctx, vcpu, vme);
 611         if (error) {
 612                 fprintf(stderr, "Unhandled %s%c 0x%04x at 0x%lx\n",
 613                     in ? "in" : "out",
 614                     bytes == 1 ? 'b' : (bytes == 2 ? 'w' : 'l'),
 615                     port, vme->rip);
 616                 return (VMEXIT_ABORT);
 617         } else {
 618                 return (VMEXIT_CONTINUE);
 619         }
 620 }
 621
 622 static int
 623 vmexit_rdmsr(struct vmctx *ctx __unused, struct vcpu *vcpu, struct vm_exit *vme)
 624 {
 625         uint64_t val;
 626         uint32_t eax, edx;
 627         int error;
 628
 629         val = 0;
 630         error = emulate_rdmsr(vcpu, vme->u.msr.code, &val);
 631         if (error != 0) {
 632                 fprintf(stderr, "rdmsr to register %#x on vcpu %d\n",
 633                     vme->u.msr.code, vcpu_id(vcpu));
 634                 if (get_config_bool("x86.strictmsr")) {
 635                         vm_inject_gp(vcpu);
 636                         return (VMEXIT_CONTINUE);
 637                 }
 638         }
 639
 640         eax = val;
 641         error = vm_set_register(vcpu, VM_REG_GUEST_RAX, eax);
 642         assert(error == 0);
 643
 644         edx = val >> 32;
 645         error = vm_set_register(vcpu, VM_REG_GUEST_RDX, edx);
 646         assert(error == 0);
 647
 648         return (VMEXIT_CONTINUE);
 649 }
 650
 651 static int
 652 vmexit_wrmsr(struct vmctx *ctx __unused, struct vcpu *vcpu, struct vm_exit *vme)
 653 {
 654         int error;
 655
 656         error = emulate_wrmsr(vcpu, vme->u.msr.code, vme->u.msr.wval);
 657         if (error != 0) {
 658                 fprintf(stderr, "wrmsr to register %#x(%#lx) on vcpu %d\n",
 659                     vme->u.msr.code, vme->u.msr.wval, vcpu_id(vcpu));
 660                 if (get_config_bool("x86.strictmsr")) {
 661                         vm_inject_gp(vcpu);
 662                         return (VMEXIT_CONTINUE);
 663                 }
 664         }
 665         return (VMEXIT_CONTINUE);
 666 }
 667
 668 #define DEBUG_EPT_MISCONFIG
 669 #ifdef DEBUG_EPT_MISCONFIG
 670 #define VMCS_GUEST_PHYSICAL_ADDRESS     0x00002400
 671
 672 static uint64_t ept_misconfig_gpa, ept_misconfig_pte[4];
 673 static int ept_misconfig_ptenum;
 674 #endif
 675
 676 static const char *
 677 vmexit_vmx_desc(uint32_t exit_reason)
 678 {
 679
 680         if (exit_reason >= nitems(vmx_exit_reason_desc) ||
 681             vmx_exit_reason_desc[exit_reason] == NULL)
 682                 return ("Unknown");
 683         return (vmx_exit_reason_desc[exit_reason]);
 684 }
 685
 686 static int
 687 vmexit_vmx(struct vmctx *ctx, struct vcpu *vcpu, struct vm_exit *vme)
 688 {
 689
 690         fprintf(stderr, "vm exit[%d]\n", vcpu_id(vcpu));
 691         fprintf(stderr, "\treason\t\tVMX\n");
 692         fprintf(stderr, "\trip\t\t0x%016lx\n", vme->rip);
 693         fprintf(stderr, "\tinst_length\t%d\n", vme->inst_length);
 694         fprintf(stderr, "\tstatus\t\t%d\n", vme->u.vmx.status);
 695         fprintf(stderr, "\texit_reason\t%u (%s)\n", vme->u.vmx.exit_reason,
 696             vmexit_vmx_desc(vme->u.vmx.exit_reason));
 697         fprintf(stderr, "\tqualification\t0x%016lx\n",
 698             vme->u.vmx.exit_qualification);
 699         fprintf(stderr, "\tinst_type\t\t%d\n", vme->u.vmx.inst_type);
 700         fprintf(stderr, "\tinst_error\t\t%d\n", vme->u.vmx.inst_error);
 701 #ifdef DEBUG_EPT_MISCONFIG
 702         if (vme->u.vmx.exit_reason == EXIT_REASON_EPT_MISCONFIG) {
 703                 vm_get_register(vcpu,
 704                     VMCS_IDENT(VMCS_GUEST_PHYSICAL_ADDRESS),
 705                     &ept_misconfig_gpa);
 706                 vm_get_gpa_pmap(ctx, ept_misconfig_gpa, ept_misconfig_pte,
 707                     &ept_misconfig_ptenum);
 708                 fprintf(stderr, "\tEPT misconfiguration:\n");
 709                 fprintf(stderr, "\t\tGPA: %#lx\n", ept_misconfig_gpa);
 710                 fprintf(stderr, "\t\tPTE(%d): %#lx %#lx %#lx %#lx\n",
 711                     ept_misconfig_ptenum, ept_misconfig_pte[0],
 712                     ept_misconfig_pte[1], ept_misconfig_pte[2],
 713                     ept_misconfig_pte[3]);
 714         }
 715 #endif  /* DEBUG_EPT_MISCONFIG */
 716         return (VMEXIT_ABORT);
 717 }
 718
 719 static int
 720 vmexit_svm(struct vmctx *ctx __unused, struct vcpu *vcpu, struct vm_exit *vme)
 721 {
 722
 723         fprintf(stderr, "vm exit[%d]\n", vcpu_id(vcpu));
 724         fprintf(stderr, "\treason\t\tSVM\n");
 725         fprintf(stderr, "\trip\t\t0x%016lx\n", vme->rip);
 726         fprintf(stderr, "\tinst_length\t%d\n", vme->inst_length);
 727         fprintf(stderr, "\texitcode\t%#lx\n", vme->u.svm.exitcode);
 728         fprintf(stderr, "\texitinfo1\t%#lx\n", vme->u.svm.exitinfo1);
 729         fprintf(stderr, "\texitinfo2\t%#lx\n", vme->u.svm.exitinfo2);
 730         return (VMEXIT_ABORT);
 731 }
 732
 733 static int
 734 vmexit_bogus(struct vmctx *ctx __unused, struct vcpu *vcpu __unused,
 735     struct vm_exit *vme)
 736 {
 737
 738         assert(vme->inst_length == 0);
 739
 740         stats.vmexit_bogus++;
 741
 742         return (VMEXIT_CONTINUE);
 743 }
 744
 745 static int
 746 vmexit_reqidle(struct vmctx *ctx __unused, struct vcpu *vcpu __unused,
 747     struct vm_exit *vme)
 748 {
 749
 750         assert(vme->inst_length == 0);
 751
 752         stats.vmexit_reqidle++;
 753
 754         return (VMEXIT_CONTINUE);
 755 }
 756
 757 static int
 758 vmexit_hlt(struct vmctx *ctx __unused, struct vcpu *vcpu __unused,
 759     struct vm_exit *vme __unused)
 760 {
 761
 762         stats.vmexit_hlt++;
 763
 764         /*
 765          * Just continue execution with the next instruction. We use
 766          * the HLT VM exit as a way to be friendly with the host
 767          * scheduler.
 768          */
 769         return (VMEXIT_CONTINUE);
 770 }
 771
 772 static int
 773 vmexit_pause(struct vmctx *ctx __unused, struct vcpu *vcpu __unused,
 774     struct vm_exit *vme __unused)
 775 {
 776
 777         stats.vmexit_pause++;
 778
 779         return (VMEXIT_CONTINUE);
 780 }
 781
 782 static int
 783 vmexit_mtrap(struct vmctx *ctx __unused, struct vcpu *vcpu,
 784     struct vm_exit *vme)
 785 {
 786
 787         assert(vme->inst_length == 0);
 788
 789         stats.vmexit_mtrap++;
 790
 791 #ifdef BHYVE_SNAPSHOT
 792         checkpoint_cpu_suspend(vcpu_id(vcpu));
 793 #endif
 794         gdb_cpu_mtrap(vcpu);
 795 #ifdef BHYVE_SNAPSHOT
 796         checkpoint_cpu_resume(vcpu_id(vcpu));
 797 #endif
 798
 799         return (VMEXIT_CONTINUE);
 800 }
 801
 802 static int
 803 vmexit_inst_emul(struct vmctx *ctx __unused, struct vcpu *vcpu,
 804     struct vm_exit *vme)
 805 {
 806         int err, i, cs_d;
 807         struct vie *vie;
 808         enum vm_cpu_mode mode;
 809
 810         stats.vmexit_inst_emul++;
 811
 812         vie = &vme->u.inst_emul.vie;
 813         if (!vie->decoded) {
 814                 /*
 815                  * Attempt to decode in userspace as a fallback.  This allows
 816                  * updating instruction decode in bhyve without rebooting the
 817                  * kernel (rapid prototyping), albeit with much slower
 818                  * emulation.
 819                  */
 820                 vie_restart(vie);
 821                 mode = vme->u.inst_emul.paging.cpu_mode;
 822                 cs_d = vme->u.inst_emul.cs_d;
 823                 if (vmm_decode_instruction(mode, cs_d, vie) != 0)
 824                         goto fail;
 825                 if (vm_set_register(vcpu, VM_REG_GUEST_RIP,
 826                     vme->rip + vie->num_processed) != 0)
 827                         goto fail;
 828         }
 829
 830         err = emulate_mem(vcpu, vme->u.inst_emul.gpa, vie,
 831             &vme->u.inst_emul.paging);
 832         if (err) {
 833                 if (err == ESRCH) {
 834                         EPRINTLN("Unhandled memory access to 0x%lx\n",
 835                             vme->u.inst_emul.gpa);
 836                 }
 837                 goto fail;
 838         }
 839
 840         return (VMEXIT_CONTINUE);
 841
 842 fail:
 843         fprintf(stderr, "Failed to emulate instruction sequence [ ");
 844         for (i = 0; i < vie->num_valid; i++)
 845                 fprintf(stderr, "%02x", vie->inst[i]);
 846         FPRINTLN(stderr, " ] at 0x%lx", vme->rip);
 847         return (VMEXIT_ABORT);
 848 }
 849
 850 static pthread_mutex_t resetcpu_mtx = PTHREAD_MUTEX_INITIALIZER;
 851 static pthread_cond_t resetcpu_cond = PTHREAD_COND_INITIALIZER;
 852
 853 static int
 854 vmexit_suspend(struct vmctx *ctx, struct vcpu *vcpu, struct vm_exit *vme)
 855 {
 856         enum vm_suspend_how how;
 857         int vcpuid = vcpu_id(vcpu);
 858
 859         how = vme->u.suspended.how;
 860
 861         fbsdrun_deletecpu(vcpuid);
 862
 863         if (vcpuid != BSP) {
 864                 pthread_mutex_lock(&resetcpu_mtx);
 865                 pthread_cond_signal(&resetcpu_cond);
 866                 pthread_mutex_unlock(&resetcpu_mtx);
 867                 pthread_exit(NULL);
 868         }
 869
 870         pthread_mutex_lock(&resetcpu_mtx);
 871         while (!CPU_EMPTY(&cpumask)) {
 872                 pthread_cond_wait(&resetcpu_cond, &resetcpu_mtx);
 873         }
 874         pthread_mutex_unlock(&resetcpu_mtx);
 875
 876         switch (how) {
 877         case VM_SUSPEND_RESET:
 878                 exit(0);
 879         case VM_SUSPEND_POWEROFF:
 880                 if (get_config_bool_default("destroy_on_poweroff", false))
 881                         vm_destroy(ctx);
 882                 exit(1);
 883         case VM_SUSPEND_HALT:
 884                 exit(2);
 885         case VM_SUSPEND_TRIPLEFAULT:
 886                 exit(3);
 887         default:
 888                 fprintf(stderr, "vmexit_suspend: invalid reason %d\n", how);
 889                 exit(100);
 890         }
 891         return (0);     /* NOTREACHED */
 892 }
 893
 894 static int
 895 vmexit_debug(struct vmctx *ctx __unused, struct vcpu *vcpu,
 896     struct vm_exit *vme __unused)
 897 {
 898
 899 #ifdef BHYVE_SNAPSHOT
 900         checkpoint_cpu_suspend(vcpu_id(vcpu));
 901 #endif
 902         gdb_cpu_suspend(vcpu);
 903 #ifdef BHYVE_SNAPSHOT
 904         checkpoint_cpu_resume(vcpu_id(vcpu));
 905 #endif
 906         /*
 907          * XXX-MJ sleep for a short period to avoid chewing up the CPU in the
 908          * window between activation of the vCPU thread and the STARTUP IPI.
 909          */
 910         usleep(1000);
 911         return (VMEXIT_CONTINUE);
 912 }
 913
 914 static int
 915 vmexit_breakpoint(struct vmctx *ctx __unused, struct vcpu *vcpu,
 916     struct vm_exit *vme)
 917 {
 918
 919         gdb_cpu_breakpoint(vcpu, vme);
 920         return (VMEXIT_CONTINUE);
 921 }
 922
 923 static int
 924 vmexit_ipi(struct vmctx *ctx __unused, struct vcpu *vcpu __unused,
 925     struct vm_exit *vme)
 926 {
 927         int error = -1;
 928         int i;
 929         switch (vme->u.ipi.mode) {
 930         case APIC_DELMODE_INIT:
 931                 CPU_FOREACH_ISSET(i, &vme->u.ipi.dmask) {
 932                         error = vm_suspend_cpu(vcpu_info[i].vcpu);
 933                         if (error) {
 934                                 warnx("%s: failed to suspend cpu %d\n",
 935                                     __func__, i);
 936                                 break;
 937                         }
 938                 }
 939                 break;
 940         case APIC_DELMODE_STARTUP:
 941                 CPU_FOREACH_ISSET(i, &vme->u.ipi.dmask) {
 942                         spinup_ap(vcpu_info[i].vcpu,
 943                             vme->u.ipi.vector << PAGE_SHIFT);
 944                 }
 945                 error = 0;
 946                 break;
 947         default:
 948                 break;
 949         }
 950
 951         return (error);
 952 }
 953
 954 static vmexit_handler_t handler[VM_EXITCODE_MAX] = {
 955         [VM_EXITCODE_INOUT]  = vmexit_inout,
 956         [VM_EXITCODE_INOUT_STR]  = vmexit_inout,
 957         [VM_EXITCODE_VMX]    = vmexit_vmx,
 958         [VM_EXITCODE_SVM]    = vmexit_svm,
 959         [VM_EXITCODE_BOGUS]  = vmexit_bogus,
 960         [VM_EXITCODE_REQIDLE] = vmexit_reqidle,
 961         [VM_EXITCODE_RDMSR]  = vmexit_rdmsr,
 962         [VM_EXITCODE_WRMSR]  = vmexit_wrmsr,
 963         [VM_EXITCODE_MTRAP]  = vmexit_mtrap,
 964         [VM_EXITCODE_INST_EMUL] = vmexit_inst_emul,
 965         [VM_EXITCODE_SUSPENDED] = vmexit_suspend,
 966         [VM_EXITCODE_TASK_SWITCH] = vmexit_task_switch,
 967         [VM_EXITCODE_DEBUG] = vmexit_debug,
 968         [VM_EXITCODE_BPT] = vmexit_breakpoint,
 969         [VM_EXITCODE_IPI] = vmexit_ipi,
 970 };
 971
 972 static void
 973 vm_loop(struct vmctx *ctx, struct vcpu *vcpu)
 974 {
 975         struct vm_exit vme;
 976         int error, rc;
 977         enum vm_exitcode exitcode;
 978         cpuset_t active_cpus;
 979
 980         error = vm_active_cpus(ctx, &active_cpus);
 981         assert(CPU_ISSET(vcpu_id(vcpu), &active_cpus));
 982
 983         while (1) {
 984                 error = vm_run(vcpu, &vme);
 985                 if (error != 0)
 986                         break;
 987
 988                 exitcode = vme.exitcode;
 989                 if (exitcode >= VM_EXITCODE_MAX || handler[exitcode] == NULL) {
 990                         fprintf(stderr, "vm_loop: unexpected exitcode 0x%x\n",
 991                             exitcode);
 992                         exit(4);
 993                 }
 994
 995                 rc = (*handler[exitcode])(ctx, vcpu, &vme);
 996
 997                 switch (rc) {
 998                 case VMEXIT_CONTINUE:
 999                         break;
1000                 case VMEXIT_ABORT:
1001                         abort();
1002                 default:
1003                         exit(4);
1004                 }
1005         }
1006         fprintf(stderr, "vm_run error %d, errno %d\n", error, errno);
1007 }
1008
1009 static int
1010 num_vcpus_allowed(struct vmctx *ctx, struct vcpu *vcpu)
1011 {
1012         uint16_t sockets, cores, threads, maxcpus;
1013         int tmp, error;
1014
1015         /*
1016          * The guest is allowed to spinup more than one processor only if the
1017          * UNRESTRICTED_GUEST capability is available.
1018          */
1019         error = vm_get_capability(vcpu, VM_CAP_UNRESTRICTED_GUEST, &tmp);
1020         if (error != 0)
1021                 return (1);
1022
1023         error = vm_get_topology(ctx, &sockets, &cores, &threads, &maxcpus);
1024         if (error == 0)
1025                 return (maxcpus);
1026         else
1027                 return (1);
1028 }
1029
1030 static void
1031 fbsdrun_set_capabilities(struct vcpu *vcpu, bool bsp)
1032 {
1033         int err, tmp;
1034
1035         if (get_config_bool_default("x86.vmexit_on_hlt", false)) {
1036                 err = vm_get_capability(vcpu, VM_CAP_HALT_EXIT, &tmp);
1037                 if (err < 0) {
1038                         fprintf(stderr, "VM exit on HLT not supported\n");
1039                         exit(4);
1040                 }
1041                 vm_set_capability(vcpu, VM_CAP_HALT_EXIT, 1);
1042                 if (bsp)
1043                         handler[VM_EXITCODE_HLT] = vmexit_hlt;
1044         }
1045
1046         if (get_config_bool_default("x86.vmexit_on_pause", false)) {
1047                 /*
1048                  * pause exit support required for this mode
1049                  */
1050                 err = vm_get_capability(vcpu, VM_CAP_PAUSE_EXIT, &tmp);
1051                 if (err < 0) {
1052                         fprintf(stderr,
1053                             "SMP mux requested, no pause support\n");
1054                         exit(4);
1055                 }
1056                 vm_set_capability(vcpu, VM_CAP_PAUSE_EXIT, 1);
1057                 if (bsp)
1058                         handler[VM_EXITCODE_PAUSE] = vmexit_pause;
1059         }
1060
1061         if (get_config_bool_default("x86.x2apic", false))
1062                 err = vm_set_x2apic_state(vcpu, X2APIC_ENABLED);
1063         else
1064                 err = vm_set_x2apic_state(vcpu, X2APIC_DISABLED);
1065
1066         if (err) {
1067                 fprintf(stderr, "Unable to set x2apic state (%d)\n", err);
1068                 exit(4);
1069         }
1070
1071         vm_set_capability(vcpu, VM_CAP_ENABLE_INVPCID, 1);
1072
1073         err = vm_set_capability(vcpu, VM_CAP_IPI_EXIT, 1);
1074         assert(err == 0);
1075 }
1076
1077 static struct vmctx *
1078 do_open(const char *vmname)
1079 {
1080         struct vmctx *ctx;
1081         int error;
1082         bool reinit, romboot;
1083
1084         reinit = romboot = false;
1085
1086         if (lpc_bootrom())
1087                 romboot = true;
1088
1089         error = vm_create(vmname);
1090         if (error) {
1091                 if (errno == EEXIST) {
1092                         if (romboot) {
1093                                 reinit = true;
1094                         } else {
1095                                 /*
1096                                  * The virtual machine has been setup by the
1097                                  * userspace bootloader.
1098                                  */
1099                         }
1100                 } else {
1101                         perror("vm_create");
1102                         exit(4);
1103                 }
1104         } else {
1105                 if (!romboot) {
1106                         /*
1107                          * If the virtual machine was just created then a
1108                          * bootrom must be configured to boot it.
1109                          */
1110                         fprintf(stderr, "virtual machine cannot be booted\n");
1111                         exit(4);
1112                 }
1113         }
1114
1115         ctx = vm_open(vmname);
1116         if (ctx == NULL) {
1117                 perror("vm_open");
1118                 exit(4);
1119         }
1120
1121 #ifndef WITHOUT_CAPSICUM
1122         if (vm_limit_rights(ctx) != 0)
1123                 err(EX_OSERR, "vm_limit_rights");
1124 #endif
1125
1126         if (reinit) {
1127                 error = vm_reinit(ctx);
1128                 if (error) {
1129                         perror("vm_reinit");
1130                         exit(4);
1131                 }
1132         }
1133         error = vm_set_topology(ctx, cpu_sockets, cpu_cores, cpu_threads,
1134             0 /* maxcpus, unimplemented */);
1135         if (error)
1136                 errx(EX_OSERR, "vm_set_topology");
1137         return (ctx);
1138 }
1139
1140 static void
1141 spinup_vcpu(struct vcpu_info *vi, bool bsp)
1142 {
1143         int error;
1144
1145         if (!bsp) {
1146                 fbsdrun_set_capabilities(vi->vcpu, false);
1147
1148                 /*
1149                  * Enable the 'unrestricted guest' mode for APs.
1150                  *
1151                  * APs startup in power-on 16-bit mode.
1152                  */
1153                 error = vm_set_capability(vi->vcpu, VM_CAP_UNRESTRICTED_GUEST, 1);
1154                 assert(error == 0);
1155         }
1156
1157         fbsdrun_addcpu(vi);
1158 }
1159
1160 static bool
1161 parse_config_option(const char *option)
1162 {
1163         const char *value;
1164         char *path;
1165
1166         value = strchr(option, '=');
1167         if (value == NULL || value[1] == '\0')
1168                 return (false);
1169         path = strndup(option, value - option);
1170         if (path == NULL)
1171                 err(4, "Failed to allocate memory");
1172         set_config_value(path, value + 1);
1173         return (true);
1174 }
1175
1176 static void
1177 parse_simple_config_file(const char *path)
1178 {
1179         FILE *fp;
1180         char *line, *cp;
1181         size_t linecap;
1182         unsigned int lineno;
1183
1184         fp = fopen(path, "r");
1185         if (fp == NULL)
1186                 err(4, "Failed to open configuration file %s", path);
1187         line = NULL;
1188         linecap = 0;
1189         lineno = 1;
1190         for (lineno = 1; getline(&line, &linecap, fp) > 0; lineno++) {
1191                 if (*line == '#' || *line == '\n')
1192                         continue;
1193                 cp = strchr(line, '\n');
1194                 if (cp != NULL)
1195                         *cp = '\0';
1196                 if (!parse_config_option(line))
1197                         errx(4, "%s line %u: invalid config option '%s'", path,
1198                             lineno, line);
1199         }
1200         free(line);
1201         fclose(fp);
1202 }
1203
1204 static void
1205 parse_gdb_options(const char *opt)
1206 {
1207         const char *sport;
1208         char *colon;
1209
1210         if (opt[0] == 'w') {
1211                 set_config_bool("gdb.wait", true);
1212                 opt++;
1213         }
1214
1215         colon = strrchr(opt, ':');
1216         if (colon == NULL) {
1217                 sport = opt;
1218         } else {
1219                 *colon = '\0';
1220                 colon++;
1221                 sport = colon;
1222                 set_config_value("gdb.address", opt);
1223         }
1224
1225         set_config_value("gdb.port", sport);
1226 }
1227
1228 static void
1229 set_defaults(void)
1230 {
1231
1232         set_config_bool("acpi_tables", false);
1233         set_config_value("memory.size", "256M");
1234         set_config_bool("x86.strictmsr", true);
1235         set_config_value("lpc.fwcfg", "bhyve");
1236 }
1237
1238 int
1239 main(int argc, char *argv[])
1240 {
1241         int c, error;
1242         int max_vcpus, memflags;
1243         struct vcpu *bsp;
1244         struct vmctx *ctx;
1245         uint64_t rip;
1246         size_t memsize;
1247         const char *optstr, *value, *vmname;
1248 #ifdef BHYVE_SNAPSHOT
1249         char *restore_file;
1250         struct restore_state rstate;
1251
1252         restore_file = NULL;
1253 #endif
1254
1255         init_config();
1256         set_defaults();
1257         progname = basename(argv[0]);
1258
1259 #ifdef BHYVE_SNAPSHOT
1260         optstr = "aehuwxACDHIPSWYk:o:p:G:c:s:m:l:K:U:r:";
1261 #else
1262         optstr = "aehuwxACDHIPSWYk:o:p:G:c:s:m:l:K:U:";
1263 #endif
1264         while ((c = getopt(argc, argv, optstr)) != -1) {
1265                 switch (c) {
1266                 case 'a':
1267                         set_config_bool("x86.x2apic", false);
1268                         break;
1269                 case 'A':
1270                         set_config_bool("acpi_tables", true);
1271                         break;
1272                 case 'D':
1273                         set_config_bool("destroy_on_poweroff", true);
1274                         break;
1275                 case 'p':
1276                         if (pincpu_parse(optarg) != 0) {
1277                             errx(EX_USAGE, "invalid vcpu pinning "
1278                                  "configuration '%s'", optarg);
1279                         }
1280                         break;
1281                 case 'c':
1282                         if (topology_parse(optarg) != 0) {
1283                             errx(EX_USAGE, "invalid cpu topology "
1284                                 "'%s'", optarg);
1285                         }
1286                         break;
1287                 case 'C':
1288                         set_config_bool("memory.guest_in_core", true);
1289                         break;
1290                 case 'G':
1291                         parse_gdb_options(optarg);
1292                         break;
1293                 case 'k':
1294                         parse_simple_config_file(optarg);
1295                         break;
1296                 case 'K':
1297                         set_config_value("keyboard.layout", optarg);
1298                         break;
1299                 case 'l':
1300                         if (strncmp(optarg, "help", strlen(optarg)) == 0) {
1301                                 lpc_print_supported_devices();
1302                                 exit(0);
1303                         } else if (lpc_device_parse(optarg) != 0) {
1304                                 errx(EX_USAGE, "invalid lpc device "
1305                                     "configuration '%s'", optarg);
1306                         }
1307                         break;
1308 #ifdef BHYVE_SNAPSHOT
1309                 case 'r':
1310                         restore_file = optarg;
1311                         break;
1312 #endif
1313                 case 's':
1314                         if (strncmp(optarg, "help", strlen(optarg)) == 0) {
1315                                 pci_print_supported_devices();
1316                                 exit(0);
1317                         } else if (pci_parse_slot(optarg) != 0)
1318                                 exit(4);
1319                         else
1320                                 break;
1321                 case 'S':
1322                         set_config_bool("memory.wired", true);
1323                         break;
1324                 case 'm':
1325                         set_config_value("memory.size", optarg);
1326                         break;
1327                 case 'o':
1328                         if (!parse_config_option(optarg))
1329                                 errx(EX_USAGE, "invalid configuration option '%s'", optarg);
1330                         break;
1331                 case 'H':
1332                         set_config_bool("x86.vmexit_on_hlt", true);
1333                         break;
1334                 case 'I':
1335                         /*
1336                          * The "-I" option was used to add an ioapic to the
1337                          * virtual machine.
1338                          *
1339                          * An ioapic is now provided unconditionally for each
1340                          * virtual machine and this option is now deprecated.
1341                          */
1342                         break;
1343                 case 'P':
1344                         set_config_bool("x86.vmexit_on_pause", true);
1345                         break;
1346                 case 'e':
1347                         set_config_bool("x86.strictio", true);
1348                         break;
1349                 case 'u':
1350                         set_config_bool("rtc.use_localtime", false);
1351                         break;
1352                 case 'U':
1353                         set_config_value("uuid", optarg);
1354                         break;
1355                 case 'w':
1356                         set_config_bool("x86.strictmsr", false);
1357                         break;
1358                 case 'W':
1359                         set_config_bool("virtio_msix", false);
1360                         break;
1361                 case 'x':
1362                         set_config_bool("x86.x2apic", true);
1363                         break;
1364                 case 'Y':
1365                         set_config_bool("x86.mptable", false);
1366                         break;
1367                 case 'h':
1368                         usage(0);
1369                 default:
1370                         usage(1);
1371                 }
1372         }
1373         argc -= optind;
1374         argv += optind;
1375
1376         if (argc > 1)
1377                 usage(1);
1378
1379 #ifdef BHYVE_SNAPSHOT
1380         if (restore_file != NULL) {
1381                 error = load_restore_file(restore_file, &rstate);
1382                 if (error) {
1383                         fprintf(stderr, "Failed to read checkpoint info from "
1384                                         "file: '%s'.\n", restore_file);
1385                         exit(1);
1386                 }
1387                 vmname = lookup_vmname(&rstate);
1388                 if (vmname != NULL)
1389                         set_config_value("name", vmname);
1390         }
1391 #endif
1392
1393         if (argc == 1)
1394                 set_config_value("name", argv[0]);
1395
1396         vmname = get_config_value("name");
1397         if (vmname == NULL)
1398                 usage(1);
1399
1400         if (get_config_bool_default("config.dump", false)) {
1401                 dump_config();
1402                 exit(1);
1403         }
1404
1405         calc_topology();
1406         build_vcpumaps();
1407
1408         value = get_config_value("memory.size");
1409         error = vm_parse_memsize(value, &memsize);
1410         if (error)
1411                 errx(EX_USAGE, "invalid memsize '%s'", value);
1412
1413         ctx = do_open(vmname);
1414
1415 #ifdef BHYVE_SNAPSHOT
1416         if (restore_file != NULL) {
1417                 guest_ncpus = lookup_guest_ncpus(&rstate);
1418                 memflags = lookup_memflags(&rstate);
1419                 memsize = lookup_memsize(&rstate);
1420         }
1421
1422         if (guest_ncpus < 1) {
1423                 fprintf(stderr, "Invalid guest vCPUs (%d)\n", guest_ncpus);
1424                 exit(1);
1425         }
1426 #endif
1427
1428         bsp = vm_vcpu_open(ctx, BSP);
1429         max_vcpus = num_vcpus_allowed(ctx, bsp);
1430         if (guest_ncpus > max_vcpus) {
1431                 fprintf(stderr, "%d vCPUs requested but only %d available\n",
1432                         guest_ncpus, max_vcpus);
1433                 exit(4);
1434         }
1435
1436         fbsdrun_set_capabilities(bsp, true);
1437
1438         /* Allocate per-VCPU resources. */
1439         vcpu_info = calloc(guest_ncpus, sizeof(*vcpu_info));
1440         for (int vcpuid = 0; vcpuid < guest_ncpus; vcpuid++) {
1441                 vcpu_info[vcpuid].ctx = ctx;
1442                 vcpu_info[vcpuid].vcpuid = vcpuid;
1443                 if (vcpuid == BSP)
1444                         vcpu_info[vcpuid].vcpu = bsp;
1445                 else
1446                         vcpu_info[vcpuid].vcpu = vm_vcpu_open(ctx, vcpuid);
1447         }
1448
1449         memflags = 0;
1450         if (get_config_bool_default("memory.wired", false))
1451                 memflags |= VM_MEM_F_WIRED;
1452         if (get_config_bool_default("memory.guest_in_core", false))
1453                 memflags |= VM_MEM_F_INCORE;
1454         vm_set_memflags(ctx, memflags);
1455         error = vm_setup_memory(ctx, memsize, VM_MMAP_ALL);
1456         if (error) {
1457                 fprintf(stderr, "Unable to setup memory (%d)\n", errno);
1458                 exit(4);
1459         }
1460
1461         error = init_msr();
1462         if (error) {
1463                 fprintf(stderr, "init_msr error %d", error);
1464                 exit(4);
1465         }
1466
1467         init_mem(guest_ncpus);
1468         init_inout();
1469         kernemu_dev_init();
1470         init_bootrom(ctx);
1471         atkbdc_init(ctx);
1472         pci_irq_init(ctx);
1473         ioapic_init(ctx);
1474
1475         rtc_init(ctx);
1476         sci_init(ctx);
1477
1478         if (qemu_fwcfg_init(ctx) != 0) {
1479                 fprintf(stderr, "qemu fwcfg initialization error");
1480                 exit(4);
1481         }
1482
1483         if (qemu_fwcfg_add_file("opt/bhyve/hw.ncpu", sizeof(guest_ncpus),
1484             &guest_ncpus) != 0) {
1485                 fprintf(stderr, "Could not add qemu fwcfg opt/bhyve/hw.ncpu");
1486                 exit(4);
1487         }
1488
1489         /*
1490          * Exit if a device emulation finds an error in its initilization
1491          */
1492         if (init_pci(ctx) != 0) {
1493                 perror("device emulation initialization error");
1494                 exit(4);
1495         }
1496
1497         /*
1498          * Initialize after PCI, to allow a bootrom file to reserve the high
1499          * region.
1500          */
1501         if (get_config_bool("acpi_tables"))
1502                 vmgenc_init(ctx);
1503
1504         init_gdb(ctx);
1505
1506         if (lpc_bootrom()) {
1507                 if (vm_set_capability(bsp, VM_CAP_UNRESTRICTED_GUEST, 1)) {
1508                         fprintf(stderr, "ROM boot failed: unrestricted guest "
1509                             "capability not available\n");
1510                         exit(4);
1511                 }
1512                 error = vcpu_reset(bsp);
1513                 assert(error == 0);
1514         }
1515
1516         /*
1517          * Add all vCPUs.
1518          */
1519         for (int vcpuid = 0; vcpuid < guest_ncpus; vcpuid++)
1520                 spinup_vcpu(&vcpu_info[vcpuid], vcpuid == BSP);
1521
1522 #ifdef BHYVE_SNAPSHOT
1523         if (restore_file != NULL) {
1524                 fprintf(stdout, "Pausing pci devs...\r\n");
1525                 if (vm_pause_user_devs() != 0) {
1526                         fprintf(stderr, "Failed to pause PCI device state.\n");
1527                         exit(1);
1528                 }
1529
1530                 fprintf(stdout, "Restoring vm mem...\r\n");
1531                 if (restore_vm_mem(ctx, &rstate) != 0) {
1532                         fprintf(stderr, "Failed to restore VM memory.\n");
1533                         exit(1);
1534                 }
1535
1536                 fprintf(stdout, "Restoring pci devs...\r\n");
1537                 if (vm_restore_user_devs(&rstate) != 0) {
1538                         fprintf(stderr, "Failed to restore PCI device state.\n");
1539                         exit(1);
1540                 }
1541
1542                 fprintf(stdout, "Restoring kernel structs...\r\n");
1543                 if (vm_restore_kern_structs(ctx, &rstate) != 0) {
1544                         fprintf(stderr, "Failed to restore kernel structs.\n");
1545                         exit(1);
1546                 }
1547
1548                 fprintf(stdout, "Resuming pci devs...\r\n");
1549                 if (vm_resume_user_devs() != 0) {
1550                         fprintf(stderr, "Failed to resume PCI device state.\n");
1551                         exit(1);
1552                 }
1553         }
1554 #endif
1555
1556         error = vm_get_register(bsp, VM_REG_GUEST_RIP, &rip);
1557         assert(error == 0);
1558
1559         /*
1560          * build the guest tables, MP etc.
1561          */
1562         if (get_config_bool_default("x86.mptable", true)) {
1563                 error = mptable_build(ctx, guest_ncpus);
1564                 if (error) {
1565                         perror("error to build the guest tables");
1566                         exit(4);
1567                 }
1568         }
1569
1570         error = smbios_build(ctx);
1571         if (error != 0)
1572                 exit(4);
1573
1574         if (get_config_bool("acpi_tables")) {
1575                 error = acpi_build(ctx, guest_ncpus);
1576                 assert(error == 0);
1577         }
1578
1579         if (lpc_bootrom() && strcmp(lpc_fwcfg(), "bhyve") == 0) {
1580                 fwctl_init();
1581         }
1582
1583         /*
1584          * Change the proc title to include the VM name.
1585          */
1586         setproctitle("%s", vmname);
1587
1588 #ifdef BHYVE_SNAPSHOT
1589         /* initialize mutex/cond variables */
1590         init_snapshot();
1591
1592         /*
1593          * checkpointing thread for communication with bhyvectl
1594          */
1595         if (init_checkpoint_thread(ctx) != 0)
1596                 errx(EX_OSERR, "Failed to start checkpoint thread");
1597 #endif
1598
1599 #ifndef WITHOUT_CAPSICUM
1600         caph_cache_catpages();
1601
1602         if (caph_limit_stdout() == -1 || caph_limit_stderr() == -1)
1603                 errx(EX_OSERR, "Unable to apply rights for sandbox");
1604
1605         if (caph_enter() == -1)
1606                 errx(EX_OSERR, "cap_enter() failed");
1607 #endif
1608
1609 #ifdef BHYVE_SNAPSHOT
1610         if (restore_file != NULL) {
1611                 destroy_restore_state(&rstate);
1612                 if (vm_restore_time(ctx) < 0)
1613                         err(EX_OSERR, "Unable to restore time");
1614
1615                 for (int vcpuid = 0; vcpuid < guest_ncpus; vcpuid++)
1616                         vm_resume_cpu(vcpu_info[vcpuid].vcpu);
1617         } else
1618 #endif
1619                 vm_resume_cpu(bsp);
1620
1621         /*
1622          * Head off to the main event dispatch loop
1623          */
1624         mevent_dispatch();
1625
1626         exit(4);
1627 }