2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2016 Flavius Anton
5 * Copyright (c) 2016 Mihai Tiganus
6 * Copyright (c) 2016-2019 Mihai Carabas
7 * Copyright (c) 2017-2019 Darius Mihai
8 * Copyright (c) 2017-2019 Elena Mihailescu
9 * Copyright (c) 2018-2019 Sergiu Weisz
10 * All rights reserved.
11 * The bhyve-snapshot feature was developed under sponsorships
12 * from Matthew Grooms.
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in the
21 * documentation and/or other materials provided with the distribution.
23 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 #include <sys/cdefs.h>
37 __FBSDID("$FreeBSD$");
39 #include <sys/types.h>
40 #ifndef WITHOUT_CAPSICUM
41 #include <sys/capsicum.h>
44 #include <sys/socket.h>
49 #include <machine/atomic.h>
50 #include <machine/segments.h>
52 #ifndef WITHOUT_CAPSICUM
53 #include <capsicum_helpers.h>
67 #include <pthread_np.h>
70 #include <sys/ioctl.h>
72 #include <machine/vmm.h>
73 #ifndef WITHOUT_CAPSICUM
74 #include <machine/vmm_dev.h>
76 #include <machine/vmm_snapshot.h>
93 #include "smbiostbl.h"
96 #include "spinup_ap.h"
102 struct spinner_info {
103 const size_t *crtval;
108 extern int guest_ncpus;
110 static struct winsize winsize;
111 static sig_t old_winch_handler;
114 #define MB (1024UL * KB)
115 #define GB (1024UL * MB)
117 #define SNAPSHOT_CHUNK (4 * MB)
118 #define PROG_BUF_SZ (8192)
120 #define SNAPSHOT_BUFFER_SIZE (20 * MB)
122 #define JSON_STRUCT_ARR_KEY "structs"
123 #define JSON_DEV_ARR_KEY "devices"
124 #define JSON_BASIC_METADATA_KEY "basic metadata"
125 #define JSON_SNAPSHOT_REQ_KEY "snapshot_req"
126 #define JSON_SIZE_KEY "size"
127 #define JSON_FILE_OFFSET_KEY "file_offset"
129 #define JSON_NCPUS_KEY "ncpus"
130 #define JSON_VMNAME_KEY "vmname"
131 #define JSON_MEMSIZE_KEY "memsize"
132 #define JSON_MEMFLAGS_KEY "memflags"
136 __typeof__ (a) _a = (a); \
137 __typeof__ (b) _b = (b); \
141 static const struct vm_snapshot_dev_info snapshot_devs[] = {
142 { "atkbdc", atkbdc_snapshot, NULL, NULL },
143 { "virtio-net", pci_snapshot, pci_pause, pci_resume },
144 { "virtio-blk", pci_snapshot, pci_pause, pci_resume },
145 { "virtio-rnd", pci_snapshot, NULL, NULL },
146 { "lpc", pci_snapshot, NULL, NULL },
147 { "fbuf", pci_snapshot, NULL, NULL },
148 { "xhci", pci_snapshot, NULL, NULL },
149 { "e1000", pci_snapshot, NULL, NULL },
150 { "ahci", pci_snapshot, pci_pause, pci_resume },
151 { "ahci-hd", pci_snapshot, pci_pause, pci_resume },
152 { "ahci-cd", pci_snapshot, pci_pause, pci_resume },
155 static const struct vm_snapshot_kern_info snapshot_kern_structs[] = {
156 { "vhpet", STRUCT_VHPET },
158 { "vmx", STRUCT_VMX },
159 { "vioapic", STRUCT_VIOAPIC },
160 { "vlapic", STRUCT_VLAPIC },
161 { "vmcx", STRUCT_VMCX },
162 { "vatpit", STRUCT_VATPIT },
163 { "vatpic", STRUCT_VATPIC },
164 { "vpmtmr", STRUCT_VPMTMR },
165 { "vrtc", STRUCT_VRTC },
168 static cpuset_t vcpus_active, vcpus_suspended;
169 static pthread_mutex_t vcpu_lock;
170 static pthread_cond_t vcpus_idle, vcpus_can_run;
171 static bool checkpoint_active;
174 * TODO: Harden this function and all of its callers since 'base_str' is a user
178 strcat_extension(const char *base_str, const char *ext)
181 size_t base_len, ext_len;
183 base_len = strnlen(base_str, NAME_MAX);
184 ext_len = strnlen(ext, NAME_MAX);
186 if (base_len + ext_len > NAME_MAX) {
187 fprintf(stderr, "Filename exceeds maximum length.\n");
191 res = malloc(base_len + ext_len + 1);
193 perror("Failed to allocate memory.");
197 memcpy(res, base_str, base_len);
198 memcpy(res + base_len, ext, ext_len);
199 res[base_len + ext_len] = 0;
205 destroy_restore_state(struct restore_state *rstate)
207 if (rstate == NULL) {
208 fprintf(stderr, "Attempting to destroy NULL restore struct.\n");
212 if (rstate->kdata_map != MAP_FAILED)
213 munmap(rstate->kdata_map, rstate->kdata_len);
215 if (rstate->kdata_fd > 0)
216 close(rstate->kdata_fd);
217 if (rstate->vmmem_fd > 0)
218 close(rstate->vmmem_fd);
220 if (rstate->meta_root_obj != NULL)
221 ucl_object_unref(rstate->meta_root_obj);
222 if (rstate->meta_parser != NULL)
223 ucl_parser_free(rstate->meta_parser);
227 load_vmmem_file(const char *filename, struct restore_state *rstate)
232 rstate->vmmem_fd = open(filename, O_RDONLY);
233 if (rstate->vmmem_fd < 0) {
234 perror("Failed to open restore file");
238 err = fstat(rstate->vmmem_fd, &sb);
240 perror("Failed to stat restore file");
244 if (sb.st_size == 0) {
245 fprintf(stderr, "Restore file is empty.\n");
249 rstate->vmmem_len = sb.st_size;
254 if (rstate->vmmem_fd > 0)
255 close(rstate->vmmem_fd);
260 load_kdata_file(const char *filename, struct restore_state *rstate)
265 rstate->kdata_fd = open(filename, O_RDONLY);
266 if (rstate->kdata_fd < 0) {
267 perror("Failed to open kernel data file");
271 err = fstat(rstate->kdata_fd, &sb);
273 perror("Failed to stat kernel data file");
277 if (sb.st_size == 0) {
278 fprintf(stderr, "Kernel data file is empty.\n");
282 rstate->kdata_len = sb.st_size;
283 rstate->kdata_map = mmap(NULL, rstate->kdata_len, PROT_READ,
284 MAP_SHARED, rstate->kdata_fd, 0);
285 if (rstate->kdata_map == MAP_FAILED) {
286 perror("Failed to map restore file");
293 if (rstate->kdata_fd > 0)
294 close(rstate->kdata_fd);
299 load_metadata_file(const char *filename, struct restore_state *rstate)
302 struct ucl_parser *parser;
305 parser = ucl_parser_new(UCL_PARSER_DEFAULT);
306 if (parser == NULL) {
307 fprintf(stderr, "Failed to initialize UCL parser.\n");
309 goto err_load_metadata;
312 err = ucl_parser_add_file(parser, filename);
314 fprintf(stderr, "Failed to parse metadata file: '%s'\n",
317 goto err_load_metadata;
320 obj = ucl_parser_get_object(parser);
322 fprintf(stderr, "Failed to parse object.\n");
324 goto err_load_metadata;
327 rstate->meta_parser = parser;
328 rstate->meta_root_obj = (ucl_object_t *)obj;
334 ucl_parser_free(parser);
339 load_restore_file(const char *filename, struct restore_state *rstate)
342 char *kdata_filename = NULL, *meta_filename = NULL;
344 assert(filename != NULL);
345 assert(rstate != NULL);
347 memset(rstate, 0, sizeof(*rstate));
348 rstate->kdata_map = MAP_FAILED;
350 err = load_vmmem_file(filename, rstate);
352 fprintf(stderr, "Failed to load guest RAM file.\n");
356 kdata_filename = strcat_extension(filename, ".kern");
357 if (kdata_filename == NULL) {
358 fprintf(stderr, "Failed to construct kernel data filename.\n");
362 err = load_kdata_file(kdata_filename, rstate);
364 fprintf(stderr, "Failed to load guest kernel data file.\n");
368 meta_filename = strcat_extension(filename, ".meta");
369 if (meta_filename == NULL) {
370 fprintf(stderr, "Failed to construct kernel metadata filename.\n");
374 err = load_metadata_file(meta_filename, rstate);
376 fprintf(stderr, "Failed to load guest metadata file.\n");
383 destroy_restore_state(rstate);
384 if (kdata_filename != NULL)
385 free(kdata_filename);
386 if (meta_filename != NULL)
391 #define JSON_GET_INT_OR_RETURN(key, obj, result_ptr, ret) \
393 const ucl_object_t *obj__; \
394 obj__ = ucl_object_lookup(obj, key); \
395 if (obj__ == NULL) { \
396 fprintf(stderr, "Missing key: '%s'", key); \
399 if (!ucl_object_toint_safe(obj__, result_ptr)) { \
400 fprintf(stderr, "Cannot convert '%s' value to int.", key); \
405 #define JSON_GET_STRING_OR_RETURN(key, obj, result_ptr, ret) \
407 const ucl_object_t *obj__; \
408 obj__ = ucl_object_lookup(obj, key); \
409 if (obj__ == NULL) { \
410 fprintf(stderr, "Missing key: '%s'", key); \
413 if (!ucl_object_tostring_safe(obj__, result_ptr)) { \
414 fprintf(stderr, "Cannot convert '%s' value to string.", key); \
420 lookup_struct(enum snapshot_req struct_id, struct restore_state *rstate,
423 const ucl_object_t *structs = NULL, *obj = NULL;
424 ucl_object_iter_t it = NULL;
425 int64_t snapshot_req, size, file_offset;
427 structs = ucl_object_lookup(rstate->meta_root_obj, JSON_STRUCT_ARR_KEY);
428 if (structs == NULL) {
429 fprintf(stderr, "Failed to find '%s' object.\n",
430 JSON_STRUCT_ARR_KEY);
434 if (ucl_object_type(structs) != UCL_ARRAY) {
435 fprintf(stderr, "Object '%s' is not an array.\n",
436 JSON_STRUCT_ARR_KEY);
440 while ((obj = ucl_object_iterate(structs, &it, true)) != NULL) {
442 JSON_GET_INT_OR_RETURN(JSON_SNAPSHOT_REQ_KEY, obj,
443 &snapshot_req, NULL);
444 assert(snapshot_req >= 0);
445 if ((enum snapshot_req) snapshot_req == struct_id) {
446 JSON_GET_INT_OR_RETURN(JSON_SIZE_KEY, obj,
450 JSON_GET_INT_OR_RETURN(JSON_FILE_OFFSET_KEY, obj,
452 assert(file_offset >= 0);
453 assert((uint64_t)file_offset + size <=
456 *struct_size = (size_t)size;
457 return ((uint8_t *)rstate->kdata_map + file_offset);
465 lookup_check_dev(const char *dev_name, struct restore_state *rstate,
466 const ucl_object_t *obj, size_t *data_size)
468 const char *snapshot_req;
469 int64_t size, file_offset;
472 JSON_GET_STRING_OR_RETURN(JSON_SNAPSHOT_REQ_KEY, obj,
473 &snapshot_req, NULL);
474 assert(snapshot_req != NULL);
475 if (!strcmp(snapshot_req, dev_name)) {
476 JSON_GET_INT_OR_RETURN(JSON_SIZE_KEY, obj,
480 JSON_GET_INT_OR_RETURN(JSON_FILE_OFFSET_KEY, obj,
482 assert(file_offset >= 0);
483 assert((uint64_t)file_offset + size <= rstate->kdata_len);
485 *data_size = (size_t)size;
486 return ((uint8_t *)rstate->kdata_map + file_offset);
493 lookup_dev(const char *dev_name, struct restore_state *rstate,
496 const ucl_object_t *devs = NULL, *obj = NULL;
497 ucl_object_iter_t it = NULL;
500 devs = ucl_object_lookup(rstate->meta_root_obj, JSON_DEV_ARR_KEY);
502 fprintf(stderr, "Failed to find '%s' object.\n",
507 if (ucl_object_type(devs) != UCL_ARRAY) {
508 fprintf(stderr, "Object '%s' is not an array.\n",
513 while ((obj = ucl_object_iterate(devs, &it, true)) != NULL) {
514 ret = lookup_check_dev(dev_name, rstate, obj, data_size);
522 static const ucl_object_t *
523 lookup_basic_metadata_object(struct restore_state *rstate)
525 const ucl_object_t *basic_meta_obj = NULL;
527 basic_meta_obj = ucl_object_lookup(rstate->meta_root_obj,
528 JSON_BASIC_METADATA_KEY);
529 if (basic_meta_obj == NULL) {
530 fprintf(stderr, "Failed to find '%s' object.\n",
531 JSON_BASIC_METADATA_KEY);
535 if (ucl_object_type(basic_meta_obj) != UCL_OBJECT) {
536 fprintf(stderr, "Object '%s' is not a JSON object.\n",
537 JSON_BASIC_METADATA_KEY);
541 return (basic_meta_obj);
545 lookup_vmname(struct restore_state *rstate)
548 const ucl_object_t *obj;
550 obj = lookup_basic_metadata_object(rstate);
554 JSON_GET_STRING_OR_RETURN(JSON_VMNAME_KEY, obj, &vmname, NULL);
559 lookup_memflags(struct restore_state *rstate)
562 const ucl_object_t *obj;
564 obj = lookup_basic_metadata_object(rstate);
568 JSON_GET_INT_OR_RETURN(JSON_MEMFLAGS_KEY, obj, &memflags, 0);
570 return ((int)memflags);
574 lookup_memsize(struct restore_state *rstate)
577 const ucl_object_t *obj;
579 obj = lookup_basic_metadata_object(rstate);
583 JSON_GET_INT_OR_RETURN(JSON_MEMSIZE_KEY, obj, &memsize, 0);
587 return ((size_t)memsize);
592 lookup_guest_ncpus(struct restore_state *rstate)
595 const ucl_object_t *obj;
597 obj = lookup_basic_metadata_object(rstate);
601 JSON_GET_INT_OR_RETURN(JSON_NCPUS_KEY, obj, &ncpus, 0);
606 winch_handler(int signal __unused)
609 ioctl(STDOUT_FILENO, TIOCGWINSZ, &winsize);
610 #endif /* TIOCGWINSZ */
614 print_progress(size_t crtval, const size_t maxval)
617 double crtval_gb, maxval_gb;
618 size_t i, win_width, prog_start, prog_done, prog_end;
621 static char prog_buf[PROG_BUF_SZ];
622 static const size_t len = sizeof(prog_buf);
625 static const char *div_str;
627 static char wip_bar[] = { '/', '-', '\\', '|' };
628 static int wip_idx = 0;
631 printf("[0B / 0B]\r\n");
638 if (maxval > 10 * GB) {
641 } else if (maxval > 10 * MB) {
649 crtval_gb = (double) crtval / div;
650 maxval_gb = (double) maxval / div;
652 rc = snprintf(prog_buf, len, "%.03lf", maxval_gb);
654 fprintf(stderr, "Maxval too big\n");
659 rc = snprintf(prog_buf, len, "\r[%*.03lf%s / %.03lf%s] |",
660 mval_len, crtval_gb, div_str, maxval_gb, div_str);
663 fprintf(stderr, "Buffer too small to print progress\n");
667 win_width = min(winsize.ws_col, len);
670 if (prog_start < (win_width - 2)) {
671 prog_end = win_width - prog_start - 2;
672 prog_done = prog_end * (crtval_gb / maxval_gb);
674 for (i = prog_start; i < prog_start + prog_done; i++)
677 if (crtval != maxval) {
678 prog_buf[i] = wip_bar[wip_idx];
679 wip_idx = (wip_idx + 1) % sizeof(wip_bar);
685 for (; i < win_width - 2; i++)
688 prog_buf[win_width - 2] = '|';
691 prog_buf[win_width - 1] = '\0';
692 write(STDOUT_FILENO, prog_buf, win_width);
698 snapshot_spinner_cb(void *arg)
701 size_t crtval, maxval, total;
702 struct spinner_info *si;
710 ts.tv_nsec = 50 * 1000 * 1000; /* 50 ms sleep time */
713 crtval = *si->crtval;
717 rc = print_progress(crtval, total);
719 fprintf(stderr, "Failed to parse progress\n");
723 nanosleep(&ts, NULL);
724 } while (crtval < maxval);
731 vm_snapshot_mem_part(const int snapfd, const size_t foff, void *src,
732 const size_t len, const size_t totalmem, const bool op_wr)
735 size_t part_done, todo, rem;
738 pthread_t spinner_th;
739 struct spinner_info *si;
741 if (lseek(snapfd, foff, SEEK_SET) < 0) {
742 perror("Failed to change file offset");
746 show_progress = false;
747 if (isatty(STDIN_FILENO) && (winsize.ws_col != 0))
748 show_progress = true;
754 si = &(struct spinner_info) {
755 .crtval = &part_done,
756 .maxval = foff + len,
760 rc = pthread_create(&spinner_th, 0, snapshot_spinner_cb, si);
762 perror("Unable to create spinner thread");
763 show_progress = false;
769 todo = min(SNAPSHOT_CHUNK, rem);
774 done = write(snapfd, src, todo);
776 done = read(snapfd, src, todo);
778 perror("Failed to write in file");
782 src = (uint8_t *)src + done;
788 rc = pthread_join(spinner_th, NULL);
790 perror("Unable to end spinner thread");
797 vm_snapshot_mem(struct vmctx *ctx, int snapfd, size_t memsz, const bool op_wr)
800 size_t lowmem, highmem, totalmem;
803 ret = vm_get_guestmem_from_ctx(ctx, &baseaddr, &lowmem, &highmem);
805 fprintf(stderr, "%s: unable to retrieve guest memory size\r\n",
809 totalmem = lowmem + highmem;
811 if ((op_wr == false) && (totalmem != memsz)) {
812 fprintf(stderr, "%s: mem size mismatch: %ld vs %ld\r\n",
813 __func__, totalmem, memsz);
819 ioctl(STDOUT_FILENO, TIOCGWINSZ, &winsize);
820 #endif /* TIOCGWINSZ */
821 old_winch_handler = signal(SIGWINCH, winch_handler);
823 ret = vm_snapshot_mem_part(snapfd, 0, baseaddr, lowmem,
826 fprintf(stderr, "%s: Could not %s lowmem\r\n",
827 __func__, op_wr ? "write" : "read");
835 ret = vm_snapshot_mem_part(snapfd, lowmem, baseaddr + 4*GB,
836 highmem, totalmem, op_wr);
838 fprintf(stderr, "%s: Could not %s highmem\r\n",
839 __func__, op_wr ? "write" : "read");
846 signal(SIGWINCH, old_winch_handler);
852 restore_vm_mem(struct vmctx *ctx, struct restore_state *rstate)
856 restored = vm_snapshot_mem(ctx, rstate->vmmem_fd, rstate->vmmem_len,
859 if (restored != rstate->vmmem_len)
866 vm_restore_kern_struct(struct vmctx *ctx, struct restore_state *rstate,
867 const struct vm_snapshot_kern_info *info)
872 struct vm_snapshot_meta *meta;
874 struct_ptr = lookup_struct(info->req, rstate, &struct_size);
875 if (struct_ptr == NULL) {
876 fprintf(stderr, "%s: Failed to lookup struct %s\r\n",
877 __func__, info->struct_name);
882 if (struct_size == 0) {
883 fprintf(stderr, "%s: Kernel struct size was 0 for: %s\r\n",
884 __func__, info->struct_name);
889 meta = &(struct vm_snapshot_meta) {
891 .dev_name = info->struct_name,
892 .dev_req = info->req,
894 .buffer.buf_start = struct_ptr,
895 .buffer.buf_size = struct_size,
897 .buffer.buf = struct_ptr,
898 .buffer.buf_rem = struct_size,
900 .op = VM_SNAPSHOT_RESTORE,
903 ret = vm_snapshot_req(meta);
905 fprintf(stderr, "%s: Failed to restore struct: %s\r\n",
906 __func__, info->struct_name);
915 vm_restore_kern_structs(struct vmctx *ctx, struct restore_state *rstate)
920 for (i = 0; i < nitems(snapshot_kern_structs); i++) {
921 ret = vm_restore_kern_struct(ctx, rstate,
922 &snapshot_kern_structs[i]);
931 vm_restore_user_dev(struct vmctx *ctx, struct restore_state *rstate,
932 const struct vm_snapshot_dev_info *info)
937 struct vm_snapshot_meta *meta;
939 dev_ptr = lookup_dev(info->dev_name, rstate, &dev_size);
940 if (dev_ptr == NULL) {
941 fprintf(stderr, "Failed to lookup dev: %s\r\n", info->dev_name);
942 fprintf(stderr, "Continuing the restore/migration process\r\n");
947 fprintf(stderr, "%s: Device size is 0. "
948 "Assuming %s is not used\r\n",
949 __func__, info->dev_name);
953 meta = &(struct vm_snapshot_meta) {
955 .dev_name = info->dev_name,
957 .buffer.buf_start = dev_ptr,
958 .buffer.buf_size = dev_size,
960 .buffer.buf = dev_ptr,
961 .buffer.buf_rem = dev_size,
963 .op = VM_SNAPSHOT_RESTORE,
966 ret = (*info->snapshot_cb)(meta);
968 fprintf(stderr, "Failed to restore dev: %s\r\n",
978 vm_restore_user_devs(struct vmctx *ctx, struct restore_state *rstate)
983 for (i = 0; i < nitems(snapshot_devs); i++) {
984 ret = vm_restore_user_dev(ctx, rstate, &snapshot_devs[i]);
993 vm_pause_user_devs(void)
995 const struct vm_snapshot_dev_info *info;
999 for (i = 0; i < nitems(snapshot_devs); i++) {
1000 info = &snapshot_devs[i];
1001 if (info->pause_cb == NULL)
1004 ret = info->pause_cb(info->dev_name);
1013 vm_resume_user_devs(void)
1015 const struct vm_snapshot_dev_info *info;
1019 for (i = 0; i < nitems(snapshot_devs); i++) {
1020 info = &snapshot_devs[i];
1021 if (info->resume_cb == NULL)
1024 ret = info->resume_cb(info->dev_name);
1033 vm_snapshot_kern_struct(int data_fd, xo_handle_t *xop, const char *array_key,
1034 struct vm_snapshot_meta *meta, off_t *offset)
1040 ret = vm_snapshot_req(meta);
1042 fprintf(stderr, "%s: Failed to snapshot struct %s\r\n",
1043 __func__, meta->dev_name);
1048 data_size = vm_get_snapshot_size(meta);
1050 /* XXX-MJ no handling for short writes. */
1051 write_cnt = write(data_fd, meta->buffer.buf_start, data_size);
1052 if (write_cnt < 0 || (size_t)write_cnt != data_size) {
1053 perror("Failed to write all snapshotted data.");
1058 /* Write metadata. */
1059 xo_open_instance_h(xop, array_key);
1060 xo_emit_h(xop, "{:debug_name/%s}\n", meta->dev_name);
1061 xo_emit_h(xop, "{:" JSON_SNAPSHOT_REQ_KEY "/%d}\n",
1063 xo_emit_h(xop, "{:" JSON_SIZE_KEY "/%lu}\n", data_size);
1064 xo_emit_h(xop, "{:" JSON_FILE_OFFSET_KEY "/%lu}\n", *offset);
1065 xo_close_instance_h(xop, JSON_STRUCT_ARR_KEY);
1067 *offset += data_size;
1074 vm_snapshot_kern_structs(struct vmctx *ctx, int data_fd, xo_handle_t *xop)
1077 size_t buf_size, i, offset;
1079 struct vm_snapshot_meta *meta;
1083 buf_size = SNAPSHOT_BUFFER_SIZE;
1085 buffer = malloc(SNAPSHOT_BUFFER_SIZE * sizeof(char));
1086 if (buffer == NULL) {
1088 perror("Failed to allocate memory for snapshot buffer");
1089 goto err_vm_snapshot_kern_data;
1092 meta = &(struct vm_snapshot_meta) {
1095 .buffer.buf_start = buffer,
1096 .buffer.buf_size = buf_size,
1098 .op = VM_SNAPSHOT_SAVE,
1101 xo_open_list_h(xop, JSON_STRUCT_ARR_KEY);
1102 for (i = 0; i < nitems(snapshot_kern_structs); i++) {
1103 meta->dev_name = snapshot_kern_structs[i].struct_name;
1104 meta->dev_req = snapshot_kern_structs[i].req;
1106 memset(meta->buffer.buf_start, 0, meta->buffer.buf_size);
1107 meta->buffer.buf = meta->buffer.buf_start;
1108 meta->buffer.buf_rem = meta->buffer.buf_size;
1110 ret = vm_snapshot_kern_struct(data_fd, xop, JSON_DEV_ARR_KEY,
1114 goto err_vm_snapshot_kern_data;
1117 xo_close_list_h(xop, JSON_STRUCT_ARR_KEY);
1119 err_vm_snapshot_kern_data:
1126 vm_snapshot_basic_metadata(struct vmctx *ctx, xo_handle_t *xop, size_t memsz)
1129 xo_open_container_h(xop, JSON_BASIC_METADATA_KEY);
1130 xo_emit_h(xop, "{:" JSON_NCPUS_KEY "/%ld}\n", guest_ncpus);
1131 xo_emit_h(xop, "{:" JSON_VMNAME_KEY "/%s}\n", vm_get_name(ctx));
1132 xo_emit_h(xop, "{:" JSON_MEMSIZE_KEY "/%lu}\n", memsz);
1133 xo_emit_h(xop, "{:" JSON_MEMFLAGS_KEY "/%d}\n", vm_get_memflags(ctx));
1134 xo_close_container_h(xop, JSON_BASIC_METADATA_KEY);
1140 vm_snapshot_dev_write_data(int data_fd, xo_handle_t *xop, const char *array_key,
1141 struct vm_snapshot_meta *meta, off_t *offset)
1146 data_size = vm_get_snapshot_size(meta);
1148 /* XXX-MJ no handling for short writes. */
1149 ret = write(data_fd, meta->buffer.buf_start, data_size);
1150 if (ret < 0 || (size_t)ret != data_size) {
1151 perror("Failed to write all snapshotted data.");
1155 /* Write metadata. */
1156 xo_open_instance_h(xop, array_key);
1157 xo_emit_h(xop, "{:" JSON_SNAPSHOT_REQ_KEY "/%s}\n", meta->dev_name);
1158 xo_emit_h(xop, "{:" JSON_SIZE_KEY "/%lu}\n", data_size);
1159 xo_emit_h(xop, "{:" JSON_FILE_OFFSET_KEY "/%lu}\n", *offset);
1160 xo_close_instance_h(xop, array_key);
1162 *offset += data_size;
1168 vm_snapshot_user_dev(const struct vm_snapshot_dev_info *info,
1169 int data_fd, xo_handle_t *xop,
1170 struct vm_snapshot_meta *meta, off_t *offset)
1174 ret = (*info->snapshot_cb)(meta);
1176 fprintf(stderr, "Failed to snapshot %s; ret=%d\r\n",
1177 meta->dev_name, ret);
1181 ret = vm_snapshot_dev_write_data(data_fd, xop, JSON_DEV_ARR_KEY, meta,
1190 vm_snapshot_user_devs(struct vmctx *ctx, int data_fd, xo_handle_t *xop)
1196 struct vm_snapshot_meta *meta;
1198 buf_size = SNAPSHOT_BUFFER_SIZE;
1200 offset = lseek(data_fd, 0, SEEK_CUR);
1202 perror("Failed to get data file current offset.");
1206 buffer = malloc(buf_size);
1207 if (buffer == NULL) {
1208 perror("Failed to allocate memory for snapshot buffer");
1213 meta = &(struct vm_snapshot_meta) {
1216 .buffer.buf_start = buffer,
1217 .buffer.buf_size = buf_size,
1219 .op = VM_SNAPSHOT_SAVE,
1222 xo_open_list_h(xop, JSON_DEV_ARR_KEY);
1224 /* Restore other devices that support this feature */
1225 for (i = 0; i < nitems(snapshot_devs); i++) {
1226 meta->dev_name = snapshot_devs[i].dev_name;
1228 memset(meta->buffer.buf_start, 0, meta->buffer.buf_size);
1229 meta->buffer.buf = meta->buffer.buf_start;
1230 meta->buffer.buf_rem = meta->buffer.buf_size;
1232 ret = vm_snapshot_user_dev(&snapshot_devs[i], data_fd, xop,
1238 xo_close_list_h(xop, JSON_DEV_ARR_KEY);
1247 checkpoint_cpu_add(int vcpu)
1250 pthread_mutex_lock(&vcpu_lock);
1251 CPU_SET(vcpu, &vcpus_active);
1253 if (checkpoint_active) {
1254 CPU_SET(vcpu, &vcpus_suspended);
1255 while (checkpoint_active)
1256 pthread_cond_wait(&vcpus_can_run, &vcpu_lock);
1257 CPU_CLR(vcpu, &vcpus_suspended);
1259 pthread_mutex_unlock(&vcpu_lock);
1263 * When a vCPU is suspended for any reason, it calls
1264 * checkpoint_cpu_suspend(). This records that the vCPU is idle.
1265 * Before returning from suspension, checkpoint_cpu_resume() is
1266 * called. In suspend we note that the vCPU is idle. In resume we
1267 * pause the vCPU thread until the checkpoint is complete. The reason
1268 * for the two-step process is that vCPUs might already be stopped in
1269 * the debug server when a checkpoint is requested. This approach
1270 * allows us to account for and handle those vCPUs.
1273 checkpoint_cpu_suspend(int vcpu)
1276 pthread_mutex_lock(&vcpu_lock);
1277 CPU_SET(vcpu, &vcpus_suspended);
1278 if (checkpoint_active && CPU_CMP(&vcpus_active, &vcpus_suspended) == 0)
1279 pthread_cond_signal(&vcpus_idle);
1280 pthread_mutex_unlock(&vcpu_lock);
1284 checkpoint_cpu_resume(int vcpu)
1287 pthread_mutex_lock(&vcpu_lock);
1288 while (checkpoint_active)
1289 pthread_cond_wait(&vcpus_can_run, &vcpu_lock);
1290 CPU_CLR(vcpu, &vcpus_suspended);
1291 pthread_mutex_unlock(&vcpu_lock);
1295 vm_vcpu_pause(struct vmctx *ctx)
1298 pthread_mutex_lock(&vcpu_lock);
1299 checkpoint_active = true;
1300 vm_suspend_cpu(ctx, -1);
1301 while (CPU_CMP(&vcpus_active, &vcpus_suspended) != 0)
1302 pthread_cond_wait(&vcpus_idle, &vcpu_lock);
1303 pthread_mutex_unlock(&vcpu_lock);
1307 vm_vcpu_resume(struct vmctx *ctx)
1310 pthread_mutex_lock(&vcpu_lock);
1311 checkpoint_active = false;
1312 pthread_mutex_unlock(&vcpu_lock);
1313 vm_resume_cpu(ctx, -1);
1314 pthread_cond_broadcast(&vcpus_can_run);
1318 vm_checkpoint(struct vmctx *ctx, const char *checkpoint_file, bool stop_vm)
1320 int fd_checkpoint = 0, kdata_fd = 0;
1324 xo_handle_t *xop = NULL;
1325 char *meta_filename = NULL;
1326 char *kdata_filename = NULL;
1327 FILE *meta_file = NULL;
1329 kdata_filename = strcat_extension(checkpoint_file, ".kern");
1330 if (kdata_filename == NULL) {
1331 fprintf(stderr, "Failed to construct kernel data filename.\n");
1335 kdata_fd = open(kdata_filename, O_WRONLY | O_CREAT | O_TRUNC, 0700);
1337 perror("Failed to open kernel data snapshot file.");
1342 fd_checkpoint = open(checkpoint_file, O_RDWR | O_CREAT | O_TRUNC, 0700);
1344 if (fd_checkpoint < 0) {
1345 perror("Failed to create checkpoint file");
1350 meta_filename = strcat_extension(checkpoint_file, ".meta");
1351 if (meta_filename == NULL) {
1352 fprintf(stderr, "Failed to construct vm metadata filename.\n");
1356 meta_file = fopen(meta_filename, "w");
1357 if (meta_file == NULL) {
1358 perror("Failed to open vm metadata snapshot file.");
1362 xop = xo_create_to_file(meta_file, XO_STYLE_JSON, XOF_PRETTY);
1364 perror("Failed to get libxo handle on metadata file.");
1370 ret = vm_pause_user_devs();
1372 fprintf(stderr, "Could not pause devices\r\n");
1377 memsz = vm_snapshot_mem(ctx, fd_checkpoint, 0, true);
1379 perror("Could not write guest memory to file");
1384 ret = vm_snapshot_basic_metadata(ctx, xop, memsz);
1386 fprintf(stderr, "Failed to snapshot vm basic metadata.\n");
1392 ret = vm_snapshot_kern_structs(ctx, kdata_fd, xop);
1394 fprintf(stderr, "Failed to snapshot vm kernel data.\n");
1399 ret = vm_snapshot_user_devs(ctx, kdata_fd, xop);
1401 fprintf(stderr, "Failed to snapshot device state.\n");
1414 ret = vm_resume_user_devs();
1416 fprintf(stderr, "Could not resume devices\r\n");
1417 vm_vcpu_resume(ctx);
1418 if (fd_checkpoint > 0)
1419 close(fd_checkpoint);
1420 if (meta_filename != NULL)
1421 free(meta_filename);
1422 if (kdata_filename != NULL)
1423 free(kdata_filename);
1426 if (meta_file != NULL)
1434 handle_message(struct vmctx *ctx, nvlist_t *nvl)
1437 struct ipc_command **ipc_cmd;
1439 if (!nvlist_exists_string(nvl, "cmd"))
1442 cmd = nvlist_get_string(nvl, "cmd");
1443 IPC_COMMAND_FOREACH(ipc_cmd, ipc_cmd_set) {
1444 if (strcmp(cmd, (*ipc_cmd)->name) == 0)
1445 return ((*ipc_cmd)->handler(ctx, nvl));
1448 return (EOPNOTSUPP);
1452 * Listen for commands from bhyvectl
1455 checkpoint_thread(void *param)
1458 struct checkpoint_thread_info *thread_info;
1461 pthread_set_name_np(pthread_self(), "checkpoint thread");
1462 thread_info = (struct checkpoint_thread_info *)param;
1464 while ((fd = accept(thread_info->socket_fd, NULL, NULL)) != -1) {
1465 nvl = nvlist_recv(fd, 0);
1467 handle_message(thread_info->ctx, nvl);
1469 EPRINTLN("nvlist_recv() failed: %s", strerror(errno));
1472 nvlist_destroy(nvl);
1479 vm_do_checkpoint(struct vmctx *ctx, const nvlist_t *nvl)
1483 if (!nvlist_exists_string(nvl, "filename") ||
1484 !nvlist_exists_bool(nvl, "suspend"))
1487 error = vm_checkpoint(ctx, nvlist_get_string(nvl, "filename"),
1488 nvlist_get_bool(nvl, "suspend"));
1492 IPC_COMMAND(ipc_cmd_set, checkpoint, vm_do_checkpoint);
1499 err = pthread_mutex_init(&vcpu_lock, NULL);
1501 errc(1, err, "checkpoint mutex init");
1502 err = pthread_cond_init(&vcpus_idle, NULL);
1504 errc(1, err, "checkpoint cv init (vcpus_idle)");
1505 err = pthread_cond_init(&vcpus_can_run, NULL);
1507 errc(1, err, "checkpoint cv init (vcpus_can_run)");
1511 * Create the listening socket for IPC with bhyvectl
1514 init_checkpoint_thread(struct vmctx *ctx)
1516 struct checkpoint_thread_info *checkpoint_info = NULL;
1517 struct sockaddr_un addr;
1519 pthread_t checkpoint_pthread;
1522 memset(&addr, 0, sizeof(addr));
1524 socket_fd = socket(PF_UNIX, SOCK_STREAM, 0);
1525 if (socket_fd < 0) {
1526 EPRINTLN("Socket creation failed: %s", strerror(errno));
1531 addr.sun_family = AF_UNIX;
1533 snprintf(addr.sun_path, sizeof(addr.sun_path), "%s%s",
1534 BHYVE_RUN_DIR, vm_get_name(ctx));
1535 addr.sun_len = SUN_LEN(&addr);
1536 unlink(addr.sun_path);
1538 if (bind(socket_fd, (struct sockaddr *)&addr, addr.sun_len) != 0) {
1539 EPRINTLN("Failed to bind socket \"%s\": %s\n",
1540 addr.sun_path, strerror(errno));
1545 if (listen(socket_fd, 10) < 0) {
1546 EPRINTLN("ipc socket listen: %s\n", strerror(errno));
1551 checkpoint_info = calloc(1, sizeof(*checkpoint_info));
1552 checkpoint_info->ctx = ctx;
1553 checkpoint_info->socket_fd = socket_fd;
1555 err = pthread_create(&checkpoint_pthread, NULL, checkpoint_thread,
1562 free(checkpoint_info);
1565 unlink(addr.sun_path);
1571 vm_snapshot_buf_err(const char *bufname, const enum vm_snapshot_op op)
1575 if (op == VM_SNAPSHOT_SAVE)
1577 else if (op == VM_SNAPSHOT_RESTORE)
1582 fprintf(stderr, "%s: snapshot-%s failed for %s\r\n",
1583 __func__, __op, bufname);
1587 vm_snapshot_buf(void *data, size_t data_size, struct vm_snapshot_meta *meta)
1589 struct vm_snapshot_buffer *buffer;
1592 buffer = &meta->buffer;
1595 if (buffer->buf_rem < data_size) {
1596 fprintf(stderr, "%s: buffer too small\r\n", __func__);
1600 if (op == VM_SNAPSHOT_SAVE)
1601 memcpy(buffer->buf, data, data_size);
1602 else if (op == VM_SNAPSHOT_RESTORE)
1603 memcpy(data, buffer->buf, data_size);
1607 buffer->buf += data_size;
1608 buffer->buf_rem -= data_size;
1614 vm_get_snapshot_size(struct vm_snapshot_meta *meta)
1617 struct vm_snapshot_buffer *buffer;
1619 buffer = &meta->buffer;
1621 if (buffer->buf_size < buffer->buf_rem) {
1622 fprintf(stderr, "%s: Invalid buffer: size = %zu, rem = %zu\r\n",
1623 __func__, buffer->buf_size, buffer->buf_rem);
1626 length = buffer->buf_size - buffer->buf_rem;
1633 vm_snapshot_guest2host_addr(void **addrp, size_t len, bool restore_null,
1634 struct vm_snapshot_meta *meta)
1639 if (meta->op == VM_SNAPSHOT_SAVE) {
1640 gaddr = paddr_host2guest(meta->ctx, *addrp);
1641 if (gaddr == (vm_paddr_t) -1) {
1642 if (!restore_null ||
1643 (restore_null && (*addrp != NULL))) {
1649 SNAPSHOT_VAR_OR_LEAVE(gaddr, meta, ret, done);
1650 } else if (meta->op == VM_SNAPSHOT_RESTORE) {
1651 SNAPSHOT_VAR_OR_LEAVE(gaddr, meta, ret, done);
1652 if (gaddr == (vm_paddr_t) -1) {
1653 if (!restore_null) {
1659 *addrp = paddr_guest2host(meta->ctx, gaddr, len);
1669 vm_snapshot_buf_cmp(void *data, size_t data_size, struct vm_snapshot_meta *meta)
1671 struct vm_snapshot_buffer *buffer;
1675 buffer = &meta->buffer;
1678 if (buffer->buf_rem < data_size) {
1679 fprintf(stderr, "%s: buffer too small\r\n", __func__);
1684 if (op == VM_SNAPSHOT_SAVE) {
1686 memcpy(buffer->buf, data, data_size);
1687 } else if (op == VM_SNAPSHOT_RESTORE) {
1688 ret = memcmp(data, buffer->buf, data_size);
1694 buffer->buf += data_size;
1695 buffer->buf_rem -= data_size;