2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2017-2018 John H. Baldwin <jhb@FreeBSD.org>
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
31 #include <sys/param.h>
32 #ifndef WITHOUT_CAPSICUM
33 #include <sys/capsicum.h>
35 #include <sys/ioctl.h>
37 #include <sys/socket.h>
38 #include <machine/atomic.h>
39 #include <machine/specialreg.h>
40 #include <machine/vmm.h>
41 #include <netinet/in.h>
43 #ifndef WITHOUT_CAPSICUM
44 #include <capsicum_helpers.h>
50 #include <pthread_np.h>
64 * GDB_SIGNAL_* numbers are part of the GDB remote protocol. Most stops
67 #define GDB_SIGNAL_TRAP 5
69 static void gdb_resume_vcpus(void);
70 static void check_command(int fd);
72 static struct mevent *read_event, *write_event;
74 static cpuset_t vcpus_active, vcpus_suspended, vcpus_waiting;
75 static pthread_mutex_t gdb_lock;
76 static pthread_cond_t idle_vcpus;
77 static bool stop_pending, first_stop;
78 static int stepping_vcpu, stopped_vcpu;
81 * An I/O buffer contains 'capacity' bytes of room at 'data'. For a
82 * read buffer, 'start' is unused and 'len' contains the number of
83 * valid bytes in the buffer. For a write buffer, 'start' is set to
84 * the index of the next byte in 'data' to send, and 'len' contains
85 * the remaining number of valid bytes to send.
94 static struct io_buffer cur_comm, cur_resp;
95 static uint8_t cur_csum;
97 static struct vmctx *ctx;
98 static int cur_fd = -1;
100 const int gdb_regset[] = {
127 const int gdb_regsize[] = {
158 static void __printflike(1, 2)
159 debug(const char *fmt, ...)
161 static FILE *logfile;
164 if (logfile == NULL) {
165 logfile = fopen("/tmp/bhyve_gdb.log", "w");
168 #ifndef WITHOUT_CAPSICUM
169 if (caph_limit_stream(fileno(logfile), CAPH_WRITE) == -1) {
178 vfprintf(logfile, fmt, ap);
186 guest_paging_info(int vcpu, struct vm_guest_paging *paging)
189 const int regset[4] = {
196 if (vm_get_register_set(ctx, vcpu, nitems(regset), regset, regs) == -1)
200 * For the debugger, always pretend to be the kernel (CPL 0),
201 * and if long-mode is enabled, always parse addresses as if
204 paging->cr3 = regs[1];
206 if (regs[3] & EFER_LMA)
207 paging->cpu_mode = CPU_MODE_64BIT;
208 else if (regs[0] & CR0_PE)
209 paging->cpu_mode = CPU_MODE_PROTECTED;
211 paging->cpu_mode = CPU_MODE_REAL;
212 if (!(regs[0] & CR0_PG))
213 paging->paging_mode = PAGING_MODE_FLAT;
214 else if (!(regs[2] & CR4_PAE))
215 paging->paging_mode = PAGING_MODE_32;
216 else if (regs[3] & EFER_LME)
217 paging->paging_mode = PAGING_MODE_64;
219 paging->paging_mode = PAGING_MODE_PAE;
224 * Map a guest virtual address to a physical address (for a given vcpu).
225 * If a guest virtual address is valid, return 1. If the address is
226 * not valid, return 0. If an error occurs obtaining the mapping,
230 guest_vaddr2paddr(int vcpu, uint64_t vaddr, uint64_t *paddr)
232 struct vm_guest_paging paging;
235 if (guest_paging_info(vcpu, &paging) == -1)
239 * Always use PROT_READ. We really care if the VA is
240 * accessible, not if the current vCPU can write.
242 if (vm_gla2gpa_nofault(ctx, vcpu, &paging, vaddr, PROT_READ, paddr,
251 io_buffer_reset(struct io_buffer *io)
258 /* Available room for adding data. */
260 io_buffer_avail(struct io_buffer *io)
263 return (io->capacity - (io->start + io->len));
267 io_buffer_head(struct io_buffer *io)
270 return (io->data + io->start);
274 io_buffer_tail(struct io_buffer *io)
277 return (io->data + io->start + io->len);
281 io_buffer_advance(struct io_buffer *io, size_t amount)
284 assert(amount <= io->len);
290 io_buffer_consume(struct io_buffer *io, size_t amount)
293 io_buffer_advance(io, amount);
300 * XXX: Consider making this move optional and compacting on a
301 * future read() before realloc().
303 memmove(io->data, io_buffer_head(io), io->len);
308 io_buffer_grow(struct io_buffer *io, size_t newsize)
311 size_t avail, new_cap;
313 avail = io_buffer_avail(io);
314 if (newsize <= avail)
317 new_cap = io->capacity + (newsize - avail);
318 new_data = realloc(io->data, new_cap);
319 if (new_data == NULL)
320 err(1, "Failed to grow GDB I/O buffer");
322 io->capacity = new_cap;
326 response_pending(void)
329 if (cur_resp.start == 0 && cur_resp.len == 0)
331 if (cur_resp.start + cur_resp.len == 1 && cur_resp.data[0] == '+')
337 close_connection(void)
341 * XXX: This triggers a warning because mevent does the close
342 * before the EV_DELETE.
344 pthread_mutex_lock(&gdb_lock);
345 mevent_delete(write_event);
346 mevent_delete_close(read_event);
349 io_buffer_reset(&cur_comm);
350 io_buffer_reset(&cur_resp);
353 /* Resume any stopped vCPUs. */
355 pthread_mutex_unlock(&gdb_lock);
359 hex_digit(uint8_t nibble)
363 return (nibble + '0');
365 return (nibble + 'a' - 10);
369 parse_digit(uint8_t v)
372 if (v >= '0' && v <= '9')
374 if (v >= 'a' && v <= 'f')
375 return (v - 'a' + 10);
376 if (v >= 'A' && v <= 'F')
377 return (v - 'A' + 10);
381 /* Parses big-endian hexadecimal. */
383 parse_integer(const uint8_t *p, size_t len)
390 v |= parse_digit(*p);
398 parse_byte(const uint8_t *p)
401 return (parse_digit(p[0]) << 4 | parse_digit(p[1]));
405 send_pending_data(int fd)
409 if (cur_resp.len == 0) {
410 mevent_disable(write_event);
413 nwritten = write(fd, io_buffer_head(&cur_resp), cur_resp.len);
414 if (nwritten == -1) {
415 warn("Write to GDB socket failed");
418 io_buffer_advance(&cur_resp, nwritten);
419 if (cur_resp.len == 0)
420 mevent_disable(write_event);
422 mevent_enable(write_event);
426 /* Append a single character to the output buffer. */
428 send_char(uint8_t data)
430 io_buffer_grow(&cur_resp, 1);
431 *io_buffer_tail(&cur_resp) = data;
435 /* Append an array of bytes to the output buffer. */
437 send_data(const uint8_t *data, size_t len)
440 io_buffer_grow(&cur_resp, len);
441 memcpy(io_buffer_tail(&cur_resp), data, len);
446 format_byte(uint8_t v, uint8_t *buf)
449 buf[0] = hex_digit(v >> 4);
450 buf[1] = hex_digit(v & 0xf);
454 * Append a single byte (formatted as two hex characters) to the
463 send_data(buf, sizeof(buf));
480 debug("-> %.*s\n", (int)cur_resp.len, io_buffer_head(&cur_resp));
484 * Append a single character (for the packet payload) and update the
488 append_char(uint8_t v)
496 * Append an array of bytes (for the packet payload) and update the
500 append_packet_data(const uint8_t *data, size_t len)
503 send_data(data, len);
512 append_string(const char *str)
515 append_packet_data(str, strlen(str));
519 append_byte(uint8_t v)
524 append_packet_data(buf, sizeof(buf));
528 append_unsigned_native(uintmax_t value, size_t len)
532 for (i = 0; i < len; i++) {
539 append_unsigned_be(uintmax_t value, size_t len)
544 for (i = 0; i < len; i++) {
545 format_byte(value, buf + (len - i - 1) * 2);
548 append_packet_data(buf, sizeof(buf));
552 append_integer(unsigned int value)
558 append_unsigned_be(value, fls(value) + 7 / 8);
562 append_asciihex(const char *str)
565 while (*str != '\0') {
572 send_empty_response(void)
580 send_error(int error)
599 parse_threadid(const uint8_t *data, size_t len)
602 if (len == 1 && *data == '0')
604 if (len == 2 && memcmp(data, "-1", 2) == 0)
608 return (parse_integer(data, len));
616 if (stopped_vcpu == -1)
620 append_byte(GDB_SIGNAL_TRAP);
621 if (stopped_vcpu != -1) {
622 append_string("thread:");
623 append_integer(stopped_vcpu + 1);
631 gdb_finish_suspend_vcpus(void)
637 } else if (response_pending())
641 send_pending_data(cur_fd);
646 _gdb_cpu_suspend(int vcpu, bool report_stop)
649 debug("$vCPU %d suspending\n", vcpu);
650 CPU_SET(vcpu, &vcpus_waiting);
651 if (report_stop && CPU_CMP(&vcpus_waiting, &vcpus_suspended) == 0)
652 gdb_finish_suspend_vcpus();
653 while (CPU_ISSET(vcpu, &vcpus_suspended) && vcpu != stepping_vcpu)
654 pthread_cond_wait(&idle_vcpus, &gdb_lock);
655 CPU_CLR(vcpu, &vcpus_waiting);
656 debug("$vCPU %d resuming\n", vcpu);
660 gdb_cpu_add(int vcpu)
663 debug("$vCPU %d starting\n", vcpu);
664 pthread_mutex_lock(&gdb_lock);
665 CPU_SET(vcpu, &vcpus_active);
668 * If a vcpu is added while vcpus are stopped, suspend the new
669 * vcpu so that it will pop back out with a debug exit before
670 * executing the first instruction.
672 if (!CPU_EMPTY(&vcpus_suspended)) {
673 CPU_SET(vcpu, &vcpus_suspended);
674 _gdb_cpu_suspend(vcpu, false);
676 pthread_mutex_unlock(&gdb_lock);
680 gdb_cpu_suspend(int vcpu)
683 pthread_mutex_lock(&gdb_lock);
684 _gdb_cpu_suspend(vcpu, true);
685 pthread_mutex_unlock(&gdb_lock);
689 gdb_cpu_mtrap(int vcpu)
692 debug("$vCPU %d MTRAP\n", vcpu);
693 pthread_mutex_lock(&gdb_lock);
694 if (vcpu == stepping_vcpu) {
696 vm_set_capability(ctx, vcpu, VM_CAP_MTRAP_EXIT, 0);
697 vm_suspend_cpu(ctx, vcpu);
698 assert(stopped_vcpu == -1);
700 _gdb_cpu_suspend(vcpu, true);
702 pthread_mutex_unlock(&gdb_lock);
706 gdb_suspend_vcpus(void)
709 assert(pthread_mutex_isowned_np(&gdb_lock));
710 debug("suspending all CPUs\n");
711 vcpus_suspended = vcpus_active;
712 vm_suspend_cpu(ctx, -1);
713 if (CPU_CMP(&vcpus_waiting, &vcpus_suspended) == 0)
714 gdb_finish_suspend_vcpus();
718 gdb_step_vcpu(int vcpu)
722 debug("$vCPU %d step\n", vcpu);
723 error = vm_get_capability(ctx, vcpu, VM_CAP_MTRAP_EXIT, &val);
726 error = vm_set_capability(ctx, vcpu, VM_CAP_MTRAP_EXIT, 1);
727 vm_resume_cpu(ctx, vcpu);
728 stepping_vcpu = vcpu;
729 pthread_cond_broadcast(&idle_vcpus);
734 gdb_resume_vcpus(void)
737 assert(pthread_mutex_isowned_np(&gdb_lock));
738 vm_resume_cpu(ctx, -1);
739 debug("resuming all CPUs\n");
740 CPU_ZERO(&vcpus_suspended);
741 pthread_cond_broadcast(&idle_vcpus);
747 uint64_t regvals[nitems(gdb_regset)];
750 if (vm_get_register_set(ctx, cur_vcpu, nitems(gdb_regset),
751 gdb_regset, regvals) == -1) {
756 for (i = 0; i < nitems(regvals); i++)
757 append_unsigned_native(regvals[i], gdb_regsize[i]);
762 gdb_read_mem(const uint8_t *data, size_t len)
764 uint64_t gpa, gva, val;
766 size_t resid, todo, bytes;
770 cp = memchr(data, ',', len);
775 gva = parse_integer(data + 1, cp - (data + 1));
776 resid = parse_integer(cp + 1, len - (cp + 1 - data));
780 error = guest_vaddr2paddr(cur_vcpu, gva, &gpa);
796 /* Read bytes from current page. */
797 todo = getpagesize() - gpa % getpagesize();
801 cp = paddr_guest2host(ctx, gpa, todo);
804 * If this page is guest RAM, read it a byte
821 * If this page isn't guest RAM, try to handle
822 * it via MMIO. For MMIO requests, use
823 * aligned reads of words when possible.
826 if (gpa & 1 || todo == 1)
828 else if (gpa & 2 || todo == 2)
832 error = read_mem(ctx, cur_vcpu, gpa, &val,
857 assert(resid == 0 || gpa % getpagesize() == 0);
865 command_equals(const uint8_t *data, size_t len, const char *cmd)
868 if (strlen(cmd) > len)
870 return (memcmp(data, cmd, strlen(cmd)) == 0);
874 gdb_query(const uint8_t *data, size_t len)
882 if (command_equals(data, len, "qAttached")) {
886 } else if (command_equals(data, len, "qC")) {
889 append_integer(cur_vcpu + 1);
891 } else if (command_equals(data, len, "qfThreadInfo")) {
896 if (CPU_EMPTY(&vcpus_active)) {
904 while (!CPU_EMPTY(&mask)) {
905 vcpu = CPU_FFS(&mask) - 1;
906 CPU_CLR(vcpu, &mask);
911 append_integer(vcpu + 1);
914 } else if (command_equals(data, len, "qsThreadInfo")) {
918 } else if (command_equals(data, len, "qThreadExtraInfo")) {
922 data += strlen("qThreadExtraInfo");
923 len -= strlen("qThreadExtraInfo");
928 tid = parse_threadid(data + 1, len - 1);
929 if (tid <= 0 || !CPU_ISSET(tid - 1, &vcpus_active)) {
934 snprintf(buf, sizeof(buf), "vCPU %d", tid - 1);
936 append_asciihex(buf);
939 send_empty_response();
943 handle_command(const uint8_t *data, size_t len)
946 /* Reject packets with a sequence-id. */
947 if (len >= 3 && data[0] >= '0' && data[0] <= '9' &&
948 data[0] >= '0' && data[0] <= '9' && data[2] == ':') {
949 send_empty_response();
960 /* Don't send a reply until a stop occurs. */
966 /* TODO: Resume any stopped CPUs. */
975 if (data[1] != 'g' && data[1] != 'c') {
979 tid = parse_threadid(data + 2, len - 2);
985 if (CPU_EMPTY(&vcpus_active)) {
989 if (tid == -1 || tid == 0)
990 cur_vcpu = CPU_FFS(&vcpus_active) - 1;
991 else if (CPU_ISSET(tid - 1, &vcpus_active))
1001 gdb_read_mem(data, len);
1006 tid = parse_threadid(data + 1, len - 1);
1007 if (tid <= 0 || !CPU_ISSET(tid - 1, &vcpus_active)) {
1015 gdb_query(data, len);
1023 /* Don't send a reply until a stop occurs. */
1024 if (!gdb_step_vcpu(cur_vcpu)) {
1025 send_error(EOPNOTSUPP);
1030 /* XXX: Only if stopped? */
1031 /* For now, just report that we are always stopped. */
1034 append_byte(GDB_SIGNAL_TRAP);
1037 case 'G': /* TODO */
1038 case 'M': /* TODO */
1040 /* Handle 'vCont' */
1042 case 'p': /* TODO */
1043 case 'P': /* TODO */
1044 case 'Q': /* TODO */
1045 case 't': /* TODO */
1046 case 'X': /* TODO */
1047 case 'z': /* TODO */
1048 case 'Z': /* TODO */
1050 send_empty_response();
1054 /* Check for a valid packet in the command buffer. */
1056 check_command(int fd)
1058 uint8_t *head, *hash, *p, sum;
1062 avail = cur_comm.len;
1065 head = io_buffer_head(&cur_comm);
1068 debug("<- Ctrl-C\n");
1069 io_buffer_consume(&cur_comm, 1);
1071 gdb_suspend_vcpus();
1074 /* ACK of previous response. */
1076 if (response_pending())
1077 io_buffer_reset(&cur_resp);
1078 io_buffer_consume(&cur_comm, 1);
1080 stop_pending = false;
1082 send_pending_data(fd);
1086 /* NACK of previous response. */
1088 if (response_pending()) {
1089 cur_resp.len += cur_resp.start;
1091 if (cur_resp.data[0] == '+')
1092 io_buffer_advance(&cur_resp, 1);
1093 debug("-> %.*s\n", (int)cur_resp.len,
1094 io_buffer_head(&cur_resp));
1096 io_buffer_consume(&cur_comm, 1);
1097 send_pending_data(fd);
1102 if (response_pending()) {
1103 warnx("New GDB command while response in "
1105 io_buffer_reset(&cur_resp);
1108 /* Is packet complete? */
1109 hash = memchr(head, '#', avail);
1112 plen = (hash - head + 1) + 2;
1115 debug("<- %.*s\n", (int)plen, head);
1117 /* Verify checksum. */
1118 for (sum = 0, p = head + 1; p < hash; p++)
1120 if (sum != parse_byte(hash + 1)) {
1121 io_buffer_consume(&cur_comm, plen);
1124 send_pending_data(fd);
1129 handle_command(head + 1, hash - (head + 1));
1130 io_buffer_consume(&cur_comm, plen);
1131 if (!response_pending())
1133 send_pending_data(fd);
1136 /* XXX: Possibly drop connection instead. */
1137 debug("-> %02x\n", *head);
1138 io_buffer_consume(&cur_comm, 1);
1145 gdb_readable(int fd, enum ev_type event, void *arg)
1150 if (ioctl(fd, FIONREAD, &pending) == -1) {
1151 warn("FIONREAD on GDB socket");
1156 * 'pending' might be zero due to EOF. We need to call read
1157 * with a non-zero length to detect EOF.
1162 /* Ensure there is room in the command buffer. */
1163 io_buffer_grow(&cur_comm, pending);
1164 assert(io_buffer_avail(&cur_comm) >= pending);
1166 nread = read(fd, io_buffer_tail(&cur_comm), io_buffer_avail(&cur_comm));
1169 } else if (nread == -1) {
1170 if (errno == EAGAIN)
1173 warn("Read from GDB socket");
1176 cur_comm.len += nread;
1177 pthread_mutex_lock(&gdb_lock);
1179 pthread_mutex_unlock(&gdb_lock);
1184 gdb_writable(int fd, enum ev_type event, void *arg)
1187 send_pending_data(fd);
1191 new_connection(int fd, enum ev_type event, void *arg)
1195 s = accept4(fd, NULL, NULL, SOCK_NONBLOCK);
1198 err(1, "Failed accepting initial GDB connection");
1200 /* Silently ignore errors post-startup. */
1205 if (setsockopt(s, SOL_SOCKET, SO_NOSIGPIPE, &optval, sizeof(optval)) ==
1207 warn("Failed to disable SIGPIPE for GDB connection");
1212 pthread_mutex_lock(&gdb_lock);
1215 warnx("Ignoring additional GDB connection.");
1218 read_event = mevent_add(s, EVF_READ, gdb_readable, NULL);
1219 if (read_event == NULL) {
1221 err(1, "Failed to setup initial GDB connection");
1222 pthread_mutex_unlock(&gdb_lock);
1225 write_event = mevent_add(s, EVF_WRITE, gdb_writable, NULL);
1226 if (write_event == NULL) {
1228 err(1, "Failed to setup initial GDB connection");
1229 mevent_delete_close(read_event);
1237 stop_pending = false;
1239 /* Break on attach. */
1241 gdb_suspend_vcpus();
1242 pthread_mutex_unlock(&gdb_lock);
1245 #ifndef WITHOUT_CAPSICUM
1247 limit_gdb_socket(int s)
1249 cap_rights_t rights;
1250 unsigned long ioctls[] = { FIONREAD };
1252 cap_rights_init(&rights, CAP_ACCEPT, CAP_EVENT, CAP_READ, CAP_WRITE,
1253 CAP_SETSOCKOPT, CAP_IOCTL);
1254 if (caph_rights_limit(s, &rights) == -1)
1255 errx(EX_OSERR, "Unable to apply rights for sandbox");
1256 if (caph_ioctls_limit(s, ioctls, nitems(ioctls)) == -1)
1257 errx(EX_OSERR, "Unable to apply rights for sandbox");
1262 init_gdb(struct vmctx *_ctx, int sport, bool wait)
1264 struct sockaddr_in sin;
1265 int error, flags, s;
1267 debug("==> starting on %d, %swaiting\n", sport, wait ? "" : "not ");
1269 error = pthread_mutex_init(&gdb_lock, NULL);
1271 errc(1, error, "gdb mutex init");
1272 error = pthread_cond_init(&idle_vcpus, NULL);
1274 errc(1, error, "gdb cv init");
1277 s = socket(PF_INET, SOCK_STREAM, 0);
1279 err(1, "gdb socket create");
1281 sin.sin_len = sizeof(sin);
1282 sin.sin_family = AF_INET;
1283 sin.sin_addr.s_addr = htonl(INADDR_ANY);
1284 sin.sin_port = htons(sport);
1286 if (bind(s, (struct sockaddr *)&sin, sizeof(sin)) < 0)
1287 err(1, "gdb socket bind");
1289 if (listen(s, 1) < 0)
1290 err(1, "gdb socket listen");
1294 * Set vcpu 0 in vcpus_suspended. This will trigger the
1295 * logic in gdb_cpu_add() to suspend the first vcpu before
1296 * it starts execution. The vcpu will remain suspended
1297 * until a debugger connects.
1301 CPU_SET(0, &vcpus_suspended);
1304 flags = fcntl(s, F_GETFL);
1305 if (fcntl(s, F_SETFL, flags | O_NONBLOCK) == -1)
1306 err(1, "Failed to mark gdb socket non-blocking");
1308 #ifndef WITHOUT_CAPSICUM
1309 limit_gdb_socket(s);
1311 mevent_add(s, EVF_READ, new_connection, NULL);