2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2017-2018 John H. Baldwin <jhb@FreeBSD.org>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
32 #include <sys/param.h>
33 #ifndef WITHOUT_CAPSICUM
34 #include <sys/capsicum.h>
36 #include <sys/ioctl.h>
38 #include <sys/socket.h>
39 #include <machine/atomic.h>
40 #include <machine/specialreg.h>
41 #include <machine/vmm.h>
42 #include <netinet/in.h>
44 #ifndef WITHOUT_CAPSICUM
45 #include <capsicum_helpers.h>
51 #include <pthread_np.h>
65 * GDB_SIGNAL_* numbers are part of the GDB remote protocol. Most stops
68 #define GDB_SIGNAL_TRAP 5
70 static void gdb_resume_vcpus(void);
71 static void check_command(int fd);
73 static struct mevent *read_event, *write_event;
75 static cpuset_t vcpus_active, vcpus_suspended, vcpus_waiting;
76 static pthread_mutex_t gdb_lock;
77 static pthread_cond_t idle_vcpus;
78 static bool stop_pending, first_stop;
79 static int stepping_vcpu, stopped_vcpu;
82 * An I/O buffer contains 'capacity' bytes of room at 'data'. For a
83 * read buffer, 'start' is unused and 'len' contains the number of
84 * valid bytes in the buffer. For a write buffer, 'start' is set to
85 * the index of the next byte in 'data' to send, and 'len' contains
86 * the remaining number of valid bytes to send.
95 static struct io_buffer cur_comm, cur_resp;
96 static uint8_t cur_csum;
98 static struct vmctx *ctx;
99 static int cur_fd = -1;
101 const int gdb_regset[] = {
128 const int gdb_regsize[] = {
159 static void __printflike(1, 2)
160 debug(const char *fmt, ...)
162 static FILE *logfile;
165 if (logfile == NULL) {
166 logfile = fopen("/tmp/bhyve_gdb.log", "w");
169 #ifndef WITHOUT_CAPSICUM
170 if (caph_limit_stream(fileno(logfile), CAPH_WRITE) == -1) {
179 vfprintf(logfile, fmt, ap);
187 guest_paging_info(int vcpu, struct vm_guest_paging *paging)
190 const int regset[4] = {
197 if (vm_get_register_set(ctx, vcpu, nitems(regset), regset, regs) == -1)
201 * For the debugger, always pretend to be the kernel (CPL 0),
202 * and if long-mode is enabled, always parse addresses as if
205 paging->cr3 = regs[1];
207 if (regs[3] & EFER_LMA)
208 paging->cpu_mode = CPU_MODE_64BIT;
209 else if (regs[0] & CR0_PE)
210 paging->cpu_mode = CPU_MODE_PROTECTED;
212 paging->cpu_mode = CPU_MODE_REAL;
213 if (!(regs[0] & CR0_PG))
214 paging->paging_mode = PAGING_MODE_FLAT;
215 else if (!(regs[2] & CR4_PAE))
216 paging->paging_mode = PAGING_MODE_32;
217 else if (regs[3] & EFER_LME)
218 paging->paging_mode = PAGING_MODE_64;
220 paging->paging_mode = PAGING_MODE_PAE;
225 * Map a guest virtual address to a physical address (for a given vcpu).
226 * If a guest virtual address is valid, return 1. If the address is
227 * not valid, return 0. If an error occurs obtaining the mapping,
231 guest_vaddr2paddr(int vcpu, uint64_t vaddr, uint64_t *paddr)
233 struct vm_guest_paging paging;
236 if (guest_paging_info(vcpu, &paging) == -1)
240 * Always use PROT_READ. We really care if the VA is
241 * accessible, not if the current vCPU can write.
243 if (vm_gla2gpa_nofault(ctx, vcpu, &paging, vaddr, PROT_READ, paddr,
252 io_buffer_reset(struct io_buffer *io)
259 /* Available room for adding data. */
261 io_buffer_avail(struct io_buffer *io)
264 return (io->capacity - (io->start + io->len));
268 io_buffer_head(struct io_buffer *io)
271 return (io->data + io->start);
275 io_buffer_tail(struct io_buffer *io)
278 return (io->data + io->start + io->len);
282 io_buffer_advance(struct io_buffer *io, size_t amount)
285 assert(amount <= io->len);
291 io_buffer_consume(struct io_buffer *io, size_t amount)
294 io_buffer_advance(io, amount);
301 * XXX: Consider making this move optional and compacting on a
302 * future read() before realloc().
304 memmove(io->data, io_buffer_head(io), io->len);
309 io_buffer_grow(struct io_buffer *io, size_t newsize)
312 size_t avail, new_cap;
314 avail = io_buffer_avail(io);
315 if (newsize <= avail)
318 new_cap = io->capacity + (newsize - avail);
319 new_data = realloc(io->data, new_cap);
320 if (new_data == NULL)
321 err(1, "Failed to grow GDB I/O buffer");
323 io->capacity = new_cap;
327 response_pending(void)
330 if (cur_resp.start == 0 && cur_resp.len == 0)
332 if (cur_resp.start + cur_resp.len == 1 && cur_resp.data[0] == '+')
338 close_connection(void)
342 * XXX: This triggers a warning because mevent does the close
343 * before the EV_DELETE.
345 pthread_mutex_lock(&gdb_lock);
346 mevent_delete(write_event);
347 mevent_delete_close(read_event);
350 io_buffer_reset(&cur_comm);
351 io_buffer_reset(&cur_resp);
354 /* Resume any stopped vCPUs. */
356 pthread_mutex_unlock(&gdb_lock);
360 hex_digit(uint8_t nibble)
364 return (nibble + '0');
366 return (nibble + 'a' - 10);
370 parse_digit(uint8_t v)
373 if (v >= '0' && v <= '9')
375 if (v >= 'a' && v <= 'f')
376 return (v - 'a' + 10);
377 if (v >= 'A' && v <= 'F')
378 return (v - 'A' + 10);
382 /* Parses big-endian hexadecimal. */
384 parse_integer(const uint8_t *p, size_t len)
391 v |= parse_digit(*p);
399 parse_byte(const uint8_t *p)
402 return (parse_digit(p[0]) << 4 | parse_digit(p[1]));
406 send_pending_data(int fd)
410 if (cur_resp.len == 0) {
411 mevent_disable(write_event);
414 nwritten = write(fd, io_buffer_head(&cur_resp), cur_resp.len);
415 if (nwritten == -1) {
416 warn("Write to GDB socket failed");
419 io_buffer_advance(&cur_resp, nwritten);
420 if (cur_resp.len == 0)
421 mevent_disable(write_event);
423 mevent_enable(write_event);
427 /* Append a single character to the output buffer. */
429 send_char(uint8_t data)
431 io_buffer_grow(&cur_resp, 1);
432 *io_buffer_tail(&cur_resp) = data;
436 /* Append an array of bytes to the output buffer. */
438 send_data(const uint8_t *data, size_t len)
441 io_buffer_grow(&cur_resp, len);
442 memcpy(io_buffer_tail(&cur_resp), data, len);
447 format_byte(uint8_t v, uint8_t *buf)
450 buf[0] = hex_digit(v >> 4);
451 buf[1] = hex_digit(v & 0xf);
455 * Append a single byte (formatted as two hex characters) to the
464 send_data(buf, sizeof(buf));
481 debug("-> %.*s\n", (int)cur_resp.len, io_buffer_head(&cur_resp));
485 * Append a single character (for the packet payload) and update the
489 append_char(uint8_t v)
497 * Append an array of bytes (for the packet payload) and update the
501 append_packet_data(const uint8_t *data, size_t len)
504 send_data(data, len);
513 append_string(const char *str)
516 append_packet_data(str, strlen(str));
520 append_byte(uint8_t v)
525 append_packet_data(buf, sizeof(buf));
529 append_unsigned_native(uintmax_t value, size_t len)
533 for (i = 0; i < len; i++) {
540 append_unsigned_be(uintmax_t value, size_t len)
545 for (i = 0; i < len; i++) {
546 format_byte(value, buf + (len - i - 1) * 2);
549 append_packet_data(buf, sizeof(buf));
553 append_integer(unsigned int value)
559 append_unsigned_be(value, fls(value) + 7 / 8);
563 append_asciihex(const char *str)
566 while (*str != '\0') {
573 send_empty_response(void)
581 send_error(int error)
600 parse_threadid(const uint8_t *data, size_t len)
603 if (len == 1 && *data == '0')
605 if (len == 2 && memcmp(data, "-1", 2) == 0)
609 return (parse_integer(data, len));
617 if (stopped_vcpu == -1)
621 append_byte(GDB_SIGNAL_TRAP);
622 if (stopped_vcpu != -1) {
623 append_string("thread:");
624 append_integer(stopped_vcpu + 1);
632 gdb_finish_suspend_vcpus(void)
638 } else if (response_pending())
642 send_pending_data(cur_fd);
647 _gdb_cpu_suspend(int vcpu, bool report_stop)
650 debug("$vCPU %d suspending\n", vcpu);
651 CPU_SET(vcpu, &vcpus_waiting);
652 if (report_stop && CPU_CMP(&vcpus_waiting, &vcpus_suspended) == 0)
653 gdb_finish_suspend_vcpus();
654 while (CPU_ISSET(vcpu, &vcpus_suspended) && vcpu != stepping_vcpu)
655 pthread_cond_wait(&idle_vcpus, &gdb_lock);
656 CPU_CLR(vcpu, &vcpus_waiting);
657 debug("$vCPU %d resuming\n", vcpu);
661 gdb_cpu_add(int vcpu)
664 debug("$vCPU %d starting\n", vcpu);
665 pthread_mutex_lock(&gdb_lock);
666 CPU_SET(vcpu, &vcpus_active);
669 * If a vcpu is added while vcpus are stopped, suspend the new
670 * vcpu so that it will pop back out with a debug exit before
671 * executing the first instruction.
673 if (!CPU_EMPTY(&vcpus_suspended)) {
674 CPU_SET(vcpu, &vcpus_suspended);
675 _gdb_cpu_suspend(vcpu, false);
677 pthread_mutex_unlock(&gdb_lock);
681 gdb_cpu_suspend(int vcpu)
684 pthread_mutex_lock(&gdb_lock);
685 _gdb_cpu_suspend(vcpu, true);
686 pthread_mutex_unlock(&gdb_lock);
690 gdb_cpu_mtrap(int vcpu)
693 debug("$vCPU %d MTRAP\n", vcpu);
694 pthread_mutex_lock(&gdb_lock);
695 if (vcpu == stepping_vcpu) {
697 vm_set_capability(ctx, vcpu, VM_CAP_MTRAP_EXIT, 0);
698 vm_suspend_cpu(ctx, vcpu);
699 assert(stopped_vcpu == -1);
701 _gdb_cpu_suspend(vcpu, true);
703 pthread_mutex_unlock(&gdb_lock);
707 gdb_suspend_vcpus(void)
710 assert(pthread_mutex_isowned_np(&gdb_lock));
711 debug("suspending all CPUs\n");
712 vcpus_suspended = vcpus_active;
713 vm_suspend_cpu(ctx, -1);
714 if (CPU_CMP(&vcpus_waiting, &vcpus_suspended) == 0)
715 gdb_finish_suspend_vcpus();
719 gdb_step_vcpu(int vcpu)
723 debug("$vCPU %d step\n", vcpu);
724 error = vm_get_capability(ctx, vcpu, VM_CAP_MTRAP_EXIT, &val);
727 error = vm_set_capability(ctx, vcpu, VM_CAP_MTRAP_EXIT, 1);
728 vm_resume_cpu(ctx, vcpu);
729 stepping_vcpu = vcpu;
730 pthread_cond_broadcast(&idle_vcpus);
735 gdb_resume_vcpus(void)
738 assert(pthread_mutex_isowned_np(&gdb_lock));
739 vm_resume_cpu(ctx, -1);
740 debug("resuming all CPUs\n");
741 CPU_ZERO(&vcpus_suspended);
742 pthread_cond_broadcast(&idle_vcpus);
748 uint64_t regvals[nitems(gdb_regset)];
751 if (vm_get_register_set(ctx, cur_vcpu, nitems(gdb_regset),
752 gdb_regset, regvals) == -1) {
757 for (i = 0; i < nitems(regvals); i++)
758 append_unsigned_native(regvals[i], gdb_regsize[i]);
763 gdb_read_mem(const uint8_t *data, size_t len)
765 uint64_t gpa, gva, val;
767 size_t resid, todo, bytes;
771 cp = memchr(data, ',', len);
776 gva = parse_integer(data + 1, cp - (data + 1));
777 resid = parse_integer(cp + 1, len - (cp + 1 - data));
781 error = guest_vaddr2paddr(cur_vcpu, gva, &gpa);
797 /* Read bytes from current page. */
798 todo = getpagesize() - gpa % getpagesize();
802 cp = paddr_guest2host(ctx, gpa, todo);
805 * If this page is guest RAM, read it a byte
822 * If this page isn't guest RAM, try to handle
823 * it via MMIO. For MMIO requests, use
824 * aligned reads of words when possible.
827 if (gpa & 1 || todo == 1)
829 else if (gpa & 2 || todo == 2)
833 error = read_mem(ctx, cur_vcpu, gpa, &val,
858 assert(resid == 0 || gpa % getpagesize() == 0);
866 command_equals(const uint8_t *data, size_t len, const char *cmd)
869 if (strlen(cmd) > len)
871 return (memcmp(data, cmd, strlen(cmd)) == 0);
875 gdb_query(const uint8_t *data, size_t len)
883 if (command_equals(data, len, "qAttached")) {
887 } else if (command_equals(data, len, "qC")) {
890 append_integer(cur_vcpu + 1);
892 } else if (command_equals(data, len, "qfThreadInfo")) {
897 if (CPU_EMPTY(&vcpus_active)) {
905 while (!CPU_EMPTY(&mask)) {
906 vcpu = CPU_FFS(&mask) - 1;
907 CPU_CLR(vcpu, &mask);
912 append_integer(vcpu + 1);
915 } else if (command_equals(data, len, "qsThreadInfo")) {
919 } else if (command_equals(data, len, "qThreadExtraInfo")) {
923 data += strlen("qThreadExtraInfo");
924 len -= strlen("qThreadExtraInfo");
929 tid = parse_threadid(data + 1, len - 1);
930 if (tid <= 0 || !CPU_ISSET(tid - 1, &vcpus_active)) {
935 snprintf(buf, sizeof(buf), "vCPU %d", tid - 1);
937 append_asciihex(buf);
940 send_empty_response();
944 handle_command(const uint8_t *data, size_t len)
947 /* Reject packets with a sequence-id. */
948 if (len >= 3 && data[0] >= '0' && data[0] <= '9' &&
949 data[0] >= '0' && data[0] <= '9' && data[2] == ':') {
950 send_empty_response();
961 /* Don't send a reply until a stop occurs. */
967 /* TODO: Resume any stopped CPUs. */
976 if (data[1] != 'g' && data[1] != 'c') {
980 tid = parse_threadid(data + 2, len - 2);
986 if (CPU_EMPTY(&vcpus_active)) {
990 if (tid == -1 || tid == 0)
991 cur_vcpu = CPU_FFS(&vcpus_active) - 1;
992 else if (CPU_ISSET(tid - 1, &vcpus_active))
1002 gdb_read_mem(data, len);
1007 tid = parse_threadid(data + 1, len - 1);
1008 if (tid <= 0 || !CPU_ISSET(tid - 1, &vcpus_active)) {
1016 gdb_query(data, len);
1024 /* Don't send a reply until a stop occurs. */
1025 if (!gdb_step_vcpu(cur_vcpu)) {
1026 send_error(EOPNOTSUPP);
1031 /* XXX: Only if stopped? */
1032 /* For now, just report that we are always stopped. */
1035 append_byte(GDB_SIGNAL_TRAP);
1038 case 'G': /* TODO */
1039 case 'M': /* TODO */
1041 /* Handle 'vCont' */
1043 case 'p': /* TODO */
1044 case 'P': /* TODO */
1045 case 'Q': /* TODO */
1046 case 't': /* TODO */
1047 case 'X': /* TODO */
1048 case 'z': /* TODO */
1049 case 'Z': /* TODO */
1051 send_empty_response();
1055 /* Check for a valid packet in the command buffer. */
1057 check_command(int fd)
1059 uint8_t *head, *hash, *p, sum;
1063 avail = cur_comm.len;
1066 head = io_buffer_head(&cur_comm);
1069 debug("<- Ctrl-C\n");
1070 io_buffer_consume(&cur_comm, 1);
1072 gdb_suspend_vcpus();
1075 /* ACK of previous response. */
1077 if (response_pending())
1078 io_buffer_reset(&cur_resp);
1079 io_buffer_consume(&cur_comm, 1);
1081 stop_pending = false;
1083 send_pending_data(fd);
1087 /* NACK of previous response. */
1089 if (response_pending()) {
1090 cur_resp.len += cur_resp.start;
1092 if (cur_resp.data[0] == '+')
1093 io_buffer_advance(&cur_resp, 1);
1094 debug("-> %.*s\n", (int)cur_resp.len,
1095 io_buffer_head(&cur_resp));
1097 io_buffer_consume(&cur_comm, 1);
1098 send_pending_data(fd);
1103 if (response_pending()) {
1104 warnx("New GDB command while response in "
1106 io_buffer_reset(&cur_resp);
1109 /* Is packet complete? */
1110 hash = memchr(head, '#', avail);
1113 plen = (hash - head + 1) + 2;
1116 debug("<- %.*s\n", (int)plen, head);
1118 /* Verify checksum. */
1119 for (sum = 0, p = head + 1; p < hash; p++)
1121 if (sum != parse_byte(hash + 1)) {
1122 io_buffer_consume(&cur_comm, plen);
1125 send_pending_data(fd);
1130 handle_command(head + 1, hash - (head + 1));
1131 io_buffer_consume(&cur_comm, plen);
1132 if (!response_pending())
1134 send_pending_data(fd);
1137 /* XXX: Possibly drop connection instead. */
1138 debug("-> %02x\n", *head);
1139 io_buffer_consume(&cur_comm, 1);
1146 gdb_readable(int fd, enum ev_type event, void *arg)
1151 if (ioctl(fd, FIONREAD, &pending) == -1) {
1152 warn("FIONREAD on GDB socket");
1157 * 'pending' might be zero due to EOF. We need to call read
1158 * with a non-zero length to detect EOF.
1163 /* Ensure there is room in the command buffer. */
1164 io_buffer_grow(&cur_comm, pending);
1165 assert(io_buffer_avail(&cur_comm) >= pending);
1167 nread = read(fd, io_buffer_tail(&cur_comm), io_buffer_avail(&cur_comm));
1170 } else if (nread == -1) {
1171 if (errno == EAGAIN)
1174 warn("Read from GDB socket");
1177 cur_comm.len += nread;
1178 pthread_mutex_lock(&gdb_lock);
1180 pthread_mutex_unlock(&gdb_lock);
1185 gdb_writable(int fd, enum ev_type event, void *arg)
1188 send_pending_data(fd);
1192 new_connection(int fd, enum ev_type event, void *arg)
1196 s = accept4(fd, NULL, NULL, SOCK_NONBLOCK);
1199 err(1, "Failed accepting initial GDB connection");
1201 /* Silently ignore errors post-startup. */
1206 if (setsockopt(s, SOL_SOCKET, SO_NOSIGPIPE, &optval, sizeof(optval)) ==
1208 warn("Failed to disable SIGPIPE for GDB connection");
1213 pthread_mutex_lock(&gdb_lock);
1216 warnx("Ignoring additional GDB connection.");
1219 read_event = mevent_add(s, EVF_READ, gdb_readable, NULL);
1220 if (read_event == NULL) {
1222 err(1, "Failed to setup initial GDB connection");
1223 pthread_mutex_unlock(&gdb_lock);
1226 write_event = mevent_add(s, EVF_WRITE, gdb_writable, NULL);
1227 if (write_event == NULL) {
1229 err(1, "Failed to setup initial GDB connection");
1230 mevent_delete_close(read_event);
1238 stop_pending = false;
1240 /* Break on attach. */
1242 gdb_suspend_vcpus();
1243 pthread_mutex_unlock(&gdb_lock);
1246 #ifndef WITHOUT_CAPSICUM
1248 limit_gdb_socket(int s)
1250 cap_rights_t rights;
1251 unsigned long ioctls[] = { FIONREAD };
1253 cap_rights_init(&rights, CAP_ACCEPT, CAP_EVENT, CAP_READ, CAP_WRITE,
1254 CAP_SETSOCKOPT, CAP_IOCTL);
1255 if (cap_rights_limit(s, &rights) == -1 && errno != ENOSYS)
1256 errx(EX_OSERR, "Unable to apply rights for sandbox");
1257 if (cap_ioctls_limit(s, ioctls, nitems(ioctls)) == -1 && errno != ENOSYS)
1258 errx(EX_OSERR, "Unable to apply rights for sandbox");
1263 init_gdb(struct vmctx *_ctx, int sport, bool wait)
1265 struct sockaddr_in sin;
1266 int error, flags, s;
1268 debug("==> starting on %d, %swaiting\n", sport, wait ? "" : "not ");
1270 error = pthread_mutex_init(&gdb_lock, NULL);
1272 errc(1, error, "gdb mutex init");
1273 error = pthread_cond_init(&idle_vcpus, NULL);
1275 errc(1, error, "gdb cv init");
1278 s = socket(PF_INET, SOCK_STREAM, 0);
1280 err(1, "gdb socket create");
1282 sin.sin_len = sizeof(sin);
1283 sin.sin_family = AF_INET;
1284 sin.sin_addr.s_addr = htonl(INADDR_ANY);
1285 sin.sin_port = htons(sport);
1287 if (bind(s, (struct sockaddr *)&sin, sizeof(sin)) < 0)
1288 err(1, "gdb socket bind");
1290 if (listen(s, 1) < 0)
1291 err(1, "gdb socket listen");
1295 * Set vcpu 0 in vcpus_suspended. This will trigger the
1296 * logic in gdb_cpu_add() to suspend the first vcpu before
1297 * it starts execution. The vcpu will remain suspended
1298 * until a debugger connects.
1302 CPU_SET(0, &vcpus_suspended);
1305 flags = fcntl(s, F_GETFL);
1306 if (fcntl(s, F_SETFL, flags | O_NONBLOCK) == -1)
1307 err(1, "Failed to mark gdb socket non-blocking");
1309 #ifndef WITHOUT_CAPSICUM
1310 limit_gdb_socket(s);
1312 mevent_add(s, EVF_READ, new_connection, NULL);