sys/arm64/arm64/debug_monitor.c

   1 /*-
   2  * Copyright (c) 2014 The FreeBSD Foundation
   3  * All rights reserved.
   4  *
   5  * This software was developed by Semihalf under
   6  * the sponsorship of the FreeBSD Foundation.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 #include "opt_ddb.h"
  31
  32 #include <sys/cdefs.h>
  33 __FBSDID("$FreeBSD$");
  34
  35 #include <sys/param.h>
  36 #include <sys/types.h>
  37 #include <sys/kdb.h>
  38 #include <sys/pcpu.h>
  39 #include <sys/proc.h>
  40 #include <sys/systm.h>
  41
  42 #include <machine/armreg.h>
  43 #include <machine/cpu.h>
  44 #include <machine/debug_monitor.h>
  45 #include <machine/kdb.h>
  46
  47 #ifdef DDB
  48 #include <ddb/ddb.h>
  49 #include <ddb/db_sym.h>
  50 #endif
  51
  52 enum dbg_t {
  53         DBG_TYPE_BREAKPOINT = 0,
  54         DBG_TYPE_WATCHPOINT = 1,
  55 };
  56
  57 static int dbg_watchpoint_num;
  58 static int dbg_breakpoint_num;
  59 static struct debug_monitor_state kernel_monitor = {
  60         .dbg_flags = DBGMON_KERNEL
  61 };
  62
  63 /* Called from the exception handlers */
  64 void dbg_monitor_enter(struct thread *);
  65 void dbg_monitor_exit(struct thread *, struct trapframe *);
  66
  67 /* Watchpoints/breakpoints control register bitfields */
  68 #define DBG_WATCH_CTRL_LEN_1            (0x1 << 5)
  69 #define DBG_WATCH_CTRL_LEN_2            (0x3 << 5)
  70 #define DBG_WATCH_CTRL_LEN_4            (0xf << 5)
  71 #define DBG_WATCH_CTRL_LEN_8            (0xff << 5)
  72 #define DBG_WATCH_CTRL_LEN_MASK(x)      ((x) & (0xff << 5))
  73 #define DBG_WATCH_CTRL_EXEC             (0x0 << 3)
  74 #define DBG_WATCH_CTRL_LOAD             (0x1 << 3)
  75 #define DBG_WATCH_CTRL_STORE            (0x2 << 3)
  76 #define DBG_WATCH_CTRL_ACCESS_MASK(x)   ((x) & (0x3 << 3))
  77
  78 /* Common for breakpoint and watchpoint */
  79 #define DBG_WB_CTRL_EL1         (0x1 << 1)
  80 #define DBG_WB_CTRL_EL0         (0x2 << 1)
  81 #define DBG_WB_CTRL_ELX_MASK(x) ((x) & (0x3 << 1))
  82 #define DBG_WB_CTRL_E           (0x1 << 0)
  83
  84 #define DBG_REG_BASE_BVR        0
  85 #define DBG_REG_BASE_BCR        (DBG_REG_BASE_BVR + 16)
  86 #define DBG_REG_BASE_WVR        (DBG_REG_BASE_BCR + 16)
  87 #define DBG_REG_BASE_WCR        (DBG_REG_BASE_WVR + 16)
  88
  89 /* Watchpoint/breakpoint helpers */
  90 #define DBG_WB_WVR      "wvr"
  91 #define DBG_WB_WCR      "wcr"
  92 #define DBG_WB_BVR      "bvr"
  93 #define DBG_WB_BCR      "bcr"
  94
  95 #define DBG_WB_READ(reg, num, val) do {                                 \
  96         __asm __volatile("mrs %0, dbg" reg #num "_el1" : "=r" (val));   \
  97 } while (0)
  98
  99 #define DBG_WB_WRITE(reg, num, val) do {                                \
 100         __asm __volatile("msr dbg" reg #num "_el1, %0" :: "r" (val));   \
 101 } while (0)
 102
 103 #define READ_WB_REG_CASE(reg, num, offset, val)         \
 104         case (num + offset):                            \
 105                 DBG_WB_READ(reg, num, val);             \
 106                 break
 107
 108 #define WRITE_WB_REG_CASE(reg, num, offset, val)        \
 109         case (num + offset):                            \
 110                 DBG_WB_WRITE(reg, num, val);            \
 111                 break
 112
 113 #define SWITCH_CASES_READ_WB_REG(reg, offset, val)      \
 114         READ_WB_REG_CASE(reg,  0, offset, val);         \
 115         READ_WB_REG_CASE(reg,  1, offset, val);         \
 116         READ_WB_REG_CASE(reg,  2, offset, val);         \
 117         READ_WB_REG_CASE(reg,  3, offset, val);         \
 118         READ_WB_REG_CASE(reg,  4, offset, val);         \
 119         READ_WB_REG_CASE(reg,  5, offset, val);         \
 120         READ_WB_REG_CASE(reg,  6, offset, val);         \
 121         READ_WB_REG_CASE(reg,  7, offset, val);         \
 122         READ_WB_REG_CASE(reg,  8, offset, val);         \
 123         READ_WB_REG_CASE(reg,  9, offset, val);         \
 124         READ_WB_REG_CASE(reg, 10, offset, val);         \
 125         READ_WB_REG_CASE(reg, 11, offset, val);         \
 126         READ_WB_REG_CASE(reg, 12, offset, val);         \
 127         READ_WB_REG_CASE(reg, 13, offset, val);         \
 128         READ_WB_REG_CASE(reg, 14, offset, val);         \
 129         READ_WB_REG_CASE(reg, 15, offset, val)
 130
 131 #define SWITCH_CASES_WRITE_WB_REG(reg, offset, val)     \
 132         WRITE_WB_REG_CASE(reg,  0, offset, val);        \
 133         WRITE_WB_REG_CASE(reg,  1, offset, val);        \
 134         WRITE_WB_REG_CASE(reg,  2, offset, val);        \
 135         WRITE_WB_REG_CASE(reg,  3, offset, val);        \
 136         WRITE_WB_REG_CASE(reg,  4, offset, val);        \
 137         WRITE_WB_REG_CASE(reg,  5, offset, val);        \
 138         WRITE_WB_REG_CASE(reg,  6, offset, val);        \
 139         WRITE_WB_REG_CASE(reg,  7, offset, val);        \
 140         WRITE_WB_REG_CASE(reg,  8, offset, val);        \
 141         WRITE_WB_REG_CASE(reg,  9, offset, val);        \
 142         WRITE_WB_REG_CASE(reg, 10, offset, val);        \
 143         WRITE_WB_REG_CASE(reg, 11, offset, val);        \
 144         WRITE_WB_REG_CASE(reg, 12, offset, val);        \
 145         WRITE_WB_REG_CASE(reg, 13, offset, val);        \
 146         WRITE_WB_REG_CASE(reg, 14, offset, val);        \
 147         WRITE_WB_REG_CASE(reg, 15, offset, val)
 148
 149 #ifdef DDB
 150 static uint64_t
 151 dbg_wb_read_reg(int reg, int n)
 152 {
 153         uint64_t val = 0;
 154
 155         switch (reg + n) {
 156         SWITCH_CASES_READ_WB_REG(DBG_WB_WVR, DBG_REG_BASE_WVR, val);
 157         SWITCH_CASES_READ_WB_REG(DBG_WB_WCR, DBG_REG_BASE_WCR, val);
 158         SWITCH_CASES_READ_WB_REG(DBG_WB_BVR, DBG_REG_BASE_BVR, val);
 159         SWITCH_CASES_READ_WB_REG(DBG_WB_BCR, DBG_REG_BASE_BCR, val);
 160         default:
 161                 printf("trying to read from wrong debug register %d\n", n);
 162         }
 163
 164         return val;
 165 }
 166 #endif /* DDB */
 167
 168 static void
 169 dbg_wb_write_reg(int reg, int n, uint64_t val)
 170 {
 171         switch (reg + n) {
 172         SWITCH_CASES_WRITE_WB_REG(DBG_WB_WVR, DBG_REG_BASE_WVR, val);
 173         SWITCH_CASES_WRITE_WB_REG(DBG_WB_WCR, DBG_REG_BASE_WCR, val);
 174         SWITCH_CASES_WRITE_WB_REG(DBG_WB_BVR, DBG_REG_BASE_BVR, val);
 175         SWITCH_CASES_WRITE_WB_REG(DBG_WB_BCR, DBG_REG_BASE_BCR, val);
 176         default:
 177                 printf("trying to write to wrong debug register %d\n", n);
 178                 return;
 179         }
 180         isb();
 181 }
 182
 183 #ifdef DDB
 184 void
 185 kdb_cpu_set_singlestep(void)
 186 {
 187
 188         kdb_frame->tf_spsr |= DBG_SPSR_SS;
 189         WRITE_SPECIALREG(mdscr_el1, READ_SPECIALREG(mdscr_el1) |
 190             DBG_MDSCR_SS | DBG_MDSCR_KDE);
 191
 192         /*
 193          * Disable breakpoints and watchpoints, e.g. stepping
 194          * over watched instruction will trigger break exception instead of
 195          * single-step exception and locks CPU on that instruction for ever.
 196          */
 197         if ((kernel_monitor.dbg_flags & DBGMON_ENABLED) != 0) {
 198                 WRITE_SPECIALREG(mdscr_el1,
 199                     READ_SPECIALREG(mdscr_el1) & ~DBG_MDSCR_MDE);
 200         }
 201 }
 202
 203 void
 204 kdb_cpu_clear_singlestep(void)
 205 {
 206
 207         WRITE_SPECIALREG(mdscr_el1, READ_SPECIALREG(mdscr_el1) &
 208             ~(DBG_MDSCR_SS | DBG_MDSCR_KDE));
 209
 210         /* Restore breakpoints and watchpoints */
 211         if ((kernel_monitor.dbg_flags & DBGMON_ENABLED) != 0) {
 212                 WRITE_SPECIALREG(mdscr_el1,
 213                     READ_SPECIALREG(mdscr_el1) | DBG_MDSCR_MDE);
 214
 215                 if ((kernel_monitor.dbg_flags & DBGMON_KERNEL) != 0) {
 216                         WRITE_SPECIALREG(mdscr_el1,
 217                             READ_SPECIALREG(mdscr_el1) | DBG_MDSCR_KDE);
 218                 }
 219         }
 220 }
 221
 222 static const char *
 223 dbg_watchtype_str(uint32_t type)
 224 {
 225         switch (type) {
 226                 case DBG_WATCH_CTRL_EXEC:
 227                         return ("execute");
 228                 case DBG_WATCH_CTRL_STORE:
 229                         return ("write");
 230                 case DBG_WATCH_CTRL_LOAD:
 231                         return ("read");
 232                 case DBG_WATCH_CTRL_LOAD | DBG_WATCH_CTRL_STORE:
 233                         return ("read/write");
 234                 default:
 235                         return ("invalid");
 236         }
 237 }
 238
 239 static int
 240 dbg_watchtype_len(uint32_t len)
 241 {
 242         switch (len) {
 243         case DBG_WATCH_CTRL_LEN_1:
 244                 return (1);
 245         case DBG_WATCH_CTRL_LEN_2:
 246                 return (2);
 247         case DBG_WATCH_CTRL_LEN_4:
 248                 return (4);
 249         case DBG_WATCH_CTRL_LEN_8:
 250                 return (8);
 251         default:
 252                 return (0);
 253         }
 254 }
 255
 256 void
 257 dbg_show_watchpoint(void)
 258 {
 259         uint32_t wcr, len, type;
 260         uint64_t addr;
 261         int i;
 262
 263         db_printf("\nhardware watchpoints:\n");
 264         db_printf("  watch    status        type  len             address              symbol\n");
 265         db_printf("  -----  --------  ----------  ---  ------------------  ------------------\n");
 266         for (i = 0; i < dbg_watchpoint_num; i++) {
 267                 wcr = dbg_wb_read_reg(DBG_REG_BASE_WCR, i);
 268                 if ((wcr & DBG_WB_CTRL_E) != 0) {
 269                         type = DBG_WATCH_CTRL_ACCESS_MASK(wcr);
 270                         len = DBG_WATCH_CTRL_LEN_MASK(wcr);
 271                         addr = dbg_wb_read_reg(DBG_REG_BASE_WVR, i);
 272                         db_printf("  %-5d  %-8s  %10s  %3d  0x%16lx  ",
 273                             i, "enabled", dbg_watchtype_str(type),
 274                             dbg_watchtype_len(len), addr);
 275                         db_printsym((db_addr_t)addr, DB_STGY_ANY);
 276                         db_printf("\n");
 277                 } else {
 278                         db_printf("  %-5d  disabled\n", i);
 279                 }
 280         }
 281 }
 282 #endif /* DDB */
 283
 284 static int
 285 dbg_find_free_slot(struct debug_monitor_state *monitor, enum dbg_t type)
 286 {
 287         uint64_t *reg;
 288         u_int max, i;
 289
 290         switch(type) {
 291         case DBG_TYPE_BREAKPOINT:
 292                 max = dbg_breakpoint_num;
 293                 reg = monitor->dbg_bcr;
 294                 break;
 295         case DBG_TYPE_WATCHPOINT:
 296                 max = dbg_watchpoint_num;
 297                 reg = monitor->dbg_wcr;
 298                 break;
 299         default:
 300                 printf("Unsupported debug type\n");
 301                 return (i);
 302         }
 303
 304         for (i = 0; i < max; i++) {
 305                 if ((reg[i] & DBG_WB_CTRL_E) == 0)
 306                         return (i);
 307         }
 308
 309         return (-1);
 310 }
 311
 312 static int
 313 dbg_find_slot(struct debug_monitor_state *monitor, enum dbg_t type,
 314     vm_offset_t addr)
 315 {
 316         uint64_t *reg_addr, *reg_ctrl;
 317         u_int max, i;
 318
 319         switch(type) {
 320         case DBG_TYPE_BREAKPOINT:
 321                 max = dbg_breakpoint_num;
 322                 reg_addr = monitor->dbg_bvr;
 323                 reg_ctrl = monitor->dbg_bcr;
 324                 break;
 325         case DBG_TYPE_WATCHPOINT:
 326                 max = dbg_watchpoint_num;
 327                 reg_addr = monitor->dbg_wvr;
 328                 reg_ctrl = monitor->dbg_wcr;
 329                 break;
 330         default:
 331                 printf("Unsupported debug type\n");
 332                 return (i);
 333         }
 334
 335         for (i = 0; i < max; i++) {
 336                 if (reg_addr[i] == addr &&
 337                     (reg_ctrl[i] & DBG_WB_CTRL_E) != 0)
 338                         return (i);
 339         }
 340
 341         return (-1);
 342 }
 343
 344 int
 345 dbg_setup_watchpoint(struct debug_monitor_state *monitor, vm_offset_t addr,
 346     vm_size_t size, enum dbg_access_t access)
 347 {
 348         uint64_t wcr_size, wcr_priv, wcr_access;
 349         u_int i;
 350
 351         if (monitor == NULL)
 352                 monitor = &kernel_monitor;
 353
 354         i = dbg_find_free_slot(monitor, DBG_TYPE_WATCHPOINT);
 355         if (i == -1) {
 356                 printf("Can not find slot for watchpoint, max %d"
 357                     " watchpoints supported\n", dbg_watchpoint_num);
 358                 return (i);
 359         }
 360
 361         switch(size) {
 362         case 1:
 363                 wcr_size = DBG_WATCH_CTRL_LEN_1;
 364                 break;
 365         case 2:
 366                 wcr_size = DBG_WATCH_CTRL_LEN_2;
 367                 break;
 368         case 4:
 369                 wcr_size = DBG_WATCH_CTRL_LEN_4;
 370                 break;
 371         case 8:
 372                 wcr_size = DBG_WATCH_CTRL_LEN_8;
 373                 break;
 374         default:
 375                 printf("Unsupported address size for watchpoint\n");
 376                 return (-1);
 377         }
 378
 379         if ((monitor->dbg_flags & DBGMON_KERNEL) == 0)
 380                 wcr_priv = DBG_WB_CTRL_EL0;
 381         else
 382                 wcr_priv = DBG_WB_CTRL_EL1;
 383
 384         switch(access) {
 385         case HW_BREAKPOINT_X:
 386                 wcr_access = DBG_WATCH_CTRL_EXEC;
 387                 break;
 388         case HW_BREAKPOINT_R:
 389                 wcr_access = DBG_WATCH_CTRL_LOAD;
 390                 break;
 391         case HW_BREAKPOINT_W:
 392                 wcr_access = DBG_WATCH_CTRL_STORE;
 393                 break;
 394         case HW_BREAKPOINT_RW:
 395                 wcr_access = DBG_WATCH_CTRL_LOAD | DBG_WATCH_CTRL_STORE;
 396                 break;
 397         default:
 398                 printf("Unsupported exception level for watchpoint\n");
 399                 return (-1);
 400         }
 401
 402         monitor->dbg_wvr[i] = addr;
 403         monitor->dbg_wcr[i] = wcr_size | wcr_access | wcr_priv | DBG_WB_CTRL_E;
 404         monitor->dbg_enable_count++;
 405         monitor->dbg_flags |= DBGMON_ENABLED;
 406
 407         dbg_register_sync(monitor);
 408         return (0);
 409 }
 410
 411 int
 412 dbg_remove_watchpoint(struct debug_monitor_state *monitor, vm_offset_t addr,
 413     vm_size_t size)
 414 {
 415         u_int i;
 416
 417         if (monitor == NULL)
 418                 monitor = &kernel_monitor;
 419
 420         i = dbg_find_slot(monitor, DBG_TYPE_WATCHPOINT, addr);
 421         if (i == -1) {
 422                 printf("Can not find watchpoint for address 0%lx\n", addr);
 423                 return (i);
 424         }
 425
 426         monitor->dbg_wvr[i] = 0;
 427         monitor->dbg_wcr[i] = 0;
 428         monitor->dbg_enable_count--;
 429         if (monitor->dbg_enable_count == 0)
 430                 monitor->dbg_flags &= ~DBGMON_ENABLED;
 431
 432         dbg_register_sync(monitor);
 433         return (0);
 434 }
 435
 436 void
 437 dbg_register_sync(struct debug_monitor_state *monitor)
 438 {
 439         uint64_t mdscr;
 440         int i;
 441
 442         if (monitor == NULL)
 443                 monitor = &kernel_monitor;
 444
 445         mdscr = READ_SPECIALREG(mdscr_el1);
 446         if ((monitor->dbg_flags & DBGMON_ENABLED) == 0) {
 447                 mdscr &= ~(DBG_MDSCR_MDE | DBG_MDSCR_KDE);
 448         } else {
 449                 for (i = 0; i < dbg_breakpoint_num; i++) {
 450                         dbg_wb_write_reg(DBG_REG_BASE_BCR, i,
 451                             monitor->dbg_bcr[i]);
 452                         dbg_wb_write_reg(DBG_REG_BASE_BVR, i,
 453                             monitor->dbg_bvr[i]);
 454                 }
 455
 456                 for (i = 0; i < dbg_watchpoint_num; i++) {
 457                         dbg_wb_write_reg(DBG_REG_BASE_WCR, i,
 458                             monitor->dbg_wcr[i]);
 459                         dbg_wb_write_reg(DBG_REG_BASE_WVR, i,
 460                             monitor->dbg_wvr[i]);
 461                 }
 462                 mdscr |= DBG_MDSCR_MDE;
 463                 if ((monitor->dbg_flags & DBGMON_KERNEL) == DBGMON_KERNEL)
 464                         mdscr |= DBG_MDSCR_KDE;
 465         }
 466         WRITE_SPECIALREG(mdscr_el1, mdscr);
 467         isb();
 468 }
 469
 470 void
 471 dbg_monitor_init(void)
 472 {
 473         u_int i;
 474
 475         /* Find out many breakpoints and watchpoints we can use */
 476         dbg_watchpoint_num = ((READ_SPECIALREG(id_aa64dfr0_el1) >> 20) & 0xf) + 1;
 477         dbg_breakpoint_num = ((READ_SPECIALREG(id_aa64dfr0_el1) >> 12) & 0xf) + 1;
 478
 479         if (bootverbose && PCPU_GET(cpuid) == 0) {
 480                 printf("%d watchpoints and %d breakpoints supported\n",
 481                     dbg_watchpoint_num, dbg_breakpoint_num);
 482         }
 483
 484         /*
 485          * We have limited number of {watch,break}points, each consists of
 486          * two registers:
 487          * - wcr/bcr regsiter configurates corresponding {watch,break}point
 488          *   behaviour
 489          * - wvr/bvr register keeps address we are hunting for
 490          *
 491          * Reset all breakpoints and watchpoints.
 492          */
 493         for (i = 0; i < dbg_watchpoint_num; i++) {
 494                 dbg_wb_write_reg(DBG_REG_BASE_WCR, i, 0);
 495                 dbg_wb_write_reg(DBG_REG_BASE_WVR, i, 0);
 496         }
 497
 498         for (i = 0; i < dbg_breakpoint_num; i++) {
 499                 dbg_wb_write_reg(DBG_REG_BASE_BCR, i, 0);
 500                 dbg_wb_write_reg(DBG_REG_BASE_BVR, i, 0);
 501         }
 502
 503         dbg_enable();
 504 }
 505
 506 void
 507 dbg_monitor_enter(struct thread *thread)
 508 {
 509         int i;
 510
 511         if ((kernel_monitor.dbg_flags & DBGMON_ENABLED) != 0) {
 512                 /* Install the kernel version of the registers */
 513                 dbg_register_sync(&kernel_monitor);
 514         } else if ((thread->td_pcb->pcb_dbg_regs.dbg_flags & DBGMON_ENABLED) != 0) {
 515                 /* Disable the user breakpoints until we return to userspace */
 516                 for (i = 0; i < dbg_watchpoint_num; i++) {
 517                         dbg_wb_write_reg(DBG_REG_BASE_WCR, i, 0);
 518                         dbg_wb_write_reg(DBG_REG_BASE_WVR, i, 0);
 519                 }
 520
 521                 for (i = 0; i < dbg_breakpoint_num; ++i) {
 522                         dbg_wb_write_reg(DBG_REG_BASE_BCR, i, 0);
 523                         dbg_wb_write_reg(DBG_REG_BASE_BVR, i, 0);
 524                 }
 525                 WRITE_SPECIALREG(mdscr_el1,
 526                     READ_SPECIALREG(mdscr_el1) &
 527                     ~(DBG_MDSCR_MDE | DBG_MDSCR_KDE));
 528                 isb();
 529         }
 530 }
 531
 532 void
 533 dbg_monitor_exit(struct thread *thread, struct trapframe *frame)
 534 {
 535         int i;
 536
 537         frame->tf_spsr |= PSR_D;
 538         if ((thread->td_pcb->pcb_dbg_regs.dbg_flags & DBGMON_ENABLED) != 0) {
 539                 /* Install the kernel version of the registers */
 540                 dbg_register_sync(&thread->td_pcb->pcb_dbg_regs);
 541                 frame->tf_spsr &= ~PSR_D;
 542         } else if ((kernel_monitor.dbg_flags & DBGMON_ENABLED) != 0) {
 543                 /* Disable the user breakpoints until we return to userspace */
 544                 for (i = 0; i < dbg_watchpoint_num; i++) {
 545                         dbg_wb_write_reg(DBG_REG_BASE_WCR, i, 0);
 546                         dbg_wb_write_reg(DBG_REG_BASE_WVR, i, 0);
 547                 }
 548
 549                 for (i = 0; i < dbg_breakpoint_num; ++i) {
 550                         dbg_wb_write_reg(DBG_REG_BASE_BCR, i, 0);
 551                         dbg_wb_write_reg(DBG_REG_BASE_BVR, i, 0);
 552                 }
 553                 WRITE_SPECIALREG(mdscr_el1,
 554                     READ_SPECIALREG(mdscr_el1) &
 555                     ~(DBG_MDSCR_MDE | DBG_MDSCR_KDE));
 556                 isb();
 557         }
 558 }