sys/mips/mips/trap.c

   1 /*      $OpenBSD: trap.c,v 1.19 1998/09/30 12:40:41 pefo Exp $  */
   2 /* tracked to 1.23 */
   3 /*-
   4  * SPDX-License-Identifier: BSD-3-Clause
   5  *
   6  * Copyright (c) 1988 University of Utah.
   7  * Copyright (c) 1992, 1993
   8  *      The Regents of the University of California.  All rights reserved.
   9  *
  10  * This code is derived from software contributed to Berkeley by
  11  * the Systems Programming Group of the University of Utah Computer
  12  * Science Department and Ralph Campbell.
  13  *
  14  * Redistribution and use in source and binary forms, with or without
  15  * modification, are permitted provided that the following conditions
  16  * are met:
  17  * 1. Redistributions of source code must retain the above copyright
  18  *    notice, this list of conditions and the following disclaimer.
  19  * 2. Redistributions in binary form must reproduce the above copyright
  20  *    notice, this list of conditions and the following disclaimer in the
  21  *    documentation and/or other materials provided with the distribution.
  22  * 3. Neither the name of the University nor the names of its contributors
  23  *    may be used to endorse or promote products derived from this software
  24  *    without specific prior written permission.
  25  *
  26  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  27  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  28  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  29  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  30  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  31  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  32  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  33  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  34  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  35  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  36  * SUCH DAMAGE.
  37  *
  38  * from: Utah Hdr: trap.c 1.32 91/04/06
  39  *
  40  *      from: @(#)trap.c        8.5 (Berkeley) 1/11/94
  41  *      JNPR: trap.c,v 1.13.2.2 2007/08/29 10:03:49 girish
  42  */
  43 #include <sys/cdefs.h>
  44 __FBSDID("$FreeBSD$");
  45
  46 #include "opt_ddb.h"
  47 #include "opt_ktrace.h"
  48
  49 #include <sys/param.h>
  50 #include <sys/systm.h>
  51 #include <sys/sysent.h>
  52 #include <sys/proc.h>
  53 #include <sys/kernel.h>
  54 #include <sys/signalvar.h>
  55 #include <sys/syscall.h>
  56 #include <sys/lock.h>
  57 #include <vm/vm.h>
  58 #include <vm/vm_extern.h>
  59 #include <vm/vm_kern.h>
  60 #include <vm/vm_page.h>
  61 #include <vm/vm_map.h>
  62 #include <vm/vm_param.h>
  63 #include <sys/vmmeter.h>
  64 #include <sys/ptrace.h>
  65 #include <sys/user.h>
  66 #include <sys/buf.h>
  67 #include <sys/vnode.h>
  68 #include <sys/pioctl.h>
  69 #include <sys/sysctl.h>
  70 #include <sys/syslog.h>
  71 #include <sys/bus.h>
  72 #ifdef KTRACE
  73 #include <sys/ktrace.h>
  74 #endif
  75 #include <net/netisr.h>
  76
  77 #include <machine/trap.h>
  78 #include <machine/cpu.h>
  79 #include <machine/cpuinfo.h>
  80 #include <machine/pte.h>
  81 #include <machine/pmap.h>
  82 #include <machine/md_var.h>
  83 #include <machine/mips_opcode.h>
  84 #include <machine/frame.h>
  85 #include <machine/regnum.h>
  86 #include <machine/tls.h>
  87
  88 #ifdef DDB
  89 #include <machine/db_machdep.h>
  90 #include <ddb/db_sym.h>
  91 #include <ddb/ddb.h>
  92 #include <sys/kdb.h>
  93 #endif
  94
  95 #ifdef KDTRACE_HOOKS
  96 #include <sys/dtrace_bsd.h>
  97 #endif
  98
  99 #ifdef TRAP_DEBUG
 100 int trap_debug = 0;
 101 SYSCTL_INT(_machdep, OID_AUTO, trap_debug, CTLFLAG_RW,
 102     &trap_debug, 0, "Debug information on all traps");
 103 #endif
 104
 105 #define lbu_macro(data, addr)                                           \
 106         __asm __volatile ("lbu %0, 0x0(%1)"                             \
 107                         : "=r" (data)   /* outputs */                   \
 108                         : "r" (addr));  /* inputs */
 109
 110 #define lb_macro(data, addr)                                            \
 111         __asm __volatile ("lb %0, 0x0(%1)"                              \
 112                         : "=r" (data)   /* outputs */                   \
 113                         : "r" (addr));  /* inputs */
 114
 115 #define lwl_macro(data, addr)                                           \
 116         __asm __volatile ("lwl %0, 0x0(%1)"                             \
 117                         : "=r" (data)   /* outputs */                   \
 118                         : "r" (addr));  /* inputs */
 119
 120 #define lwr_macro(data, addr)                                           \
 121         __asm __volatile ("lwr %0, 0x0(%1)"                             \
 122                         : "=r" (data)   /* outputs */                   \
 123                         : "r" (addr));  /* inputs */
 124
 125 #define ldl_macro(data, addr)                                           \
 126         __asm __volatile ("ldl %0, 0x0(%1)"                             \
 127                         : "=r" (data)   /* outputs */                   \
 128                         : "r" (addr));  /* inputs */
 129
 130 #define ldr_macro(data, addr)                                           \
 131         __asm __volatile ("ldr %0, 0x0(%1)"                             \
 132                         : "=r" (data)   /* outputs */                   \
 133                         : "r" (addr));  /* inputs */
 134
 135 #define sb_macro(data, addr)                                            \
 136         __asm __volatile ("sb %0, 0x0(%1)"                              \
 137                         :                               /* outputs */   \
 138                         : "r" (data), "r" (addr));      /* inputs */
 139
 140 #define swl_macro(data, addr)                                           \
 141         __asm __volatile ("swl %0, 0x0(%1)"                             \
 142                         :                               /* outputs */   \
 143                         : "r" (data), "r" (addr));      /* inputs */
 144
 145 #define swr_macro(data, addr)                                           \
 146         __asm __volatile ("swr %0, 0x0(%1)"                             \
 147                         :                               /* outputs */   \
 148                         : "r" (data), "r" (addr));      /* inputs */
 149
 150 #define sdl_macro(data, addr)                                           \
 151         __asm __volatile ("sdl %0, 0x0(%1)"                             \
 152                         :                               /* outputs */   \
 153                         : "r" (data), "r" (addr));      /* inputs */
 154
 155 #define sdr_macro(data, addr)                                           \
 156         __asm __volatile ("sdr %0, 0x0(%1)"                             \
 157                         :                               /* outputs */   \
 158                         : "r" (data), "r" (addr));      /* inputs */
 159
 160 static void log_illegal_instruction(const char *, struct trapframe *);
 161 static void log_bad_page_fault(char *, struct trapframe *, int);
 162 static void log_frame_dump(struct trapframe *frame);
 163 static void get_mapping_info(vm_offset_t, pd_entry_t **, pt_entry_t **);
 164
 165 int (*dtrace_invop_jump_addr)(struct trapframe *);
 166
 167 #ifdef TRAP_DEBUG
 168 static void trap_frame_dump(struct trapframe *frame);
 169 #endif
 170
 171 void (*machExceptionTable[]) (void)= {
 172 /*
 173  * The kernel exception handlers.
 174  */
 175         MipsKernIntr,           /* external interrupt */
 176         MipsKernGenException,   /* TLB modification */
 177         MipsTLBInvalidException,/* TLB miss (load or instr. fetch) */
 178         MipsTLBInvalidException,/* TLB miss (store) */
 179         MipsKernGenException,   /* address error (load or I-fetch) */
 180         MipsKernGenException,   /* address error (store) */
 181         MipsKernGenException,   /* bus error (I-fetch) */
 182         MipsKernGenException,   /* bus error (load or store) */
 183         MipsKernGenException,   /* system call */
 184         MipsKernGenException,   /* breakpoint */
 185         MipsKernGenException,   /* reserved instruction */
 186         MipsKernGenException,   /* coprocessor unusable */
 187         MipsKernGenException,   /* arithmetic overflow */
 188         MipsKernGenException,   /* trap exception */
 189         MipsKernGenException,   /* virtual coherence exception inst */
 190         MipsKernGenException,   /* floating point exception */
 191         MipsKernGenException,   /* reserved */
 192         MipsKernGenException,   /* reserved */
 193         MipsKernGenException,   /* reserved */
 194         MipsKernGenException,   /* reserved */
 195         MipsKernGenException,   /* reserved */
 196         MipsKernGenException,   /* reserved */
 197         MipsKernGenException,   /* reserved */
 198         MipsKernGenException,   /* watch exception */
 199         MipsKernGenException,   /* reserved */
 200         MipsKernGenException,   /* reserved */
 201         MipsKernGenException,   /* reserved */
 202         MipsKernGenException,   /* reserved */
 203         MipsKernGenException,   /* reserved */
 204         MipsKernGenException,   /* reserved */
 205         MipsKernGenException,   /* reserved */
 206         MipsKernGenException,   /* virtual coherence exception data */
 207 /*
 208  * The user exception handlers.
 209  */
 210         MipsUserIntr,           /* 0 */
 211         MipsUserGenException,   /* 1 */
 212         MipsTLBInvalidException,/* 2 */
 213         MipsTLBInvalidException,/* 3 */
 214         MipsUserGenException,   /* 4 */
 215         MipsUserGenException,   /* 5 */
 216         MipsUserGenException,   /* 6 */
 217         MipsUserGenException,   /* 7 */
 218         MipsUserGenException,   /* 8 */
 219         MipsUserGenException,   /* 9 */
 220         MipsUserGenException,   /* 10 */
 221         MipsUserGenException,   /* 11 */
 222         MipsUserGenException,   /* 12 */
 223         MipsUserGenException,   /* 13 */
 224         MipsUserGenException,   /* 14 */
 225         MipsUserGenException,   /* 15 */
 226         MipsUserGenException,   /* 16 */
 227         MipsUserGenException,   /* 17 */
 228         MipsUserGenException,   /* 18 */
 229         MipsUserGenException,   /* 19 */
 230         MipsUserGenException,   /* 20 */
 231         MipsUserGenException,   /* 21 */
 232         MipsUserGenException,   /* 22 */
 233         MipsUserGenException,   /* 23 */
 234         MipsUserGenException,   /* 24 */
 235         MipsUserGenException,   /* 25 */
 236         MipsUserGenException,   /* 26 */
 237         MipsUserGenException,   /* 27 */
 238         MipsUserGenException,   /* 28 */
 239         MipsUserGenException,   /* 29 */
 240         MipsUserGenException,   /* 20 */
 241         MipsUserGenException,   /* 31 */
 242 };
 243
 244 char *trap_type[] = {
 245         "external interrupt",
 246         "TLB modification",
 247         "TLB miss (load or instr. fetch)",
 248         "TLB miss (store)",
 249         "address error (load or I-fetch)",
 250         "address error (store)",
 251         "bus error (I-fetch)",
 252         "bus error (load or store)",
 253         "system call",
 254         "breakpoint",
 255         "reserved instruction",
 256         "coprocessor unusable",
 257         "arithmetic overflow",
 258         "trap",
 259         "virtual coherency instruction",
 260         "floating point",
 261         "reserved 16",
 262         "reserved 17",
 263         "reserved 18",
 264         "reserved 19",
 265         "reserved 20",
 266         "reserved 21",
 267         "reserved 22",
 268         "watch",
 269         "reserved 24",
 270         "reserved 25",
 271         "reserved 26",
 272         "reserved 27",
 273         "reserved 28",
 274         "reserved 29",
 275         "reserved 30",
 276         "virtual coherency data",
 277 };
 278
 279 #if !defined(SMP) && (defined(DDB) || defined(DEBUG))
 280 struct trapdebug trapdebug[TRAPSIZE], *trp = trapdebug;
 281 #endif
 282
 283 #define KERNLAND(x)     ((vm_offset_t)(x) >= VM_MIN_KERNEL_ADDRESS && (vm_offset_t)(x) < VM_MAX_KERNEL_ADDRESS)
 284 #define DELAYBRANCH(x)  ((x) & MIPS_CR_BR_DELAY)
 285
 286 /*
 287  * MIPS load/store access type
 288  */
 289 enum {
 290         MIPS_LHU_ACCESS = 1,
 291         MIPS_LH_ACCESS,
 292         MIPS_LWU_ACCESS,
 293         MIPS_LW_ACCESS,
 294         MIPS_LD_ACCESS,
 295         MIPS_SH_ACCESS,
 296         MIPS_SW_ACCESS,
 297         MIPS_SD_ACCESS
 298 };
 299
 300 char *access_name[] = {
 301         "Load Halfword Unsigned",
 302         "Load Halfword",
 303         "Load Word Unsigned",
 304         "Load Word",
 305         "Load Doubleword",
 306         "Store Halfword",
 307         "Store Word",
 308         "Store Doubleword"
 309 };
 310
 311 #ifdef  CPU_CNMIPS
 312 #include <machine/octeon_cop2.h>
 313 #endif
 314
 315 static int allow_unaligned_acc = 1;
 316
 317 SYSCTL_INT(_vm, OID_AUTO, allow_unaligned_acc, CTLFLAG_RW,
 318     &allow_unaligned_acc, 0, "Allow unaligned accesses");
 319
 320 /*
 321  * FP emulation is assumed to work on O32, but the code is outdated and crufty
 322  * enough that it's a more sensible default to have it disabled when using
 323  * other ABIs.  At the very least, it needs a lot of help in using
 324  * type-semantic ABI-oblivious macros for everything it does.
 325  */
 326 #if defined(__mips_o32)
 327 static int emulate_fp = 1;
 328 #else
 329 static int emulate_fp = 0;
 330 #endif
 331 SYSCTL_INT(_machdep, OID_AUTO, emulate_fp, CTLFLAG_RW,
 332     &emulate_fp, 0, "Emulate unimplemented FPU instructions");
 333
 334 static int emulate_unaligned_access(struct trapframe *frame, int mode);
 335
 336 extern void fswintrberr(void); /* XXX */
 337
 338 int
 339 cpu_fetch_syscall_args(struct thread *td)
 340 {
 341         struct trapframe *locr0;
 342         struct sysentvec *se;
 343         struct syscall_args *sa;
 344         int error, nsaved;
 345
 346         locr0 = td->td_frame;
 347         sa = &td->td_sa;
 348
 349         bzero(sa->args, sizeof(sa->args));
 350
 351         /* compute next PC after syscall instruction */
 352         td->td_pcb->pcb_tpc = sa->trapframe->pc; /* Remember if restart */
 353         if (DELAYBRANCH(sa->trapframe->cause))   /* Check BD bit */
 354                 locr0->pc = MipsEmulateBranch(locr0, sa->trapframe->pc, 0, 0);
 355         else
 356                 locr0->pc += sizeof(int);
 357         sa->code = locr0->v0;
 358
 359         switch (sa->code) {
 360         case SYS___syscall:
 361         case SYS_syscall:
 362                 /*
 363                  * This is an indirect syscall, in which the code is the first argument.
 364                  */
 365 #if (!defined(__mips_n32) && !defined(__mips_n64)) || defined(COMPAT_FREEBSD32)
 366                 if (sa->code == SYS___syscall && SV_PROC_FLAG(td->td_proc, SV_ILP32)) {
 367                         /*
 368                          * Like syscall, but code is a quad, so as to maintain alignment
 369                          * for the rest of the arguments.
 370                          */
 371                         if (_QUAD_LOWWORD == 0)
 372                                 sa->code = locr0->a0;
 373                         else
 374                                 sa->code = locr0->a1;
 375                         sa->args[0] = locr0->a2;
 376                         sa->args[1] = locr0->a3;
 377                         nsaved = 2;
 378                         break;
 379                 }
 380 #endif
 381                 /*
 382                  * This is either not a quad syscall, or is a quad syscall with a
 383                  * new ABI in which quads fit in a single register.
 384                  */
 385                 sa->code = locr0->a0;
 386                 sa->args[0] = locr0->a1;
 387                 sa->args[1] = locr0->a2;
 388                 sa->args[2] = locr0->a3;
 389                 nsaved = 3;
 390 #if defined(__mips_n32) || defined(__mips_n64)
 391 #ifdef COMPAT_FREEBSD32
 392                 if (!SV_PROC_FLAG(td->td_proc, SV_ILP32)) {
 393 #endif
 394                         /*
 395                          * Non-o32 ABIs support more arguments in registers.
 396                          */
 397                         sa->args[3] = locr0->a4;
 398                         sa->args[4] = locr0->a5;
 399                         sa->args[5] = locr0->a6;
 400                         sa->args[6] = locr0->a7;
 401                         nsaved += 4;
 402 #ifdef COMPAT_FREEBSD32
 403                 }
 404 #endif
 405 #endif
 406                 break;
 407         default:
 408                 /*
 409                  * A direct syscall, arguments are just parameters to the syscall.
 410                  */
 411                 sa->args[0] = locr0->a0;
 412                 sa->args[1] = locr0->a1;
 413                 sa->args[2] = locr0->a2;
 414                 sa->args[3] = locr0->a3;
 415                 nsaved = 4;
 416 #if defined (__mips_n32) || defined(__mips_n64)
 417 #ifdef COMPAT_FREEBSD32
 418                 if (!SV_PROC_FLAG(td->td_proc, SV_ILP32)) {
 419 #endif
 420                         /*
 421                          * Non-o32 ABIs support more arguments in registers.
 422                          */
 423                         sa->args[4] = locr0->a4;
 424                         sa->args[5] = locr0->a5;
 425                         sa->args[6] = locr0->a6;
 426                         sa->args[7] = locr0->a7;
 427                         nsaved += 4;
 428 #ifdef COMPAT_FREEBSD32
 429                 }
 430 #endif
 431 #endif
 432                 break;
 433         }
 434
 435 #ifdef TRAP_DEBUG
 436         if (trap_debug)
 437                 printf("SYSCALL #%d pid:%u\n", sa->code, td->td_proc->p_pid);
 438 #endif
 439
 440         se = td->td_proc->p_sysent;
 441         /*
 442          * XXX
 443          * Shouldn't this go before switching on the code?
 444          */
 445         if (se->sv_mask)
 446                 sa->code &= se->sv_mask;
 447
 448         if (sa->code >= se->sv_size)
 449                 sa->callp = &se->sv_table[0];
 450         else
 451                 sa->callp = &se->sv_table[sa->code];
 452
 453         sa->narg = sa->callp->sy_narg;
 454
 455         if (sa->narg > nsaved) {
 456 #if defined(__mips_n32) || defined(__mips_n64)
 457                 /*
 458                  * XXX
 459                  * Is this right for new ABIs?  I think the 4 there
 460                  * should be 8, size there are 8 registers to skip,
 461                  * not 4, but I'm not certain.
 462                  */
 463 #ifdef COMPAT_FREEBSD32
 464                 if (!SV_PROC_FLAG(td->td_proc, SV_ILP32))
 465 #endif
 466                         printf("SYSCALL #%u pid:%u, narg (%u) > nsaved (%u).\n",
 467                             sa->code, td->td_proc->p_pid, sa->narg, nsaved);
 468 #endif
 469 #if (defined(__mips_n32) || defined(__mips_n64)) && defined(COMPAT_FREEBSD32)
 470                 if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) {
 471                         unsigned i;
 472                         int32_t arg;
 473
 474                         error = 0; /* XXX GCC is awful.  */
 475                         for (i = nsaved; i < sa->narg; i++) {
 476                                 error = copyin((caddr_t)(intptr_t)(locr0->sp +
 477                                     (4 + (i - nsaved)) * sizeof(int32_t)),
 478                                     (caddr_t)&arg, sizeof arg);
 479                                 if (error != 0)
 480                                         break;
 481                                 sa->args[i] = arg;
 482                         }
 483                 } else
 484 #endif
 485                 error = copyin((caddr_t)(intptr_t)(locr0->sp +
 486                     4 * sizeof(register_t)), (caddr_t)&sa->args[nsaved],
 487                    (u_int)(sa->narg - nsaved) * sizeof(register_t));
 488                 if (error != 0) {
 489                         locr0->v0 = error;
 490                         locr0->a3 = 1;
 491                 }
 492         } else
 493                 error = 0;
 494
 495         if (error == 0) {
 496                 td->td_retval[0] = 0;
 497                 td->td_retval[1] = locr0->v1;
 498         }
 499
 500         return (error);
 501 }
 502
 503 #undef __FBSDID
 504 #define __FBSDID(x)
 505 #include "../../kern/subr_syscall.c"
 506
 507 /*
 508  * Handle an exception.
 509  * Called from MipsKernGenException() or MipsUserGenException()
 510  * when a processor trap occurs.
 511  * In the case of a kernel trap, we return the pc where to resume if
 512  * p->p_addr->u_pcb.pcb_onfault is set, otherwise, return old pc.
 513  */
 514 register_t
 515 trap(struct trapframe *trapframe)
 516 {
 517         int type, usermode;
 518         int i = 0;
 519         unsigned ucode = 0;
 520         struct thread *td = curthread;
 521         struct proc *p = curproc;
 522         vm_prot_t ftype;
 523         pmap_t pmap;
 524         int access_type;
 525         ksiginfo_t ksi;
 526         char *msg = NULL;
 527         intptr_t addr = 0;
 528         register_t pc;
 529         int cop, error;
 530         register_t *frame_regs;
 531
 532         trapdebug_enter(trapframe, 0);
 533 #ifdef KDB
 534         if (kdb_active) {
 535                 kdb_reenter();
 536                 return (0);
 537         }
 538 #endif
 539         type = (trapframe->cause & MIPS_CR_EXC_CODE) >> MIPS_CR_EXC_CODE_SHIFT;
 540         if (TRAPF_USERMODE(trapframe)) {
 541                 type |= T_USER;
 542                 usermode = 1;
 543         } else {
 544                 usermode = 0;
 545         }
 546
 547         /*
 548          * Enable hardware interrupts if they were on before the trap. If it
 549          * was off disable all so we don't accidently enable it when doing a
 550          * return to userland.
 551          */
 552         if (trapframe->sr & MIPS_SR_INT_IE) {
 553                 set_intr_mask(trapframe->sr & MIPS_SR_INT_MASK);
 554                 intr_enable();
 555         } else {
 556                 intr_disable();
 557         }
 558
 559 #ifdef TRAP_DEBUG
 560         if (trap_debug) {
 561                 static vm_offset_t last_badvaddr = 0;
 562                 static vm_offset_t this_badvaddr = 0;
 563                 static int count = 0;
 564                 u_int32_t pid;
 565
 566                 printf("trap type %x (%s - ", type,
 567                     trap_type[type & (~T_USER)]);
 568
 569                 if (type & T_USER)
 570                         printf("user mode)\n");
 571                 else
 572                         printf("kernel mode)\n");
 573
 574 #ifdef SMP
 575                 printf("cpuid = %d\n", PCPU_GET(cpuid));
 576 #endif
 577                 pid = mips_rd_entryhi() & TLBHI_ASID_MASK;
 578                 printf("badaddr = %#jx, pc = %#jx, ra = %#jx, sp = %#jx, sr = %jx, pid = %d, ASID = %u\n",
 579                     (intmax_t)trapframe->badvaddr, (intmax_t)trapframe->pc, (intmax_t)trapframe->ra,
 580                     (intmax_t)trapframe->sp, (intmax_t)trapframe->sr,
 581                     (curproc ? curproc->p_pid : -1), pid);
 582
 583                 switch (type & ~T_USER) {
 584                 case T_TLB_MOD:
 585                 case T_TLB_LD_MISS:
 586                 case T_TLB_ST_MISS:
 587                 case T_ADDR_ERR_LD:
 588                 case T_ADDR_ERR_ST:
 589                         this_badvaddr = trapframe->badvaddr;
 590                         break;
 591                 case T_SYSCALL:
 592                         this_badvaddr = trapframe->ra;
 593                         break;
 594                 default:
 595                         this_badvaddr = trapframe->pc;
 596                         break;
 597                 }
 598                 if ((last_badvaddr == this_badvaddr) &&
 599                     ((type & ~T_USER) != T_SYSCALL) &&
 600                     ((type & ~T_USER) != T_COP_UNUSABLE)) {
 601                         if (++count == 3) {
 602                                 trap_frame_dump(trapframe);
 603                                 panic("too many faults at %p\n", (void *)last_badvaddr);
 604                         }
 605                 } else {
 606                         last_badvaddr = this_badvaddr;
 607                         count = 0;
 608                 }
 609         }
 610 #endif
 611
 612 #ifdef KDTRACE_HOOKS
 613         /*
 614          * A trap can occur while DTrace executes a probe. Before
 615          * executing the probe, DTrace blocks re-scheduling and sets
 616          * a flag in its per-cpu flags to indicate that it doesn't
 617          * want to fault. On returning from the probe, the no-fault
 618          * flag is cleared and finally re-scheduling is enabled.
 619          *
 620          * If the DTrace kernel module has registered a trap handler,
 621          * call it and if it returns non-zero, assume that it has
 622          * handled the trap and modified the trap frame so that this
 623          * function can return normally.
 624          */
 625         /*
 626          * XXXDTRACE: add pid probe handler here (if ever)
 627          */
 628         if (!usermode) {
 629                 if (dtrace_trap_func != NULL &&
 630                     (*dtrace_trap_func)(trapframe, type) != 0)
 631                         return (trapframe->pc);
 632         }
 633 #endif
 634
 635         switch (type) {
 636         case T_MCHECK:
 637 #ifdef DDB
 638                 kdb_trap(type, 0, trapframe);
 639 #endif
 640                 panic("MCHECK\n");
 641                 break;
 642         case T_TLB_MOD:
 643                 /* check for kernel address */
 644                 if (KERNLAND(trapframe->badvaddr)) {
 645                         if (pmap_emulate_modified(kernel_pmap,
 646                             trapframe->badvaddr) != 0) {
 647                                 ftype = VM_PROT_WRITE;
 648                                 goto kernel_fault;
 649                         }
 650                         return (trapframe->pc);
 651                 }
 652                 /* FALLTHROUGH */
 653
 654         case T_TLB_MOD + T_USER:
 655                 pmap = &p->p_vmspace->vm_pmap;
 656                 if (pmap_emulate_modified(pmap, trapframe->badvaddr) != 0) {
 657                         ftype = VM_PROT_WRITE;
 658                         goto dofault;
 659                 }
 660                 if (!usermode)
 661                         return (trapframe->pc);
 662                 goto out;
 663
 664         case T_TLB_LD_MISS:
 665         case T_TLB_ST_MISS:
 666                 ftype = (type == T_TLB_ST_MISS) ? VM_PROT_WRITE : VM_PROT_READ;
 667                 /* check for kernel address */
 668                 if (KERNLAND(trapframe->badvaddr)) {
 669                         vm_offset_t va;
 670                         int rv;
 671
 672         kernel_fault:
 673                         va = trunc_page((vm_offset_t)trapframe->badvaddr);
 674                         rv = vm_fault(kernel_map, va, ftype, VM_FAULT_NORMAL);
 675                         if (rv == KERN_SUCCESS)
 676                                 return (trapframe->pc);
 677                         if (td->td_pcb->pcb_onfault != NULL) {
 678                                 pc = (register_t)(intptr_t)td->td_pcb->pcb_onfault;
 679                                 td->td_pcb->pcb_onfault = NULL;
 680                                 return (pc);
 681                         }
 682                         goto err;
 683                 }
 684
 685                 /*
 686                  * It is an error for the kernel to access user space except
 687                  * through the copyin/copyout routines.
 688                  */
 689                 if (td->td_pcb->pcb_onfault == NULL)
 690                         goto err;
 691
 692                 goto dofault;
 693
 694         case T_TLB_LD_MISS + T_USER:
 695                 ftype = VM_PROT_READ;
 696                 goto dofault;
 697
 698         case T_TLB_ST_MISS + T_USER:
 699                 ftype = VM_PROT_WRITE;
 700 dofault:
 701                 {
 702                         vm_offset_t va;
 703                         struct vmspace *vm;
 704                         vm_map_t map;
 705                         int rv = 0;
 706
 707                         vm = p->p_vmspace;
 708                         map = &vm->vm_map;
 709                         va = trunc_page((vm_offset_t)trapframe->badvaddr);
 710                         if (KERNLAND(trapframe->badvaddr)) {
 711                                 /*
 712                                  * Don't allow user-mode faults in kernel
 713                                  * address space.
 714                                  */
 715                                 goto nogo;
 716                         }
 717
 718                         rv = vm_fault(map, va, ftype, VM_FAULT_NORMAL);
 719                         /*
 720                          * XXXDTRACE: add dtrace_doubletrap_func here?
 721                          */
 722 #ifdef VMFAULT_TRACE
 723                         printf("vm_fault(%p (pmap %p), %p (%p), %x, %d) -> %x at pc %p\n",
 724                             map, &vm->vm_pmap, (void *)va, (void *)(intptr_t)trapframe->badvaddr,
 725                             ftype, VM_FAULT_NORMAL, rv, (void *)(intptr_t)trapframe->pc);
 726 #endif
 727
 728                         if (rv == KERN_SUCCESS) {
 729                                 if (!usermode) {
 730                                         return (trapframe->pc);
 731                                 }
 732                                 goto out;
 733                         }
 734         nogo:
 735                         if (!usermode) {
 736                                 if (td->td_pcb->pcb_onfault != NULL) {
 737                                         pc = (register_t)(intptr_t)td->td_pcb->pcb_onfault;
 738                                         td->td_pcb->pcb_onfault = NULL;
 739                                         return (pc);
 740                                 }
 741                                 goto err;
 742                         }
 743                         i = SIGSEGV;
 744                         if (rv == KERN_PROTECTION_FAILURE)
 745                                 ucode = SEGV_ACCERR;
 746                         else
 747                                 ucode = SEGV_MAPERR;
 748                         addr = trapframe->pc;
 749
 750                         msg = "BAD_PAGE_FAULT";
 751                         log_bad_page_fault(msg, trapframe, type);
 752
 753                         break;
 754                 }
 755
 756         case T_ADDR_ERR_LD + T_USER:    /* misaligned or kseg access */
 757         case T_ADDR_ERR_ST + T_USER:    /* misaligned or kseg access */
 758                 if (trapframe->badvaddr < 0 ||
 759                     trapframe->badvaddr >= VM_MAXUSER_ADDRESS) {
 760                         msg = "ADDRESS_SPACE_ERR";
 761                 } else if (allow_unaligned_acc) {
 762                         int mode;
 763
 764                         if (type == (T_ADDR_ERR_LD + T_USER))
 765                                 mode = VM_PROT_READ;
 766                         else
 767                                 mode = VM_PROT_WRITE;
 768
 769                         access_type = emulate_unaligned_access(trapframe, mode);
 770                         if (access_type != 0)
 771                                 goto out;
 772                         msg = "ALIGNMENT_FIX_ERR";
 773                 } else {
 774                         msg = "ADDRESS_ERR";
 775                 }
 776
 777                 /* FALL THROUGH */
 778
 779         case T_BUS_ERR_IFETCH + T_USER: /* BERR asserted to cpu */
 780         case T_BUS_ERR_LD_ST + T_USER:  /* BERR asserted to cpu */
 781                 ucode = 0;      /* XXX should be VM_PROT_something */
 782                 i = SIGBUS;
 783                 addr = trapframe->pc;
 784                 if (!msg)
 785                         msg = "BUS_ERR";
 786                 log_bad_page_fault(msg, trapframe, type);
 787                 break;
 788
 789         case T_SYSCALL + T_USER:
 790                 {
 791                         int error;
 792
 793                         td->td_sa.trapframe = trapframe;
 794                         error = syscallenter(td);
 795
 796 #if !defined(SMP) && (defined(DDB) || defined(DEBUG))
 797                         if (trp == trapdebug)
 798                                 trapdebug[TRAPSIZE - 1].code = td->td_sa.code;
 799                         else
 800                                 trp[-1].code = td->td_sa.code;
 801 #endif
 802                         trapdebug_enter(td->td_frame, -td->td_sa.code);
 803
 804                         /*
 805                          * The sync'ing of I & D caches for SYS_ptrace() is
 806                          * done by procfs_domem() through procfs_rwmem()
 807                          * instead of being done here under a special check
 808                          * for SYS_ptrace().
 809                          */
 810                         syscallret(td, error);
 811                         return (trapframe->pc);
 812                 }
 813
 814 #if defined(KDTRACE_HOOKS) || defined(DDB)
 815         case T_BREAK:
 816 #ifdef KDTRACE_HOOKS
 817                 if (!usermode && dtrace_invop_jump_addr != 0) {
 818                         dtrace_invop_jump_addr(trapframe);
 819                         return (trapframe->pc);
 820                 }
 821 #endif
 822 #ifdef DDB
 823                 kdb_trap(type, 0, trapframe);
 824                 return (trapframe->pc);
 825 #endif
 826 #endif
 827
 828         case T_BREAK + T_USER:
 829                 {
 830                         intptr_t va;
 831                         uint32_t instr;
 832
 833                         i = SIGTRAP;
 834                         ucode = TRAP_BRKPT;
 835                         addr = trapframe->pc;
 836
 837                         /* compute address of break instruction */
 838                         va = trapframe->pc;
 839                         if (DELAYBRANCH(trapframe->cause))
 840                                 va += sizeof(int);
 841
 842                         if (td->td_md.md_ss_addr != va)
 843                                 break;
 844
 845                         /* read break instruction */
 846                         instr = fuword32((caddr_t)va);
 847
 848                         if (instr != MIPS_BREAK_SSTEP)
 849                                 break;
 850
 851                         CTR3(KTR_PTRACE,
 852                             "trap: tid %d, single step at %#lx: %#08x",
 853                             td->td_tid, va, instr);
 854                         PROC_LOCK(p);
 855                         _PHOLD(p);
 856                         error = ptrace_clear_single_step(td);
 857                         _PRELE(p);
 858                         PROC_UNLOCK(p);
 859                         if (error == 0)
 860                                 ucode = TRAP_TRACE;
 861                         break;
 862                 }
 863
 864         case T_IWATCH + T_USER:
 865         case T_DWATCH + T_USER:
 866                 {
 867                         intptr_t va;
 868
 869                         /* compute address of trapped instruction */
 870                         va = trapframe->pc;
 871                         if (DELAYBRANCH(trapframe->cause))
 872                                 va += sizeof(int);
 873                         printf("watch exception @ %p\n", (void *)va);
 874                         i = SIGTRAP;
 875                         ucode = TRAP_BRKPT;
 876                         addr = va;
 877                         break;
 878                 }
 879
 880         case T_TRAP + T_USER:
 881                 {
 882                         intptr_t va;
 883                         struct trapframe *locr0 = td->td_frame;
 884
 885                         /* compute address of trap instruction */
 886                         va = trapframe->pc;
 887                         if (DELAYBRANCH(trapframe->cause))
 888                                 va += sizeof(int);
 889
 890                         if (DELAYBRANCH(trapframe->cause)) {    /* Check BD bit */
 891                                 locr0->pc = MipsEmulateBranch(locr0, trapframe->pc, 0,
 892                                     0);
 893                         } else {
 894                                 locr0->pc += sizeof(int);
 895                         }
 896                         addr = va;
 897                         i = SIGEMT;     /* Stuff it with something for now */
 898                         break;
 899                 }
 900
 901         case T_RES_INST + T_USER:
 902                 {
 903                         InstFmt inst;
 904                         inst = *(InstFmt *)(intptr_t)trapframe->pc;
 905                         switch (inst.RType.op) {
 906                         case OP_SPECIAL3:
 907                                 switch (inst.RType.func) {
 908                                 case OP_RDHWR:
 909                                         /* Register 29 used for TLS */
 910                                         if (inst.RType.rd == 29) {
 911                                                 frame_regs = &(trapframe->zero);
 912                                                 frame_regs[inst.RType.rt] = (register_t)(intptr_t)td->td_md.md_tls;
 913                                                 frame_regs[inst.RType.rt] += td->td_md.md_tls_tcb_offset;
 914                                                 trapframe->pc += sizeof(int);
 915                                                 goto out;
 916                                         }
 917                                 break;
 918                                 }
 919                         break;
 920                         }
 921
 922                         log_illegal_instruction("RES_INST", trapframe);
 923                         i = SIGILL;
 924                         addr = trapframe->pc;
 925                 }
 926                 break;
 927         case T_C2E:
 928         case T_C2E + T_USER:
 929                 goto err;
 930                 break;
 931         case T_COP_UNUSABLE:
 932 #ifdef  CPU_CNMIPS
 933                 cop = (trapframe->cause & MIPS_CR_COP_ERR) >> MIPS_CR_COP_ERR_SHIFT;
 934                 /* Handle only COP2 exception */
 935                 if (cop != 2)
 936                         goto err;
 937
 938                 addr = trapframe->pc;
 939                 /* save userland cop2 context if it has been touched */
 940                 if ((td->td_md.md_flags & MDTD_COP2USED) &&
 941                     (td->td_md.md_cop2owner == COP2_OWNER_USERLAND)) {
 942                         if (td->td_md.md_ucop2)
 943                                 octeon_cop2_save(td->td_md.md_ucop2);
 944                         else
 945                                 panic("COP2 was used in user mode but md_ucop2 is NULL");
 946                 }
 947
 948                 if (td->td_md.md_cop2 == NULL) {
 949                         td->td_md.md_cop2 = octeon_cop2_alloc_ctx();
 950                         if (td->td_md.md_cop2 == NULL)
 951                                 panic("Failed to allocate COP2 context");
 952                         memset(td->td_md.md_cop2, 0, sizeof(*td->td_md.md_cop2));
 953                 }
 954
 955                 octeon_cop2_restore(td->td_md.md_cop2);
 956
 957                 /* Make userland re-request its context */
 958                 td->td_frame->sr &= ~MIPS_SR_COP_2_BIT;
 959                 td->td_md.md_flags |= MDTD_COP2USED;
 960                 td->td_md.md_cop2owner = COP2_OWNER_KERNEL;
 961                 /* Enable COP2, it will be disabled in cpu_switch */
 962                 mips_wr_status(mips_rd_status() | MIPS_SR_COP_2_BIT);
 963                 return (trapframe->pc);
 964 #else
 965                 goto err;
 966                 break;
 967 #endif
 968
 969         case T_COP_UNUSABLE + T_USER:
 970                 cop = (trapframe->cause & MIPS_CR_COP_ERR) >> MIPS_CR_COP_ERR_SHIFT;
 971                 if (cop == 1) {
 972                         /* FP (COP1) instruction */
 973                         if (cpuinfo.fpu_id == 0) {
 974                                 log_illegal_instruction("COP1_UNUSABLE",
 975                                     trapframe);
 976                                 i = SIGILL;
 977                                 break;
 978                         }
 979                         addr = trapframe->pc;
 980                         MipsSwitchFPState(PCPU_GET(fpcurthread), td->td_frame);
 981                         PCPU_SET(fpcurthread, td);
 982 #if defined(__mips_n32) || defined(__mips_n64)
 983                         td->td_frame->sr |= MIPS_SR_COP_1_BIT | MIPS_SR_FR;
 984 #else
 985                         td->td_frame->sr |= MIPS_SR_COP_1_BIT;
 986 #endif
 987                         td->td_md.md_flags |= MDTD_FPUSED;
 988                         goto out;
 989                 }
 990 #ifdef  CPU_CNMIPS
 991                 else  if (cop == 2) {
 992                         addr = trapframe->pc;
 993                         if ((td->td_md.md_flags & MDTD_COP2USED) &&
 994                             (td->td_md.md_cop2owner == COP2_OWNER_KERNEL)) {
 995                                 if (td->td_md.md_cop2)
 996                                         octeon_cop2_save(td->td_md.md_cop2);
 997                                 else
 998                                         panic("COP2 was used in kernel mode but md_cop2 is NULL");
 999                         }
1000
1001                         if (td->td_md.md_ucop2 == NULL) {
1002                                 td->td_md.md_ucop2 = octeon_cop2_alloc_ctx();
1003                                 if (td->td_md.md_ucop2 == NULL)
1004                                         panic("Failed to allocate userland COP2 context");
1005                                 memset(td->td_md.md_ucop2, 0, sizeof(*td->td_md.md_ucop2));
1006                         }
1007
1008                         octeon_cop2_restore(td->td_md.md_ucop2);
1009
1010                         td->td_frame->sr |= MIPS_SR_COP_2_BIT;
1011                         td->td_md.md_flags |= MDTD_COP2USED;
1012                         td->td_md.md_cop2owner = COP2_OWNER_USERLAND;
1013                         goto out;
1014                 }
1015 #endif
1016                 else {
1017                         log_illegal_instruction("COPn_UNUSABLE", trapframe);
1018                         i = SIGILL;     /* only FPU instructions allowed */
1019                         break;
1020                 }
1021
1022         case T_FPE:
1023 #if !defined(SMP) && (defined(DDB) || defined(DEBUG))
1024                 trapDump("fpintr");
1025 #else
1026                 printf("FPU Trap: PC %#jx CR %x SR %x\n",
1027                     (intmax_t)trapframe->pc, (unsigned)trapframe->cause, (unsigned)trapframe->sr);
1028                 goto err;
1029 #endif
1030
1031         case T_FPE + T_USER:
1032                 if (!emulate_fp) {
1033                         i = SIGFPE;
1034                         addr = trapframe->pc;
1035                         break;
1036                 }
1037                 MipsFPTrap(trapframe->sr, trapframe->cause, trapframe->pc);
1038                 goto out;
1039
1040         case T_OVFLOW + T_USER:
1041                 i = SIGFPE;
1042                 addr = trapframe->pc;
1043                 break;
1044
1045         case T_ADDR_ERR_LD:     /* misaligned access */
1046         case T_ADDR_ERR_ST:     /* misaligned access */
1047 #ifdef TRAP_DEBUG
1048                 if (trap_debug) {
1049                         printf("+++ ADDR_ERR: type = %d, badvaddr = %#jx\n", type,
1050                             (intmax_t)trapframe->badvaddr);
1051                 }
1052 #endif
1053                 /* Only allow emulation on a user address */
1054                 if (allow_unaligned_acc &&
1055                     ((vm_offset_t)trapframe->badvaddr < VM_MAXUSER_ADDRESS)) {
1056                         int mode;
1057
1058                         if (type == T_ADDR_ERR_LD)
1059                                 mode = VM_PROT_READ;
1060                         else
1061                                 mode = VM_PROT_WRITE;
1062
1063                         access_type = emulate_unaligned_access(trapframe, mode);
1064                         if (access_type != 0)
1065                                 return (trapframe->pc);
1066                 }
1067                 /* FALLTHROUGH */
1068
1069         case T_BUS_ERR_LD_ST:   /* BERR asserted to cpu */
1070                 if (td->td_pcb->pcb_onfault != NULL) {
1071                         pc = (register_t)(intptr_t)td->td_pcb->pcb_onfault;
1072                         td->td_pcb->pcb_onfault = NULL;
1073                         return (pc);
1074                 }
1075
1076                 /* FALLTHROUGH */
1077
1078         default:
1079 err:
1080
1081 #if !defined(SMP) && defined(DEBUG)
1082                 trapDump("trap");
1083 #endif
1084 #ifdef SMP
1085                 printf("cpu:%d-", PCPU_GET(cpuid));
1086 #endif
1087                 printf("Trap cause = %d (%s - ", type,
1088                     trap_type[type & (~T_USER)]);
1089
1090                 if (type & T_USER)
1091                         printf("user mode)\n");
1092                 else
1093                         printf("kernel mode)\n");
1094
1095 #ifdef TRAP_DEBUG
1096                 if (trap_debug)
1097                         printf("badvaddr = %#jx, pc = %#jx, ra = %#jx, sr = %#jxx\n",
1098                                (intmax_t)trapframe->badvaddr, (intmax_t)trapframe->pc, (intmax_t)trapframe->ra,
1099                                (intmax_t)trapframe->sr);
1100 #endif
1101
1102 #ifdef KDB
1103                 if (debugger_on_panic) {
1104                         kdb_why = KDB_WHY_TRAP;
1105                         kdb_trap(type, 0, trapframe);
1106                         kdb_why = KDB_WHY_UNSET;
1107                 }
1108 #endif
1109                 panic("trap");
1110         }
1111         td->td_frame->pc = trapframe->pc;
1112         td->td_frame->cause = trapframe->cause;
1113         td->td_frame->badvaddr = trapframe->badvaddr;
1114         ksiginfo_init_trap(&ksi);
1115         ksi.ksi_signo = i;
1116         ksi.ksi_code = ucode;
1117         ksi.ksi_addr = (void *)addr;
1118         ksi.ksi_trapno = type;
1119         trapsignal(td, &ksi);
1120 out:
1121
1122         /*
1123          * Note: we should only get here if returning to user mode.
1124          */
1125         userret(td, trapframe);
1126         return (trapframe->pc);
1127 }
1128
1129 #if !defined(SMP) && (defined(DDB) || defined(DEBUG))
1130 void
1131 trapDump(char *msg)
1132 {
1133         register_t s;
1134         int i;
1135
1136         s = intr_disable();
1137         printf("trapDump(%s)\n", msg);
1138         for (i = 0; i < TRAPSIZE; i++) {
1139                 if (trp == trapdebug) {
1140                         trp = &trapdebug[TRAPSIZE - 1];
1141                 } else {
1142                         trp--;
1143                 }
1144
1145                 if (trp->cause == 0)
1146                         break;
1147
1148                 printf("%s: ADR %jx PC %jx CR %jx SR %jx\n",
1149                     trap_type[(trp->cause & MIPS_CR_EXC_CODE) >>
1150                         MIPS_CR_EXC_CODE_SHIFT],
1151                     (intmax_t)trp->vadr, (intmax_t)trp->pc,
1152                     (intmax_t)trp->cause, (intmax_t)trp->status);
1153
1154                 printf("   RA %jx SP %jx code %d\n", (intmax_t)trp->ra,
1155                     (intmax_t)trp->sp, (int)trp->code);
1156         }
1157         intr_restore(s);
1158 }
1159 #endif
1160
1161
1162 /*
1163  * Return the resulting PC as if the branch was executed.
1164  */
1165 uintptr_t
1166 MipsEmulateBranch(struct trapframe *framePtr, uintptr_t instPC, int fpcCSR,
1167     uintptr_t instptr)
1168 {
1169         InstFmt inst;
1170         register_t *regsPtr = (register_t *) framePtr;
1171         uintptr_t retAddr = 0;
1172         int condition;
1173
1174 #define GetBranchDest(InstPtr, inst) \
1175         (InstPtr + 4 + ((short)inst.IType.imm << 2))
1176
1177
1178         if (instptr) {
1179                 if (instptr < MIPS_KSEG0_START)
1180                         inst.word = fuword32((void *)instptr);
1181                 else
1182                         inst = *(InstFmt *) instptr;
1183         } else {
1184                 if ((vm_offset_t)instPC < MIPS_KSEG0_START)
1185                         inst.word = fuword32((void *)instPC);
1186                 else
1187                         inst = *(InstFmt *) instPC;
1188         }
1189
1190         switch ((int)inst.JType.op) {
1191         case OP_SPECIAL:
1192                 switch ((int)inst.RType.func) {
1193                 case OP_JR:
1194                 case OP_JALR:
1195                         retAddr = regsPtr[inst.RType.rs];
1196                         break;
1197
1198                 default:
1199                         retAddr = instPC + 4;
1200                         break;
1201                 }
1202                 break;
1203
1204         case OP_BCOND:
1205                 switch ((int)inst.IType.rt) {
1206                 case OP_BLTZ:
1207                 case OP_BLTZL:
1208                 case OP_BLTZAL:
1209                 case OP_BLTZALL:
1210                         if ((int)(regsPtr[inst.RType.rs]) < 0)
1211                                 retAddr = GetBranchDest(instPC, inst);
1212                         else
1213                                 retAddr = instPC + 8;
1214                         break;
1215
1216                 case OP_BGEZ:
1217                 case OP_BGEZL:
1218                 case OP_BGEZAL:
1219                 case OP_BGEZALL:
1220                         if ((int)(regsPtr[inst.RType.rs]) >= 0)
1221                                 retAddr = GetBranchDest(instPC, inst);
1222                         else
1223                                 retAddr = instPC + 8;
1224                         break;
1225
1226                 case OP_TGEI:
1227                 case OP_TGEIU:
1228                 case OP_TLTI:
1229                 case OP_TLTIU:
1230                 case OP_TEQI:
1231                 case OP_TNEI:
1232                         retAddr = instPC + 4;   /* Like syscall... */
1233                         break;
1234
1235                 default:
1236                         panic("MipsEmulateBranch: Bad branch cond");
1237                 }
1238                 break;
1239
1240         case OP_J:
1241         case OP_JAL:
1242                 retAddr = (inst.JType.target << 2) |
1243                     ((unsigned)(instPC + 4) & 0xF0000000);
1244                 break;
1245
1246         case OP_BEQ:
1247         case OP_BEQL:
1248                 if (regsPtr[inst.RType.rs] == regsPtr[inst.RType.rt])
1249                         retAddr = GetBranchDest(instPC, inst);
1250                 else
1251                         retAddr = instPC + 8;
1252                 break;
1253
1254         case OP_BNE:
1255         case OP_BNEL:
1256                 if (regsPtr[inst.RType.rs] != regsPtr[inst.RType.rt])
1257                         retAddr = GetBranchDest(instPC, inst);
1258                 else
1259                         retAddr = instPC + 8;
1260                 break;
1261
1262         case OP_BLEZ:
1263         case OP_BLEZL:
1264                 if ((int)(regsPtr[inst.RType.rs]) <= 0)
1265                         retAddr = GetBranchDest(instPC, inst);
1266                 else
1267                         retAddr = instPC + 8;
1268                 break;
1269
1270         case OP_BGTZ:
1271         case OP_BGTZL:
1272                 if ((int)(regsPtr[inst.RType.rs]) > 0)
1273                         retAddr = GetBranchDest(instPC, inst);
1274                 else
1275                         retAddr = instPC + 8;
1276                 break;
1277
1278         case OP_COP1:
1279                 switch (inst.RType.rs) {
1280                 case OP_BCx:
1281                 case OP_BCy:
1282                         if ((inst.RType.rt & COPz_BC_TF_MASK) == COPz_BC_TRUE)
1283                                 condition = fpcCSR & MIPS_FPU_COND_BIT;
1284                         else
1285                                 condition = !(fpcCSR & MIPS_FPU_COND_BIT);
1286                         if (condition)
1287                                 retAddr = GetBranchDest(instPC, inst);
1288                         else
1289                                 retAddr = instPC + 8;
1290                         break;
1291
1292                 default:
1293                         retAddr = instPC + 4;
1294                 }
1295                 break;
1296
1297         default:
1298                 retAddr = instPC + 4;
1299         }
1300         return (retAddr);
1301 }
1302
1303 static void
1304 log_frame_dump(struct trapframe *frame)
1305 {
1306         log(LOG_ERR, "Trapframe Register Dump:\n");
1307         log(LOG_ERR, "\tzero: %#jx\tat: %#jx\tv0: %#jx\tv1: %#jx\n",
1308             (intmax_t)0, (intmax_t)frame->ast, (intmax_t)frame->v0, (intmax_t)frame->v1);
1309
1310         log(LOG_ERR, "\ta0: %#jx\ta1: %#jx\ta2: %#jx\ta3: %#jx\n",
1311             (intmax_t)frame->a0, (intmax_t)frame->a1, (intmax_t)frame->a2, (intmax_t)frame->a3);
1312
1313 #if defined(__mips_n32) || defined(__mips_n64)
1314         log(LOG_ERR, "\ta4: %#jx\ta5: %#jx\ta6: %#jx\ta6: %#jx\n",
1315             (intmax_t)frame->a4, (intmax_t)frame->a5, (intmax_t)frame->a6, (intmax_t)frame->a7);
1316
1317         log(LOG_ERR, "\tt0: %#jx\tt1: %#jx\tt2: %#jx\tt3: %#jx\n",
1318             (intmax_t)frame->t0, (intmax_t)frame->t1, (intmax_t)frame->t2, (intmax_t)frame->t3);
1319 #else
1320         log(LOG_ERR, "\tt0: %#jx\tt1: %#jx\tt2: %#jx\tt3: %#jx\n",
1321             (intmax_t)frame->t0, (intmax_t)frame->t1, (intmax_t)frame->t2, (intmax_t)frame->t3);
1322
1323         log(LOG_ERR, "\tt4: %#jx\tt5: %#jx\tt6: %#jx\tt7: %#jx\n",
1324             (intmax_t)frame->t4, (intmax_t)frame->t5, (intmax_t)frame->t6, (intmax_t)frame->t7);
1325 #endif
1326         log(LOG_ERR, "\tt8: %#jx\tt9: %#jx\ts0: %#jx\ts1: %#jx\n",
1327             (intmax_t)frame->t8, (intmax_t)frame->t9, (intmax_t)frame->s0, (intmax_t)frame->s1);
1328
1329         log(LOG_ERR, "\ts2: %#jx\ts3: %#jx\ts4: %#jx\ts5: %#jx\n",
1330             (intmax_t)frame->s2, (intmax_t)frame->s3, (intmax_t)frame->s4, (intmax_t)frame->s5);
1331
1332         log(LOG_ERR, "\ts6: %#jx\ts7: %#jx\tk0: %#jx\tk1: %#jx\n",
1333             (intmax_t)frame->s6, (intmax_t)frame->s7, (intmax_t)frame->k0, (intmax_t)frame->k1);
1334
1335         log(LOG_ERR, "\tgp: %#jx\tsp: %#jx\ts8: %#jx\tra: %#jx\n",
1336             (intmax_t)frame->gp, (intmax_t)frame->sp, (intmax_t)frame->s8, (intmax_t)frame->ra);
1337
1338         log(LOG_ERR, "\tsr: %#jx\tmullo: %#jx\tmulhi: %#jx\tbadvaddr: %#jx\n",
1339             (intmax_t)frame->sr, (intmax_t)frame->mullo, (intmax_t)frame->mulhi, (intmax_t)frame->badvaddr);
1340
1341         log(LOG_ERR, "\tcause: %#jx\tpc: %#jx\n",
1342             (intmax_t)frame->cause, (intmax_t)frame->pc);
1343 }
1344
1345 #ifdef TRAP_DEBUG
1346 static void
1347 trap_frame_dump(struct trapframe *frame)
1348 {
1349         printf("Trapframe Register Dump:\n");
1350         printf("\tzero: %#jx\tat: %#jx\tv0: %#jx\tv1: %#jx\n",
1351             (intmax_t)0, (intmax_t)frame->ast, (intmax_t)frame->v0, (intmax_t)frame->v1);
1352
1353         printf("\ta0: %#jx\ta1: %#jx\ta2: %#jx\ta3: %#jx\n",
1354             (intmax_t)frame->a0, (intmax_t)frame->a1, (intmax_t)frame->a2, (intmax_t)frame->a3);
1355 #if defined(__mips_n32) || defined(__mips_n64)
1356         printf("\ta4: %#jx\ta5: %#jx\ta6: %#jx\ta7: %#jx\n",
1357             (intmax_t)frame->a4, (intmax_t)frame->a5, (intmax_t)frame->a6, (intmax_t)frame->a7);
1358
1359         printf("\tt0: %#jx\tt1: %#jx\tt2: %#jx\tt3: %#jx\n",
1360             (intmax_t)frame->t0, (intmax_t)frame->t1, (intmax_t)frame->t2, (intmax_t)frame->t3);
1361 #else
1362         printf("\tt0: %#jx\tt1: %#jx\tt2: %#jx\tt3: %#jx\n",
1363             (intmax_t)frame->t0, (intmax_t)frame->t1, (intmax_t)frame->t2, (intmax_t)frame->t3);
1364
1365         printf("\tt4: %#jx\tt5: %#jx\tt6: %#jx\tt7: %#jx\n",
1366             (intmax_t)frame->t4, (intmax_t)frame->t5, (intmax_t)frame->t6, (intmax_t)frame->t7);
1367 #endif
1368         printf("\tt8: %#jx\tt9: %#jx\ts0: %#jx\ts1: %#jx\n",
1369             (intmax_t)frame->t8, (intmax_t)frame->t9, (intmax_t)frame->s0, (intmax_t)frame->s1);
1370
1371         printf("\ts2: %#jx\ts3: %#jx\ts4: %#jx\ts5: %#jx\n",
1372             (intmax_t)frame->s2, (intmax_t)frame->s3, (intmax_t)frame->s4, (intmax_t)frame->s5);
1373
1374         printf("\ts6: %#jx\ts7: %#jx\tk0: %#jx\tk1: %#jx\n",
1375             (intmax_t)frame->s6, (intmax_t)frame->s7, (intmax_t)frame->k0, (intmax_t)frame->k1);
1376
1377         printf("\tgp: %#jx\tsp: %#jx\ts8: %#jx\tra: %#jx\n",
1378             (intmax_t)frame->gp, (intmax_t)frame->sp, (intmax_t)frame->s8, (intmax_t)frame->ra);
1379
1380         printf("\tsr: %#jx\tmullo: %#jx\tmulhi: %#jx\tbadvaddr: %#jx\n",
1381             (intmax_t)frame->sr, (intmax_t)frame->mullo, (intmax_t)frame->mulhi, (intmax_t)frame->badvaddr);
1382
1383         printf("\tcause: %#jx\tpc: %#jx\n",
1384             (intmax_t)frame->cause, (intmax_t)frame->pc);
1385 }
1386
1387 #endif
1388
1389
1390 static void
1391 get_mapping_info(vm_offset_t va, pd_entry_t **pdepp, pt_entry_t **ptepp)
1392 {
1393         pt_entry_t *ptep;
1394         pd_entry_t *pdep;
1395         struct proc *p = curproc;
1396
1397         pdep = (&(p->p_vmspace->vm_pmap.pm_segtab[(va >> SEGSHIFT) & (NPDEPG - 1)]));
1398         if (*pdep)
1399                 ptep = pmap_pte(&p->p_vmspace->vm_pmap, va);
1400         else
1401                 ptep = (pt_entry_t *)0;
1402
1403         *pdepp = pdep;
1404         *ptepp = ptep;
1405 }
1406
1407 static void
1408 log_illegal_instruction(const char *msg, struct trapframe *frame)
1409 {
1410         pt_entry_t *ptep;
1411         pd_entry_t *pdep;
1412         unsigned int *addr;
1413         struct thread *td;
1414         struct proc *p;
1415         register_t pc;
1416
1417         td = curthread;
1418         p = td->td_proc;
1419
1420 #ifdef SMP
1421         printf("cpuid = %d\n", PCPU_GET(cpuid));
1422 #endif
1423         pc = frame->pc + (DELAYBRANCH(frame->cause) ? 4 : 0);
1424         log(LOG_ERR, "%s: pid %d tid %ld (%s), uid %d: pc %#jx ra %#jx\n",
1425             msg, p->p_pid, (long)td->td_tid, p->p_comm,
1426             p->p_ucred ? p->p_ucred->cr_uid : -1,
1427             (intmax_t)pc,
1428             (intmax_t)frame->ra);
1429
1430         /* log registers in trap frame */
1431         log_frame_dump(frame);
1432
1433         get_mapping_info((vm_offset_t)pc, &pdep, &ptep);
1434
1435         /*
1436          * Dump a few words around faulting instruction, if the addres is
1437          * valid.
1438          */
1439         if (!(pc & 3) &&
1440             useracc((caddr_t)(intptr_t)pc, sizeof(int) * 4, VM_PROT_READ)) {
1441                 /* dump page table entry for faulting instruction */
1442                 log(LOG_ERR, "Page table info for pc address %#jx: pde = %p, pte = %#jx\n",
1443                     (intmax_t)pc, (void *)(intptr_t)*pdep, (uintmax_t)(ptep ? *ptep : 0));
1444
1445                 addr = (unsigned int *)(intptr_t)pc;
1446                 log(LOG_ERR, "Dumping 4 words starting at pc address %p: \n",
1447                     addr);
1448                 log(LOG_ERR, "%08x %08x %08x %08x\n",
1449                     addr[0], addr[1], addr[2], addr[3]);
1450         } else {
1451                 log(LOG_ERR, "pc address %#jx is inaccessible, pde = %p, pte = %#jx\n",
1452                     (intmax_t)pc, (void *)(intptr_t)*pdep, (uintmax_t)(ptep ? *ptep : 0));
1453         }
1454 }
1455
1456 static void
1457 log_bad_page_fault(char *msg, struct trapframe *frame, int trap_type)
1458 {
1459         pt_entry_t *ptep;
1460         pd_entry_t *pdep;
1461         unsigned int *addr;
1462         struct thread *td;
1463         struct proc *p;
1464         char *read_or_write;
1465         register_t pc;
1466
1467         trap_type &= ~T_USER;
1468
1469         td = curthread;
1470         p = td->td_proc;
1471
1472 #ifdef SMP
1473         printf("cpuid = %d\n", PCPU_GET(cpuid));
1474 #endif
1475         switch (trap_type) {
1476         case T_TLB_MOD:
1477         case T_TLB_ST_MISS:
1478         case T_ADDR_ERR_ST:
1479                 read_or_write = "write";
1480                 break;
1481         case T_TLB_LD_MISS:
1482         case T_ADDR_ERR_LD:
1483         case T_BUS_ERR_IFETCH:
1484                 read_or_write = "read";
1485                 break;
1486         default:
1487                 read_or_write = "unknown";
1488         }
1489
1490         pc = frame->pc + (DELAYBRANCH(frame->cause) ? 4 : 0);
1491         log(LOG_ERR, "%s: pid %d tid %ld (%s), uid %d: pc %#jx got a %s fault "
1492             "(type %#x) at %#jx\n",
1493             msg, p->p_pid, (long)td->td_tid, p->p_comm,
1494             p->p_ucred ? p->p_ucred->cr_uid : -1,
1495             (intmax_t)pc,
1496             read_or_write,
1497             trap_type,
1498             (intmax_t)frame->badvaddr);
1499
1500         /* log registers in trap frame */
1501         log_frame_dump(frame);
1502
1503         get_mapping_info((vm_offset_t)pc, &pdep, &ptep);
1504
1505         /*
1506          * Dump a few words around faulting instruction, if the addres is
1507          * valid.
1508          */
1509         if (!(pc & 3) && (pc != frame->badvaddr) &&
1510             (trap_type != T_BUS_ERR_IFETCH) &&
1511             useracc((caddr_t)(intptr_t)pc, sizeof(int) * 4, VM_PROT_READ)) {
1512                 /* dump page table entry for faulting instruction */
1513                 log(LOG_ERR, "Page table info for pc address %#jx: pde = %p, pte = %#jx\n",
1514                     (intmax_t)pc, (void *)(intptr_t)*pdep, (uintmax_t)(ptep ? *ptep : 0));
1515
1516                 addr = (unsigned int *)(intptr_t)pc;
1517                 log(LOG_ERR, "Dumping 4 words starting at pc address %p: \n",
1518                     addr);
1519                 log(LOG_ERR, "%08x %08x %08x %08x\n",
1520                     addr[0], addr[1], addr[2], addr[3]);
1521         } else {
1522                 log(LOG_ERR, "pc address %#jx is inaccessible, pde = %p, pte = %#jx\n",
1523                     (intmax_t)pc, (void *)(intptr_t)*pdep, (uintmax_t)(ptep ? *ptep : 0));
1524         }
1525
1526         get_mapping_info((vm_offset_t)frame->badvaddr, &pdep, &ptep);
1527         log(LOG_ERR, "Page table info for bad address %#jx: pde = %p, pte = %#jx\n",
1528             (intmax_t)frame->badvaddr, (void *)(intptr_t)*pdep, (uintmax_t)(ptep ? *ptep : 0));
1529 }
1530
1531
1532 /*
1533  * Unaligned load/store emulation
1534  */
1535 static int
1536 mips_unaligned_load_store(struct trapframe *frame, int mode, register_t addr, register_t pc)
1537 {
1538         register_t *reg = (register_t *) frame;
1539         u_int32_t inst = *((u_int32_t *)(intptr_t)pc);
1540         register_t value_msb, value;
1541         unsigned size;
1542
1543         /*
1544          * ADDR_ERR faults have higher priority than TLB
1545          * Miss faults.  Therefore, it is necessary to
1546          * verify that the faulting address is a valid
1547          * virtual address within the process' address space
1548          * before trying to emulate the unaligned access.
1549          */
1550         switch (MIPS_INST_OPCODE(inst)) {
1551         case OP_LHU: case OP_LH:
1552         case OP_SH:
1553                 size = 2;
1554                 break;
1555         case OP_LWU: case OP_LW:
1556         case OP_SW:
1557                 size = 4;
1558                 break;
1559         case OP_LD:
1560         case OP_SD:
1561                 size = 8;
1562                 break;
1563         default:
1564                 printf("%s: unhandled opcode in address error: %#x\n", __func__, MIPS_INST_OPCODE(inst));
1565                 return (0);
1566         }
1567
1568         if (!useracc((void *)rounddown2((vm_offset_t)addr, size), size * 2, mode))
1569                 return (0);
1570
1571         /*
1572          * XXX
1573          * Handle LL/SC LLD/SCD.
1574          */
1575         switch (MIPS_INST_OPCODE(inst)) {
1576         case OP_LHU:
1577                 KASSERT(mode == VM_PROT_READ, ("access mode must be read for load instruction."));
1578                 lbu_macro(value_msb, addr);
1579                 addr += 1;
1580                 lbu_macro(value, addr);
1581                 value |= value_msb << 8;
1582                 reg[MIPS_INST_RT(inst)] = value;
1583                 return (MIPS_LHU_ACCESS);
1584
1585         case OP_LH:
1586                 KASSERT(mode == VM_PROT_READ, ("access mode must be read for load instruction."));
1587                 lb_macro(value_msb, addr);
1588                 addr += 1;
1589                 lbu_macro(value, addr);
1590                 value |= value_msb << 8;
1591                 reg[MIPS_INST_RT(inst)] = value;
1592                 return (MIPS_LH_ACCESS);
1593
1594         case OP_LWU:
1595                 KASSERT(mode == VM_PROT_READ, ("access mode must be read for load instruction."));
1596                 lwl_macro(value, addr);
1597                 addr += 3;
1598                 lwr_macro(value, addr);
1599                 value &= 0xffffffff;
1600                 reg[MIPS_INST_RT(inst)] = value;
1601                 return (MIPS_LWU_ACCESS);
1602
1603         case OP_LW:
1604                 KASSERT(mode == VM_PROT_READ, ("access mode must be read for load instruction."));
1605                 lwl_macro(value, addr);
1606                 addr += 3;
1607                 lwr_macro(value, addr);
1608                 reg[MIPS_INST_RT(inst)] = value;
1609                 return (MIPS_LW_ACCESS);
1610
1611 #if defined(__mips_n32) || defined(__mips_n64)
1612         case OP_LD:
1613                 KASSERT(mode == VM_PROT_READ, ("access mode must be read for load instruction."));
1614                 ldl_macro(value, addr);
1615                 addr += 7;
1616                 ldr_macro(value, addr);
1617                 reg[MIPS_INST_RT(inst)] = value;
1618                 return (MIPS_LD_ACCESS);
1619 #endif
1620
1621         case OP_SH:
1622                 KASSERT(mode == VM_PROT_WRITE, ("access mode must be write for store instruction."));
1623                 value = reg[MIPS_INST_RT(inst)];
1624                 value_msb = value >> 8;
1625                 sb_macro(value_msb, addr);
1626                 addr += 1;
1627                 sb_macro(value, addr);
1628                 return (MIPS_SH_ACCESS);
1629
1630         case OP_SW:
1631                 KASSERT(mode == VM_PROT_WRITE, ("access mode must be write for store instruction."));
1632                 value = reg[MIPS_INST_RT(inst)];
1633                 swl_macro(value, addr);
1634                 addr += 3;
1635                 swr_macro(value, addr);
1636                 return (MIPS_SW_ACCESS);
1637
1638 #if defined(__mips_n32) || defined(__mips_n64)
1639         case OP_SD:
1640                 KASSERT(mode == VM_PROT_WRITE, ("access mode must be write for store instruction."));
1641                 value = reg[MIPS_INST_RT(inst)];
1642                 sdl_macro(value, addr);
1643                 addr += 7;
1644                 sdr_macro(value, addr);
1645                 return (MIPS_SD_ACCESS);
1646 #endif
1647         }
1648         panic("%s: should not be reached.", __func__);
1649 }
1650
1651
1652 /*
1653  * XXX TODO: SMP?
1654  */
1655 static struct timeval unaligned_lasterr;
1656 static int unaligned_curerr;
1657
1658 static int unaligned_pps_log_limit = 4;
1659
1660 SYSCTL_INT(_machdep, OID_AUTO, unaligned_log_pps_limit, CTLFLAG_RWTUN,
1661     &unaligned_pps_log_limit, 0,
1662     "limit number of userland unaligned log messages per second");
1663
1664 static int
1665 emulate_unaligned_access(struct trapframe *frame, int mode)
1666 {
1667         register_t pc;
1668         int access_type = 0;
1669         struct thread *td = curthread;
1670         struct proc *p = curproc;
1671
1672         pc = frame->pc + (DELAYBRANCH(frame->cause) ? 4 : 0);
1673
1674         /*
1675          * Fall through if it's instruction fetch exception
1676          */
1677         if (!((pc & 3) || (pc == frame->badvaddr))) {
1678
1679                 /*
1680                  * Handle unaligned load and store
1681                  */
1682
1683                 /*
1684                  * Return access type if the instruction was emulated.
1685                  * Otherwise restore pc and fall through.
1686                  */
1687                 access_type = mips_unaligned_load_store(frame,
1688                     mode, frame->badvaddr, pc);
1689
1690                 if (access_type) {
1691                         if (DELAYBRANCH(frame->cause))
1692                                 frame->pc = MipsEmulateBranch(frame, frame->pc,
1693                                     0, 0);
1694                         else
1695                                 frame->pc += 4;
1696
1697                         if (ppsratecheck(&unaligned_lasterr,
1698                             &unaligned_curerr, unaligned_pps_log_limit)) {
1699                                 /* XXX TODO: keep global/tid/pid counters? */
1700                                 log(LOG_INFO,
1701                                     "Unaligned %s: pid=%ld (%s), tid=%ld, "
1702                                     "pc=%#jx, badvaddr=%#jx\n",
1703                                     access_name[access_type - 1],
1704                                     (long) p->p_pid,
1705                                     p->p_comm,
1706                                     (long) td->td_tid,
1707                                     (intmax_t)pc,
1708                                     (intmax_t)frame->badvaddr);
1709                         }
1710                 }
1711         }
1712         return access_type;
1713 }