sys/sparc64/sparc64/trap.c

   1 /*-
   2  * Copyright (c) 2001, Jake Burkholder
   3  * Copyright (C) 1994, David Greenman
   4  * Copyright (c) 1990, 1993
   5  *      The Regents of the University of California.  All rights reserved.
   6  *
   7  * This code is derived from software contributed to Berkeley by
   8  * the University of Utah, and William Jolitz.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  * 3. All advertising materials mentioning features or use of this software
  19  *    must display the following acknowledgement:
  20  *      This product includes software developed by the University of
  21  *      California, Berkeley and its contributors.
  22  * 4. Neither the name of the University nor the names of its contributors
  23  *    may be used to endorse or promote products derived from this software
  24  *    without specific prior written permission.
  25  *
  26  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  27  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  28  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  29  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  30  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  31  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  32  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  33  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  34  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  35  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  36  * SUCH DAMAGE.
  37  *
  38  *      from: @(#)trap.c        7.4 (Berkeley) 5/13/91
  39  *      from: FreeBSD: src/sys/i386/i386/trap.c,v 1.197 2001/07/19
  40  */
  41
  42 #include <sys/cdefs.h>
  43 __FBSDID("$FreeBSD$");
  44
  45 #include "opt_ddb.h"
  46 #include "opt_ktr.h"
  47 #include "opt_ktrace.h"
  48
  49 #include <sys/param.h>
  50 #include <sys/kdb.h>
  51 #include <sys/kernel.h>
  52 #include <sys/bus.h>
  53 #include <sys/interrupt.h>
  54 #include <sys/ktr.h>
  55 #include <sys/lock.h>
  56 #include <sys/mutex.h>
  57 #include <sys/systm.h>
  58 #include <sys/pioctl.h>
  59 #include <sys/ptrace.h>
  60 #include <sys/proc.h>
  61 #include <sys/smp.h>
  62 #include <sys/signalvar.h>
  63 #include <sys/syscall.h>
  64 #include <sys/sysctl.h>
  65 #include <sys/sysent.h>
  66 #include <sys/vmmeter.h>
  67 #ifdef KTRACE
  68 #include <sys/uio.h>
  69 #include <sys/ktrace.h>
  70 #endif
  71 #include <security/audit/audit.h>
  72
  73 #include <dev/ofw/openfirm.h>
  74
  75 #include <vm/vm.h>
  76 #include <vm/pmap.h>
  77 #include <vm/vm_extern.h>
  78 #include <vm/vm_param.h>
  79 #include <vm/vm_kern.h>
  80 #include <vm/vm_map.h>
  81 #include <vm/vm_page.h>
  82
  83 #include <machine/cpu.h>
  84 #include <machine/frame.h>
  85 #include <machine/intr_machdep.h>
  86 #include <machine/pcb.h>
  87 #include <machine/smp.h>
  88 #include <machine/trap.h>
  89 #include <machine/tstate.h>
  90 #include <machine/tte.h>
  91 #include <machine/tlb.h>
  92 #include <machine/tsb.h>
  93 #include <machine/watch.h>
  94
  95 void trap(struct trapframe *tf);
  96 void syscall(struct trapframe *tf);
  97
  98 static int trap_pfault(struct thread *td, struct trapframe *tf);
  99
 100 extern char copy_fault[];
 101 extern char copy_nofault_begin[];
 102 extern char copy_nofault_end[];
 103
 104 extern char fs_fault[];
 105 extern char fs_nofault_begin[];
 106 extern char fs_nofault_end[];
 107 extern char fs_nofault_intr_begin[];
 108 extern char fs_nofault_intr_end[];
 109
 110 extern char fas_fault[];
 111 extern char fas_nofault_begin[];
 112 extern char fas_nofault_end[];
 113
 114 extern char *syscallnames[];
 115
 116 const char *trap_msg[] = {
 117         "reserved",
 118         "instruction access exception",
 119         "instruction access error",
 120         "instruction access protection",
 121         "illtrap instruction",
 122         "illegal instruction",
 123         "privileged opcode",
 124         "floating point disabled",
 125         "floating point exception ieee 754",
 126         "floating point exception other",
 127         "tag overflow",
 128         "division by zero",
 129         "data access exception",
 130         "data access error",
 131         "data access protection",
 132         "memory address not aligned",
 133         "privileged action",
 134         "async data error",
 135         "trap instruction 16",
 136         "trap instruction 17",
 137         "trap instruction 18",
 138         "trap instruction 19",
 139         "trap instruction 20",
 140         "trap instruction 21",
 141         "trap instruction 22",
 142         "trap instruction 23",
 143         "trap instruction 24",
 144         "trap instruction 25",
 145         "trap instruction 26",
 146         "trap instruction 27",
 147         "trap instruction 28",
 148         "trap instruction 29",
 149         "trap instruction 30",
 150         "trap instruction 31",
 151         "fast instruction access mmu miss",
 152         "fast data access mmu miss",
 153         "interrupt",
 154         "physical address watchpoint",
 155         "virtual address watchpoint",
 156         "corrected ecc error",
 157         "spill",
 158         "fill",
 159         "fill",
 160         "breakpoint",
 161         "clean window",
 162         "range check",
 163         "fix alignment",
 164         "integer overflow",
 165         "syscall",
 166         "restore physical watchpoint",
 167         "restore virtual watchpoint",
 168         "kernel stack fault",
 169 };
 170
 171 static const int trap_sig[] = {
 172         SIGILL,                 /* reserved */
 173         SIGILL,                 /* instruction access exception */
 174         SIGILL,                 /* instruction access error */
 175         SIGILL,                 /* instruction access protection */
 176         SIGILL,                 /* illtrap instruction */
 177         SIGILL,                 /* illegal instruction */
 178         SIGBUS,                 /* privileged opcode */
 179         SIGFPE,                 /* floating point disabled */
 180         SIGFPE,                 /* floating point exception ieee 754 */
 181         SIGFPE,                 /* floating point exception other */
 182         SIGEMT,                 /* tag overflow */
 183         SIGFPE,                 /* division by zero */
 184         SIGILL,                 /* data access exception */
 185         SIGILL,                 /* data access error */
 186         SIGBUS,                 /* data access protection */
 187         SIGBUS,                 /* memory address not aligned */
 188         SIGBUS,                 /* privileged action */
 189         SIGBUS,                 /* async data error */
 190         SIGILL,                 /* trap instruction 16 */
 191         SIGILL,                 /* trap instruction 17 */
 192         SIGILL,                 /* trap instruction 18 */
 193         SIGILL,                 /* trap instruction 19 */
 194         SIGILL,                 /* trap instruction 20 */
 195         SIGILL,                 /* trap instruction 21 */
 196         SIGILL,                 /* trap instruction 22 */
 197         SIGILL,                 /* trap instruction 23 */
 198         SIGILL,                 /* trap instruction 24 */
 199         SIGILL,                 /* trap instruction 25 */
 200         SIGILL,                 /* trap instruction 26 */
 201         SIGILL,                 /* trap instruction 27 */
 202         SIGILL,                 /* trap instruction 28 */
 203         SIGILL,                 /* trap instruction 29 */
 204         SIGILL,                 /* trap instruction 30 */
 205         SIGILL,                 /* trap instruction 31 */
 206         SIGSEGV,                /* fast instruction access mmu miss */
 207         SIGSEGV,                /* fast data access mmu miss */
 208         -1,                     /* interrupt */
 209         -1,                     /* physical address watchpoint */
 210         -1,                     /* virtual address watchpoint */
 211         -1,                     /* corrected ecc error */
 212         SIGILL,                 /* spill */
 213         SIGILL,                 /* fill */
 214         SIGILL,                 /* fill */
 215         SIGTRAP,                /* breakpoint */
 216         SIGILL,                 /* clean window */
 217         SIGILL,                 /* range check */
 218         SIGILL,                 /* fix alignment */
 219         SIGILL,                 /* integer overflow */
 220         SIGSYS,                 /* syscall */
 221         -1,                     /* restore physical watchpoint */
 222         -1,                     /* restore virtual watchpoint */
 223         -1,                     /* kernel stack fault */
 224 };
 225
 226 CTASSERT(sizeof(struct trapframe) == 256);
 227
 228 int debugger_on_signal = 0;
 229 SYSCTL_INT(_debug, OID_AUTO, debugger_on_signal, CTLFLAG_RW,
 230     &debugger_on_signal, 0, "");
 231
 232 /*
 233  * SUNW,set-trap-table allows to take over %tba from the PROM, which
 234  * will turn off interrupts and handle outstanding ones while doing so,
 235  * in a safe way.
 236  */
 237 void
 238 sun4u_set_traptable(void *tba_addr)
 239 {
 240         static struct {
 241                 cell_t name;
 242                 cell_t nargs;
 243                 cell_t nreturns;
 244                 cell_t tba_addr;
 245         } args = {
 246                 (cell_t)"SUNW,set-trap-table",
 247                 2,
 248         };
 249
 250         args.tba_addr = (cell_t)tba_addr;
 251         openfirmware(&args);
 252 }
 253
 254 void
 255 trap(struct trapframe *tf)
 256 {
 257         struct thread *td;
 258         struct proc *p;
 259         int error;
 260         int sig;
 261         register_t addr;
 262         ksiginfo_t ksi;
 263
 264         td = curthread;
 265
 266         CTR4(KTR_TRAP, "trap: %p type=%s (%s) pil=%#lx", td,
 267             trap_msg[tf->tf_type & ~T_KERNEL],
 268             (TRAPF_USERMODE(tf) ? "user" : "kernel"), rdpr(pil));
 269
 270         PCPU_INC(cnt.v_trap);
 271
 272         if ((tf->tf_tstate & TSTATE_PRIV) == 0) {
 273                 KASSERT(td != NULL, ("trap: curthread NULL"));
 274                 KASSERT(td->td_proc != NULL, ("trap: curproc NULL"));
 275
 276                 p = td->td_proc;
 277                 td->td_pticks = 0;
 278                 td->td_frame = tf;
 279                 addr = tf->tf_tpc;
 280                 if (td->td_ucred != p->p_ucred)
 281                         cred_update_thread(td);
 282
 283                 switch (tf->tf_type) {
 284                 case T_DATA_MISS:
 285                 case T_DATA_PROTECTION:
 286                         addr = tf->tf_sfar;
 287                         /* FALLTHROUGH */
 288                 case T_INSTRUCTION_MISS:
 289                         sig = trap_pfault(td, tf);
 290                         break;
 291                 case T_FILL:
 292                         sig = rwindow_load(td, tf, 2);
 293                         break;
 294                 case T_FILL_RET:
 295                         sig = rwindow_load(td, tf, 1);
 296                         break;
 297                 case T_SPILL:
 298                         sig = rwindow_save(td);
 299                         break;
 300                 default:
 301                         if (tf->tf_type < 0 || tf->tf_type >= T_MAX ||
 302                             trap_sig[tf->tf_type] == -1)
 303                                 panic("trap: bad trap type");
 304                         sig = trap_sig[tf->tf_type];
 305                         break;
 306                 }
 307
 308                 if (sig != 0) {
 309                         /* Translate fault for emulators. */
 310                         if (p->p_sysent->sv_transtrap != NULL) {
 311                                 sig = p->p_sysent->sv_transtrap(sig,
 312                                     tf->tf_type);
 313                         }
 314                         if (debugger_on_signal &&
 315                             (sig == 4 || sig == 10 || sig == 11))
 316                                 kdb_enter(KDB_WHY_TRAPSIG, "trapsig");
 317                         ksiginfo_init_trap(&ksi);
 318                         ksi.ksi_signo = sig;
 319                         ksi.ksi_code = (int)tf->tf_type; /* XXX not POSIX */
 320                         ksi.ksi_addr = (void *)addr;
 321                         ksi.ksi_trapno = (int)tf->tf_type;
 322                         trapsignal(td, &ksi);
 323                 }
 324
 325                 userret(td, tf);
 326                 mtx_assert(&Giant, MA_NOTOWNED);
 327         } else {
 328                 KASSERT((tf->tf_type & T_KERNEL) != 0,
 329                     ("trap: kernel trap isn't"));
 330
 331 #ifdef KDB
 332                 if (kdb_active) {
 333                         kdb_reenter();
 334                         return;
 335                 }
 336 #endif
 337
 338                 switch (tf->tf_type & ~T_KERNEL) {
 339 #ifdef KDB
 340                 case T_BREAKPOINT:
 341                 case T_KSTACK_FAULT:
 342                         error = (kdb_trap(tf->tf_type, 0, tf) == 0);
 343                         TF_DONE(tf);
 344                         break;
 345 #ifdef notyet
 346                 case T_PA_WATCHPOINT:
 347                 case T_VA_WATCHPOINT:
 348                         error = db_watch_trap(tf);
 349                         break;
 350 #endif
 351 #endif
 352                 case T_DATA_MISS:
 353                 case T_DATA_PROTECTION:
 354                 case T_INSTRUCTION_MISS:
 355                         error = trap_pfault(td, tf);
 356                         break;
 357                 case T_DATA_EXCEPTION:
 358                 case T_MEM_ADDRESS_NOT_ALIGNED:
 359                         if ((tf->tf_sfsr & MMU_SFSR_FV) != 0 &&
 360                             MMU_SFSR_GET_ASI(tf->tf_sfsr) == ASI_AIUP) {
 361                                 if (tf->tf_tpc >= (u_long)copy_nofault_begin &&
 362                                     tf->tf_tpc <= (u_long)copy_nofault_end) {
 363                                         tf->tf_tpc = (u_long)copy_fault;
 364                                         tf->tf_tnpc = tf->tf_tpc + 4;
 365                                         error = 0;
 366                                         break;
 367                                 }
 368                                 if (tf->tf_tpc >= (u_long)fs_nofault_begin &&
 369                                     tf->tf_tpc <= (u_long)fs_nofault_end) {
 370                                         tf->tf_tpc = (u_long)fs_fault;
 371                                         tf->tf_tnpc = tf->tf_tpc + 4;
 372                                         error = 0;
 373                                         break;
 374                                 }
 375                         }
 376                         error = 1;
 377                         break;
 378                 case T_DATA_ERROR:
 379                         /*
 380                          * Handle PCI poke/peek as per UltraSPARC IIi
 381                          * User's Manual 16.2.1, modulo checking the
 382                          * TPC as USIII CPUs generate a precise trap
 383                          * instead of a special deferred one.
 384                          */
 385                         if (tf->tf_tpc > (u_long)fas_nofault_begin &&
 386                             tf->tf_tpc < (u_long)fas_nofault_end) {
 387                                 cache_flush();
 388                                 cache_enable();
 389                                 tf->tf_tpc = (u_long)fas_fault;
 390                                 tf->tf_tnpc = tf->tf_tpc + 4;
 391                                 error = 0;
 392                                 break;
 393                         }
 394                         error = 1;
 395                         break;
 396                 default:
 397                         error = 1;
 398                         break;
 399                 }
 400
 401                 if (error != 0)
 402                         panic("trap: %s", trap_msg[tf->tf_type & ~T_KERNEL]);
 403         }
 404         CTR1(KTR_TRAP, "trap: td=%p return", td);
 405 }
 406
 407 static int
 408 trap_pfault(struct thread *td, struct trapframe *tf)
 409 {
 410         struct vmspace *vm;
 411         struct pcb *pcb;
 412         struct proc *p;
 413         vm_offset_t va;
 414         vm_prot_t prot;
 415         vm_map_entry_t entry;
 416         u_long ctx;
 417         int flags;
 418         int type;
 419         int rv;
 420
 421         if (td == NULL)
 422                 return (-1);
 423         KASSERT(td->td_pcb != NULL, ("trap_pfault: pcb NULL"));
 424         KASSERT(td->td_proc != NULL, ("trap_pfault: curproc NULL"));
 425         KASSERT(td->td_proc->p_vmspace != NULL, ("trap_pfault: vmspace NULL"));
 426
 427         p = td->td_proc;
 428
 429         rv = KERN_SUCCESS;
 430         ctx = TLB_TAR_CTX(tf->tf_tar);
 431         pcb = td->td_pcb;
 432         type = tf->tf_type & ~T_KERNEL;
 433         va = TLB_TAR_VA(tf->tf_tar);
 434
 435         CTR4(KTR_TRAP, "trap_pfault: td=%p pm_ctx=%#lx va=%#lx ctx=%#lx",
 436             td, p->p_vmspace->vm_pmap.pm_context[curcpu], va, ctx);
 437
 438         if (type == T_DATA_PROTECTION) {
 439                 prot = VM_PROT_WRITE;
 440                 flags = VM_FAULT_DIRTY;
 441         } else {
 442                 if (type == T_DATA_MISS)
 443                         prot = VM_PROT_READ;
 444                 else
 445                         prot = VM_PROT_READ | VM_PROT_EXECUTE;
 446                 flags = VM_FAULT_NORMAL;
 447         }
 448
 449         if (ctx != TLB_CTX_KERNEL) {
 450                 if ((tf->tf_tstate & TSTATE_PRIV) != 0 &&
 451                     (tf->tf_tpc >= (u_long)fs_nofault_intr_begin &&
 452                     tf->tf_tpc <= (u_long)fs_nofault_intr_end)) {
 453                         tf->tf_tpc = (u_long)fs_fault;
 454                         tf->tf_tnpc = tf->tf_tpc + 4;
 455                         return (0);
 456                 }
 457
 458                 /*
 459                  * This is a fault on non-kernel virtual memory.
 460                  */
 461                 vm = p->p_vmspace;
 462
 463                 /*
 464                  * Keep swapout from messing with us during this
 465                  * critical time.
 466                  */
 467                 PROC_LOCK(p);
 468                 ++p->p_lock;
 469                 PROC_UNLOCK(p);
 470
 471                 /* Fault in the user page. */
 472                 rv = vm_fault(&vm->vm_map, va, prot, flags);
 473
 474                 /*
 475                  * Now the process can be swapped again.
 476                  */
 477                 PROC_LOCK(p);
 478                 --p->p_lock;
 479                 PROC_UNLOCK(p);
 480         } else {
 481                 /*
 482                  * This is a fault on kernel virtual memory.  Attempts to
 483                  * access kernel memory from user mode cause privileged
 484                  * action traps, not page fault.
 485                  */
 486                 KASSERT(tf->tf_tstate & TSTATE_PRIV,
 487                     ("trap_pfault: fault on nucleus context from user mode"));
 488
 489                 if (tf->tf_tpc >= (u_long)copy_nofault_begin &&
 490                     tf->tf_tpc <= (u_long)copy_nofault_end) {
 491                         vm_map_lock_read(kernel_map);
 492                         if (vm_map_lookup_entry(kernel_map, va, &entry) &&
 493                             (entry->eflags & MAP_ENTRY_NOFAULT) != 0) {
 494                                 tf->tf_tpc = (u_long)copy_fault;
 495                                 tf->tf_tnpc = tf->tf_tpc + 4;
 496                                 vm_map_unlock_read(kernel_map);
 497                                 return (0);
 498                         }
 499                         vm_map_unlock_read(kernel_map);
 500                 }
 501
 502                 /*
 503                  * We don't have to worry about process locking or stacks in
 504                  * the kernel.
 505                  */
 506                 rv = vm_fault(kernel_map, va, prot, VM_FAULT_NORMAL);
 507         }
 508
 509         CTR3(KTR_TRAP, "trap_pfault: return td=%p va=%#lx rv=%d",
 510             td, va, rv);
 511         if (rv == KERN_SUCCESS)
 512                 return (0);
 513         if (ctx != TLB_CTX_KERNEL && (tf->tf_tstate & TSTATE_PRIV) != 0) {
 514                 if (tf->tf_tpc >= (u_long)fs_nofault_begin &&
 515                     tf->tf_tpc <= (u_long)fs_nofault_end) {
 516                         tf->tf_tpc = (u_long)fs_fault;
 517                         tf->tf_tnpc = tf->tf_tpc + 4;
 518                         return (0);
 519                 }
 520                 if (tf->tf_tpc >= (u_long)copy_nofault_begin &&
 521                     tf->tf_tpc <= (u_long)copy_nofault_end) {
 522                         tf->tf_tpc = (u_long)copy_fault;
 523                         tf->tf_tnpc = tf->tf_tpc + 4;
 524                         return (0);
 525                 }
 526         }
 527         return ((rv == KERN_PROTECTION_FAILURE) ? SIGBUS : SIGSEGV);
 528 }
 529
 530 /* Maximum number of arguments that can be passed via the out registers. */
 531 #define REG_MAXARGS     6
 532
 533 /*
 534  * Syscall handler. The arguments to the syscall are passed in the o registers
 535  * by the caller, and are saved in the trap frame. The syscall number is passed
 536  * in %g1 (and also saved in the trap frame).
 537  */
 538 void
 539 syscall(struct trapframe *tf)
 540 {
 541         struct sysent *callp;
 542         struct thread *td;
 543         register_t args[8];
 544         register_t *argp;
 545         struct proc *p;
 546         u_long code;
 547         u_long tpc;
 548         int reg;
 549         int regcnt;
 550         int narg;
 551         int error;
 552
 553         td = curthread;
 554         KASSERT(td != NULL, ("trap: curthread NULL"));
 555         KASSERT(td->td_proc != NULL, ("trap: curproc NULL"));
 556
 557         p = td->td_proc;
 558
 559         PCPU_INC(cnt.v_syscall);
 560
 561         narg = 0;
 562         error = 0;
 563         reg = 0;
 564         regcnt = REG_MAXARGS;
 565
 566         td->td_pticks = 0;
 567         td->td_frame = tf;
 568         if (td->td_ucred != p->p_ucred)
 569                 cred_update_thread(td);
 570         code = tf->tf_global[1];
 571
 572         /*
 573          * For syscalls, we don't want to retry the faulting instruction
 574          * (usually), instead we need to advance one instruction.
 575          */
 576         tpc = tf->tf_tpc;
 577         TF_DONE(tf);
 578
 579         if (p->p_sysent->sv_prepsyscall) {
 580                 /*
 581                  * The prep code is MP aware.
 582                  */
 583 #if 0
 584                 (*p->p_sysent->sv_prepsyscall)(tf, args, &code, &params);
 585 #endif
 586         } else if (code == SYS_syscall || code == SYS___syscall) {
 587                 code = tf->tf_out[reg++];
 588                 regcnt--;
 589         }
 590
 591         if (p->p_sysent->sv_mask)
 592                 code &= p->p_sysent->sv_mask;
 593
 594         if (code >= p->p_sysent->sv_size)
 595                 callp = &p->p_sysent->sv_table[0];
 596         else
 597                 callp = &p->p_sysent->sv_table[code];
 598
 599         narg = callp->sy_narg;
 600
 601         KASSERT(narg <= sizeof(args) / sizeof(args[0]),
 602             ("Too many syscall arguments!"));
 603         error = 0;
 604         argp = args;
 605         bcopy(&tf->tf_out[reg], args, sizeof(args[0]) * regcnt);
 606         if (narg > regcnt)
 607                 error = copyin((void *)(tf->tf_out[6] + SPOFF +
 608                     offsetof(struct frame, fr_pad[6])),
 609                     &args[regcnt], (narg - regcnt) * sizeof(args[0]));
 610
 611         CTR5(KTR_SYSC, "syscall: td=%p %s(%#lx, %#lx, %#lx)", td,
 612             syscallnames[code], argp[0], argp[1], argp[2]);
 613
 614 #ifdef KTRACE
 615         if (KTRPOINT(td, KTR_SYSCALL))
 616                 ktrsyscall(code, narg, argp);
 617 #endif
 618
 619         td->td_syscalls++;
 620
 621         if (error == 0) {
 622                 td->td_retval[0] = 0;
 623                 td->td_retval[1] = 0;
 624
 625                 STOPEVENT(p, S_SCE, narg);      /* MP aware */
 626
 627                 PTRACESTOP_SC(p, td, S_PT_SCE);
 628
 629                 AUDIT_SYSCALL_ENTER(code, td);
 630                 error = (*callp->sy_call)(td, argp);
 631                 AUDIT_SYSCALL_EXIT(error, td);
 632
 633                 CTR5(KTR_SYSC, "syscall: p=%p error=%d %s return %#lx %#lx ", p,
 634                     error, syscallnames[code], td->td_retval[0],
 635                     td->td_retval[1]);
 636         }
 637
 638         /*
 639          * MP SAFE (we may or may not have the MP lock at this point)
 640          */
 641         switch (error) {
 642         case 0:
 643                 tf->tf_out[0] = td->td_retval[0];
 644                 tf->tf_out[1] = td->td_retval[1];
 645                 tf->tf_tstate &= ~TSTATE_XCC_C;
 646                 break;
 647
 648         case ERESTART:
 649                 /*
 650                  * Undo the tpc advancement we have done above, we want to
 651                  * reexecute the system call.
 652                  */
 653                 tf->tf_tpc = tpc;
 654                 tf->tf_tnpc -= 4;
 655                 break;
 656
 657         case EJUSTRETURN:
 658                 break;
 659
 660         default:
 661                 if (p->p_sysent->sv_errsize) {
 662                         if (error >= p->p_sysent->sv_errsize)
 663                                 error = -1;     /* XXX */
 664                         else
 665                                 error = p->p_sysent->sv_errtbl[error];
 666                 }
 667                 tf->tf_out[0] = error;
 668                 tf->tf_tstate |= TSTATE_XCC_C;
 669                 break;
 670         }
 671
 672         /*
 673          * Check for misbehavior.
 674          */
 675         WITNESS_WARN(WARN_PANIC, NULL, "System call %s returning",
 676             (code >= 0 && code < SYS_MAXSYSCALL) ? syscallnames[code] : "???");
 677         KASSERT(td->td_critnest == 0,
 678             ("System call %s returning in a critical section",
 679             (code >= 0 && code < SYS_MAXSYSCALL) ? syscallnames[code] : "???"));
 680         KASSERT(td->td_locks == 0,
 681             ("System call %s returning with %d locks held",
 682             (code >= 0 && code < SYS_MAXSYSCALL) ? syscallnames[code] : "???",
 683             td->td_locks));
 684
 685         /*
 686          * Handle reschedule and other end-of-syscall issues
 687          */
 688         userret(td, tf);
 689
 690 #ifdef KTRACE
 691         if (KTRPOINT(td, KTR_SYSRET))
 692                 ktrsysret(code, error, td->td_retval[0]);
 693 #endif
 694         /*
 695          * This works because errno is findable through the
 696          * register set.  If we ever support an emulation where this
 697          * is not the case, this code will need to be revisited.
 698          */
 699         STOPEVENT(p, S_SCX, code);
 700
 701         PTRACESTOP_SC(p, td, S_PT_SCX);
 702 }