sys/sparc64/sparc64/trap.c

   1 /*-
   2  * Copyright (c) 2001, Jake Burkholder
   3  * Copyright (C) 1994, David Greenman
   4  * Copyright (c) 1990, 1993
   5  *      The Regents of the University of California.  All rights reserved.
   6  *
   7  * This code is derived from software contributed to Berkeley by
   8  * the University of Utah, and William Jolitz.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  * 3. All advertising materials mentioning features or use of this software
  19  *    must display the following acknowledgement:
  20  *      This product includes software developed by the University of
  21  *      California, Berkeley and its contributors.
  22  * 4. Neither the name of the University nor the names of its contributors
  23  *    may be used to endorse or promote products derived from this software
  24  *    without specific prior written permission.
  25  *
  26  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  27  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  28  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  29  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  30  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  31  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  32  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  33  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  34  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  35  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  36  * SUCH DAMAGE.
  37  *
  38  *      from: @(#)trap.c        7.4 (Berkeley) 5/13/91
  39  *      from: FreeBSD: src/sys/i386/i386/trap.c,v 1.197 2001/07/19
  40  */
  41
  42 #include <sys/cdefs.h>
  43 __FBSDID("$FreeBSD$");
  44
  45 #include "opt_ddb.h"
  46 #include "opt_ktr.h"
  47 #include "opt_ktrace.h"
  48
  49 #include <sys/param.h>
  50 #include <sys/kdb.h>
  51 #include <sys/kernel.h>
  52 #include <sys/bus.h>
  53 #include <sys/interrupt.h>
  54 #include <sys/ktr.h>
  55 #include <sys/lock.h>
  56 #include <sys/mutex.h>
  57 #include <sys/systm.h>
  58 #include <sys/pioctl.h>
  59 #include <sys/ptrace.h>
  60 #include <sys/proc.h>
  61 #include <sys/smp.h>
  62 #include <sys/signalvar.h>
  63 #include <sys/syscall.h>
  64 #include <sys/sysctl.h>
  65 #include <sys/sysent.h>
  66 #include <sys/vmmeter.h>
  67 #ifdef KTRACE
  68 #include <sys/uio.h>
  69 #include <sys/ktrace.h>
  70 #endif
  71 #include <security/audit/audit.h>
  72
  73 #include <dev/ofw/openfirm.h>
  74
  75 #include <vm/vm.h>
  76 #include <vm/pmap.h>
  77 #include <vm/vm_extern.h>
  78 #include <vm/vm_param.h>
  79 #include <vm/vm_kern.h>
  80 #include <vm/vm_map.h>
  81 #include <vm/vm_page.h>
  82
  83 #include <machine/cpu.h>
  84 #include <machine/frame.h>
  85 #include <machine/intr_machdep.h>
  86 #include <machine/ofw_machdep.h>
  87 #include <machine/smp.h>
  88 #include <machine/trap.h>
  89 #include <machine/tstate.h>
  90 #include <machine/tte.h>
  91 #include <machine/tlb.h>
  92 #include <machine/tsb.h>
  93 #include <machine/watch.h>
  94
  95 void trap(struct trapframe *tf);
  96 void syscall(struct trapframe *tf);
  97
  98 static int trap_pfault(struct thread *td, struct trapframe *tf);
  99
 100 extern char copy_fault[];
 101 extern char copy_nofault_begin[];
 102 extern char copy_nofault_end[];
 103
 104 extern char fs_fault[];
 105 extern char fs_nofault_begin[];
 106 extern char fs_nofault_end[];
 107 extern char fs_nofault_intr_begin[];
 108 extern char fs_nofault_intr_end[];
 109
 110 extern char fas_fault[];
 111 extern char fas_nofault_begin[];
 112 extern char fas_nofault_end[];
 113
 114 extern char *syscallnames[];
 115
 116 const char *const trap_msg[] = {
 117         "reserved",
 118         "instruction access exception",
 119         "instruction access error",
 120         "instruction access protection",
 121         "illtrap instruction",
 122         "illegal instruction",
 123         "privileged opcode",
 124         "floating point disabled",
 125         "floating point exception ieee 754",
 126         "floating point exception other",
 127         "tag overflow",
 128         "division by zero",
 129         "data access exception",
 130         "data access error",
 131         "data access protection",
 132         "memory address not aligned",
 133         "privileged action",
 134         "async data error",
 135         "trap instruction 16",
 136         "trap instruction 17",
 137         "trap instruction 18",
 138         "trap instruction 19",
 139         "trap instruction 20",
 140         "trap instruction 21",
 141         "trap instruction 22",
 142         "trap instruction 23",
 143         "trap instruction 24",
 144         "trap instruction 25",
 145         "trap instruction 26",
 146         "trap instruction 27",
 147         "trap instruction 28",
 148         "trap instruction 29",
 149         "trap instruction 30",
 150         "trap instruction 31",
 151         "fast instruction access mmu miss",
 152         "fast data access mmu miss",
 153         "interrupt",
 154         "physical address watchpoint",
 155         "virtual address watchpoint",
 156         "corrected ecc error",
 157         "spill",
 158         "fill",
 159         "fill",
 160         "breakpoint",
 161         "clean window",
 162         "range check",
 163         "fix alignment",
 164         "integer overflow",
 165         "syscall",
 166         "restore physical watchpoint",
 167         "restore virtual watchpoint",
 168         "kernel stack fault",
 169 };
 170
 171 static const int trap_sig[] = {
 172         SIGILL,                 /* reserved */
 173         SIGILL,                 /* instruction access exception */
 174         SIGILL,                 /* instruction access error */
 175         SIGILL,                 /* instruction access protection */
 176         SIGILL,                 /* illtrap instruction */
 177         SIGILL,                 /* illegal instruction */
 178         SIGBUS,                 /* privileged opcode */
 179         SIGFPE,                 /* floating point disabled */
 180         SIGFPE,                 /* floating point exception ieee 754 */
 181         SIGFPE,                 /* floating point exception other */
 182         SIGEMT,                 /* tag overflow */
 183         SIGFPE,                 /* division by zero */
 184         SIGILL,                 /* data access exception */
 185         SIGILL,                 /* data access error */
 186         SIGBUS,                 /* data access protection */
 187         SIGBUS,                 /* memory address not aligned */
 188         SIGBUS,                 /* privileged action */
 189         SIGBUS,                 /* async data error */
 190         SIGILL,                 /* trap instruction 16 */
 191         SIGILL,                 /* trap instruction 17 */
 192         SIGILL,                 /* trap instruction 18 */
 193         SIGILL,                 /* trap instruction 19 */
 194         SIGILL,                 /* trap instruction 20 */
 195         SIGILL,                 /* trap instruction 21 */
 196         SIGILL,                 /* trap instruction 22 */
 197         SIGILL,                 /* trap instruction 23 */
 198         SIGILL,                 /* trap instruction 24 */
 199         SIGILL,                 /* trap instruction 25 */
 200         SIGILL,                 /* trap instruction 26 */
 201         SIGILL,                 /* trap instruction 27 */
 202         SIGILL,                 /* trap instruction 28 */
 203         SIGILL,                 /* trap instruction 29 */
 204         SIGILL,                 /* trap instruction 30 */
 205         SIGILL,                 /* trap instruction 31 */
 206         SIGSEGV,                /* fast instruction access mmu miss */
 207         SIGSEGV,                /* fast data access mmu miss */
 208         -1,                     /* interrupt */
 209         -1,                     /* physical address watchpoint */
 210         -1,                     /* virtual address watchpoint */
 211         -1,                     /* corrected ecc error */
 212         SIGILL,                 /* spill */
 213         SIGILL,                 /* fill */
 214         SIGILL,                 /* fill */
 215         SIGTRAP,                /* breakpoint */
 216         SIGILL,                 /* clean window */
 217         SIGILL,                 /* range check */
 218         SIGILL,                 /* fix alignment */
 219         SIGILL,                 /* integer overflow */
 220         SIGSYS,                 /* syscall */
 221         -1,                     /* restore physical watchpoint */
 222         -1,                     /* restore virtual watchpoint */
 223         -1,                     /* kernel stack fault */
 224 };
 225
 226 CTASSERT(sizeof(struct trapframe) == 256);
 227
 228 int debugger_on_signal = 0;
 229 SYSCTL_INT(_debug, OID_AUTO, debugger_on_signal, CTLFLAG_RW,
 230     &debugger_on_signal, 0, "");
 231
 232 /*
 233  * SUNW,set-trap-table allows to take over %tba from the PROM, which
 234  * will turn off interrupts and handle outstanding ones while doing so,
 235  * in a safe way.
 236  */
 237 void
 238 sun4u_set_traptable(void *tba_addr)
 239 {
 240         static struct {
 241                 cell_t name;
 242                 cell_t nargs;
 243                 cell_t nreturns;
 244                 cell_t tba_addr;
 245         } args = {
 246                 (cell_t)"SUNW,set-trap-table",
 247                 2,
 248         };
 249
 250         args.tba_addr = (cell_t)tba_addr;
 251         ofw_entry(&args);
 252 }
 253
 254 void
 255 trap(struct trapframe *tf)
 256 {
 257         struct thread *td;
 258         struct proc *p;
 259         int error;
 260         int sig;
 261         register_t addr;
 262         ksiginfo_t ksi;
 263
 264         td = curthread;
 265
 266         CTR4(KTR_TRAP, "trap: %p type=%s (%s) pil=%#lx", td,
 267             trap_msg[tf->tf_type & ~T_KERNEL],
 268             (TRAPF_USERMODE(tf) ? "user" : "kernel"), rdpr(pil));
 269
 270         PCPU_INC(cnt.v_trap);
 271
 272         if ((tf->tf_tstate & TSTATE_PRIV) == 0) {
 273                 KASSERT(td != NULL, ("trap: curthread NULL"));
 274                 KASSERT(td->td_proc != NULL, ("trap: curproc NULL"));
 275
 276                 p = td->td_proc;
 277                 td->td_pticks = 0;
 278                 td->td_frame = tf;
 279                 addr = tf->tf_tpc;
 280                 if (td->td_ucred != p->p_ucred)
 281                         cred_update_thread(td);
 282
 283                 switch (tf->tf_type) {
 284                 case T_DATA_MISS:
 285                 case T_DATA_PROTECTION:
 286                         addr = tf->tf_sfar;
 287                         /* FALLTHROUGH */
 288                 case T_INSTRUCTION_MISS:
 289                         sig = trap_pfault(td, tf);
 290                         break;
 291                 case T_FILL:
 292                         sig = rwindow_load(td, tf, 2);
 293                         break;
 294                 case T_FILL_RET:
 295                         sig = rwindow_load(td, tf, 1);
 296                         break;
 297                 case T_SPILL:
 298                         sig = rwindow_save(td);
 299                         break;
 300                 default:
 301                         if (tf->tf_type < 0 || tf->tf_type >= T_MAX ||
 302                             trap_sig[tf->tf_type] == -1)
 303                                 panic("trap: bad trap type");
 304                         sig = trap_sig[tf->tf_type];
 305                         break;
 306                 }
 307
 308                 if (sig != 0) {
 309                         /* Translate fault for emulators. */
 310                         if (p->p_sysent->sv_transtrap != NULL) {
 311                                 sig = p->p_sysent->sv_transtrap(sig,
 312                                     tf->tf_type);
 313                         }
 314                         if (debugger_on_signal &&
 315                             (sig == 4 || sig == 10 || sig == 11))
 316                                 kdb_enter(KDB_WHY_TRAPSIG, "trapsig");
 317                         ksiginfo_init_trap(&ksi);
 318                         ksi.ksi_signo = sig;
 319                         ksi.ksi_code = (int)tf->tf_type; /* XXX not POSIX */
 320                         ksi.ksi_addr = (void *)addr;
 321                         ksi.ksi_trapno = (int)tf->tf_type;
 322                         trapsignal(td, &ksi);
 323                 }
 324
 325                 userret(td, tf);
 326                 mtx_assert(&Giant, MA_NOTOWNED);
 327         } else {
 328                 KASSERT((tf->tf_type & T_KERNEL) != 0,
 329                     ("trap: kernel trap isn't"));
 330
 331                 if (kdb_active) {
 332                         kdb_reenter();
 333                         return;
 334                 }
 335
 336                 switch (tf->tf_type & ~T_KERNEL) {
 337                 case T_BREAKPOINT:
 338                 case T_KSTACK_FAULT:
 339                         error = (kdb_trap(tf->tf_type, 0, tf) == 0);
 340                         TF_DONE(tf);
 341                         break;
 342 #ifdef notyet
 343                 case T_PA_WATCHPOINT:
 344                 case T_VA_WATCHPOINT:
 345                         error = db_watch_trap(tf);
 346                         break;
 347 #endif
 348                 case T_DATA_MISS:
 349                 case T_DATA_PROTECTION:
 350                 case T_INSTRUCTION_MISS:
 351                         error = trap_pfault(td, tf);
 352                         break;
 353                 case T_DATA_EXCEPTION:
 354                 case T_MEM_ADDRESS_NOT_ALIGNED:
 355                         if ((tf->tf_sfsr & MMU_SFSR_FV) != 0 &&
 356                             MMU_SFSR_GET_ASI(tf->tf_sfsr) == ASI_AIUP) {
 357                                 if (tf->tf_tpc >= (u_long)copy_nofault_begin &&
 358                                     tf->tf_tpc <= (u_long)copy_nofault_end) {
 359                                         tf->tf_tpc = (u_long)copy_fault;
 360                                         tf->tf_tnpc = tf->tf_tpc + 4;
 361                                         error = 0;
 362                                         break;
 363                                 }
 364                                 if (tf->tf_tpc >= (u_long)fs_nofault_begin &&
 365                                     tf->tf_tpc <= (u_long)fs_nofault_end) {
 366                                         tf->tf_tpc = (u_long)fs_fault;
 367                                         tf->tf_tnpc = tf->tf_tpc + 4;
 368                                         error = 0;
 369                                         break;
 370                                 }
 371                         }
 372                         error = 1;
 373                         break;
 374                 case T_DATA_ERROR:
 375                         /*
 376                          * Handle PCI poke/peek as per UltraSPARC IIi
 377                          * User's Manual 16.2.1, modulo checking the
 378                          * TPC as USIII CPUs generate a precise trap
 379                          * instead of a special deferred one.
 380                          */
 381                         if (tf->tf_tpc > (u_long)fas_nofault_begin &&
 382                             tf->tf_tpc < (u_long)fas_nofault_end) {
 383                                 cache_flush();
 384                                 cache_enable();
 385                                 tf->tf_tpc = (u_long)fas_fault;
 386                                 tf->tf_tnpc = tf->tf_tpc + 4;
 387                                 error = 0;
 388                                 break;
 389                         }
 390                         error = 1;
 391                         break;
 392                 default:
 393                         error = 1;
 394                         break;
 395                 }
 396
 397                 if (error != 0)
 398                         panic("trap: %s", trap_msg[tf->tf_type & ~T_KERNEL]);
 399         }
 400         CTR1(KTR_TRAP, "trap: td=%p return", td);
 401 }
 402
 403 static int
 404 trap_pfault(struct thread *td, struct trapframe *tf)
 405 {
 406         struct vmspace *vm;
 407         struct proc *p;
 408         vm_offset_t va;
 409         vm_prot_t prot;
 410         vm_map_entry_t entry;
 411         u_long ctx;
 412         int flags;
 413         int type;
 414         int rv;
 415
 416         if (td == NULL)
 417                 return (-1);
 418         KASSERT(td->td_pcb != NULL, ("trap_pfault: pcb NULL"));
 419         KASSERT(td->td_proc != NULL, ("trap_pfault: curproc NULL"));
 420         KASSERT(td->td_proc->p_vmspace != NULL, ("trap_pfault: vmspace NULL"));
 421
 422         p = td->td_proc;
 423
 424         rv = KERN_SUCCESS;
 425         ctx = TLB_TAR_CTX(tf->tf_tar);
 426         type = tf->tf_type & ~T_KERNEL;
 427         va = TLB_TAR_VA(tf->tf_tar);
 428
 429         CTR4(KTR_TRAP, "trap_pfault: td=%p pm_ctx=%#lx va=%#lx ctx=%#lx",
 430             td, p->p_vmspace->vm_pmap.pm_context[curcpu], va, ctx);
 431
 432         if (type == T_DATA_PROTECTION) {
 433                 prot = VM_PROT_WRITE;
 434                 flags = VM_FAULT_DIRTY;
 435         } else {
 436                 if (type == T_DATA_MISS)
 437                         prot = VM_PROT_READ;
 438                 else
 439                         prot = VM_PROT_READ | VM_PROT_EXECUTE;
 440                 flags = VM_FAULT_NORMAL;
 441         }
 442
 443         if (ctx != TLB_CTX_KERNEL) {
 444                 if ((tf->tf_tstate & TSTATE_PRIV) != 0 &&
 445                     (tf->tf_tpc >= (u_long)fs_nofault_intr_begin &&
 446                     tf->tf_tpc <= (u_long)fs_nofault_intr_end)) {
 447                         tf->tf_tpc = (u_long)fs_fault;
 448                         tf->tf_tnpc = tf->tf_tpc + 4;
 449                         return (0);
 450                 }
 451
 452                 /*
 453                  * This is a fault on non-kernel virtual memory.
 454                  */
 455                 vm = p->p_vmspace;
 456
 457                 /*
 458                  * Keep swapout from messing with us during this
 459                  * critical time.
 460                  */
 461                 PROC_LOCK(p);
 462                 ++p->p_lock;
 463                 PROC_UNLOCK(p);
 464
 465                 /* Fault in the user page. */
 466                 rv = vm_fault(&vm->vm_map, va, prot, flags);
 467
 468                 /*
 469                  * Now the process can be swapped again.
 470                  */
 471                 PROC_LOCK(p);
 472                 --p->p_lock;
 473                 PROC_UNLOCK(p);
 474         } else {
 475                 /*
 476                  * This is a fault on kernel virtual memory.  Attempts to
 477                  * access kernel memory from user mode cause privileged
 478                  * action traps, not page fault.
 479                  */
 480                 KASSERT(tf->tf_tstate & TSTATE_PRIV,
 481                     ("trap_pfault: fault on nucleus context from user mode"));
 482
 483                 if (tf->tf_tpc >= (u_long)copy_nofault_begin &&
 484                     tf->tf_tpc <= (u_long)copy_nofault_end) {
 485                         vm_map_lock_read(kernel_map);
 486                         if (vm_map_lookup_entry(kernel_map, va, &entry) &&
 487                             (entry->eflags & MAP_ENTRY_NOFAULT) != 0) {
 488                                 tf->tf_tpc = (u_long)copy_fault;
 489                                 tf->tf_tnpc = tf->tf_tpc + 4;
 490                                 vm_map_unlock_read(kernel_map);
 491                                 return (0);
 492                         }
 493                         vm_map_unlock_read(kernel_map);
 494                 }
 495
 496                 /*
 497                  * We don't have to worry about process locking or stacks in
 498                  * the kernel.
 499                  */
 500                 rv = vm_fault(kernel_map, va, prot, VM_FAULT_NORMAL);
 501         }
 502
 503         CTR3(KTR_TRAP, "trap_pfault: return td=%p va=%#lx rv=%d",
 504             td, va, rv);
 505         if (rv == KERN_SUCCESS)
 506                 return (0);
 507         if (ctx != TLB_CTX_KERNEL && (tf->tf_tstate & TSTATE_PRIV) != 0) {
 508                 if (tf->tf_tpc >= (u_long)fs_nofault_begin &&
 509                     tf->tf_tpc <= (u_long)fs_nofault_end) {
 510                         tf->tf_tpc = (u_long)fs_fault;
 511                         tf->tf_tnpc = tf->tf_tpc + 4;
 512                         return (0);
 513                 }
 514                 if (tf->tf_tpc >= (u_long)copy_nofault_begin &&
 515                     tf->tf_tpc <= (u_long)copy_nofault_end) {
 516                         tf->tf_tpc = (u_long)copy_fault;
 517                         tf->tf_tnpc = tf->tf_tpc + 4;
 518                         return (0);
 519                 }
 520         }
 521         return ((rv == KERN_PROTECTION_FAILURE) ? SIGBUS : SIGSEGV);
 522 }
 523
 524 /* Maximum number of arguments that can be passed via the out registers. */
 525 #define REG_MAXARGS     6
 526
 527 /*
 528  * Syscall handler. The arguments to the syscall are passed in the o registers
 529  * by the caller, and are saved in the trap frame. The syscall number is passed
 530  * in %g1 (and also saved in the trap frame).
 531  */
 532 void
 533 syscall(struct trapframe *tf)
 534 {
 535         struct sysent *callp;
 536         struct thread *td;
 537         register_t args[8];
 538         register_t *argp;
 539         struct proc *p;
 540         u_long code;
 541         u_long tpc;
 542         int reg;
 543         int regcnt;
 544         int narg;
 545         int error;
 546
 547         td = curthread;
 548         KASSERT(td != NULL, ("trap: curthread NULL"));
 549         KASSERT(td->td_proc != NULL, ("trap: curproc NULL"));
 550
 551         p = td->td_proc;
 552
 553         PCPU_INC(cnt.v_syscall);
 554
 555         td->td_pticks = 0;
 556         td->td_frame = tf;
 557         if (td->td_ucred != p->p_ucred)
 558                 cred_update_thread(td);
 559         code = tf->tf_global[1];
 560
 561         /*
 562          * For syscalls, we don't want to retry the faulting instruction
 563          * (usually), instead we need to advance one instruction.
 564          */
 565         tpc = tf->tf_tpc;
 566         TF_DONE(tf);
 567
 568         reg = 0;
 569         regcnt = REG_MAXARGS;
 570         if (p->p_sysent->sv_prepsyscall) {
 571                 /*
 572                  * The prep code is MP aware.
 573                  */
 574 #if 0
 575                 (*p->p_sysent->sv_prepsyscall)(tf, args, &code, &params);
 576 #endif
 577         } else if (code == SYS_syscall || code == SYS___syscall) {
 578                 code = tf->tf_out[reg++];
 579                 regcnt--;
 580         }
 581
 582         if (p->p_sysent->sv_mask)
 583                 code &= p->p_sysent->sv_mask;
 584
 585         if (code >= p->p_sysent->sv_size)
 586                 callp = &p->p_sysent->sv_table[0];
 587         else
 588                 callp = &p->p_sysent->sv_table[code];
 589
 590         narg = callp->sy_narg;
 591
 592         KASSERT(narg <= sizeof(args) / sizeof(args[0]),
 593             ("Too many syscall arguments!"));
 594         error = 0;
 595         argp = args;
 596         bcopy(&tf->tf_out[reg], args, sizeof(args[0]) * regcnt);
 597         if (narg > regcnt)
 598                 error = copyin((void *)(tf->tf_out[6] + SPOFF +
 599                     offsetof(struct frame, fr_pad[6])),
 600                     &args[regcnt], (narg - regcnt) * sizeof(args[0]));
 601
 602         CTR5(KTR_SYSC, "syscall: td=%p %s(%#lx, %#lx, %#lx)", td,
 603             syscallnames[code], argp[0], argp[1], argp[2]);
 604
 605 #ifdef KTRACE
 606         if (KTRPOINT(td, KTR_SYSCALL))
 607                 ktrsyscall(code, narg, argp);
 608 #endif
 609
 610         td->td_syscalls++;
 611
 612         if (error == 0) {
 613                 td->td_retval[0] = 0;
 614                 td->td_retval[1] = 0;
 615
 616                 STOPEVENT(p, S_SCE, narg);      /* MP aware */
 617
 618                 PTRACESTOP_SC(p, td, S_PT_SCE);
 619
 620                 AUDIT_SYSCALL_ENTER(code, td);
 621                 error = (*callp->sy_call)(td, argp);
 622                 AUDIT_SYSCALL_EXIT(error, td);
 623
 624                 CTR5(KTR_SYSC, "syscall: p=%p error=%d %s return %#lx %#lx ", p,
 625                     error, syscallnames[code], td->td_retval[0],
 626                     td->td_retval[1]);
 627         }
 628
 629         /*
 630          * MP SAFE (we may or may not have the MP lock at this point)
 631          */
 632         switch (error) {
 633         case 0:
 634                 tf->tf_out[0] = td->td_retval[0];
 635                 tf->tf_out[1] = td->td_retval[1];
 636                 tf->tf_tstate &= ~TSTATE_XCC_C;
 637                 break;
 638
 639         case ERESTART:
 640                 /*
 641                  * Undo the tpc advancement we have done above, we want to
 642                  * reexecute the system call.
 643                  */
 644                 tf->tf_tpc = tpc;
 645                 tf->tf_tnpc -= 4;
 646                 break;
 647
 648         case EJUSTRETURN:
 649                 break;
 650
 651         default:
 652                 if (p->p_sysent->sv_errsize) {
 653                         if (error >= p->p_sysent->sv_errsize)
 654                                 error = -1;     /* XXX */
 655                         else
 656                                 error = p->p_sysent->sv_errtbl[error];
 657                 }
 658                 tf->tf_out[0] = error;
 659                 tf->tf_tstate |= TSTATE_XCC_C;
 660                 break;
 661         }
 662
 663         /*
 664          * Check for misbehavior.
 665          */
 666         WITNESS_WARN(WARN_PANIC, NULL, "System call %s returning",
 667             (code >= 0 && code < SYS_MAXSYSCALL) ? syscallnames[code] : "???");
 668         KASSERT(td->td_critnest == 0,
 669             ("System call %s returning in a critical section",
 670             (code >= 0 && code < SYS_MAXSYSCALL) ? syscallnames[code] : "???"));
 671         KASSERT(td->td_locks == 0,
 672             ("System call %s returning with %d locks held",
 673             (code >= 0 && code < SYS_MAXSYSCALL) ? syscallnames[code] : "???",
 674             td->td_locks));
 675
 676         /*
 677          * Handle reschedule and other end-of-syscall issues
 678          */
 679         userret(td, tf);
 680
 681 #ifdef KTRACE
 682         if (KTRPOINT(td, KTR_SYSRET))
 683                 ktrsysret(code, error, td->td_retval[0]);
 684 #endif
 685         /*
 686          * This works because errno is findable through the
 687          * register set.  If we ever support an emulation where this
 688          * is not the case, this code will need to be revisited.
 689          */
 690         STOPEVENT(p, S_SCX, code);
 691
 692         PTRACESTOP_SC(p, td, S_PT_SCX);
 693 }