sys/sparc64/sparc64/trap.c

   1 /*-
   2  * Copyright (c) 2001, Jake Burkholder
   3  * Copyright (C) 1994, David Greenman
   4  * Copyright (c) 1990, 1993
   5  *      The Regents of the University of California.  All rights reserved.
   6  *
   7  * This code is derived from software contributed to Berkeley by
   8  * the University of Utah, and William Jolitz.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  * 3. All advertising materials mentioning features or use of this software
  19  *    must display the following acknowledgement:
  20  *      This product includes software developed by the University of
  21  *      California, Berkeley and its contributors.
  22  * 4. Neither the name of the University nor the names of its contributors
  23  *    may be used to endorse or promote products derived from this software
  24  *    without specific prior written permission.
  25  *
  26  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  27  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  28  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  29  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  30  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  31  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  32  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  33  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  34  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  35  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  36  * SUCH DAMAGE.
  37  *
  38  *      from: @(#)trap.c        7.4 (Berkeley) 5/13/91
  39  *      from: FreeBSD: src/sys/i386/i386/trap.c,v 1.197 2001/07/19
  40  */
  41
  42 #include <sys/cdefs.h>
  43 __FBSDID("$FreeBSD$");
  44
  45 #include "opt_ddb.h"
  46 #include "opt_ktr.h"
  47 #include "opt_ktrace.h"
  48
  49 #include <sys/param.h>
  50 #include <sys/kdb.h>
  51 #include <sys/kernel.h>
  52 #include <sys/bus.h>
  53 #include <sys/interrupt.h>
  54 #include <sys/ktr.h>
  55 #include <sys/lock.h>
  56 #include <sys/mutex.h>
  57 #include <sys/systm.h>
  58 #include <sys/pioctl.h>
  59 #include <sys/ptrace.h>
  60 #include <sys/proc.h>
  61 #include <sys/smp.h>
  62 #include <sys/signalvar.h>
  63 #include <sys/syscall.h>
  64 #include <sys/sysctl.h>
  65 #include <sys/sysent.h>
  66 #include <sys/vmmeter.h>
  67 #ifdef KTRACE
  68 #include <sys/uio.h>
  69 #include <sys/ktrace.h>
  70 #endif
  71 #include <security/audit/audit.h>
  72
  73 #include <dev/ofw/openfirm.h>
  74
  75 #include <vm/vm.h>
  76 #include <vm/pmap.h>
  77 #include <vm/vm_extern.h>
  78 #include <vm/vm_param.h>
  79 #include <vm/vm_kern.h>
  80 #include <vm/vm_map.h>
  81 #include <vm/vm_page.h>
  82
  83 #include <machine/cpu.h>
  84 #include <machine/frame.h>
  85 #include <machine/intr_machdep.h>
  86 #include <machine/smp.h>
  87 #include <machine/trap.h>
  88 #include <machine/tstate.h>
  89 #include <machine/tte.h>
  90 #include <machine/tlb.h>
  91 #include <machine/tsb.h>
  92 #include <machine/watch.h>
  93
  94 void trap(struct trapframe *tf);
  95 void syscall(struct trapframe *tf);
  96
  97 static int trap_pfault(struct thread *td, struct trapframe *tf);
  98
  99 extern char copy_fault[];
 100 extern char copy_nofault_begin[];
 101 extern char copy_nofault_end[];
 102
 103 extern char fs_fault[];
 104 extern char fs_nofault_begin[];
 105 extern char fs_nofault_end[];
 106 extern char fs_nofault_intr_begin[];
 107 extern char fs_nofault_intr_end[];
 108
 109 extern char fas_fault[];
 110 extern char fas_nofault_begin[];
 111 extern char fas_nofault_end[];
 112
 113 extern char *syscallnames[];
 114
 115 const char *const trap_msg[] = {
 116         "reserved",
 117         "instruction access exception",
 118         "instruction access error",
 119         "instruction access protection",
 120         "illtrap instruction",
 121         "illegal instruction",
 122         "privileged opcode",
 123         "floating point disabled",
 124         "floating point exception ieee 754",
 125         "floating point exception other",
 126         "tag overflow",
 127         "division by zero",
 128         "data access exception",
 129         "data access error",
 130         "data access protection",
 131         "memory address not aligned",
 132         "privileged action",
 133         "async data error",
 134         "trap instruction 16",
 135         "trap instruction 17",
 136         "trap instruction 18",
 137         "trap instruction 19",
 138         "trap instruction 20",
 139         "trap instruction 21",
 140         "trap instruction 22",
 141         "trap instruction 23",
 142         "trap instruction 24",
 143         "trap instruction 25",
 144         "trap instruction 26",
 145         "trap instruction 27",
 146         "trap instruction 28",
 147         "trap instruction 29",
 148         "trap instruction 30",
 149         "trap instruction 31",
 150         "fast instruction access mmu miss",
 151         "fast data access mmu miss",
 152         "interrupt",
 153         "physical address watchpoint",
 154         "virtual address watchpoint",
 155         "corrected ecc error",
 156         "spill",
 157         "fill",
 158         "fill",
 159         "breakpoint",
 160         "clean window",
 161         "range check",
 162         "fix alignment",
 163         "integer overflow",
 164         "syscall",
 165         "restore physical watchpoint",
 166         "restore virtual watchpoint",
 167         "kernel stack fault",
 168 };
 169
 170 static const int trap_sig[] = {
 171         SIGILL,                 /* reserved */
 172         SIGILL,                 /* instruction access exception */
 173         SIGILL,                 /* instruction access error */
 174         SIGILL,                 /* instruction access protection */
 175         SIGILL,                 /* illtrap instruction */
 176         SIGILL,                 /* illegal instruction */
 177         SIGBUS,                 /* privileged opcode */
 178         SIGFPE,                 /* floating point disabled */
 179         SIGFPE,                 /* floating point exception ieee 754 */
 180         SIGFPE,                 /* floating point exception other */
 181         SIGEMT,                 /* tag overflow */
 182         SIGFPE,                 /* division by zero */
 183         SIGILL,                 /* data access exception */
 184         SIGILL,                 /* data access error */
 185         SIGBUS,                 /* data access protection */
 186         SIGBUS,                 /* memory address not aligned */
 187         SIGBUS,                 /* privileged action */
 188         SIGBUS,                 /* async data error */
 189         SIGILL,                 /* trap instruction 16 */
 190         SIGILL,                 /* trap instruction 17 */
 191         SIGILL,                 /* trap instruction 18 */
 192         SIGILL,                 /* trap instruction 19 */
 193         SIGILL,                 /* trap instruction 20 */
 194         SIGILL,                 /* trap instruction 21 */
 195         SIGILL,                 /* trap instruction 22 */
 196         SIGILL,                 /* trap instruction 23 */
 197         SIGILL,                 /* trap instruction 24 */
 198         SIGILL,                 /* trap instruction 25 */
 199         SIGILL,                 /* trap instruction 26 */
 200         SIGILL,                 /* trap instruction 27 */
 201         SIGILL,                 /* trap instruction 28 */
 202         SIGILL,                 /* trap instruction 29 */
 203         SIGILL,                 /* trap instruction 30 */
 204         SIGILL,                 /* trap instruction 31 */
 205         SIGSEGV,                /* fast instruction access mmu miss */
 206         SIGSEGV,                /* fast data access mmu miss */
 207         -1,                     /* interrupt */
 208         -1,                     /* physical address watchpoint */
 209         -1,                     /* virtual address watchpoint */
 210         -1,                     /* corrected ecc error */
 211         SIGILL,                 /* spill */
 212         SIGILL,                 /* fill */
 213         SIGILL,                 /* fill */
 214         SIGTRAP,                /* breakpoint */
 215         SIGILL,                 /* clean window */
 216         SIGILL,                 /* range check */
 217         SIGILL,                 /* fix alignment */
 218         SIGILL,                 /* integer overflow */
 219         SIGSYS,                 /* syscall */
 220         -1,                     /* restore physical watchpoint */
 221         -1,                     /* restore virtual watchpoint */
 222         -1,                     /* kernel stack fault */
 223 };
 224
 225 CTASSERT(sizeof(struct trapframe) == 256);
 226
 227 int debugger_on_signal = 0;
 228 SYSCTL_INT(_debug, OID_AUTO, debugger_on_signal, CTLFLAG_RW,
 229     &debugger_on_signal, 0, "");
 230
 231 /*
 232  * SUNW,set-trap-table allows to take over %tba from the PROM, which
 233  * will turn off interrupts and handle outstanding ones while doing so,
 234  * in a safe way.
 235  */
 236 void
 237 sun4u_set_traptable(void *tba_addr)
 238 {
 239         static struct {
 240                 cell_t name;
 241                 cell_t nargs;
 242                 cell_t nreturns;
 243                 cell_t tba_addr;
 244         } args = {
 245                 (cell_t)"SUNW,set-trap-table",
 246                 2,
 247         };
 248
 249         args.tba_addr = (cell_t)tba_addr;
 250         openfirmware(&args);
 251 }
 252
 253 void
 254 trap(struct trapframe *tf)
 255 {
 256         struct thread *td;
 257         struct proc *p;
 258         int error;
 259         int sig;
 260         register_t addr;
 261         ksiginfo_t ksi;
 262
 263         td = curthread;
 264
 265         CTR4(KTR_TRAP, "trap: %p type=%s (%s) pil=%#lx", td,
 266             trap_msg[tf->tf_type & ~T_KERNEL],
 267             (TRAPF_USERMODE(tf) ? "user" : "kernel"), rdpr(pil));
 268
 269         PCPU_INC(cnt.v_trap);
 270
 271         if ((tf->tf_tstate & TSTATE_PRIV) == 0) {
 272                 KASSERT(td != NULL, ("trap: curthread NULL"));
 273                 KASSERT(td->td_proc != NULL, ("trap: curproc NULL"));
 274
 275                 p = td->td_proc;
 276                 td->td_pticks = 0;
 277                 td->td_frame = tf;
 278                 addr = tf->tf_tpc;
 279                 if (td->td_ucred != p->p_ucred)
 280                         cred_update_thread(td);
 281
 282                 switch (tf->tf_type) {
 283                 case T_DATA_MISS:
 284                 case T_DATA_PROTECTION:
 285                         addr = tf->tf_sfar;
 286                         /* FALLTHROUGH */
 287                 case T_INSTRUCTION_MISS:
 288                         sig = trap_pfault(td, tf);
 289                         break;
 290                 case T_FILL:
 291                         sig = rwindow_load(td, tf, 2);
 292                         break;
 293                 case T_FILL_RET:
 294                         sig = rwindow_load(td, tf, 1);
 295                         break;
 296                 case T_SPILL:
 297                         sig = rwindow_save(td);
 298                         break;
 299                 default:
 300                         if (tf->tf_type < 0 || tf->tf_type >= T_MAX ||
 301                             trap_sig[tf->tf_type] == -1)
 302                                 panic("trap: bad trap type");
 303                         sig = trap_sig[tf->tf_type];
 304                         break;
 305                 }
 306
 307                 if (sig != 0) {
 308                         /* Translate fault for emulators. */
 309                         if (p->p_sysent->sv_transtrap != NULL) {
 310                                 sig = p->p_sysent->sv_transtrap(sig,
 311                                     tf->tf_type);
 312                         }
 313                         if (debugger_on_signal &&
 314                             (sig == 4 || sig == 10 || sig == 11))
 315                                 kdb_enter_why(KDB_WHY_TRAPSIG, "trapsig");
 316                         ksiginfo_init_trap(&ksi);
 317                         ksi.ksi_signo = sig;
 318                         ksi.ksi_code = (int)tf->tf_type; /* XXX not POSIX */
 319                         ksi.ksi_addr = (void *)addr;
 320                         ksi.ksi_trapno = (int)tf->tf_type;
 321                         trapsignal(td, &ksi);
 322                 }
 323
 324                 userret(td, tf);
 325                 mtx_assert(&Giant, MA_NOTOWNED);
 326         } else {
 327                 KASSERT((tf->tf_type & T_KERNEL) != 0,
 328                     ("trap: kernel trap isn't"));
 329
 330                 if (kdb_active) {
 331                         kdb_reenter();
 332                         return;
 333                 }
 334
 335                 switch (tf->tf_type & ~T_KERNEL) {
 336                 case T_BREAKPOINT:
 337                 case T_KSTACK_FAULT:
 338                         error = (kdb_trap(tf->tf_type, 0, tf) == 0);
 339                         TF_DONE(tf);
 340                         break;
 341 #ifdef notyet
 342                 case T_PA_WATCHPOINT:
 343                 case T_VA_WATCHPOINT:
 344                         error = db_watch_trap(tf);
 345                         break;
 346 #endif
 347                 case T_DATA_MISS:
 348                 case T_DATA_PROTECTION:
 349                 case T_INSTRUCTION_MISS:
 350                         error = trap_pfault(td, tf);
 351                         break;
 352                 case T_DATA_EXCEPTION:
 353                 case T_MEM_ADDRESS_NOT_ALIGNED:
 354                         if ((tf->tf_sfsr & MMU_SFSR_FV) != 0 &&
 355                             MMU_SFSR_GET_ASI(tf->tf_sfsr) == ASI_AIUP) {
 356                                 if (tf->tf_tpc >= (u_long)copy_nofault_begin &&
 357                                     tf->tf_tpc <= (u_long)copy_nofault_end) {
 358                                         tf->tf_tpc = (u_long)copy_fault;
 359                                         tf->tf_tnpc = tf->tf_tpc + 4;
 360                                         error = 0;
 361                                         break;
 362                                 }
 363                                 if (tf->tf_tpc >= (u_long)fs_nofault_begin &&
 364                                     tf->tf_tpc <= (u_long)fs_nofault_end) {
 365                                         tf->tf_tpc = (u_long)fs_fault;
 366                                         tf->tf_tnpc = tf->tf_tpc + 4;
 367                                         error = 0;
 368                                         break;
 369                                 }
 370                         }
 371                         error = 1;
 372                         break;
 373                 case T_DATA_ERROR:
 374                         /*
 375                          * Handle PCI poke/peek as per UltraSPARC IIi
 376                          * User's Manual 16.2.1, modulo checking the
 377                          * TPC as USIII CPUs generate a precise trap
 378                          * instead of a special deferred one.
 379                          */
 380                         if (tf->tf_tpc > (u_long)fas_nofault_begin &&
 381                             tf->tf_tpc < (u_long)fas_nofault_end) {
 382                                 cache_flush();
 383                                 cache_enable();
 384                                 tf->tf_tpc = (u_long)fas_fault;
 385                                 tf->tf_tnpc = tf->tf_tpc + 4;
 386                                 error = 0;
 387                                 break;
 388                         }
 389                         error = 1;
 390                         break;
 391                 default:
 392                         error = 1;
 393                         break;
 394                 }
 395
 396                 if (error != 0)
 397                         panic("trap: %s", trap_msg[tf->tf_type & ~T_KERNEL]);
 398         }
 399         CTR1(KTR_TRAP, "trap: td=%p return", td);
 400 }
 401
 402 static int
 403 trap_pfault(struct thread *td, struct trapframe *tf)
 404 {
 405         struct vmspace *vm;
 406         struct proc *p;
 407         vm_offset_t va;
 408         vm_prot_t prot;
 409         vm_map_entry_t entry;
 410         u_long ctx;
 411         int flags;
 412         int type;
 413         int rv;
 414
 415         if (td == NULL)
 416                 return (-1);
 417         KASSERT(td->td_pcb != NULL, ("trap_pfault: pcb NULL"));
 418         KASSERT(td->td_proc != NULL, ("trap_pfault: curproc NULL"));
 419         KASSERT(td->td_proc->p_vmspace != NULL, ("trap_pfault: vmspace NULL"));
 420
 421         p = td->td_proc;
 422
 423         rv = KERN_SUCCESS;
 424         ctx = TLB_TAR_CTX(tf->tf_tar);
 425         type = tf->tf_type & ~T_KERNEL;
 426         va = TLB_TAR_VA(tf->tf_tar);
 427
 428         CTR4(KTR_TRAP, "trap_pfault: td=%p pm_ctx=%#lx va=%#lx ctx=%#lx",
 429             td, p->p_vmspace->vm_pmap.pm_context[curcpu], va, ctx);
 430
 431         if (type == T_DATA_PROTECTION) {
 432                 prot = VM_PROT_WRITE;
 433                 flags = VM_FAULT_DIRTY;
 434         } else {
 435                 if (type == T_DATA_MISS)
 436                         prot = VM_PROT_READ;
 437                 else
 438                         prot = VM_PROT_READ | VM_PROT_EXECUTE;
 439                 flags = VM_FAULT_NORMAL;
 440         }
 441
 442         if (ctx != TLB_CTX_KERNEL) {
 443                 if ((tf->tf_tstate & TSTATE_PRIV) != 0 &&
 444                     (tf->tf_tpc >= (u_long)fs_nofault_intr_begin &&
 445                     tf->tf_tpc <= (u_long)fs_nofault_intr_end)) {
 446                         tf->tf_tpc = (u_long)fs_fault;
 447                         tf->tf_tnpc = tf->tf_tpc + 4;
 448                         return (0);
 449                 }
 450
 451                 /*
 452                  * This is a fault on non-kernel virtual memory.
 453                  */
 454                 vm = p->p_vmspace;
 455
 456                 /*
 457                  * Keep swapout from messing with us during this
 458                  * critical time.
 459                  */
 460                 PROC_LOCK(p);
 461                 ++p->p_lock;
 462                 PROC_UNLOCK(p);
 463
 464                 /* Fault in the user page. */
 465                 rv = vm_fault(&vm->vm_map, va, prot, flags);
 466
 467                 /*
 468                  * Now the process can be swapped again.
 469                  */
 470                 PROC_LOCK(p);
 471                 --p->p_lock;
 472                 PROC_UNLOCK(p);
 473         } else {
 474                 /*
 475                  * This is a fault on kernel virtual memory.  Attempts to
 476                  * access kernel memory from user mode cause privileged
 477                  * action traps, not page fault.
 478                  */
 479                 KASSERT(tf->tf_tstate & TSTATE_PRIV,
 480                     ("trap_pfault: fault on nucleus context from user mode"));
 481
 482                 if (tf->tf_tpc >= (u_long)copy_nofault_begin &&
 483                     tf->tf_tpc <= (u_long)copy_nofault_end) {
 484                         vm_map_lock_read(kernel_map);
 485                         if (vm_map_lookup_entry(kernel_map, va, &entry) &&
 486                             (entry->eflags & MAP_ENTRY_NOFAULT) != 0) {
 487                                 tf->tf_tpc = (u_long)copy_fault;
 488                                 tf->tf_tnpc = tf->tf_tpc + 4;
 489                                 vm_map_unlock_read(kernel_map);
 490                                 return (0);
 491                         }
 492                         vm_map_unlock_read(kernel_map);
 493                 }
 494
 495                 /*
 496                  * We don't have to worry about process locking or stacks in
 497                  * the kernel.
 498                  */
 499                 rv = vm_fault(kernel_map, va, prot, VM_FAULT_NORMAL);
 500         }
 501
 502         CTR3(KTR_TRAP, "trap_pfault: return td=%p va=%#lx rv=%d",
 503             td, va, rv);
 504         if (rv == KERN_SUCCESS)
 505                 return (0);
 506         if (ctx != TLB_CTX_KERNEL && (tf->tf_tstate & TSTATE_PRIV) != 0) {
 507                 if (tf->tf_tpc >= (u_long)fs_nofault_begin &&
 508                     tf->tf_tpc <= (u_long)fs_nofault_end) {
 509                         tf->tf_tpc = (u_long)fs_fault;
 510                         tf->tf_tnpc = tf->tf_tpc + 4;
 511                         return (0);
 512                 }
 513                 if (tf->tf_tpc >= (u_long)copy_nofault_begin &&
 514                     tf->tf_tpc <= (u_long)copy_nofault_end) {
 515                         tf->tf_tpc = (u_long)copy_fault;
 516                         tf->tf_tnpc = tf->tf_tpc + 4;
 517                         return (0);
 518                 }
 519         }
 520         return ((rv == KERN_PROTECTION_FAILURE) ? SIGBUS : SIGSEGV);
 521 }
 522
 523 /* Maximum number of arguments that can be passed via the out registers. */
 524 #define REG_MAXARGS     6
 525
 526 /*
 527  * Syscall handler. The arguments to the syscall are passed in the o registers
 528  * by the caller, and are saved in the trap frame. The syscall number is passed
 529  * in %g1 (and also saved in the trap frame).
 530  */
 531 void
 532 syscall(struct trapframe *tf)
 533 {
 534         struct sysent *callp;
 535         struct thread *td;
 536         register_t args[8];
 537         register_t *argp;
 538         struct proc *p;
 539         u_long code;
 540         u_long tpc;
 541         int reg;
 542         int regcnt;
 543         int narg;
 544         int error;
 545
 546         td = curthread;
 547         KASSERT(td != NULL, ("trap: curthread NULL"));
 548         KASSERT(td->td_proc != NULL, ("trap: curproc NULL"));
 549
 550         p = td->td_proc;
 551
 552         PCPU_INC(cnt.v_syscall);
 553
 554         td->td_pticks = 0;
 555         td->td_frame = tf;
 556         if (td->td_ucred != p->p_ucred)
 557                 cred_update_thread(td);
 558 #ifdef KSE
 559         if (p->p_flag & P_SA)
 560                 thread_user_enter(td);
 561 #endif
 562         code = tf->tf_global[1];
 563
 564         /*
 565          * For syscalls, we don't want to retry the faulting instruction
 566          * (usually), instead we need to advance one instruction.
 567          */
 568         tpc = tf->tf_tpc;
 569         TF_DONE(tf);
 570
 571         reg = 0;
 572         regcnt = REG_MAXARGS;
 573         if (p->p_sysent->sv_prepsyscall) {
 574                 /*
 575                  * The prep code is MP aware.
 576                  */
 577 #if 0
 578                 (*p->p_sysent->sv_prepsyscall)(tf, args, &code, &params);
 579 #endif
 580         } else if (code == SYS_syscall || code == SYS___syscall) {
 581                 code = tf->tf_out[reg++];
 582                 regcnt--;
 583         }
 584
 585         if (p->p_sysent->sv_mask)
 586                 code &= p->p_sysent->sv_mask;
 587
 588         if (code >= p->p_sysent->sv_size)
 589                 callp = &p->p_sysent->sv_table[0];
 590         else
 591                 callp = &p->p_sysent->sv_table[code];
 592
 593         narg = callp->sy_narg;
 594
 595         KASSERT(narg <= sizeof(args) / sizeof(args[0]),
 596             ("Too many syscall arguments!"));
 597         error = 0;
 598         argp = args;
 599         bcopy(&tf->tf_out[reg], args, sizeof(args[0]) * regcnt);
 600         if (narg > regcnt)
 601                 error = copyin((void *)(tf->tf_out[6] + SPOFF +
 602                     offsetof(struct frame, fr_pad[6])),
 603                     &args[regcnt], (narg - regcnt) * sizeof(args[0]));
 604
 605         CTR5(KTR_SYSC, "syscall: td=%p %s(%#lx, %#lx, %#lx)", td,
 606             syscallnames[code], argp[0], argp[1], argp[2]);
 607
 608 #ifdef KTRACE
 609         if (KTRPOINT(td, KTR_SYSCALL))
 610                 ktrsyscall(code, narg, argp);
 611 #endif
 612
 613         td->td_syscalls++;
 614
 615         if (error == 0) {
 616                 td->td_retval[0] = 0;
 617                 td->td_retval[1] = 0;
 618
 619                 STOPEVENT(p, S_SCE, narg);      /* MP aware */
 620
 621                 PTRACESTOP_SC(p, td, S_PT_SCE);
 622
 623                 AUDIT_SYSCALL_ENTER(code, td);
 624                 error = (*callp->sy_call)(td, argp);
 625                 AUDIT_SYSCALL_EXIT(error, td);
 626
 627                 CTR5(KTR_SYSC, "syscall: p=%p error=%d %s return %#lx %#lx ", p,
 628                     error, syscallnames[code], td->td_retval[0],
 629                     td->td_retval[1]);
 630         }
 631
 632         /*
 633          * MP SAFE (we may or may not have the MP lock at this point)
 634          */
 635         switch (error) {
 636         case 0:
 637                 tf->tf_out[0] = td->td_retval[0];
 638                 tf->tf_out[1] = td->td_retval[1];
 639                 tf->tf_tstate &= ~TSTATE_XCC_C;
 640                 break;
 641
 642         case ERESTART:
 643                 /*
 644                  * Undo the tpc advancement we have done above, we want to
 645                  * reexecute the system call.
 646                  */
 647                 tf->tf_tpc = tpc;
 648                 tf->tf_tnpc -= 4;
 649                 break;
 650
 651         case EJUSTRETURN:
 652                 break;
 653
 654         default:
 655                 if (p->p_sysent->sv_errsize) {
 656                         if (error >= p->p_sysent->sv_errsize)
 657                                 error = -1;     /* XXX */
 658                         else
 659                                 error = p->p_sysent->sv_errtbl[error];
 660                 }
 661                 tf->tf_out[0] = error;
 662                 tf->tf_tstate |= TSTATE_XCC_C;
 663                 break;
 664         }
 665
 666         /*
 667          * Check for misbehavior.
 668          */
 669         WITNESS_WARN(WARN_PANIC, NULL, "System call %s returning",
 670             (code >= 0 && code < SYS_MAXSYSCALL) ? syscallnames[code] : "???");
 671         KASSERT(td->td_critnest == 0,
 672             ("System call %s returning in a critical section",
 673             (code >= 0 && code < SYS_MAXSYSCALL) ? syscallnames[code] : "???"));
 674         KASSERT(td->td_locks == 0,
 675             ("System call %s returning with %d locks held",
 676             (code >= 0 && code < SYS_MAXSYSCALL) ? syscallnames[code] : "???",
 677             td->td_locks));
 678
 679         /*
 680          * Handle reschedule and other end-of-syscall issues
 681          */
 682         userret(td, tf);
 683
 684 #ifdef KTRACE
 685         if (KTRPOINT(td, KTR_SYSRET))
 686                 ktrsysret(code, error, td->td_retval[0]);
 687 #endif
 688         /*
 689          * This works because errno is findable through the
 690          * register set.  If we ever support an emulation where this
 691          * is not the case, this code will need to be revisited.
 692          */
 693         STOPEVENT(p, S_SCX, code);
 694
 695         PTRACESTOP_SC(p, td, S_PT_SCX);
 696 }