sys/sun4v/sun4v/trap.c

   1 /*-
   2  * Copyright (c) 2001, Jake Burkholder
   3  * Copyright (C) 1994, David Greenman
   4  * Copyright (c) 1990, 1993
   5  *      The Regents of the University of California.  All rights reserved.
   6  *
   7  * This code is derived from software contributed to Berkeley by
   8  * the University of Utah, and William Jolitz.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  * 3. All advertising materials mentioning features or use of this software
  19  *    must display the following acknowledgement:
  20  *      This product includes software developed by the University of
  21  *      California, Berkeley and its contributors.
  22  * 4. Neither the name of the University nor the names of its contributors
  23  *    may be used to endorse or promote products derived from this software
  24  *    without specific prior written permission.
  25  *
  26  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  27  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  28  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  29  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  30  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  31  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  32  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  33  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  34  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  35  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  36  * SUCH DAMAGE.
  37  *
  38  *      from: @(#)trap.c        7.4 (Berkeley) 5/13/91
  39  *      from: FreeBSD: src/sys/i386/i386/trap.c,v 1.197 2001/07/19
  40  * $FreeBSD$
  41  */
  42
  43 #include "opt_ddb.h"
  44 #include "opt_ktr.h"
  45 #include "opt_ktrace.h"
  46
  47 #include <sys/param.h>
  48 #include <sys/kdb.h>
  49 #include <sys/kernel.h>
  50 #include <sys/bus.h>
  51 #include <sys/interrupt.h>
  52 #include <sys/ktr.h>
  53 #include <sys/lock.h>
  54 #include <sys/mutex.h>
  55 #include <sys/systm.h>
  56 #include <sys/pioctl.h>
  57 #include <sys/ptrace.h>
  58 #include <sys/proc.h>
  59 #include <sys/smp.h>
  60 #include <sys/signalvar.h>
  61 #include <sys/syscall.h>
  62 #include <sys/sysctl.h>
  63 #include <sys/sysent.h>
  64 #include <sys/vmmeter.h>
  65 #ifdef KTRACE
  66 #include <sys/uio.h>
  67 #include <sys/ktrace.h>
  68 #endif
  69
  70 #include <dev/ofw/openfirm.h>
  71
  72 #include <vm/vm.h>
  73 #include <vm/pmap.h>
  74 #include <vm/vm_extern.h>
  75 #include <vm/vm_param.h>
  76 #include <vm/vm_kern.h>
  77 #include <vm/vm_map.h>
  78 #include <vm/vm_page.h>
  79
  80 #include <machine/clock.h>
  81 #include <machine/cpu.h>
  82 #include <machine/frame.h>
  83 #include <machine/intr_machdep.h>
  84 #include <machine/pcb.h>
  85 #include <machine/smp.h>
  86 #include <machine/trap.h>
  87 #include <machine/tstate.h>
  88 #include <machine/tte.h>
  89 #include <machine/tlb.h>
  90 #include <machine/tsb.h>
  91 #include <machine/watch.h>
  92 #include <machine/wstate.h>
  93
  94 #include <machine/md_var.h>
  95 #include <machine/hypervisorvar.h>
  96
  97 #include <security/audit/audit.h>
  98
  99 void trap(struct trapframe *tf, int64_t type, uint64_t data);
 100 void syscall(struct trapframe *tf);
 101
 102 extern vm_paddr_t mmu_fault_status_area;
 103
 104 static int trap_pfault(struct thread *td, struct trapframe *tf, int64_t type, uint64_t data);
 105
 106 extern char copy_fault[];
 107 extern char copy_nofault_begin[];
 108 extern char copy_nofault_end[];
 109
 110 extern char fs_fault[];
 111 extern char fs_nofault_begin[];
 112 extern char fs_nofault_end[];
 113 extern char fs_nofault_intr_begin[];
 114 extern char fs_nofault_intr_end[];
 115
 116 extern char fas_fault[];
 117 extern char fas_nofault_begin[];
 118 extern char fas_nofault_end[];
 119
 120 extern char *syscallnames[];
 121
 122 const char *trap_msg[] = {
 123         "reserved",
 124         "instruction access exception",
 125         "instruction access error",
 126         "instruction access protection",
 127         "illtrap instruction",
 128         "illegal instruction",
 129         "privileged opcode",
 130         "floating point disabled",
 131         "floating point exception ieee 754",
 132         "floating point exception other",
 133         "tag overflow",
 134         "division by zero",
 135         "data access exception",
 136         "data access error",
 137         "data access protection",
 138         "memory address not aligned",
 139         "privileged action",
 140         "async data error",
 141         "trap instruction 16",
 142         "trap instruction 17",
 143         "trap instruction 18",
 144         "trap instruction 19",
 145         "trap instruction 20",
 146         "trap instruction 21",
 147         "trap instruction 22",
 148         "trap instruction 23",
 149         "trap instruction 24",
 150         "trap instruction 25",
 151         "trap instruction 26",
 152         "trap instruction 27",
 153         "trap instruction 28",
 154         "trap instruction 29",
 155         "trap instruction 30",
 156         "trap instruction 31",
 157         "fast instruction access mmu miss",
 158         "fast data access mmu miss",
 159         "interrupt",
 160         "physical address watchpoint",
 161         "virtual address watchpoint",
 162         "corrected ecc error",
 163         "spill",
 164         "fill",
 165         "fill",
 166         "breakpoint",
 167         "clean window",
 168         "range check",
 169         "fix alignment",
 170         "integer overflow",
 171         "syscall",
 172         "restore physical watchpoint",
 173         "restore virtual watchpoint",
 174         "kernel stack fault",
 175         "resumable error",
 176         "nonresumable error"
 177 };
 178
 179 const int trap_sig[] = {
 180         SIGILL,                 /* reserved */
 181         SIGILL,                 /* instruction access exception */
 182         SIGILL,                 /* instruction access error */
 183         SIGILL,                 /* instruction access protection */
 184         SIGILL,                 /* illtrap instruction */
 185         SIGILL,                 /* illegal instruction */
 186         SIGBUS,                 /* privileged opcode */
 187         SIGFPE,                 /* floating point disabled */
 188         SIGFPE,                 /* floating point exception ieee 754 */
 189         SIGFPE,                 /* floating point exception other */
 190         SIGEMT,                 /* tag overflow */
 191         SIGFPE,                 /* division by zero */
 192         SIGILL,                 /* data access exception */
 193         SIGILL,                 /* data access error */
 194         SIGBUS,                 /* data access protection */
 195         SIGBUS,                 /* memory address not aligned */
 196         SIGBUS,                 /* privileged action */
 197         SIGBUS,                 /* async data error */
 198         SIGILL,                 /* trap instruction 16 */
 199         SIGILL,                 /* trap instruction 17 */
 200         SIGILL,                 /* trap instruction 18 */
 201         SIGILL,                 /* trap instruction 19 */
 202         SIGILL,                 /* trap instruction 20 */
 203         SIGILL,                 /* trap instruction 21 */
 204         SIGILL,                 /* trap instruction 22 */
 205         SIGILL,                 /* trap instruction 23 */
 206         SIGILL,                 /* trap instruction 24 */
 207         SIGILL,                 /* trap instruction 25 */
 208         SIGILL,                 /* trap instruction 26 */
 209         SIGILL,                 /* trap instruction 27 */
 210         SIGILL,                 /* trap instruction 28 */
 211         SIGILL,                 /* trap instruction 29 */
 212         SIGILL,                 /* trap instruction 30 */
 213         SIGILL,                 /* trap instruction 31 */
 214         SIGFPE,                 /* floating point error */
 215         SIGSEGV,                /* fast data access mmu miss */
 216         -1,                     /* interrupt */
 217         -1,                     /* physical address watchpoint */
 218         -1,                     /* virtual address watchpoint */
 219         -1,                     /* corrected ecc error */
 220         SIGILL,                 /* spill */
 221         SIGILL,                 /* fill */
 222         SIGILL,                 /* fill */
 223         SIGTRAP,                /* breakpoint */
 224         SIGILL,                 /* clean window */
 225         SIGILL,                 /* range check */
 226         SIGILL,                 /* fix alignment */
 227         SIGILL,                 /* integer overflow */
 228         SIGSYS,                 /* syscall */
 229         -1,                     /* restore physical watchpoint */
 230         -1,                     /* restore virtual watchpoint */
 231         -1,                     /* kernel stack fault */
 232 };
 233
 234 CTASSERT(sizeof(struct trapframe) == 256);
 235
 236 int debugger_on_signal = 0;
 237 #ifdef DEBUG
 238 SYSCTL_INT(_debug, OID_AUTO, debugger_on_signal, CTLFLAG_RW,
 239     &debugger_on_signal, 0, "");
 240 #endif
 241
 242 /*
 243  * This interface allows the client to safely take over the %tba by
 244  * the prom's service. The prom will take care of the quiescence of
 245  * interrupts and handle any pending soft interrupts.
 246  * This call also sets the MMU fault status area for the CPU.
 247  */
 248 void
 249 set_mmfsa_traptable(void *tba_addr, uint64_t mmfsa_ra)
 250 {
 251         static struct {
 252         cell_t name;
 253                 cell_t nargs;
 254                 cell_t nreturns;
 255                 cell_t tba_addr;
 256                 cell_t mmfsa_ra;
 257         } args = {
 258                 (cell_t)"SUNW,set-trap-table",
 259                 2,
 260         };
 261
 262         args.tba_addr = (cell_t)tba_addr;
 263         args.mmfsa_ra = mmfsa_ra;
 264         openfirmware(&args);
 265 }
 266
 267 void
 268 trap_init(void)
 269 {
 270         vm_paddr_t mmfsa;
 271
 272         mmfsa = mmu_fault_status_area + (MMFSA_SIZE*curcpu);
 273
 274         set_wstate(WSTATE_KERN);
 275         set_mmfsa_scratchpad(mmfsa);
 276
 277         init_mondo_queue();
 278         set_mmfsa_traptable(&tl0_base, mmfsa);
 279 }
 280
 281 void
 282 trap(struct trapframe *tf, int64_t type, uint64_t data)
 283 {
 284         struct thread *td;
 285         struct proc *p;
 286         int error, sig, ctx;
 287         uint64_t trapno;
 288         register_t addr;
 289         ksiginfo_t ksi;
 290
 291         td = curthread;
 292
 293         CTR4(KTR_TRAP, "trap: %p type=%s (%s) pil=%#lx", td,
 294             trap_msg[trapno],
 295             (TRAPF_USERMODE(tf) ? "user" : "kernel"), rdpr(pil));
 296
 297         PCPU_INC(cnt.v_trap);
 298
 299         trapno = (type & TRAP_MASK);
 300         ctx = (type >> TRAP_CTX_SHIFT);
 301
 302         if (((tf->tf_tstate & TSTATE_PRIV) == 0) || (ctx != 0)) {
 303                 KASSERT(td != NULL, ("trap: curthread NULL"));
 304                 KASSERT(td->td_proc != NULL, ("trap: curproc NULL"));
 305
 306                 p = td->td_proc;
 307                 td->td_pticks = 0;
 308                 td->td_frame = tf;
 309                 addr = tf->tf_tpc;
 310                 if (td->td_ucred != p->p_ucred)
 311                         cred_update_thread(td);
 312
 313                 switch (trapno) {
 314                 case T_DATA_MISS:
 315                 case T_DATA_PROTECTION:
 316                         addr = TLB_TAR_VA(data);
 317                 case T_INSTRUCTION_MISS:
 318                         sig = trap_pfault(td, tf, trapno, data);
 319                         break;
 320                 case T_FILL:
 321                         sig = rwindow_load(td, tf, 2);
 322                         break;
 323                 case T_FILL_RET:
 324                         sig = rwindow_load(td, tf, 1);
 325                         break;
 326                 case T_SPILL:
 327                         sig = rwindow_save(td);
 328                         break;
 329                 case T_DATA_EXCEPTION:
 330                 case T_DATA_ERROR:
 331                 case T_MEM_ADDRESS_NOT_ALIGNED:
 332                         printf("bad trap trapno=%ld data=0x%lx pc=0x%lx\n",
 333                                trapno, data, tf->tf_tpc);
 334                         if (tf->tf_tpc >= (u_long)copy_nofault_begin &&
 335                             tf->tf_tpc <= (u_long)copy_nofault_end) {
 336                                 tf->tf_tpc = (u_long)copy_fault;
 337                                 tf->tf_tnpc = tf->tf_tpc + 4;
 338                                 sig = 0;
 339                                 break;
 340                         }
 341                         if (tf->tf_tpc >= (u_long)fs_nofault_begin &&
 342                             tf->tf_tpc <= (u_long)fs_nofault_end) {
 343                                 tf->tf_tpc = (u_long)fs_fault;
 344                                 tf->tf_tnpc = tf->tf_tpc + 4;
 345                                 sig = 0;
 346                                 break;
 347                         }
 348
 349                         addr = data;
 350                         sig = trap_sig[trapno];
 351                         break;
 352
 353                 default:
 354                         if (trapno < 0 || trapno >= T_MAX ||
 355                             trap_sig[trapno] == -1)
 356                                 panic("trap: bad trap type");
 357                         sig = trap_sig[trapno];
 358                         break;
 359                 }
 360
 361                 if (sig != 0) {
 362                         /* Translate fault for emulators. */
 363                         if (p->p_sysent->sv_transtrap != NULL) {
 364                                 sig = p->p_sysent->sv_transtrap(sig,
 365                                     trapno);
 366                         }
 367                         if (debugger_on_signal &&
 368                             (sig == 4 || sig == 10 || sig == 11))
 369                                 kdb_enter("trapsig");
 370 #ifdef VERBOSE
 371                         if (sig == 4 || sig == 10 || sig == 11)
 372                                 printf("trap: %ld:%s: 0x%lx at 0x%lx on cpu=%d sig=%d proc=%s\n",
 373                                        trapno, trap_msg[trapno], data, tf->tf_tpc, curcpu, sig, curthread->td_proc->p_comm);
 374 #endif
 375                         /* XXX I've renumbered the traps to largely reflect what the hardware uses
 376                          * so this will need to be re-visited
 377                          */
 378                         ksiginfo_init_trap(&ksi);
 379                         ksi.ksi_signo = sig;
 380                         ksi.ksi_code = (int)trapno; /* XXX not POSIX */
 381                         ksi.ksi_addr = (void *)addr;
 382                         ksi.ksi_trapno = (int)trapno;
 383                         trapsignal(td, &ksi);
 384                 }
 385
 386                 userret(td, tf);
 387                 mtx_assert(&Giant, MA_NOTOWNED);
 388         } else {
 389                 KASSERT((type & T_KERNEL) != 0,
 390                     ("trap: kernel trap isn't - trap: %ld:%s: 0x%lx at 0x%lx on cpu=%d\n",
 391                      trapno, trap_msg[trapno], data, tf->tf_tpc, curcpu));
 392
 393 #ifdef KDB
 394                 if (kdb_active) {
 395                         kdb_reenter();
 396                         return;
 397                 }
 398 #endif
 399
 400                 switch (trapno) {
 401 #ifdef KDB
 402                 case T_BREAKPOINT:
 403                 case T_KSTACK_FAULT:
 404                         error = (kdb_trap(trapno, 0, tf) == 0);
 405                         TF_DONE(tf);
 406                         break;
 407 #endif
 408                 case T_DATA_MISS:
 409                 case T_DATA_PROTECTION:
 410                 case T_INSTRUCTION_MISS:
 411                         error = trap_pfault(td, tf, trapno, data);
 412                         break;
 413                 case T_DATA_EXCEPTION:
 414                         printf("data exception on 0x%lx at 0x%lx\n", data, tf->tf_tpc);
 415                         printf("trap: %ld=%s: 0x%lx at 0x%lx:0x%lx\n", trapno,
 416                                trap_msg[trapno], data, tf->tf_tpc, tf->tf_tnpc);
 417                case T_ILLEGAL_INSTRUCTION:
 418                        if (tf->tf_tpc > KERNBASE) {
 419                                printf("illinstr: 0x%lx\n", tf->tf_tpc);
 420                                printf("illinstr: 0x%x\n", *((uint32_t *)tf->tf_tpc));
 421                        }
 422                 case T_DATA_ERROR:
 423                 case T_ALIGNMENT:
 424                         if (tf->tf_asi == ASI_AIUS) {
 425                                 if (tf->tf_tpc >= (u_long)copy_nofault_begin &&
 426                                     tf->tf_tpc <= (u_long)copy_nofault_end) {
 427                                         tf->tf_tpc = (u_long)copy_fault;
 428                                         tf->tf_tnpc = tf->tf_tpc + 4;
 429                                         error = 0;
 430                                         break;
 431                                 }
 432
 433                                 printf("ASI_AIUS but bad tpc\n");
 434                         }
 435                         if (tf->tf_tpc >= (u_long)fs_nofault_begin &&
 436                             tf->tf_tpc <= (u_long)fs_nofault_end) {
 437                                 tf->tf_tpc = (u_long)fs_fault;
 438                                 tf->tf_tnpc = tf->tf_tpc + 4;
 439                                 error = 0;
 440                                         break;
 441                         }
 442                         printf("asi=0x%lx\n", tf->tf_asi);
 443                         error = 1;
 444                         break;
 445                 default:
 446                         printf("unchecked trap 0x%lx asi=0x%lx\n", trapno, tf->tf_asi);
 447                         error = 1;
 448                         break;
 449                 }
 450
 451                 if (error != 0)
 452                         panic("trap: %ld=%s: 0x%lx at 0x%lx:0x%lx error=%d asi=0x%lx",
 453                               trapno, trap_msg[trapno], data, tf->tf_tpc,
 454                               tf->tf_tnpc, error, tf->tf_asi);
 455         }
 456         CTR1(KTR_TRAP, "trap: td=%p return", td);
 457 }
 458
 459 static int
 460 trap_pfault(struct thread *td, struct trapframe *tf, int64_t type, uint64_t data)
 461 {
 462         struct vmspace *vm;
 463         struct pcb *pcb;
 464         struct proc *p;
 465         vm_offset_t va;
 466         vm_prot_t prot;
 467         u_long ctx;
 468         int flags;
 469         int rv;
 470
 471         if (td == NULL)
 472                 return (-1);
 473
 474         p = td->td_proc;
 475
 476         rv = KERN_SUCCESS;
 477         ctx = TLB_TAR_CTX(data);
 478         pcb = td->td_pcb;
 479         type = type & ~T_KERNEL;
 480         va = TLB_TAR_VA(data);
 481
 482
 483         if (data > VM_MIN_DIRECT_ADDRESS)
 484                 printf("trap_pfault(type=%ld, data=0x%lx, tpc=0x%lx, ctx=0x%lx)\n",
 485                        type, data, tf->tf_tpc, ctx);
 486
 487 #if 0
 488         CTR4(KTR_TRAP, "trap_pfault: td=%p pm_ctx=%#lx va=%#lx ctx=%#lx",
 489             td, p->p_vmspace->vm_pmap.pm_context, va, ctx);
 490 #endif
 491         KASSERT(td->td_pcb != NULL, ("trap_pfault: pcb NULL"));
 492         KASSERT(td->td_proc != NULL, ("trap_pfault: curproc NULL"));
 493         KASSERT(td->td_proc->p_vmspace != NULL, ("trap_pfault: vmspace NULL"));
 494
 495
 496         if (type == T_DATA_PROTECTION) {
 497                 prot = VM_PROT_WRITE;
 498                 flags = VM_FAULT_DIRTY;
 499         } else {
 500                 if (type == T_DATA_MISS)
 501                         prot = VM_PROT_READ;
 502                 else
 503                         prot = VM_PROT_READ | VM_PROT_EXECUTE;
 504                 flags = VM_FAULT_NORMAL;
 505         }
 506
 507         if (ctx != TLB_CTX_KERNEL) {
 508                 if ((tf->tf_tstate & TSTATE_PRIV) != 0 &&
 509                     (tf->tf_tpc >= (u_long)fs_nofault_intr_begin &&
 510                      tf->tf_tpc <= (u_long)fs_nofault_intr_end)) {
 511                         tf->tf_tpc = (u_long)fs_fault;
 512                         tf->tf_tnpc = tf->tf_tpc + 4;
 513                         return (0);
 514                 }
 515
 516                 /*
 517                  * This is a fault on non-kernel virtual memory.
 518                  */
 519                 vm = p->p_vmspace;
 520
 521                 /*
 522                  * Keep swapout from messing with us during this
 523                  * critical time.
 524                  */
 525                 PROC_LOCK(p);
 526                 ++p->p_lock;
 527                 PROC_UNLOCK(p);
 528
 529                 /* Fault in the user page. */
 530                 rv = vm_fault(&vm->vm_map, va, prot, flags);
 531
 532                 /*
 533                  * Now the process can be swapped again.
 534                  */
 535                 PROC_LOCK(p);
 536                 --p->p_lock;
 537                 PROC_UNLOCK(p);
 538         } else {
 539                 /*
 540                  * This is a fault on kernel virtual memory.  Attempts to
 541                  * access kernel memory from user mode cause privileged
 542                  * action traps, not page fault.
 543                  */
 544                 KASSERT(tf->tf_tstate & TSTATE_PRIV,
 545                     ("trap_pfault: fault on nucleus context from user mode"));
 546
 547                 /*
 548                  * Don't have to worry about process locking or stacks in the
 549                  * kernel.
 550                  */
 551                 rv = vm_fault(kernel_map, va, prot, VM_FAULT_NORMAL);
 552         }
 553
 554         CTR3(KTR_TRAP, "trap_pfault: return td=%p va=%#lx rv=%d",
 555             td, va, rv);
 556         if (rv == KERN_SUCCESS)
 557                 return (0);
 558         if (ctx != TLB_CTX_KERNEL && (tf->tf_tstate & TSTATE_PRIV) != 0) {
 559                 if (tf->tf_tpc >= (u_long)fs_nofault_begin &&
 560                     tf->tf_tpc <= (u_long)fs_nofault_end) {
 561                         tf->tf_tpc = (u_long)fs_fault;
 562                         tf->tf_tnpc = tf->tf_tpc + 4;
 563                         return (0);
 564                 }
 565                 if (tf->tf_tpc >= (u_long)copy_nofault_begin &&
 566                     tf->tf_tpc <= (u_long)copy_nofault_end) {
 567                         tf->tf_tpc = (u_long)copy_fault;
 568                         tf->tf_tnpc = tf->tf_tpc + 4;
 569                         return (0);
 570                 }
 571         }
 572         return ((rv == KERN_PROTECTION_FAILURE) ? SIGBUS : SIGSEGV);
 573 }
 574
 575 /* Maximum number of arguments that can be passed via the out registers. */
 576 #define REG_MAXARGS     6
 577
 578 /*
 579  * Syscall handler. The arguments to the syscall are passed in the o registers
 580  * by the caller, and are saved in the trap frame. The syscall number is passed
 581  * in %g1 (and also saved in the trap frame).
 582  */
 583 void
 584 syscall(struct trapframe *tf)
 585 {
 586         struct sysent *callp;
 587         struct thread *td;
 588         register_t args[8];
 589         register_t *argp;
 590         struct proc *p;
 591         u_long code;
 592         u_long tpc;
 593         int reg;
 594         int regcnt;
 595         int narg;
 596         int error;
 597
 598         td = curthread;
 599         KASSERT(td != NULL, ("trap: curthread NULL"));
 600         KASSERT(td->td_proc != NULL, ("trap: curproc NULL"));
 601
 602         p = td->td_proc;
 603
 604         PCPU_INC(cnt.v_syscall);
 605
 606         narg = 0;
 607         error = 0;
 608         reg = 0;
 609         regcnt = REG_MAXARGS;
 610
 611         td->td_pticks = 0;
 612         td->td_frame = tf;
 613         if (td->td_ucred != p->p_ucred)
 614                 cred_update_thread(td);
 615         code = tf->tf_global[1];
 616
 617         /*
 618          * For syscalls, we don't want to retry the faulting instruction
 619          * (usually), instead we need to advance one instruction.
 620          */
 621         tpc = tf->tf_tpc;
 622         TF_DONE(tf);
 623
 624         if (p->p_sysent->sv_prepsyscall) {
 625                 /*
 626                  * The prep code is MP aware.
 627                  */
 628 #if 0
 629                 (*p->p_sysent->sv_prepsyscall)(tf, args, &code, &params);
 630 #endif
 631         } else  if (code == SYS_syscall || code == SYS___syscall) {
 632                 code = tf->tf_out[reg++];
 633                 regcnt--;
 634         }
 635
 636         if (p->p_sysent->sv_mask)
 637                 code &= p->p_sysent->sv_mask;
 638
 639         if (code >= p->p_sysent->sv_size)
 640                 callp = &p->p_sysent->sv_table[0];
 641         else
 642                 callp = &p->p_sysent->sv_table[code];
 643
 644         narg = callp->sy_narg;
 645
 646         if (narg <= regcnt) {
 647                 argp = &tf->tf_out[reg];
 648                 error = 0;
 649         } else {
 650                 KASSERT(narg <= sizeof(args) / sizeof(args[0]),
 651                     ("Too many syscall arguments!"));
 652                 argp = args;
 653                 bcopy(&tf->tf_out[reg], args, sizeof(args[0]) * regcnt);
 654                 error = copyin((void *)(tf->tf_out[6] + SPOFF +
 655                     offsetof(struct frame, fr_pad[6])),
 656                     &args[regcnt], (narg - regcnt) * sizeof(args[0]));
 657         }
 658
 659         CTR5(KTR_SYSC, "syscall: td=%p %s(%#lx, %#lx, %#lx)", td,
 660             syscallnames[code], argp[0], argp[1], argp[2]);
 661
 662         /*
 663          * Try to run the syscall without the MP lock if the syscall
 664          * is MP safe.
 665          */
 666 #ifdef KTRACE
 667         if (KTRPOINT(td, KTR_SYSCALL))
 668                 ktrsyscall(code, narg, argp);
 669 #endif
 670         if (error == 0) {
 671                 td->td_retval[0] = 0;
 672                 td->td_retval[1] = 0;
 673
 674                 STOPEVENT(p, S_SCE, narg);      /* MP aware */
 675
 676                 PTRACESTOP_SC(p, td, S_PT_SCE);
 677
 678                 AUDIT_SYSCALL_ENTER(code, td);
 679                 error = (*callp->sy_call)(td, argp);
 680                 AUDIT_SYSCALL_EXIT(error, td);
 681
 682                 CTR5(KTR_SYSC, "syscall: p=%p error=%d %s return %#lx %#lx ", p,
 683                     error, syscallnames[code], td->td_retval[0],
 684                     td->td_retval[1]);
 685         }
 686
 687         /*
 688          * MP SAFE (we may or may not have the MP lock at this point)
 689          */
 690         switch (error) {
 691         case 0:
 692                 tf->tf_out[0] = td->td_retval[0];
 693                 tf->tf_out[1] = td->td_retval[1];
 694                 tf->tf_tstate &= ~TSTATE_XCC_C;
 695                 break;
 696
 697         case ERESTART:
 698                 /*
 699                  * Undo the tpc advancement we have done above, we want to
 700                  * reexecute the system call.
 701                  */
 702                 tf->tf_tpc = tpc;
 703                 tf->tf_tnpc -= 4;
 704                 break;
 705
 706         case EJUSTRETURN:
 707                 break;
 708
 709         default:
 710                 if (p->p_sysent->sv_errsize) {
 711                         if (error >= p->p_sysent->sv_errsize)
 712                                 error = -1;     /* XXX */
 713                         else
 714                                 error = p->p_sysent->sv_errtbl[error];
 715                 }
 716                 tf->tf_out[0] = error;
 717                 tf->tf_tstate |= TSTATE_XCC_C;
 718                 break;
 719         }
 720
 721         /*
 722          * Handle reschedule and other end-of-syscall issues
 723          */
 724         userret(td, tf);
 725
 726 #ifdef KTRACE
 727         if (KTRPOINT(td, KTR_SYSRET))
 728                 ktrsysret(code, error, td->td_retval[0]);
 729 #endif
 730         /*
 731          * This works because errno is findable through the
 732          * register set.  If we ever support an emulation where this
 733          * is not the case, this code will need to be revisited.
 734          */
 735         STOPEVENT(p, S_SCX, code);
 736
 737         PTRACESTOP_SC(p, td, S_PT_SCX);
 738
 739         WITNESS_WARN(WARN_PANIC, NULL, "System call %s returning",
 740             (code >= 0 && code < SYS_MAXSYSCALL) ? syscallnames[code] : "???");
 741         mtx_assert(&Giant, MA_NOTOWNED);
 742 }